Merge pull request #1270 from bilawalsidhu/codex/fix-space-provider-cooldowns

fix(space): respect provider cooldowns and bound TLE groups

changelog.d/1270-space-issue-fix.md

@@ -0,0 +1 @@
+- Respect provider cooldowns and bound tle groups (asturksever, #1270).

docs/CURRENT-STATE.md

@@ -9,18 +9,15 @@ broader suffix variants. Follow-ups retain boundary provenance, and cancelled
 lookups preserve the previous analyst cohort and boundary source. Navigation
 search keeps a successful geocode, with no search or view-based region recovery.
 
-Radio tags are lowercased and whitespace-normalized before the final 80-character
+CelesTrak accepts only the eight satellite groups used by the app (the seven
-cap, including requested-tag coverage matching. Malformed station rows are skipped
+satellite-layer groups plus `active` for Space Missions) and gates cross-site
-individually on the server and the client; valid rows retain their accepted
+requests before provider work with the keyless same-site policy, so the
-catalog generation. The server's `/stations` envelope carries
+documented reverse proxy still works. CelesTrak and Launch Library failures
-`rejectedStationCount`, the structurally malformed upstream rows it skipped
+start a provider-wide cooldown that a later failure never shortens: Retry-After
-(capped at 10,000; policy-filtered rows such as offline or non-HTTPS streams are
+seconds or HTTP dates are bounded to 5 seconds through 30 minutes, with a
-not counted). It is diagnostic only and never fails catalog health. The client
+60-second fallback. During cooldown dated last-good data is served with stale
-adds its own rejected rows after admission, so the UI snapshot reports the
+provenance, otherwise the proxy returns a bounded error without another upstream
-combined, capped `rejectedStationCount` and degraded coverage when either side
+attempt. Error bodies are cancelled before reading.
-skipped rows. Destroy resets the count. Malformed envelope, freshness, generation
-or rejection-count metadata and all-invalid responses still preserve the previous
-usable catalog.
 
 Terrain height requests accept at most 64 points, matching the browser service's
 existing batch size and one bounded upstream chunk. Larger requests return HTTP

scripts/package-boundaries.json

@@ -103,6 +103,9 @@
     "modules": [
       "server/providers/space.js",
       "server/providers/space/celestrak.js",
+      "server/providers/common/same-site.js",
+      "src/localRequestGate.mjs",
+      "server/providers/space/cooldown.js",
       "server/providers/space/launch-library.js",
       "server/providers/common/http.js",
       "src/data/spaceProviderRequests.js",

server/providers/space/celestrak.js

@@ -1,6 +1,8 @@
 import path from 'node:path';
 import { promises as fsp } from 'node:fs';
 import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
+import { providerCooldownMs } from './cooldown.js';
+import { admitSameSiteKeyless } from '../common/same-site.js';
 import { readResponseTextCapped } from '../common/http.js';
 
 /**
@@ -24,6 +26,18 @@ import { readResponseTextCapped } from '../common/http.js';
 export function celestrakProxy() {
   const TLE_TTL_MS = 6 * 3600_000;
   const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
+  const groups = new Set([
+    'stations',
+    'visual',
+    'gps-ops',
+    'glo-ops',
+    'galileo',
+    'geo',
+    'starlink',
+    // Space Missions (`getActiveTle`) reads the full active catalog.
+    'active',
+  ]);
+  let cooldownUntil = 0;
   const mem = new Map(); // group -> { at: epochMs, body: string }
   const inflight = new Map(); // group -> Promise<{at, body}|null>
 
@@ -64,7 +78,16 @@ export function celestrakProxy() {
           'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
       },
     });
-    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    if (!res.ok) {
+      // Group refreshes run in parallel: a later failure without Retry-After
+      // must never shorten a longer deadline an earlier one already set.
+      cooldownUntil = Math.max(
+        cooldownUntil,
+        Date.now() + providerCooldownMs(res),
+      );
+      void res.body?.cancel().catch(() => {});
+      throw new Error(`HTTP ${res.status}`);
+    }
     const body = await readResponseTextCapped(
       res,
       CELESTRAK_MAX_RESPONSE_BYTES,
@@ -80,12 +103,20 @@ export function celestrakProxy() {
       const group = String(req.url || '')
         .replace(/^\//, '')
         .split('?')[0];
-      if (!/^[a-z0-9-]+$/i.test(group)) {
+      // Keyless and needed to render the satellites layer: refuse cross-site
+      // browsers but keep the documented reverse-proxy deployment working.
+      if (admitSameSiteKeyless(req, res)) return;
+      if (req.method !== 'GET') {
+        res.writeHead(405);
+        res.end();
+        return;
+      }
+      if (!groups.has(group)) {
         res.writeHead(400, { 'Content-Type': 'text/plain' });
         res.end('invalid group');
         return;
       }
-      const send = (status, body, cacheStatus) => {
+      const send = (status, body, cacheStatus, entry) => {
         // Guard against a double-send (e.g. a throw AFTER a response already
         // went out routing into the catch's send): writeHead after headersSent
         // throws "Cannot set headers after they are sent".
@@ -93,6 +124,17 @@ export function celestrakProxy() {
         res.writeHead(status, {
           'Content-Type': 'text/plain',
           'x-tle-cache': cacheStatus,
+          'Cache-Control': 'no-store',
+          ...(entry
+            ? { 'X-TLE-Cached-At': new Date(entry.at).toISOString() }
+            : {}),
+          ...(cooldownUntil > 0 && Date.now() < cooldownUntil
+            ? {
+                'Retry-After': String(
+                  Math.ceil((cooldownUntil - Date.now()) / 1000),
+                ),
+              }
+            : {}),
         });
         res.end(body);
       };
@@ -104,7 +146,16 @@ export function celestrakProxy() {
           if (entry) mem.set(group, entry);
         }
         if (entry && now - entry.at < TLE_TTL_MS) {
-          send(200, entry.body, 'HIT');
+          send(200, entry.body, 'HIT', entry);
+          return;
+        }
+        if (Date.now() < cooldownUntil) {
+          send(
+            entry ? 200 : 502,
+            entry?.body || 'celestrak upstream cooling down',
+            entry ? 'STALE-ERROR' : 'NONE',
+            entry,
+          );
           return;
         }
         // Stale or missing → refresh, single-flight per group.
@@ -118,6 +169,8 @@ export function celestrakProxy() {
                 return fresh;
               })
               .catch((err) => {
+                if (Date.now() >= cooldownUntil)
+                  cooldownUntil = Date.now() + 60000;
                 console.warn(
                   '[celestrak-proxy] refresh failed — serving cache if any',
                 );
@@ -128,9 +181,9 @@ export function celestrakProxy() {
         }
         const fresh = await inflight.get(group);
         if (fresh) {
-          send(200, fresh.body, 'MISS');
+          send(200, fresh.body, 'MISS', fresh);
         } else if (entry) {
-          send(200, entry.body, 'STALE-ERROR'); // upstream down — stale beats empty
+          send(200, entry.body, 'STALE-ERROR', entry); // upstream down — stale beats empty
         } else {
           send(502, 'celestrak fetch failed and no cache available', 'NONE');
         }

server/providers/space/cooldown.js

@@ -0,0 +1,12 @@
+/** Bound an upstream Retry-After delay (seconds or HTTP date). */
+export function providerCooldownMs(response, now = Date.now()) {
+  const raw = response?.headers?.get('retry-after')?.trim();
+  const seconds = raw && /^\d+(?:\.\d+)?$/.test(raw) ? Number(raw) : NaN;
+  const deadline = raw && !Number.isFinite(seconds) ? Date.parse(raw) : NaN;
+  const delay = Number.isFinite(seconds)
+    ? seconds * 1000
+    : Number.isFinite(deadline)
+      ? deadline - now
+      : 60000;
+  return Math.min(30 * 60000, Math.max(5000, delay));
+}

server/providers/space/launch-library.js

@@ -1,4 +1,5 @@
 import path from 'node:path';
+import { providerCooldownMs } from './cooldown.js';
 import { promises as fsp } from 'node:fs';
 import {
   readResponseTextCapped,
@@ -30,6 +31,8 @@ export function rocketLaunchesProxy() {
     'launch-library-2-v2.3.json',
   );
   let cache = null;
+  let cooldownUntil = 0;
+  let cooldownStatus = 502;
   let diskLoaded = false;
   const inFlight = new Map();
 
@@ -62,7 +65,18 @@ export function rocketLaunchesProxy() {
   function send(res, status, body, cacheState) {
     res.writeHead(status, {
       'Content-Type': 'application/json',
-      'Cache-Control': status === 200 ? 'public, max-age=900' : 'no-store',
+      'Cache-Control':
+        status === 200 && cacheState !== 'STALE-ERROR'
+          ? 'public, max-age=900'
+          : 'no-store',
+      ...(cache ? { 'X-GEV-Cached-At': new Date(cache.at).toISOString() } : {}),
+      ...(Date.now() < cooldownUntil
+        ? {
+            'Retry-After': String(
+              Math.ceil((cooldownUntil - Date.now()) / 1000),
+            ),
+          }
+        : {}),
       'X-GEV-Cache': cacheState,
     });
     res.end(body);
@@ -75,17 +89,25 @@ export function rocketLaunchesProxy() {
       signal: AbortSignal.timeout(20000),
       headers: launchLibraryRequestHeaders(),
     });
-    const body = await readResponseTextCapped(upstream, maxResponseBytes);
     if (!upstream.ok) {
+      const deadline = Date.now() + providerCooldownMs(upstream);
+      // Never shorten a longer cooldown that is already in force.
+      if (deadline >= cooldownUntil) {
+        cooldownUntil = deadline;
+        cooldownStatus = upstream.status;
+      }
+      void upstream.body?.cancel().catch(() => {});
       const error = new Error(`upstream HTTP ${upstream.status}`);
       error.upstreamStatus = upstream.status;
       throw error;
     }
+    const body = await readResponseTextCapped(upstream, maxResponseBytes);
     const parsed = JSON.parse(body);
     if (!Array.isArray(parsed?.results))
       throw new Error('malformed upstream response');
     const fresh = { at: Date.now(), body };
     cache = fresh;
+    cooldownUntil = 0;
     void saveDiskCache(fresh);
     return fresh;
   }
@@ -103,6 +125,16 @@ export function rocketLaunchesProxy() {
         return;
       }
       const stale = cache;
+      if (now < cooldownUntil) {
+        send(
+          res,
+          stale ? 200 : cooldownStatus,
+          stale?.body ||
+            JSON.stringify({ error: 'Launch Library 2 unavailable' }),
+          stale ? 'STALE-ERROR' : 'NONE',
+        );
+        return;
+      }
       const request = coalesceProxyRequest(
         inFlight,
         'recent-launches',
@@ -112,6 +144,10 @@ export function rocketLaunchesProxy() {
         const fresh = await request.promise;
         send(res, 200, fresh.body, request.shared ? 'INFLIGHT' : 'MISS');
       } catch (error) {
+        if (Date.now() >= cooldownUntil) {
+          cooldownUntil = Date.now() + 60000;
+          cooldownStatus = 502;
+        }
         // Log only a bounded status, never upstream bodies, URLs, or credentials.
         const status = Number.isInteger(error?.upstreamStatus)
           ? error.upstreamStatus

src/proxyErrorResponses.test.mjs

@@ -2,6 +2,7 @@ import test from 'node:test';
 import assert from 'node:assert/strict';
 import { readFileSync } from 'node:fs';
 import path from 'node:path';
+import { providerCooldownMs } from '../server/providers/space/cooldown.js';
 import { parseTerrainPoints } from './data/terrainHeightsProxy.js';
 import { readResponseTextCapped, coalesceProxyRequest } from './sources/httpBody.js';
 import { makeRateLimiter, clientKey } from '../server/providers/common/rate-limit.js';
@@ -23,7 +24,7 @@ function extract(name) {
 function fixture(name, overrides = {}, preview = false) {
   const logs = [];
   const deps = {
-    readResponseTextCapped, coalesceProxyRequest,
+    readResponseTextCapped, coalesceProxyRequest, providerCooldownMs,
     // The rate-limited proxies reach for these; give them the real ones so a
     // fixture request is throttled exactly as production would be.
     makeRateLimiter, clientKey,
@@ -117,7 +118,7 @@ test('Launch Library retains single-flight, fresh cache, stale fallback, and met
 
 test('CelesTrak unexpected failures hide details', async () => {
   const app = fixture('celestrakProxy', { Date: { now() { throw new Error(detail); } } });
-  const res = await app.request('/active');
+  const res = await app.request('/stations');
   assert.equal(res.status, 500);
   assert.equal(res.body, 'celestrak proxy error');
   assert.equal(res.headers['x-tle-cache'], 'ERROR');
@@ -126,7 +127,7 @@ test('CelesTrak unexpected failures hide details', async () => {
 test('CelesTrak retains invalid-group and unavailable responses', async () => {
   const app = fixture('celestrakProxy');
   assert.equal((await app.request('/../')).status, 400);
-  const res = await app.request('/active');
+  const res = await app.request('/stations');
   assert.equal(res.status, 502);
   assert.equal(res.headers['x-tle-cache'], 'NONE');
 });
@@ -135,13 +136,13 @@ test('CelesTrak retains fresh and stale TLE caches', async () => {
   let now = Date.now();
   let calls = 0;
   const app = fixture('celestrakProxy', {
-    Date: { now: () => now },
+    Date: class extends Date { static now() { return now; } },
     fetch: async () => { if (++calls > 1) throw new Error(detail); return new Response('1 valid-fixture-TLE'); },
   });
-  assert.equal((await app.request('/active')).headers['x-tle-cache'], 'MISS');
+  assert.equal((await app.request('/stations')).headers['x-tle-cache'], 'MISS');
-  assert.equal((await app.request('/active')).headers['x-tle-cache'], 'HIT');
+  assert.equal((await app.request('/stations')).headers['x-tle-cache'], 'HIT');
   now += 7 * 3600_000;
-  const stale = await app.request('/active');
+  const stale = await app.request('/stations');
   assert.equal(stale.status, 200);
   assert.equal(stale.body, '1 valid-fixture-TLE');
   assert.equal(stale.headers['x-tle-cache'], 'STALE-ERROR');

src/tooling/spaceProviders.test.mjs

@@ -12,6 +12,7 @@ import {
   launchLibraryRecentUrl,
 } from 'gods-eye-view/sources/space';
 import * as compatibility from '../../server/providers/local.js';
+import { createLaunchSource } from 'gods-eye-view/layers/launches/source';
 
 function install(plugin, preview = false) {
   const routes = new Map();
@@ -22,17 +23,23 @@ function install(plugin, preview = false) {
       },
     },
   });
-  return async (route, url = '/', method = 'GET') => {
+  return async (route, url = '/', method = 'GET', headers = {}) => {
     const res = {
       headersSent: false,
+      headers: {},
       writeHead(status, headers) {
         Object.assign(this, { status, headers, headersSent: true });
       },
+      // The same-site gates answer 403 through statusCode + setHeader.
+      setHeader(name, value) {
+        this.headers[name] = value;
+      },
       end(body) {
         this.body = body;
+        this.status ??= this.statusCode;
       },
     };
-    await routes.get(route)({ url, method }, res);
+    await routes.get(route)({ url, method, headers, socket: {} }, res);
     return res;
   };
 }
@@ -161,7 +168,11 @@ test('CelesTrak answers 502 to an oversized TLE body without caching it', async
   assert.ok(bodies[0].pulled <= 17, 'reading stops at the 16 MiB cap');
   assert.equal(fsp.writeFile.mock.callCount(), 0);
   assert.equal((await request('/api/celestrak', '/stations')).status, 502);
-  assert.equal(bodies.length, 2, 'nothing was cached in memory');
+  assert.equal(
+    bodies.length,
+    1,
+    'failed bodies are not cached but a cooldown bounds retry work',
+  );
 });
 
 for (const preview of [false, true])
@@ -190,3 +201,194 @@ for (const preview of [false, true])
     assert.equal(hit.body, first.body);
     assert.equal(calls, 1);
   });
+
+test('space providers honor Retry-After, gate groups, preserve stale dates and recover', async (t) => {
+  isolateDisk(t);
+  t.mock.method(console, 'warn', () => {});
+  let now = Date.now();
+  t.mock.method(Date, 'now', () => now);
+  for (const kind of ['tle', 'launches']) {
+    for (const retryAfter of ['90', new Date(now + 90000).toUTCString()]) {
+      let calls = 0;
+      let fail = false;
+      let cancelled = false;
+      t.mock.method(globalThis, 'fetch', async () => {
+        calls++;
+        if (fail)
+          return new Response(
+            new ReadableStream({
+              cancel() {
+                cancelled = true;
+              },
+            }),
+            {
+              status: kind === 'tle' ? 403 : 429,
+              headers: { 'Retry-After': retryAfter },
+            },
+          );
+        return new Response(
+          kind === 'tle'
+            ? 'ISS\n1 25544U fixture\n2 25544 fixture\n'
+            : '{"results":[]}',
+        );
+      });
+      const request = install(
+        kind === 'tle' ? celestrakProxy() : rocketLaunchesProxy(),
+      );
+      const route = kind === 'tle' ? '/api/celestrak' : '/api/launches';
+      const url = kind === 'tle' ? '/stations' : '/';
+      if (kind === 'tle') {
+        assert.equal(
+          (await request(route, '/not-a-supported-group')).status,
+          400,
+        );
+        assert.equal(calls, 0);
+      }
+      const first = await request(route, url);
+      now += 7 * 3600000;
+      // Recompute the HTTP-date relative to the clock at the failed refresh.
+      const originalFetch = globalThis.fetch;
+      t.mock.method(globalThis, 'fetch', async (...args) => {
+        const response = await originalFetch(...args);
+        if (fail && !/^\d+$/.test(retryAfter))
+          response.headers.set(
+            'Retry-After',
+            new Date(now + 90000).toUTCString(),
+          );
+        return response;
+      });
+      fail = true;
+      const stale = await request(route, url);
+      assert.equal(stale.status, 200);
+      assert.equal(stale.body, first.body);
+      assert.equal(stale.headers['Retry-After'], '90');
+      assert.ok(
+        stale.headers[kind === 'tle' ? 'X-TLE-Cached-At' : 'X-GEV-Cached-At'],
+      );
+      assert.equal(cancelled, true);
+      await request(route, url);
+      if (kind === 'tle')
+        assert.equal((await request(route, '/visual')).status, 502);
+      assert.equal(calls, 2, 'cooldown applies across TLE groups');
+      now += 91000;
+      fail = false;
+      assert.equal((await request(route, url)).status, 200);
+      assert.equal(calls, 3);
+    }
+  }
+});
+
+test('Space Missions reads the active catalog through the real proxy while unknown groups stay rejected', async (t) => {
+  isolateDisk(t);
+  const tle = 'ISS\n1 25544U fixture\n2 25544 fixture\n';
+  const fetched = [];
+  t.mock.method(globalThis, 'fetch', async (url) => {
+    fetched.push(new URL(url).searchParams.get('GROUP'));
+    return new Response(tle);
+  });
+  const request = install(celestrakProxy());
+  // Route the browser source's real request into the middleware.
+  const source = createLaunchSource({
+    fetchImpl: async (url) => {
+      const prefix = '/api/celestrak';
+      assert.ok(url.startsWith(`${prefix}/`));
+      const res = await request(prefix, url.slice(prefix.length));
+      return new Response(res.body, { status: res.status });
+    },
+  });
+  assert.equal(await source.getActiveTle(), tle);
+  assert.deepEqual(fetched, ['active']);
+  for (const group of ['/not-a-group', '/active-extra', '/../active'])
+    assert.equal((await request('/api/celestrak', group)).status, 400);
+  assert.deepEqual(fetched, ['active'], 'rejected groups never reach upstream');
+});
+
+test('CelesTrak admits a same-origin request behind the documented reverse proxy and refuses foreign origins', async (t) => {
+  isolateDisk(t);
+  let calls = 0;
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls++;
+    return new Response('ISS\n1 25544U fixture\n2 25544 fixture\n');
+  });
+  const request = install(celestrakProxy());
+  const proxied = {
+    host: 'gev.example.test',
+    'x-forwarded-for': '203.0.113.7',
+    'x-forwarded-host': 'gev.example.test',
+    'x-forwarded-proto': 'https',
+    via: '1.1 proxy',
+  };
+  const app = await request('/api/celestrak', '/stations', 'GET', {
+    ...proxied,
+    origin: 'https://gev.example.test',
+    'sec-fetch-site': 'same-origin',
+  });
+  assert.equal(app.status, 200);
+  assert.equal(app.headers['x-tle-cache'], 'MISS');
+  assert.equal(calls, 1);
+  for (const headers of [
+    { ...proxied, origin: 'https://unrelated.example' },
+    { ...proxied, origin: 'null' },
+    { ...proxied, 'sec-fetch-site': 'cross-site' },
+  ]) {
+    const refused = await request('/api/celestrak', '/visual', 'GET', headers);
+    assert.equal(refused.status, 403);
+  }
+  assert.equal(calls, 1, 'refused requests do no provider or cache work');
+});
+
+for (const longFirst of [true, false])
+  test(`CelesTrak keeps the longest cooldown when concurrent failures finish ${longFirst ? 'long then short' : 'short then long'}`, async (t) => {
+    isolateDisk(t);
+    t.mock.method(console, 'warn', () => {});
+    let now = Date.now();
+    t.mock.method(Date, 'now', () => now);
+    const pending = new Map();
+    let calls = 0;
+    t.mock.method(globalThis, 'fetch', async (url) => {
+      calls++;
+      const group = new URL(url).searchParams.get('GROUP');
+      await new Promise((resolve) => pending.set(group, resolve));
+      return group === 'stations'
+        ? new Response('slow down', {
+            status: 429,
+            headers: { 'Retry-After': '1800' },
+          })
+        : new Response('unavailable', { status: 503 });
+    });
+    const request = install(celestrakProxy());
+    const long = request('/api/celestrak', '/stations');
+    const short = request('/api/celestrak', '/visual');
+    while (pending.size < 2) await new Promise(setImmediate);
+    const order = longFirst
+      ? [
+          ['stations', long],
+          ['visual', short],
+        ]
+      : [
+          ['visual', short],
+          ['stations', long],
+        ];
+    for (const [group, response] of order) {
+      pending.get(group)();
+      assert.equal((await response).status, 502);
+    }
+    assert.equal(calls, 2);
+    const cooling = await request('/api/celestrak', '/geo');
+    assert.equal(cooling.status, 502);
+    assert.equal(cooling.headers['Retry-After'], '1800');
+    now += 61_000;
+    assert.equal((await request('/api/celestrak', '/geo')).status, 502);
+    assert.equal(
+      calls,
+      2,
+      'the 60 s fallback did not replace the 30 min deadline',
+    );
+    now += 1800_000;
+    t.mock.method(
+      globalThis,
+      'fetch',
+      async () => new Response('ISS\n1 25544U fixture\n2 25544 fixture\n'),
+    );
+    assert.equal((await request('/api/celestrak', '/geo')).status, 200);
+  });