Merge branch 'main' into codex/fix-military-history-failures

server/providers/radio/catalog.js

@@ -5,7 +5,7 @@ import { readResponseTextCapped } from '../common/http.js';
 import {
   normalizeRadioBrowserStation,
   publicRadioStation,
-  cleanRadioText,
+  normalizeRadioTag,
 } from './stations.js';
 import {
   radioMirrorOrigin,
@@ -25,6 +25,7 @@ import {
   RADIO_USER_AGENT,
   RADIO_UUID_RE,
   RADIO_FALLBACK_MIRRORS,
+  RADIO_REJECTED_STATION_COUNT_MAX,
 } from './constants.js';
 async function mapRadioConcurrent(values, concurrency, mapper) {
   const results = new Array(values.length);
@@ -177,29 +178,25 @@ export function createRadioProxyMiddleware({
           const rows = await fetchPath(`/json/stations/search?${params}`);
           if (!Array.isArray(rows))
             throw new Error('Radio Browser catalog payload was not an array');
-          if (
-            !rows.every(
-              (row) =>
-                row &&
-                typeof row === 'object' &&
-                !Array.isArray(row) &&
-                typeof row.stationuuid === 'string' &&
-                typeof row.name === 'string' &&
-                (typeof row.url_resolved === 'string' ||
-                  typeof row.url === 'string'),
-            )
-          )
-            throw new Error(
-              'Radio Browser catalog contained a malformed station row',
-            );
-          const stations = rows
+          const structuralRows = rows.filter(
+            (row) =>
+              row &&
+              typeof row === 'object' &&
+              !Array.isArray(row) &&
+              typeof row.stationuuid === 'string' &&
+              typeof row.name === 'string' &&
+              (typeof row.url_resolved === 'string' ||
+                typeof row.url === 'string'),
+          );
+          // Malformed rows are skipped individually, but counted so the
+          // catalog keeps a diagnostic for upstream schema drift. Policy
+          // rejections in normalization (offline, non-HTTPS, codec) are normal
+          // filtering and are not counted.
+          const rejectedRows = rows.length - structuralRows.length;
+          const stations = structuralRows
             .map(normalizeRadioBrowserStation)
             .filter(Boolean);
-          const requestedTag = cleanRadioText(tag, 80)
-            .toLocaleLowerCase()
-            .replace(/[_-]+/g, ' ')
-            .replace(/\s+/g, ' ')
-            .trim();
+          const requestedTag = normalizeRadioTag(tag);
           const requestedTagCovered =
             !requestedTag ||
             stations.some((station) =>
@@ -215,9 +212,10 @@ export function createRadioProxyMiddleware({
             // also contain an accepted station tagged for the requested category.
             succeeded: stations.length > 0 && requestedTagCovered,
             stations,
+            rejectedRows,
           };
         } catch {
-          return { succeeded: false, stations: [] };
+          return { succeeded: false, stations: [], rejectedRows: 0 };
         }
       },
     );
@@ -257,6 +255,10 @@ export function createRadioProxyMiddleware({
     if (selected.length < RADIO_CATALOG_HEALTHY_MIN_STATIONS)
       healthReasons.push('station-coverage-below-policy');
     const degraded = healthReasons.length > 0;
+    const rejectedStationCount = Math.min(
+      RADIO_REJECTED_STATION_COUNT_MAX,
+      outcomes.reduce((total, outcome) => total + outcome.rejectedRows, 0),
+    );
     const coverage = {
       successfulQueries,
       totalQueries: queries.length,
@@ -271,6 +273,9 @@ export function createRadioProxyMiddleware({
       degraded,
       degradedReason: degraded ? healthReasons.join(',') : null,
       coverage,
+      // Diagnostic only: it never fails health policy, so a usable generation
+      // with a few skipped rows is still admitted. The client reports it.
+      rejectedStationCount,
     };
     if (degraded && catalogCache) {
       const error = new Error(
@@ -360,6 +365,7 @@ export function createRadioProxyMiddleware({
           coverage: catalog.coverage || null,
           acceptedGeneration: catalog.acceptedGeneration ?? null,
           catalogInstance,
+          rejectedStationCount: catalog.rejectedStationCount ?? 0,
         });
       } catch (error) {
         sendJson(res, 503, {

server/providers/radio/constants.js

@@ -11,6 +11,9 @@ export const RADIO_CATALOG_HEALTHY_MIN_STATIONS = Math.ceil(
 export const RADIO_USER_AGENT =
   'GodsEyeView/1.0 (Radio Browser directory client)';
 export { RADIO_UUID_RE } from '../../../src/sources/radioBrowser.js';
+// Upper bound for the structurally malformed upstream rows reported in the
+// /stations envelope. The client layer validates against the same bound.
+export const RADIO_REJECTED_STATION_COUNT_MAX = 10_000;
 export const RADIO_FALLBACK_MIRRORS = Object.freeze([
   'https://de1.api.radio-browser.info',
   'https://de2.api.radio-browser.info',

server/providers/radio/stations.js

@@ -1,5 +1,6 @@
 export {
   cleanRadioText,
+  normalizeRadioTag,
   isNonGlobalIpv4,
   publicRadioHttpsUrl,
   normalizeRadioBrowserStation,

server/providers/space/celestrak.js

@@ -1,6 +1,8 @@
 import path from 'node:path';
 import { promises as fsp } from 'node:fs';
 import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
+import { providerCooldownMs } from './cooldown.js';
+import { admitSameSiteKeyless } from '../common/same-site.js';
 import { readResponseTextCapped } from '../common/http.js';
 
 /**
@@ -24,6 +26,18 @@ import { readResponseTextCapped } from '../common/http.js';
 export function celestrakProxy() {
   const TLE_TTL_MS = 6 * 3600_000;
   const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
+  const groups = new Set([
+    'stations',
+    'visual',
+    'gps-ops',
+    'glo-ops',
+    'galileo',
+    'geo',
+    'starlink',
+    // Space Missions (`getActiveTle`) reads the full active catalog.
+    'active',
+  ]);
+  let cooldownUntil = 0;
   const mem = new Map(); // group -> { at: epochMs, body: string }
   const inflight = new Map(); // group -> Promise<{at, body}|null>
 
@@ -64,7 +78,16 @@ export function celestrakProxy() {
           'gods-eye-view-celestrak-proxy/1.0 (+https://github.com/bilawalsidhu/gods-eye-view)',
       },
     });
-    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    if (!res.ok) {
+      // Group refreshes run in parallel: a later failure without Retry-After
+      // must never shorten a longer deadline an earlier one already set.
+      cooldownUntil = Math.max(
+        cooldownUntil,
+        Date.now() + providerCooldownMs(res),
+      );
+      void res.body?.cancel().catch(() => {});
+      throw new Error(`HTTP ${res.status}`);
+    }
     const body = await readResponseTextCapped(
       res,
       CELESTRAK_MAX_RESPONSE_BYTES,
@@ -80,12 +103,20 @@ export function celestrakProxy() {
       const group = String(req.url || '')
         .replace(/^\//, '')
         .split('?')[0];
-      if (!/^[a-z0-9-]+$/i.test(group)) {
+      // Keyless and needed to render the satellites layer: refuse cross-site
+      // browsers but keep the documented reverse-proxy deployment working.
+      if (admitSameSiteKeyless(req, res)) return;
+      if (req.method !== 'GET') {
+        res.writeHead(405);
+        res.end();
+        return;
+      }
+      if (!groups.has(group)) {
         res.writeHead(400, { 'Content-Type': 'text/plain' });
         res.end('invalid group');
         return;
       }
-      const send = (status, body, cacheStatus) => {
+      const send = (status, body, cacheStatus, entry) => {
         // Guard against a double-send (e.g. a throw AFTER a response already
         // went out routing into the catch's send): writeHead after headersSent
         // throws "Cannot set headers after they are sent".
@@ -93,6 +124,17 @@ export function celestrakProxy() {
         res.writeHead(status, {
           'Content-Type': 'text/plain',
           'x-tle-cache': cacheStatus,
+          'Cache-Control': 'no-store',
+          ...(entry
+            ? { 'X-TLE-Cached-At': new Date(entry.at).toISOString() }
+            : {}),
+          ...(cooldownUntil > 0 && Date.now() < cooldownUntil
+            ? {
+                'Retry-After': String(
+                  Math.ceil((cooldownUntil - Date.now()) / 1000),
+                ),
+              }
+            : {}),
         });
         res.end(body);
       };
@@ -104,7 +146,16 @@ export function celestrakProxy() {
           if (entry) mem.set(group, entry);
         }
         if (entry && now - entry.at < TLE_TTL_MS) {
-          send(200, entry.body, 'HIT');
+          send(200, entry.body, 'HIT', entry);
+          return;
+        }
+        if (Date.now() < cooldownUntil) {
+          send(
+            entry ? 200 : 502,
+            entry?.body || 'celestrak upstream cooling down',
+            entry ? 'STALE-ERROR' : 'NONE',
+            entry,
+          );
           return;
         }
         // Stale or missing → refresh, single-flight per group.
@@ -118,6 +169,8 @@ export function celestrakProxy() {
                 return fresh;
               })
               .catch((err) => {
+                if (Date.now() >= cooldownUntil)
+                  cooldownUntil = Date.now() + 60000;
                 console.warn(
                   '[celestrak-proxy] refresh failed — serving cache if any',
                 );
@@ -128,9 +181,9 @@ export function celestrakProxy() {
         }
         const fresh = await inflight.get(group);
         if (fresh) {
-          send(200, fresh.body, 'MISS');
+          send(200, fresh.body, 'MISS', fresh);
         } else if (entry) {
-          send(200, entry.body, 'STALE-ERROR'); // upstream down — stale beats empty
+          send(200, entry.body, 'STALE-ERROR', entry); // upstream down — stale beats empty
         } else {
           send(502, 'celestrak fetch failed and no cache available', 'NONE');
         }

server/providers/space/cooldown.js

@@ -0,0 +1,12 @@
+/** Bound an upstream Retry-After delay (seconds or HTTP date). */
+export function providerCooldownMs(response, now = Date.now()) {
+  const raw = response?.headers?.get('retry-after')?.trim();
+  const seconds = raw && /^\d+(?:\.\d+)?$/.test(raw) ? Number(raw) : NaN;
+  const deadline = raw && !Number.isFinite(seconds) ? Date.parse(raw) : NaN;
+  const delay = Number.isFinite(seconds)
+    ? seconds * 1000
+    : Number.isFinite(deadline)
+      ? deadline - now
+      : 60000;
+  return Math.min(30 * 60000, Math.max(5000, delay));
+}

server/providers/space/launch-library.js

@@ -1,4 +1,5 @@
 import path from 'node:path';
+import { providerCooldownMs } from './cooldown.js';
 import { promises as fsp } from 'node:fs';
 import {
   readResponseTextCapped,
@@ -30,6 +31,8 @@ export function rocketLaunchesProxy() {
     'launch-library-2-v2.3.json',
   );
   let cache = null;
+  let cooldownUntil = 0;
+  let cooldownStatus = 502;
   let diskLoaded = false;
   const inFlight = new Map();
 
@@ -62,7 +65,18 @@ export function rocketLaunchesProxy() {
   function send(res, status, body, cacheState) {
     res.writeHead(status, {
       'Content-Type': 'application/json',
-      'Cache-Control': status === 200 ? 'public, max-age=900' : 'no-store',
+      'Cache-Control':
+        status === 200 && cacheState !== 'STALE-ERROR'
+          ? 'public, max-age=900'
+          : 'no-store',
+      ...(cache ? { 'X-GEV-Cached-At': new Date(cache.at).toISOString() } : {}),
+      ...(Date.now() < cooldownUntil
+        ? {
+            'Retry-After': String(
+              Math.ceil((cooldownUntil - Date.now()) / 1000),
+            ),
+          }
+        : {}),
       'X-GEV-Cache': cacheState,
     });
     res.end(body);
@@ -75,17 +89,25 @@ export function rocketLaunchesProxy() {
       signal: AbortSignal.timeout(20000),
       headers: launchLibraryRequestHeaders(),
     });
-    const body = await readResponseTextCapped(upstream, maxResponseBytes);
     if (!upstream.ok) {
+      const deadline = Date.now() + providerCooldownMs(upstream);
+      // Never shorten a longer cooldown that is already in force.
+      if (deadline >= cooldownUntil) {
+        cooldownUntil = deadline;
+        cooldownStatus = upstream.status;
+      }
+      void upstream.body?.cancel().catch(() => {});
       const error = new Error(`upstream HTTP ${upstream.status}`);
       error.upstreamStatus = upstream.status;
       throw error;
     }
+    const body = await readResponseTextCapped(upstream, maxResponseBytes);
     const parsed = JSON.parse(body);
     if (!Array.isArray(parsed?.results))
       throw new Error('malformed upstream response');
     const fresh = { at: Date.now(), body };
     cache = fresh;
+    cooldownUntil = 0;
     void saveDiskCache(fresh);
     return fresh;
   }
@@ -103,6 +125,16 @@ export function rocketLaunchesProxy() {
         return;
       }
       const stale = cache;
+      if (now < cooldownUntil) {
+        send(
+          res,
+          stale ? 200 : cooldownStatus,
+          stale?.body ||
+            JSON.stringify({ error: 'Launch Library 2 unavailable' }),
+          stale ? 'STALE-ERROR' : 'NONE',
+        );
+        return;
+      }
       const request = coalesceProxyRequest(
         inFlight,
         'recent-launches',
@@ -112,6 +144,10 @@ export function rocketLaunchesProxy() {
         const fresh = await request.promise;
         send(res, 200, fresh.body, request.shared ? 'INFLIGHT' : 'MISS');
       } catch (error) {
+        if (Date.now() >= cooldownUntil) {
+          cooldownUntil = Date.now() + 60000;
+          cooldownStatus = 502;
+        }
         // Log only a bounded status, never upstream bodies, URLs, or credentials.
         const status = Number.isInteger(error?.upstreamStatus)
           ? error.upstreamStatus