Merge branch 'main' into feat/bikeshare-toronto-and-failure-fix

CHANGELOG.md

@@ -178,6 +178,17 @@
 
 ## [Unreleased]
 
+- Bound six upstream body reads (FIRMS CSV, CelesTrak TLE groups, adsb.lol
+  `/v2/mil`, TomTom flow tiles, OpenSky `states/all` and adsbdb lookups) with
+  streaming size caps that cancel the body at the cap or the route's deadline;
+  the adsb.lol military poll gains a deadline of its own. An oversized OpenSky
+  snapshot is not retried, and an OpenSky 429 starts its cooldown before the
+  body is read.
+
+- Keep later satellites when a TLE catalog entry is missing an orbital line.
+  The parser skips the incomplete set and resumes at the next complete named
+  set, instead of losing the rest of the group (#906).
+
 - Satellite clouds keep Clouds only / Full as quick endpoints and add a continuous
   cloud-filter slider: 0% is Full, 100% is the existing brightness key, and
   intermediate values blend the keyed alpha. The slider commits on release to

docs/CURRENT-STATE.md

@@ -1192,6 +1192,28 @@ Sharing asks of clients of the Oslo, Bergen and Trondheim Bysykkel feeds
 their GBFS tree: `api.entur.io` to `/mobility/v2/gbfs/` and `api.cyclocity.fr`
 (Bysykkel Lillestrøm) to `/contracts/`.
 
+## Upstream body bounds
+
+FIRMS CSV, CelesTrak TLE groups, adsb.lol military responses, TomTom flow
+tiles, OpenSky states/all and adsbdb lookups now read their bodies through the
+shared streaming readers in `src/sources/httpBody.js`, so a declared or observed body
+over the cap is refused with `RESPONSE_TOO_LARGE` and cancelled rather than
+buffered. TomTom flow tiles use `readResponseBytesCapped` alongside
+GTFS-Realtime. Each read takes the route's abort signal, so a body still
+streaming when the deadline fires is cancelled. Caps are bounds, not budgets, sized well
+above measured payloads: FIRMS 64 MiB per satellite source, CelesTrak 16 MiB
+per group, OpenSky `states/all` 8 MiB, adsb.lol `/v2/mil` 8 MiB, TomTom tiles
+8 MiB and adsbdb lookups 1 MiB. Where a route already had a deadline (FIRMS
+60 s, CelesTrak 20 s, TomTom 15 s, adsbdb 8 s, and each 10 s OpenSky
+`states/all` attempt) it already bounded the body read, because aborting a
+fetch signal also aborts its body; the capped reader takes the same signal.
+The adsb.lol military poll, which had no deadline, now waits at most 12
+seconds for headers and body together. An oversized OpenSky snapshot is not
+retried, and an OpenSky 429 starts its cooldown from the response headers and
+cancels the body unread. A rejected or timed-out body is an upstream miss and
+follows each route's existing error and cached-data paths. FIRMS map-key
+status, the OpenSky token exchange and CCTV media keep their current handling.
+
 ## Remaining local service modules
 
 Overpass query validation, geometry simplification, disk caching and upstream

scripts/format-scope.json

@@ -734,6 +734,7 @@
   "src/search/nominatim.js",
   "src/search/nominatim.test.mjs",
   "src/sources/httpBody.js",
+  "src/sources/httpBody.test.mjs",
   "src/sources/overpass.js",
   "src/sources/nominatim.js",
   "server/providers/transit.js",

scripts/package-boundaries.json

@@ -123,7 +123,12 @@
   "traffic-provider": {
     "runtime": "node",
     "exports": ["./server/providers/traffic"],
-    "modules": ["server/providers/traffic.js", "src/data/tomtomTiles.js"],
+    "modules": [
+      "server/providers/traffic.js",
+      "server/providers/common/http.js",
+      "src/sources/httpBody.js",
+      "src/data/tomtomTiles.js"
+    ],
     "external": []
   },
   "traffic-source": {
@@ -139,7 +144,12 @@
   "firms-provider": {
     "runtime": "node",
     "exports": ["./server/providers/firms"],
-    "modules": ["server/providers/firms.js", "src/data/firmsCsv.js"],
+    "modules": [
+      "server/providers/firms.js",
+      "server/providers/common/http.js",
+      "src/sources/httpBody.js",
+      "src/data/firmsCsv.js"
+    ],
     "external": []
   },
   "gbfs-provider": {

server/providers/aircraft/adsb-lol.js

@@ -1,3 +1,5 @@
+import { readResponseTextCapped } from '../common/http.js';
+
 /**
  * Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
  *
@@ -25,6 +27,10 @@ export function adsbLolProxy() {
   let _cooldownStatus = 0;
   /** Response cache TTL (ms). */
   const CACHE_MS = 12000;
+  /** Upstream deadline (ms); the fetch previously had none and could hang a poll. */
+  const UPSTREAM_TIMEOUT_MS = 12000;
+  /** /v2/mil measures ~67 KB; a relayed error page is bounded by the same cap. */
+  const ADSBLOL_MIL_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
   /** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */
   const RATE_LIMIT_COOLDOWN_MS = 30000;
   /** Cooldown after a 5xx (ms). */
@@ -98,11 +104,17 @@ export function adsbLolProxy() {
           );
           return;
         }
+        const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
         const upstream = await fetch('https://api.adsb.lol/v2/mil', {
           headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
+          signal,
         });
         if (upstream.ok) {
-          const body = await upstream.text();
+          const body = await readResponseTextCapped(
+            upstream,
+            ADSBLOL_MIL_MAX_RESPONSE_BYTES,
+            signal,
+          );
           _cache = body;
           _cacheAt = Date.now();
           _cooldownUntil = 0;
@@ -129,7 +141,11 @@ export function adsbLolProxy() {
             return;
           }
         }
-        const body = await upstream.text();
+        const body = await readResponseTextCapped(
+          upstream,
+          ADSBLOL_MIL_MAX_RESPONSE_BYTES,
+          signal,
+        );
         serve(
           res,
           upstream.status,

server/providers/aircraft/enrichment.js

@@ -1,5 +1,9 @@
 import path from 'node:path';
 import { promises as fsp } from 'node:fs';
+import { readResponseJsonCapped } from '../common/http.js';
+
+/** Route and aircraft lookups are a few hundred bytes; 1 MiB is a hard bound. */
+const ADSBDB_MAX_RESPONSE_BYTES = 1024 * 1024;
 /**
  * adsbdb.com enrichment proxy: callsign → route (airline + origin/destination
  * airports) and hex → aircraft type/registration. Free community API — cached
@@ -80,12 +84,16 @@ export function adsbdbProxy() {
               kind === 'route'
                 ? `https://api.adsbdb.com/v0/callsign/${encodeURIComponent(key)}`
                 : `https://api.adsbdb.com/v0/aircraft/${encodeURIComponent(key)}`;
-            const res = await fetch(url, { signal: AbortSignal.timeout(8000) });
+            const signal = AbortSignal.timeout(8000);
+            const res = await fetch(url, { signal });
             if (res.ok) {
+              const payload = await readResponseJsonCapped(
+                res,
+                ADSBDB_MAX_RESPONSE_BYTES,
+                signal,
+              );
               const data =
-                kind === 'route'
+                kind === 'route' ? parseRoute(payload) : parseAircraft(payload);
-                  ? parseRoute(await res.json())
-                  : parseAircraft(await res.json());
               store[key] = { at: Date.now(), data }; // data may be null — negative cache
               dirty = true;
               return data;

server/providers/aircraft/opensky.js

@@ -2,6 +2,7 @@ import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback
 import {
   coalesceProxyRequest,
   readResponseJsonCapped,
+  readResponseTextCapped,
 } from '../common/http.js';
 import { requiredFiniteQueryNumber } from '../common/query.js';
 // ---------------------------------------------------------------------------
@@ -29,11 +30,34 @@ const OPENSKY_CACHE_MS = 9000;
 const OPENSKY_ATTEMPT_TIMEOUT_MS = 10_000;
 const OPENSKY_STATES_URL =
   'https://opensky-network.org/api/states/all?extended=1';
+/** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */
+const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
+
+/**
+ * Credit-governor cooldown for a 429: OpenSky's retry-after bounded to
+ * 30 s … 30 min, or 2 min when the header is absent.
+ * @param {Response} upstream Rate-limited response.
+ * @returns {number} Cooldown (ms).
+ */
+function openSkyRateLimitCooldownMs(upstream) {
+  // headers.get() gives null when absent, and Number(null) is 0, not NaN.
+  const header = upstream.headers.get('x-rate-limit-retry-after-seconds');
+  const retryAfterSec =
+    header == null || header.trim() === '' ? NaN : Number(header);
+  return Math.min(
+    Math.max(
+      Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
+      30_000,
+    ),
+    30 * 60_000,
+  );
+}
 
 /**
  * Fetch and read the global snapshot, giving each attempt
  * OPENSKY_ATTEMPT_TIMEOUT_MS and retrying once when an attempt times out or
- * the connection fails. A second failure throws, which the proxy answers
+ * the connection fails. An oversized body is not retried: it would fail again
+ * and spend more credits. A second failure throws, which the proxy answers
  * from its stale cache or the regional fallback.
  * @param {Record<string, string>} headers Request headers.
  * @returns {Promise<{upstream: Response, body: string}>}
@@ -41,13 +65,27 @@ const OPENSKY_STATES_URL =
 async function fetchOpenSkyStates(headers) {
   for (let attempt = 1; ; attempt++) {
     try {
-      const upstream = await fetch(OPENSKY_STATES_URL, {
+      const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
-        headers,
+      const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
-        signal: AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS),
+      if (upstream.status === 429) {
-      });
+        // Start the cooldown from the headers and drop the rejected body
-      return { upstream, body: await upstream.text() };
+        // unread, so a stalled or oversized 429 body cannot skip it.
+        _openskyCooldownUntil =
+          Date.now() + openSkyRateLimitCooldownMs(upstream);
+        void upstream.body?.cancel().catch(() => {});
+        return {
+          upstream,
+          body: JSON.stringify({ error: 'OpenSky rate limited.' }),
+        };
+      }
+      const body = await readResponseTextCapped(
+        upstream,
+        OPENSKY_STATES_MAX_RESPONSE_BYTES,
+        signal,
+      );
+      return { upstream, body };
     } catch (error) {
-      if (attempt >= 2) throw error;
+      if (attempt >= 2 || error?.code === 'RESPONSE_TOO_LARGE') throw error;
       console.warn(
         '[OpenSky] global snapshot attempt failed, retrying:',
         error?.message || error,
@@ -544,19 +582,9 @@ export function openSkyProxy() {
         }
         if (upstream.status === 429) {
           reason = 'rate_limited';
-          // Credit governor: honor OpenSky's retry-after (bounded 30 s … 30 min;
+          // Credit governor: fetchOpenSkyStates already started the cooldown
-          // 2 min when the header is absent) — no upstream attempts until then.
+          // from OpenSky's retry-after — no upstream attempts until then.
-          const retryAfterSec = Number(
+          const cooldownMs = openSkyRateLimitCooldownMs(upstream);
-            upstream.headers.get('x-rate-limit-retry-after-seconds'),
-          );
-          const cooldownMs = Math.min(
-            Math.max(
-              Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
-              30_000,
-            ),
-            30 * 60_000,
-          );
-          _openskyCooldownUntil = now + cooldownMs;
           // Serve the last-good body instead of the 429 when we have one —
           // the layer keeps rendering (STALE-cued) instead of dying.
           if (_openskyCacheBody && _openskyCacheStatus === 200) {

server/providers/firms.js

@@ -2,6 +2,7 @@ import path from 'node:path';
 import { promises as fsp } from 'node:fs';
 
 import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
+import { readResponseTextCapped } from './common/http.js';
 
 /**
  * NASA FIRMS live active-fire proxy with a memory + disk cache.
@@ -35,6 +36,12 @@ export function firmsProxy() {
     'VIIRS_SNPP_NRT',
     'MODIS_NRT',
   ];
+  /**
+   * Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
+   * 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
+   * bound, not a budget.
+   */
+  const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024;
   const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
   const CACHE_PATH = path.join(CACHE_DIR, 'firms.json');
 
@@ -83,9 +90,12 @@ export function firmsProxy() {
    */
   async function fetchSource(key, source) {
     const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
-    const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
+    const signal = AbortSignal.timeout(60_000);
+    const res = await fetch(url, { signal });
     if (!res.ok) throw new Error(`HTTP ${res.status}`);
-    const records = parseFirmsCsv(await res.text());
+    const records = parseFirmsCsv(
+      await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
+    );
     if (records === null) throw new Error('non-CSV upstream response');
     return records;
   }

server/providers/space/celestrak.js

@@ -1,6 +1,7 @@
 import path from 'node:path';
 import { promises as fsp } from 'node:fs';
 import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
+import { readResponseTextCapped } from '../common/http.js';
 
 /**
  * Vite plugin: CelesTrak TLE proxy.
@@ -48,10 +49,14 @@ export function celestrakProxy() {
     }
   }
 
+  /** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */
+  const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
+
   async function fetchUpstream(group) {
     const url = celestrakTleUrl(group);
+    const signal = AbortSignal.timeout(20000);
     const res = await fetch(url.toString(), {
-      signal: AbortSignal.timeout(20000),
+      signal,
       // CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
       // descriptive User-Agent with a contact point.
       headers: {
@@ -60,7 +65,11 @@ export function celestrakProxy() {
       },
     });
     if (!res.ok) throw new Error(`HTTP ${res.status}`);
-    const body = await res.text();
+    const body = await readResponseTextCapped(
+      res,
+      CELESTRAK_MAX_RESPONSE_BYTES,
+      signal,
+    );
     // An upstream error page parses to zero TLEs — treat as failure, keep cache.
     if (!/^1 /m.test(body)) throw new Error('no TLE lines in response');
     return { at: Date.now(), body };

server/providers/traffic.js

@@ -1,6 +1,7 @@
 import path from 'node:path';
 import { promises as fsp } from 'node:fs';
 
+import { readResponseBytesCapped } from './common/http.js';
 import {
   isValidTileCoord as isValidTomTomTile,
   utcDayKey as tomtomUtcDayKey,
@@ -48,6 +49,8 @@ export function tomtomProxy() {
   const DEFAULT_DAILY_BUDGET = 6000;
   const MEM_MAX_ENTRIES = 256;
   const UPSTREAM_TIMEOUT_MS = 15000;
+  /** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */
+  const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
 
   /** @type {Map<string, {at:number, buf:Buffer}>} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */
   const mem = new Map();
@@ -143,11 +146,15 @@ export function tomtomProxy() {
       'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' +
       `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`;
     recordUpstreamFetch(); // attempts count — upstream bills the request either way
-    const res = await fetch(url, {
+    const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
-      signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
+    const res = await fetch(url, { signal });
-    });
     if (!res.ok) throw new Error(`HTTP ${res.status}`);
-    const buf = Buffer.from(await res.arrayBuffer());
+    const bytes = await readResponseBytesCapped(
+      res,
+      TOMTOM_TILE_MAX_RESPONSE_BYTES,
+      signal,
+    );
+    const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
     if (buf.length === 0) throw new Error('empty tile body');
     return buf;
   }

src/sources/httpBody.js

@@ -55,7 +55,9 @@ export async function readResponseJsonCapped(response, maxBytes, signal) {
 
 /**
  * Read a fetch() Response body as bytes with the same hard cap as
- * readResponseTextCapped — for protobuf upstreams (GTFS-Realtime).
+ * readResponseTextCapped — for protobuf and tile upstreams (GTFS-Realtime,
+ * TomTom flow). Same early Content-Length rejection, running cap while
+ * streaming and cancellation on abort, so callers can treat the two alike.
  * Throws { code:'RESPONSE_TOO_LARGE' }.
  */
 export async function readResponseBytesCapped(response, maxBytes, signal) {

src/sources/httpBody.test.mjs

@@ -0,0 +1,133 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { readResponseBytesCapped } from './httpBody.js';
+
+/**
+ * A Response that streams `chunk` forever until the reader cancels it. The
+ * zero high-water mark keeps the stream from pulling eagerly at construction,
+ * so `pulls()` counts only what a reader actually asked for.
+ */
+function endlessResponse(chunk, onCancel, headers = {}) {
+  let pulls = 0;
+  const stream = new ReadableStream(
+    {
+      pull(controller) {
+        pulls += 1;
+        controller.enqueue(chunk);
+      },
+      cancel: onCancel,
+    },
+    { highWaterMark: 0 },
+  );
+  return { response: new Response(stream, { headers }), pulls: () => pulls };
+}
+
+function chunkedResponse(chunks, headers = {}) {
+  const stream = new ReadableStream(
+    {
+      start(controller) {
+        for (const chunk of chunks) controller.enqueue(chunk);
+        controller.close();
+      },
+    },
+    { highWaterMark: 0 },
+  );
+  return new Response(stream, { headers });
+}
+
+const bytes = (n, fill) => new Uint8Array(n).fill(fill);
+
+test('returns the exact bytes of a body that fits, untouched by text decoding', async () => {
+  // 0xFF and 0xFE are invalid UTF-8 lead bytes; a text path would mangle them.
+  const payload = Uint8Array.from([0xff, 0xfe, 0x00, 0x7f, 0x80, 0x01]);
+  const buf = await readResponseBytesCapped(chunkedResponse([payload]), 1024);
+  assert.ok(buf instanceof Uint8Array);
+  assert.deepEqual([...buf], [...payload]);
+});
+
+test('joins chunks in order and reports the total length', async () => {
+  const buf = await readResponseBytesCapped(
+    chunkedResponse([bytes(3, 1), bytes(2, 2), bytes(1, 3)]),
+    1024,
+  );
+  assert.deepEqual([...buf], [1, 1, 1, 2, 2, 3]);
+});
+
+test('refuses on a declared Content-Length without pulling the body', async () => {
+  let cancelled = false;
+  const { response, pulls } = endlessResponse(
+    bytes(512, 0),
+    () => {
+      cancelled = true;
+    },
+    { 'content-length': '4096' },
+  );
+  await assert.rejects(readResponseBytesCapped(response, 1024), {
+    code: 'RESPONSE_TOO_LARGE',
+  });
+  assert.equal(pulls(), 0);
+  assert.ok(cancelled);
+});
+
+test('refuses a chunked body the moment it passes the cap and stops pulling', async () => {
+  let cancelled = false;
+  const { response, pulls } = endlessResponse(bytes(512, 0), () => {
+    cancelled = true;
+  });
+  await assert.rejects(readResponseBytesCapped(response, 1024), {
+    code: 'RESPONSE_TOO_LARGE',
+  });
+  // Three 512-byte chunks is the first total over 1024; an uncapped read of
+  // this never-ending stream would not terminate at all.
+  assert.ok(pulls() <= 3, `pulled ${pulls()} chunks`);
+  assert.ok(cancelled);
+});
+
+test('accepts a body exactly at the cap', async () => {
+  const buf = await readResponseBytesCapped(
+    chunkedResponse([bytes(512, 7), bytes(512, 7)]),
+    1024,
+  );
+  assert.equal(buf.byteLength, 1024);
+});
+
+test('cancels a body still streaming when the signal aborts, and never returns a partial one', async () => {
+  const controller = new AbortController();
+  let cancelled = false;
+  let pulls = 0;
+  const stream = new ReadableStream(
+    {
+      pull(ctrl) {
+        pulls += 1;
+        ctrl.enqueue(bytes(64, 0));
+        if (pulls === 2) controller.abort();
+      },
+      cancel() {
+        cancelled = true;
+      },
+    },
+    { highWaterMark: 0 },
+  );
+  await assert.rejects(
+    readResponseBytesCapped(new Response(stream), 1 << 20, controller.signal),
+  );
+  assert.ok(cancelled);
+  assert.ok(pulls <= 3, `pulled ${pulls} chunks after abort`);
+});
+
+test('refuses immediately when the signal is already aborted', async () => {
+  const controller = new AbortController();
+  controller.abort();
+  const { response, pulls } = endlessResponse(bytes(8, 0), () => {});
+  await assert.rejects(
+    readResponseBytesCapped(response, 1024, controller.signal),
+  );
+  assert.equal(pulls(), 0);
+});
+
+test('falls back to arrayBuffer() for a body with no stream reader', async () => {
+  const response = new Response(bytes(16, 9));
+  const buf = await readResponseBytesCapped(response, 1024);
+  assert.equal(buf.byteLength, 16);
+  assert.equal(buf[0], 9);
+});

src/tooling/environmentProviders.test.mjs

@@ -53,6 +53,29 @@ function isolate(t, env = {}) {
 }
 const json = (res) => JSON.parse(res.body);
 
+/** A response that streams 1 MiB chunks of `text` well past every cap, then ends. */
+function oversizedResponse(text) {
+  const state = { pulled: 0, cancelled: false };
+  const chunk = new TextEncoder().encode(
+    text.repeat(Math.ceil((1 << 20) / text.length)),
+  );
+  const response = new Response(
+    new ReadableStream(
+      {
+        pull(controller) {
+          if (++state.pulled > 40) controller.close();
+          else controller.enqueue(chunk);
+        },
+        cancel() {
+          state.cancelled = true;
+        },
+      },
+      { highWaterMark: 0 },
+    ),
+  );
+  return { response, state };
+}
+
 test('standalone composition mounts every extracted provider exactly once without acquisition', (t) => {
   t.mock.method(globalThis, 'fetch', () => {
     throw Error('construction must not fetch');
@@ -168,6 +191,29 @@ test('traffic middleware preserves keyless mode, caching, stale budget fallback
   assert.equal(calls, 2);
 });
 
+test('traffic middleware answers 502 to an oversized tile without caching it', async (t) => {
+  isolate(t, { TOMTOM_API_KEY: 'fixture-key', TOMTOM_DAILY_TILE_BUDGET: '' });
+  const bodies = [];
+  t.mock.method(globalThis, 'fetch', async () => {
+    const oversized = oversizedResponse('\x01');
+    bodies.push(oversized.state);
+    return oversized.response;
+  });
+  const request = install(tomtomProxy());
+  const first = await request('/flow/8/1/1.pbf');
+  assert.equal(first.status, 502);
+  assert.equal(bodies[0].cancelled, true);
+  assert.ok(bodies[0].pulled <= 9, 'reading stops at the 8 MiB cap');
+  assert.ok(
+    !fsp.writeFile.mock.calls.some(({ arguments: [file] }) =>
+      String(file).endsWith('.pbf'),
+    ),
+    'no tile reached the disk cache',
+  );
+  assert.equal((await request('/flow/8/1/1.pbf')).status, 502);
+  assert.equal(bodies.length, 2, 'nothing was cached in memory');
+});
+
 test('FIRMS retains a large successful source during partial failure and filters stale data at serve time', async (t) => {
   isolate(t, { FIRMS_MAP_KEY: '' });
   let now = Date.UTC(2026, 8, 12, 12);

src/tooling/liveProviders.test.mjs

@@ -37,6 +37,30 @@ function install(plugin, preview = false) {
   };
 }
 
+/** A response that streams 1 MiB chunks of `text` well past every cap, then ends. */
+function oversizedResponse(text, init = {}) {
+  const state = { pulled: 0, cancelled: false };
+  const chunk = new TextEncoder().encode(
+    text.repeat(Math.ceil((1 << 20) / text.length)),
+  );
+  const response = new Response(
+    new ReadableStream(
+      {
+        pull(controller) {
+          if (++state.pulled > 40) controller.close();
+          else controller.enqueue(chunk);
+        },
+        cancel() {
+          state.cancelled = true;
+        },
+      },
+      { highWaterMark: 0 },
+    ),
+    init,
+  );
+  return { response, state };
+}
+
 function environment(t, values) {
   for (const [key, value] of Object.entries(values)) {
     const original = process.env[key];
@@ -193,6 +217,130 @@ test('OpenSky global snapshot attempts are time-limited, retried once, then fall
   assert.equal(stalled.response.headers['x-flight-source'], 'adsb.lol');
 });
 
+const openSkyAnonEnv = {
+  OPENSKY_CLIENT_ID: undefined,
+  OPENSKY_CLIENT_SECRET: undefined,
+  OPENSKY_AUTH_MODE: 'anon',
+  OPENSKY_USERNAME: undefined,
+  OPENSKY_PASSWORD: undefined,
+};
+const regionalFallback = () =>
+  Response.json({
+    now: Date.now() / 1000,
+    ac: [{ hex: 'abc123', lat: 30, lon: -97, alt_baro: 10000 }],
+  });
+
+test('OpenSky 429 starts the cooldown from headers and cancels the body unread', async (t) => {
+  environment(t, openSkyAnonEnv);
+  t.mock.method(console, 'warn', () => {});
+  t.mock.method(console, 'error', () => {});
+  let states = 0;
+  let limited;
+  t.mock.method(globalThis, 'fetch', async (url) => {
+    if (url.includes('/states/')) {
+      states++;
+      limited = oversizedResponse('x', {
+        status: 429,
+        headers: { 'x-rate-limit-retry-after-seconds': '600' },
+      });
+      return limited.response;
+    }
+    if (url.includes('/lat/')) return regionalFallback();
+    throw Error(`Unexpected URL: ${url}`);
+  });
+  const fresh = await import(
+    `../../server/providers/aircraft/opensky.js?ratelimit=${Math.random()}`
+  );
+  const request = install(fresh.openSkyProxy());
+  const first = await request('/api/flights', '?lat=30&lon=-97');
+  assert.equal(first.statusCode, 200);
+  assert.equal(first.headers['x-flight-source'], 'adsb.lol');
+  assert.equal(states, 1, 'a rate-limited snapshot is not retried');
+  assert.equal(limited.state.cancelled, true);
+  assert.ok(limited.state.pulled <= 1, 'the 429 body is not read');
+  const cooling = await request('/api/flights', '?lat=30&lon=-97');
+  assert.equal(cooling.statusCode, 200);
+  assert.equal(
+    cooling.headers['x-opensky-auth-reason'],
+    'opensky_cooldown_regional_fallback',
+  );
+  assert.equal(states, 1, 'the cooldown holds off the next upstream attempt');
+});
+
+test('OpenSky 429 without retry-after cools down for the 2 minute default', async (t) => {
+  environment(t, openSkyAnonEnv);
+  t.mock.method(console, 'warn', () => {});
+  t.mock.method(console, 'error', () => {});
+  const start = Date.now();
+  let now = start;
+  t.mock.method(Date, 'now', () => now);
+  let states = 0;
+  t.mock.method(globalThis, 'fetch', async (url) => {
+    if (url.includes('/states/')) {
+      states++;
+      return new Response('{}', { status: 429 });
+    }
+    if (url.includes('/lat/')) return regionalFallback();
+    throw Error(`Unexpected URL: ${url}`);
+  });
+  const fresh = await import(
+    `../../server/providers/aircraft/opensky.js?ratelimit-default=${Math.random()}`
+  );
+  const request = install(fresh.openSkyProxy());
+  await request('/api/flights', '?lat=30&lon=-97');
+  assert.equal(states, 1);
+  // Past the 30 s floor but inside the 2 minute default.
+  now = start + 90_000;
+  await request('/api/flights', '?lat=30&lon=-97');
+  assert.equal(states, 1, 'a missing header is not read as a zero wait');
+});
+
+test('OpenSky does not retry a snapshot body over the cap', async (t) => {
+  environment(t, openSkyAnonEnv);
+  t.mock.method(console, 'warn', () => {});
+  t.mock.method(console, 'error', () => {});
+  const bodies = [];
+  t.mock.method(globalThis, 'fetch', async (url) => {
+    if (url.includes('/states/')) {
+      const oversized = oversizedResponse(' ');
+      bodies.push(oversized.state);
+      return oversized.response;
+    }
+    if (url.includes('/lat/')) return regionalFallback();
+    throw Error(`Unexpected URL: ${url}`);
+  });
+  const fresh = await import(
+    `../../server/providers/aircraft/opensky.js?oversize=${Math.random()}`
+  );
+  const response = await install(fresh.openSkyProxy())(
+    '/api/flights',
+    '?lat=30&lon=-97',
+  );
+  assert.equal(response.statusCode, 200);
+  assert.equal(response.headers['x-flight-source'], 'adsb.lol');
+  assert.equal(bodies.length, 1, 'an oversized body is not fetched twice');
+  assert.equal(bodies[0].cancelled, true);
+  assert.ok(bodies[0].pulled <= 9, 'reading stops at the 8 MiB cap');
+});
+
+test('military aircraft route answers 502 to an oversized body without caching it', async (t) => {
+  t.mock.method(console, 'error', () => {});
+  const bodies = [];
+  t.mock.method(globalThis, 'fetch', async () => {
+    const oversized = oversizedResponse('{"ac":[]}');
+    bodies.push(oversized.state);
+    return oversized.response;
+  });
+  const request = install(providers.adsbLolProxy());
+  const first = await request('/api/military');
+  assert.equal(first.statusCode, 502);
+  assert.equal(bodies[0].cancelled, true);
+  assert.ok(bodies[0].pulled <= 9, 'reading stops at the 8 MiB cap');
+  const again = await request('/api/military');
+  assert.equal(again.statusCode, 502, 'nothing was cached');
+  assert.equal(bodies.length, 2);
+});
+
 test('military aircraft route preserves fresh cache and stale response after upstream failure', async (t) => {
   let now = Date.now();
   let calls = 0;

src/tooling/spaceProviders.test.mjs

@@ -47,6 +47,29 @@ function isolateDisk(t) {
   t.mock.method(fsp, 'writeFile', async () => {});
 }
 
+/** A response that streams 1 MiB chunks of `text` well past every cap, then ends. */
+function oversizedResponse(text) {
+  const state = { pulled: 0, cancelled: false };
+  const chunk = new TextEncoder().encode(
+    text.repeat(Math.ceil((1 << 20) / text.length)),
+  );
+  const response = new Response(
+    new ReadableStream(
+      {
+        pull(controller) {
+          if (++state.pulled > 40) controller.close();
+          else controller.enqueue(chunk);
+        },
+        cancel() {
+          state.cancelled = true;
+        },
+      },
+      { highWaterMark: 0 },
+    ),
+  );
+  return { response, state };
+}
+
 test('portable requests keep fixed origins, encode group data and preserve the 30-day UTC window', () => {
   const tle = celestrakTleUrl('stations&FORMAT=json');
   assert.equal(tle.origin, 'https://celestrak.org');
@@ -120,6 +143,27 @@ test('exported CelesTrak plugin coalesces refreshes, retains stale TLEs, and rea
   assert.equal(disk.body, tle);
 });
 
+test('CelesTrak answers 502 to an oversized TLE body without caching it', async (t) => {
+  isolateDisk(t);
+  t.mock.method(console, 'warn', () => {});
+  const bodies = [];
+  t.mock.method(globalThis, 'fetch', async () => {
+    const oversized = oversizedResponse(
+      'ISS\n1 25544U fixture\n2 25544 fixture\n',
+    );
+    bodies.push(oversized.state);
+    return oversized.response;
+  });
+  const request = install(celestrakProxy());
+  const first = await request('/api/celestrak', '/stations');
+  assert.equal(first.status, 502);
+  assert.equal(bodies[0].cancelled, true);
+  assert.ok(bodies[0].pulled <= 17, 'reading stops at the 16 MiB cap');
+  assert.equal(fsp.writeFile.mock.callCount(), 0);
+  assert.equal((await request('/api/celestrak', '/stations')).status, 502);
+  assert.equal(bodies.length, 2, 'nothing was cached in memory');
+});
+
 for (const preview of [false, true])
   test(`exported launch plugin preserves optional server auth and cache in ${preview ? 'preview' : 'development'}`, async (t) => {
     isolateDisk(t);