fix(providers): bound the remaining upstream body reads

Route the six proxied fetches that still buffered upstream bodies without
a limit through the shared capped readers in src/sources/httpBody.js:
FIRMS CSV, CelesTrak TLE groups, adsb.lol /v2/mil (both the OK and
relayed-status paths), TomTom flow tiles, OpenSky states/all and adsbdb
enrichment lookups. Each cap is a bound sized well above the measured
payload, not a budget.

readResponseBytesCapped gains the same optional abort signal as the text
reader, with the same cancel-on-abort plumbing and reader release, so the
TomTom tile path can share the route deadline instead of reading
arrayBuffer() unbounded. Where a route already had a deadline, the same
signal now covers the body read as well as the headers. Two polls had no
deadline at all: the adsb.lol military poll gains a 12-second one and the
OpenSky states/all exchange, including its Basic-auth retry, a 20-second
one; both miss into their existing STALE paths.

FIRMS map-key status, the OpenSky token exchange and CCTV media are left
as they are.

Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Michael JohnstonandClaude Opus 5 committed 2026-09-26 14:40:08 -04:00
1 parent b210ab0fe4
commit 5f8a275487
11 files changed
+278 -34

No files matched your search

+12 -2
View File
@@ -2,6 +2,7 @@ import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
import { readResponseTextCapped } from './common/http.js';
/**
* NASA FIRMS live active-fire proxy with a memory + disk cache.
@@ -35,6 +36,12 @@ export function firmsProxy() {
'VIIRS_SNPP_NRT',
'MODIS_NRT',
];
/**
* Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
* 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
* bound, not a budget.
*/
const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024;
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
const CACHE_PATH = path.join(CACHE_DIR, 'firms.json');
@@ -83,9 +90,12 @@ export function firmsProxy() {
*/
async function fetchSource(key, source) {
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
const signal = AbortSignal.timeout(60_000);
const res = await fetch(url, { signal });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const records = parseFirmsCsv(await res.text());
const records = parseFirmsCsv(
await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
);
if (records === null) throw new Error('non-CSV upstream response');
return records;
}