fix(providers): bound the remaining upstream body reads

Route the six proxied fetches that still buffered upstream bodies without
a limit through the shared capped readers in src/sources/httpBody.js:
FIRMS CSV, CelesTrak TLE groups, adsb.lol /v2/mil (both the OK and
relayed-status paths), TomTom flow tiles, OpenSky states/all and adsbdb
enrichment lookups. Each cap is a bound sized well above the measured
payload, not a budget.

readResponseBytesCapped gains the same optional abort signal as the text
reader, with the same cancel-on-abort plumbing and reader release, so the
TomTom tile path can share the route deadline instead of reading
arrayBuffer() unbounded. Where a route already had a deadline, the same
signal now covers the body read as well as the headers. Two polls had no
deadline at all: the adsb.lol military poll gains a 12-second one and the
OpenSky states/all exchange, including its Basic-auth retry, a 20-second
one; both miss into their existing STALE paths.

FIRMS map-key status, the OpenSky token exchange and CCTV media are left
as they are.

Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Michael JohnstonandClaude Opus 5 committed 2026-09-26 14:40:08 -04:00
1 parent b210ab0fe4
commit 5f8a275487
11 files changed
+278 -34

No files matched your search

+29 -15
View File
@@ -55,38 +55,52 @@ export async function readResponseJsonCapped(response, maxBytes, signal) {
/**
* Read a fetch() Response body as bytes with the same hard cap as
* readResponseTextCapped — for protobuf upstreams (GTFS-Realtime).
* readResponseTextCapped — for protobuf and tile upstreams (GTFS-Realtime,
* TomTom flow). Same early Content-Length rejection, running cap while
* streaming and cancellation on abort, so callers can treat the two alike.
* Throws { code:'RESPONSE_TOO_LARGE' }.
*/
export async function readResponseBytesCapped(response, maxBytes) {
export async function readResponseBytesCapped(response, maxBytes, signal) {
const tooLarge = () => {
const err = new Error('Upstream response too large');
err.code = 'RESPONSE_TOO_LARGE';
return err;
};
const declared = Number(response.headers.get('content-length'));
if (Number.isFinite(declared) && declared > maxBytes) throw tooLarge();
if (Number.isFinite(declared) && declared > maxBytes) {
void response.body?.cancel().catch(() => {});
throw tooLarge();
}
signal?.throwIfAborted();
const reader = response.body?.getReader?.();
if (!reader) {
const bytes = new Uint8Array(await response.arrayBuffer());
signal?.throwIfAborted();
if (bytes.byteLength > maxBytes) throw tooLarge();
return bytes;
}
const cancel = () => {
void reader.cancel().catch(() => {});
};
signal?.addEventListener('abort', cancel, { once: true });
const chunks = [];
let total = 0;
for (;;) {
const { done, value } = await reader.read();
if (done) break;
total += value.byteLength;
if (total > maxBytes) {
try {
await reader.cancel();
} catch {
/* no-op */
}
throw tooLarge();
try {
for (;;) {
signal?.throwIfAborted();
const { done, value } = await reader.read();
signal?.throwIfAborted();
if (done) break;
total += value.byteLength;
if (total > maxBytes) throw tooLarge();
chunks.push(value);
}
chunks.push(value);
} catch (error) {
cancel();
throw error;
} finally {
signal?.removeEventListener('abort', cancel);
reader.releaseLock();
}
const out = new Uint8Array(total);
let offset = 0;