fix(providers): bound the remaining upstream body reads

Route the six proxied fetches that still buffered upstream bodies without
a limit through the shared capped readers in src/sources/httpBody.js:
FIRMS CSV, CelesTrak TLE groups, adsb.lol /v2/mil (both the OK and
relayed-status paths), TomTom flow tiles, OpenSky states/all and adsbdb
enrichment lookups. Each cap is a bound sized well above the measured
payload, not a budget.

readResponseBytesCapped gains the same optional abort signal as the text
reader, with the same cancel-on-abort plumbing and reader release, so the
TomTom tile path can share the route deadline instead of reading
arrayBuffer() unbounded. Where a route already had a deadline, the same
signal now covers the body read as well as the headers. Two polls had no
deadline at all: the adsb.lol military poll gains a 12-second one and the
OpenSky states/all exchange, including its Basic-auth retry, a 20-second
one; both miss into their existing STALE paths.

FIRMS map-key status, the OpenSky token exchange and CCTV media are left
as they are.

Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Michael JohnstonandClaude Opus 5 committed 2026-09-26 14:40:08 -04:00
1 parent b210ab0fe4
commit 5f8a275487
11 files changed
+278 -34

No files matched your search

+11 -4
View File
@@ -1,6 +1,7 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { readResponseBytesCapped } from './common/http.js';
import {
isValidTileCoord as isValidTomTomTile,
utcDayKey as tomtomUtcDayKey,
@@ -48,6 +49,8 @@ export function tomtomProxy() {
const DEFAULT_DAILY_BUDGET = 6000;
const MEM_MAX_ENTRIES = 256;
const UPSTREAM_TIMEOUT_MS = 15000;
/** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */
const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
/** @type {Map<string, {at:number, buf:Buffer}>} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */
const mem = new Map();
@@ -143,11 +146,15 @@ export function tomtomProxy() {
'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' +
`${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`;
recordUpstreamFetch(); // attempts count — upstream bills the request either way
const res = await fetch(url, {
signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
});
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const res = await fetch(url, { signal });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const buf = Buffer.from(await res.arrayBuffer());
const bytes = await readResponseBytesCapped(
res,
TOMTOM_TILE_MAX_RESPONSE_BYTES,
signal,
);
const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
if (buf.length === 0) throw new Error('empty tile body');
return buf;
}