mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-13 14:52:19 +00:00
Merge upstream/main: bikeshare Toronto and scoped city failures
Upstream's #641 (bikeshare Toronto, city failures scoped to range with partial outages read DEGRADED/STALE) replaced the layer-level GBFS error strings this branch had msg()-marked, in the two paths that conflicted. - ingestion.js/viewport.js taken as upstream's per-city _failedCityIds tracking; the layer-level error assignments are gone. - 'GBFS fetch error' still reaches the panel via stats.error from controls.js, so its mark moved there (translateFeedError keeps translating it). - 'GBFS status update failed' no longer exists anywhere: pruned from es/ru and from the layerStatusText SITES table, which now pins the controls.js site instead of the removed ones. Also brings #581 (upstream-read caps, OpenSky 429 cooldown) - no i18n surface. npm run i18n: complete (2553), no unmarked strings. npm test: only the pre-existing environmental OPENAI_API_KEY leak in devFreshDotenv fails. Build passes.
This commit is contained in:
commit
6fe8327ccb
30 files changed
+1102
-96
No files matched your search
@@ -1,3 +1,5 @@
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
|
||||
*
|
||||
@@ -25,6 +27,10 @@ export function adsbLolProxy() {
|
||||
let _cooldownStatus = 0;
|
||||
/** Response cache TTL (ms). */
|
||||
const CACHE_MS = 12000;
|
||||
/** Upstream deadline (ms); the fetch previously had none and could hang a poll. */
|
||||
const UPSTREAM_TIMEOUT_MS = 12000;
|
||||
/** /v2/mil measures ~67 KB; a relayed error page is bounded by the same cap. */
|
||||
const ADSBLOL_MIL_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
/** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */
|
||||
const RATE_LIMIT_COOLDOWN_MS = 30000;
|
||||
/** Cooldown after a 5xx (ms). */
|
||||
@@ -98,11 +104,17 @@ export function adsbLolProxy() {
|
||||
);
|
||||
return;
|
||||
}
|
||||
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
|
||||
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
|
||||
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
|
||||
signal,
|
||||
});
|
||||
if (upstream.ok) {
|
||||
const body = await upstream.text();
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
_cache = body;
|
||||
_cacheAt = Date.now();
|
||||
_cooldownUntil = 0;
|
||||
@@ -129,7 +141,11 @@ export function adsbLolProxy() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
const body = await upstream.text();
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
serve(
|
||||
res,
|
||||
upstream.status,
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
|
||||
/** Route and aircraft lookups are a few hundred bytes; 1 MiB is a hard bound. */
|
||||
const ADSBDB_MAX_RESPONSE_BYTES = 1024 * 1024;
|
||||
/**
|
||||
* adsbdb.com enrichment proxy: callsign → route (airline + origin/destination
|
||||
* airports) and hex → aircraft type/registration. Free community API — cached
|
||||
@@ -80,12 +84,16 @@ export function adsbdbProxy() {
|
||||
kind === 'route'
|
||||
? `https://api.adsbdb.com/v0/callsign/${encodeURIComponent(key)}`
|
||||
: `https://api.adsbdb.com/v0/aircraft/${encodeURIComponent(key)}`;
|
||||
const res = await fetch(url, { signal: AbortSignal.timeout(8000) });
|
||||
const signal = AbortSignal.timeout(8000);
|
||||
const res = await fetch(url, { signal });
|
||||
if (res.ok) {
|
||||
const payload = await readResponseJsonCapped(
|
||||
res,
|
||||
ADSBDB_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
const data =
|
||||
kind === 'route'
|
||||
? parseRoute(await res.json())
|
||||
: parseAircraft(await res.json());
|
||||
kind === 'route' ? parseRoute(payload) : parseAircraft(payload);
|
||||
store[key] = { at: Date.now(), data }; // data may be null — negative cache
|
||||
dirty = true;
|
||||
return data;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback
|
||||
import {
|
||||
coalesceProxyRequest,
|
||||
readResponseJsonCapped,
|
||||
readResponseTextCapped,
|
||||
} from '../common/http.js';
|
||||
import { requiredFiniteQueryNumber } from '../common/query.js';
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -29,11 +30,34 @@ const OPENSKY_CACHE_MS = 9000;
|
||||
const OPENSKY_ATTEMPT_TIMEOUT_MS = 10_000;
|
||||
const OPENSKY_STATES_URL =
|
||||
'https://opensky-network.org/api/states/all?extended=1';
|
||||
/** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */
|
||||
const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Credit-governor cooldown for a 429: OpenSky's retry-after bounded to
|
||||
* 30 s … 30 min, or 2 min when the header is absent.
|
||||
* @param {Response} upstream Rate-limited response.
|
||||
* @returns {number} Cooldown (ms).
|
||||
*/
|
||||
function openSkyRateLimitCooldownMs(upstream) {
|
||||
// headers.get() gives null when absent, and Number(null) is 0, not NaN.
|
||||
const header = upstream.headers.get('x-rate-limit-retry-after-seconds');
|
||||
const retryAfterSec =
|
||||
header == null || header.trim() === '' ? NaN : Number(header);
|
||||
return Math.min(
|
||||
Math.max(
|
||||
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
|
||||
30_000,
|
||||
),
|
||||
30 * 60_000,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch and read the global snapshot, giving each attempt
|
||||
* OPENSKY_ATTEMPT_TIMEOUT_MS and retrying once when an attempt times out or
|
||||
* the connection fails. A second failure throws, which the proxy answers
|
||||
* the connection fails. An oversized body is not retried: it would fail again
|
||||
* and spend more credits. A second failure throws, which the proxy answers
|
||||
* from its stale cache or the regional fallback.
|
||||
* @param {Record<string, string>} headers Request headers.
|
||||
* @returns {Promise<{upstream: Response, body: string}>}
|
||||
@@ -41,13 +65,27 @@ const OPENSKY_STATES_URL =
|
||||
async function fetchOpenSkyStates(headers) {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
const upstream = await fetch(OPENSKY_STATES_URL, {
|
||||
headers,
|
||||
signal: AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS),
|
||||
});
|
||||
return { upstream, body: await upstream.text() };
|
||||
const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
|
||||
const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
|
||||
if (upstream.status === 429) {
|
||||
// Start the cooldown from the headers and drop the rejected body
|
||||
// unread, so a stalled or oversized 429 body cannot skip it.
|
||||
_openskyCooldownUntil =
|
||||
Date.now() + openSkyRateLimitCooldownMs(upstream);
|
||||
void upstream.body?.cancel().catch(() => {});
|
||||
return {
|
||||
upstream,
|
||||
body: JSON.stringify({ error: 'OpenSky rate limited.' }),
|
||||
};
|
||||
}
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
OPENSKY_STATES_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
return { upstream, body };
|
||||
} catch (error) {
|
||||
if (attempt >= 2) throw error;
|
||||
if (attempt >= 2 || error?.code === 'RESPONSE_TOO_LARGE') throw error;
|
||||
console.warn(
|
||||
'[OpenSky] global snapshot attempt failed, retrying:',
|
||||
error?.message || error,
|
||||
@@ -544,19 +582,9 @@ export function openSkyProxy() {
|
||||
}
|
||||
if (upstream.status === 429) {
|
||||
reason = 'rate_limited';
|
||||
// Credit governor: honor OpenSky's retry-after (bounded 30 s … 30 min;
|
||||
// 2 min when the header is absent) — no upstream attempts until then.
|
||||
const retryAfterSec = Number(
|
||||
upstream.headers.get('x-rate-limit-retry-after-seconds'),
|
||||
);
|
||||
const cooldownMs = Math.min(
|
||||
Math.max(
|
||||
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
|
||||
30_000,
|
||||
),
|
||||
30 * 60_000,
|
||||
);
|
||||
_openskyCooldownUntil = now + cooldownMs;
|
||||
// Credit governor: fetchOpenSkyStates already started the cooldown
|
||||
// from OpenSky's retry-after — no upstream attempts until then.
|
||||
const cooldownMs = openSkyRateLimitCooldownMs(upstream);
|
||||
// Serve the last-good body instead of the 429 when we have one —
|
||||
// the layer keeps rendering (STALE-cued) instead of dying.
|
||||
if (_openskyCacheBody && _openskyCacheStatus === 200) {
|
||||
|
||||
@@ -2,6 +2,7 @@ import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
|
||||
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
|
||||
import { readResponseTextCapped } from './common/http.js';
|
||||
|
||||
/**
|
||||
* NASA FIRMS live active-fire proxy with a memory + disk cache.
|
||||
@@ -35,6 +36,12 @@ export function firmsProxy() {
|
||||
'VIIRS_SNPP_NRT',
|
||||
'MODIS_NRT',
|
||||
];
|
||||
/**
|
||||
* Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
|
||||
* 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
|
||||
* bound, not a budget.
|
||||
*/
|
||||
const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024;
|
||||
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
|
||||
const CACHE_PATH = path.join(CACHE_DIR, 'firms.json');
|
||||
|
||||
@@ -83,9 +90,12 @@ export function firmsProxy() {
|
||||
*/
|
||||
async function fetchSource(key, source) {
|
||||
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
|
||||
const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
|
||||
const signal = AbortSignal.timeout(60_000);
|
||||
const res = await fetch(url, { signal });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const records = parseFirmsCsv(await res.text());
|
||||
const records = parseFirmsCsv(
|
||||
await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
|
||||
);
|
||||
if (records === null) throw new Error('non-CSV upstream response');
|
||||
return records;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: CelesTrak TLE proxy.
|
||||
@@ -48,10 +49,14 @@ export function celestrakProxy() {
|
||||
}
|
||||
}
|
||||
|
||||
/** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */
|
||||
const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
async function fetchUpstream(group) {
|
||||
const url = celestrakTleUrl(group);
|
||||
const signal = AbortSignal.timeout(20000);
|
||||
const res = await fetch(url.toString(), {
|
||||
signal: AbortSignal.timeout(20000),
|
||||
signal,
|
||||
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
|
||||
// descriptive User-Agent with a contact point.
|
||||
headers: {
|
||||
@@ -60,7 +65,11 @@ export function celestrakProxy() {
|
||||
},
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const body = await res.text();
|
||||
const body = await readResponseTextCapped(
|
||||
res,
|
||||
CELESTRAK_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
|
||||
if (!/^1 /m.test(body)) throw new Error('no TLE lines in response');
|
||||
return { at: Date.now(), body };
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
|
||||
import { readResponseBytesCapped } from './common/http.js';
|
||||
import {
|
||||
isValidTileCoord as isValidTomTomTile,
|
||||
utcDayKey as tomtomUtcDayKey,
|
||||
@@ -48,6 +49,8 @@ export function tomtomProxy() {
|
||||
const DEFAULT_DAILY_BUDGET = 6000;
|
||||
const MEM_MAX_ENTRIES = 256;
|
||||
const UPSTREAM_TIMEOUT_MS = 15000;
|
||||
/** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */
|
||||
const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/** @type {Map<string, {at:number, buf:Buffer}>} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */
|
||||
const mem = new Map();
|
||||
@@ -143,11 +146,15 @@ export function tomtomProxy() {
|
||||
'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' +
|
||||
`${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`;
|
||||
recordUpstreamFetch(); // attempts count — upstream bills the request either way
|
||||
const res = await fetch(url, {
|
||||
signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
|
||||
});
|
||||
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
|
||||
const res = await fetch(url, { signal });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
const bytes = await readResponseBytesCapped(
|
||||
res,
|
||||
TOMTOM_TILE_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
if (buf.length === 0) throw new Error('empty tile body');
|
||||
return buf;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user