From f4054469d567b73ba225f9e251100ab4628efd0d Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Fri, 11 Sep 2026 21:57:50 -0700 Subject: [PATCH] refactor: split terrain traffic fire and bike-share providers --- CHANGELOG.md | 3 + docs/CODE-BOUNDARIES.md | 20 + docs/CURRENT-STATE.md | 10 + package.json | 17 +- scripts/package-boundaries.json | 42 ++ server/providers/firms.js | 227 ++++++ server/providers/gbfs.js | 126 ++++ server/providers/local.js | 802 +--------------------- server/providers/terrain.js | 162 +++++ server/providers/traffic.js | 239 +++++++ src/data/gbfsSource.js | 52 ++ src/proxyErrorResponses.test.mjs | 2 +- src/tooling/environmentProviders.test.mjs | 119 ++++ 13 files changed, 1024 insertions(+), 797 deletions(-) create mode 100644 server/providers/firms.js create mode 100644 server/providers/gbfs.js create mode 100644 server/providers/terrain.js create mode 100644 server/providers/traffic.js create mode 100644 src/data/gbfsSource.js create mode 100644 src/tooling/environmentProviders.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index ceb2c887c..44ce2aff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md ## [Unreleased] +- Separate terrain, traffic, FIRMS and GBFS middleware into focused provider + modules, preserving local configuration, routes and cache/error behavior. + - Split satellite and launch-feed server providers into focused modules with portable request URL builders, preserving routes and cache/error behavior. diff --git a/docs/CODE-BOUNDARIES.md b/docs/CODE-BOUNDARIES.md index bf7525e4e..5fbdba635 100644 --- a/docs/CODE-BOUNDARIES.md +++ b/docs/CODE-BOUNDARIES.md @@ -132,3 +132,23 @@ Callers supply the group or end date and own validation, credentials, transport, response limits and cache policy. The boundary gate checks this portable entry separately from the Node providers. Satellite rendering and launch replay remain in their existing browser modules. + +## Terrain, traffic, fires and bike-share providers + +`gods-eye-view/server/providers/terrain`, `/traffic`, `/firms` and `/gbfs` +are separate Node-only entries. Each owns its existing middleware and +process-scoped cache or request handling. Standalone composition mounts them in +the original order; their imports do not start acquisition. + +`gods-eye-view/sources/terrain` exports existing point-key, retry and cache +reconstruction mechanics with injectable acquisition dependencies. +`gods-eye-view/sources/traffic` exports tile math and budget calculations. +`gods-eye-view/sources/gbfs` exports host/path acceptance and cache-header rules. +These entries import no Node middleware, application configuration or rendering. +Callers retain their request admission and transport policy. FIRMS CSV parsing +remains an owned dependency of the FIRMS Node provider. The boundary gate +checks each entry independently. + +Browser terrain sampling, traffic matching/drawing, fire overlays and bike-share +layer lifecycle remain in their current modules. This extraction changes no +source defaults, credentials, quotas, data interpretation or visual behavior. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 14bc6c4ec..250a2f106 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,15 @@ # God's Eye View Current State +## Terrain, traffic, fire and bike-share provider modules + +Local composition now imports separate Node modules for Re:Earth heights, +TomTom flow tiles, NASA FIRMS detections and GBFS station feeds. Existing routes, +plugin order, server-key selection, validation, disk caches, budgets, retries +and stale/error responses remain unchanged. Each has a Node-only package entry +under `gods-eye-view/server/providers/`. Portable terrain mechanics, traffic tile +math and GBFS source rules are available under `gods-eye-view/sources/`. +The browser layers and their rendering remain in their existing modules. + ## Landmark annotation identity When a landmark geocode contains only address components, annotations retain diff --git a/package.json b/package.json index 407c374d6..b93c900ff 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,21 @@ "./server/providers/space": { "node": "./server/providers/space.js" }, - "./sources/space": "./src/data/spaceProviderRequests.js" + "./sources/space": "./src/data/spaceProviderRequests.js", + "./server/providers/terrain": { + "node": "./server/providers/terrain.js" + }, + "./sources/terrain": "./src/data/terrainHeightsProxy.js", + "./server/providers/traffic": { + "node": "./server/providers/traffic.js" + }, + "./sources/traffic": "./src/data/tomtomTiles.js", + "./server/providers/firms": { + "node": "./server/providers/firms.js" + }, + "./server/providers/gbfs": { + "node": "./server/providers/gbfs.js" + }, + "./sources/gbfs": "./src/data/gbfsSource.js" } } diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 1b93c61e4..06d90f55f 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -89,5 +89,47 @@ "exports": ["./sources/space"], "modules": ["src/data/spaceProviderRequests.js"], "external": [] + }, + "terrain-provider": { + "runtime": "node", + "exports": ["./server/providers/terrain"], + "modules": [ + "server/providers/terrain.js", + "src/data/terrainHeightsProxy.js" + ], + "external": [] + }, + "terrain-source": { + "exports": ["./sources/terrain"], + "modules": ["src/data/terrainHeightsProxy.js"], + "external": [] + }, + "traffic-provider": { + "runtime": "node", + "exports": ["./server/providers/traffic"], + "modules": ["server/providers/traffic.js", "src/data/tomtomTiles.js"], + "external": [] + }, + "traffic-source": { + "exports": ["./sources/traffic"], + "modules": ["src/data/tomtomTiles.js"], + "external": [] + }, + "firms-provider": { + "runtime": "node", + "exports": ["./server/providers/firms"], + "modules": ["server/providers/firms.js", "src/data/firmsCsv.js"], + "external": [] + }, + "gbfs-provider": { + "runtime": "node", + "exports": ["./server/providers/gbfs"], + "modules": ["server/providers/gbfs.js", "src/data/gbfsSource.js"], + "external": [] + }, + "gbfs-source": { + "exports": ["./sources/gbfs"], + "modules": ["src/data/gbfsSource.js"], + "external": [] } } diff --git a/server/providers/firms.js b/server/providers/firms.js new file mode 100644 index 000000000..8326a612d --- /dev/null +++ b/server/providers/firms.js @@ -0,0 +1,227 @@ +import path from 'node:path'; +import { promises as fsp } from 'node:fs'; + +import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js'; + +/** + * NASA FIRMS live active-fire proxy with a memory + disk cache. + * Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2 + * + * Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent + * satellites, no cross-source dedup) fetched sequentially with `days=2` + * (`days=1` means "current UTC day", nearly empty just after 00:00Z) and + * clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS quota is + * 5,000 transactions / 10 min per MAP_KEY, so the cache is the point: + * TTL 30 min, single-flight refresh, serve-stale-on-failure, and a + * fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream + * fetch across dev-server restarts. Pattern mirrors celestrakProxy. + * + * Routes: + * GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires} + * GET /api/firms/status → {hasKey, lastFetch, count, stale, ttlMs, transactions} + * + * Keyless (no FIRMS_MAP_KEY): /api/firms → 503 {error:'no_key'}; status → + * {hasKey:false}. Upstream is never touched without a key. + * + * @returns {import('vite').Plugin} + */ +export function firmsProxy() { + const TTL_MS = 30 * 60_000; + const STATUS_TTL_MS = 5 * 60_000; + const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'VIIRS_SNPP_NRT']; + const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); + const CACHE_PATH = path.join(CACHE_DIR, 'firms.json'); + + /** @type {?{at: number, sources: Array, fires: Array}} */ + let mem = null; + let diskChecked = false; + /** @type {?Promise, fires: Array}>} single-flight refresh */ + let inflight = null; + /** @type {?{at: number, transactions: ?{used: number, limit: number}}} mapkey_status cache */ + let statusCache = null; + /** @type {?Promise} */ + let statusInflight = null; + + const mapKey = () => String(process.env.FIRMS_MAP_KEY || '').trim(); + + async function readDiskOnce() { + if (diskChecked) return; + diskChecked = true; + try { + const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); + if (Number.isFinite(parsed?.at) && Array.isArray(parsed?.sources) && Array.isArray(parsed?.fires)) { + mem = parsed; + } + } catch { /* no disk cache yet */ } + } + + async function writeDisk(entry) { + try { + await fsp.mkdir(CACHE_DIR, { recursive: true }); + await fsp.writeFile(CACHE_PATH, JSON.stringify(entry), 'utf8'); + } catch (err) { + console.warn('[firms-proxy] cache write failed:', err?.message || err); + } + } + + /** + * Fetch + parse one FIRMS source. Throws on HTTP error or a non-CSV body + * (FIRMS reports errors as HTML/plain text, never CSV). Never log the URL — + * it embeds the MAP_KEY. + */ + async function fetchSource(key, source) { + const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`; + const res = await fetch(url, { signal: AbortSignal.timeout(60_000) }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const records = parseFirmsCsv(await res.text()); + if (records === null) throw new Error('non-CSV upstream response'); + return records; + } + + /** + * Refresh all sources sequentially (quota courtesy — never in parallel). + * Partial success (≥1 source ok) still produces a cacheable entry with the + * failed sources marked ok:false; total failure throws so the caller can + * serve stale. + */ + async function refreshUpstream(key) { + const now = Date.now(); + const sources = []; + const fires = []; + for (const source of SOURCES) { + try { + const records = filterTrailing24h(await fetchSource(key, source), now); + // NOT fires.push(...records): spread passes each record as an argument, + // and a world/2 VIIRS pull exceeds V8's argument limit (~125k) at + // ~131k records — RangeError, and the whole source is silently dropped. + for (const record of records) fires.push(record); + sources.push({ source, count: records.length, ok: true }); + } catch (err) { + console.warn(`[firms-proxy] ${source} fetch failed:`, err?.message || err); + sources.push({ source, count: 0, ok: false }); + } + } + if (!sources.some((s) => s.ok)) throw new Error('all FIRMS sources failed'); + return { at: now, sources, fires }; + } + + /** + * Cache entry → response payload. Fires are RE-filtered to the trailing + * 24 h at serve time so a stale cache never serves >24h-old detections. + */ + function buildPayload(entry, stale) { + const fires = filterTrailing24h(entry.fires, Date.now()); + return { + fetchedAt: entry.at, + stale, + ttlMs: TTL_MS, + sources: entry.sources, + count: fires.length, + fires, + }; + } + + /** mapkey_status transactions, cached 5 min, best-effort (null on failure). */ + function getTransactions(key) { + const now = Date.now(); + if (statusCache && now - statusCache.at < STATUS_TTL_MS) { + return Promise.resolve(statusCache.transactions); + } + if (!statusInflight) { + statusInflight = (async () => { + try { + const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`; + const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const body = await res.json(); + const used = Number(body?.current_transactions); + const limit = Number(body?.transaction_limit); + return Number.isFinite(used) && Number.isFinite(limit) ? { used, limit } : null; + } catch (err) { + console.warn('[firms-proxy] mapkey status failed:', err?.message || err); + return null; + } + })() + .then((transactions) => { + statusCache = { at: Date.now(), transactions }; + return transactions; + }) + .finally(() => { statusInflight = null; }); + } + return statusInflight; + } + + return { + name: 'firms-proxy', + configureServer(server) { + server.middlewares.use('/api/firms', async (req, res) => { + const sendJson = (status, obj) => { + if (res.headersSent) return; + res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify(obj)); + }; + try { + const subPath = String(req.url || '').split('?')[0]; + const key = mapKey(); + await readDiskOnce(); + + if (subPath === '/status') { + if (!key) { + sendJson(200, { hasKey: false, lastFetch: null, count: null, stale: false, ttlMs: TTL_MS, transactions: null }); + return; + } + const transactions = await getTransactions(key); + sendJson(200, { + hasKey: true, + lastFetch: mem ? mem.at : null, + count: mem ? mem.fires.length : null, + stale: mem ? Date.now() - mem.at >= TTL_MS : false, + ttlMs: TTL_MS, + transactions, + }); + return; + } + + if (!key) { + sendJson(503, { error: 'no_key' }); + return; + } + + const entry = mem; + if (entry && Date.now() - entry.at < TTL_MS) { + sendJson(200, buildPayload(entry, false)); + return; + } + // Stale or missing → refresh, single-flight (concurrent requests + // share one upstream pass). Capture the promise locally BEFORE + // awaiting: the .finally() nulls `inflight` the moment it settles. + if (!inflight) { + inflight = refreshUpstream(key) + .then(async (fresh) => { + mem = fresh; + await writeDisk(fresh); + return fresh; + }) + .catch((err) => { + console.warn(`[firms-proxy] refresh failed (${err?.message || err}) — serving cache if any`); + return null; + }) + .finally(() => { inflight = null; }); + } + const pending = inflight; + const fresh = await pending; + if (fresh) { + sendJson(200, buildPayload(fresh, false)); + } else if (entry) { + sendJson(200, buildPayload(entry, true)); // upstream down — stale beats empty + } else { + sendJson(502, { error: 'firms fetch failed and no cache available' }); + } + } catch (err) { + console.warn('[firms-proxy] error:', err?.message || err); + sendJson(500, { error: 'firms proxy error' }); + } + }); + }, + }; +} diff --git a/server/providers/gbfs.js b/server/providers/gbfs.js new file mode 100644 index 000000000..bc9bb0ed6 --- /dev/null +++ b/server/providers/gbfs.js @@ -0,0 +1,126 @@ +import { isAllowedGbfsHost, isAllowedGbfsPath, gbfsCacheControl } from '../../src/data/gbfsSource.js'; + +// --------------------------------------------------------------------------- +// GBFS (General Bikeshare Feed Specification) proxy constants +// --------------------------------------------------------------------------- +/** Upstream fetch timeout for GBFS requests (ms). */ +const GBFS_PROXY_TIMEOUT_MS = 12000; + +/** + * Vite plugin: GBFS bike-share proxy with host allowlisting and size limits. + * + * Accepts GET /api/gbfs/ and proxies the request + * to the upstream GBFS provider. Validates hostname against an allowlist, + * restricts to station_information/station_status paths, enforces HTTPS, + * and caps response body at 5 MB. + * + * @returns {import('vite').Plugin} + */ +export function gbfsProxy() { + return { + name: 'gbfs-proxy', + configureServer(server) { + server.middlewares.use('/api/gbfs', async (req, res) => { + try { + if (req.method !== 'GET') { + res.writeHead(405, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Method Not Allowed' })); + return; + } + + const url = new URL(req.url || '/', 'http://localhost'); + const encodedTarget = url.pathname.replace(/^\/+/, ''); + if (!encodedTarget) { + res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Missing GBFS upstream target' })); + return; + } + + let decodedTarget = ''; + try { + decodedTarget = decodeURIComponent(encodedTarget); + } catch { + res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Invalid GBFS target encoding' })); + return; + } + + let upstreamUrl = null; + try { + upstreamUrl = new URL(decodedTarget); + } catch { + res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Invalid GBFS upstream URL' })); + return; + } + + if (upstreamUrl.protocol !== 'https:') { + res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Only https GBFS targets are allowed' })); + return; + } + + if (!isAllowedGbfsHost(upstreamUrl.hostname)) { + res.writeHead(403, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'GBFS host not allowed' })); + return; + } + + if (!isAllowedGbfsPath(upstreamUrl.pathname)) { + res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Only station_information/station_status endpoints are allowed' })); + return; + } + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), GBFS_PROXY_TIMEOUT_MS); + let upstream; + try { + upstream = await fetch(upstreamUrl.toString(), { + method: 'GET', + headers: { + Accept: 'application/json', + 'User-Agent': 'gods-eye-view-gbfs-proxy/1.0', + }, + signal: controller.signal, + }); + } finally { + clearTimeout(timeoutId); + } + + // Limit response size to prevent memory exhaustion from malicious upstream + const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB + const contentLength = Number(upstream.headers.get('content-length')); + if (Number.isFinite(contentLength) && contentLength > GBFS_MAX_BODY_BYTES) { + res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'GBFS upstream response too large' })); + return; + } + const body = await upstream.text(); + if (Buffer.byteLength(body, 'utf8') > GBFS_MAX_BODY_BYTES) { + res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'GBFS upstream response too large' })); + return; + } + const contentType = upstream.headers.get('content-type') || 'application/json'; + res.writeHead(upstream.status, { + 'Content-Type': contentType, + 'Cache-Control': gbfsCacheControl(upstreamUrl.pathname), + 'X-GBFS-Upstream': upstreamUrl.hostname, + 'X-GBFS-Cache': 'MISS', + }); + res.end(body); + } catch (error) { + if (error?.name === 'AbortError') { + res.writeHead(504, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'GBFS upstream timeout' })); + return; + } + console.error('[GBFS Proxy]', error?.message || String(error)); + res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'GBFS proxy error' })); + } + }); + }, + }; +} diff --git a/server/providers/local.js b/server/providers/local.js index 81f9b3fdb..934e7c442 100644 --- a/server/providers/local.js +++ b/server/providers/local.js @@ -1,3 +1,7 @@ +import { terrainHeightsProxy } from './terrain.js'; +import { tomtomProxy } from './traffic.js'; +import { firmsProxy } from './firms.js'; +import { gbfsProxy } from './gbfs.js'; import { celestrakProxy, rocketLaunchesProxy } from './space.js'; export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js'; /** @@ -48,13 +52,8 @@ import { Readable } from 'node:stream'; import https from 'node:https'; import { lookup as lookupDns } from 'node:dns/promises'; import { directionToHeading } from '../../src/data/directionText.js'; -import { - isValidTileCoord as isValidTomTomTile, - utcDayKey as tomtomUtcDayKey, - normalizeBudget as normalizeTomTomBudget, - isOverBudget as isTomTomOverBudget, -} from '../../src/data/tomtomTiles.js'; -import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js'; + + import { fileURLToPath } from 'node:url'; import { normalizeRadioCountryInput } from '../../src/data/radioCountry.js'; import { @@ -74,13 +73,7 @@ import { validateKeySetupUpdates, } from '../../src/keySetupCore.mjs'; import { hardenCredentialFile } from '../../src/keySetupHardening.mjs'; -import { - fetchTerrainChunkWithRetry, - parseTerrainPoints, - resolveTerrainHeightRequest, - terrainPointKey, - validTerrainResult, -} from '../../src/data/terrainHeightsProxy.js'; + import { VOICE_MODELS, isKnownVoiceTier, resolveVoiceModel } from '../../src/voice/voiceCost.js'; /** Resolve __dirname for ESM context. */ @@ -1111,22 +1104,6 @@ function radioBrowserProxy() { configurePreviewServer: install, }; } -// --------------------------------------------------------------------------- -// GBFS (General Bikeshare Feed Specification) proxy constants -// --------------------------------------------------------------------------- -/** Upstream fetch timeout for GBFS requests (ms). */ -const GBFS_PROXY_TIMEOUT_MS = 12000; -/** Allowlisted GBFS hostnames; wildcard *.publicbikesystem.net also accepted. */ -const GBFS_ALLOWED_HOSTS = new Set([ - 'gbfs.lyft.com', - 'gbfs.bluebikes.com', - 'gbfs.bcycle.com', - 'gbfs.biketownpdx.com', - 'gbfs.cogobikeshare.com', - 'austin.publicbikesystem.net', - 'hon.publicbikesystem.net', - 'chat.publicbikesystem.net', -]); // Sourced from the shared voice-model registry so the client's cost estimate // can never be computed against a different model than the session runs on. const OPENAI_REALTIME_MODEL_DEFAULT = VOICE_MODELS.standard.id; @@ -1140,611 +1117,6 @@ const REALTIME_DEBUG_LOG_DIR = path.join(__dirname, '.gev-logs'); const REALTIME_DEBUG_LOG_FILE = path.join(REALTIME_DEBUG_LOG_DIR, 'realtime-conversations.jsonl'); const REALTIME_DEBUG_LOG_MAX_BYTES = 8 * 1024 * 1024; -/** - * TomTom traffic-flow vector-tile proxy with a daily budget governor. - * - * Upstream: https://api.tomtom.com/traffic/map/4/tile/flow/relative/{z}/{x}/{y}.pbf - * (style `relative`; the response is an UNCOMPRESSED Mapbox Vector Tile, layer - * "Traffic flow"). The key comes from TOMTOM_API_KEY server-side only — the - * browser fetches same-origin `/api/tomtom/flow/{z}/{x}/{y}.pbf`. - * - * Cache: memory + disk (.gev-cache/tomtom/), TTL 120 s (traffic is fresh - * data), single-flight per tile, serve-stale-on-failure — the celestrakProxy - * pattern. Cache hits never count against the budget. - * - * Budget governor (mirrors the OpenSky credit-governor philosophy — last-good - * data beats a dead layer): a persistent counter (.gev-cache/tomtom/budget.json, - * keyed by UTC date, reset on day change) counts upstream fetch attempts - * against a soft cap (TOMTOM_DAILY_TILE_BUDGET, default 40,000 of the free - * tier's ~50k/day). Over the cap the proxy serves stale tiles when available, - * else 429 {error:'budget'}. - * - * GET /api/tomtom/status → {hasKey, dailyCount, budget, date}. Keyless mode: - * status reports hasKey:false and the tile endpoint 503s {error:'no_key'} - * without touching upstream — the traffic layer then stays in simulation mode. - * - * @returns {import('vite').Plugin} - */ -function tomtomProxy() { - const TILE_TTL_MS = 120_000; - const CACHE_DIR = path.join(process.cwd(), '.gev-cache', 'tomtom'); - const BUDGET_PATH = path.join(CACHE_DIR, 'budget.json'); - const DEFAULT_DAILY_BUDGET = 40000; - const MEM_MAX_ENTRIES = 256; - const UPSTREAM_TIMEOUT_MS = 15000; - - /** @type {Map} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */ - const mem = new Map(); - /** @type {Map>} single-flight per tile. */ - const inflight = new Map(); - - /** @type {{date:string, count:number}|null} lazily-loaded persistent counter. */ - let budget = null; - let budgetLoaded = false; - - function dailyBudgetLimit() { - const raw = Number.parseInt(process.env.TOMTOM_DAILY_TILE_BUDGET || '', 10); - return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_DAILY_BUDGET; - } - - async function loadBudgetOnce() { - if (budgetLoaded) return; - budgetLoaded = true; - try { - const parsed = JSON.parse(await fsp.readFile(BUDGET_PATH, 'utf8')); - if (parsed && typeof parsed.date === 'string' && Number.isFinite(parsed.count)) { - budget = parsed; - } - } catch { /* no budget file yet */ } - } - - async function persistBudget() { - try { - await fsp.mkdir(CACHE_DIR, { recursive: true }); - await fsp.writeFile(BUDGET_PATH, JSON.stringify(budget), 'utf8'); - } catch (err) { - console.warn('[tomtom-proxy] budget write failed:', err?.message || err); - } - } - - /** Roll the counter to today (UTC) and return it. */ - function currentBudget() { - budget = normalizeTomTomBudget(budget, tomtomUtcDayKey()); - return budget; - } - - /** Count one upstream fetch attempt against today's budget (async persist). */ - function recordUpstreamFetch() { - currentBudget().count += 1; - void persistBudget(); - } - - const tilePath = (key) => path.join(CACHE_DIR, `flow-${key.replaceAll('/', '-')}.pbf`); - - /** Disk-cache read; tile age comes from the file's mtime. */ - async function readDiskTile(key) { - try { - const [stat, buf] = await Promise.all([ - fsp.stat(tilePath(key)), - fsp.readFile(tilePath(key)), - ]); - return { at: stat.mtimeMs, buf }; - } catch { return null; } - } - - async function writeDiskTile(key, buf) { - try { - await fsp.mkdir(CACHE_DIR, { recursive: true }); - await fsp.writeFile(tilePath(key), buf); - } catch (err) { - console.warn(`[tomtom-proxy] tile cache write failed for ${key}:`, err?.message || err); - } - } - - /** LRU-ish memory insert (Map preserves insertion order; evict the oldest). */ - function memSet(key, entry) { - if (!mem.has(key) && mem.size >= MEM_MAX_ENTRIES) { - const oldest = mem.keys().next().value; - mem.delete(oldest); - } - mem.set(key, entry); - } - - async function fetchUpstream(z, x, y) { - const url = 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' - + `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`; - recordUpstreamFetch(); // attempts count — upstream bills the request either way - const res = await fetch(url, { signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS) }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const buf = Buffer.from(await res.arrayBuffer()); - if (buf.length === 0) throw new Error('empty tile body'); - return buf; - } - - return { - name: 'tomtom-proxy', - configureServer(server) { - server.middlewares.use('/api/tomtom', async (req, res) => { - // Sanitized responses only (proxy/security baseline): no upstream - // error details, and never echo the key or the upstream URL. - const sendJson = (status, obj, extraHeaders = {}) => { - if (res.headersSent) return; - res.writeHead(status, { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - ...extraHeaders, - }); - res.end(JSON.stringify(obj)); - }; - const sendTile = (buf, cacheStatus) => { - if (res.headersSent) return; - res.writeHead(200, { - 'Content-Type': 'application/x-protobuf', - 'Cache-Control': 'no-store', - 'x-tomtom-cache': cacheStatus, - }); - res.end(buf); - }; - - try { - await loadBudgetOnce(); - const urlPath = String(req.url || '').split('?')[0]; - - if (urlPath === '/status') { - const hasKey = Boolean(process.env.TOMTOM_API_KEY); - const b = currentBudget(); - sendJson(200, { hasKey, dailyCount: b.count, budget: dailyBudgetLimit(), date: b.date }); - return; - } - - const m = urlPath.match(/^\/flow\/(\d+)\/(\d+)\/(\d+)\.pbf$/); - if (!m) { - sendJson(404, { error: 'not_found' }); - return; - } - const z = Number(m[1]); - const x = Number(m[2]); - const y = Number(m[3]); - if (!isValidTomTomTile(z, x, y)) { - sendJson(400, { error: 'invalid_tile' }); - return; - } - if (!process.env.TOMTOM_API_KEY) { - sendJson(503, { error: 'no_key' }); - return; - } - - const key = `${z}/${x}/${y}`; - const now = Date.now(); - - let entry = mem.get(key); - if (!entry) { - entry = await readDiskTile(key); - if (entry) memSet(key, entry); - } - // Fresh cache hit — never counts against the budget. - if (entry && now - entry.at < TILE_TTL_MS) { - sendTile(entry.buf, 'HIT'); - return; - } - - // Budget governor: over the soft cap, last-good data beats a dead layer. - if (isTomTomOverBudget(currentBudget(), dailyBudgetLimit())) { - if (entry) { - sendTile(entry.buf, 'STALE-BUDGET'); - } else { - sendJson(429, { error: 'budget' }); - } - return; - } - - // Stale or missing → refresh, single-flight per tile. - if (!inflight.has(key)) { - inflight.set(key, fetchUpstream(z, x, y) - .then(async (buf) => { - const fresh = { at: Date.now(), buf }; - memSet(key, fresh); - await writeDiskTile(key, buf); - return fresh; - }) - .catch((err) => { - console.warn(`[tomtom-proxy] ${key} fetch failed (${err?.message || err}) — serving stale if any`); - return null; - }) - .finally(() => inflight.delete(key))); - } - const fresh = await inflight.get(key); - if (fresh) { - sendTile(fresh.buf, 'MISS'); - } else if (entry) { - sendTile(entry.buf, 'STALE-ERROR'); // upstream down — stale beats empty - } else { - sendJson(502, { error: 'upstream' }); - } - } catch (err) { - console.warn('[tomtom-proxy] error:', err?.message || err); - sendJson(500, { error: 'proxy' }); - } - }); - }, - }; -} - -/** - * NASA FIRMS live active-fire proxy with a memory + disk cache. - * Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2 - * - * Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent - * satellites, no cross-source dedup) fetched sequentially with `days=2` - * (`days=1` means "current UTC day", nearly empty just after 00:00Z) and - * clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS quota is - * 5,000 transactions / 10 min per MAP_KEY, so the cache is the point: - * TTL 30 min, single-flight refresh, serve-stale-on-failure, and a - * fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream - * fetch across dev-server restarts. Pattern mirrors celestrakProxy. - * - * Routes: - * GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires} - * GET /api/firms/status → {hasKey, lastFetch, count, stale, ttlMs, transactions} - * - * Keyless (no FIRMS_MAP_KEY): /api/firms → 503 {error:'no_key'}; status → - * {hasKey:false}. Upstream is never touched without a key. - * - * @returns {import('vite').Plugin} - */ -function firmsProxy() { - const TTL_MS = 30 * 60_000; - const STATUS_TTL_MS = 5 * 60_000; - const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'VIIRS_SNPP_NRT']; - const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); - const CACHE_PATH = path.join(CACHE_DIR, 'firms.json'); - - /** @type {?{at: number, sources: Array, fires: Array}} */ - let mem = null; - let diskChecked = false; - /** @type {?Promise, fires: Array}>} single-flight refresh */ - let inflight = null; - /** @type {?{at: number, transactions: ?{used: number, limit: number}}} mapkey_status cache */ - let statusCache = null; - /** @type {?Promise} */ - let statusInflight = null; - - const mapKey = () => String(process.env.FIRMS_MAP_KEY || '').trim(); - - async function readDiskOnce() { - if (diskChecked) return; - diskChecked = true; - try { - const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); - if (Number.isFinite(parsed?.at) && Array.isArray(parsed?.sources) && Array.isArray(parsed?.fires)) { - mem = parsed; - } - } catch { /* no disk cache yet */ } - } - - async function writeDisk(entry) { - try { - await fsp.mkdir(CACHE_DIR, { recursive: true }); - await fsp.writeFile(CACHE_PATH, JSON.stringify(entry), 'utf8'); - } catch (err) { - console.warn('[firms-proxy] cache write failed:', err?.message || err); - } - } - - /** - * Fetch + parse one FIRMS source. Throws on HTTP error or a non-CSV body - * (FIRMS reports errors as HTML/plain text, never CSV). Never log the URL — - * it embeds the MAP_KEY. - */ - async function fetchSource(key, source) { - const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`; - const res = await fetch(url, { signal: AbortSignal.timeout(60_000) }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const records = parseFirmsCsv(await res.text()); - if (records === null) throw new Error('non-CSV upstream response'); - return records; - } - - /** - * Refresh all sources sequentially (quota courtesy — never in parallel). - * Partial success (≥1 source ok) still produces a cacheable entry with the - * failed sources marked ok:false; total failure throws so the caller can - * serve stale. - */ - async function refreshUpstream(key) { - const now = Date.now(); - const sources = []; - const fires = []; - for (const source of SOURCES) { - try { - const records = filterTrailing24h(await fetchSource(key, source), now); - // NOT fires.push(...records): spread passes each record as an argument, - // and a world/2 VIIRS pull exceeds V8's argument limit (~125k) at - // ~131k records — RangeError, and the whole source is silently dropped. - for (const record of records) fires.push(record); - sources.push({ source, count: records.length, ok: true }); - } catch (err) { - console.warn(`[firms-proxy] ${source} fetch failed:`, err?.message || err); - sources.push({ source, count: 0, ok: false }); - } - } - if (!sources.some((s) => s.ok)) throw new Error('all FIRMS sources failed'); - return { at: now, sources, fires }; - } - - /** - * Cache entry → response payload. Fires are RE-filtered to the trailing - * 24 h at serve time so a stale cache never serves >24h-old detections. - */ - function buildPayload(entry, stale) { - const fires = filterTrailing24h(entry.fires, Date.now()); - return { - fetchedAt: entry.at, - stale, - ttlMs: TTL_MS, - sources: entry.sources, - count: fires.length, - fires, - }; - } - - /** mapkey_status transactions, cached 5 min, best-effort (null on failure). */ - function getTransactions(key) { - const now = Date.now(); - if (statusCache && now - statusCache.at < STATUS_TTL_MS) { - return Promise.resolve(statusCache.transactions); - } - if (!statusInflight) { - statusInflight = (async () => { - try { - const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`; - const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const body = await res.json(); - const used = Number(body?.current_transactions); - const limit = Number(body?.transaction_limit); - return Number.isFinite(used) && Number.isFinite(limit) ? { used, limit } : null; - } catch (err) { - console.warn('[firms-proxy] mapkey status failed:', err?.message || err); - return null; - } - })() - .then((transactions) => { - statusCache = { at: Date.now(), transactions }; - return transactions; - }) - .finally(() => { statusInflight = null; }); - } - return statusInflight; - } - - return { - name: 'firms-proxy', - configureServer(server) { - server.middlewares.use('/api/firms', async (req, res) => { - const sendJson = (status, obj) => { - if (res.headersSent) return; - res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify(obj)); - }; - try { - const subPath = String(req.url || '').split('?')[0]; - const key = mapKey(); - await readDiskOnce(); - - if (subPath === '/status') { - if (!key) { - sendJson(200, { hasKey: false, lastFetch: null, count: null, stale: false, ttlMs: TTL_MS, transactions: null }); - return; - } - const transactions = await getTransactions(key); - sendJson(200, { - hasKey: true, - lastFetch: mem ? mem.at : null, - count: mem ? mem.fires.length : null, - stale: mem ? Date.now() - mem.at >= TTL_MS : false, - ttlMs: TTL_MS, - transactions, - }); - return; - } - - if (!key) { - sendJson(503, { error: 'no_key' }); - return; - } - - const entry = mem; - if (entry && Date.now() - entry.at < TTL_MS) { - sendJson(200, buildPayload(entry, false)); - return; - } - // Stale or missing → refresh, single-flight (concurrent requests - // share one upstream pass). Capture the promise locally BEFORE - // awaiting: the .finally() nulls `inflight` the moment it settles. - if (!inflight) { - inflight = refreshUpstream(key) - .then(async (fresh) => { - mem = fresh; - await writeDisk(fresh); - return fresh; - }) - .catch((err) => { - console.warn(`[firms-proxy] refresh failed (${err?.message || err}) — serving cache if any`); - return null; - }) - .finally(() => { inflight = null; }); - } - const pending = inflight; - const fresh = await pending; - if (fresh) { - sendJson(200, buildPayload(fresh, false)); - } else if (entry) { - sendJson(200, buildPayload(entry, true)); // upstream down — stale beats empty - } else { - sendJson(502, { error: 'firms fetch failed and no cache available' }); - } - } catch (err) { - console.warn('[firms-proxy] error:', err?.message || err); - sendJson(500, { error: 'firms proxy error' }); - } - }); - }, - }; -} - -/** - * Re:Earth terrain point-height proxy: batched lon/lat → ellipsoidal height - * lookups, keyless. Upstream: https://terrain.reearth.land/heights.json - * (≤256 points per call). Terrain doesn't move, so results are cached to - * disk with a long TTL (30 days) — mirrors celestrakProxy's memory+disk - * cache and serve-stale shape. Cache entries and stale fallback are keyed per - * 5dp point, so reordered and partially overlapping batches reuse prior work. - * Only missing/stale points go upstream; the response is rebuilt in exact - * request order. Oversized requests (>256 points) are chunked sequentially. - */ -function terrainHeightsProxy() { - const TTL_MS = 30 * 24 * 3600_000; - const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); - const CACHE_PATH = path.join(CACHE_DIR, 'terrain-heights.json'); - const UPSTREAM_CHUNK = 256; - const MAX_POINTS = 2000; - - /** @type {Map} keyed by canonical 5dp lon/lat. */ - const mem = new Map(); - /** @type {Map>>} single-flight per missing-point subset. */ - const inflight = new Map(); - let diskLoaded = false; - let diskDirty = false; - - /** Load the on-disk cache into memory once, lazily (first request only). */ - async function loadDiskOnce() { - if (diskLoaded) return; - diskLoaded = true; - try { - const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); - const pointEntries = parsed?.version === 2 && parsed.points && typeof parsed.points === 'object' - ? parsed.points - : null; - if (pointEntries) { - for (const [key, entry] of Object.entries(pointEntries)) { - if (entry && Number.isFinite(entry.at) && validTerrainResult(entry.result)) { - mem.set(key, entry); - } - } - } else if (parsed && typeof parsed === 'object') { - // One-time migration from the former raw-batch cache. Zip only real, - // positionally present results; an absent value never becomes height 0. - for (const [rawPoints, entry] of Object.entries(parsed)) { - const points = parseTerrainPoints(rawPoints); - if (!points || !entry || !Number.isFinite(entry.at) || !Array.isArray(entry.results)) continue; - for (let i = 0; i < points.length; i += 1) { - const result = entry.results[i]; - if (!validTerrainResult(result)) continue; - const key = terrainPointKey(points[i]); - const existing = mem.get(key); - if (!existing || entry.at > existing.at) mem.set(key, { at: entry.at, result }); - } - } - diskDirty = mem.size > 0; - } - } catch { /* no disk cache yet */ } - // Periodic flush, same shape as adsbdbProxy: coalesce writes instead of - // hitting disk on every request. - setInterval(async () => { - if (!diskDirty) return; - diskDirty = false; - try { - await fsp.mkdir(CACHE_DIR, { recursive: true }); - const obj = { version: 2, points: Object.fromEntries(mem.entries()) }; - await fsp.writeFile(CACHE_PATH, JSON.stringify(obj), 'utf8'); - } catch (err) { - diskDirty = true; // retry next tick - console.warn('[terrain-heights-proxy] cache write failed'); - } - }, 15_000).unref?.(); - } - - /** - * Fetch all missing chunks sequentially (upstream caps each call at 256). - * - * The first try keeps its empirically required 30s timeout; network errors, - * 429, and 5xx receive up to three jittered retries sharing a 10s added-time - * budget, with Retry-After honored within that bound. - * @param {Array<[number, number]>} points - * @returns {Promise>} - */ - async function fetchUpstreamAll(points) { - const results = []; - for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) { - const chunk = points.slice(i, i + UPSTREAM_CHUNK); - const chunkResults = await fetchTerrainChunkWithRetry(chunk); - // Keep later chunks aligned even if a malformed upstream response omits - // trailing positions. The resolver will reject each null individually. - for (let j = 0; j < chunk.length; j += 1) results.push(chunkResults[j] ?? null); - } - return results; - } - - /** Coalesce concurrent requests for the same canonical missing-point list. */ - function fetchMissingSingleFlight(points) { - const key = points.map(terrainPointKey).join(';'); - if (!inflight.has(key)) { - const request = fetchUpstreamAll(points) - .finally(() => { - if (inflight.get(key) === request) inflight.delete(key); - }); - inflight.set(key, request); - } - return inflight.get(key); - } - - return { - name: 'terrain-heights-proxy', - configureServer(server) { - server.middlewares.use('/api/terrain/heights', async (req, res) => { - const send = (status, bodyObj) => { - if (res.headersSent) return; - res.writeHead(status, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(bodyObj)); - }; - try { - await loadDiskOnce(); - const parsedUrl = new URL(req.url || '', 'http://internal'); - const rawPoints = parsedUrl.searchParams.get('points'); - const points = parseTerrainPoints(rawPoints); - if (!points) { - send(400, { error: 'invalid points parameter — expected "lon,lat;lon,lat;…" with finite numbers' }); - return; - } - if (points.length > MAX_POINTS) { - send(500, { error: `too many points (${points.length}); max ${MAX_POINTS} per request` }); - return; - } - - const outcome = await resolveTerrainHeightRequest({ - points, - cache: mem, - fetchMissing: fetchMissingSingleFlight, - ttlMs: TTL_MS, - }); - if (outcome.cacheChanged) diskDirty = true; - if (outcome.upstreamError) { - console.warn( - '[terrain-heights-proxy] refresh incomplete' - + ' — serving stale points when available' - ); - } - send(outcome.status, outcome.body); - } catch (err) { - console.error('[terrain-heights-proxy] request failed'); - send(500, { error: 'terrain heights proxy error' }); - } - }); - }, - }; -} - /** * Detect whether an Overpass API response body indicates rate-limiting. * @@ -2053,166 +1425,6 @@ function overpassProxy() { }; } -/** - * Check whether a hostname is in the GBFS allowlist. - * - * Also accepts any subdomain of publicbikesystem.net. - * - * @param {string} hostname - * @returns {boolean} - */ -function isAllowedGbfsHost(hostname) { - const host = String(hostname || '').trim().toLowerCase(); - if (!host) return false; - if (GBFS_ALLOWED_HOSTS.has(host)) return true; - return host.endsWith('.publicbikesystem.net'); -} - -/** - * Only allow station_information.json and station_status.json endpoints. - * - * @param {string} pathname - * @returns {boolean} - */ -function isAllowedGbfsPath(pathname) { - return /\/station_(information|status)\.json$/i.test(String(pathname || '')); -} - -/** - * Return an appropriate Cache-Control header for a GBFS endpoint. - * - * station_information is semi-static (5 min cache); station_status is - * real-time (no-store). - * - * @param {string} pathname - * @returns {string} Cache-Control header value. - */ -function gbfsCacheControl(pathname) { - if (/\/station_information\.json$/i.test(String(pathname || ''))) { - return 'public, max-age=300'; - } - return 'no-store'; -} - -/** - * Vite plugin: GBFS bike-share proxy with host allowlisting and size limits. - * - * Accepts GET /api/gbfs/ and proxies the request - * to the upstream GBFS provider. Validates hostname against an allowlist, - * restricts to station_information/station_status paths, enforces HTTPS, - * and caps response body at 5 MB. - * - * @returns {import('vite').Plugin} - */ -function gbfsProxy() { - return { - name: 'gbfs-proxy', - configureServer(server) { - server.middlewares.use('/api/gbfs', async (req, res) => { - try { - if (req.method !== 'GET') { - res.writeHead(405, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Method Not Allowed' })); - return; - } - - const url = new URL(req.url || '/', 'http://localhost'); - const encodedTarget = url.pathname.replace(/^\/+/, ''); - if (!encodedTarget) { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Missing GBFS upstream target' })); - return; - } - - let decodedTarget = ''; - try { - decodedTarget = decodeURIComponent(encodedTarget); - } catch { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Invalid GBFS target encoding' })); - return; - } - - let upstreamUrl = null; - try { - upstreamUrl = new URL(decodedTarget); - } catch { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Invalid GBFS upstream URL' })); - return; - } - - if (upstreamUrl.protocol !== 'https:') { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Only https GBFS targets are allowed' })); - return; - } - - if (!isAllowedGbfsHost(upstreamUrl.hostname)) { - res.writeHead(403, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS host not allowed' })); - return; - } - - if (!isAllowedGbfsPath(upstreamUrl.pathname)) { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Only station_information/station_status endpoints are allowed' })); - return; - } - - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), GBFS_PROXY_TIMEOUT_MS); - let upstream; - try { - upstream = await fetch(upstreamUrl.toString(), { - method: 'GET', - headers: { - Accept: 'application/json', - 'User-Agent': 'gods-eye-view-gbfs-proxy/1.0', - }, - signal: controller.signal, - }); - } finally { - clearTimeout(timeoutId); - } - - // Limit response size to prevent memory exhaustion from malicious upstream - const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB - const contentLength = Number(upstream.headers.get('content-length')); - if (Number.isFinite(contentLength) && contentLength > GBFS_MAX_BODY_BYTES) { - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS upstream response too large' })); - return; - } - const body = await upstream.text(); - if (Buffer.byteLength(body, 'utf8') > GBFS_MAX_BODY_BYTES) { - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS upstream response too large' })); - return; - } - const contentType = upstream.headers.get('content-type') || 'application/json'; - res.writeHead(upstream.status, { - 'Content-Type': contentType, - 'Cache-Control': gbfsCacheControl(upstreamUrl.pathname), - 'X-GBFS-Upstream': upstreamUrl.hostname, - 'X-GBFS-Cache': 'MISS', - }); - res.end(body); - } catch (error) { - if (error?.name === 'AbortError') { - res.writeHead(504, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS upstream timeout' })); - return; - } - console.error('[GBFS Proxy]', error?.message || String(error)); - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS proxy error' })); - } - }); - }, - }; -} - /** * FNV-1a 32-bit hash of a string, used to derive deterministic pseudo-random * values (e.g. hue for synthetic SVG billboards, fallback heading angles). diff --git a/server/providers/terrain.js b/server/providers/terrain.js new file mode 100644 index 000000000..fa015cc0e --- /dev/null +++ b/server/providers/terrain.js @@ -0,0 +1,162 @@ +import path from 'node:path'; +import { promises as fsp } from 'node:fs'; + +import { + fetchTerrainChunkWithRetry, + parseTerrainPoints, + resolveTerrainHeightRequest, + terrainPointKey, + validTerrainResult, +} from '../../src/data/terrainHeightsProxy.js'; + +/** + * Re:Earth terrain point-height proxy: batched lon/lat → ellipsoidal height + * lookups, keyless. Upstream: https://terrain.reearth.land/heights.json + * (≤256 points per call). Terrain doesn't move, so results are cached to + * disk with a long TTL (30 days) — mirrors celestrakProxy's memory+disk + * cache and serve-stale shape. Cache entries and stale fallback are keyed per + * 5dp point, so reordered and partially overlapping batches reuse prior work. + * Only missing/stale points go upstream; the response is rebuilt in exact + * request order. Oversized requests (>256 points) are chunked sequentially. + */ +export function terrainHeightsProxy() { + const TTL_MS = 30 * 24 * 3600_000; + const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); + const CACHE_PATH = path.join(CACHE_DIR, 'terrain-heights.json'); + const UPSTREAM_CHUNK = 256; + const MAX_POINTS = 2000; + + /** @type {Map} keyed by canonical 5dp lon/lat. */ + const mem = new Map(); + /** @type {Map>>} single-flight per missing-point subset. */ + const inflight = new Map(); + let diskLoaded = false; + let diskDirty = false; + + /** Load the on-disk cache into memory once, lazily (first request only). */ + async function loadDiskOnce() { + if (diskLoaded) return; + diskLoaded = true; + try { + const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); + const pointEntries = parsed?.version === 2 && parsed.points && typeof parsed.points === 'object' + ? parsed.points + : null; + if (pointEntries) { + for (const [key, entry] of Object.entries(pointEntries)) { + if (entry && Number.isFinite(entry.at) && validTerrainResult(entry.result)) { + mem.set(key, entry); + } + } + } else if (parsed && typeof parsed === 'object') { + // One-time migration from the former raw-batch cache. Zip only real, + // positionally present results; an absent value never becomes height 0. + for (const [rawPoints, entry] of Object.entries(parsed)) { + const points = parseTerrainPoints(rawPoints); + if (!points || !entry || !Number.isFinite(entry.at) || !Array.isArray(entry.results)) continue; + for (let i = 0; i < points.length; i += 1) { + const result = entry.results[i]; + if (!validTerrainResult(result)) continue; + const key = terrainPointKey(points[i]); + const existing = mem.get(key); + if (!existing || entry.at > existing.at) mem.set(key, { at: entry.at, result }); + } + } + diskDirty = mem.size > 0; + } + } catch { /* no disk cache yet */ } + // Periodic flush, same shape as adsbdbProxy: coalesce writes instead of + // hitting disk on every request. + setInterval(async () => { + if (!diskDirty) return; + diskDirty = false; + try { + await fsp.mkdir(CACHE_DIR, { recursive: true }); + const obj = { version: 2, points: Object.fromEntries(mem.entries()) }; + await fsp.writeFile(CACHE_PATH, JSON.stringify(obj), 'utf8'); + } catch (err) { + diskDirty = true; // retry next tick + console.warn('[terrain-heights-proxy] cache write failed'); + } + }, 15_000).unref?.(); + } + + /** + * Fetch all missing chunks sequentially (upstream caps each call at 256). + * + * The first try keeps its empirically required 30s timeout; network errors, + * 429, and 5xx receive up to three jittered retries sharing a 10s added-time + * budget, with Retry-After honored within that bound. + * @param {Array<[number, number]>} points + * @returns {Promise>} + */ + async function fetchUpstreamAll(points) { + const results = []; + for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) { + const chunk = points.slice(i, i + UPSTREAM_CHUNK); + const chunkResults = await fetchTerrainChunkWithRetry(chunk); + // Keep later chunks aligned even if a malformed upstream response omits + // trailing positions. The resolver will reject each null individually. + for (let j = 0; j < chunk.length; j += 1) results.push(chunkResults[j] ?? null); + } + return results; + } + + /** Coalesce concurrent requests for the same canonical missing-point list. */ + function fetchMissingSingleFlight(points) { + const key = points.map(terrainPointKey).join(';'); + if (!inflight.has(key)) { + const request = fetchUpstreamAll(points) + .finally(() => { + if (inflight.get(key) === request) inflight.delete(key); + }); + inflight.set(key, request); + } + return inflight.get(key); + } + + return { + name: 'terrain-heights-proxy', + configureServer(server) { + server.middlewares.use('/api/terrain/heights', async (req, res) => { + const send = (status, bodyObj) => { + if (res.headersSent) return; + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(bodyObj)); + }; + try { + await loadDiskOnce(); + const parsedUrl = new URL(req.url || '', 'http://internal'); + const rawPoints = parsedUrl.searchParams.get('points'); + const points = parseTerrainPoints(rawPoints); + if (!points) { + send(400, { error: 'invalid points parameter — expected "lon,lat;lon,lat;…" with finite numbers' }); + return; + } + if (points.length > MAX_POINTS) { + send(500, { error: `too many points (${points.length}); max ${MAX_POINTS} per request` }); + return; + } + + const outcome = await resolveTerrainHeightRequest({ + points, + cache: mem, + fetchMissing: fetchMissingSingleFlight, + ttlMs: TTL_MS, + }); + if (outcome.cacheChanged) diskDirty = true; + if (outcome.upstreamError) { + console.warn( + '[terrain-heights-proxy] refresh incomplete' + + ' — serving stale points when available' + ); + } + send(outcome.status, outcome.body); + } catch (err) { + console.error('[terrain-heights-proxy] request failed'); + send(500, { error: 'terrain heights proxy error' }); + } + }); + }, + }; +} diff --git a/server/providers/traffic.js b/server/providers/traffic.js new file mode 100644 index 000000000..f11a32c1b --- /dev/null +++ b/server/providers/traffic.js @@ -0,0 +1,239 @@ +import path from 'node:path'; +import { promises as fsp } from 'node:fs'; + +import { + isValidTileCoord as isValidTomTomTile, + utcDayKey as tomtomUtcDayKey, + normalizeBudget as normalizeTomTomBudget, + isOverBudget as isTomTomOverBudget, +} from '../../src/data/tomtomTiles.js'; + +/** + * TomTom traffic-flow vector-tile proxy with a daily budget governor. + * + * Upstream: https://api.tomtom.com/traffic/map/4/tile/flow/relative/{z}/{x}/{y}.pbf + * (style `relative`; the response is an UNCOMPRESSED Mapbox Vector Tile, layer + * "Traffic flow"). The key comes from TOMTOM_API_KEY server-side only — the + * browser fetches same-origin `/api/tomtom/flow/{z}/{x}/{y}.pbf`. + * + * Cache: memory + disk (.gev-cache/tomtom/), TTL 120 s (traffic is fresh + * data), single-flight per tile, serve-stale-on-failure — the celestrakProxy + * pattern. Cache hits never count against the budget. + * + * Budget governor (mirrors the OpenSky credit-governor philosophy — last-good + * data beats a dead layer): a persistent counter (.gev-cache/tomtom/budget.json, + * keyed by UTC date, reset on day change) counts upstream fetch attempts + * against a soft cap (TOMTOM_DAILY_TILE_BUDGET, default 40,000 of the free + * tier's ~50k/day). Over the cap the proxy serves stale tiles when available, + * else 429 {error:'budget'}. + * + * GET /api/tomtom/status → {hasKey, dailyCount, budget, date}. Keyless mode: + * status reports hasKey:false and the tile endpoint 503s {error:'no_key'} + * without touching upstream — the traffic layer then stays in simulation mode. + * + * @returns {import('vite').Plugin} + */ +export function tomtomProxy() { + const TILE_TTL_MS = 120_000; + const CACHE_DIR = path.join(process.cwd(), '.gev-cache', 'tomtom'); + const BUDGET_PATH = path.join(CACHE_DIR, 'budget.json'); + const DEFAULT_DAILY_BUDGET = 40000; + const MEM_MAX_ENTRIES = 256; + const UPSTREAM_TIMEOUT_MS = 15000; + + /** @type {Map} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */ + const mem = new Map(); + /** @type {Map>} single-flight per tile. */ + const inflight = new Map(); + + /** @type {{date:string, count:number}|null} lazily-loaded persistent counter. */ + let budget = null; + let budgetLoaded = false; + + function dailyBudgetLimit() { + const raw = Number.parseInt(process.env.TOMTOM_DAILY_TILE_BUDGET || '', 10); + return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_DAILY_BUDGET; + } + + async function loadBudgetOnce() { + if (budgetLoaded) return; + budgetLoaded = true; + try { + const parsed = JSON.parse(await fsp.readFile(BUDGET_PATH, 'utf8')); + if (parsed && typeof parsed.date === 'string' && Number.isFinite(parsed.count)) { + budget = parsed; + } + } catch { /* no budget file yet */ } + } + + async function persistBudget() { + try { + await fsp.mkdir(CACHE_DIR, { recursive: true }); + await fsp.writeFile(BUDGET_PATH, JSON.stringify(budget), 'utf8'); + } catch (err) { + console.warn('[tomtom-proxy] budget write failed:', err?.message || err); + } + } + + /** Roll the counter to today (UTC) and return it. */ + function currentBudget() { + budget = normalizeTomTomBudget(budget, tomtomUtcDayKey()); + return budget; + } + + /** Count one upstream fetch attempt against today's budget (async persist). */ + function recordUpstreamFetch() { + currentBudget().count += 1; + void persistBudget(); + } + + const tilePath = (key) => path.join(CACHE_DIR, `flow-${key.replaceAll('/', '-')}.pbf`); + + /** Disk-cache read; tile age comes from the file's mtime. */ + async function readDiskTile(key) { + try { + const [stat, buf] = await Promise.all([ + fsp.stat(tilePath(key)), + fsp.readFile(tilePath(key)), + ]); + return { at: stat.mtimeMs, buf }; + } catch { return null; } + } + + async function writeDiskTile(key, buf) { + try { + await fsp.mkdir(CACHE_DIR, { recursive: true }); + await fsp.writeFile(tilePath(key), buf); + } catch (err) { + console.warn(`[tomtom-proxy] tile cache write failed for ${key}:`, err?.message || err); + } + } + + /** LRU-ish memory insert (Map preserves insertion order; evict the oldest). */ + function memSet(key, entry) { + if (!mem.has(key) && mem.size >= MEM_MAX_ENTRIES) { + const oldest = mem.keys().next().value; + mem.delete(oldest); + } + mem.set(key, entry); + } + + async function fetchUpstream(z, x, y) { + const url = 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' + + `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`; + recordUpstreamFetch(); // attempts count — upstream bills the request either way + const res = await fetch(url, { signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS) }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const buf = Buffer.from(await res.arrayBuffer()); + if (buf.length === 0) throw new Error('empty tile body'); + return buf; + } + + return { + name: 'tomtom-proxy', + configureServer(server) { + server.middlewares.use('/api/tomtom', async (req, res) => { + // Sanitized responses only (proxy/security baseline): no upstream + // error details, and never echo the key or the upstream URL. + const sendJson = (status, obj, extraHeaders = {}) => { + if (res.headersSent) return; + res.writeHead(status, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + ...extraHeaders, + }); + res.end(JSON.stringify(obj)); + }; + const sendTile = (buf, cacheStatus) => { + if (res.headersSent) return; + res.writeHead(200, { + 'Content-Type': 'application/x-protobuf', + 'Cache-Control': 'no-store', + 'x-tomtom-cache': cacheStatus, + }); + res.end(buf); + }; + + try { + await loadBudgetOnce(); + const urlPath = String(req.url || '').split('?')[0]; + + if (urlPath === '/status') { + const hasKey = Boolean(process.env.TOMTOM_API_KEY); + const b = currentBudget(); + sendJson(200, { hasKey, dailyCount: b.count, budget: dailyBudgetLimit(), date: b.date }); + return; + } + + const m = urlPath.match(/^\/flow\/(\d+)\/(\d+)\/(\d+)\.pbf$/); + if (!m) { + sendJson(404, { error: 'not_found' }); + return; + } + const z = Number(m[1]); + const x = Number(m[2]); + const y = Number(m[3]); + if (!isValidTomTomTile(z, x, y)) { + sendJson(400, { error: 'invalid_tile' }); + return; + } + if (!process.env.TOMTOM_API_KEY) { + sendJson(503, { error: 'no_key' }); + return; + } + + const key = `${z}/${x}/${y}`; + const now = Date.now(); + + let entry = mem.get(key); + if (!entry) { + entry = await readDiskTile(key); + if (entry) memSet(key, entry); + } + // Fresh cache hit — never counts against the budget. + if (entry && now - entry.at < TILE_TTL_MS) { + sendTile(entry.buf, 'HIT'); + return; + } + + // Budget governor: over the soft cap, last-good data beats a dead layer. + if (isTomTomOverBudget(currentBudget(), dailyBudgetLimit())) { + if (entry) { + sendTile(entry.buf, 'STALE-BUDGET'); + } else { + sendJson(429, { error: 'budget' }); + } + return; + } + + // Stale or missing → refresh, single-flight per tile. + if (!inflight.has(key)) { + inflight.set(key, fetchUpstream(z, x, y) + .then(async (buf) => { + const fresh = { at: Date.now(), buf }; + memSet(key, fresh); + await writeDiskTile(key, buf); + return fresh; + }) + .catch((err) => { + console.warn(`[tomtom-proxy] ${key} fetch failed (${err?.message || err}) — serving stale if any`); + return null; + }) + .finally(() => inflight.delete(key))); + } + const fresh = await inflight.get(key); + if (fresh) { + sendTile(fresh.buf, 'MISS'); + } else if (entry) { + sendTile(entry.buf, 'STALE-ERROR'); // upstream down — stale beats empty + } else { + sendJson(502, { error: 'upstream' }); + } + } catch (err) { + console.warn('[tomtom-proxy] error:', err?.message || err); + sendJson(500, { error: 'proxy' }); + } + }); + }, + }; +} diff --git a/src/data/gbfsSource.js b/src/data/gbfsSource.js new file mode 100644 index 000000000..d44aadac9 --- /dev/null +++ b/src/data/gbfsSource.js @@ -0,0 +1,52 @@ +/** Allowlisted GBFS hostnames; wildcard *.publicbikesystem.net also accepted. */ +const GBFS_ALLOWED_HOSTS = new Set([ + 'gbfs.lyft.com', + 'gbfs.bluebikes.com', + 'gbfs.bcycle.com', + 'gbfs.biketownpdx.com', + 'gbfs.cogobikeshare.com', + 'austin.publicbikesystem.net', + 'hon.publicbikesystem.net', + 'chat.publicbikesystem.net', +]); + +/** + * Check whether a hostname is in the GBFS allowlist. + * + * Also accepts any subdomain of publicbikesystem.net. + * + * @param {string} hostname + * @returns {boolean} + */ +export function isAllowedGbfsHost(hostname) { + const host = String(hostname || '').trim().toLowerCase(); + if (!host) return false; + if (GBFS_ALLOWED_HOSTS.has(host)) return true; + return host.endsWith('.publicbikesystem.net'); +} + +/** + * Only allow station_information.json and station_status.json endpoints. + * + * @param {string} pathname + * @returns {boolean} + */ +export function isAllowedGbfsPath(pathname) { + return /\/station_(information|status)\.json$/i.test(String(pathname || '')); +} + +/** + * Return an appropriate Cache-Control header for a GBFS endpoint. + * + * station_information is semi-static (5 min cache); station_status is + * real-time (no-store). + * + * @param {string} pathname + * @returns {string} Cache-Control header value. + */ +export function gbfsCacheControl(pathname) { + if (/\/station_information\.json$/i.test(String(pathname || ''))) { + return 'public, max-age=300'; + } + return 'no-store'; +} diff --git a/src/proxyErrorResponses.test.mjs b/src/proxyErrorResponses.test.mjs index 31760460f..0ba4ac80f 100644 --- a/src/proxyErrorResponses.test.mjs +++ b/src/proxyErrorResponses.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import path from 'node:path'; -const source = ['local.js', 'common/http.js', 'aircraft/enrichment.js', 'space/celestrak.js', 'space/launch-library.js', '../../src/data/spaceProviderRequests.js'] +const source = ['local.js', 'common/http.js', 'aircraft/enrichment.js', 'terrain.js', 'space/celestrak.js', 'space/launch-library.js', '../../src/data/spaceProviderRequests.js'] .map(file => readFileSync(new URL(`../server/providers/${file}`, import.meta.url), 'utf8')) .join('\n'); const detail = 'fixture-secret-token /internal/example '; diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs new file mode 100644 index 000000000..5f1340e10 --- /dev/null +++ b/src/tooling/environmentProviders.test.mjs @@ -0,0 +1,119 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { promises as fsp } from 'node:fs'; +import { terrainHeightsProxy } from 'gods-eye-view/server/providers/terrain'; +import { tomtomProxy } from 'gods-eye-view/server/providers/traffic'; +import { firmsProxy } from 'gods-eye-view/server/providers/firms'; +import { gbfsProxy } from 'gods-eye-view/server/providers/gbfs'; +import { localProviderPlugins } from '../../server/providers/local.js'; + +function install(plugin) { + const routes = new Map(); + plugin.configureServer({ middlewares: { use(route, handler) { routes.set(route, handler); } } }); + assert.equal(routes.size, 1); + return async (url = '/', method = 'GET') => { + const res = { headersSent: false, writeHead(status, headers) { Object.assign(this, { status, headers, headersSent: true }); }, end(body) { this.body = body; } }; + await [...routes.values()][0]({ url, method }, res); + return res; + }; +} +function isolate(t, env = {}) { + for(const [name,value] of Object.entries(env)) { + const previous = process.env[name]; + t.after(() => { if(previous === undefined) delete process.env[name]; else process.env[name] = previous; }); + process.env[name] = value; + } + t.mock.method(fsp, 'readFile', async () => { throw Error('no disk cache'); }); + t.mock.method(fsp, 'stat', async () => { throw Error('no disk cache'); }); + t.mock.method(fsp, 'mkdir', async () => {}); + t.mock.method(fsp, 'writeFile', async () => {}); + t.mock.method(globalThis, 'setInterval', () => ({ unref() {} })); + t.mock.method(console, 'warn', () => {}); +} +const json = res => JSON.parse(res.body); + +test('standalone composition mounts every extracted provider exactly once without acquisition', (t) => { + t.mock.method(globalThis, 'fetch', () => { throw Error('construction must not fetch'); }); + const plugins = localProviderPlugins(); + for(const factory of [terrainHeightsProxy,tomtomProxy,firmsProxy,gbfsProxy]) assert.equal(plugins.filter(p=>p.name===factory().name).length,1); +}); + +test('terrain middleware chunks missing points and reconstructs repeated/reordered requests from cache', async (t) => { + isolate(t);let calls=0;const sizes=[]; + t.mock.method(globalThis, 'fetch', async (raw) => { + calls++;const url=new URL(raw);assert.equal(url.origin,'https://terrain.reearth.land'); + const points=url.searchParams.get('points').split(';').map(p=>p.split(',').map(Number));sizes.push(points.length); + return Response.json({results:points.map(([lon])=>({ellipsoid:lon+100}))}); + }); + const request=install(terrainHeightsProxy()); + const points=Array.from({length:257},(_,i)=>`${i/100},1`); + const res=await request('/?points='+points.join(';')); + assert.equal(res.status,200);assert.deepEqual(sizes,[256,1]); + const reordered=await request('/?points=2.56,1;0,1;2.56,1'); + assert.deepEqual(json(reordered),{results:[{ellipsoid:102.56},{ellipsoid:100},{ellipsoid:102.56}]});assert.equal(calls,2); + assert.equal((await request('/?points=invalid')).status,400); + assert.equal((await request('/?points='+Array(2001).fill('0,1').join(';'))).status,500); + assert.equal(calls,2); +}); + +test('terrain middleware migrates valid legacy disk points without fabricating omitted heights', async (t) => { + isolate(t); + t.mock.method(fsp,'readFile',async()=>JSON.stringify({'1,2;3,4':{at:Date.now(),results:[{ellipsoid:77}]}})); + let calls=0; + t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).searchParams.get('points'),'3.00000,4.00000');return Response.json({results:[{ellipsoid:88}]});}); + const res=await install(terrainHeightsProxy())('/?points=3,4;1,2'); + assert.deepEqual(json(res),{results:[{ellipsoid:88},{ellipsoid:77}]});assert.equal(calls,1); +}); + +test('traffic middleware preserves keyless mode, caching, stale budget fallback and UTC rollover', async (t) => { + isolate(t,{TOMTOM_API_KEY:'',TOMTOM_DAILY_TILE_BUDGET:'1'}); + let now=Date.UTC(2026,8,12,12);t.mock.method(Date,'now',()=>now); + let calls=0;t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).searchParams.get('key'),'fixture-key');return new Response(new Uint8Array([1,2,3]));}); + const request=install(tomtomProxy()); + assert.equal(json(await request('/status')).hasKey,false); + assert.equal((await request('/flow/8/1/1.pbf')).status,503); + assert.equal((await request('/flow/7/1/1.pbf')).status,400);assert.equal(calls,0); + process.env.TOMTOM_API_KEY='fixture-key'; + assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'MISS'); + assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'HIT'); + assert.equal(json(await request('/status')).dailyCount,1);assert.equal(calls,1); + now+=120001; + assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'STALE-BUDGET'); + assert.equal((await request('/flow/8/2/1.pbf')).status,429);assert.equal(calls,1); + now+=86400000; + assert.equal(json(await request('/status')).dailyCount,0); + assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'MISS');assert.equal(calls,2); +}); + +test('FIRMS retains a large successful source during partial failure and filters stale data at serve time', async (t) => { + isolate(t,{FIRMS_MAP_KEY:''});let now=Date.UTC(2026,8,12,12);t.mock.method(Date,'now',()=>now); + let calls=0; + const header='latitude,longitude,acq_date,acq_time,confidence,frp\n'; + const csv=header+'30,-97,2026-09-12,1200,h,10\n'.repeat(130001); + t.mock.method(globalThis,'fetch',async(raw)=>{ + calls++;const url=new URL(raw);assert.equal(url.hostname,'firms.modaps.eosdis.nasa.gov'); + if(url.pathname.includes('mapkey_status')) return Response.json({current_transactions:3,transaction_limit:5000}); + return url.pathname.includes('VIIRS_NOAA20')?new Response(csv):new Response('offline',{status:503}); + }); + const request=install(firmsProxy()); + assert.equal((await request()).status,503);assert.equal(json(await request('/status')).hasKey,false);assert.equal(calls,0); + process.env.FIRMS_MAP_KEY='fixture-key'; + const first=json(await request());assert.equal(first.count,130001);assert.equal(first.sources.filter(s=>s.ok).length,1);assert.equal(calls,3); + assert.equal(json(await request()).count,130001);assert.equal(calls,3); + assert.deepEqual(json(await request('/status')).transactions,{used:3,limit:5000}); + now+=25*3600000;t.mock.method(globalThis,'fetch',async()=>new Response('offline',{status:503})); + const stale=json(await request());assert.equal(stale.stale,true);assert.equal(stale.count,0); +}); + +test('GBFS keeps host/path/method guards, response caps and distinct information/status cache headers', async (t) => { + let calls=0;t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).hostname,'gbfs.lyft.com');return Response.json({data:{stations:[]}});}); + const request=install(gbfsProxy());const target=p=>'/'+encodeURIComponent('https://gbfs.lyft.com/'+p); + assert.equal((await request(target('station_status.json'),'POST')).status,405); + assert.equal((await request('/'+encodeURIComponent('https://example.com/station_status.json'))).status,403); + assert.equal((await request(target('gbfs.json'))).status,400); + assert.equal((await request('/%zz')).status,400);assert.equal(calls,0); + assert.equal((await request(target('station_information.json'))).headers['Cache-Control'],'public, max-age=300'); + assert.equal((await request(target('station_status.json'))).headers['Cache-Control'],'no-store'); + t.mock.method(globalThis,'fetch',async()=>new Response('x',{headers:{'content-length':String(6*1024*1024)}})); + assert.equal((await request(target('station_status.json'))).status,502); +});