From f4054469d567b73ba225f9e251100ab4628efd0d Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Fri, 11 Sep 2026 21:57:50 -0700 Subject: [PATCH 1/3] refactor: split terrain traffic fire and bike-share providers --- CHANGELOG.md | 3 + docs/CODE-BOUNDARIES.md | 20 + docs/CURRENT-STATE.md | 10 + package.json | 17 +- scripts/package-boundaries.json | 42 ++ server/providers/firms.js | 227 ++++++ server/providers/gbfs.js | 126 ++++ server/providers/local.js | 802 +--------------------- server/providers/terrain.js | 162 +++++ server/providers/traffic.js | 239 +++++++ src/data/gbfsSource.js | 52 ++ src/proxyErrorResponses.test.mjs | 2 +- src/tooling/environmentProviders.test.mjs | 119 ++++ 13 files changed, 1024 insertions(+), 797 deletions(-) create mode 100644 server/providers/firms.js create mode 100644 server/providers/gbfs.js create mode 100644 server/providers/terrain.js create mode 100644 server/providers/traffic.js create mode 100644 src/data/gbfsSource.js create mode 100644 src/tooling/environmentProviders.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index ceb2c887c..44ce2aff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md ## [Unreleased] +- Separate terrain, traffic, FIRMS and GBFS middleware into focused provider + modules, preserving local configuration, routes and cache/error behavior. + - Split satellite and launch-feed server providers into focused modules with portable request URL builders, preserving routes and cache/error behavior. diff --git a/docs/CODE-BOUNDARIES.md b/docs/CODE-BOUNDARIES.md index bf7525e4e..5fbdba635 100644 --- a/docs/CODE-BOUNDARIES.md +++ b/docs/CODE-BOUNDARIES.md @@ -132,3 +132,23 @@ Callers supply the group or end date and own validation, credentials, transport, response limits and cache policy. The boundary gate checks this portable entry separately from the Node providers. Satellite rendering and launch replay remain in their existing browser modules. + +## Terrain, traffic, fires and bike-share providers + +`gods-eye-view/server/providers/terrain`, `/traffic`, `/firms` and `/gbfs` +are separate Node-only entries. Each owns its existing middleware and +process-scoped cache or request handling. Standalone composition mounts them in +the original order; their imports do not start acquisition. + +`gods-eye-view/sources/terrain` exports existing point-key, retry and cache +reconstruction mechanics with injectable acquisition dependencies. +`gods-eye-view/sources/traffic` exports tile math and budget calculations. +`gods-eye-view/sources/gbfs` exports host/path acceptance and cache-header rules. +These entries import no Node middleware, application configuration or rendering. +Callers retain their request admission and transport policy. FIRMS CSV parsing +remains an owned dependency of the FIRMS Node provider. The boundary gate +checks each entry independently. + +Browser terrain sampling, traffic matching/drawing, fire overlays and bike-share +layer lifecycle remain in their current modules. This extraction changes no +source defaults, credentials, quotas, data interpretation or visual behavior. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 14bc6c4ec..250a2f106 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1,5 +1,15 @@ # God's Eye View Current State +## Terrain, traffic, fire and bike-share provider modules + +Local composition now imports separate Node modules for Re:Earth heights, +TomTom flow tiles, NASA FIRMS detections and GBFS station feeds. Existing routes, +plugin order, server-key selection, validation, disk caches, budgets, retries +and stale/error responses remain unchanged. Each has a Node-only package entry +under `gods-eye-view/server/providers/`. Portable terrain mechanics, traffic tile +math and GBFS source rules are available under `gods-eye-view/sources/`. +The browser layers and their rendering remain in their existing modules. + ## Landmark annotation identity When a landmark geocode contains only address components, annotations retain diff --git a/package.json b/package.json index 407c374d6..b93c900ff 100644 --- a/package.json +++ b/package.json @@ -81,6 +81,21 @@ "./server/providers/space": { "node": "./server/providers/space.js" }, - "./sources/space": "./src/data/spaceProviderRequests.js" + "./sources/space": "./src/data/spaceProviderRequests.js", + "./server/providers/terrain": { + "node": "./server/providers/terrain.js" + }, + "./sources/terrain": "./src/data/terrainHeightsProxy.js", + "./server/providers/traffic": { + "node": "./server/providers/traffic.js" + }, + "./sources/traffic": "./src/data/tomtomTiles.js", + "./server/providers/firms": { + "node": "./server/providers/firms.js" + }, + "./server/providers/gbfs": { + "node": "./server/providers/gbfs.js" + }, + "./sources/gbfs": "./src/data/gbfsSource.js" } } diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 1b93c61e4..06d90f55f 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -89,5 +89,47 @@ "exports": ["./sources/space"], "modules": ["src/data/spaceProviderRequests.js"], "external": [] + }, + "terrain-provider": { + "runtime": "node", + "exports": ["./server/providers/terrain"], + "modules": [ + "server/providers/terrain.js", + "src/data/terrainHeightsProxy.js" + ], + "external": [] + }, + "terrain-source": { + "exports": ["./sources/terrain"], + "modules": ["src/data/terrainHeightsProxy.js"], + "external": [] + }, + "traffic-provider": { + "runtime": "node", + "exports": ["./server/providers/traffic"], + "modules": ["server/providers/traffic.js", "src/data/tomtomTiles.js"], + "external": [] + }, + "traffic-source": { + "exports": ["./sources/traffic"], + "modules": ["src/data/tomtomTiles.js"], + "external": [] + }, + "firms-provider": { + "runtime": "node", + "exports": ["./server/providers/firms"], + "modules": ["server/providers/firms.js", "src/data/firmsCsv.js"], + "external": [] + }, + "gbfs-provider": { + "runtime": "node", + "exports": ["./server/providers/gbfs"], + "modules": ["server/providers/gbfs.js", "src/data/gbfsSource.js"], + "external": [] + }, + "gbfs-source": { + "exports": ["./sources/gbfs"], + "modules": ["src/data/gbfsSource.js"], + "external": [] } } diff --git a/server/providers/firms.js b/server/providers/firms.js new file mode 100644 index 000000000..8326a612d --- /dev/null +++ b/server/providers/firms.js @@ -0,0 +1,227 @@ +import path from 'node:path'; +import { promises as fsp } from 'node:fs'; + +import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js'; + +/** + * NASA FIRMS live active-fire proxy with a memory + disk cache. + * Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2 + * + * Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent + * satellites, no cross-source dedup) fetched sequentially with `days=2` + * (`days=1` means "current UTC day", nearly empty just after 00:00Z) and + * clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS quota is + * 5,000 transactions / 10 min per MAP_KEY, so the cache is the point: + * TTL 30 min, single-flight refresh, serve-stale-on-failure, and a + * fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream + * fetch across dev-server restarts. Pattern mirrors celestrakProxy. + * + * Routes: + * GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires} + * GET /api/firms/status → {hasKey, lastFetch, count, stale, ttlMs, transactions} + * + * Keyless (no FIRMS_MAP_KEY): /api/firms → 503 {error:'no_key'}; status → + * {hasKey:false}. Upstream is never touched without a key. + * + * @returns {import('vite').Plugin} + */ +export function firmsProxy() { + const TTL_MS = 30 * 60_000; + const STATUS_TTL_MS = 5 * 60_000; + const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'VIIRS_SNPP_NRT']; + const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); + const CACHE_PATH = path.join(CACHE_DIR, 'firms.json'); + + /** @type {?{at: number, sources: Array, fires: Array}} */ + let mem = null; + let diskChecked = false; + /** @type {?Promise, fires: Array}>} single-flight refresh */ + let inflight = null; + /** @type {?{at: number, transactions: ?{used: number, limit: number}}} mapkey_status cache */ + let statusCache = null; + /** @type {?Promise} */ + let statusInflight = null; + + const mapKey = () => String(process.env.FIRMS_MAP_KEY || '').trim(); + + async function readDiskOnce() { + if (diskChecked) return; + diskChecked = true; + try { + const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); + if (Number.isFinite(parsed?.at) && Array.isArray(parsed?.sources) && Array.isArray(parsed?.fires)) { + mem = parsed; + } + } catch { /* no disk cache yet */ } + } + + async function writeDisk(entry) { + try { + await fsp.mkdir(CACHE_DIR, { recursive: true }); + await fsp.writeFile(CACHE_PATH, JSON.stringify(entry), 'utf8'); + } catch (err) { + console.warn('[firms-proxy] cache write failed:', err?.message || err); + } + } + + /** + * Fetch + parse one FIRMS source. Throws on HTTP error or a non-CSV body + * (FIRMS reports errors as HTML/plain text, never CSV). Never log the URL — + * it embeds the MAP_KEY. + */ + async function fetchSource(key, source) { + const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`; + const res = await fetch(url, { signal: AbortSignal.timeout(60_000) }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const records = parseFirmsCsv(await res.text()); + if (records === null) throw new Error('non-CSV upstream response'); + return records; + } + + /** + * Refresh all sources sequentially (quota courtesy — never in parallel). + * Partial success (≥1 source ok) still produces a cacheable entry with the + * failed sources marked ok:false; total failure throws so the caller can + * serve stale. + */ + async function refreshUpstream(key) { + const now = Date.now(); + const sources = []; + const fires = []; + for (const source of SOURCES) { + try { + const records = filterTrailing24h(await fetchSource(key, source), now); + // NOT fires.push(...records): spread passes each record as an argument, + // and a world/2 VIIRS pull exceeds V8's argument limit (~125k) at + // ~131k records — RangeError, and the whole source is silently dropped. + for (const record of records) fires.push(record); + sources.push({ source, count: records.length, ok: true }); + } catch (err) { + console.warn(`[firms-proxy] ${source} fetch failed:`, err?.message || err); + sources.push({ source, count: 0, ok: false }); + } + } + if (!sources.some((s) => s.ok)) throw new Error('all FIRMS sources failed'); + return { at: now, sources, fires }; + } + + /** + * Cache entry → response payload. Fires are RE-filtered to the trailing + * 24 h at serve time so a stale cache never serves >24h-old detections. + */ + function buildPayload(entry, stale) { + const fires = filterTrailing24h(entry.fires, Date.now()); + return { + fetchedAt: entry.at, + stale, + ttlMs: TTL_MS, + sources: entry.sources, + count: fires.length, + fires, + }; + } + + /** mapkey_status transactions, cached 5 min, best-effort (null on failure). */ + function getTransactions(key) { + const now = Date.now(); + if (statusCache && now - statusCache.at < STATUS_TTL_MS) { + return Promise.resolve(statusCache.transactions); + } + if (!statusInflight) { + statusInflight = (async () => { + try { + const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`; + const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const body = await res.json(); + const used = Number(body?.current_transactions); + const limit = Number(body?.transaction_limit); + return Number.isFinite(used) && Number.isFinite(limit) ? { used, limit } : null; + } catch (err) { + console.warn('[firms-proxy] mapkey status failed:', err?.message || err); + return null; + } + })() + .then((transactions) => { + statusCache = { at: Date.now(), transactions }; + return transactions; + }) + .finally(() => { statusInflight = null; }); + } + return statusInflight; + } + + return { + name: 'firms-proxy', + configureServer(server) { + server.middlewares.use('/api/firms', async (req, res) => { + const sendJson = (status, obj) => { + if (res.headersSent) return; + res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify(obj)); + }; + try { + const subPath = String(req.url || '').split('?')[0]; + const key = mapKey(); + await readDiskOnce(); + + if (subPath === '/status') { + if (!key) { + sendJson(200, { hasKey: false, lastFetch: null, count: null, stale: false, ttlMs: TTL_MS, transactions: null }); + return; + } + const transactions = await getTransactions(key); + sendJson(200, { + hasKey: true, + lastFetch: mem ? mem.at : null, + count: mem ? mem.fires.length : null, + stale: mem ? Date.now() - mem.at >= TTL_MS : false, + ttlMs: TTL_MS, + transactions, + }); + return; + } + + if (!key) { + sendJson(503, { error: 'no_key' }); + return; + } + + const entry = mem; + if (entry && Date.now() - entry.at < TTL_MS) { + sendJson(200, buildPayload(entry, false)); + return; + } + // Stale or missing → refresh, single-flight (concurrent requests + // share one upstream pass). Capture the promise locally BEFORE + // awaiting: the .finally() nulls `inflight` the moment it settles. + if (!inflight) { + inflight = refreshUpstream(key) + .then(async (fresh) => { + mem = fresh; + await writeDisk(fresh); + return fresh; + }) + .catch((err) => { + console.warn(`[firms-proxy] refresh failed (${err?.message || err}) — serving cache if any`); + return null; + }) + .finally(() => { inflight = null; }); + } + const pending = inflight; + const fresh = await pending; + if (fresh) { + sendJson(200, buildPayload(fresh, false)); + } else if (entry) { + sendJson(200, buildPayload(entry, true)); // upstream down — stale beats empty + } else { + sendJson(502, { error: 'firms fetch failed and no cache available' }); + } + } catch (err) { + console.warn('[firms-proxy] error:', err?.message || err); + sendJson(500, { error: 'firms proxy error' }); + } + }); + }, + }; +} diff --git a/server/providers/gbfs.js b/server/providers/gbfs.js new file mode 100644 index 000000000..bc9bb0ed6 --- /dev/null +++ b/server/providers/gbfs.js @@ -0,0 +1,126 @@ +import { isAllowedGbfsHost, isAllowedGbfsPath, gbfsCacheControl } from '../../src/data/gbfsSource.js'; + +// --------------------------------------------------------------------------- +// GBFS (General Bikeshare Feed Specification) proxy constants +// --------------------------------------------------------------------------- +/** Upstream fetch timeout for GBFS requests (ms). */ +const GBFS_PROXY_TIMEOUT_MS = 12000; + +/** + * Vite plugin: GBFS bike-share proxy with host allowlisting and size limits. + * + * Accepts GET /api/gbfs/ and proxies the request + * to the upstream GBFS provider. Validates hostname against an allowlist, + * restricts to station_information/station_status paths, enforces HTTPS, + * and caps response body at 5 MB. + * + * @returns {import('vite').Plugin} + */ +export function gbfsProxy() { + return { + name: 'gbfs-proxy', + configureServer(server) { + server.middlewares.use('/api/gbfs', async (req, res) => { + try { + if (req.method !== 'GET') { + res.writeHead(405, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Method Not Allowed' })); + return; + } + + const url = new URL(req.url || '/', 'http://localhost'); + const encodedTarget = url.pathname.replace(/^\/+/, ''); + if (!encodedTarget) { + res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Missing GBFS upstream target' })); + return; + } + + let decodedTarget = ''; + try { + decodedTarget = decodeURIComponent(encodedTarget); + } catch { + res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Invalid GBFS target encoding' })); + return; + } + + let upstreamUrl = null; + try { + upstreamUrl = new URL(decodedTarget); + } catch { + res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Invalid GBFS upstream URL' })); + return; + } + + if (upstreamUrl.protocol !== 'https:') { + res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Only https GBFS targets are allowed' })); + return; + } + + if (!isAllowedGbfsHost(upstreamUrl.hostname)) { + res.writeHead(403, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'GBFS host not allowed' })); + return; + } + + if (!isAllowedGbfsPath(upstreamUrl.pathname)) { + res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'Only station_information/station_status endpoints are allowed' })); + return; + } + + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), GBFS_PROXY_TIMEOUT_MS); + let upstream; + try { + upstream = await fetch(upstreamUrl.toString(), { + method: 'GET', + headers: { + Accept: 'application/json', + 'User-Agent': 'gods-eye-view-gbfs-proxy/1.0', + }, + signal: controller.signal, + }); + } finally { + clearTimeout(timeoutId); + } + + // Limit response size to prevent memory exhaustion from malicious upstream + const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB + const contentLength = Number(upstream.headers.get('content-length')); + if (Number.isFinite(contentLength) && contentLength > GBFS_MAX_BODY_BYTES) { + res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'GBFS upstream response too large' })); + return; + } + const body = await upstream.text(); + if (Buffer.byteLength(body, 'utf8') > GBFS_MAX_BODY_BYTES) { + res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'GBFS upstream response too large' })); + return; + } + const contentType = upstream.headers.get('content-type') || 'application/json'; + res.writeHead(upstream.status, { + 'Content-Type': contentType, + 'Cache-Control': gbfsCacheControl(upstreamUrl.pathname), + 'X-GBFS-Upstream': upstreamUrl.hostname, + 'X-GBFS-Cache': 'MISS', + }); + res.end(body); + } catch (error) { + if (error?.name === 'AbortError') { + res.writeHead(504, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'GBFS upstream timeout' })); + return; + } + console.error('[GBFS Proxy]', error?.message || String(error)); + res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.end(JSON.stringify({ error: 'GBFS proxy error' })); + } + }); + }, + }; +} diff --git a/server/providers/local.js b/server/providers/local.js index 81f9b3fdb..934e7c442 100644 --- a/server/providers/local.js +++ b/server/providers/local.js @@ -1,3 +1,7 @@ +import { terrainHeightsProxy } from './terrain.js'; +import { tomtomProxy } from './traffic.js'; +import { firmsProxy } from './firms.js'; +import { gbfsProxy } from './gbfs.js'; import { celestrakProxy, rocketLaunchesProxy } from './space.js'; export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js'; /** @@ -48,13 +52,8 @@ import { Readable } from 'node:stream'; import https from 'node:https'; import { lookup as lookupDns } from 'node:dns/promises'; import { directionToHeading } from '../../src/data/directionText.js'; -import { - isValidTileCoord as isValidTomTomTile, - utcDayKey as tomtomUtcDayKey, - normalizeBudget as normalizeTomTomBudget, - isOverBudget as isTomTomOverBudget, -} from '../../src/data/tomtomTiles.js'; -import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js'; + + import { fileURLToPath } from 'node:url'; import { normalizeRadioCountryInput } from '../../src/data/radioCountry.js'; import { @@ -74,13 +73,7 @@ import { validateKeySetupUpdates, } from '../../src/keySetupCore.mjs'; import { hardenCredentialFile } from '../../src/keySetupHardening.mjs'; -import { - fetchTerrainChunkWithRetry, - parseTerrainPoints, - resolveTerrainHeightRequest, - terrainPointKey, - validTerrainResult, -} from '../../src/data/terrainHeightsProxy.js'; + import { VOICE_MODELS, isKnownVoiceTier, resolveVoiceModel } from '../../src/voice/voiceCost.js'; /** Resolve __dirname for ESM context. */ @@ -1111,22 +1104,6 @@ function radioBrowserProxy() { configurePreviewServer: install, }; } -// --------------------------------------------------------------------------- -// GBFS (General Bikeshare Feed Specification) proxy constants -// --------------------------------------------------------------------------- -/** Upstream fetch timeout for GBFS requests (ms). */ -const GBFS_PROXY_TIMEOUT_MS = 12000; -/** Allowlisted GBFS hostnames; wildcard *.publicbikesystem.net also accepted. */ -const GBFS_ALLOWED_HOSTS = new Set([ - 'gbfs.lyft.com', - 'gbfs.bluebikes.com', - 'gbfs.bcycle.com', - 'gbfs.biketownpdx.com', - 'gbfs.cogobikeshare.com', - 'austin.publicbikesystem.net', - 'hon.publicbikesystem.net', - 'chat.publicbikesystem.net', -]); // Sourced from the shared voice-model registry so the client's cost estimate // can never be computed against a different model than the session runs on. const OPENAI_REALTIME_MODEL_DEFAULT = VOICE_MODELS.standard.id; @@ -1140,611 +1117,6 @@ const REALTIME_DEBUG_LOG_DIR = path.join(__dirname, '.gev-logs'); const REALTIME_DEBUG_LOG_FILE = path.join(REALTIME_DEBUG_LOG_DIR, 'realtime-conversations.jsonl'); const REALTIME_DEBUG_LOG_MAX_BYTES = 8 * 1024 * 1024; -/** - * TomTom traffic-flow vector-tile proxy with a daily budget governor. - * - * Upstream: https://api.tomtom.com/traffic/map/4/tile/flow/relative/{z}/{x}/{y}.pbf - * (style `relative`; the response is an UNCOMPRESSED Mapbox Vector Tile, layer - * "Traffic flow"). The key comes from TOMTOM_API_KEY server-side only — the - * browser fetches same-origin `/api/tomtom/flow/{z}/{x}/{y}.pbf`. - * - * Cache: memory + disk (.gev-cache/tomtom/), TTL 120 s (traffic is fresh - * data), single-flight per tile, serve-stale-on-failure — the celestrakProxy - * pattern. Cache hits never count against the budget. - * - * Budget governor (mirrors the OpenSky credit-governor philosophy — last-good - * data beats a dead layer): a persistent counter (.gev-cache/tomtom/budget.json, - * keyed by UTC date, reset on day change) counts upstream fetch attempts - * against a soft cap (TOMTOM_DAILY_TILE_BUDGET, default 40,000 of the free - * tier's ~50k/day). Over the cap the proxy serves stale tiles when available, - * else 429 {error:'budget'}. - * - * GET /api/tomtom/status → {hasKey, dailyCount, budget, date}. Keyless mode: - * status reports hasKey:false and the tile endpoint 503s {error:'no_key'} - * without touching upstream — the traffic layer then stays in simulation mode. - * - * @returns {import('vite').Plugin} - */ -function tomtomProxy() { - const TILE_TTL_MS = 120_000; - const CACHE_DIR = path.join(process.cwd(), '.gev-cache', 'tomtom'); - const BUDGET_PATH = path.join(CACHE_DIR, 'budget.json'); - const DEFAULT_DAILY_BUDGET = 40000; - const MEM_MAX_ENTRIES = 256; - const UPSTREAM_TIMEOUT_MS = 15000; - - /** @type {Map} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */ - const mem = new Map(); - /** @type {Map>} single-flight per tile. */ - const inflight = new Map(); - - /** @type {{date:string, count:number}|null} lazily-loaded persistent counter. */ - let budget = null; - let budgetLoaded = false; - - function dailyBudgetLimit() { - const raw = Number.parseInt(process.env.TOMTOM_DAILY_TILE_BUDGET || '', 10); - return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_DAILY_BUDGET; - } - - async function loadBudgetOnce() { - if (budgetLoaded) return; - budgetLoaded = true; - try { - const parsed = JSON.parse(await fsp.readFile(BUDGET_PATH, 'utf8')); - if (parsed && typeof parsed.date === 'string' && Number.isFinite(parsed.count)) { - budget = parsed; - } - } catch { /* no budget file yet */ } - } - - async function persistBudget() { - try { - await fsp.mkdir(CACHE_DIR, { recursive: true }); - await fsp.writeFile(BUDGET_PATH, JSON.stringify(budget), 'utf8'); - } catch (err) { - console.warn('[tomtom-proxy] budget write failed:', err?.message || err); - } - } - - /** Roll the counter to today (UTC) and return it. */ - function currentBudget() { - budget = normalizeTomTomBudget(budget, tomtomUtcDayKey()); - return budget; - } - - /** Count one upstream fetch attempt against today's budget (async persist). */ - function recordUpstreamFetch() { - currentBudget().count += 1; - void persistBudget(); - } - - const tilePath = (key) => path.join(CACHE_DIR, `flow-${key.replaceAll('/', '-')}.pbf`); - - /** Disk-cache read; tile age comes from the file's mtime. */ - async function readDiskTile(key) { - try { - const [stat, buf] = await Promise.all([ - fsp.stat(tilePath(key)), - fsp.readFile(tilePath(key)), - ]); - return { at: stat.mtimeMs, buf }; - } catch { return null; } - } - - async function writeDiskTile(key, buf) { - try { - await fsp.mkdir(CACHE_DIR, { recursive: true }); - await fsp.writeFile(tilePath(key), buf); - } catch (err) { - console.warn(`[tomtom-proxy] tile cache write failed for ${key}:`, err?.message || err); - } - } - - /** LRU-ish memory insert (Map preserves insertion order; evict the oldest). */ - function memSet(key, entry) { - if (!mem.has(key) && mem.size >= MEM_MAX_ENTRIES) { - const oldest = mem.keys().next().value; - mem.delete(oldest); - } - mem.set(key, entry); - } - - async function fetchUpstream(z, x, y) { - const url = 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' - + `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`; - recordUpstreamFetch(); // attempts count — upstream bills the request either way - const res = await fetch(url, { signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS) }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const buf = Buffer.from(await res.arrayBuffer()); - if (buf.length === 0) throw new Error('empty tile body'); - return buf; - } - - return { - name: 'tomtom-proxy', - configureServer(server) { - server.middlewares.use('/api/tomtom', async (req, res) => { - // Sanitized responses only (proxy/security baseline): no upstream - // error details, and never echo the key or the upstream URL. - const sendJson = (status, obj, extraHeaders = {}) => { - if (res.headersSent) return; - res.writeHead(status, { - 'Content-Type': 'application/json', - 'Cache-Control': 'no-store', - ...extraHeaders, - }); - res.end(JSON.stringify(obj)); - }; - const sendTile = (buf, cacheStatus) => { - if (res.headersSent) return; - res.writeHead(200, { - 'Content-Type': 'application/x-protobuf', - 'Cache-Control': 'no-store', - 'x-tomtom-cache': cacheStatus, - }); - res.end(buf); - }; - - try { - await loadBudgetOnce(); - const urlPath = String(req.url || '').split('?')[0]; - - if (urlPath === '/status') { - const hasKey = Boolean(process.env.TOMTOM_API_KEY); - const b = currentBudget(); - sendJson(200, { hasKey, dailyCount: b.count, budget: dailyBudgetLimit(), date: b.date }); - return; - } - - const m = urlPath.match(/^\/flow\/(\d+)\/(\d+)\/(\d+)\.pbf$/); - if (!m) { - sendJson(404, { error: 'not_found' }); - return; - } - const z = Number(m[1]); - const x = Number(m[2]); - const y = Number(m[3]); - if (!isValidTomTomTile(z, x, y)) { - sendJson(400, { error: 'invalid_tile' }); - return; - } - if (!process.env.TOMTOM_API_KEY) { - sendJson(503, { error: 'no_key' }); - return; - } - - const key = `${z}/${x}/${y}`; - const now = Date.now(); - - let entry = mem.get(key); - if (!entry) { - entry = await readDiskTile(key); - if (entry) memSet(key, entry); - } - // Fresh cache hit — never counts against the budget. - if (entry && now - entry.at < TILE_TTL_MS) { - sendTile(entry.buf, 'HIT'); - return; - } - - // Budget governor: over the soft cap, last-good data beats a dead layer. - if (isTomTomOverBudget(currentBudget(), dailyBudgetLimit())) { - if (entry) { - sendTile(entry.buf, 'STALE-BUDGET'); - } else { - sendJson(429, { error: 'budget' }); - } - return; - } - - // Stale or missing → refresh, single-flight per tile. - if (!inflight.has(key)) { - inflight.set(key, fetchUpstream(z, x, y) - .then(async (buf) => { - const fresh = { at: Date.now(), buf }; - memSet(key, fresh); - await writeDiskTile(key, buf); - return fresh; - }) - .catch((err) => { - console.warn(`[tomtom-proxy] ${key} fetch failed (${err?.message || err}) — serving stale if any`); - return null; - }) - .finally(() => inflight.delete(key))); - } - const fresh = await inflight.get(key); - if (fresh) { - sendTile(fresh.buf, 'MISS'); - } else if (entry) { - sendTile(entry.buf, 'STALE-ERROR'); // upstream down — stale beats empty - } else { - sendJson(502, { error: 'upstream' }); - } - } catch (err) { - console.warn('[tomtom-proxy] error:', err?.message || err); - sendJson(500, { error: 'proxy' }); - } - }); - }, - }; -} - -/** - * NASA FIRMS live active-fire proxy with a memory + disk cache. - * Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2 - * - * Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent - * satellites, no cross-source dedup) fetched sequentially with `days=2` - * (`days=1` means "current UTC day", nearly empty just after 00:00Z) and - * clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS quota is - * 5,000 transactions / 10 min per MAP_KEY, so the cache is the point: - * TTL 30 min, single-flight refresh, serve-stale-on-failure, and a - * fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream - * fetch across dev-server restarts. Pattern mirrors celestrakProxy. - * - * Routes: - * GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires} - * GET /api/firms/status → {hasKey, lastFetch, count, stale, ttlMs, transactions} - * - * Keyless (no FIRMS_MAP_KEY): /api/firms → 503 {error:'no_key'}; status → - * {hasKey:false}. Upstream is never touched without a key. - * - * @returns {import('vite').Plugin} - */ -function firmsProxy() { - const TTL_MS = 30 * 60_000; - const STATUS_TTL_MS = 5 * 60_000; - const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'VIIRS_SNPP_NRT']; - const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); - const CACHE_PATH = path.join(CACHE_DIR, 'firms.json'); - - /** @type {?{at: number, sources: Array, fires: Array}} */ - let mem = null; - let diskChecked = false; - /** @type {?Promise, fires: Array}>} single-flight refresh */ - let inflight = null; - /** @type {?{at: number, transactions: ?{used: number, limit: number}}} mapkey_status cache */ - let statusCache = null; - /** @type {?Promise} */ - let statusInflight = null; - - const mapKey = () => String(process.env.FIRMS_MAP_KEY || '').trim(); - - async function readDiskOnce() { - if (diskChecked) return; - diskChecked = true; - try { - const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); - if (Number.isFinite(parsed?.at) && Array.isArray(parsed?.sources) && Array.isArray(parsed?.fires)) { - mem = parsed; - } - } catch { /* no disk cache yet */ } - } - - async function writeDisk(entry) { - try { - await fsp.mkdir(CACHE_DIR, { recursive: true }); - await fsp.writeFile(CACHE_PATH, JSON.stringify(entry), 'utf8'); - } catch (err) { - console.warn('[firms-proxy] cache write failed:', err?.message || err); - } - } - - /** - * Fetch + parse one FIRMS source. Throws on HTTP error or a non-CSV body - * (FIRMS reports errors as HTML/plain text, never CSV). Never log the URL — - * it embeds the MAP_KEY. - */ - async function fetchSource(key, source) { - const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`; - const res = await fetch(url, { signal: AbortSignal.timeout(60_000) }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const records = parseFirmsCsv(await res.text()); - if (records === null) throw new Error('non-CSV upstream response'); - return records; - } - - /** - * Refresh all sources sequentially (quota courtesy — never in parallel). - * Partial success (≥1 source ok) still produces a cacheable entry with the - * failed sources marked ok:false; total failure throws so the caller can - * serve stale. - */ - async function refreshUpstream(key) { - const now = Date.now(); - const sources = []; - const fires = []; - for (const source of SOURCES) { - try { - const records = filterTrailing24h(await fetchSource(key, source), now); - // NOT fires.push(...records): spread passes each record as an argument, - // and a world/2 VIIRS pull exceeds V8's argument limit (~125k) at - // ~131k records — RangeError, and the whole source is silently dropped. - for (const record of records) fires.push(record); - sources.push({ source, count: records.length, ok: true }); - } catch (err) { - console.warn(`[firms-proxy] ${source} fetch failed:`, err?.message || err); - sources.push({ source, count: 0, ok: false }); - } - } - if (!sources.some((s) => s.ok)) throw new Error('all FIRMS sources failed'); - return { at: now, sources, fires }; - } - - /** - * Cache entry → response payload. Fires are RE-filtered to the trailing - * 24 h at serve time so a stale cache never serves >24h-old detections. - */ - function buildPayload(entry, stale) { - const fires = filterTrailing24h(entry.fires, Date.now()); - return { - fetchedAt: entry.at, - stale, - ttlMs: TTL_MS, - sources: entry.sources, - count: fires.length, - fires, - }; - } - - /** mapkey_status transactions, cached 5 min, best-effort (null on failure). */ - function getTransactions(key) { - const now = Date.now(); - if (statusCache && now - statusCache.at < STATUS_TTL_MS) { - return Promise.resolve(statusCache.transactions); - } - if (!statusInflight) { - statusInflight = (async () => { - try { - const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`; - const res = await fetch(url, { signal: AbortSignal.timeout(10_000) }); - if (!res.ok) throw new Error(`HTTP ${res.status}`); - const body = await res.json(); - const used = Number(body?.current_transactions); - const limit = Number(body?.transaction_limit); - return Number.isFinite(used) && Number.isFinite(limit) ? { used, limit } : null; - } catch (err) { - console.warn('[firms-proxy] mapkey status failed:', err?.message || err); - return null; - } - })() - .then((transactions) => { - statusCache = { at: Date.now(), transactions }; - return transactions; - }) - .finally(() => { statusInflight = null; }); - } - return statusInflight; - } - - return { - name: 'firms-proxy', - configureServer(server) { - server.middlewares.use('/api/firms', async (req, res) => { - const sendJson = (status, obj) => { - if (res.headersSent) return; - res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify(obj)); - }; - try { - const subPath = String(req.url || '').split('?')[0]; - const key = mapKey(); - await readDiskOnce(); - - if (subPath === '/status') { - if (!key) { - sendJson(200, { hasKey: false, lastFetch: null, count: null, stale: false, ttlMs: TTL_MS, transactions: null }); - return; - } - const transactions = await getTransactions(key); - sendJson(200, { - hasKey: true, - lastFetch: mem ? mem.at : null, - count: mem ? mem.fires.length : null, - stale: mem ? Date.now() - mem.at >= TTL_MS : false, - ttlMs: TTL_MS, - transactions, - }); - return; - } - - if (!key) { - sendJson(503, { error: 'no_key' }); - return; - } - - const entry = mem; - if (entry && Date.now() - entry.at < TTL_MS) { - sendJson(200, buildPayload(entry, false)); - return; - } - // Stale or missing → refresh, single-flight (concurrent requests - // share one upstream pass). Capture the promise locally BEFORE - // awaiting: the .finally() nulls `inflight` the moment it settles. - if (!inflight) { - inflight = refreshUpstream(key) - .then(async (fresh) => { - mem = fresh; - await writeDisk(fresh); - return fresh; - }) - .catch((err) => { - console.warn(`[firms-proxy] refresh failed (${err?.message || err}) — serving cache if any`); - return null; - }) - .finally(() => { inflight = null; }); - } - const pending = inflight; - const fresh = await pending; - if (fresh) { - sendJson(200, buildPayload(fresh, false)); - } else if (entry) { - sendJson(200, buildPayload(entry, true)); // upstream down — stale beats empty - } else { - sendJson(502, { error: 'firms fetch failed and no cache available' }); - } - } catch (err) { - console.warn('[firms-proxy] error:', err?.message || err); - sendJson(500, { error: 'firms proxy error' }); - } - }); - }, - }; -} - -/** - * Re:Earth terrain point-height proxy: batched lon/lat → ellipsoidal height - * lookups, keyless. Upstream: https://terrain.reearth.land/heights.json - * (≤256 points per call). Terrain doesn't move, so results are cached to - * disk with a long TTL (30 days) — mirrors celestrakProxy's memory+disk - * cache and serve-stale shape. Cache entries and stale fallback are keyed per - * 5dp point, so reordered and partially overlapping batches reuse prior work. - * Only missing/stale points go upstream; the response is rebuilt in exact - * request order. Oversized requests (>256 points) are chunked sequentially. - */ -function terrainHeightsProxy() { - const TTL_MS = 30 * 24 * 3600_000; - const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); - const CACHE_PATH = path.join(CACHE_DIR, 'terrain-heights.json'); - const UPSTREAM_CHUNK = 256; - const MAX_POINTS = 2000; - - /** @type {Map} keyed by canonical 5dp lon/lat. */ - const mem = new Map(); - /** @type {Map>>} single-flight per missing-point subset. */ - const inflight = new Map(); - let diskLoaded = false; - let diskDirty = false; - - /** Load the on-disk cache into memory once, lazily (first request only). */ - async function loadDiskOnce() { - if (diskLoaded) return; - diskLoaded = true; - try { - const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); - const pointEntries = parsed?.version === 2 && parsed.points && typeof parsed.points === 'object' - ? parsed.points - : null; - if (pointEntries) { - for (const [key, entry] of Object.entries(pointEntries)) { - if (entry && Number.isFinite(entry.at) && validTerrainResult(entry.result)) { - mem.set(key, entry); - } - } - } else if (parsed && typeof parsed === 'object') { - // One-time migration from the former raw-batch cache. Zip only real, - // positionally present results; an absent value never becomes height 0. - for (const [rawPoints, entry] of Object.entries(parsed)) { - const points = parseTerrainPoints(rawPoints); - if (!points || !entry || !Number.isFinite(entry.at) || !Array.isArray(entry.results)) continue; - for (let i = 0; i < points.length; i += 1) { - const result = entry.results[i]; - if (!validTerrainResult(result)) continue; - const key = terrainPointKey(points[i]); - const existing = mem.get(key); - if (!existing || entry.at > existing.at) mem.set(key, { at: entry.at, result }); - } - } - diskDirty = mem.size > 0; - } - } catch { /* no disk cache yet */ } - // Periodic flush, same shape as adsbdbProxy: coalesce writes instead of - // hitting disk on every request. - setInterval(async () => { - if (!diskDirty) return; - diskDirty = false; - try { - await fsp.mkdir(CACHE_DIR, { recursive: true }); - const obj = { version: 2, points: Object.fromEntries(mem.entries()) }; - await fsp.writeFile(CACHE_PATH, JSON.stringify(obj), 'utf8'); - } catch (err) { - diskDirty = true; // retry next tick - console.warn('[terrain-heights-proxy] cache write failed'); - } - }, 15_000).unref?.(); - } - - /** - * Fetch all missing chunks sequentially (upstream caps each call at 256). - * - * The first try keeps its empirically required 30s timeout; network errors, - * 429, and 5xx receive up to three jittered retries sharing a 10s added-time - * budget, with Retry-After honored within that bound. - * @param {Array<[number, number]>} points - * @returns {Promise>} - */ - async function fetchUpstreamAll(points) { - const results = []; - for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) { - const chunk = points.slice(i, i + UPSTREAM_CHUNK); - const chunkResults = await fetchTerrainChunkWithRetry(chunk); - // Keep later chunks aligned even if a malformed upstream response omits - // trailing positions. The resolver will reject each null individually. - for (let j = 0; j < chunk.length; j += 1) results.push(chunkResults[j] ?? null); - } - return results; - } - - /** Coalesce concurrent requests for the same canonical missing-point list. */ - function fetchMissingSingleFlight(points) { - const key = points.map(terrainPointKey).join(';'); - if (!inflight.has(key)) { - const request = fetchUpstreamAll(points) - .finally(() => { - if (inflight.get(key) === request) inflight.delete(key); - }); - inflight.set(key, request); - } - return inflight.get(key); - } - - return { - name: 'terrain-heights-proxy', - configureServer(server) { - server.middlewares.use('/api/terrain/heights', async (req, res) => { - const send = (status, bodyObj) => { - if (res.headersSent) return; - res.writeHead(status, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify(bodyObj)); - }; - try { - await loadDiskOnce(); - const parsedUrl = new URL(req.url || '', 'http://internal'); - const rawPoints = parsedUrl.searchParams.get('points'); - const points = parseTerrainPoints(rawPoints); - if (!points) { - send(400, { error: 'invalid points parameter — expected "lon,lat;lon,lat;…" with finite numbers' }); - return; - } - if (points.length > MAX_POINTS) { - send(500, { error: `too many points (${points.length}); max ${MAX_POINTS} per request` }); - return; - } - - const outcome = await resolveTerrainHeightRequest({ - points, - cache: mem, - fetchMissing: fetchMissingSingleFlight, - ttlMs: TTL_MS, - }); - if (outcome.cacheChanged) diskDirty = true; - if (outcome.upstreamError) { - console.warn( - '[terrain-heights-proxy] refresh incomplete' - + ' — serving stale points when available' - ); - } - send(outcome.status, outcome.body); - } catch (err) { - console.error('[terrain-heights-proxy] request failed'); - send(500, { error: 'terrain heights proxy error' }); - } - }); - }, - }; -} - /** * Detect whether an Overpass API response body indicates rate-limiting. * @@ -2053,166 +1425,6 @@ function overpassProxy() { }; } -/** - * Check whether a hostname is in the GBFS allowlist. - * - * Also accepts any subdomain of publicbikesystem.net. - * - * @param {string} hostname - * @returns {boolean} - */ -function isAllowedGbfsHost(hostname) { - const host = String(hostname || '').trim().toLowerCase(); - if (!host) return false; - if (GBFS_ALLOWED_HOSTS.has(host)) return true; - return host.endsWith('.publicbikesystem.net'); -} - -/** - * Only allow station_information.json and station_status.json endpoints. - * - * @param {string} pathname - * @returns {boolean} - */ -function isAllowedGbfsPath(pathname) { - return /\/station_(information|status)\.json$/i.test(String(pathname || '')); -} - -/** - * Return an appropriate Cache-Control header for a GBFS endpoint. - * - * station_information is semi-static (5 min cache); station_status is - * real-time (no-store). - * - * @param {string} pathname - * @returns {string} Cache-Control header value. - */ -function gbfsCacheControl(pathname) { - if (/\/station_information\.json$/i.test(String(pathname || ''))) { - return 'public, max-age=300'; - } - return 'no-store'; -} - -/** - * Vite plugin: GBFS bike-share proxy with host allowlisting and size limits. - * - * Accepts GET /api/gbfs/ and proxies the request - * to the upstream GBFS provider. Validates hostname against an allowlist, - * restricts to station_information/station_status paths, enforces HTTPS, - * and caps response body at 5 MB. - * - * @returns {import('vite').Plugin} - */ -function gbfsProxy() { - return { - name: 'gbfs-proxy', - configureServer(server) { - server.middlewares.use('/api/gbfs', async (req, res) => { - try { - if (req.method !== 'GET') { - res.writeHead(405, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Method Not Allowed' })); - return; - } - - const url = new URL(req.url || '/', 'http://localhost'); - const encodedTarget = url.pathname.replace(/^\/+/, ''); - if (!encodedTarget) { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Missing GBFS upstream target' })); - return; - } - - let decodedTarget = ''; - try { - decodedTarget = decodeURIComponent(encodedTarget); - } catch { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Invalid GBFS target encoding' })); - return; - } - - let upstreamUrl = null; - try { - upstreamUrl = new URL(decodedTarget); - } catch { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Invalid GBFS upstream URL' })); - return; - } - - if (upstreamUrl.protocol !== 'https:') { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Only https GBFS targets are allowed' })); - return; - } - - if (!isAllowedGbfsHost(upstreamUrl.hostname)) { - res.writeHead(403, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS host not allowed' })); - return; - } - - if (!isAllowedGbfsPath(upstreamUrl.pathname)) { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Only station_information/station_status endpoints are allowed' })); - return; - } - - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), GBFS_PROXY_TIMEOUT_MS); - let upstream; - try { - upstream = await fetch(upstreamUrl.toString(), { - method: 'GET', - headers: { - Accept: 'application/json', - 'User-Agent': 'gods-eye-view-gbfs-proxy/1.0', - }, - signal: controller.signal, - }); - } finally { - clearTimeout(timeoutId); - } - - // Limit response size to prevent memory exhaustion from malicious upstream - const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB - const contentLength = Number(upstream.headers.get('content-length')); - if (Number.isFinite(contentLength) && contentLength > GBFS_MAX_BODY_BYTES) { - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS upstream response too large' })); - return; - } - const body = await upstream.text(); - if (Buffer.byteLength(body, 'utf8') > GBFS_MAX_BODY_BYTES) { - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS upstream response too large' })); - return; - } - const contentType = upstream.headers.get('content-type') || 'application/json'; - res.writeHead(upstream.status, { - 'Content-Type': contentType, - 'Cache-Control': gbfsCacheControl(upstreamUrl.pathname), - 'X-GBFS-Upstream': upstreamUrl.hostname, - 'X-GBFS-Cache': 'MISS', - }); - res.end(body); - } catch (error) { - if (error?.name === 'AbortError') { - res.writeHead(504, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS upstream timeout' })); - return; - } - console.error('[GBFS Proxy]', error?.message || String(error)); - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS proxy error' })); - } - }); - }, - }; -} - /** * FNV-1a 32-bit hash of a string, used to derive deterministic pseudo-random * values (e.g. hue for synthetic SVG billboards, fallback heading angles). diff --git a/server/providers/terrain.js b/server/providers/terrain.js new file mode 100644 index 000000000..fa015cc0e --- /dev/null +++ b/server/providers/terrain.js @@ -0,0 +1,162 @@ +import path from 'node:path'; +import { promises as fsp } from 'node:fs'; + +import { + fetchTerrainChunkWithRetry, + parseTerrainPoints, + resolveTerrainHeightRequest, + terrainPointKey, + validTerrainResult, +} from '../../src/data/terrainHeightsProxy.js'; + +/** + * Re:Earth terrain point-height proxy: batched lon/lat → ellipsoidal height + * lookups, keyless. Upstream: https://terrain.reearth.land/heights.json + * (≤256 points per call). Terrain doesn't move, so results are cached to + * disk with a long TTL (30 days) — mirrors celestrakProxy's memory+disk + * cache and serve-stale shape. Cache entries and stale fallback are keyed per + * 5dp point, so reordered and partially overlapping batches reuse prior work. + * Only missing/stale points go upstream; the response is rebuilt in exact + * request order. Oversized requests (>256 points) are chunked sequentially. + */ +export function terrainHeightsProxy() { + const TTL_MS = 30 * 24 * 3600_000; + const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); + const CACHE_PATH = path.join(CACHE_DIR, 'terrain-heights.json'); + const UPSTREAM_CHUNK = 256; + const MAX_POINTS = 2000; + + /** @type {Map} keyed by canonical 5dp lon/lat. */ + const mem = new Map(); + /** @type {Map>>} single-flight per missing-point subset. */ + const inflight = new Map(); + let diskLoaded = false; + let diskDirty = false; + + /** Load the on-disk cache into memory once, lazily (first request only). */ + async function loadDiskOnce() { + if (diskLoaded) return; + diskLoaded = true; + try { + const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); + const pointEntries = parsed?.version === 2 && parsed.points && typeof parsed.points === 'object' + ? parsed.points + : null; + if (pointEntries) { + for (const [key, entry] of Object.entries(pointEntries)) { + if (entry && Number.isFinite(entry.at) && validTerrainResult(entry.result)) { + mem.set(key, entry); + } + } + } else if (parsed && typeof parsed === 'object') { + // One-time migration from the former raw-batch cache. Zip only real, + // positionally present results; an absent value never becomes height 0. + for (const [rawPoints, entry] of Object.entries(parsed)) { + const points = parseTerrainPoints(rawPoints); + if (!points || !entry || !Number.isFinite(entry.at) || !Array.isArray(entry.results)) continue; + for (let i = 0; i < points.length; i += 1) { + const result = entry.results[i]; + if (!validTerrainResult(result)) continue; + const key = terrainPointKey(points[i]); + const existing = mem.get(key); + if (!existing || entry.at > existing.at) mem.set(key, { at: entry.at, result }); + } + } + diskDirty = mem.size > 0; + } + } catch { /* no disk cache yet */ } + // Periodic flush, same shape as adsbdbProxy: coalesce writes instead of + // hitting disk on every request. + setInterval(async () => { + if (!diskDirty) return; + diskDirty = false; + try { + await fsp.mkdir(CACHE_DIR, { recursive: true }); + const obj = { version: 2, points: Object.fromEntries(mem.entries()) }; + await fsp.writeFile(CACHE_PATH, JSON.stringify(obj), 'utf8'); + } catch (err) { + diskDirty = true; // retry next tick + console.warn('[terrain-heights-proxy] cache write failed'); + } + }, 15_000).unref?.(); + } + + /** + * Fetch all missing chunks sequentially (upstream caps each call at 256). + * + * The first try keeps its empirically required 30s timeout; network errors, + * 429, and 5xx receive up to three jittered retries sharing a 10s added-time + * budget, with Retry-After honored within that bound. + * @param {Array<[number, number]>} points + * @returns {Promise>} + */ + async function fetchUpstreamAll(points) { + const results = []; + for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) { + const chunk = points.slice(i, i + UPSTREAM_CHUNK); + const chunkResults = await fetchTerrainChunkWithRetry(chunk); + // Keep later chunks aligned even if a malformed upstream response omits + // trailing positions. The resolver will reject each null individually. + for (let j = 0; j < chunk.length; j += 1) results.push(chunkResults[j] ?? null); + } + return results; + } + + /** Coalesce concurrent requests for the same canonical missing-point list. */ + function fetchMissingSingleFlight(points) { + const key = points.map(terrainPointKey).join(';'); + if (!inflight.has(key)) { + const request = fetchUpstreamAll(points) + .finally(() => { + if (inflight.get(key) === request) inflight.delete(key); + }); + inflight.set(key, request); + } + return inflight.get(key); + } + + return { + name: 'terrain-heights-proxy', + configureServer(server) { + server.middlewares.use('/api/terrain/heights', async (req, res) => { + const send = (status, bodyObj) => { + if (res.headersSent) return; + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(bodyObj)); + }; + try { + await loadDiskOnce(); + const parsedUrl = new URL(req.url || '', 'http://internal'); + const rawPoints = parsedUrl.searchParams.get('points'); + const points = parseTerrainPoints(rawPoints); + if (!points) { + send(400, { error: 'invalid points parameter — expected "lon,lat;lon,lat;…" with finite numbers' }); + return; + } + if (points.length > MAX_POINTS) { + send(500, { error: `too many points (${points.length}); max ${MAX_POINTS} per request` }); + return; + } + + const outcome = await resolveTerrainHeightRequest({ + points, + cache: mem, + fetchMissing: fetchMissingSingleFlight, + ttlMs: TTL_MS, + }); + if (outcome.cacheChanged) diskDirty = true; + if (outcome.upstreamError) { + console.warn( + '[terrain-heights-proxy] refresh incomplete' + + ' — serving stale points when available' + ); + } + send(outcome.status, outcome.body); + } catch (err) { + console.error('[terrain-heights-proxy] request failed'); + send(500, { error: 'terrain heights proxy error' }); + } + }); + }, + }; +} diff --git a/server/providers/traffic.js b/server/providers/traffic.js new file mode 100644 index 000000000..f11a32c1b --- /dev/null +++ b/server/providers/traffic.js @@ -0,0 +1,239 @@ +import path from 'node:path'; +import { promises as fsp } from 'node:fs'; + +import { + isValidTileCoord as isValidTomTomTile, + utcDayKey as tomtomUtcDayKey, + normalizeBudget as normalizeTomTomBudget, + isOverBudget as isTomTomOverBudget, +} from '../../src/data/tomtomTiles.js'; + +/** + * TomTom traffic-flow vector-tile proxy with a daily budget governor. + * + * Upstream: https://api.tomtom.com/traffic/map/4/tile/flow/relative/{z}/{x}/{y}.pbf + * (style `relative`; the response is an UNCOMPRESSED Mapbox Vector Tile, layer + * "Traffic flow"). The key comes from TOMTOM_API_KEY server-side only — the + * browser fetches same-origin `/api/tomtom/flow/{z}/{x}/{y}.pbf`. + * + * Cache: memory + disk (.gev-cache/tomtom/), TTL 120 s (traffic is fresh + * data), single-flight per tile, serve-stale-on-failure — the celestrakProxy + * pattern. Cache hits never count against the budget. + * + * Budget governor (mirrors the OpenSky credit-governor philosophy — last-good + * data beats a dead layer): a persistent counter (.gev-cache/tomtom/budget.json, + * keyed by UTC date, reset on day change) counts upstream fetch attempts + * against a soft cap (TOMTOM_DAILY_TILE_BUDGET, default 40,000 of the free + * tier's ~50k/day). Over the cap the proxy serves stale tiles when available, + * else 429 {error:'budget'}. + * + * GET /api/tomtom/status → {hasKey, dailyCount, budget, date}. Keyless mode: + * status reports hasKey:false and the tile endpoint 503s {error:'no_key'} + * without touching upstream — the traffic layer then stays in simulation mode. + * + * @returns {import('vite').Plugin} + */ +export function tomtomProxy() { + const TILE_TTL_MS = 120_000; + const CACHE_DIR = path.join(process.cwd(), '.gev-cache', 'tomtom'); + const BUDGET_PATH = path.join(CACHE_DIR, 'budget.json'); + const DEFAULT_DAILY_BUDGET = 40000; + const MEM_MAX_ENTRIES = 256; + const UPSTREAM_TIMEOUT_MS = 15000; + + /** @type {Map} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */ + const mem = new Map(); + /** @type {Map>} single-flight per tile. */ + const inflight = new Map(); + + /** @type {{date:string, count:number}|null} lazily-loaded persistent counter. */ + let budget = null; + let budgetLoaded = false; + + function dailyBudgetLimit() { + const raw = Number.parseInt(process.env.TOMTOM_DAILY_TILE_BUDGET || '', 10); + return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_DAILY_BUDGET; + } + + async function loadBudgetOnce() { + if (budgetLoaded) return; + budgetLoaded = true; + try { + const parsed = JSON.parse(await fsp.readFile(BUDGET_PATH, 'utf8')); + if (parsed && typeof parsed.date === 'string' && Number.isFinite(parsed.count)) { + budget = parsed; + } + } catch { /* no budget file yet */ } + } + + async function persistBudget() { + try { + await fsp.mkdir(CACHE_DIR, { recursive: true }); + await fsp.writeFile(BUDGET_PATH, JSON.stringify(budget), 'utf8'); + } catch (err) { + console.warn('[tomtom-proxy] budget write failed:', err?.message || err); + } + } + + /** Roll the counter to today (UTC) and return it. */ + function currentBudget() { + budget = normalizeTomTomBudget(budget, tomtomUtcDayKey()); + return budget; + } + + /** Count one upstream fetch attempt against today's budget (async persist). */ + function recordUpstreamFetch() { + currentBudget().count += 1; + void persistBudget(); + } + + const tilePath = (key) => path.join(CACHE_DIR, `flow-${key.replaceAll('/', '-')}.pbf`); + + /** Disk-cache read; tile age comes from the file's mtime. */ + async function readDiskTile(key) { + try { + const [stat, buf] = await Promise.all([ + fsp.stat(tilePath(key)), + fsp.readFile(tilePath(key)), + ]); + return { at: stat.mtimeMs, buf }; + } catch { return null; } + } + + async function writeDiskTile(key, buf) { + try { + await fsp.mkdir(CACHE_DIR, { recursive: true }); + await fsp.writeFile(tilePath(key), buf); + } catch (err) { + console.warn(`[tomtom-proxy] tile cache write failed for ${key}:`, err?.message || err); + } + } + + /** LRU-ish memory insert (Map preserves insertion order; evict the oldest). */ + function memSet(key, entry) { + if (!mem.has(key) && mem.size >= MEM_MAX_ENTRIES) { + const oldest = mem.keys().next().value; + mem.delete(oldest); + } + mem.set(key, entry); + } + + async function fetchUpstream(z, x, y) { + const url = 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' + + `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`; + recordUpstreamFetch(); // attempts count — upstream bills the request either way + const res = await fetch(url, { signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS) }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const buf = Buffer.from(await res.arrayBuffer()); + if (buf.length === 0) throw new Error('empty tile body'); + return buf; + } + + return { + name: 'tomtom-proxy', + configureServer(server) { + server.middlewares.use('/api/tomtom', async (req, res) => { + // Sanitized responses only (proxy/security baseline): no upstream + // error details, and never echo the key or the upstream URL. + const sendJson = (status, obj, extraHeaders = {}) => { + if (res.headersSent) return; + res.writeHead(status, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + ...extraHeaders, + }); + res.end(JSON.stringify(obj)); + }; + const sendTile = (buf, cacheStatus) => { + if (res.headersSent) return; + res.writeHead(200, { + 'Content-Type': 'application/x-protobuf', + 'Cache-Control': 'no-store', + 'x-tomtom-cache': cacheStatus, + }); + res.end(buf); + }; + + try { + await loadBudgetOnce(); + const urlPath = String(req.url || '').split('?')[0]; + + if (urlPath === '/status') { + const hasKey = Boolean(process.env.TOMTOM_API_KEY); + const b = currentBudget(); + sendJson(200, { hasKey, dailyCount: b.count, budget: dailyBudgetLimit(), date: b.date }); + return; + } + + const m = urlPath.match(/^\/flow\/(\d+)\/(\d+)\/(\d+)\.pbf$/); + if (!m) { + sendJson(404, { error: 'not_found' }); + return; + } + const z = Number(m[1]); + const x = Number(m[2]); + const y = Number(m[3]); + if (!isValidTomTomTile(z, x, y)) { + sendJson(400, { error: 'invalid_tile' }); + return; + } + if (!process.env.TOMTOM_API_KEY) { + sendJson(503, { error: 'no_key' }); + return; + } + + const key = `${z}/${x}/${y}`; + const now = Date.now(); + + let entry = mem.get(key); + if (!entry) { + entry = await readDiskTile(key); + if (entry) memSet(key, entry); + } + // Fresh cache hit — never counts against the budget. + if (entry && now - entry.at < TILE_TTL_MS) { + sendTile(entry.buf, 'HIT'); + return; + } + + // Budget governor: over the soft cap, last-good data beats a dead layer. + if (isTomTomOverBudget(currentBudget(), dailyBudgetLimit())) { + if (entry) { + sendTile(entry.buf, 'STALE-BUDGET'); + } else { + sendJson(429, { error: 'budget' }); + } + return; + } + + // Stale or missing → refresh, single-flight per tile. + if (!inflight.has(key)) { + inflight.set(key, fetchUpstream(z, x, y) + .then(async (buf) => { + const fresh = { at: Date.now(), buf }; + memSet(key, fresh); + await writeDiskTile(key, buf); + return fresh; + }) + .catch((err) => { + console.warn(`[tomtom-proxy] ${key} fetch failed (${err?.message || err}) — serving stale if any`); + return null; + }) + .finally(() => inflight.delete(key))); + } + const fresh = await inflight.get(key); + if (fresh) { + sendTile(fresh.buf, 'MISS'); + } else if (entry) { + sendTile(entry.buf, 'STALE-ERROR'); // upstream down — stale beats empty + } else { + sendJson(502, { error: 'upstream' }); + } + } catch (err) { + console.warn('[tomtom-proxy] error:', err?.message || err); + sendJson(500, { error: 'proxy' }); + } + }); + }, + }; +} diff --git a/src/data/gbfsSource.js b/src/data/gbfsSource.js new file mode 100644 index 000000000..d44aadac9 --- /dev/null +++ b/src/data/gbfsSource.js @@ -0,0 +1,52 @@ +/** Allowlisted GBFS hostnames; wildcard *.publicbikesystem.net also accepted. */ +const GBFS_ALLOWED_HOSTS = new Set([ + 'gbfs.lyft.com', + 'gbfs.bluebikes.com', + 'gbfs.bcycle.com', + 'gbfs.biketownpdx.com', + 'gbfs.cogobikeshare.com', + 'austin.publicbikesystem.net', + 'hon.publicbikesystem.net', + 'chat.publicbikesystem.net', +]); + +/** + * Check whether a hostname is in the GBFS allowlist. + * + * Also accepts any subdomain of publicbikesystem.net. + * + * @param {string} hostname + * @returns {boolean} + */ +export function isAllowedGbfsHost(hostname) { + const host = String(hostname || '').trim().toLowerCase(); + if (!host) return false; + if (GBFS_ALLOWED_HOSTS.has(host)) return true; + return host.endsWith('.publicbikesystem.net'); +} + +/** + * Only allow station_information.json and station_status.json endpoints. + * + * @param {string} pathname + * @returns {boolean} + */ +export function isAllowedGbfsPath(pathname) { + return /\/station_(information|status)\.json$/i.test(String(pathname || '')); +} + +/** + * Return an appropriate Cache-Control header for a GBFS endpoint. + * + * station_information is semi-static (5 min cache); station_status is + * real-time (no-store). + * + * @param {string} pathname + * @returns {string} Cache-Control header value. + */ +export function gbfsCacheControl(pathname) { + if (/\/station_information\.json$/i.test(String(pathname || ''))) { + return 'public, max-age=300'; + } + return 'no-store'; +} diff --git a/src/proxyErrorResponses.test.mjs b/src/proxyErrorResponses.test.mjs index 31760460f..0ba4ac80f 100644 --- a/src/proxyErrorResponses.test.mjs +++ b/src/proxyErrorResponses.test.mjs @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; import path from 'node:path'; -const source = ['local.js', 'common/http.js', 'aircraft/enrichment.js', 'space/celestrak.js', 'space/launch-library.js', '../../src/data/spaceProviderRequests.js'] +const source = ['local.js', 'common/http.js', 'aircraft/enrichment.js', 'terrain.js', 'space/celestrak.js', 'space/launch-library.js', '../../src/data/spaceProviderRequests.js'] .map(file => readFileSync(new URL(`../server/providers/${file}`, import.meta.url), 'utf8')) .join('\n'); const detail = 'fixture-secret-token /internal/example '; diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs new file mode 100644 index 000000000..5f1340e10 --- /dev/null +++ b/src/tooling/environmentProviders.test.mjs @@ -0,0 +1,119 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { promises as fsp } from 'node:fs'; +import { terrainHeightsProxy } from 'gods-eye-view/server/providers/terrain'; +import { tomtomProxy } from 'gods-eye-view/server/providers/traffic'; +import { firmsProxy } from 'gods-eye-view/server/providers/firms'; +import { gbfsProxy } from 'gods-eye-view/server/providers/gbfs'; +import { localProviderPlugins } from '../../server/providers/local.js'; + +function install(plugin) { + const routes = new Map(); + plugin.configureServer({ middlewares: { use(route, handler) { routes.set(route, handler); } } }); + assert.equal(routes.size, 1); + return async (url = '/', method = 'GET') => { + const res = { headersSent: false, writeHead(status, headers) { Object.assign(this, { status, headers, headersSent: true }); }, end(body) { this.body = body; } }; + await [...routes.values()][0]({ url, method }, res); + return res; + }; +} +function isolate(t, env = {}) { + for(const [name,value] of Object.entries(env)) { + const previous = process.env[name]; + t.after(() => { if(previous === undefined) delete process.env[name]; else process.env[name] = previous; }); + process.env[name] = value; + } + t.mock.method(fsp, 'readFile', async () => { throw Error('no disk cache'); }); + t.mock.method(fsp, 'stat', async () => { throw Error('no disk cache'); }); + t.mock.method(fsp, 'mkdir', async () => {}); + t.mock.method(fsp, 'writeFile', async () => {}); + t.mock.method(globalThis, 'setInterval', () => ({ unref() {} })); + t.mock.method(console, 'warn', () => {}); +} +const json = res => JSON.parse(res.body); + +test('standalone composition mounts every extracted provider exactly once without acquisition', (t) => { + t.mock.method(globalThis, 'fetch', () => { throw Error('construction must not fetch'); }); + const plugins = localProviderPlugins(); + for(const factory of [terrainHeightsProxy,tomtomProxy,firmsProxy,gbfsProxy]) assert.equal(plugins.filter(p=>p.name===factory().name).length,1); +}); + +test('terrain middleware chunks missing points and reconstructs repeated/reordered requests from cache', async (t) => { + isolate(t);let calls=0;const sizes=[]; + t.mock.method(globalThis, 'fetch', async (raw) => { + calls++;const url=new URL(raw);assert.equal(url.origin,'https://terrain.reearth.land'); + const points=url.searchParams.get('points').split(';').map(p=>p.split(',').map(Number));sizes.push(points.length); + return Response.json({results:points.map(([lon])=>({ellipsoid:lon+100}))}); + }); + const request=install(terrainHeightsProxy()); + const points=Array.from({length:257},(_,i)=>`${i/100},1`); + const res=await request('/?points='+points.join(';')); + assert.equal(res.status,200);assert.deepEqual(sizes,[256,1]); + const reordered=await request('/?points=2.56,1;0,1;2.56,1'); + assert.deepEqual(json(reordered),{results:[{ellipsoid:102.56},{ellipsoid:100},{ellipsoid:102.56}]});assert.equal(calls,2); + assert.equal((await request('/?points=invalid')).status,400); + assert.equal((await request('/?points='+Array(2001).fill('0,1').join(';'))).status,500); + assert.equal(calls,2); +}); + +test('terrain middleware migrates valid legacy disk points without fabricating omitted heights', async (t) => { + isolate(t); + t.mock.method(fsp,'readFile',async()=>JSON.stringify({'1,2;3,4':{at:Date.now(),results:[{ellipsoid:77}]}})); + let calls=0; + t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).searchParams.get('points'),'3.00000,4.00000');return Response.json({results:[{ellipsoid:88}]});}); + const res=await install(terrainHeightsProxy())('/?points=3,4;1,2'); + assert.deepEqual(json(res),{results:[{ellipsoid:88},{ellipsoid:77}]});assert.equal(calls,1); +}); + +test('traffic middleware preserves keyless mode, caching, stale budget fallback and UTC rollover', async (t) => { + isolate(t,{TOMTOM_API_KEY:'',TOMTOM_DAILY_TILE_BUDGET:'1'}); + let now=Date.UTC(2026,8,12,12);t.mock.method(Date,'now',()=>now); + let calls=0;t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).searchParams.get('key'),'fixture-key');return new Response(new Uint8Array([1,2,3]));}); + const request=install(tomtomProxy()); + assert.equal(json(await request('/status')).hasKey,false); + assert.equal((await request('/flow/8/1/1.pbf')).status,503); + assert.equal((await request('/flow/7/1/1.pbf')).status,400);assert.equal(calls,0); + process.env.TOMTOM_API_KEY='fixture-key'; + assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'MISS'); + assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'HIT'); + assert.equal(json(await request('/status')).dailyCount,1);assert.equal(calls,1); + now+=120001; + assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'STALE-BUDGET'); + assert.equal((await request('/flow/8/2/1.pbf')).status,429);assert.equal(calls,1); + now+=86400000; + assert.equal(json(await request('/status')).dailyCount,0); + assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'MISS');assert.equal(calls,2); +}); + +test('FIRMS retains a large successful source during partial failure and filters stale data at serve time', async (t) => { + isolate(t,{FIRMS_MAP_KEY:''});let now=Date.UTC(2026,8,12,12);t.mock.method(Date,'now',()=>now); + let calls=0; + const header='latitude,longitude,acq_date,acq_time,confidence,frp\n'; + const csv=header+'30,-97,2026-09-12,1200,h,10\n'.repeat(130001); + t.mock.method(globalThis,'fetch',async(raw)=>{ + calls++;const url=new URL(raw);assert.equal(url.hostname,'firms.modaps.eosdis.nasa.gov'); + if(url.pathname.includes('mapkey_status')) return Response.json({current_transactions:3,transaction_limit:5000}); + return url.pathname.includes('VIIRS_NOAA20')?new Response(csv):new Response('offline',{status:503}); + }); + const request=install(firmsProxy()); + assert.equal((await request()).status,503);assert.equal(json(await request('/status')).hasKey,false);assert.equal(calls,0); + process.env.FIRMS_MAP_KEY='fixture-key'; + const first=json(await request());assert.equal(first.count,130001);assert.equal(first.sources.filter(s=>s.ok).length,1);assert.equal(calls,3); + assert.equal(json(await request()).count,130001);assert.equal(calls,3); + assert.deepEqual(json(await request('/status')).transactions,{used:3,limit:5000}); + now+=25*3600000;t.mock.method(globalThis,'fetch',async()=>new Response('offline',{status:503})); + const stale=json(await request());assert.equal(stale.stale,true);assert.equal(stale.count,0); +}); + +test('GBFS keeps host/path/method guards, response caps and distinct information/status cache headers', async (t) => { + let calls=0;t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).hostname,'gbfs.lyft.com');return Response.json({data:{stations:[]}});}); + const request=install(gbfsProxy());const target=p=>'/'+encodeURIComponent('https://gbfs.lyft.com/'+p); + assert.equal((await request(target('station_status.json'),'POST')).status,405); + assert.equal((await request('/'+encodeURIComponent('https://example.com/station_status.json'))).status,403); + assert.equal((await request(target('gbfs.json'))).status,400); + assert.equal((await request('/%zz')).status,400);assert.equal(calls,0); + assert.equal((await request(target('station_information.json'))).headers['Cache-Control'],'public, max-age=300'); + assert.equal((await request(target('station_status.json'))).headers['Cache-Control'],'no-store'); + t.mock.method(globalThis,'fetch',async()=>new Response('x',{headers:{'content-length':String(6*1024*1024)}})); + assert.equal((await request(target('station_status.json'))).status,502); +}); From a2d6461b52e5217908291da133b07ba6e9d2df44 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Fri, 11 Sep 2026 21:57:51 -0700 Subject: [PATCH 2/3] style: format terrain traffic fire and bike-share modules --- scripts/format-scope.json | 14 +- server/providers/firms.js | 54 ++++- server/providers/gbfs.js | 93 ++++++-- server/providers/terrain.js | 53 +++-- server/providers/traffic.js | 67 ++++-- src/data/firmsCsv.js | 37 ++- src/data/firmsCsv.test.mjs | 27 ++- src/data/gbfsSource.js | 4 +- src/data/terrainHeightsProxy.js | 64 +++-- src/data/terrainHeightsProxy.test.mjs | 66 ++++-- src/data/tomtomTiles.js | 22 +- src/data/tomtomTiles.test.mjs | 73 ++++-- src/tooling/environmentProviders.test.mjs | 277 ++++++++++++++++------ 13 files changed, 638 insertions(+), 213 deletions(-) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index aff90961f..e822ab5a2 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -57,5 +57,17 @@ "server/providers/space/celestrak.js", "server/providers/space/launch-library.js", "src/data/spaceProviderRequests.js", - "src/tooling/spaceProviders.test.mjs" + "src/tooling/spaceProviders.test.mjs", + "server/providers/terrain.js", + "server/providers/traffic.js", + "server/providers/firms.js", + "server/providers/gbfs.js", + "src/data/gbfsSource.js", + "src/data/terrainHeightsProxy.js", + "src/data/terrainHeightsProxy.test.mjs", + "src/data/tomtomTiles.js", + "src/data/tomtomTiles.test.mjs", + "src/data/firmsCsv.js", + "src/data/firmsCsv.test.mjs", + "src/tooling/environmentProviders.test.mjs" ] diff --git a/server/providers/firms.js b/server/providers/firms.js index 8326a612d..afed2a20c 100644 --- a/server/providers/firms.js +++ b/server/providers/firms.js @@ -49,10 +49,16 @@ export function firmsProxy() { diskChecked = true; try { const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); - if (Number.isFinite(parsed?.at) && Array.isArray(parsed?.sources) && Array.isArray(parsed?.fires)) { + if ( + Number.isFinite(parsed?.at) && + Array.isArray(parsed?.sources) && + Array.isArray(parsed?.fires) + ) { mem = parsed; } - } catch { /* no disk cache yet */ } + } catch { + /* no disk cache yet */ + } } async function writeDisk(entry) { @@ -97,7 +103,10 @@ export function firmsProxy() { for (const record of records) fires.push(record); sources.push({ source, count: records.length, ok: true }); } catch (err) { - console.warn(`[firms-proxy] ${source} fetch failed:`, err?.message || err); + console.warn( + `[firms-proxy] ${source} fetch failed:`, + err?.message || err, + ); sources.push({ source, count: 0, ok: false }); } } @@ -136,9 +145,14 @@ export function firmsProxy() { const body = await res.json(); const used = Number(body?.current_transactions); const limit = Number(body?.transaction_limit); - return Number.isFinite(used) && Number.isFinite(limit) ? { used, limit } : null; + return Number.isFinite(used) && Number.isFinite(limit) + ? { used, limit } + : null; } catch (err) { - console.warn('[firms-proxy] mapkey status failed:', err?.message || err); + console.warn( + '[firms-proxy] mapkey status failed:', + err?.message || err, + ); return null; } })() @@ -146,7 +160,9 @@ export function firmsProxy() { statusCache = { at: Date.now(), transactions }; return transactions; }) - .finally(() => { statusInflight = null; }); + .finally(() => { + statusInflight = null; + }); } return statusInflight; } @@ -157,7 +173,10 @@ export function firmsProxy() { server.middlewares.use('/api/firms', async (req, res) => { const sendJson = (status, obj) => { if (res.headersSent) return; - res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(status, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify(obj)); }; try { @@ -167,7 +186,14 @@ export function firmsProxy() { if (subPath === '/status') { if (!key) { - sendJson(200, { hasKey: false, lastFetch: null, count: null, stale: false, ttlMs: TTL_MS, transactions: null }); + sendJson(200, { + hasKey: false, + lastFetch: null, + count: null, + stale: false, + ttlMs: TTL_MS, + transactions: null, + }); return; } const transactions = await getTransactions(key); @@ -203,10 +229,14 @@ export function firmsProxy() { return fresh; }) .catch((err) => { - console.warn(`[firms-proxy] refresh failed (${err?.message || err}) — serving cache if any`); + console.warn( + `[firms-proxy] refresh failed (${err?.message || err}) — serving cache if any`, + ); return null; }) - .finally(() => { inflight = null; }); + .finally(() => { + inflight = null; + }); } const pending = inflight; const fresh = await pending; @@ -215,7 +245,9 @@ export function firmsProxy() { } else if (entry) { sendJson(200, buildPayload(entry, true)); // upstream down — stale beats empty } else { - sendJson(502, { error: 'firms fetch failed and no cache available' }); + sendJson(502, { + error: 'firms fetch failed and no cache available', + }); } } catch (err) { console.warn('[firms-proxy] error:', err?.message || err); diff --git a/server/providers/gbfs.js b/server/providers/gbfs.js index bc9bb0ed6..4e1457fce 100644 --- a/server/providers/gbfs.js +++ b/server/providers/gbfs.js @@ -1,4 +1,8 @@ -import { isAllowedGbfsHost, isAllowedGbfsPath, gbfsCacheControl } from '../../src/data/gbfsSource.js'; +import { + isAllowedGbfsHost, + isAllowedGbfsPath, + gbfsCacheControl, +} from '../../src/data/gbfsSource.js'; // --------------------------------------------------------------------------- // GBFS (General Bikeshare Feed Specification) proxy constants @@ -23,7 +27,10 @@ export function gbfsProxy() { server.middlewares.use('/api/gbfs', async (req, res) => { try { if (req.method !== 'GET') { - res.writeHead(405, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(405, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify({ error: 'Method Not Allowed' })); return; } @@ -31,7 +38,10 @@ export function gbfsProxy() { const url = new URL(req.url || '/', 'http://localhost'); const encodedTarget = url.pathname.replace(/^\/+/, ''); if (!encodedTarget) { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(400, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify({ error: 'Missing GBFS upstream target' })); return; } @@ -40,7 +50,10 @@ export function gbfsProxy() { try { decodedTarget = decodeURIComponent(encodedTarget); } catch { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(400, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify({ error: 'Invalid GBFS target encoding' })); return; } @@ -49,31 +62,53 @@ export function gbfsProxy() { try { upstreamUrl = new URL(decodedTarget); } catch { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(400, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify({ error: 'Invalid GBFS upstream URL' })); return; } if (upstreamUrl.protocol !== 'https:') { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Only https GBFS targets are allowed' })); + res.writeHead(400, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); + res.end( + JSON.stringify({ error: 'Only https GBFS targets are allowed' }), + ); return; } if (!isAllowedGbfsHost(upstreamUrl.hostname)) { - res.writeHead(403, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(403, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify({ error: 'GBFS host not allowed' })); return; } if (!isAllowedGbfsPath(upstreamUrl.pathname)) { - res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'Only station_information/station_status endpoints are allowed' })); + res.writeHead(400, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); + res.end( + JSON.stringify({ + error: + 'Only station_information/station_status endpoints are allowed', + }), + ); return; } const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), GBFS_PROXY_TIMEOUT_MS); + const timeoutId = setTimeout( + () => controller.abort(), + GBFS_PROXY_TIMEOUT_MS, + ); let upstream; try { upstream = await fetch(upstreamUrl.toString(), { @@ -91,18 +126,32 @@ export function gbfsProxy() { // Limit response size to prevent memory exhaustion from malicious upstream const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB const contentLength = Number(upstream.headers.get('content-length')); - if (Number.isFinite(contentLength) && contentLength > GBFS_MAX_BODY_BYTES) { - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS upstream response too large' })); + if ( + Number.isFinite(contentLength) && + contentLength > GBFS_MAX_BODY_BYTES + ) { + res.writeHead(502, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); + res.end( + JSON.stringify({ error: 'GBFS upstream response too large' }), + ); return; } const body = await upstream.text(); if (Buffer.byteLength(body, 'utf8') > GBFS_MAX_BODY_BYTES) { - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); - res.end(JSON.stringify({ error: 'GBFS upstream response too large' })); + res.writeHead(502, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); + res.end( + JSON.stringify({ error: 'GBFS upstream response too large' }), + ); return; } - const contentType = upstream.headers.get('content-type') || 'application/json'; + const contentType = + upstream.headers.get('content-type') || 'application/json'; res.writeHead(upstream.status, { 'Content-Type': contentType, 'Cache-Control': gbfsCacheControl(upstreamUrl.pathname), @@ -112,12 +161,18 @@ export function gbfsProxy() { res.end(body); } catch (error) { if (error?.name === 'AbortError') { - res.writeHead(504, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(504, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify({ error: 'GBFS upstream timeout' })); return; } console.error('[GBFS Proxy]', error?.message || String(error)); - res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' }); + res.writeHead(502, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); res.end(JSON.stringify({ error: 'GBFS proxy error' })); } }); diff --git a/server/providers/terrain.js b/server/providers/terrain.js index fa015cc0e..5c7e6c609 100644 --- a/server/providers/terrain.js +++ b/server/providers/terrain.js @@ -39,12 +39,19 @@ export function terrainHeightsProxy() { diskLoaded = true; try { const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8')); - const pointEntries = parsed?.version === 2 && parsed.points && typeof parsed.points === 'object' - ? parsed.points - : null; + const pointEntries = + parsed?.version === 2 && + parsed.points && + typeof parsed.points === 'object' + ? parsed.points + : null; if (pointEntries) { for (const [key, entry] of Object.entries(pointEntries)) { - if (entry && Number.isFinite(entry.at) && validTerrainResult(entry.result)) { + if ( + entry && + Number.isFinite(entry.at) && + validTerrainResult(entry.result) + ) { mem.set(key, entry); } } @@ -53,18 +60,27 @@ export function terrainHeightsProxy() { // positionally present results; an absent value never becomes height 0. for (const [rawPoints, entry] of Object.entries(parsed)) { const points = parseTerrainPoints(rawPoints); - if (!points || !entry || !Number.isFinite(entry.at) || !Array.isArray(entry.results)) continue; + if ( + !points || + !entry || + !Number.isFinite(entry.at) || + !Array.isArray(entry.results) + ) + continue; for (let i = 0; i < points.length; i += 1) { const result = entry.results[i]; if (!validTerrainResult(result)) continue; const key = terrainPointKey(points[i]); const existing = mem.get(key); - if (!existing || entry.at > existing.at) mem.set(key, { at: entry.at, result }); + if (!existing || entry.at > existing.at) + mem.set(key, { at: entry.at, result }); } } diskDirty = mem.size > 0; } - } catch { /* no disk cache yet */ } + } catch { + /* no disk cache yet */ + } // Periodic flush, same shape as adsbdbProxy: coalesce writes instead of // hitting disk on every request. setInterval(async () => { @@ -97,7 +113,8 @@ export function terrainHeightsProxy() { const chunkResults = await fetchTerrainChunkWithRetry(chunk); // Keep later chunks aligned even if a malformed upstream response omits // trailing positions. The resolver will reject each null individually. - for (let j = 0; j < chunk.length; j += 1) results.push(chunkResults[j] ?? null); + for (let j = 0; j < chunk.length; j += 1) + results.push(chunkResults[j] ?? null); } return results; } @@ -106,10 +123,9 @@ export function terrainHeightsProxy() { function fetchMissingSingleFlight(points) { const key = points.map(terrainPointKey).join(';'); if (!inflight.has(key)) { - const request = fetchUpstreamAll(points) - .finally(() => { - if (inflight.get(key) === request) inflight.delete(key); - }); + const request = fetchUpstreamAll(points).finally(() => { + if (inflight.get(key) === request) inflight.delete(key); + }); inflight.set(key, request); } return inflight.get(key); @@ -130,11 +146,16 @@ export function terrainHeightsProxy() { const rawPoints = parsedUrl.searchParams.get('points'); const points = parseTerrainPoints(rawPoints); if (!points) { - send(400, { error: 'invalid points parameter — expected "lon,lat;lon,lat;…" with finite numbers' }); + send(400, { + error: + 'invalid points parameter — expected "lon,lat;lon,lat;…" with finite numbers', + }); return; } if (points.length > MAX_POINTS) { - send(500, { error: `too many points (${points.length}); max ${MAX_POINTS} per request` }); + send(500, { + error: `too many points (${points.length}); max ${MAX_POINTS} per request`, + }); return; } @@ -147,8 +168,8 @@ export function terrainHeightsProxy() { if (outcome.cacheChanged) diskDirty = true; if (outcome.upstreamError) { console.warn( - '[terrain-heights-proxy] refresh incomplete' - + ' — serving stale points when available' + '[terrain-heights-proxy] refresh incomplete' + + ' — serving stale points when available', ); } send(outcome.status, outcome.body); diff --git a/server/providers/traffic.js b/server/providers/traffic.js index f11a32c1b..a259ccaa3 100644 --- a/server/providers/traffic.js +++ b/server/providers/traffic.js @@ -60,10 +60,16 @@ export function tomtomProxy() { budgetLoaded = true; try { const parsed = JSON.parse(await fsp.readFile(BUDGET_PATH, 'utf8')); - if (parsed && typeof parsed.date === 'string' && Number.isFinite(parsed.count)) { + if ( + parsed && + typeof parsed.date === 'string' && + Number.isFinite(parsed.count) + ) { budget = parsed; } - } catch { /* no budget file yet */ } + } catch { + /* no budget file yet */ + } } async function persistBudget() { @@ -87,7 +93,8 @@ export function tomtomProxy() { void persistBudget(); } - const tilePath = (key) => path.join(CACHE_DIR, `flow-${key.replaceAll('/', '-')}.pbf`); + const tilePath = (key) => + path.join(CACHE_DIR, `flow-${key.replaceAll('/', '-')}.pbf`); /** Disk-cache read; tile age comes from the file's mtime. */ async function readDiskTile(key) { @@ -97,7 +104,9 @@ export function tomtomProxy() { fsp.readFile(tilePath(key)), ]); return { at: stat.mtimeMs, buf }; - } catch { return null; } + } catch { + return null; + } } async function writeDiskTile(key, buf) { @@ -105,7 +114,10 @@ export function tomtomProxy() { await fsp.mkdir(CACHE_DIR, { recursive: true }); await fsp.writeFile(tilePath(key), buf); } catch (err) { - console.warn(`[tomtom-proxy] tile cache write failed for ${key}:`, err?.message || err); + console.warn( + `[tomtom-proxy] tile cache write failed for ${key}:`, + err?.message || err, + ); } } @@ -119,10 +131,13 @@ export function tomtomProxy() { } async function fetchUpstream(z, x, y) { - const url = 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' - + `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`; + const url = + 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' + + `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`; recordUpstreamFetch(); // attempts count — upstream bills the request either way - const res = await fetch(url, { signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS) }); + const res = await fetch(url, { + signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), + }); if (!res.ok) throw new Error(`HTTP ${res.status}`); const buf = Buffer.from(await res.arrayBuffer()); if (buf.length === 0) throw new Error('empty tile body'); @@ -161,7 +176,12 @@ export function tomtomProxy() { if (urlPath === '/status') { const hasKey = Boolean(process.env.TOMTOM_API_KEY); const b = currentBudget(); - sendJson(200, { hasKey, dailyCount: b.count, budget: dailyBudgetLimit(), date: b.date }); + sendJson(200, { + hasKey, + dailyCount: b.count, + budget: dailyBudgetLimit(), + date: b.date, + }); return; } @@ -208,18 +228,23 @@ export function tomtomProxy() { // Stale or missing → refresh, single-flight per tile. if (!inflight.has(key)) { - inflight.set(key, fetchUpstream(z, x, y) - .then(async (buf) => { - const fresh = { at: Date.now(), buf }; - memSet(key, fresh); - await writeDiskTile(key, buf); - return fresh; - }) - .catch((err) => { - console.warn(`[tomtom-proxy] ${key} fetch failed (${err?.message || err}) — serving stale if any`); - return null; - }) - .finally(() => inflight.delete(key))); + inflight.set( + key, + fetchUpstream(z, x, y) + .then(async (buf) => { + const fresh = { at: Date.now(), buf }; + memSet(key, fresh); + await writeDiskTile(key, buf); + return fresh; + }) + .catch((err) => { + console.warn( + `[tomtom-proxy] ${key} fetch failed (${err?.message || err}) — serving stale if any`, + ); + return null; + }) + .finally(() => inflight.delete(key)), + ); } const fresh = await inflight.get(key); if (fresh) { diff --git a/src/data/firmsCsv.js b/src/data/firmsCsv.js index 2eeb0c307..138b1a4db 100644 --- a/src/data/firmsCsv.js +++ b/src/data/firmsCsv.js @@ -17,7 +17,14 @@ */ /** Header fields that must all be present for a payload to count as FIRMS CSV. */ -const REQUIRED_HEADER_FIELDS = ['latitude', 'longitude', 'acq_date', 'acq_time', 'confidence', 'frp']; +const REQUIRED_HEADER_FIELDS = [ + 'latitude', + 'longitude', + 'acq_date', + 'acq_time', + 'confidence', + 'frp', +]; const HOUR_MS = 3600_000; /** Trailing window size for {@link filterTrailing24h}. */ @@ -36,8 +43,10 @@ export function isLikelyCsv(text) { if (typeof text !== 'string') return false; const trimmed = text.trimStart(); if (!trimmed || trimmed[0] === '<') return false; - const headerLine = trimmed.slice(0, trimmed.indexOf('\n') === -1 ? undefined : trimmed.indexOf('\n')) - .trim().toLowerCase(); + const headerLine = trimmed + .slice(0, trimmed.indexOf('\n') === -1 ? undefined : trimmed.indexOf('\n')) + .trim() + .toLowerCase(); const fields = headerLine.split(',').map((f) => f.trim()); return REQUIRED_HEADER_FIELDS.every((required) => fields.includes(required)); } @@ -62,8 +71,13 @@ export function parseFirmsCsv(text) { // Locate the header (first non-empty line) and build a column index so the // parser survives column reordering across FIRMS product versions. let headerIndex = 0; - while (headerIndex < lines.length && !lines[headerIndex].trim()) headerIndex += 1; - const header = lines[headerIndex].trim().toLowerCase().split(',').map((f) => f.trim()); + while (headerIndex < lines.length && !lines[headerIndex].trim()) + headerIndex += 1; + const header = lines[headerIndex] + .trim() + .toLowerCase() + .split(',') + .map((f) => f.trim()); const col = new Map(header.map((name, i) => [name, i])); const iLat = col.get('latitude'); const iLon = col.get('longitude'); @@ -114,7 +128,8 @@ export function parseFirmsCsv(text) { * @returns {number} Epoch ms, or NaN when unparseable. */ export function acquisitionMsUtc(acqDate, acqTime) { - if (typeof acqDate !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(acqDate)) return NaN; + if (typeof acqDate !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(acqDate)) + return NaN; const timeText = String(acqTime ?? '').trim(); if (!/^\d{1,4}$/.test(timeText)) return NaN; const hhmm = timeText.padStart(4, '0'); @@ -123,7 +138,15 @@ export function acquisitionMsUtc(acqDate, acqTime) { const day = Number(acqDate.slice(8, 10)); const hours = Number(hhmm.slice(0, 2)); const minutes = Number(hhmm.slice(2, 4)); - if (month < 1 || month > 12 || day < 1 || day > 31 || hours > 23 || minutes > 59) return NaN; + if ( + month < 1 || + month > 12 || + day < 1 || + day > 31 || + hours > 23 || + minutes > 59 + ) + return NaN; return Date.UTC(year, month - 1, day, hours, minutes); } diff --git a/src/data/firmsCsv.test.mjs b/src/data/firmsCsv.test.mjs index 8dcdc9c29..438f728cf 100644 --- a/src/data/firmsCsv.test.mjs +++ b/src/data/firmsCsv.test.mjs @@ -14,10 +14,11 @@ import { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const FIXTURE = fs.readFileSync( path.join(__dirname, 'fixtures', 'firms-viirs-noaa20-sample.csv'), - 'utf8' + 'utf8', ); -const HEADER = 'latitude,longitude,bright_ti4,scan,track,acq_date,acq_time,satellite,instrument,confidence,version,bright_ti5,frp,daynight'; +const HEADER = + 'latitude,longitude,bright_ti4,scan,track,acq_date,acq_time,satellite,instrument,confidence,version,bright_ti5,frp,daynight'; test('fixture parses to 45 records with finite lat/lon/frp', () => { const records = parseFirmsCsv(FIXTURE); @@ -48,15 +49,24 @@ test('fixture first row maps every field, categorical confidence preserved raw', }); test('acquisitionMsUtc: "1006" = 10:06 UTC', () => { - assert.equal(acquisitionMsUtc('2026-07-16', '1006'), Date.UTC(2026, 6, 16, 10, 6)); + assert.equal( + acquisitionMsUtc('2026-07-16', '1006'), + Date.UTC(2026, 6, 16, 10, 6), + ); }); test('acquisitionMsUtc: non-zero-padded "45" = 00:45 UTC', () => { - assert.equal(acquisitionMsUtc('2026-07-16', '45'), Date.UTC(2026, 6, 16, 0, 45)); + assert.equal( + acquisitionMsUtc('2026-07-16', '45'), + Date.UTC(2026, 6, 16, 0, 45), + ); }); test('acquisitionMsUtc: "0" = midnight UTC; garbage → NaN', () => { - assert.equal(acquisitionMsUtc('2026-07-16', '0'), Date.UTC(2026, 6, 16, 0, 0)); + assert.equal( + acquisitionMsUtc('2026-07-16', '0'), + Date.UTC(2026, 6, 16, 0, 0), + ); assert.ok(Number.isNaN(acquisitionMsUtc('not-a-date', '1006'))); assert.ok(Number.isNaN(acquisitionMsUtc('2026-07-16', 'xx'))); }); @@ -118,7 +128,12 @@ test('filterTrailing24h: window is [now − 24 h, now + 2 h] inclusive', () => { const kept = filterTrailing24h(records, now); assert.deepEqual( kept.map((r) => `${r.acqDate} ${r.acqTime}`), - ['2026-07-15 1200', '2026-07-15 1201', '2026-07-16 1130', '2026-07-16 1400'] + [ + '2026-07-15 1200', + '2026-07-15 1201', + '2026-07-16 1130', + '2026-07-16 1400', + ], ); }); diff --git a/src/data/gbfsSource.js b/src/data/gbfsSource.js index d44aadac9..a15a806d9 100644 --- a/src/data/gbfsSource.js +++ b/src/data/gbfsSource.js @@ -19,7 +19,9 @@ const GBFS_ALLOWED_HOSTS = new Set([ * @returns {boolean} */ export function isAllowedGbfsHost(hostname) { - const host = String(hostname || '').trim().toLowerCase(); + const host = String(hostname || '') + .trim() + .toLowerCase(); if (!host) return false; if (GBFS_ALLOWED_HOSTS.has(host)) return true; return host.endsWith('.publicbikesystem.net'); diff --git a/src/data/terrainHeightsProxy.js b/src/data/terrainHeightsProxy.js index 525f4d489..7e6eb406f 100644 --- a/src/data/terrainHeightsProxy.js +++ b/src/data/terrainHeightsProxy.js @@ -24,7 +24,10 @@ function defaultSleep(ms) { export function parseTerrainPoints(raw) { const text = String(raw || '').trim(); if (!text) return null; - const pairs = text.split(';').map((part) => part.trim()).filter(Boolean); + const pairs = text + .split(';') + .map((part) => part.trim()) + .filter(Boolean); if (pairs.length === 0) return null; const points = []; for (const pair of pairs) { @@ -83,16 +86,19 @@ export function terrainRetryAfterMs(value, nowMs = Date.now()) { * @param {object} [options] * @returns {Promise>} */ -export async function fetchTerrainChunkWithRetry(points, { - fetchImpl = globalThis.fetch, - sleep = defaultSleep, - random = Math.random, - now = Date.now, - makeSignal = (timeoutMs) => AbortSignal.timeout(timeoutMs), - attemptTimeoutMs = 30_000, - retryBudgetMs = TERRAIN_RETRY_BUDGET_MS, - maxAttempts = TERRAIN_MAX_ATTEMPTS, -} = {}) { +export async function fetchTerrainChunkWithRetry( + points, + { + fetchImpl = globalThis.fetch, + sleep = defaultSleep, + random = Math.random, + now = Date.now, + makeSignal = (timeoutMs) => AbortSignal.timeout(timeoutMs), + attemptTimeoutMs = 30_000, + retryBudgetMs = TERRAIN_RETRY_BUDGET_MS, + maxAttempts = TERRAIN_MAX_ATTEMPTS, + } = {}, +) { const pointsParam = points.map(terrainPointKey).join(';'); const url = `https://terrain.reearth.land/heights.json?points=${encodeURIComponent(pointsParam)}`; let retryStartedAt = null; @@ -117,7 +123,9 @@ export async function fetchTerrainChunkWithRetry(points, { } const json = await res.json(); if (!Array.isArray(json?.results)) { - const error = new Error('malformed upstream response (no results array)'); + const error = new Error( + 'malformed upstream response (no results array)', + ); error.retryable = false; throw error; } @@ -131,14 +139,18 @@ export async function fetchTerrainChunkWithRetry(points, { if (remaining <= 0) break; const retryAfterMs = terrainRetryAfterMs(error?.retryAfter, now()); - const backoffMs = Math.round(350 * (2 ** attempt) * (0.75 + random() * 0.5)); + const backoffMs = Math.round( + 350 * 2 ** attempt * (0.75 + random() * 0.5), + ); const requestedDelay = retryAfterMs ?? backoffMs; const delayMs = Math.min(requestedDelay, remaining); if (delayMs > 0) await sleep(delayMs); } } - throw lastError || new Error('terrain heights upstream retry budget exhausted'); + throw ( + lastError || new Error('terrain heights upstream retry budget exhausted') + ); } /** @@ -171,7 +183,12 @@ export async function resolveTerrainHeightRequest({ const missing = []; for (const [key, point] of unique) { const entry = cache.get(key); - if (entry && validTerrainResult(entry.result) && requestStartedAt - entry.at < ttlMs) continue; + if ( + entry && + validTerrainResult(entry.result) && + requestStartedAt - entry.at < ttlMs + ) + continue; missing.push({ key, point }); } @@ -180,7 +197,8 @@ export async function resolveTerrainHeightRequest({ if (missing.length > 0) { try { const fetched = await fetchMissing(missing.map((item) => item.point)); - if (!Array.isArray(fetched)) throw new Error('malformed upstream response (no results array)'); + if (!Array.isArray(fetched)) + throw new Error('malformed upstream response (no results array)'); const fetchedAt = now(); for (let i = 0; i < missing.length; i += 1) { const result = fetched[i]; @@ -188,8 +206,13 @@ export async function resolveTerrainHeightRequest({ cache.set(missing[i].key, { at: fetchedAt, result }); cacheChanged = true; } - if (fetched.length !== missing.length || missing.some((_, i) => !validTerrainResult(fetched[i]))) { - upstreamError = new Error('upstream omitted one or more terrain heights'); + if ( + fetched.length !== missing.length || + missing.some((_, i) => !validTerrainResult(fetched[i])) + ) { + upstreamError = new Error( + 'upstream omitted one or more terrain heights', + ); } } catch (error) { upstreamError = error; @@ -203,7 +226,10 @@ export async function resolveTerrainHeightRequest({ if (results.some((result) => result == null)) { return { status: 502, - body: { error: 'terrain heights fetch failed and no cache available for every point' }, + body: { + error: + 'terrain heights fetch failed and no cache available for every point', + }, cacheChanged, upstreamError, }; diff --git a/src/data/terrainHeightsProxy.test.mjs b/src/data/terrainHeightsProxy.test.mjs index 3e9ed38c5..cd05dca8b 100644 --- a/src/data/terrainHeightsProxy.test.mjs +++ b/src/data/terrainHeightsProxy.test.mjs @@ -18,7 +18,13 @@ test('per-point cache reconstruction preserves exact request order with mixed hi [terrainPointKey([10, 1]), { at: now, result: result('cached-a', 101) }], [terrainPointKey([30, 3]), { at: now, result: result('cached-c', 303) }], ]); - const points = [[30, 3], [20, 2], [10, 1], [40, 4], [20, 2]]; + const points = [ + [30, 3], + [20, 2], + [10, 1], + [40, 4], + [20, 2], + ]; let fetchedPoints = null; const response = await resolveTerrainHeightRequest({ @@ -33,11 +39,18 @@ test('per-point cache reconstruction preserves exact request order with mixed hi }); assert.equal(response.status, 200); - assert.deepEqual(fetchedPoints, [[20, 2], [40, 4]], 'only unique cache misses go upstream'); + assert.deepEqual( + fetchedPoints, + [ + [20, 2], + [40, 4], + ], + 'only unique cache misses go upstream', + ); assert.deepEqual( response.body.results.map((item) => item.id), ['cached-c', 'fresh-b', 'cached-a', 'fresh-d', 'fresh-b'], - 'each output position must map back to its own input point' + 'each output position must map back to its own input point', ); }); @@ -59,7 +72,9 @@ test('429 retry honors Retry-After before the next attempt', async () => { return { ok: false, status: 429, - headers: { get: (name) => name.toLowerCase() === 'retry-after' ? '3' : null }, + headers: { + get: (name) => (name.toLowerCase() === 'retry-after' ? '3' : null), + }, }; } return { @@ -83,21 +98,28 @@ test('permanently failing upstream exhausts retries and returns 502 without fake cache: new Map(), ttlMs: 10_000, now: () => clock, - fetchMissing: (missing) => fetchTerrainChunkWithRetry(missing, { - now: () => clock, - random: () => 0.5, - makeSignal: () => undefined, - sleep: async (ms) => { clock += ms; }, - fetchImpl: async () => { - attempts += 1; - return { ok: false, status: 503, headers: { get: () => null } }; - }, - }), + fetchMissing: (missing) => + fetchTerrainChunkWithRetry(missing, { + now: () => clock, + random: () => 0.5, + makeSignal: () => undefined, + sleep: async (ms) => { + clock += ms; + }, + fetchImpl: async () => { + attempts += 1; + return { ok: false, status: 503, headers: { get: () => null } }; + }, + }), }); assert.equal(attempts, 4, 'one initial attempt plus three retries'); assert.equal(response.status, 502); - assert.equal('results' in response.body, false, 'failure body must not fabricate positional heights'); + assert.equal( + 'results' in response.body, + false, + 'failure body must not fabricate positional heights', + ); }); test('stale per-point entries serve through a failed refresh only when every point is real', async () => { @@ -106,12 +128,20 @@ test('stale per-point entries serve through a failed refresh only when every poi [terrainPointKey([2, 2]), { at: 0, result: result('stale-b', 22) }], ]); const response = await resolveTerrainHeightRequest({ - points: [[2, 2], [1, 1]], + points: [ + [2, 2], + [1, 1], + ], cache, ttlMs: 100, now: () => 1000, - fetchMissing: async () => { throw new Error('offline'); }, + fetchMissing: async () => { + throw new Error('offline'); + }, }); assert.equal(response.status, 200); - assert.deepEqual(response.body.results.map((item) => item.id), ['stale-b', 'stale-a']); + assert.deepEqual( + response.body.results.map((item) => item.id), + ['stale-b', 'stale-a'], + ); }); diff --git a/src/data/tomtomTiles.js b/src/data/tomtomTiles.js index b78550ed4..f0da57e46 100644 --- a/src/data/tomtomTiles.js +++ b/src/data/tomtomTiles.js @@ -29,7 +29,8 @@ const MERCATOR_LAT_LIMIT = 85.05112878; * @returns {boolean} True when the coordinate is a fetchable tile. */ export function isValidTileCoord(z, x, y) { - if (!Number.isInteger(z) || !Number.isInteger(x) || !Number.isInteger(y)) return false; + if (!Number.isInteger(z) || !Number.isInteger(x) || !Number.isInteger(y)) + return false; if (z < MIN_TILE_ZOOM || z > MAX_TILE_ZOOM) return false; const n = 2 ** z; return x >= 0 && x < n && y >= 0 && y < n; @@ -47,11 +48,14 @@ export function isValidTileCoord(z, x, y) { */ export function lonLatToTile(lon, lat, z) { const n = 2 ** z; - const clampedLat = Math.max(-MERCATOR_LAT_LIMIT, Math.min(MERCATOR_LAT_LIMIT, lat)); + const clampedLat = Math.max( + -MERCATOR_LAT_LIMIT, + Math.min(MERCATOR_LAT_LIMIT, lat), + ); const latRad = (clampedLat * Math.PI) / 180; const x = Math.floor(((lon + 180) / 360) * n); const y = Math.floor( - ((1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI) / 2) * n + ((1 - Math.log(Math.tan(latRad) + 1 / Math.cos(latRad)) / Math.PI) / 2) * n, ); return { x: Math.max(0, Math.min(n - 1, x)), @@ -70,7 +74,8 @@ export function lonLatToTile(lon, lat, z) { export function tileToBBox(z, x, y) { const n = 2 ** z; const lonAt = (col) => (col / n) * 360 - 180; - const latAt = (row) => (Math.atan(Math.sinh(Math.PI * (1 - (2 * row) / n))) * 180) / Math.PI; + const latAt = (row) => + (Math.atan(Math.sinh(Math.PI * (1 - (2 * row) / n))) * 180) / Math.PI; return { west: lonAt(x), east: lonAt(x + 1), @@ -132,10 +137,11 @@ export function utcDayKey(epochMs = Date.now()) { * @returns {{date:string, count:number}} Valid state for `dayKey`. */ export function normalizeBudget(state, dayKey) { - const valid = Boolean(state) - && state.date === dayKey - && Number.isFinite(state.count) - && state.count >= 0; + const valid = + Boolean(state) && + state.date === dayKey && + Number.isFinite(state.count) && + state.count >= 0; return valid ? state : { date: dayKey, count: 0 }; } diff --git a/src/data/tomtomTiles.test.mjs b/src/data/tomtomTiles.test.mjs index 523ce54b8..1c9d289c0 100644 --- a/src/data/tomtomTiles.test.mjs +++ b/src/data/tomtomTiles.test.mjs @@ -53,22 +53,39 @@ test('lonLatToTile: downtown Austin @ z12 -> x935 y1686 (matches fixture)', () = test('tileToBBox: fixture tile bbox contains downtown Austin', () => { const bbox = tileToBBox(12, 935, 1686); - assert.ok(bbox.west <= AUSTIN.lon && AUSTIN.lon <= bbox.east, `lon outside [${bbox.west}, ${bbox.east}]`); - assert.ok(bbox.south <= AUSTIN.lat && AUSTIN.lat <= bbox.north, `lat outside [${bbox.south}, ${bbox.north}]`); + assert.ok( + bbox.west <= AUSTIN.lon && AUSTIN.lon <= bbox.east, + `lon outside [${bbox.west}, ${bbox.east}]`, + ); + assert.ok( + bbox.south <= AUSTIN.lat && AUSTIN.lat <= bbox.north, + `lat outside [${bbox.south}, ${bbox.north}]`, + ); assert.ok(bbox.north > bbox.south && bbox.east > bbox.west); }); test('tileToBBox <-> lonLatToTile roundtrip at the bbox center', () => { const bbox = tileToBBox(12, 935, 1686); - const center = { lon: (bbox.west + bbox.east) / 2, lat: (bbox.south + bbox.north) / 2 }; + const center = { + lon: (bbox.west + bbox.east) / 2, + lat: (bbox.south + bbox.north) / 2, + }; const t = lonLatToTile(center.lon, center.lat, 12); assert.deepEqual({ x: t.x, y: t.y }, { x: 935, y: 1686 }); }); test('lonLatToTile clamps poles and antimeridian into valid range', () => { - for (const [lon, lat] of [[-180, 89.9], [180, -89.9], [179.9999, 0]]) { + for (const [lon, lat] of [ + [-180, 89.9], + [180, -89.9], + [179.9999, 0], + ]) { const t = lonLatToTile(lon, lat, 12); - assert.equal(isValidTileCoord(12, t.x, t.y), true, `invalid tile for ${lon},${lat}: ${t.x},${t.y}`); + assert.equal( + isValidTileCoord(12, t.x, t.y), + true, + `invalid tile for ${lon},${lat}: ${t.x},${t.y}`, + ); } }); @@ -76,14 +93,19 @@ test('lonLatToTile clamps poles and antimeridian into valid range', () => { test('tilesForBounds: clamped Austin viewport covers the fixture tile', () => { const bounds = { - south: AUSTIN.lat - 0.025, north: AUSTIN.lat + 0.025, - west: AUSTIN.lon - 0.025, east: AUSTIN.lon + 0.025, + south: AUSTIN.lat - 0.025, + north: AUSTIN.lat + 0.025, + west: AUSTIN.lon - 0.025, + east: AUSTIN.lon + 0.025, }; const tiles = tilesForBounds(bounds, 12); - assert.ok(tiles.length >= 1 && tiles.length <= 4, `tile count ${tiles.length}`); + assert.ok( + tiles.length >= 1 && tiles.length <= 4, + `tile count ${tiles.length}`, + ); assert.ok( tiles.some((t) => t.z === 12 && t.x === 935 && t.y === 1686), - `fixture tile missing from ${JSON.stringify(tiles)}` + `fixture tile missing from ${JSON.stringify(tiles)}`, ); }); @@ -102,7 +124,7 @@ test('tilesForBounds: bounds straddling a tile edge return both tiles', () => { }); test('tilesForBounds: default zoom is 12 and every tile is valid', () => { - const bounds = { south: 51.49, north: 51.52, west: -0.14, east: -0.10 }; // London + const bounds = { south: 51.49, north: 51.52, west: -0.14, east: -0.1 }; // London const tiles = tilesForBounds(bounds); assert.ok(tiles.length >= 1); for (const t of tiles) { @@ -112,7 +134,11 @@ test('tilesForBounds: default zoom is 12 and every tile is valid', () => { }); test('tilesForBounds: runaway bounds are truncated by the safety cap', () => { - const tiles = tilesForBounds({ south: -60, north: 60, west: -170, east: 170 }, 12, { maxTiles: 16 }); + const tiles = tilesForBounds( + { south: -60, north: 60, west: -170, east: 170 }, + 12, + { maxTiles: 16 }, + ); assert.ok(tiles.length <= 16, `got ${tiles.length}`); }); @@ -129,15 +155,30 @@ test('normalizeBudget: same-day state passes through untouched', () => { }); test('normalizeBudget: day rollover resets the counter', () => { - const rolled = normalizeBudget({ date: '2026-07-16', count: 39999 }, '2026-07-17'); + const rolled = normalizeBudget( + { date: '2026-07-16', count: 39999 }, + '2026-07-17', + ); assert.deepEqual(rolled, { date: '2026-07-17', count: 0 }); }); test('normalizeBudget: missing/corrupt state starts fresh', () => { - assert.deepEqual(normalizeBudget(null, '2026-07-16'), { date: '2026-07-16', count: 0 }); - assert.deepEqual(normalizeBudget({ date: '2026-07-16', count: NaN }, '2026-07-16'), { date: '2026-07-16', count: 0 }); - assert.deepEqual(normalizeBudget({ count: 5 }, '2026-07-16'), { date: '2026-07-16', count: 0 }); - assert.deepEqual(normalizeBudget({ date: '2026-07-16', count: -3 }, '2026-07-16'), { date: '2026-07-16', count: 0 }); + assert.deepEqual(normalizeBudget(null, '2026-07-16'), { + date: '2026-07-16', + count: 0, + }); + assert.deepEqual( + normalizeBudget({ date: '2026-07-16', count: NaN }, '2026-07-16'), + { date: '2026-07-16', count: 0 }, + ); + assert.deepEqual(normalizeBudget({ count: 5 }, '2026-07-16'), { + date: '2026-07-16', + count: 0, + }); + assert.deepEqual( + normalizeBudget({ date: '2026-07-16', count: -3 }, '2026-07-16'), + { date: '2026-07-16', count: 0 }, + ); }); test('isOverBudget: at or above the limit is over, below is not', () => { diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs index 5f1340e10..38e87c6f6 100644 --- a/src/tooling/environmentProviders.test.mjs +++ b/src/tooling/environmentProviders.test.mjs @@ -9,111 +9,248 @@ import { localProviderPlugins } from '../../server/providers/local.js'; function install(plugin) { const routes = new Map(); - plugin.configureServer({ middlewares: { use(route, handler) { routes.set(route, handler); } } }); + plugin.configureServer({ + middlewares: { + use(route, handler) { + routes.set(route, handler); + }, + }, + }); assert.equal(routes.size, 1); return async (url = '/', method = 'GET') => { - const res = { headersSent: false, writeHead(status, headers) { Object.assign(this, { status, headers, headersSent: true }); }, end(body) { this.body = body; } }; + const res = { + headersSent: false, + writeHead(status, headers) { + Object.assign(this, { status, headers, headersSent: true }); + }, + end(body) { + this.body = body; + }, + }; await [...routes.values()][0]({ url, method }, res); return res; }; } function isolate(t, env = {}) { - for(const [name,value] of Object.entries(env)) { + for (const [name, value] of Object.entries(env)) { const previous = process.env[name]; - t.after(() => { if(previous === undefined) delete process.env[name]; else process.env[name] = previous; }); + t.after(() => { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + }); process.env[name] = value; } - t.mock.method(fsp, 'readFile', async () => { throw Error('no disk cache'); }); - t.mock.method(fsp, 'stat', async () => { throw Error('no disk cache'); }); + t.mock.method(fsp, 'readFile', async () => { + throw Error('no disk cache'); + }); + t.mock.method(fsp, 'stat', async () => { + throw Error('no disk cache'); + }); t.mock.method(fsp, 'mkdir', async () => {}); t.mock.method(fsp, 'writeFile', async () => {}); t.mock.method(globalThis, 'setInterval', () => ({ unref() {} })); t.mock.method(console, 'warn', () => {}); } -const json = res => JSON.parse(res.body); +const json = (res) => JSON.parse(res.body); test('standalone composition mounts every extracted provider exactly once without acquisition', (t) => { - t.mock.method(globalThis, 'fetch', () => { throw Error('construction must not fetch'); }); + t.mock.method(globalThis, 'fetch', () => { + throw Error('construction must not fetch'); + }); const plugins = localProviderPlugins(); - for(const factory of [terrainHeightsProxy,tomtomProxy,firmsProxy,gbfsProxy]) assert.equal(plugins.filter(p=>p.name===factory().name).length,1); + for (const factory of [ + terrainHeightsProxy, + tomtomProxy, + firmsProxy, + gbfsProxy, + ]) + assert.equal(plugins.filter((p) => p.name === factory().name).length, 1); }); test('terrain middleware chunks missing points and reconstructs repeated/reordered requests from cache', async (t) => { - isolate(t);let calls=0;const sizes=[]; + isolate(t); + let calls = 0; + const sizes = []; t.mock.method(globalThis, 'fetch', async (raw) => { - calls++;const url=new URL(raw);assert.equal(url.origin,'https://terrain.reearth.land'); - const points=url.searchParams.get('points').split(';').map(p=>p.split(',').map(Number));sizes.push(points.length); - return Response.json({results:points.map(([lon])=>({ellipsoid:lon+100}))}); + calls++; + const url = new URL(raw); + assert.equal(url.origin, 'https://terrain.reearth.land'); + const points = url.searchParams + .get('points') + .split(';') + .map((p) => p.split(',').map(Number)); + sizes.push(points.length); + return Response.json({ + results: points.map(([lon]) => ({ ellipsoid: lon + 100 })), + }); }); - const request=install(terrainHeightsProxy()); - const points=Array.from({length:257},(_,i)=>`${i/100},1`); - const res=await request('/?points='+points.join(';')); - assert.equal(res.status,200);assert.deepEqual(sizes,[256,1]); - const reordered=await request('/?points=2.56,1;0,1;2.56,1'); - assert.deepEqual(json(reordered),{results:[{ellipsoid:102.56},{ellipsoid:100},{ellipsoid:102.56}]});assert.equal(calls,2); - assert.equal((await request('/?points=invalid')).status,400); - assert.equal((await request('/?points='+Array(2001).fill('0,1').join(';'))).status,500); - assert.equal(calls,2); + const request = install(terrainHeightsProxy()); + const points = Array.from({ length: 257 }, (_, i) => `${i / 100},1`); + const res = await request('/?points=' + points.join(';')); + assert.equal(res.status, 200); + assert.deepEqual(sizes, [256, 1]); + const reordered = await request('/?points=2.56,1;0,1;2.56,1'); + assert.deepEqual(json(reordered), { + results: [{ ellipsoid: 102.56 }, { ellipsoid: 100 }, { ellipsoid: 102.56 }], + }); + assert.equal(calls, 2); + assert.equal((await request('/?points=invalid')).status, 400); + assert.equal( + (await request('/?points=' + Array(2001).fill('0,1').join(';'))).status, + 500, + ); + assert.equal(calls, 2); }); test('terrain middleware migrates valid legacy disk points without fabricating omitted heights', async (t) => { isolate(t); - t.mock.method(fsp,'readFile',async()=>JSON.stringify({'1,2;3,4':{at:Date.now(),results:[{ellipsoid:77}]}})); - let calls=0; - t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).searchParams.get('points'),'3.00000,4.00000');return Response.json({results:[{ellipsoid:88}]});}); - const res=await install(terrainHeightsProxy())('/?points=3,4;1,2'); - assert.deepEqual(json(res),{results:[{ellipsoid:88},{ellipsoid:77}]});assert.equal(calls,1); + t.mock.method(fsp, 'readFile', async () => + JSON.stringify({ + '1,2;3,4': { at: Date.now(), results: [{ ellipsoid: 77 }] }, + }), + ); + let calls = 0; + t.mock.method(globalThis, 'fetch', async (raw) => { + calls++; + assert.equal(new URL(raw).searchParams.get('points'), '3.00000,4.00000'); + return Response.json({ results: [{ ellipsoid: 88 }] }); + }); + const res = await install(terrainHeightsProxy())('/?points=3,4;1,2'); + assert.deepEqual(json(res), { + results: [{ ellipsoid: 88 }, { ellipsoid: 77 }], + }); + assert.equal(calls, 1); }); test('traffic middleware preserves keyless mode, caching, stale budget fallback and UTC rollover', async (t) => { - isolate(t,{TOMTOM_API_KEY:'',TOMTOM_DAILY_TILE_BUDGET:'1'}); - let now=Date.UTC(2026,8,12,12);t.mock.method(Date,'now',()=>now); - let calls=0;t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).searchParams.get('key'),'fixture-key');return new Response(new Uint8Array([1,2,3]));}); - const request=install(tomtomProxy()); - assert.equal(json(await request('/status')).hasKey,false); - assert.equal((await request('/flow/8/1/1.pbf')).status,503); - assert.equal((await request('/flow/7/1/1.pbf')).status,400);assert.equal(calls,0); - process.env.TOMTOM_API_KEY='fixture-key'; - assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'MISS'); - assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'HIT'); - assert.equal(json(await request('/status')).dailyCount,1);assert.equal(calls,1); - now+=120001; - assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'STALE-BUDGET'); - assert.equal((await request('/flow/8/2/1.pbf')).status,429);assert.equal(calls,1); - now+=86400000; - assert.equal(json(await request('/status')).dailyCount,0); - assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'MISS');assert.equal(calls,2); + isolate(t, { TOMTOM_API_KEY: '', TOMTOM_DAILY_TILE_BUDGET: '1' }); + let now = Date.UTC(2026, 8, 12, 12); + t.mock.method(Date, 'now', () => now); + let calls = 0; + t.mock.method(globalThis, 'fetch', async (raw) => { + calls++; + assert.equal(new URL(raw).searchParams.get('key'), 'fixture-key'); + return new Response(new Uint8Array([1, 2, 3])); + }); + const request = install(tomtomProxy()); + assert.equal(json(await request('/status')).hasKey, false); + assert.equal((await request('/flow/8/1/1.pbf')).status, 503); + assert.equal((await request('/flow/7/1/1.pbf')).status, 400); + assert.equal(calls, 0); + process.env.TOMTOM_API_KEY = 'fixture-key'; + assert.equal( + (await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'], + 'MISS', + ); + assert.equal( + (await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'], + 'HIT', + ); + assert.equal(json(await request('/status')).dailyCount, 1); + assert.equal(calls, 1); + now += 120001; + assert.equal( + (await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'], + 'STALE-BUDGET', + ); + assert.equal((await request('/flow/8/2/1.pbf')).status, 429); + assert.equal(calls, 1); + now += 86400000; + assert.equal(json(await request('/status')).dailyCount, 0); + assert.equal( + (await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'], + 'MISS', + ); + assert.equal(calls, 2); }); test('FIRMS retains a large successful source during partial failure and filters stale data at serve time', async (t) => { - isolate(t,{FIRMS_MAP_KEY:''});let now=Date.UTC(2026,8,12,12);t.mock.method(Date,'now',()=>now); - let calls=0; - const header='latitude,longitude,acq_date,acq_time,confidence,frp\n'; - const csv=header+'30,-97,2026-09-12,1200,h,10\n'.repeat(130001); - t.mock.method(globalThis,'fetch',async(raw)=>{ - calls++;const url=new URL(raw);assert.equal(url.hostname,'firms.modaps.eosdis.nasa.gov'); - if(url.pathname.includes('mapkey_status')) return Response.json({current_transactions:3,transaction_limit:5000}); - return url.pathname.includes('VIIRS_NOAA20')?new Response(csv):new Response('offline',{status:503}); + isolate(t, { FIRMS_MAP_KEY: '' }); + let now = Date.UTC(2026, 8, 12, 12); + t.mock.method(Date, 'now', () => now); + let calls = 0; + const header = 'latitude,longitude,acq_date,acq_time,confidence,frp\n'; + const csv = header + '30,-97,2026-09-12,1200,h,10\n'.repeat(130001); + t.mock.method(globalThis, 'fetch', async (raw) => { + calls++; + const url = new URL(raw); + assert.equal(url.hostname, 'firms.modaps.eosdis.nasa.gov'); + if (url.pathname.includes('mapkey_status')) + return Response.json({ + current_transactions: 3, + transaction_limit: 5000, + }); + return url.pathname.includes('VIIRS_NOAA20') + ? new Response(csv) + : new Response('offline', { status: 503 }); }); - const request=install(firmsProxy()); - assert.equal((await request()).status,503);assert.equal(json(await request('/status')).hasKey,false);assert.equal(calls,0); - process.env.FIRMS_MAP_KEY='fixture-key'; - const first=json(await request());assert.equal(first.count,130001);assert.equal(first.sources.filter(s=>s.ok).length,1);assert.equal(calls,3); - assert.equal(json(await request()).count,130001);assert.equal(calls,3); - assert.deepEqual(json(await request('/status')).transactions,{used:3,limit:5000}); - now+=25*3600000;t.mock.method(globalThis,'fetch',async()=>new Response('offline',{status:503})); - const stale=json(await request());assert.equal(stale.stale,true);assert.equal(stale.count,0); + const request = install(firmsProxy()); + assert.equal((await request()).status, 503); + assert.equal(json(await request('/status')).hasKey, false); + assert.equal(calls, 0); + process.env.FIRMS_MAP_KEY = 'fixture-key'; + const first = json(await request()); + assert.equal(first.count, 130001); + assert.equal(first.sources.filter((s) => s.ok).length, 1); + assert.equal(calls, 3); + assert.equal(json(await request()).count, 130001); + assert.equal(calls, 3); + assert.deepEqual(json(await request('/status')).transactions, { + used: 3, + limit: 5000, + }); + now += 25 * 3600000; + t.mock.method( + globalThis, + 'fetch', + async () => new Response('offline', { status: 503 }), + ); + const stale = json(await request()); + assert.equal(stale.stale, true); + assert.equal(stale.count, 0); }); test('GBFS keeps host/path/method guards, response caps and distinct information/status cache headers', async (t) => { - let calls=0;t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).hostname,'gbfs.lyft.com');return Response.json({data:{stations:[]}});}); - const request=install(gbfsProxy());const target=p=>'/'+encodeURIComponent('https://gbfs.lyft.com/'+p); - assert.equal((await request(target('station_status.json'),'POST')).status,405); - assert.equal((await request('/'+encodeURIComponent('https://example.com/station_status.json'))).status,403); - assert.equal((await request(target('gbfs.json'))).status,400); - assert.equal((await request('/%zz')).status,400);assert.equal(calls,0); - assert.equal((await request(target('station_information.json'))).headers['Cache-Control'],'public, max-age=300'); - assert.equal((await request(target('station_status.json'))).headers['Cache-Control'],'no-store'); - t.mock.method(globalThis,'fetch',async()=>new Response('x',{headers:{'content-length':String(6*1024*1024)}})); - assert.equal((await request(target('station_status.json'))).status,502); + let calls = 0; + t.mock.method(globalThis, 'fetch', async (raw) => { + calls++; + assert.equal(new URL(raw).hostname, 'gbfs.lyft.com'); + return Response.json({ data: { stations: [] } }); + }); + const request = install(gbfsProxy()); + const target = (p) => '/' + encodeURIComponent('https://gbfs.lyft.com/' + p); + assert.equal( + (await request(target('station_status.json'), 'POST')).status, + 405, + ); + assert.equal( + ( + await request( + '/' + encodeURIComponent('https://example.com/station_status.json'), + ) + ).status, + 403, + ); + assert.equal((await request(target('gbfs.json'))).status, 400); + assert.equal((await request('/%zz')).status, 400); + assert.equal(calls, 0); + assert.equal( + (await request(target('station_information.json'))).headers[ + 'Cache-Control' + ], + 'public, max-age=300', + ); + assert.equal( + (await request(target('station_status.json'))).headers['Cache-Control'], + 'no-store', + ); + t.mock.method( + globalThis, + 'fetch', + async () => + new Response('x', { + headers: { 'content-length': String(6 * 1024 * 1024) }, + }), + ); + assert.equal((await request(target('station_status.json'))).status, 502); }); From eadeac6c1055c0a720df2291b05a4cc1ff4566e6 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Fri, 11 Sep 2026 22:01:12 -0700 Subject: [PATCH 3/3] test: follow extracted FIRMS aggregation code --- src/data/firmsProxy.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/firmsProxy.test.mjs b/src/data/firmsProxy.test.mjs index b65b9ae72..a6dcdb774 100644 --- a/src/data/firmsProxy.test.mjs +++ b/src/data/firmsProxy.test.mjs @@ -3,7 +3,7 @@ import fs from 'node:fs'; import test from 'node:test'; import { filterTrailing24h, parseFirmsCsv } from './firmsCsv.js'; -const config = fs.readFileSync(new URL('../../server/providers/local.js', import.meta.url), 'utf8'); +const config = fs.readFileSync(new URL('../../server/providers/firms.js', import.meta.url), 'utf8'); const start = config.indexOf(' async function refreshUpstream(key) {'); assert.notEqual(start, -1, 'FIRMS refresh function exists'); const end = config.indexOf('\n }', start);