mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-06 11:24:00 +00:00
refactor: split terrain traffic fire and bike-share providers
This commit is contained in:
1 parent
3ec3fe6822
commit
f4054469d5
13 files changed
+1024
-797
No files matched your search
@@ -13,6 +13,9 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
- Separate terrain, traffic, FIRMS and GBFS middleware into focused provider
|
||||||
|
modules, preserving local configuration, routes and cache/error behavior.
|
||||||
|
|
||||||
- Split satellite and launch-feed server providers into focused modules with
|
- Split satellite and launch-feed server providers into focused modules with
|
||||||
portable request URL builders, preserving routes and cache/error behavior.
|
portable request URL builders, preserving routes and cache/error behavior.
|
||||||
|
|
||||||
|
|||||||
@@ -132,3 +132,23 @@ Callers supply the group or end date and own validation, credentials, transport,
|
|||||||
response limits and cache policy. The boundary gate checks this portable entry
|
response limits and cache policy. The boundary gate checks this portable entry
|
||||||
separately from the Node providers. Satellite rendering and launch replay remain
|
separately from the Node providers. Satellite rendering and launch replay remain
|
||||||
in their existing browser modules.
|
in their existing browser modules.
|
||||||
|
|
||||||
|
## Terrain, traffic, fires and bike-share providers
|
||||||
|
|
||||||
|
`gods-eye-view/server/providers/terrain`, `/traffic`, `/firms` and `/gbfs`
|
||||||
|
are separate Node-only entries. Each owns its existing middleware and
|
||||||
|
process-scoped cache or request handling. Standalone composition mounts them in
|
||||||
|
the original order; their imports do not start acquisition.
|
||||||
|
|
||||||
|
`gods-eye-view/sources/terrain` exports existing point-key, retry and cache
|
||||||
|
reconstruction mechanics with injectable acquisition dependencies.
|
||||||
|
`gods-eye-view/sources/traffic` exports tile math and budget calculations.
|
||||||
|
`gods-eye-view/sources/gbfs` exports host/path acceptance and cache-header rules.
|
||||||
|
These entries import no Node middleware, application configuration or rendering.
|
||||||
|
Callers retain their request admission and transport policy. FIRMS CSV parsing
|
||||||
|
remains an owned dependency of the FIRMS Node provider. The boundary gate
|
||||||
|
checks each entry independently.
|
||||||
|
|
||||||
|
Browser terrain sampling, traffic matching/drawing, fire overlays and bike-share
|
||||||
|
layer lifecycle remain in their current modules. This extraction changes no
|
||||||
|
source defaults, credentials, quotas, data interpretation or visual behavior.
|
||||||
@@ -1,5 +1,15 @@
|
|||||||
# God's Eye View Current State
|
# God's Eye View Current State
|
||||||
|
|
||||||
|
## Terrain, traffic, fire and bike-share provider modules
|
||||||
|
|
||||||
|
Local composition now imports separate Node modules for Re:Earth heights,
|
||||||
|
TomTom flow tiles, NASA FIRMS detections and GBFS station feeds. Existing routes,
|
||||||
|
plugin order, server-key selection, validation, disk caches, budgets, retries
|
||||||
|
and stale/error responses remain unchanged. Each has a Node-only package entry
|
||||||
|
under `gods-eye-view/server/providers/`. Portable terrain mechanics, traffic tile
|
||||||
|
math and GBFS source rules are available under `gods-eye-view/sources/`.
|
||||||
|
The browser layers and their rendering remain in their existing modules.
|
||||||
|
|
||||||
## Landmark annotation identity
|
## Landmark annotation identity
|
||||||
|
|
||||||
When a landmark geocode contains only address components, annotations retain
|
When a landmark geocode contains only address components, annotations retain
|
||||||
|
|||||||
+16
-1
@@ -81,6 +81,21 @@
|
|||||||
"./server/providers/space": {
|
"./server/providers/space": {
|
||||||
"node": "./server/providers/space.js"
|
"node": "./server/providers/space.js"
|
||||||
},
|
},
|
||||||
"./sources/space": "./src/data/spaceProviderRequests.js"
|
"./sources/space": "./src/data/spaceProviderRequests.js",
|
||||||
|
"./server/providers/terrain": {
|
||||||
|
"node": "./server/providers/terrain.js"
|
||||||
|
},
|
||||||
|
"./sources/terrain": "./src/data/terrainHeightsProxy.js",
|
||||||
|
"./server/providers/traffic": {
|
||||||
|
"node": "./server/providers/traffic.js"
|
||||||
|
},
|
||||||
|
"./sources/traffic": "./src/data/tomtomTiles.js",
|
||||||
|
"./server/providers/firms": {
|
||||||
|
"node": "./server/providers/firms.js"
|
||||||
|
},
|
||||||
|
"./server/providers/gbfs": {
|
||||||
|
"node": "./server/providers/gbfs.js"
|
||||||
|
},
|
||||||
|
"./sources/gbfs": "./src/data/gbfsSource.js"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -89,5 +89,47 @@
|
|||||||
"exports": ["./sources/space"],
|
"exports": ["./sources/space"],
|
||||||
"modules": ["src/data/spaceProviderRequests.js"],
|
"modules": ["src/data/spaceProviderRequests.js"],
|
||||||
"external": []
|
"external": []
|
||||||
|
},
|
||||||
|
"terrain-provider": {
|
||||||
|
"runtime": "node",
|
||||||
|
"exports": ["./server/providers/terrain"],
|
||||||
|
"modules": [
|
||||||
|
"server/providers/terrain.js",
|
||||||
|
"src/data/terrainHeightsProxy.js"
|
||||||
|
],
|
||||||
|
"external": []
|
||||||
|
},
|
||||||
|
"terrain-source": {
|
||||||
|
"exports": ["./sources/terrain"],
|
||||||
|
"modules": ["src/data/terrainHeightsProxy.js"],
|
||||||
|
"external": []
|
||||||
|
},
|
||||||
|
"traffic-provider": {
|
||||||
|
"runtime": "node",
|
||||||
|
"exports": ["./server/providers/traffic"],
|
||||||
|
"modules": ["server/providers/traffic.js", "src/data/tomtomTiles.js"],
|
||||||
|
"external": []
|
||||||
|
},
|
||||||
|
"traffic-source": {
|
||||||
|
"exports": ["./sources/traffic"],
|
||||||
|
"modules": ["src/data/tomtomTiles.js"],
|
||||||
|
"external": []
|
||||||
|
},
|
||||||
|
"firms-provider": {
|
||||||
|
"runtime": "node",
|
||||||
|
"exports": ["./server/providers/firms"],
|
||||||
|
"modules": ["server/providers/firms.js", "src/data/firmsCsv.js"],
|
||||||
|
"external": []
|
||||||
|
},
|
||||||
|
"gbfs-provider": {
|
||||||
|
"runtime": "node",
|
||||||
|
"exports": ["./server/providers/gbfs"],
|
||||||
|
"modules": ["server/providers/gbfs.js", "src/data/gbfsSource.js"],
|
||||||
|
"external": []
|
||||||
|
},
|
||||||
|
"gbfs-source": {
|
||||||
|
"exports": ["./sources/gbfs"],
|
||||||
|
"modules": ["src/data/gbfsSource.js"],
|
||||||
|
"external": []
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
import path from 'node:path';
|
||||||
|
import { promises as fsp } from 'node:fs';
|
||||||
|
|
||||||
|
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* NASA FIRMS live active-fire proxy with a memory + disk cache.
|
||||||
|
* Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2
|
||||||
|
*
|
||||||
|
* Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent
|
||||||
|
* satellites, no cross-source dedup) fetched sequentially with `days=2`
|
||||||
|
* (`days=1` means "current UTC day", nearly empty just after 00:00Z) and
|
||||||
|
* clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS quota is
|
||||||
|
* 5,000 transactions / 10 min per MAP_KEY, so the cache is the point:
|
||||||
|
* TTL 30 min, single-flight refresh, serve-stale-on-failure, and a
|
||||||
|
* fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream
|
||||||
|
* fetch across dev-server restarts. Pattern mirrors celestrakProxy.
|
||||||
|
*
|
||||||
|
* Routes:
|
||||||
|
* GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires}
|
||||||
|
* GET /api/firms/status → {hasKey, lastFetch, count, stale, ttlMs, transactions}
|
||||||
|
*
|
||||||
|
* Keyless (no FIRMS_MAP_KEY): /api/firms → 503 {error:'no_key'}; status →
|
||||||
|
* {hasKey:false}. Upstream is never touched without a key.
|
||||||
|
*
|
||||||
|
* @returns {import('vite').Plugin}
|
||||||
|
*/
|
||||||
|
export function firmsProxy() {
|
||||||
|
const TTL_MS = 30 * 60_000;
|
||||||
|
const STATUS_TTL_MS = 5 * 60_000;
|
||||||
|
const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'VIIRS_SNPP_NRT'];
|
||||||
|
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
|
||||||
|
const CACHE_PATH = path.join(CACHE_DIR, 'firms.json');
|
||||||
|
|
||||||
|
/** @type {?{at: number, sources: Array<object>, fires: Array<object>}} */
|
||||||
|
let mem = null;
|
||||||
|
let diskChecked = false;
|
||||||
|
/** @type {?Promise<?{at: number, sources: Array<object>, fires: Array<object>}>} single-flight refresh */
|
||||||
|
let inflight = null;
|
||||||
|
/** @type {?{at: number, transactions: ?{used: number, limit: number}}} mapkey_status cache */
|
||||||
|
let statusCache = null;
|
||||||
|
/** @type {?Promise<?{used: number, limit: number}>} */
|
||||||
|
let statusInflight = null;
|
||||||
|
|
||||||
|
const mapKey = () => String(process.env.FIRMS_MAP_KEY || '').trim();
|
||||||
|
|
||||||
|
async function readDiskOnce() {
|
||||||
|
if (diskChecked) return;
|
||||||
|
diskChecked = true;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8'));
|
||||||
|
if (Number.isFinite(parsed?.at) && Array.isArray(parsed?.sources) && Array.isArray(parsed?.fires)) {
|
||||||
|
mem = parsed;
|
||||||
|
}
|
||||||
|
} catch { /* no disk cache yet */ }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeDisk(entry) {
|
||||||
|
try {
|
||||||
|
await fsp.mkdir(CACHE_DIR, { recursive: true });
|
||||||
|
await fsp.writeFile(CACHE_PATH, JSON.stringify(entry), 'utf8');
|
||||||
|
} catch (err) {
|
||||||
|
console.warn('[firms-proxy] cache write failed:', err?.message || err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch + parse one FIRMS source. Throws on HTTP error or a non-CSV body
|
||||||
|
* (FIRMS reports errors as HTML/plain text, never CSV). Never log the URL —
|
||||||
|
* it embeds the MAP_KEY.
|
||||||
|
*/
|
||||||
|
async function fetchSource(key, source) {
|
||||||
|
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
|
||||||
|
const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
|
||||||
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||||
|
const records = parseFirmsCsv(await res.text());
|
||||||
|
if (records === null) throw new Error('non-CSV upstream response');
|
||||||
|
return records;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Refresh all sources sequentially (quota courtesy — never in parallel).
|
||||||
|
* Partial success (≥1 source ok) still produces a cacheable entry with the
|
||||||
|
* failed sources marked ok:false; total failure throws so the caller can
|
||||||
|
* serve stale.
|
||||||
|
*/
|
||||||
|
async function refreshUpstream(key) {
|
||||||
|
const now = Date.now();
|
||||||
|
const sources = [];
|
||||||
|
const fires = [];
|
||||||
|
for (const source of SOURCES) {
|
||||||
|
try {
|
||||||
|
const records = filterTrailing24h(await fetchSource(key, source), now);
|
||||||
|
// NOT fires.push(...records): spread passes each record as an argument,
|
||||||
|
// and a world/2 VIIRS pull exceeds V8's argument limit (~125k) at
|
||||||
|
// ~131k records — RangeError, and the whole source is silently dropped.
|
||||||
|
for (const record of records) fires.push(record);
|
||||||
|
sources.push({ source, count: records.length, ok: true });
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[firms-proxy] ${source} fetch failed:`, err?.message || err);
|
||||||
|
sources.push({ source, count: 0, ok: false });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!sources.some((s) => s.ok)) throw new Error('all FIRMS sources failed');
|
||||||
|
return { at: now, sources, fires };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Cache entry → response payload. Fires are RE-filtered to the trailing
|
||||||
|
* 24 h at serve time so a stale cache never serves >24h-old detections.
|
||||||
|
*/
|
||||||
|
function buildPayload(entry, stale) {
|
||||||
|
const fires = filterTrailing24h(entry.fires, Date.now());
|
||||||
|
return {
|
||||||
|
fetchedAt: entry.at,
|
||||||
|
stale,
|
||||||
|
ttlMs: TTL_MS,
|
||||||
|
sources: entry.sources,
|
||||||
|
count: fires.length,
|
||||||
|
fires,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** mapkey_status transactions, cached 5 min, best-effort (null on failure). */
|
||||||
|
function getTransactions(key) {
|
||||||
|
const now = Date.now();
|
||||||
|
if (statusCache && now - statusCache.at < STATUS_TTL_MS) {
|
||||||
|
return Promise.resolve(statusCache.transactions);
|
||||||
|
}
|
||||||
|
if (!statusInflight) {
|
||||||
|
statusInflight = (async () => {
|
||||||
|
try {
|
||||||
|
const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`;
|
||||||
|
const res = await fetch(url, { signal: AbortSignal.timeout(10_000) });
|
||||||
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||||
|
const body = await res.json();
|
||||||
|
const used = Number(body?.current_transactions);
|
||||||
|
const limit = Number(body?.transaction_limit);
|
||||||
|
return Number.isFinite(used) && Number.isFinite(limit) ? { used, limit } : null;
|
||||||
|
} catch (err) {
|
||||||
|
console.warn('[firms-proxy] mapkey status failed:', err?.message || err);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
})()
|
||||||
|
.then((transactions) => {
|
||||||
|
statusCache = { at: Date.now(), transactions };
|
||||||
|
return transactions;
|
||||||
|
})
|
||||||
|
.finally(() => { statusInflight = null; });
|
||||||
|
}
|
||||||
|
return statusInflight;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: 'firms-proxy',
|
||||||
|
configureServer(server) {
|
||||||
|
server.middlewares.use('/api/firms', async (req, res) => {
|
||||||
|
const sendJson = (status, obj) => {
|
||||||
|
if (res.headersSent) return;
|
||||||
|
res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify(obj));
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const subPath = String(req.url || '').split('?')[0];
|
||||||
|
const key = mapKey();
|
||||||
|
await readDiskOnce();
|
||||||
|
|
||||||
|
if (subPath === '/status') {
|
||||||
|
if (!key) {
|
||||||
|
sendJson(200, { hasKey: false, lastFetch: null, count: null, stale: false, ttlMs: TTL_MS, transactions: null });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const transactions = await getTransactions(key);
|
||||||
|
sendJson(200, {
|
||||||
|
hasKey: true,
|
||||||
|
lastFetch: mem ? mem.at : null,
|
||||||
|
count: mem ? mem.fires.length : null,
|
||||||
|
stale: mem ? Date.now() - mem.at >= TTL_MS : false,
|
||||||
|
ttlMs: TTL_MS,
|
||||||
|
transactions,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!key) {
|
||||||
|
sendJson(503, { error: 'no_key' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const entry = mem;
|
||||||
|
if (entry && Date.now() - entry.at < TTL_MS) {
|
||||||
|
sendJson(200, buildPayload(entry, false));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Stale or missing → refresh, single-flight (concurrent requests
|
||||||
|
// share one upstream pass). Capture the promise locally BEFORE
|
||||||
|
// awaiting: the .finally() nulls `inflight` the moment it settles.
|
||||||
|
if (!inflight) {
|
||||||
|
inflight = refreshUpstream(key)
|
||||||
|
.then(async (fresh) => {
|
||||||
|
mem = fresh;
|
||||||
|
await writeDisk(fresh);
|
||||||
|
return fresh;
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
console.warn(`[firms-proxy] refresh failed (${err?.message || err}) — serving cache if any`);
|
||||||
|
return null;
|
||||||
|
})
|
||||||
|
.finally(() => { inflight = null; });
|
||||||
|
}
|
||||||
|
const pending = inflight;
|
||||||
|
const fresh = await pending;
|
||||||
|
if (fresh) {
|
||||||
|
sendJson(200, buildPayload(fresh, false));
|
||||||
|
} else if (entry) {
|
||||||
|
sendJson(200, buildPayload(entry, true)); // upstream down — stale beats empty
|
||||||
|
} else {
|
||||||
|
sendJson(502, { error: 'firms fetch failed and no cache available' });
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.warn('[firms-proxy] error:', err?.message || err);
|
||||||
|
sendJson(500, { error: 'firms proxy error' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
import { isAllowedGbfsHost, isAllowedGbfsPath, gbfsCacheControl } from '../../src/data/gbfsSource.js';
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// GBFS (General Bikeshare Feed Specification) proxy constants
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
/** Upstream fetch timeout for GBFS requests (ms). */
|
||||||
|
const GBFS_PROXY_TIMEOUT_MS = 12000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Vite plugin: GBFS bike-share proxy with host allowlisting and size limits.
|
||||||
|
*
|
||||||
|
* Accepts GET /api/gbfs/<encoded-upstream-URL> and proxies the request
|
||||||
|
* to the upstream GBFS provider. Validates hostname against an allowlist,
|
||||||
|
* restricts to station_information/station_status paths, enforces HTTPS,
|
||||||
|
* and caps response body at 5 MB.
|
||||||
|
*
|
||||||
|
* @returns {import('vite').Plugin}
|
||||||
|
*/
|
||||||
|
export function gbfsProxy() {
|
||||||
|
return {
|
||||||
|
name: 'gbfs-proxy',
|
||||||
|
configureServer(server) {
|
||||||
|
server.middlewares.use('/api/gbfs', async (req, res) => {
|
||||||
|
try {
|
||||||
|
if (req.method !== 'GET') {
|
||||||
|
res.writeHead(405, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = new URL(req.url || '/', 'http://localhost');
|
||||||
|
const encodedTarget = url.pathname.replace(/^\/+/, '');
|
||||||
|
if (!encodedTarget) {
|
||||||
|
res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'Missing GBFS upstream target' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let decodedTarget = '';
|
||||||
|
try {
|
||||||
|
decodedTarget = decodeURIComponent(encodedTarget);
|
||||||
|
} catch {
|
||||||
|
res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'Invalid GBFS target encoding' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let upstreamUrl = null;
|
||||||
|
try {
|
||||||
|
upstreamUrl = new URL(decodedTarget);
|
||||||
|
} catch {
|
||||||
|
res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'Invalid GBFS upstream URL' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (upstreamUrl.protocol !== 'https:') {
|
||||||
|
res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'Only https GBFS targets are allowed' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isAllowedGbfsHost(upstreamUrl.hostname)) {
|
||||||
|
res.writeHead(403, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'GBFS host not allowed' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isAllowedGbfsPath(upstreamUrl.pathname)) {
|
||||||
|
res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'Only station_information/station_status endpoints are allowed' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timeoutId = setTimeout(() => controller.abort(), GBFS_PROXY_TIMEOUT_MS);
|
||||||
|
let upstream;
|
||||||
|
try {
|
||||||
|
upstream = await fetch(upstreamUrl.toString(), {
|
||||||
|
method: 'GET',
|
||||||
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'User-Agent': 'gods-eye-view-gbfs-proxy/1.0',
|
||||||
|
},
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timeoutId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Limit response size to prevent memory exhaustion from malicious upstream
|
||||||
|
const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB
|
||||||
|
const contentLength = Number(upstream.headers.get('content-length'));
|
||||||
|
if (Number.isFinite(contentLength) && contentLength > GBFS_MAX_BODY_BYTES) {
|
||||||
|
res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'GBFS upstream response too large' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const body = await upstream.text();
|
||||||
|
if (Buffer.byteLength(body, 'utf8') > GBFS_MAX_BODY_BYTES) {
|
||||||
|
res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'GBFS upstream response too large' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const contentType = upstream.headers.get('content-type') || 'application/json';
|
||||||
|
res.writeHead(upstream.status, {
|
||||||
|
'Content-Type': contentType,
|
||||||
|
'Cache-Control': gbfsCacheControl(upstreamUrl.pathname),
|
||||||
|
'X-GBFS-Upstream': upstreamUrl.hostname,
|
||||||
|
'X-GBFS-Cache': 'MISS',
|
||||||
|
});
|
||||||
|
res.end(body);
|
||||||
|
} catch (error) {
|
||||||
|
if (error?.name === 'AbortError') {
|
||||||
|
res.writeHead(504, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'GBFS upstream timeout' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
console.error('[GBFS Proxy]', error?.message || String(error));
|
||||||
|
res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
||||||
|
res.end(JSON.stringify({ error: 'GBFS proxy error' }));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
+7
-795
@@ -1,3 +1,7 @@
|
|||||||
|
import { terrainHeightsProxy } from './terrain.js';
|
||||||
|
import { tomtomProxy } from './traffic.js';
|
||||||
|
import { firmsProxy } from './firms.js';
|
||||||
|
import { gbfsProxy } from './gbfs.js';
|
||||||
import { celestrakProxy, rocketLaunchesProxy } from './space.js';
|
import { celestrakProxy, rocketLaunchesProxy } from './space.js';
|
||||||
export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js';
|
export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js';
|
||||||
/**
|
/**
|
||||||
@@ -48,13 +52,8 @@ import { Readable } from 'node:stream';
|
|||||||
import https from 'node:https';
|
import https from 'node:https';
|
||||||
import { lookup as lookupDns } from 'node:dns/promises';
|
import { lookup as lookupDns } from 'node:dns/promises';
|
||||||
import { directionToHeading } from '../../src/data/directionText.js';
|
import { directionToHeading } from '../../src/data/directionText.js';
|
||||||
import {
|
|
||||||
isValidTileCoord as isValidTomTomTile,
|
|
||||||
utcDayKey as tomtomUtcDayKey,
|
|
||||||
normalizeBudget as normalizeTomTomBudget,
|
|
||||||
isOverBudget as isTomTomOverBudget,
|
|
||||||
} from '../../src/data/tomtomTiles.js';
|
|
||||||
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
|
|
||||||
import { fileURLToPath } from 'node:url';
|
import { fileURLToPath } from 'node:url';
|
||||||
import { normalizeRadioCountryInput } from '../../src/data/radioCountry.js';
|
import { normalizeRadioCountryInput } from '../../src/data/radioCountry.js';
|
||||||
import {
|
import {
|
||||||
@@ -74,13 +73,7 @@ import {
|
|||||||
validateKeySetupUpdates,
|
validateKeySetupUpdates,
|
||||||
} from '../../src/keySetupCore.mjs';
|
} from '../../src/keySetupCore.mjs';
|
||||||
import { hardenCredentialFile } from '../../src/keySetupHardening.mjs';
|
import { hardenCredentialFile } from '../../src/keySetupHardening.mjs';
|
||||||
import {
|
|
||||||
fetchTerrainChunkWithRetry,
|
|
||||||
parseTerrainPoints,
|
|
||||||
resolveTerrainHeightRequest,
|
|
||||||
terrainPointKey,
|
|
||||||
validTerrainResult,
|
|
||||||
} from '../../src/data/terrainHeightsProxy.js';
|
|
||||||
import { VOICE_MODELS, isKnownVoiceTier, resolveVoiceModel } from '../../src/voice/voiceCost.js';
|
import { VOICE_MODELS, isKnownVoiceTier, resolveVoiceModel } from '../../src/voice/voiceCost.js';
|
||||||
|
|
||||||
/** Resolve __dirname for ESM context. */
|
/** Resolve __dirname for ESM context. */
|
||||||
@@ -1111,22 +1104,6 @@ function radioBrowserProxy() {
|
|||||||
configurePreviewServer: install,
|
configurePreviewServer: install,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// GBFS (General Bikeshare Feed Specification) proxy constants
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
/** Upstream fetch timeout for GBFS requests (ms). */
|
|
||||||
const GBFS_PROXY_TIMEOUT_MS = 12000;
|
|
||||||
/** Allowlisted GBFS hostnames; wildcard *.publicbikesystem.net also accepted. */
|
|
||||||
const GBFS_ALLOWED_HOSTS = new Set([
|
|
||||||
'gbfs.lyft.com',
|
|
||||||
'gbfs.bluebikes.com',
|
|
||||||
'gbfs.bcycle.com',
|
|
||||||
'gbfs.biketownpdx.com',
|
|
||||||
'gbfs.cogobikeshare.com',
|
|
||||||
'austin.publicbikesystem.net',
|
|
||||||
'hon.publicbikesystem.net',
|
|
||||||
'chat.publicbikesystem.net',
|
|
||||||
]);
|
|
||||||
// Sourced from the shared voice-model registry so the client's cost estimate
|
// Sourced from the shared voice-model registry so the client's cost estimate
|
||||||
// can never be computed against a different model than the session runs on.
|
// can never be computed against a different model than the session runs on.
|
||||||
const OPENAI_REALTIME_MODEL_DEFAULT = VOICE_MODELS.standard.id;
|
const OPENAI_REALTIME_MODEL_DEFAULT = VOICE_MODELS.standard.id;
|
||||||
@@ -1140,611 +1117,6 @@ const REALTIME_DEBUG_LOG_DIR = path.join(__dirname, '.gev-logs');
|
|||||||
const REALTIME_DEBUG_LOG_FILE = path.join(REALTIME_DEBUG_LOG_DIR, 'realtime-conversations.jsonl');
|
const REALTIME_DEBUG_LOG_FILE = path.join(REALTIME_DEBUG_LOG_DIR, 'realtime-conversations.jsonl');
|
||||||
const REALTIME_DEBUG_LOG_MAX_BYTES = 8 * 1024 * 1024;
|
const REALTIME_DEBUG_LOG_MAX_BYTES = 8 * 1024 * 1024;
|
||||||
|
|
||||||
/**
|
|
||||||
* TomTom traffic-flow vector-tile proxy with a daily budget governor.
|
|
||||||
*
|
|
||||||
* Upstream: https://api.tomtom.com/traffic/map/4/tile/flow/relative/{z}/{x}/{y}.pbf
|
|
||||||
* (style `relative`; the response is an UNCOMPRESSED Mapbox Vector Tile, layer
|
|
||||||
* "Traffic flow"). The key comes from TOMTOM_API_KEY server-side only — the
|
|
||||||
* browser fetches same-origin `/api/tomtom/flow/{z}/{x}/{y}.pbf`.
|
|
||||||
*
|
|
||||||
* Cache: memory + disk (.gev-cache/tomtom/), TTL 120 s (traffic is fresh
|
|
||||||
* data), single-flight per tile, serve-stale-on-failure — the celestrakProxy
|
|
||||||
* pattern. Cache hits never count against the budget.
|
|
||||||
*
|
|
||||||
* Budget governor (mirrors the OpenSky credit-governor philosophy — last-good
|
|
||||||
* data beats a dead layer): a persistent counter (.gev-cache/tomtom/budget.json,
|
|
||||||
* keyed by UTC date, reset on day change) counts upstream fetch attempts
|
|
||||||
* against a soft cap (TOMTOM_DAILY_TILE_BUDGET, default 40,000 of the free
|
|
||||||
* tier's ~50k/day). Over the cap the proxy serves stale tiles when available,
|
|
||||||
* else 429 {error:'budget'}.
|
|
||||||
*
|
|
||||||
* GET /api/tomtom/status → {hasKey, dailyCount, budget, date}. Keyless mode:
|
|
||||||
* status reports hasKey:false and the tile endpoint 503s {error:'no_key'}
|
|
||||||
* without touching upstream — the traffic layer then stays in simulation mode.
|
|
||||||
*
|
|
||||||
* @returns {import('vite').Plugin}
|
|
||||||
*/
|
|
||||||
function tomtomProxy() {
|
|
||||||
const TILE_TTL_MS = 120_000;
|
|
||||||
const CACHE_DIR = path.join(process.cwd(), '.gev-cache', 'tomtom');
|
|
||||||
const BUDGET_PATH = path.join(CACHE_DIR, 'budget.json');
|
|
||||||
const DEFAULT_DAILY_BUDGET = 40000;
|
|
||||||
const MEM_MAX_ENTRIES = 256;
|
|
||||||
const UPSTREAM_TIMEOUT_MS = 15000;
|
|
||||||
|
|
||||||
/** @type {Map<string, {at:number, buf:Buffer}>} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */
|
|
||||||
const mem = new Map();
|
|
||||||
/** @type {Map<string, Promise<{at:number, buf:Buffer}|null>>} single-flight per tile. */
|
|
||||||
const inflight = new Map();
|
|
||||||
|
|
||||||
/** @type {{date:string, count:number}|null} lazily-loaded persistent counter. */
|
|
||||||
let budget = null;
|
|
||||||
let budgetLoaded = false;
|
|
||||||
|
|
||||||
function dailyBudgetLimit() {
|
|
||||||
const raw = Number.parseInt(process.env.TOMTOM_DAILY_TILE_BUDGET || '', 10);
|
|
||||||
return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_DAILY_BUDGET;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadBudgetOnce() {
|
|
||||||
if (budgetLoaded) return;
|
|
||||||
budgetLoaded = true;
|
|
||||||
try {
|
|
||||||
const parsed = JSON.parse(await fsp.readFile(BUDGET_PATH, 'utf8'));
|
|
||||||
if (parsed && typeof parsed.date === 'string' && Number.isFinite(parsed.count)) {
|
|
||||||
budget = parsed;
|
|
||||||
}
|
|
||||||
} catch { /* no budget file yet */ }
|
|
||||||
}
|
|
||||||
|
|
||||||
async function persistBudget() {
|
|
||||||
try {
|
|
||||||
await fsp.mkdir(CACHE_DIR, { recursive: true });
|
|
||||||
await fsp.writeFile(BUDGET_PATH, JSON.stringify(budget), 'utf8');
|
|
||||||
} catch (err) {
|
|
||||||
console.warn('[tomtom-proxy] budget write failed:', err?.message || err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Roll the counter to today (UTC) and return it. */
|
|
||||||
function currentBudget() {
|
|
||||||
budget = normalizeTomTomBudget(budget, tomtomUtcDayKey());
|
|
||||||
return budget;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Count one upstream fetch attempt against today's budget (async persist). */
|
|
||||||
function recordUpstreamFetch() {
|
|
||||||
currentBudget().count += 1;
|
|
||||||
void persistBudget();
|
|
||||||
}
|
|
||||||
|
|
||||||
const tilePath = (key) => path.join(CACHE_DIR, `flow-${key.replaceAll('/', '-')}.pbf`);
|
|
||||||
|
|
||||||
/** Disk-cache read; tile age comes from the file's mtime. */
|
|
||||||
async function readDiskTile(key) {
|
|
||||||
try {
|
|
||||||
const [stat, buf] = await Promise.all([
|
|
||||||
fsp.stat(tilePath(key)),
|
|
||||||
fsp.readFile(tilePath(key)),
|
|
||||||
]);
|
|
||||||
return { at: stat.mtimeMs, buf };
|
|
||||||
} catch { return null; }
|
|
||||||
}
|
|
||||||
|
|
||||||
async function writeDiskTile(key, buf) {
|
|
||||||
try {
|
|
||||||
await fsp.mkdir(CACHE_DIR, { recursive: true });
|
|
||||||
await fsp.writeFile(tilePath(key), buf);
|
|
||||||
} catch (err) {
|
|
||||||
console.warn(`[tomtom-proxy] tile cache write failed for ${key}:`, err?.message || err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** LRU-ish memory insert (Map preserves insertion order; evict the oldest). */
|
|
||||||
function memSet(key, entry) {
|
|
||||||
if (!mem.has(key) && mem.size >= MEM_MAX_ENTRIES) {
|
|
||||||
const oldest = mem.keys().next().value;
|
|
||||||
mem.delete(oldest);
|
|
||||||
}
|
|
||||||
mem.set(key, entry);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fetchUpstream(z, x, y) {
|
|
||||||
const url = 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/'
|
|
||||||
+ `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`;
|
|
||||||
recordUpstreamFetch(); // attempts count — upstream bills the request either way
|
|
||||||
const res = await fetch(url, { signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS) });
|
|
||||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
||||||
const buf = Buffer.from(await res.arrayBuffer());
|
|
||||||
if (buf.length === 0) throw new Error('empty tile body');
|
|
||||||
return buf;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
name: 'tomtom-proxy',
|
|
||||||
configureServer(server) {
|
|
||||||
server.middlewares.use('/api/tomtom', async (req, res) => {
|
|
||||||
// Sanitized responses only (proxy/security baseline): no upstream
|
|
||||||
// error details, and never echo the key or the upstream URL.
|
|
||||||
const sendJson = (status, obj, extraHeaders = {}) => {
|
|
||||||
if (res.headersSent) return;
|
|
||||||
res.writeHead(status, {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Cache-Control': 'no-store',
|
|
||||||
...extraHeaders,
|
|
||||||
});
|
|
||||||
res.end(JSON.stringify(obj));
|
|
||||||
};
|
|
||||||
const sendTile = (buf, cacheStatus) => {
|
|
||||||
if (res.headersSent) return;
|
|
||||||
res.writeHead(200, {
|
|
||||||
'Content-Type': 'application/x-protobuf',
|
|
||||||
'Cache-Control': 'no-store',
|
|
||||||
'x-tomtom-cache': cacheStatus,
|
|
||||||
});
|
|
||||||
res.end(buf);
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
await loadBudgetOnce();
|
|
||||||
const urlPath = String(req.url || '').split('?')[0];
|
|
||||||
|
|
||||||
if (urlPath === '/status') {
|
|
||||||
const hasKey = Boolean(process.env.TOMTOM_API_KEY);
|
|
||||||
const b = currentBudget();
|
|
||||||
sendJson(200, { hasKey, dailyCount: b.count, budget: dailyBudgetLimit(), date: b.date });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const m = urlPath.match(/^\/flow\/(\d+)\/(\d+)\/(\d+)\.pbf$/);
|
|
||||||
if (!m) {
|
|
||||||
sendJson(404, { error: 'not_found' });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const z = Number(m[1]);
|
|
||||||
const x = Number(m[2]);
|
|
||||||
const y = Number(m[3]);
|
|
||||||
if (!isValidTomTomTile(z, x, y)) {
|
|
||||||
sendJson(400, { error: 'invalid_tile' });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (!process.env.TOMTOM_API_KEY) {
|
|
||||||
sendJson(503, { error: 'no_key' });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const key = `${z}/${x}/${y}`;
|
|
||||||
const now = Date.now();
|
|
||||||
|
|
||||||
let entry = mem.get(key);
|
|
||||||
if (!entry) {
|
|
||||||
entry = await readDiskTile(key);
|
|
||||||
if (entry) memSet(key, entry);
|
|
||||||
}
|
|
||||||
// Fresh cache hit — never counts against the budget.
|
|
||||||
if (entry && now - entry.at < TILE_TTL_MS) {
|
|
||||||
sendTile(entry.buf, 'HIT');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Budget governor: over the soft cap, last-good data beats a dead layer.
|
|
||||||
if (isTomTomOverBudget(currentBudget(), dailyBudgetLimit())) {
|
|
||||||
if (entry) {
|
|
||||||
sendTile(entry.buf, 'STALE-BUDGET');
|
|
||||||
} else {
|
|
||||||
sendJson(429, { error: 'budget' });
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stale or missing → refresh, single-flight per tile.
|
|
||||||
if (!inflight.has(key)) {
|
|
||||||
inflight.set(key, fetchUpstream(z, x, y)
|
|
||||||
.then(async (buf) => {
|
|
||||||
const fresh = { at: Date.now(), buf };
|
|
||||||
memSet(key, fresh);
|
|
||||||
await writeDiskTile(key, buf);
|
|
||||||
return fresh;
|
|
||||||
})
|
|
||||||
.catch((err) => {
|
|
||||||
console.warn(`[tomtom-proxy] ${key} fetch failed (${err?.message || err}) — serving stale if any`);
|
|
||||||
return null;
|
|
||||||
})
|
|
||||||
.finally(() => inflight.delete(key)));
|
|
||||||
}
|
|
||||||
const fresh = await inflight.get(key);
|
|
||||||
if (fresh) {
|
|
||||||
sendTile(fresh.buf, 'MISS');
|
|
||||||
} else if (entry) {
|
|
||||||
sendTile(entry.buf, 'STALE-ERROR'); // upstream down — stale beats empty
|
|
||||||
} else {
|
|
||||||
sendJson(502, { error: 'upstream' });
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.warn('[tomtom-proxy] error:', err?.message || err);
|
|
||||||
sendJson(500, { error: 'proxy' });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* NASA FIRMS live active-fire proxy with a memory + disk cache.
|
|
||||||
* Upstream: https://firms.modaps.eosdis.nasa.gov/api/area/csv/{KEY}/{SOURCE}/world/2
|
|
||||||
*
|
|
||||||
* Merges three VIIRS NRT sources (NOAA-20, NOAA-21, Suomi-NPP — independent
|
|
||||||
* satellites, no cross-source dedup) fetched sequentially with `days=2`
|
|
||||||
* (`days=1` means "current UTC day", nearly empty just after 00:00Z) and
|
|
||||||
* clamps to the trailing 24 h via src/data/firmsCsv.js. FIRMS quota is
|
|
||||||
* 5,000 transactions / 10 min per MAP_KEY, so the cache is the point:
|
|
||||||
* TTL 30 min, single-flight refresh, serve-stale-on-failure, and a
|
|
||||||
* fresh-enough disk cache (.gev-cache/firms.json) prevents ANY upstream
|
|
||||||
* fetch across dev-server restarts. Pattern mirrors celestrakProxy.
|
|
||||||
*
|
|
||||||
* Routes:
|
|
||||||
* GET /api/firms → {fetchedAt, stale, ttlMs, sources, count, fires}
|
|
||||||
* GET /api/firms/status → {hasKey, lastFetch, count, stale, ttlMs, transactions}
|
|
||||||
*
|
|
||||||
* Keyless (no FIRMS_MAP_KEY): /api/firms → 503 {error:'no_key'}; status →
|
|
||||||
* {hasKey:false}. Upstream is never touched without a key.
|
|
||||||
*
|
|
||||||
* @returns {import('vite').Plugin}
|
|
||||||
*/
|
|
||||||
function firmsProxy() {
|
|
||||||
const TTL_MS = 30 * 60_000;
|
|
||||||
const STATUS_TTL_MS = 5 * 60_000;
|
|
||||||
const SOURCES = ['VIIRS_NOAA20_NRT', 'VIIRS_NOAA21_NRT', 'VIIRS_SNPP_NRT'];
|
|
||||||
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
|
|
||||||
const CACHE_PATH = path.join(CACHE_DIR, 'firms.json');
|
|
||||||
|
|
||||||
/** @type {?{at: number, sources: Array<object>, fires: Array<object>}} */
|
|
||||||
let mem = null;
|
|
||||||
let diskChecked = false;
|
|
||||||
/** @type {?Promise<?{at: number, sources: Array<object>, fires: Array<object>}>} single-flight refresh */
|
|
||||||
let inflight = null;
|
|
||||||
/** @type {?{at: number, transactions: ?{used: number, limit: number}}} mapkey_status cache */
|
|
||||||
let statusCache = null;
|
|
||||||
/** @type {?Promise<?{used: number, limit: number}>} */
|
|
||||||
let statusInflight = null;
|
|
||||||
|
|
||||||
const mapKey = () => String(process.env.FIRMS_MAP_KEY || '').trim();
|
|
||||||
|
|
||||||
async function readDiskOnce() {
|
|
||||||
if (diskChecked) return;
|
|
||||||
diskChecked = true;
|
|
||||||
try {
|
|
||||||
const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8'));
|
|
||||||
if (Number.isFinite(parsed?.at) && Array.isArray(parsed?.sources) && Array.isArray(parsed?.fires)) {
|
|
||||||
mem = parsed;
|
|
||||||
}
|
|
||||||
} catch { /* no disk cache yet */ }
|
|
||||||
}
|
|
||||||
|
|
||||||
async function writeDisk(entry) {
|
|
||||||
try {
|
|
||||||
await fsp.mkdir(CACHE_DIR, { recursive: true });
|
|
||||||
await fsp.writeFile(CACHE_PATH, JSON.stringify(entry), 'utf8');
|
|
||||||
} catch (err) {
|
|
||||||
console.warn('[firms-proxy] cache write failed:', err?.message || err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fetch + parse one FIRMS source. Throws on HTTP error or a non-CSV body
|
|
||||||
* (FIRMS reports errors as HTML/plain text, never CSV). Never log the URL —
|
|
||||||
* it embeds the MAP_KEY.
|
|
||||||
*/
|
|
||||||
async function fetchSource(key, source) {
|
|
||||||
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
|
|
||||||
const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
|
|
||||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
||||||
const records = parseFirmsCsv(await res.text());
|
|
||||||
if (records === null) throw new Error('non-CSV upstream response');
|
|
||||||
return records;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Refresh all sources sequentially (quota courtesy — never in parallel).
|
|
||||||
* Partial success (≥1 source ok) still produces a cacheable entry with the
|
|
||||||
* failed sources marked ok:false; total failure throws so the caller can
|
|
||||||
* serve stale.
|
|
||||||
*/
|
|
||||||
async function refreshUpstream(key) {
|
|
||||||
const now = Date.now();
|
|
||||||
const sources = [];
|
|
||||||
const fires = [];
|
|
||||||
for (const source of SOURCES) {
|
|
||||||
try {
|
|
||||||
const records = filterTrailing24h(await fetchSource(key, source), now);
|
|
||||||
// NOT fires.push(...records): spread passes each record as an argument,
|
|
||||||
// and a world/2 VIIRS pull exceeds V8's argument limit (~125k) at
|
|
||||||
// ~131k records — RangeError, and the whole source is silently dropped.
|
|
||||||
for (const record of records) fires.push(record);
|
|
||||||
sources.push({ source, count: records.length, ok: true });
|
|
||||||
} catch (err) {
|
|
||||||
console.warn(`[firms-proxy] ${source} fetch failed:`, err?.message || err);
|
|
||||||
sources.push({ source, count: 0, ok: false });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!sources.some((s) => s.ok)) throw new Error('all FIRMS sources failed');
|
|
||||||
return { at: now, sources, fires };
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Cache entry → response payload. Fires are RE-filtered to the trailing
|
|
||||||
* 24 h at serve time so a stale cache never serves >24h-old detections.
|
|
||||||
*/
|
|
||||||
function buildPayload(entry, stale) {
|
|
||||||
const fires = filterTrailing24h(entry.fires, Date.now());
|
|
||||||
return {
|
|
||||||
fetchedAt: entry.at,
|
|
||||||
stale,
|
|
||||||
ttlMs: TTL_MS,
|
|
||||||
sources: entry.sources,
|
|
||||||
count: fires.length,
|
|
||||||
fires,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/** mapkey_status transactions, cached 5 min, best-effort (null on failure). */
|
|
||||||
function getTransactions(key) {
|
|
||||||
const now = Date.now();
|
|
||||||
if (statusCache && now - statusCache.at < STATUS_TTL_MS) {
|
|
||||||
return Promise.resolve(statusCache.transactions);
|
|
||||||
}
|
|
||||||
if (!statusInflight) {
|
|
||||||
statusInflight = (async () => {
|
|
||||||
try {
|
|
||||||
const url = `https://firms.modaps.eosdis.nasa.gov/mapserver/mapkey_status/?MAP_KEY=${encodeURIComponent(key)}`;
|
|
||||||
const res = await fetch(url, { signal: AbortSignal.timeout(10_000) });
|
|
||||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
||||||
const body = await res.json();
|
|
||||||
const used = Number(body?.current_transactions);
|
|
||||||
const limit = Number(body?.transaction_limit);
|
|
||||||
return Number.isFinite(used) && Number.isFinite(limit) ? { used, limit } : null;
|
|
||||||
} catch (err) {
|
|
||||||
console.warn('[firms-proxy] mapkey status failed:', err?.message || err);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
})()
|
|
||||||
.then((transactions) => {
|
|
||||||
statusCache = { at: Date.now(), transactions };
|
|
||||||
return transactions;
|
|
||||||
})
|
|
||||||
.finally(() => { statusInflight = null; });
|
|
||||||
}
|
|
||||||
return statusInflight;
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
name: 'firms-proxy',
|
|
||||||
configureServer(server) {
|
|
||||||
server.middlewares.use('/api/firms', async (req, res) => {
|
|
||||||
const sendJson = (status, obj) => {
|
|
||||||
if (res.headersSent) return;
|
|
||||||
res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify(obj));
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
const subPath = String(req.url || '').split('?')[0];
|
|
||||||
const key = mapKey();
|
|
||||||
await readDiskOnce();
|
|
||||||
|
|
||||||
if (subPath === '/status') {
|
|
||||||
if (!key) {
|
|
||||||
sendJson(200, { hasKey: false, lastFetch: null, count: null, stale: false, ttlMs: TTL_MS, transactions: null });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const transactions = await getTransactions(key);
|
|
||||||
sendJson(200, {
|
|
||||||
hasKey: true,
|
|
||||||
lastFetch: mem ? mem.at : null,
|
|
||||||
count: mem ? mem.fires.length : null,
|
|
||||||
stale: mem ? Date.now() - mem.at >= TTL_MS : false,
|
|
||||||
ttlMs: TTL_MS,
|
|
||||||
transactions,
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!key) {
|
|
||||||
sendJson(503, { error: 'no_key' });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const entry = mem;
|
|
||||||
if (entry && Date.now() - entry.at < TTL_MS) {
|
|
||||||
sendJson(200, buildPayload(entry, false));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Stale or missing → refresh, single-flight (concurrent requests
|
|
||||||
// share one upstream pass). Capture the promise locally BEFORE
|
|
||||||
// awaiting: the .finally() nulls `inflight` the moment it settles.
|
|
||||||
if (!inflight) {
|
|
||||||
inflight = refreshUpstream(key)
|
|
||||||
.then(async (fresh) => {
|
|
||||||
mem = fresh;
|
|
||||||
await writeDisk(fresh);
|
|
||||||
return fresh;
|
|
||||||
})
|
|
||||||
.catch((err) => {
|
|
||||||
console.warn(`[firms-proxy] refresh failed (${err?.message || err}) — serving cache if any`);
|
|
||||||
return null;
|
|
||||||
})
|
|
||||||
.finally(() => { inflight = null; });
|
|
||||||
}
|
|
||||||
const pending = inflight;
|
|
||||||
const fresh = await pending;
|
|
||||||
if (fresh) {
|
|
||||||
sendJson(200, buildPayload(fresh, false));
|
|
||||||
} else if (entry) {
|
|
||||||
sendJson(200, buildPayload(entry, true)); // upstream down — stale beats empty
|
|
||||||
} else {
|
|
||||||
sendJson(502, { error: 'firms fetch failed and no cache available' });
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.warn('[firms-proxy] error:', err?.message || err);
|
|
||||||
sendJson(500, { error: 'firms proxy error' });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Re:Earth terrain point-height proxy: batched lon/lat → ellipsoidal height
|
|
||||||
* lookups, keyless. Upstream: https://terrain.reearth.land/heights.json
|
|
||||||
* (≤256 points per call). Terrain doesn't move, so results are cached to
|
|
||||||
* disk with a long TTL (30 days) — mirrors celestrakProxy's memory+disk
|
|
||||||
* cache and serve-stale shape. Cache entries and stale fallback are keyed per
|
|
||||||
* 5dp point, so reordered and partially overlapping batches reuse prior work.
|
|
||||||
* Only missing/stale points go upstream; the response is rebuilt in exact
|
|
||||||
* request order. Oversized requests (>256 points) are chunked sequentially.
|
|
||||||
*/
|
|
||||||
function terrainHeightsProxy() {
|
|
||||||
const TTL_MS = 30 * 24 * 3600_000;
|
|
||||||
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
|
|
||||||
const CACHE_PATH = path.join(CACHE_DIR, 'terrain-heights.json');
|
|
||||||
const UPSTREAM_CHUNK = 256;
|
|
||||||
const MAX_POINTS = 2000;
|
|
||||||
|
|
||||||
/** @type {Map<string, {at:number, result:object}>} keyed by canonical 5dp lon/lat. */
|
|
||||||
const mem = new Map();
|
|
||||||
/** @type {Map<string, Promise<Array<object>>>} single-flight per missing-point subset. */
|
|
||||||
const inflight = new Map();
|
|
||||||
let diskLoaded = false;
|
|
||||||
let diskDirty = false;
|
|
||||||
|
|
||||||
/** Load the on-disk cache into memory once, lazily (first request only). */
|
|
||||||
async function loadDiskOnce() {
|
|
||||||
if (diskLoaded) return;
|
|
||||||
diskLoaded = true;
|
|
||||||
try {
|
|
||||||
const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8'));
|
|
||||||
const pointEntries = parsed?.version === 2 && parsed.points && typeof parsed.points === 'object'
|
|
||||||
? parsed.points
|
|
||||||
: null;
|
|
||||||
if (pointEntries) {
|
|
||||||
for (const [key, entry] of Object.entries(pointEntries)) {
|
|
||||||
if (entry && Number.isFinite(entry.at) && validTerrainResult(entry.result)) {
|
|
||||||
mem.set(key, entry);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if (parsed && typeof parsed === 'object') {
|
|
||||||
// One-time migration from the former raw-batch cache. Zip only real,
|
|
||||||
// positionally present results; an absent value never becomes height 0.
|
|
||||||
for (const [rawPoints, entry] of Object.entries(parsed)) {
|
|
||||||
const points = parseTerrainPoints(rawPoints);
|
|
||||||
if (!points || !entry || !Number.isFinite(entry.at) || !Array.isArray(entry.results)) continue;
|
|
||||||
for (let i = 0; i < points.length; i += 1) {
|
|
||||||
const result = entry.results[i];
|
|
||||||
if (!validTerrainResult(result)) continue;
|
|
||||||
const key = terrainPointKey(points[i]);
|
|
||||||
const existing = mem.get(key);
|
|
||||||
if (!existing || entry.at > existing.at) mem.set(key, { at: entry.at, result });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
diskDirty = mem.size > 0;
|
|
||||||
}
|
|
||||||
} catch { /* no disk cache yet */ }
|
|
||||||
// Periodic flush, same shape as adsbdbProxy: coalesce writes instead of
|
|
||||||
// hitting disk on every request.
|
|
||||||
setInterval(async () => {
|
|
||||||
if (!diskDirty) return;
|
|
||||||
diskDirty = false;
|
|
||||||
try {
|
|
||||||
await fsp.mkdir(CACHE_DIR, { recursive: true });
|
|
||||||
const obj = { version: 2, points: Object.fromEntries(mem.entries()) };
|
|
||||||
await fsp.writeFile(CACHE_PATH, JSON.stringify(obj), 'utf8');
|
|
||||||
} catch (err) {
|
|
||||||
diskDirty = true; // retry next tick
|
|
||||||
console.warn('[terrain-heights-proxy] cache write failed');
|
|
||||||
}
|
|
||||||
}, 15_000).unref?.();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Fetch all missing chunks sequentially (upstream caps each call at 256).
|
|
||||||
*
|
|
||||||
* The first try keeps its empirically required 30s timeout; network errors,
|
|
||||||
* 429, and 5xx receive up to three jittered retries sharing a 10s added-time
|
|
||||||
* budget, with Retry-After honored within that bound.
|
|
||||||
* @param {Array<[number, number]>} points
|
|
||||||
* @returns {Promise<Array<object>>}
|
|
||||||
*/
|
|
||||||
async function fetchUpstreamAll(points) {
|
|
||||||
const results = [];
|
|
||||||
for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) {
|
|
||||||
const chunk = points.slice(i, i + UPSTREAM_CHUNK);
|
|
||||||
const chunkResults = await fetchTerrainChunkWithRetry(chunk);
|
|
||||||
// Keep later chunks aligned even if a malformed upstream response omits
|
|
||||||
// trailing positions. The resolver will reject each null individually.
|
|
||||||
for (let j = 0; j < chunk.length; j += 1) results.push(chunkResults[j] ?? null);
|
|
||||||
}
|
|
||||||
return results;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Coalesce concurrent requests for the same canonical missing-point list. */
|
|
||||||
function fetchMissingSingleFlight(points) {
|
|
||||||
const key = points.map(terrainPointKey).join(';');
|
|
||||||
if (!inflight.has(key)) {
|
|
||||||
const request = fetchUpstreamAll(points)
|
|
||||||
.finally(() => {
|
|
||||||
if (inflight.get(key) === request) inflight.delete(key);
|
|
||||||
});
|
|
||||||
inflight.set(key, request);
|
|
||||||
}
|
|
||||||
return inflight.get(key);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
name: 'terrain-heights-proxy',
|
|
||||||
configureServer(server) {
|
|
||||||
server.middlewares.use('/api/terrain/heights', async (req, res) => {
|
|
||||||
const send = (status, bodyObj) => {
|
|
||||||
if (res.headersSent) return;
|
|
||||||
res.writeHead(status, { 'Content-Type': 'application/json' });
|
|
||||||
res.end(JSON.stringify(bodyObj));
|
|
||||||
};
|
|
||||||
try {
|
|
||||||
await loadDiskOnce();
|
|
||||||
const parsedUrl = new URL(req.url || '', 'http://internal');
|
|
||||||
const rawPoints = parsedUrl.searchParams.get('points');
|
|
||||||
const points = parseTerrainPoints(rawPoints);
|
|
||||||
if (!points) {
|
|
||||||
send(400, { error: 'invalid points parameter — expected "lon,lat;lon,lat;…" with finite numbers' });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (points.length > MAX_POINTS) {
|
|
||||||
send(500, { error: `too many points (${points.length}); max ${MAX_POINTS} per request` });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const outcome = await resolveTerrainHeightRequest({
|
|
||||||
points,
|
|
||||||
cache: mem,
|
|
||||||
fetchMissing: fetchMissingSingleFlight,
|
|
||||||
ttlMs: TTL_MS,
|
|
||||||
});
|
|
||||||
if (outcome.cacheChanged) diskDirty = true;
|
|
||||||
if (outcome.upstreamError) {
|
|
||||||
console.warn(
|
|
||||||
'[terrain-heights-proxy] refresh incomplete'
|
|
||||||
+ ' — serving stale points when available'
|
|
||||||
);
|
|
||||||
}
|
|
||||||
send(outcome.status, outcome.body);
|
|
||||||
} catch (err) {
|
|
||||||
console.error('[terrain-heights-proxy] request failed');
|
|
||||||
send(500, { error: 'terrain heights proxy error' });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Detect whether an Overpass API response body indicates rate-limiting.
|
* Detect whether an Overpass API response body indicates rate-limiting.
|
||||||
*
|
*
|
||||||
@@ -2053,166 +1425,6 @@ function overpassProxy() {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Check whether a hostname is in the GBFS allowlist.
|
|
||||||
*
|
|
||||||
* Also accepts any subdomain of publicbikesystem.net.
|
|
||||||
*
|
|
||||||
* @param {string} hostname
|
|
||||||
* @returns {boolean}
|
|
||||||
*/
|
|
||||||
function isAllowedGbfsHost(hostname) {
|
|
||||||
const host = String(hostname || '').trim().toLowerCase();
|
|
||||||
if (!host) return false;
|
|
||||||
if (GBFS_ALLOWED_HOSTS.has(host)) return true;
|
|
||||||
return host.endsWith('.publicbikesystem.net');
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Only allow station_information.json and station_status.json endpoints.
|
|
||||||
*
|
|
||||||
* @param {string} pathname
|
|
||||||
* @returns {boolean}
|
|
||||||
*/
|
|
||||||
function isAllowedGbfsPath(pathname) {
|
|
||||||
return /\/station_(information|status)\.json$/i.test(String(pathname || ''));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Return an appropriate Cache-Control header for a GBFS endpoint.
|
|
||||||
*
|
|
||||||
* station_information is semi-static (5 min cache); station_status is
|
|
||||||
* real-time (no-store).
|
|
||||||
*
|
|
||||||
* @param {string} pathname
|
|
||||||
* @returns {string} Cache-Control header value.
|
|
||||||
*/
|
|
||||||
function gbfsCacheControl(pathname) {
|
|
||||||
if (/\/station_information\.json$/i.test(String(pathname || ''))) {
|
|
||||||
return 'public, max-age=300';
|
|
||||||
}
|
|
||||||
return 'no-store';
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Vite plugin: GBFS bike-share proxy with host allowlisting and size limits.
|
|
||||||
*
|
|
||||||
* Accepts GET /api/gbfs/<encoded-upstream-URL> and proxies the request
|
|
||||||
* to the upstream GBFS provider. Validates hostname against an allowlist,
|
|
||||||
* restricts to station_information/station_status paths, enforces HTTPS,
|
|
||||||
* and caps response body at 5 MB.
|
|
||||||
*
|
|
||||||
* @returns {import('vite').Plugin}
|
|
||||||
*/
|
|
||||||
function gbfsProxy() {
|
|
||||||
return {
|
|
||||||
name: 'gbfs-proxy',
|
|
||||||
configureServer(server) {
|
|
||||||
server.middlewares.use('/api/gbfs', async (req, res) => {
|
|
||||||
try {
|
|
||||||
if (req.method !== 'GET') {
|
|
||||||
res.writeHead(405, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'Method Not Allowed' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const url = new URL(req.url || '/', 'http://localhost');
|
|
||||||
const encodedTarget = url.pathname.replace(/^\/+/, '');
|
|
||||||
if (!encodedTarget) {
|
|
||||||
res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'Missing GBFS upstream target' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let decodedTarget = '';
|
|
||||||
try {
|
|
||||||
decodedTarget = decodeURIComponent(encodedTarget);
|
|
||||||
} catch {
|
|
||||||
res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'Invalid GBFS target encoding' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let upstreamUrl = null;
|
|
||||||
try {
|
|
||||||
upstreamUrl = new URL(decodedTarget);
|
|
||||||
} catch {
|
|
||||||
res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'Invalid GBFS upstream URL' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (upstreamUrl.protocol !== 'https:') {
|
|
||||||
res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'Only https GBFS targets are allowed' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isAllowedGbfsHost(upstreamUrl.hostname)) {
|
|
||||||
res.writeHead(403, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'GBFS host not allowed' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isAllowedGbfsPath(upstreamUrl.pathname)) {
|
|
||||||
res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'Only station_information/station_status endpoints are allowed' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const controller = new AbortController();
|
|
||||||
const timeoutId = setTimeout(() => controller.abort(), GBFS_PROXY_TIMEOUT_MS);
|
|
||||||
let upstream;
|
|
||||||
try {
|
|
||||||
upstream = await fetch(upstreamUrl.toString(), {
|
|
||||||
method: 'GET',
|
|
||||||
headers: {
|
|
||||||
Accept: 'application/json',
|
|
||||||
'User-Agent': 'gods-eye-view-gbfs-proxy/1.0',
|
|
||||||
},
|
|
||||||
signal: controller.signal,
|
|
||||||
});
|
|
||||||
} finally {
|
|
||||||
clearTimeout(timeoutId);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Limit response size to prevent memory exhaustion from malicious upstream
|
|
||||||
const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB
|
|
||||||
const contentLength = Number(upstream.headers.get('content-length'));
|
|
||||||
if (Number.isFinite(contentLength) && contentLength > GBFS_MAX_BODY_BYTES) {
|
|
||||||
res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'GBFS upstream response too large' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const body = await upstream.text();
|
|
||||||
if (Buffer.byteLength(body, 'utf8') > GBFS_MAX_BODY_BYTES) {
|
|
||||||
res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'GBFS upstream response too large' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const contentType = upstream.headers.get('content-type') || 'application/json';
|
|
||||||
res.writeHead(upstream.status, {
|
|
||||||
'Content-Type': contentType,
|
|
||||||
'Cache-Control': gbfsCacheControl(upstreamUrl.pathname),
|
|
||||||
'X-GBFS-Upstream': upstreamUrl.hostname,
|
|
||||||
'X-GBFS-Cache': 'MISS',
|
|
||||||
});
|
|
||||||
res.end(body);
|
|
||||||
} catch (error) {
|
|
||||||
if (error?.name === 'AbortError') {
|
|
||||||
res.writeHead(504, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'GBFS upstream timeout' }));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
console.error('[GBFS Proxy]', error?.message || String(error));
|
|
||||||
res.writeHead(502, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store' });
|
|
||||||
res.end(JSON.stringify({ error: 'GBFS proxy error' }));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* FNV-1a 32-bit hash of a string, used to derive deterministic pseudo-random
|
* FNV-1a 32-bit hash of a string, used to derive deterministic pseudo-random
|
||||||
* values (e.g. hue for synthetic SVG billboards, fallback heading angles).
|
* values (e.g. hue for synthetic SVG billboards, fallback heading angles).
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
import path from 'node:path';
|
||||||
|
import { promises as fsp } from 'node:fs';
|
||||||
|
|
||||||
|
import {
|
||||||
|
fetchTerrainChunkWithRetry,
|
||||||
|
parseTerrainPoints,
|
||||||
|
resolveTerrainHeightRequest,
|
||||||
|
terrainPointKey,
|
||||||
|
validTerrainResult,
|
||||||
|
} from '../../src/data/terrainHeightsProxy.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Re:Earth terrain point-height proxy: batched lon/lat → ellipsoidal height
|
||||||
|
* lookups, keyless. Upstream: https://terrain.reearth.land/heights.json
|
||||||
|
* (≤256 points per call). Terrain doesn't move, so results are cached to
|
||||||
|
* disk with a long TTL (30 days) — mirrors celestrakProxy's memory+disk
|
||||||
|
* cache and serve-stale shape. Cache entries and stale fallback are keyed per
|
||||||
|
* 5dp point, so reordered and partially overlapping batches reuse prior work.
|
||||||
|
* Only missing/stale points go upstream; the response is rebuilt in exact
|
||||||
|
* request order. Oversized requests (>256 points) are chunked sequentially.
|
||||||
|
*/
|
||||||
|
export function terrainHeightsProxy() {
|
||||||
|
const TTL_MS = 30 * 24 * 3600_000;
|
||||||
|
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
|
||||||
|
const CACHE_PATH = path.join(CACHE_DIR, 'terrain-heights.json');
|
||||||
|
const UPSTREAM_CHUNK = 256;
|
||||||
|
const MAX_POINTS = 2000;
|
||||||
|
|
||||||
|
/** @type {Map<string, {at:number, result:object}>} keyed by canonical 5dp lon/lat. */
|
||||||
|
const mem = new Map();
|
||||||
|
/** @type {Map<string, Promise<Array<object>>>} single-flight per missing-point subset. */
|
||||||
|
const inflight = new Map();
|
||||||
|
let diskLoaded = false;
|
||||||
|
let diskDirty = false;
|
||||||
|
|
||||||
|
/** Load the on-disk cache into memory once, lazily (first request only). */
|
||||||
|
async function loadDiskOnce() {
|
||||||
|
if (diskLoaded) return;
|
||||||
|
diskLoaded = true;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(await fsp.readFile(CACHE_PATH, 'utf8'));
|
||||||
|
const pointEntries = parsed?.version === 2 && parsed.points && typeof parsed.points === 'object'
|
||||||
|
? parsed.points
|
||||||
|
: null;
|
||||||
|
if (pointEntries) {
|
||||||
|
for (const [key, entry] of Object.entries(pointEntries)) {
|
||||||
|
if (entry && Number.isFinite(entry.at) && validTerrainResult(entry.result)) {
|
||||||
|
mem.set(key, entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (parsed && typeof parsed === 'object') {
|
||||||
|
// One-time migration from the former raw-batch cache. Zip only real,
|
||||||
|
// positionally present results; an absent value never becomes height 0.
|
||||||
|
for (const [rawPoints, entry] of Object.entries(parsed)) {
|
||||||
|
const points = parseTerrainPoints(rawPoints);
|
||||||
|
if (!points || !entry || !Number.isFinite(entry.at) || !Array.isArray(entry.results)) continue;
|
||||||
|
for (let i = 0; i < points.length; i += 1) {
|
||||||
|
const result = entry.results[i];
|
||||||
|
if (!validTerrainResult(result)) continue;
|
||||||
|
const key = terrainPointKey(points[i]);
|
||||||
|
const existing = mem.get(key);
|
||||||
|
if (!existing || entry.at > existing.at) mem.set(key, { at: entry.at, result });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
diskDirty = mem.size > 0;
|
||||||
|
}
|
||||||
|
} catch { /* no disk cache yet */ }
|
||||||
|
// Periodic flush, same shape as adsbdbProxy: coalesce writes instead of
|
||||||
|
// hitting disk on every request.
|
||||||
|
setInterval(async () => {
|
||||||
|
if (!diskDirty) return;
|
||||||
|
diskDirty = false;
|
||||||
|
try {
|
||||||
|
await fsp.mkdir(CACHE_DIR, { recursive: true });
|
||||||
|
const obj = { version: 2, points: Object.fromEntries(mem.entries()) };
|
||||||
|
await fsp.writeFile(CACHE_PATH, JSON.stringify(obj), 'utf8');
|
||||||
|
} catch (err) {
|
||||||
|
diskDirty = true; // retry next tick
|
||||||
|
console.warn('[terrain-heights-proxy] cache write failed');
|
||||||
|
}
|
||||||
|
}, 15_000).unref?.();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch all missing chunks sequentially (upstream caps each call at 256).
|
||||||
|
*
|
||||||
|
* The first try keeps its empirically required 30s timeout; network errors,
|
||||||
|
* 429, and 5xx receive up to three jittered retries sharing a 10s added-time
|
||||||
|
* budget, with Retry-After honored within that bound.
|
||||||
|
* @param {Array<[number, number]>} points
|
||||||
|
* @returns {Promise<Array<object>>}
|
||||||
|
*/
|
||||||
|
async function fetchUpstreamAll(points) {
|
||||||
|
const results = [];
|
||||||
|
for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) {
|
||||||
|
const chunk = points.slice(i, i + UPSTREAM_CHUNK);
|
||||||
|
const chunkResults = await fetchTerrainChunkWithRetry(chunk);
|
||||||
|
// Keep later chunks aligned even if a malformed upstream response omits
|
||||||
|
// trailing positions. The resolver will reject each null individually.
|
||||||
|
for (let j = 0; j < chunk.length; j += 1) results.push(chunkResults[j] ?? null);
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Coalesce concurrent requests for the same canonical missing-point list. */
|
||||||
|
function fetchMissingSingleFlight(points) {
|
||||||
|
const key = points.map(terrainPointKey).join(';');
|
||||||
|
if (!inflight.has(key)) {
|
||||||
|
const request = fetchUpstreamAll(points)
|
||||||
|
.finally(() => {
|
||||||
|
if (inflight.get(key) === request) inflight.delete(key);
|
||||||
|
});
|
||||||
|
inflight.set(key, request);
|
||||||
|
}
|
||||||
|
return inflight.get(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: 'terrain-heights-proxy',
|
||||||
|
configureServer(server) {
|
||||||
|
server.middlewares.use('/api/terrain/heights', async (req, res) => {
|
||||||
|
const send = (status, bodyObj) => {
|
||||||
|
if (res.headersSent) return;
|
||||||
|
res.writeHead(status, { 'Content-Type': 'application/json' });
|
||||||
|
res.end(JSON.stringify(bodyObj));
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
await loadDiskOnce();
|
||||||
|
const parsedUrl = new URL(req.url || '', 'http://internal');
|
||||||
|
const rawPoints = parsedUrl.searchParams.get('points');
|
||||||
|
const points = parseTerrainPoints(rawPoints);
|
||||||
|
if (!points) {
|
||||||
|
send(400, { error: 'invalid points parameter — expected "lon,lat;lon,lat;…" with finite numbers' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (points.length > MAX_POINTS) {
|
||||||
|
send(500, { error: `too many points (${points.length}); max ${MAX_POINTS} per request` });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const outcome = await resolveTerrainHeightRequest({
|
||||||
|
points,
|
||||||
|
cache: mem,
|
||||||
|
fetchMissing: fetchMissingSingleFlight,
|
||||||
|
ttlMs: TTL_MS,
|
||||||
|
});
|
||||||
|
if (outcome.cacheChanged) diskDirty = true;
|
||||||
|
if (outcome.upstreamError) {
|
||||||
|
console.warn(
|
||||||
|
'[terrain-heights-proxy] refresh incomplete'
|
||||||
|
+ ' — serving stale points when available'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
send(outcome.status, outcome.body);
|
||||||
|
} catch (err) {
|
||||||
|
console.error('[terrain-heights-proxy] request failed');
|
||||||
|
send(500, { error: 'terrain heights proxy error' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,239 @@
|
|||||||
|
import path from 'node:path';
|
||||||
|
import { promises as fsp } from 'node:fs';
|
||||||
|
|
||||||
|
import {
|
||||||
|
isValidTileCoord as isValidTomTomTile,
|
||||||
|
utcDayKey as tomtomUtcDayKey,
|
||||||
|
normalizeBudget as normalizeTomTomBudget,
|
||||||
|
isOverBudget as isTomTomOverBudget,
|
||||||
|
} from '../../src/data/tomtomTiles.js';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TomTom traffic-flow vector-tile proxy with a daily budget governor.
|
||||||
|
*
|
||||||
|
* Upstream: https://api.tomtom.com/traffic/map/4/tile/flow/relative/{z}/{x}/{y}.pbf
|
||||||
|
* (style `relative`; the response is an UNCOMPRESSED Mapbox Vector Tile, layer
|
||||||
|
* "Traffic flow"). The key comes from TOMTOM_API_KEY server-side only — the
|
||||||
|
* browser fetches same-origin `/api/tomtom/flow/{z}/{x}/{y}.pbf`.
|
||||||
|
*
|
||||||
|
* Cache: memory + disk (.gev-cache/tomtom/), TTL 120 s (traffic is fresh
|
||||||
|
* data), single-flight per tile, serve-stale-on-failure — the celestrakProxy
|
||||||
|
* pattern. Cache hits never count against the budget.
|
||||||
|
*
|
||||||
|
* Budget governor (mirrors the OpenSky credit-governor philosophy — last-good
|
||||||
|
* data beats a dead layer): a persistent counter (.gev-cache/tomtom/budget.json,
|
||||||
|
* keyed by UTC date, reset on day change) counts upstream fetch attempts
|
||||||
|
* against a soft cap (TOMTOM_DAILY_TILE_BUDGET, default 40,000 of the free
|
||||||
|
* tier's ~50k/day). Over the cap the proxy serves stale tiles when available,
|
||||||
|
* else 429 {error:'budget'}.
|
||||||
|
*
|
||||||
|
* GET /api/tomtom/status → {hasKey, dailyCount, budget, date}. Keyless mode:
|
||||||
|
* status reports hasKey:false and the tile endpoint 503s {error:'no_key'}
|
||||||
|
* without touching upstream — the traffic layer then stays in simulation mode.
|
||||||
|
*
|
||||||
|
* @returns {import('vite').Plugin}
|
||||||
|
*/
|
||||||
|
export function tomtomProxy() {
|
||||||
|
const TILE_TTL_MS = 120_000;
|
||||||
|
const CACHE_DIR = path.join(process.cwd(), '.gev-cache', 'tomtom');
|
||||||
|
const BUDGET_PATH = path.join(CACHE_DIR, 'budget.json');
|
||||||
|
const DEFAULT_DAILY_BUDGET = 40000;
|
||||||
|
const MEM_MAX_ENTRIES = 256;
|
||||||
|
const UPSTREAM_TIMEOUT_MS = 15000;
|
||||||
|
|
||||||
|
/** @type {Map<string, {at:number, buf:Buffer}>} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */
|
||||||
|
const mem = new Map();
|
||||||
|
/** @type {Map<string, Promise<{at:number, buf:Buffer}|null>>} single-flight per tile. */
|
||||||
|
const inflight = new Map();
|
||||||
|
|
||||||
|
/** @type {{date:string, count:number}|null} lazily-loaded persistent counter. */
|
||||||
|
let budget = null;
|
||||||
|
let budgetLoaded = false;
|
||||||
|
|
||||||
|
function dailyBudgetLimit() {
|
||||||
|
const raw = Number.parseInt(process.env.TOMTOM_DAILY_TILE_BUDGET || '', 10);
|
||||||
|
return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_DAILY_BUDGET;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadBudgetOnce() {
|
||||||
|
if (budgetLoaded) return;
|
||||||
|
budgetLoaded = true;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(await fsp.readFile(BUDGET_PATH, 'utf8'));
|
||||||
|
if (parsed && typeof parsed.date === 'string' && Number.isFinite(parsed.count)) {
|
||||||
|
budget = parsed;
|
||||||
|
}
|
||||||
|
} catch { /* no budget file yet */ }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function persistBudget() {
|
||||||
|
try {
|
||||||
|
await fsp.mkdir(CACHE_DIR, { recursive: true });
|
||||||
|
await fsp.writeFile(BUDGET_PATH, JSON.stringify(budget), 'utf8');
|
||||||
|
} catch (err) {
|
||||||
|
console.warn('[tomtom-proxy] budget write failed:', err?.message || err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Roll the counter to today (UTC) and return it. */
|
||||||
|
function currentBudget() {
|
||||||
|
budget = normalizeTomTomBudget(budget, tomtomUtcDayKey());
|
||||||
|
return budget;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Count one upstream fetch attempt against today's budget (async persist). */
|
||||||
|
function recordUpstreamFetch() {
|
||||||
|
currentBudget().count += 1;
|
||||||
|
void persistBudget();
|
||||||
|
}
|
||||||
|
|
||||||
|
const tilePath = (key) => path.join(CACHE_DIR, `flow-${key.replaceAll('/', '-')}.pbf`);
|
||||||
|
|
||||||
|
/** Disk-cache read; tile age comes from the file's mtime. */
|
||||||
|
async function readDiskTile(key) {
|
||||||
|
try {
|
||||||
|
const [stat, buf] = await Promise.all([
|
||||||
|
fsp.stat(tilePath(key)),
|
||||||
|
fsp.readFile(tilePath(key)),
|
||||||
|
]);
|
||||||
|
return { at: stat.mtimeMs, buf };
|
||||||
|
} catch { return null; }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeDiskTile(key, buf) {
|
||||||
|
try {
|
||||||
|
await fsp.mkdir(CACHE_DIR, { recursive: true });
|
||||||
|
await fsp.writeFile(tilePath(key), buf);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[tomtom-proxy] tile cache write failed for ${key}:`, err?.message || err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** LRU-ish memory insert (Map preserves insertion order; evict the oldest). */
|
||||||
|
function memSet(key, entry) {
|
||||||
|
if (!mem.has(key) && mem.size >= MEM_MAX_ENTRIES) {
|
||||||
|
const oldest = mem.keys().next().value;
|
||||||
|
mem.delete(oldest);
|
||||||
|
}
|
||||||
|
mem.set(key, entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchUpstream(z, x, y) {
|
||||||
|
const url = 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/'
|
||||||
|
+ `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`;
|
||||||
|
recordUpstreamFetch(); // attempts count — upstream bills the request either way
|
||||||
|
const res = await fetch(url, { signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS) });
|
||||||
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||||
|
const buf = Buffer.from(await res.arrayBuffer());
|
||||||
|
if (buf.length === 0) throw new Error('empty tile body');
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: 'tomtom-proxy',
|
||||||
|
configureServer(server) {
|
||||||
|
server.middlewares.use('/api/tomtom', async (req, res) => {
|
||||||
|
// Sanitized responses only (proxy/security baseline): no upstream
|
||||||
|
// error details, and never echo the key or the upstream URL.
|
||||||
|
const sendJson = (status, obj, extraHeaders = {}) => {
|
||||||
|
if (res.headersSent) return;
|
||||||
|
res.writeHead(status, {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
'Cache-Control': 'no-store',
|
||||||
|
...extraHeaders,
|
||||||
|
});
|
||||||
|
res.end(JSON.stringify(obj));
|
||||||
|
};
|
||||||
|
const sendTile = (buf, cacheStatus) => {
|
||||||
|
if (res.headersSent) return;
|
||||||
|
res.writeHead(200, {
|
||||||
|
'Content-Type': 'application/x-protobuf',
|
||||||
|
'Cache-Control': 'no-store',
|
||||||
|
'x-tomtom-cache': cacheStatus,
|
||||||
|
});
|
||||||
|
res.end(buf);
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
await loadBudgetOnce();
|
||||||
|
const urlPath = String(req.url || '').split('?')[0];
|
||||||
|
|
||||||
|
if (urlPath === '/status') {
|
||||||
|
const hasKey = Boolean(process.env.TOMTOM_API_KEY);
|
||||||
|
const b = currentBudget();
|
||||||
|
sendJson(200, { hasKey, dailyCount: b.count, budget: dailyBudgetLimit(), date: b.date });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const m = urlPath.match(/^\/flow\/(\d+)\/(\d+)\/(\d+)\.pbf$/);
|
||||||
|
if (!m) {
|
||||||
|
sendJson(404, { error: 'not_found' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const z = Number(m[1]);
|
||||||
|
const x = Number(m[2]);
|
||||||
|
const y = Number(m[3]);
|
||||||
|
if (!isValidTomTomTile(z, x, y)) {
|
||||||
|
sendJson(400, { error: 'invalid_tile' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!process.env.TOMTOM_API_KEY) {
|
||||||
|
sendJson(503, { error: 'no_key' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const key = `${z}/${x}/${y}`;
|
||||||
|
const now = Date.now();
|
||||||
|
|
||||||
|
let entry = mem.get(key);
|
||||||
|
if (!entry) {
|
||||||
|
entry = await readDiskTile(key);
|
||||||
|
if (entry) memSet(key, entry);
|
||||||
|
}
|
||||||
|
// Fresh cache hit — never counts against the budget.
|
||||||
|
if (entry && now - entry.at < TILE_TTL_MS) {
|
||||||
|
sendTile(entry.buf, 'HIT');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Budget governor: over the soft cap, last-good data beats a dead layer.
|
||||||
|
if (isTomTomOverBudget(currentBudget(), dailyBudgetLimit())) {
|
||||||
|
if (entry) {
|
||||||
|
sendTile(entry.buf, 'STALE-BUDGET');
|
||||||
|
} else {
|
||||||
|
sendJson(429, { error: 'budget' });
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stale or missing → refresh, single-flight per tile.
|
||||||
|
if (!inflight.has(key)) {
|
||||||
|
inflight.set(key, fetchUpstream(z, x, y)
|
||||||
|
.then(async (buf) => {
|
||||||
|
const fresh = { at: Date.now(), buf };
|
||||||
|
memSet(key, fresh);
|
||||||
|
await writeDiskTile(key, buf);
|
||||||
|
return fresh;
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
console.warn(`[tomtom-proxy] ${key} fetch failed (${err?.message || err}) — serving stale if any`);
|
||||||
|
return null;
|
||||||
|
})
|
||||||
|
.finally(() => inflight.delete(key)));
|
||||||
|
}
|
||||||
|
const fresh = await inflight.get(key);
|
||||||
|
if (fresh) {
|
||||||
|
sendTile(fresh.buf, 'MISS');
|
||||||
|
} else if (entry) {
|
||||||
|
sendTile(entry.buf, 'STALE-ERROR'); // upstream down — stale beats empty
|
||||||
|
} else {
|
||||||
|
sendJson(502, { error: 'upstream' });
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.warn('[tomtom-proxy] error:', err?.message || err);
|
||||||
|
sendJson(500, { error: 'proxy' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
/** Allowlisted GBFS hostnames; wildcard *.publicbikesystem.net also accepted. */
|
||||||
|
const GBFS_ALLOWED_HOSTS = new Set([
|
||||||
|
'gbfs.lyft.com',
|
||||||
|
'gbfs.bluebikes.com',
|
||||||
|
'gbfs.bcycle.com',
|
||||||
|
'gbfs.biketownpdx.com',
|
||||||
|
'gbfs.cogobikeshare.com',
|
||||||
|
'austin.publicbikesystem.net',
|
||||||
|
'hon.publicbikesystem.net',
|
||||||
|
'chat.publicbikesystem.net',
|
||||||
|
]);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check whether a hostname is in the GBFS allowlist.
|
||||||
|
*
|
||||||
|
* Also accepts any subdomain of publicbikesystem.net.
|
||||||
|
*
|
||||||
|
* @param {string} hostname
|
||||||
|
* @returns {boolean}
|
||||||
|
*/
|
||||||
|
export function isAllowedGbfsHost(hostname) {
|
||||||
|
const host = String(hostname || '').trim().toLowerCase();
|
||||||
|
if (!host) return false;
|
||||||
|
if (GBFS_ALLOWED_HOSTS.has(host)) return true;
|
||||||
|
return host.endsWith('.publicbikesystem.net');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Only allow station_information.json and station_status.json endpoints.
|
||||||
|
*
|
||||||
|
* @param {string} pathname
|
||||||
|
* @returns {boolean}
|
||||||
|
*/
|
||||||
|
export function isAllowedGbfsPath(pathname) {
|
||||||
|
return /\/station_(information|status)\.json$/i.test(String(pathname || ''));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return an appropriate Cache-Control header for a GBFS endpoint.
|
||||||
|
*
|
||||||
|
* station_information is semi-static (5 min cache); station_status is
|
||||||
|
* real-time (no-store).
|
||||||
|
*
|
||||||
|
* @param {string} pathname
|
||||||
|
* @returns {string} Cache-Control header value.
|
||||||
|
*/
|
||||||
|
export function gbfsCacheControl(pathname) {
|
||||||
|
if (/\/station_information\.json$/i.test(String(pathname || ''))) {
|
||||||
|
return 'public, max-age=300';
|
||||||
|
}
|
||||||
|
return 'no-store';
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
|
|||||||
import { readFileSync } from 'node:fs';
|
import { readFileSync } from 'node:fs';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
|
|
||||||
const source = ['local.js', 'common/http.js', 'aircraft/enrichment.js', 'space/celestrak.js', 'space/launch-library.js', '../../src/data/spaceProviderRequests.js']
|
const source = ['local.js', 'common/http.js', 'aircraft/enrichment.js', 'terrain.js', 'space/celestrak.js', 'space/launch-library.js', '../../src/data/spaceProviderRequests.js']
|
||||||
.map(file => readFileSync(new URL(`../server/providers/${file}`, import.meta.url), 'utf8'))
|
.map(file => readFileSync(new URL(`../server/providers/${file}`, import.meta.url), 'utf8'))
|
||||||
.join('\n');
|
.join('\n');
|
||||||
const detail = 'fixture-secret-token /internal/example <html>';
|
const detail = 'fixture-secret-token /internal/example <html>';
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { promises as fsp } from 'node:fs';
|
||||||
|
import { terrainHeightsProxy } from 'gods-eye-view/server/providers/terrain';
|
||||||
|
import { tomtomProxy } from 'gods-eye-view/server/providers/traffic';
|
||||||
|
import { firmsProxy } from 'gods-eye-view/server/providers/firms';
|
||||||
|
import { gbfsProxy } from 'gods-eye-view/server/providers/gbfs';
|
||||||
|
import { localProviderPlugins } from '../../server/providers/local.js';
|
||||||
|
|
||||||
|
function install(plugin) {
|
||||||
|
const routes = new Map();
|
||||||
|
plugin.configureServer({ middlewares: { use(route, handler) { routes.set(route, handler); } } });
|
||||||
|
assert.equal(routes.size, 1);
|
||||||
|
return async (url = '/', method = 'GET') => {
|
||||||
|
const res = { headersSent: false, writeHead(status, headers) { Object.assign(this, { status, headers, headersSent: true }); }, end(body) { this.body = body; } };
|
||||||
|
await [...routes.values()][0]({ url, method }, res);
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function isolate(t, env = {}) {
|
||||||
|
for(const [name,value] of Object.entries(env)) {
|
||||||
|
const previous = process.env[name];
|
||||||
|
t.after(() => { if(previous === undefined) delete process.env[name]; else process.env[name] = previous; });
|
||||||
|
process.env[name] = value;
|
||||||
|
}
|
||||||
|
t.mock.method(fsp, 'readFile', async () => { throw Error('no disk cache'); });
|
||||||
|
t.mock.method(fsp, 'stat', async () => { throw Error('no disk cache'); });
|
||||||
|
t.mock.method(fsp, 'mkdir', async () => {});
|
||||||
|
t.mock.method(fsp, 'writeFile', async () => {});
|
||||||
|
t.mock.method(globalThis, 'setInterval', () => ({ unref() {} }));
|
||||||
|
t.mock.method(console, 'warn', () => {});
|
||||||
|
}
|
||||||
|
const json = res => JSON.parse(res.body);
|
||||||
|
|
||||||
|
test('standalone composition mounts every extracted provider exactly once without acquisition', (t) => {
|
||||||
|
t.mock.method(globalThis, 'fetch', () => { throw Error('construction must not fetch'); });
|
||||||
|
const plugins = localProviderPlugins();
|
||||||
|
for(const factory of [terrainHeightsProxy,tomtomProxy,firmsProxy,gbfsProxy]) assert.equal(plugins.filter(p=>p.name===factory().name).length,1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('terrain middleware chunks missing points and reconstructs repeated/reordered requests from cache', async (t) => {
|
||||||
|
isolate(t);let calls=0;const sizes=[];
|
||||||
|
t.mock.method(globalThis, 'fetch', async (raw) => {
|
||||||
|
calls++;const url=new URL(raw);assert.equal(url.origin,'https://terrain.reearth.land');
|
||||||
|
const points=url.searchParams.get('points').split(';').map(p=>p.split(',').map(Number));sizes.push(points.length);
|
||||||
|
return Response.json({results:points.map(([lon])=>({ellipsoid:lon+100}))});
|
||||||
|
});
|
||||||
|
const request=install(terrainHeightsProxy());
|
||||||
|
const points=Array.from({length:257},(_,i)=>`${i/100},1`);
|
||||||
|
const res=await request('/?points='+points.join(';'));
|
||||||
|
assert.equal(res.status,200);assert.deepEqual(sizes,[256,1]);
|
||||||
|
const reordered=await request('/?points=2.56,1;0,1;2.56,1');
|
||||||
|
assert.deepEqual(json(reordered),{results:[{ellipsoid:102.56},{ellipsoid:100},{ellipsoid:102.56}]});assert.equal(calls,2);
|
||||||
|
assert.equal((await request('/?points=invalid')).status,400);
|
||||||
|
assert.equal((await request('/?points='+Array(2001).fill('0,1').join(';'))).status,500);
|
||||||
|
assert.equal(calls,2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('terrain middleware migrates valid legacy disk points without fabricating omitted heights', async (t) => {
|
||||||
|
isolate(t);
|
||||||
|
t.mock.method(fsp,'readFile',async()=>JSON.stringify({'1,2;3,4':{at:Date.now(),results:[{ellipsoid:77}]}}));
|
||||||
|
let calls=0;
|
||||||
|
t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).searchParams.get('points'),'3.00000,4.00000');return Response.json({results:[{ellipsoid:88}]});});
|
||||||
|
const res=await install(terrainHeightsProxy())('/?points=3,4;1,2');
|
||||||
|
assert.deepEqual(json(res),{results:[{ellipsoid:88},{ellipsoid:77}]});assert.equal(calls,1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('traffic middleware preserves keyless mode, caching, stale budget fallback and UTC rollover', async (t) => {
|
||||||
|
isolate(t,{TOMTOM_API_KEY:'',TOMTOM_DAILY_TILE_BUDGET:'1'});
|
||||||
|
let now=Date.UTC(2026,8,12,12);t.mock.method(Date,'now',()=>now);
|
||||||
|
let calls=0;t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).searchParams.get('key'),'fixture-key');return new Response(new Uint8Array([1,2,3]));});
|
||||||
|
const request=install(tomtomProxy());
|
||||||
|
assert.equal(json(await request('/status')).hasKey,false);
|
||||||
|
assert.equal((await request('/flow/8/1/1.pbf')).status,503);
|
||||||
|
assert.equal((await request('/flow/7/1/1.pbf')).status,400);assert.equal(calls,0);
|
||||||
|
process.env.TOMTOM_API_KEY='fixture-key';
|
||||||
|
assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'MISS');
|
||||||
|
assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'HIT');
|
||||||
|
assert.equal(json(await request('/status')).dailyCount,1);assert.equal(calls,1);
|
||||||
|
now+=120001;
|
||||||
|
assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'STALE-BUDGET');
|
||||||
|
assert.equal((await request('/flow/8/2/1.pbf')).status,429);assert.equal(calls,1);
|
||||||
|
now+=86400000;
|
||||||
|
assert.equal(json(await request('/status')).dailyCount,0);
|
||||||
|
assert.equal((await request('/flow/8/1/1.pbf')).headers['x-tomtom-cache'],'MISS');assert.equal(calls,2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('FIRMS retains a large successful source during partial failure and filters stale data at serve time', async (t) => {
|
||||||
|
isolate(t,{FIRMS_MAP_KEY:''});let now=Date.UTC(2026,8,12,12);t.mock.method(Date,'now',()=>now);
|
||||||
|
let calls=0;
|
||||||
|
const header='latitude,longitude,acq_date,acq_time,confidence,frp\n';
|
||||||
|
const csv=header+'30,-97,2026-09-12,1200,h,10\n'.repeat(130001);
|
||||||
|
t.mock.method(globalThis,'fetch',async(raw)=>{
|
||||||
|
calls++;const url=new URL(raw);assert.equal(url.hostname,'firms.modaps.eosdis.nasa.gov');
|
||||||
|
if(url.pathname.includes('mapkey_status')) return Response.json({current_transactions:3,transaction_limit:5000});
|
||||||
|
return url.pathname.includes('VIIRS_NOAA20')?new Response(csv):new Response('offline',{status:503});
|
||||||
|
});
|
||||||
|
const request=install(firmsProxy());
|
||||||
|
assert.equal((await request()).status,503);assert.equal(json(await request('/status')).hasKey,false);assert.equal(calls,0);
|
||||||
|
process.env.FIRMS_MAP_KEY='fixture-key';
|
||||||
|
const first=json(await request());assert.equal(first.count,130001);assert.equal(first.sources.filter(s=>s.ok).length,1);assert.equal(calls,3);
|
||||||
|
assert.equal(json(await request()).count,130001);assert.equal(calls,3);
|
||||||
|
assert.deepEqual(json(await request('/status')).transactions,{used:3,limit:5000});
|
||||||
|
now+=25*3600000;t.mock.method(globalThis,'fetch',async()=>new Response('offline',{status:503}));
|
||||||
|
const stale=json(await request());assert.equal(stale.stale,true);assert.equal(stale.count,0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('GBFS keeps host/path/method guards, response caps and distinct information/status cache headers', async (t) => {
|
||||||
|
let calls=0;t.mock.method(globalThis,'fetch',async(raw)=>{calls++;assert.equal(new URL(raw).hostname,'gbfs.lyft.com');return Response.json({data:{stations:[]}});});
|
||||||
|
const request=install(gbfsProxy());const target=p=>'/'+encodeURIComponent('https://gbfs.lyft.com/'+p);
|
||||||
|
assert.equal((await request(target('station_status.json'),'POST')).status,405);
|
||||||
|
assert.equal((await request('/'+encodeURIComponent('https://example.com/station_status.json'))).status,403);
|
||||||
|
assert.equal((await request(target('gbfs.json'))).status,400);
|
||||||
|
assert.equal((await request('/%zz')).status,400);assert.equal(calls,0);
|
||||||
|
assert.equal((await request(target('station_information.json'))).headers['Cache-Control'],'public, max-age=300');
|
||||||
|
assert.equal((await request(target('station_status.json'))).headers['Cache-Control'],'no-store');
|
||||||
|
t.mock.method(globalThis,'fetch',async()=>new Response('x',{headers:{'content-length':String(6*1024*1024)}}));
|
||||||
|
assert.equal((await request(target('station_status.json'))).status,502);
|
||||||
|
});
|
||||||
Reference in new issue
Block a user