Merge branch 'main' into feat/cctv-toronto

This commit is contained in:
Sameh Khamis committed 2026-10-15 14:23:19 -07:00
commit 7561a5ab82
314 files changed
+36277 -1808

No files matched your search

+18 -2
View File
@@ -1,3 +1,5 @@
import { readResponseTextCapped } from '../common/http.js';
/**
* Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
*
@@ -25,6 +27,10 @@ export function adsbLolProxy() {
let _cooldownStatus = 0;
/** Response cache TTL (ms). */
const CACHE_MS = 12000;
/** Upstream deadline (ms); the fetch previously had none and could hang a poll. */
const UPSTREAM_TIMEOUT_MS = 12000;
/** /v2/mil measures ~67 KB; a relayed error page is bounded by the same cap. */
const ADSBLOL_MIL_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
/** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */
const RATE_LIMIT_COOLDOWN_MS = 30000;
/** Cooldown after a 5xx (ms). */
@@ -98,11 +104,17 @@ export function adsbLolProxy() {
);
return;
}
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
signal,
});
if (upstream.ok) {
const body = await upstream.text();
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
_cache = body;
_cacheAt = Date.now();
_cooldownUntil = 0;
@@ -129,7 +141,11 @@ export function adsbLolProxy() {
return;
}
}
const body = await upstream.text();
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
serve(
res,
upstream.status,
+12 -4
View File
@@ -1,5 +1,9 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { readResponseJsonCapped } from '../common/http.js';
/** Route and aircraft lookups are a few hundred bytes; 1 MiB is a hard bound. */
const ADSBDB_MAX_RESPONSE_BYTES = 1024 * 1024;
/**
* adsbdb.com enrichment proxy: callsign → route (airline + origin/destination
* airports) and hex → aircraft type/registration. Free community API — cached
@@ -80,12 +84,16 @@ export function adsbdbProxy() {
kind === 'route'
? `https://api.adsbdb.com/v0/callsign/${encodeURIComponent(key)}`
: `https://api.adsbdb.com/v0/aircraft/${encodeURIComponent(key)}`;
const res = await fetch(url, { signal: AbortSignal.timeout(8000) });
const signal = AbortSignal.timeout(8000);
const res = await fetch(url, { signal });
if (res.ok) {
const payload = await readResponseJsonCapped(
res,
ADSBDB_MAX_RESPONSE_BYTES,
signal,
);
const data =
kind === 'route'
? parseRoute(await res.json())
: parseAircraft(await res.json());
kind === 'route' ? parseRoute(payload) : parseAircraft(payload);
store[key] = { at: Date.now(), data }; // data may be null — negative cache
dirty = true;
return data;
+48 -20
View File
@@ -2,6 +2,7 @@ import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback
import {
coalesceProxyRequest,
readResponseJsonCapped,
readResponseTextCapped,
} from '../common/http.js';
import { requiredFiniteQueryNumber } from '../common/query.js';
// ---------------------------------------------------------------------------
@@ -29,11 +30,34 @@ const OPENSKY_CACHE_MS = 9000;
const OPENSKY_ATTEMPT_TIMEOUT_MS = 10_000;
const OPENSKY_STATES_URL =
'https://opensky-network.org/api/states/all?extended=1';
/** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */
const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
/**
* Credit-governor cooldown for a 429: OpenSky's retry-after bounded to
* 30 s … 30 min, or 2 min when the header is absent.
* @param {Response} upstream Rate-limited response.
* @returns {number} Cooldown (ms).
*/
function openSkyRateLimitCooldownMs(upstream) {
// headers.get() gives null when absent, and Number(null) is 0, not NaN.
const header = upstream.headers.get('x-rate-limit-retry-after-seconds');
const retryAfterSec =
header == null || header.trim() === '' ? NaN : Number(header);
return Math.min(
Math.max(
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
30_000,
),
30 * 60_000,
);
}
/**
* Fetch and read the global snapshot, giving each attempt
* OPENSKY_ATTEMPT_TIMEOUT_MS and retrying once when an attempt times out or
* the connection fails. A second failure throws, which the proxy answers
* the connection fails. An oversized body is not retried: it would fail again
* and spend more credits. A second failure throws, which the proxy answers
* from its stale cache or the regional fallback.
* @param {Record<string, string>} headers Request headers.
* @returns {Promise<{upstream: Response, body: string}>}
@@ -41,13 +65,27 @@ const OPENSKY_STATES_URL =
async function fetchOpenSkyStates(headers) {
for (let attempt = 1; ; attempt++) {
try {
const upstream = await fetch(OPENSKY_STATES_URL, {
headers,
signal: AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS),
});
return { upstream, body: await upstream.text() };
const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
if (upstream.status === 429) {
// Start the cooldown from the headers and drop the rejected body
// unread, so a stalled or oversized 429 body cannot skip it.
_openskyCooldownUntil =
Date.now() + openSkyRateLimitCooldownMs(upstream);
void upstream.body?.cancel().catch(() => {});
return {
upstream,
body: JSON.stringify({ error: 'OpenSky rate limited.' }),
};
}
const body = await readResponseTextCapped(
upstream,
OPENSKY_STATES_MAX_RESPONSE_BYTES,
signal,
);
return { upstream, body };
} catch (error) {
if (attempt >= 2) throw error;
if (attempt >= 2 || error?.code === 'RESPONSE_TOO_LARGE') throw error;
console.warn(
'[OpenSky] global snapshot attempt failed, retrying:',
error?.message || error,
@@ -544,19 +582,9 @@ export function openSkyProxy() {
}
if (upstream.status === 429) {
reason = 'rate_limited';
// Credit governor: honor OpenSky's retry-after (bounded 30 s … 30 min;
// 2 min when the header is absent) — no upstream attempts until then.
const retryAfterSec = Number(
upstream.headers.get('x-rate-limit-retry-after-seconds'),
);
const cooldownMs = Math.min(
Math.max(
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
30_000,
),
30 * 60_000,
);
_openskyCooldownUntil = now + cooldownMs;
// Credit governor: fetchOpenSkyStates already started the cooldown
// from OpenSky's retry-after — no upstream attempts until then.
const cooldownMs = openSkyRateLimitCooldownMs(upstream);
// Serve the last-good body instead of the 429 when we have one —
// the layer keeps rendering (STALE-cued) instead of dying.
if (_openskyCacheBody && _openskyCacheStatus === 200) {
+12 -1
View File
@@ -19,18 +19,22 @@ import {
loadCalgarySourcesFromOpenData,
loadDelDOTSourcesFromOpenData,
loadVegvesenSourcesFromOpenData,
loadNycSourcesFromOpenData,
loadTorontoSourcesFromOpenData,
} from './sources.js';
/** Env kill switch: unset or anything but "0" means enabled. */
const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0';
/** Env opt-in: only "1" enables. For packs whose feed access needs an
* operator agreement, so an unset flag never reaches the upstream. */
const envOptIn = (name) => String(process.env[name] || '').trim() === '1';
/**
* Live open-data packs, in merge order. Adding a region is one entry here
* plus its loader in sources.js; the catalog cap is shared across entries
* round-robin (cap.js), so a new pack never silently evicts an older one.
* Each pack fails independently (allSettled) and is gated by its own env
* kill switch.
* kill switch, or by an opt-in flag where access needs an agreement.
*/
const LIVE_PACKS = [
{ name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData },
@@ -99,6 +103,13 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'),
load: loadVegvesenSourcesFromOpenData,
},
{
// NYC DOT requires a data-sharing agreement for this feed, so the pack is
// off unless the operator opts in (see DATA_SOURCES.md).
name: 'nyc',
enabled: () => envOptIn('CCTV_NYC_ENABLED'),
load: loadNycSourcesFromOpenData,
},
{
name: 'toronto',
enabled: () => envEnabled('CCTV_TORONTO_ENABLED'),
+30
View File
@@ -267,6 +267,36 @@ export const NORWAY_ANCHORS = [
{ lat: 69.6492, lon: 18.9553 }, // Tromsø
];
/** NYC DOT Traffic Management Center cameras: one keyless JSON list for the
* five boroughs (~980 cameras). Frames are stills synthesized from each
* camera UUID on the city's webcam host — never taken from the payload. */
export const NYC_CAMERAS_URL = 'https://webcams.nyctmc.org/api/cameras';
/** The only origin NYC camera frames may come from. The catalog's `imageUrl`
* is ignored; the registered path is always this origin plus a validated
* UUID plus `/image`. */
export const NYC_IMAGE_ORIGIN = 'https://webcams.nyctmc.org/api/cameras/';
export const NYC_IMAGE_URL = (id) => `${NYC_IMAGE_ORIGIN}${id}/image`;
export const NYC_CAMERA_ID_RE =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
export const DEFAULT_NYC_MAX_SOURCES = 250;
/** Hard ceiling on the NYC catalog body. The live list is ~980 rows / ~250 KB;
* this only exists so an upstream that streams an unbounded body cannot be
* buffered without limit. */
export const NYC_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Ground-elevation prior in metres. Matches the New York CITY_POIS prior;
* the client's one-shot ground snap corrects it wherever 3D tiles load. */
export const NYC_GROUND_ELEVATION_M = 10;
/** Prioritization anchors: one per borough so a cap keeps citywide coverage
* rather than Midtown alone. */
export const NYC_ANCHORS = [
{ lat: 40.758, lon: -73.9855 }, // Manhattan (Times Square)
{ lat: 40.7128, lon: -74.006 }, // Lower Manhattan
{ lat: 40.6782, lon: -73.9442 }, // Brooklyn
{ lat: 40.7282, lon: -73.7949 }, // Queens
{ lat: 40.8448, lon: -73.8648 }, // Bronx
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
];
/** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */
export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json';
export const DEFAULT_DELDOT_MAX_SOURCES = 300;
+11
View File
@@ -367,6 +367,17 @@ export function isLikelyCalgaryCoordinate(lat, lon) {
);
}
/** New York City's five boroughs, with slack for the harbor and city line. */
export function isLikelyNycCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 40.45 &&
lat <= 40.95 &&
lon >= -74.3 &&
lon <= -73.65
);
}
/** Mainland Norway, with slack for the coast and the Finnmark border. */
export function isLikelyNorwayCoordinate(lat, lon) {
return (
+153
View File
@@ -71,6 +71,13 @@ import {
DEFAULT_VEGVESEN_MAX_SOURCES,
VEGVESEN_MAX_CATALOG_BYTES,
NORWAY_ANCHORS,
NYC_CAMERAS_URL,
NYC_IMAGE_URL,
NYC_CAMERA_ID_RE,
DEFAULT_NYC_MAX_SOURCES,
NYC_MAX_CATALOG_BYTES,
NYC_GROUND_ELEVATION_M,
NYC_ANCHORS,
} from './constants.js';
import {
toFiniteNumber,
@@ -88,6 +95,7 @@ import {
isLikelyTexasCoordinate,
isLikelyNswCoordinate,
isLikelyCalgaryCoordinate,
isLikelyNycCoordinate,
isLikelyNorwayCoordinate,
isLikelyTorontoCoordinate,
cameraDisplayCode,
@@ -1874,6 +1882,151 @@ export async function loadVegvesenSourcesFromOpenData() {
}
}
const NYC_BOROUGHS = new Set([
'Manhattan',
'Brooklyn',
'Queens',
'Bronx',
'Staten Island',
]);
/** NYC TMC publishes `isOnline` as the strings "true"/"false". */
export function nycCameraIsOnline(value) {
return (
value === true ||
String(value ?? '')
.trim()
.toLowerCase() === 'true'
);
}
/**
* Label for one NYC camera: the intersection an operator recognises
* ("Central Park West @ 86 St"). The catalog's `area` is the borough, not
* a facing, and is not part of the name.
*
* @param {object} record - Raw TMC camera row.
* @param {string} cameraId - Derived stable id.
* @returns {string}
*/
export function nycCameraName(record, cameraId) {
const name = String(record?.name ?? '').trim();
if (name) return name;
const short = String(cameraId || '')
.replace(/^nyc-/, '')
.slice(0, 8);
return short ? `NYC Camera ${short}` : 'NYC Camera';
}
/**
* One NYC DOT TMC camera row -> one catalog source, or null.
*
* Frame URLs are synthesized from a validated UUID on the pinned TMC image
* origin. The payload's `imageUrl` is ignored so a catalog edit cannot steer
* the frame proxy off-host. There is no heading in the feed; headings use
* the shared id-hash fallback at low confidence, like Calgary and Fintraffic.
*
* @param {object} record - Raw TMC camera row.
* @returns {?object}
*/
export function nycCameraToSource(record) {
if (!record || typeof record !== 'object') return null;
if (!nycCameraIsOnline(record.isOnline)) return null;
const lat = toFiniteNumber(record.latitude);
const lon = toFiniteNumber(record.longitude);
if (!isLikelyNycCoordinate(lat, lon)) return null;
const uuid = String(record.id ?? '').trim();
if (!NYC_CAMERA_ID_RE.test(uuid)) return null;
const cameraId = `nyc-${uuid.toLowerCase()}`;
const imageUrl = NYC_IMAGE_URL(uuid);
const name = nycCameraName(record, cameraId);
const area = String(record.area ?? '').trim();
const city = NYC_BOROUGHS.has(area) ? area : 'New York';
return {
id: cameraId,
name,
city,
cityId: 'nyc',
provider: 'NYC DOT Traffic Management Center',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
groundElevationM: NYC_GROUND_ELEVATION_M,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'nyc-dot-tmc',
license: 'Public NYC DOT Traffic Management Center traffic camera',
code: cameraDisplayCode(name.toUpperCase()),
};
}
/**
* Fetch NYC DOT Traffic Management Center cameras from the keyless
* `webcams.nyctmc.org` catalog. Frames are stills on that same host,
* registered from a validated UUID rather than from the payload URL.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadNycSourcesFromOpenData() {
try {
const endpoint = process.env.CCTV_NYC_CAMERAS_URL || NYC_CAMERAS_URL;
const resp = await fetch(endpoint, {
headers: { Accept: 'application/json' },
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn('[CCTV] NYC catalog redirected; redirects are not followed');
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] NYC camera download failed:', resp.status);
return discard();
}
const rows = await readResponseJsonCapped(resp, NYC_MAX_CATALOG_BYTES);
if (!Array.isArray(rows)) return [];
const cameras = [];
const seen = new Set();
for (const record of rows) {
const camera = nycCameraToSource(record);
if (!camera || seen.has(camera.id)) continue;
seen.add(camera.id);
cameras.push(camera);
}
const maxRaw = Number(
process.env.CCTV_NYC_MAX_SOURCES || DEFAULT_NYC_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(400, Math.floor(maxRaw)))
: DEFAULT_NYC_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, NYC_ANCHORS);
console.log(
`[CCTV] Loaded NYC camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn('[CCTV] NYC camera download error:', error?.message || error);
return [];
}
}
/**
* The JSONP envelope the City of Toronto wraps its camera list in. Anchored at
* both ends, so only that exact callback is unwrapped — a payload that merely
+12 -2
View File
@@ -2,6 +2,7 @@ import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
import { readResponseTextCapped } from './common/http.js';
/**
* NASA FIRMS live active-fire proxy with a memory + disk cache.
@@ -35,6 +36,12 @@ export function firmsProxy() {
'VIIRS_SNPP_NRT',
'MODIS_NRT',
];
/**
* Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
* 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
* bound, not a budget.
*/
const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024;
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
const CACHE_PATH = path.join(CACHE_DIR, 'firms.json');
@@ -83,9 +90,12 @@ export function firmsProxy() {
*/
async function fetchSource(key, source) {
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
const signal = AbortSignal.timeout(60_000);
const res = await fetch(url, { signal });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const records = parseFirmsCsv(await res.text());
const records = parseFirmsCsv(
await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
);
if (records === null) throw new Error('non-CSV upstream response');
return records;
}
+9 -1
View File
@@ -3,6 +3,7 @@ import {
isAllowedGbfsHost,
isAllowedGbfsPath,
gbfsCacheControl,
gbfsUpstreamHeaders,
} from '../../src/data/gbfsSource.js';
// ---------------------------------------------------------------------------
@@ -11,6 +12,9 @@ import {
/** Upstream fetch timeout for GBFS requests (ms). */
const GBFS_PROXY_TIMEOUT_MS = 12000;
/** Value of the Client-Identifier header sent with every upstream request. */
export const GBFS_CLIENT_IDENTIFIER = 'gods-eye-view';
export const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB
function gbfsRedirectHost(location, requestUrl) {
@@ -60,6 +64,10 @@ export async function fetchGbfsUpstream(
headers: {
Accept: 'application/json',
'User-Agent': 'gods-eye-view-gbfs-proxy/1.0',
// Urban Sharing (the Norwegian city-bike feeds) asks every client to
// name itself in this header; other operators ignore it.
'Client-Identifier': GBFS_CLIENT_IDENTIFIER,
...gbfsUpstreamHeaders(new URL(url).hostname),
},
redirect: 'manual',
signal: controller.signal,
@@ -181,7 +189,7 @@ export function gbfsProxy() {
return;
}
if (!isAllowedGbfsPath(upstreamUrl.pathname)) {
if (!isAllowedGbfsPath(upstreamUrl.pathname, upstreamUrl.hostname)) {
res.writeHead(400, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
+238
View File
@@ -0,0 +1,238 @@
import {
GNSS_CLASSIFIER,
GNSS_WINDOW_MS,
normalizeGnssAircraft,
} from '../../src/layers/gnss/records.js';
import { adsbLolFallbackAnchor } from './aircraft/opensky.js';
import { coalesceProxyRequest, readResponseJsonCapped } from './common/http.js';
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
// adsb.lol regional snapshot (keyless, ODbL). Only the integrity fields the
// GNSS layer needs leave this proxy, so the browser never sees the full feed.
// This is a separate read from the flights fallback on purpose: that point
// cache keeps only the fields the flights layer renders (no nic / nac_p), is
// keyed on 0.25° anchors and lives for 12 s, while integrity cells need
// whole-degree anchors held for a minute.
const RADIUS_NM = 250;
const CACHE_MS = 60_000;
const CACHE_MAX = 64;
/** Default 429 cooldown when upstream sends no usable Retry-After. */
const COOLDOWN_MS = 60_000;
/** Bound sane upstream deadlines to one hour; never shorten a deadline within that range. */
const COOLDOWN_MIN_MS = 5_000;
const COOLDOWN_MAX_MS = 60 * 60_000;
const FAILURE_COOLDOWN_MS = 15_000;
const MAX_IN_FLIGHT = 8;
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
const TIMEOUT_MS = 10_000;
/** Snap a view anchor to a whole degree so nearby views share one upstream read. */
export function gnssIntegrityAnchor(url) {
const anchor = adsbLolFallbackAnchor({ url });
if (!anchor) return null;
return {
lat: Math.round(anchor.latitude),
lon: Math.round(anchor.longitude),
};
}
/** Cooldown (ms) an upstream 429 earns, honouring Retry-After when sane. */
export function gnssRetryCooldownMs(raw, nowMs) {
if (raw) {
const seconds = /^\d+(?:\.\d+)?$/.test(raw.trim()) ? Number(raw) : NaN;
const ms = seconds * 1000;
if (Number.isFinite(seconds) && seconds > 0)
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, ms));
const at =
Number.isNaN(seconds) && /[a-z]/i.test(raw) ? Date.parse(raw) : NaN;
if (Number.isFinite(at) && at > nowMs)
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, at - nowMs));
}
return COOLDOWN_MS;
}
/** Fixed-origin, bounded adsb.lol integrity route for dev and preview. */
export function gnssIntegrityProxy({
fetchImpl = (...args) => globalThis.fetch(...args),
now = () => Date.now(),
monotonicNow = () => performance.now(),
warn = (...args) => console.warn(...args),
} = {}) {
const cache = new Map();
const inFlight = new Map();
const allow = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 600 });
let cooldownUntil = 0;
let cooldownStatus = 429;
async function load({ lat, lon }) {
if (monotonicNow() < cooldownUntil)
throw Object.assign(new Error('cooling_down'), {
status: cooldownStatus,
});
const signal = AbortSignal.timeout(TIMEOUT_MS);
try {
const response = await fetchImpl(
`https://api.adsb.lol/v2/lat/${lat}/lon/${lon}/dist/${RADIUS_NM}`,
{
signal,
redirect: 'error',
headers: {
Accept: 'application/json',
'User-Agent': 'gods-eye-view-gnss-integrity/1.0',
},
},
);
if (!response.ok) {
await response.body?.cancel();
if (response.status === 429) {
const at = monotonicNow();
const raw = response.headers.get('retry-after');
const delay = gnssRetryCooldownMs(raw, now());
if (delay === COOLDOWN_MAX_MS)
warn('[gnss-integrity] upstream cooldown capped at one hour');
cooldownStatus = 429;
cooldownUntil = Math.max(cooldownUntil, at + delay);
throw Object.assign(new Error('upstream_rate_limited'), {
status: 429,
});
}
throw new Error('upstream_unavailable');
}
const payload = await readResponseJsonCapped(
response,
MAX_RESPONSE_BYTES,
signal,
);
const rows = normalizeGnssAircraft(payload);
if (!rows) throw new Error('invalid_snapshot');
const sourceNow = payload.now;
if (!Number.isFinite(sourceNow) || sourceNow <= 0)
throw new Error('undated_snapshot');
const fetchedAt = Math.min(
now(),
sourceNow > 10_000_000_000 ? sourceNow : sourceNow * 1000,
);
if (now() - fetchedAt > GNSS_WINDOW_MS)
throw new Error('expired_snapshot');
return {
fetchedAt,
anchor: { lat, lon },
radiusNm: RADIUS_NM,
// `degraded` on each row is this classifier's verdict, not gpsjam's.
classifier: GNSS_CLASSIFIER.id,
rows,
};
} catch (error) {
if (error.status !== 429) {
const until = monotonicNow() + FAILURE_COOLDOWN_MS;
if (until > cooldownUntil) {
cooldownUntil = until;
cooldownStatus = 502;
}
}
throw error;
}
}
function age(entry) {
if (!entry) return Infinity;
const elapsed = monotonicNow() - entry.savedAt;
// A backwards injected clock cannot rejuvenate a snapshot.
return elapsed < 0 ? Infinity : entry.ageAtReceipt + elapsed;
}
async function acquire(anchor, peer) {
const key = `${anchor.lat},${anchor.lon}`;
const previous = cache.get(key);
if (age(previous) < CACHE_MS)
return { value: previous.value, ageMs: age(previous), stale: false };
try {
if (!inFlight.has(key) && inFlight.size >= MAX_IN_FLIGHT)
throw Object.assign(new Error('busy'), { status: 429, local: true });
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
if (monotonicNow() >= cooldownUntil && !allow(peer))
throw Object.assign(new Error('rate_limited'), {
status: 429,
local: true,
});
const value = await load(anchor);
const entry = {
value,
savedAt: monotonicNow(),
ageAtReceipt: Math.max(0, now() - value.fetchedAt),
};
cache.delete(key);
cache.set(key, entry);
while (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value);
return entry;
});
const entry = await promise;
const ageMs = age(entry);
return { value: entry.value, ageMs, stale: ageMs >= CACHE_MS };
} catch (error) {
if (age(previous) <= GNSS_WINDOW_MS)
return { value: previous.value, ageMs: age(previous), stale: true };
throw error;
}
}
async function handler(req, res) {
const json = (status, value, stale = false, retryAfterS = 60) => {
if (res.destroyed) return;
res.writeHead(status, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
...(status === 405 ? { Allow: 'GET' } : {}),
...([429, 502].includes(status)
? { 'Retry-After': String(retryAfterS) }
: {}),
...(stale ? { 'X-Data-Stale': 'true' } : {}),
});
res.end(JSON.stringify(value));
};
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
const path = (req.url || '/').split('?')[0];
if (path !== '/' && path !== '')
return json(404, { error: 'unknown_route' });
const anchor = gnssIntegrityAnchor(req.url);
if (!anchor) return json(400, { error: 'invalid_anchor' });
try {
const { value, stale, ageMs } = await acquire(anchor, clientKey(req));
// ageMs is measured on this server's clock, so the browser can date
// the rows without comparing its clock to ours.
json(200, { ...value, ageMs, ...(stale ? { stale: true } : {}) }, stale);
} catch (error) {
if (error.status === 429) {
// Tell the browser how long the shared upstream cooldown has left.
const retryAfterS = Math.max(
1,
error.local ? 60 : Math.ceil((cooldownUntil - monotonicNow()) / 1000),
);
return json(
429,
{
error: error.local ? error.message : 'gnss_integrity_rate_limited',
},
false,
retryAfterS,
);
}
json(
502,
{ error: 'gnss_integrity_unavailable' },
false,
Math.max(1, Math.ceil((cooldownUntil - monotonicNow()) / 1000)),
);
}
}
return {
name: 'gnss-integrity',
configureServer({ middlewares }) {
middlewares.use('/api/gnss-integrity', handler);
},
configurePreviewServer({ middlewares }) {
middlewares.use('/api/gnss-integrity', handler);
},
};
}
+5
View File
@@ -24,6 +24,7 @@ import { mapillaryProxy } from './mapillary.js';
import { keySetupEndpoint } from '../standalone/key-setup.js';
import { weatherProxy } from './weather.js';
import { firePerimetersProxy } from './firePerimeters.js';
import { gnssIntegrityProxy } from './gnssIntegrity.js';
import { cycloneProxy } from './cyclones.js';
import { windProxy } from './wind.js';
@@ -60,6 +61,7 @@ function localProviderPlugins({ realtime } = {}) {
cycloneProxy(),
mapillaryProxy(),
firePerimetersProxy(),
gnssIntegrityProxy(),
keySetupEndpoint(),
];
}
@@ -81,6 +83,9 @@ export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js';
export { googlePlacesContextProxy } from './places.js';
export { googleServerApiKey } from './places.js';
export { keylessGooglePlacesResponse } from './places.js';
export { keylessGoogleGeocodeResponse } from './places.js';
export { validateGeocodeAddress } from './places.js';
export { geocodeBounds } from './places.js';
export { adsbLolFallbackAnchor } from './aircraft/opensky.js';
export { readResponseTextCapped } from './common/http.js';
export { readResponseJsonCapped } from './common/http.js';
+11 -2
View File
@@ -9,7 +9,10 @@ import {
fetchOverpassPayload,
overpassNotConfigured,
} from './overpass/transport.js';
import { resolveOverpassUpstreams } from './overpass/constants.js';
import {
resolveOverpassUpstreams,
resolveOverpassUpstreamTag,
} from './overpass/constants.js';
import {
_militaryInstallationCache,
trimMilitaryInstallationCache,
@@ -74,7 +77,13 @@ function militaryInstallationsProxy() {
retrievedAt: new Date().toISOString(),
status: 'ready',
};
const entry = { payload, cachedAt: Date.now() };
// Tagged with the configured upstream set so the disk layer does not serve
// this answer back after the operator repoints (see overpass/cache.js).
const entry = {
payload,
cachedAt: Date.now(),
upstreams: resolveOverpassUpstreamTag(),
};
_militaryInstallationCache.set(key, entry);
trimMilitaryInstallationCache();
writeMilitaryInstallationDisk(key, entry);
@@ -5,6 +5,7 @@ import {
MILITARY_INSTALLATION_DISK_DIR,
MILITARY_INSTALLATION_MAX_CACHE,
} from './constants.js';
import { cachedByConfiguredUpstream } from '../overpass/constants.js';
import path from 'node:path';
import { createHash } from 'node:crypto';
import { promises as fsp } from 'node:fs';
@@ -114,6 +115,9 @@ async function readMilitaryInstallationDisk(
const entry = JSON.parse(
await fsp.readFile(militaryInstallationDiskPath(cacheKey, dir), 'utf8'),
);
// Same reason as the Overpass disk cache: the key is the bbox alone, so an
// entry written against a different OVERPASS_UPSTREAMS is otherwise a hit.
if (!cachedByConfiguredUpstream(entry)) return null;
if (!militaryInstallationDiskFresh(entry, maxAgeMs)) return null;
return migrateMilitaryInstallationEntry(entry);
} catch {
+2 -2
View File
@@ -179,7 +179,7 @@ export const ACTION_DESCRIPTIONS = {
},
get_current_view_state: {
description:
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action.',
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action. Tracked aircraft emergency status is a crowd-sourced transponder broadcast, not a confirmed incident: quote emergencyBroadcast and never infer a hijack or an incident ending from a code change.',
$position: 1,
},
set_hud: {
@@ -543,7 +543,7 @@ export const ACTION_DESCRIPTIONS = {
},
analyst_query: {
description:
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values).',
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
$position: 1,
parameters: {
properties: {
+6
View File
@@ -5,6 +5,7 @@ import {
OVERPASS_DISK_DIR,
OVERPASS_CACHE_MS,
OVERPASS_CACHE_MAX_ENTRIES,
cachedByConfiguredUpstream,
} from './constants.js';
import path from 'node:path';
import { createHash } from 'node:crypto';
@@ -47,6 +48,11 @@ async function readOverpassDisk(cacheKey, maxAgeMs) {
// Older versions persisted 4xx refusals with normal data TTLs. Ignore
// them on both fresh and stale reads so an upgrade can recover immediately.
if (!overpassPayloadIsData(payload)) return null;
// The key is the query alone, so a file written by a different upstream —
// including the public instances that were the default before #742 — is a
// hit for the same query. Refuse it: the operator's configured instance
// must be what answers, on the stale path too.
if (!cachedByConfiguredUpstream(payload)) return null;
if (Date.now() - payload.cachedAt > maxAgeMs) return null;
return payload;
} catch {
+59 -4
View File
@@ -1,4 +1,5 @@
import path from 'node:path';
import { createHash } from 'node:crypto';
// ---------------------------------------------------------------------------
// Overpass API proxy constants and cache state
@@ -27,16 +28,68 @@ function parseOverpassUpstreams(raw) {
return endpoints.slice(0, 8);
}
let upstreamMemo = { raw: null, endpoints: [] };
let upstreamMemo = { raw: null, endpoints: [], tag: 'none' };
/**
* Short, stable identifier for the configured upstream set. Cached responses
* carry it so an answer produced by a different instance is not served after
* the operator repoints `OVERPASS_UPSTREAMS` — the disk cache outlives the
* configuration change by 7 days, a month for boundary queries, and forever on
* the serve-stale path. It is a hash, never the URLs, and it is only ever
* compared on read: it is not sent to clients and does not appear in errors.
*/
function upstreamTag(endpoints) {
if (!endpoints.length) return 'none';
// Sorted, so reordering the same instances for failover keeps the cache.
return createHash('sha256')
.update([...endpoints].sort().join('\n'))
.digest('hex');
}
/** Resolve after environment loading. Public Overpass instances are not used by default. */
function resolveOverpassUpstreams() {
const raw = process.env.OVERPASS_UPSTREAMS || '';
if (upstreamMemo.raw !== raw)
upstreamMemo = { raw, endpoints: parseOverpassUpstreams(raw) };
readUpstreamMemo();
return [...upstreamMemo.endpoints];
}
/** The configured-upstream tag stored alongside, and checked against, cache entries. */
function resolveOverpassUpstreamTag() {
return readUpstreamMemo().tag;
}
let reportedForeignCache = false;
/**
* Was this cached entry produced by the upstream set configured now?
*
* With nothing configured there is no instance to contradict, and serving
* last-good data at any age is the point of the cache (the proxy's
* not-configured path depends on it), so any entry is accepted. Once an
* operator does configure an instance, only its own answers are served: an
* entry with no tag, or one from a different instance, is refused rather than
* matched on the query alone. Shared by both caches fed by these upstreams.
*/
function cachedByConfiguredUpstream(entry) {
const tag = resolveOverpassUpstreamTag();
if (tag === 'none' || entry?.upstreams === tag) return true;
if (!reportedForeignCache) {
reportedForeignCache = true;
console.warn(
'[Overpass] ignoring cached responses produced by a different OVERPASS_UPSTREAMS; they will be refetched from the configured instance.',
);
}
return false;
}
function readUpstreamMemo() {
const raw = process.env.OVERPASS_UPSTREAMS || '';
if (upstreamMemo.raw !== raw) {
const endpoints = parseOverpassUpstreams(raw);
upstreamMemo = { raw, endpoints, tag: upstreamTag(endpoints) };
}
return upstreamMemo;
}
/**
* TTL for FRESH cached Overpass responses (ms). Road geometry is static for
* months — the original 45 s TTL forced a public-mirror round-trip on nearly
@@ -153,6 +206,8 @@ export {
OVERPASS_MAX_RESPONSE_BYTES,
parseOverpassUpstreams,
resolveOverpassUpstreams,
resolveOverpassUpstreamTag,
cachedByConfiguredUpstream,
OVERPASS_USER_AGENT,
OVERPASS_TIMEOUT_MS,
};
+4
View File
@@ -1,6 +1,7 @@
import {
OVERPASS_MAX_RESPONSE_BYTES,
resolveOverpassUpstreams,
resolveOverpassUpstreamTag,
OVERPASS_USER_AGENT,
OVERPASS_TIMEOUT_MS,
} from './constants.js';
@@ -187,6 +188,9 @@ async function fetchOverpassPayload(
contentType: 'application/json',
// Never retain a secret-bearing endpoint in cache or response metadata.
endpoint: 'configured',
// Which configured instance produced this, as a hash rather than a URL,
// so a cached answer is not reused after the operator repoints.
upstreams: resolveOverpassUpstreamTag(),
rateLimited: false,
};
} catch {
+6 -1
View File
@@ -1,8 +1,13 @@
/** Place-search and route middleware for Node development servers. */
export { googlePlacesContextProxy } from './places/google.js';
export {
googlePlacesContextProxy,
validateGeocodeAddress,
geocodeBounds,
} from './places/google.js';
export {
googleServerApiKey,
keylessGooglePlacesResponse,
keylessGoogleGeocodeResponse,
} from './places/google-key.js';
export { installRouteMiddleware } from './places/routes.js';
export {
+28 -6
View File
@@ -14,12 +14,34 @@ export function keylessGooglePlacesResponse(apiKey) {
}
/**
* Google API key for the SERVER-SIDE calls (Places nearby/text search, the
* CCTV Street View fallback). These never reach the browser, so this key can
* be restricted by server IP and scoped to Places API + Street View Static
* API — while GOOGLE_MAPS_API_KEY stays referrer-restricted to Map Tiles +
* Geocoding for the browser (#33). Splitting them is opt-in: unset, this
* falls back to the shared browser key and nothing changes.
* Geocoding speaks Google's own response shape rather than the `places: []`
* contract its two siblings use, because the browser adapters read `status`
* and `results` directly. The keyless capability reads the same way: a status
* that is not `OK` and no results, which the adapters already treat as "this
* provider did not answer" and pass to the next geocoder in the chain.
*/
export function keylessGoogleGeocodeResponse(apiKey) {
if (String(apiKey ?? '').trim()) return null;
return {
statusCode: 200,
payload: {
configured: false,
status: 'REQUEST_DENIED',
results: [],
error: null,
},
};
}
/**
* Google API key for the SERVER-SIDE calls (Places nearby/text search,
* geocoding, the CCTV Street View fallback). These never reach the browser, so
* this key can be restricted by server IP and scoped to Places API + Geocoding
* API + Street View Static API — while GOOGLE_MAPS_API_KEY stays
* referrer-restricted to Map Tiles + Maps JavaScript API for the browser
* (#33, #363). Splitting them is opt-in: unset, this falls back to the shared
* browser key, which then has to carry every API both sides use and cannot
* take a referrer restriction.
*/
export function googleServerApiKey() {
return resolveGoogleServerKey(process.env);
+191 -1
View File
@@ -1,9 +1,11 @@
import {
googleServerApiKey,
keylessGooglePlacesResponse,
keylessGoogleGeocodeResponse,
} from './google-key.js';
import { makeCostRateLimiter, clientKey } from '../common/rate-limit.js';
import { admitSameSite } from '../common/same-site.js';
import { readResponseJsonCapped } from '../common/http.js';
import {
projectNearbyPlaces,
projectTextSearchPlaces,
@@ -25,7 +27,7 @@ let _googleRateLimiter;
*/
export const GOOGLE_DEFAULT_PER_MIN = 120;
/** Google cost endpoints (nearby-places + text-search). Null only when set to 0. */
/** Google cost endpoints (nearby-places, text-search, geocoding). Null only when set to 0. */
function googleRateLimiter() {
if (_googleRateLimiter === undefined)
_googleRateLimiter = makeCostRateLimiter(
@@ -56,6 +58,51 @@ export function validatePlacesCoordinates(searchParams) {
return { ok: true, latitude, longitude };
}
/** Longest accepted geocoder query. Real place names are far shorter. */
const GEOCODE_MAX_QUERY = 200;
/**
* Upstream deadline and body ceiling for the geocode routes. The deadline stays
* armed through body consumption, so a stalled upstream cannot hold a handler
* open, and the cap bounds a chunked or length-omitted reply. A geocode answer
* is a handful of results; 256 KiB is far above any honest one.
*/
const GEOCODE_UPSTREAM_TIMEOUT_MS = 10_000;
const GEOCODE_MAX_RESPONSE_BYTES = 256 * 1024;
/** Validate a free-text geocoder query before consuming request quota. */
export function validateGeocodeAddress(searchParams) {
const address = String(searchParams.get('address') || '').trim();
if (!address) return { ok: false, error: 'address is required' };
if (address.length > GEOCODE_MAX_QUERY)
return { ok: false, error: 'address is too long' };
return { ok: true, address };
}
/**
* Validate the optional viewport bias, which Google reads as
* `sw_lat,sw_lng|ne_lat,ne_lng`. An unparsed value is dropped rather than
* refused: the bias only ranks results, so a malformed one costs relevance,
* not an answer.
*/
export function geocodeBounds(searchParams) {
const raw = String(searchParams.get('bounds') || '').trim();
if (!raw) return null;
const corners = raw.split('|');
if (corners.length !== 2) return null;
const parts = corners.flatMap((corner) => corner.split(','));
// A blank component must be refused before Number() turns it into 0 and a
// half-written corner like `30.4,` reads as the equator.
if (parts.length !== 4 || parts.some((part) => !part.trim())) return null;
const numbers = parts.map(Number);
if (!numbers.every(Number.isFinite)) return null;
const [swLat, swLon, neLat, neLon] = numbers;
if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null;
if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null;
return raw;
}
/** Nearby place labels and view-biased text search, with request-time key resolution. */
export function googlePlacesContextProxy({
resolveApiKey = googleServerApiKey,
@@ -293,6 +340,149 @@ export function googlePlacesContextProxy({
);
}
});
/**
* Geocoding belongs on the server, not in the page. Google's Geocoding web
* service refuses referrer-restricted keys, so calling it from the browser
* forces GOOGLE_MAPS_API_KEY — which ships in the bundle by design — to be
* left unrestricted (#363). Here the request carries the server key, and
* the browser key no longer needs the Geocoding API at all.
*
* Unlike the two Places routes above, these answer in Google's own shape
* (`status` + `results`) rather than the `places: []` contract: the browser
* normalizers read that shape directly, and keeping it means the proxy adds
* a hop without adding a translation nobody asked for.
*/
function installGeocodeRoute(path, readQuery) {
middlewares.use(path, async (req, res) => {
const refuse = (statusCode, error) => {
res.statusCode = statusCode;
res.setHeader('Content-Type', 'application/json');
res.end(
JSON.stringify({ status: 'REQUEST_DENIED', results: [], error }),
);
};
// Gate first, like the Places routes above: a cross-site caller learns
// nothing about this endpoint's method surface.
if (admitSameSite(req, res)) return;
if (req.method !== 'GET') {
refuse(405, 'Method not allowed');
return;
}
// A keyless geocode has no provider cost, so it resolves before the
// paid-endpoint limiter can consume or exhaust quota (mirrors the
// Places routes above).
const apiKey = resolveApiKey();
const keyless = keylessGoogleGeocodeResponse(apiKey);
if (keyless) {
res.statusCode = keyless.statusCode;
res.setHeader('Content-Type', 'application/json');
res.setHeader('Cache-Control', 'no-store');
res.end(JSON.stringify(keyless.payload));
return;
}
const requestUrl = new URL(req.url || '', 'http://localhost');
const query = readQuery(requestUrl.searchParams);
if (!query.ok) {
refuse(400, query.error);
return;
}
// Per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN), shared with the
// Places routes because both spend the same Google budget.
const _grl = googleRateLimiter();
if (_grl && !_grl(clientKey(req))) {
res.statusCode = 429;
res.setHeader('Content-Type', 'application/json');
res.setHeader('Retry-After', '5');
res.end(
JSON.stringify({
status: 'OVER_QUERY_LIMIT',
results: [],
error: 'Rate limit exceeded',
}),
);
return;
}
try {
const upstream = new URL(
endpoints.geocode ||
'https://maps.googleapis.com/maps/api/geocode/json',
);
for (const [name, value] of Object.entries(query.params))
upstream.searchParams.set(name, value);
upstream.searchParams.set('key', apiKey);
const deadline = AbortSignal.timeout(GEOCODE_UPSTREAM_TIMEOUT_MS);
const response = await fetchImpl(upstream.toString(), {
redirect: 'error',
signal: deadline,
});
// An unreadable, oversized or timed-out body is not an answer: it
// falls through to the same shaped failure as a refusal below.
const data = await readResponseJsonCapped(
response,
GEOCODE_MAX_RESPONSE_BYTES,
deadline,
).catch(() => null);
// A body we could not read is an upstream failure, not an answer with
// a status of its own, so it does not borrow the upstream's code.
if (!data) {
res.statusCode = 502;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.end(
JSON.stringify({
status: 'UNKNOWN_ERROR',
results: [],
error: 'Google Geocoding request failed',
}),
);
return;
}
res.statusCode = response.ok ? 200 : response.status;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.setHeader('Cache-Control', 'private, max-age=300');
res.end(JSON.stringify(data));
} catch {
// Fixed public text: a transport, URL or runtime message would
// describe the server to the browser (SECURITY.md, sanitized errors).
res.statusCode = 502;
res.setHeader('Content-Type', 'application/json; charset=utf-8');
res.end(
JSON.stringify({
status: 'UNKNOWN_ERROR',
results: [],
error: 'Google Geocoding request failed',
}),
);
}
});
}
installGeocodeRoute('/api/google/geocode', (searchParams) => {
const address = validateGeocodeAddress(searchParams);
if (!address.ok) return address;
const bounds = geocodeBounds(searchParams);
return {
ok: true,
params: {
address: address.address,
...(bounds ? { bounds } : {}),
},
};
});
installGeocodeRoute('/api/google/reverse-geocode', (searchParams) => {
const coordinates = validatePlacesCoordinates(searchParams);
if (!coordinates.ok) return coordinates;
const { latitude, longitude } = coordinates;
return { ok: true, params: { latlng: `${latitude},${longitude}` } };
});
}
return {
+29 -12
View File
@@ -1,6 +1,9 @@
import https from 'node:https';
import { Readable } from 'node:stream';
import { isNonGlobalIpv4 } from './stations.js';
const NULL_BODY_STATUSES = new Set([204, 205, 304]);
export function radioMirrorOrigin(value) {
const hostname = String(value ?? '')
.toLowerCase()
@@ -124,19 +127,33 @@ export function fetchPinnedRadioResponse(url, options, addresses) {
},
},
(response) => {
const headers = new Headers();
for (const [name, value] of Object.entries(response.headers)) {
if (Array.isArray(value))
value.forEach((item) => headers.append(name, item));
else if (value !== undefined) headers.set(name, String(value));
// Nothing thrown here may escape: this callback runs outside the
// promise, so an exception would be uncaught and crash the server.
try {
const status = response.statusCode || 500;
// A `Response` only represents statuses up to 599; Node never hands
// this callback a 1xx.
if (status > 599)
throw new Error(
`Radio Browser returned unsupported status ${status}`,
);
const headers = new Headers();
for (const [name, value] of Object.entries(response.headers)) {
if (Array.isArray(value))
value.forEach((item) => headers.append(name, item));
else if (value !== undefined) headers.set(name, String(value));
}
const nullBody = NULL_BODY_STATUSES.has(status);
const upstream = new Response(
nullBody ? null : Readable.toWeb(response),
{ status, statusText: response.statusMessage || '', headers },
);
if (nullBody) response.resume();
resolve(upstream);
} catch (error) {
response.destroy();
reject(error);
}
resolve(
new Response(Readable.toWeb(response), {
status: response.statusCode || 500,
statusText: response.statusMessage || '',
headers,
}),
);
},
);
request.on('error', reject);
+11 -2
View File
@@ -1,6 +1,7 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
import { readResponseTextCapped } from '../common/http.js';
/**
* Vite plugin: CelesTrak TLE proxy.
@@ -48,10 +49,14 @@ export function celestrakProxy() {
}
}
/** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */
const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
async function fetchUpstream(group) {
const url = celestrakTleUrl(group);
const signal = AbortSignal.timeout(20000);
const res = await fetch(url.toString(), {
signal: AbortSignal.timeout(20000),
signal,
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
// descriptive User-Agent with a contact point.
headers: {
@@ -60,7 +65,11 @@ export function celestrakProxy() {
},
});
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const body = await res.text();
const body = await readResponseTextCapped(
res,
CELESTRAK_MAX_RESPONSE_BYTES,
signal,
);
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
if (!/^1 /m.test(body)) throw new Error('no TLE lines in response');
return { at: Date.now(), body };
+11 -4
View File
@@ -1,6 +1,7 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { readResponseBytesCapped } from './common/http.js';
import {
isValidTileCoord as isValidTomTomTile,
utcDayKey as tomtomUtcDayKey,
@@ -48,6 +49,8 @@ export function tomtomProxy() {
const DEFAULT_DAILY_BUDGET = 6000;
const MEM_MAX_ENTRIES = 256;
const UPSTREAM_TIMEOUT_MS = 15000;
/** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */
const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
/** @type {Map<string, {at:number, buf:Buffer}>} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */
const mem = new Map();
@@ -143,11 +146,15 @@ export function tomtomProxy() {
'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' +
`${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`;
recordUpstreamFetch(); // attempts count — upstream bills the request either way
const res = await fetch(url, {
signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
});
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const res = await fetch(url, { signal });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const buf = Buffer.from(await res.arrayBuffer());
const bytes = await readResponseBytesCapped(
res,
TOMTOM_TILE_MAX_RESPONSE_BYTES,
signal,
);
const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
if (buf.length === 0) throw new Error('empty tile body');
return buf;
}
+9 -7
View File
@@ -142,9 +142,11 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
const rows = Array.from({ length: 15000 }, (_, i) =>
row(now, `vehicle-${i}`),
);
const start = performance.now();
// Main-thread CPU: excludes scheduler waits and parallel GC helper threads.
const start = process.threadCpuUsage();
h.ingest(feed, rows, now);
times.push(performance.now() - start);
const usage = process.threadCpuUsage(start);
times.push((usage.user + usage.system) / 1000);
}
assert.equal(h.diagnostics().keys, 15000);
for (let i = 0; i < 15000; i++) {
@@ -154,12 +156,12 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
}
assert.ok(h.diagnostics().allocatedBytes < 16 * 1024 * 1024);
t.diagnostic(
`15000-vehicle ingestion ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
`15000-vehicle ingestion CPU ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
);
// A ceiling that says "not quadratic", not a benchmark: 15,000 rows in
// well under two seconds on a loaded laptop or a slow CI runner. The
// per-run figure above is the diagnostic to read.
assert.ok(Math.max(...times) < 2000, `ingestion ms: ${times}`);
// CPU time measures ingestion work without counting scheduler delays on
// loaded runners. Keep the two-second ceiling as a guard against quadratic
// work, not a benchmark; the per-poll figures above show any slowdown.
assert.ok(Math.max(...times) < 2000, `ingestion CPU ms: ${times}`);
});
test('snapshot capacity pressure preserves members not yet visited and discloses recreated history', (t) => {