mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 10:53:59 +00:00
Merge remote-tracking branch 'upstream/pr-686' into fix/terrain-same-site-gate-on-686
# Conflicts: # CHANGELOG.md
This commit is contained in:
commit
5d0c367315
400 files changed
+48271
-2164
No files matched your search
@@ -1,3 +1,5 @@
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
|
||||
*
|
||||
@@ -25,6 +27,10 @@ export function adsbLolProxy() {
|
||||
let _cooldownStatus = 0;
|
||||
/** Response cache TTL (ms). */
|
||||
const CACHE_MS = 12000;
|
||||
/** Upstream deadline (ms); the fetch previously had none and could hang a poll. */
|
||||
const UPSTREAM_TIMEOUT_MS = 12000;
|
||||
/** /v2/mil measures ~67 KB; a relayed error page is bounded by the same cap. */
|
||||
const ADSBLOL_MIL_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
/** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */
|
||||
const RATE_LIMIT_COOLDOWN_MS = 30000;
|
||||
/** Cooldown after a 5xx (ms). */
|
||||
@@ -98,11 +104,17 @@ export function adsbLolProxy() {
|
||||
);
|
||||
return;
|
||||
}
|
||||
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
|
||||
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
|
||||
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
|
||||
signal,
|
||||
});
|
||||
if (upstream.ok) {
|
||||
const body = await upstream.text();
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
_cache = body;
|
||||
_cacheAt = Date.now();
|
||||
_cooldownUntil = 0;
|
||||
@@ -129,7 +141,11 @@ export function adsbLolProxy() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
const body = await upstream.text();
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
serve(
|
||||
res,
|
||||
upstream.status,
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
|
||||
/** Route and aircraft lookups are a few hundred bytes; 1 MiB is a hard bound. */
|
||||
const ADSBDB_MAX_RESPONSE_BYTES = 1024 * 1024;
|
||||
/**
|
||||
* adsbdb.com enrichment proxy: callsign → route (airline + origin/destination
|
||||
* airports) and hex → aircraft type/registration. Free community API — cached
|
||||
@@ -80,12 +84,16 @@ export function adsbdbProxy() {
|
||||
kind === 'route'
|
||||
? `https://api.adsbdb.com/v0/callsign/${encodeURIComponent(key)}`
|
||||
: `https://api.adsbdb.com/v0/aircraft/${encodeURIComponent(key)}`;
|
||||
const res = await fetch(url, { signal: AbortSignal.timeout(8000) });
|
||||
const signal = AbortSignal.timeout(8000);
|
||||
const res = await fetch(url, { signal });
|
||||
if (res.ok) {
|
||||
const payload = await readResponseJsonCapped(
|
||||
res,
|
||||
ADSBDB_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
const data =
|
||||
kind === 'route'
|
||||
? parseRoute(await res.json())
|
||||
: parseAircraft(await res.json());
|
||||
kind === 'route' ? parseRoute(payload) : parseAircraft(payload);
|
||||
store[key] = { at: Date.now(), data }; // data may be null — negative cache
|
||||
dirty = true;
|
||||
return data;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback
|
||||
import {
|
||||
coalesceProxyRequest,
|
||||
readResponseJsonCapped,
|
||||
readResponseTextCapped,
|
||||
} from '../common/http.js';
|
||||
import { requiredFiniteQueryNumber } from '../common/query.js';
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -29,11 +30,34 @@ const OPENSKY_CACHE_MS = 9000;
|
||||
const OPENSKY_ATTEMPT_TIMEOUT_MS = 10_000;
|
||||
const OPENSKY_STATES_URL =
|
||||
'https://opensky-network.org/api/states/all?extended=1';
|
||||
/** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */
|
||||
const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Credit-governor cooldown for a 429: OpenSky's retry-after bounded to
|
||||
* 30 s … 30 min, or 2 min when the header is absent.
|
||||
* @param {Response} upstream Rate-limited response.
|
||||
* @returns {number} Cooldown (ms).
|
||||
*/
|
||||
function openSkyRateLimitCooldownMs(upstream) {
|
||||
// headers.get() gives null when absent, and Number(null) is 0, not NaN.
|
||||
const header = upstream.headers.get('x-rate-limit-retry-after-seconds');
|
||||
const retryAfterSec =
|
||||
header == null || header.trim() === '' ? NaN : Number(header);
|
||||
return Math.min(
|
||||
Math.max(
|
||||
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
|
||||
30_000,
|
||||
),
|
||||
30 * 60_000,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch and read the global snapshot, giving each attempt
|
||||
* OPENSKY_ATTEMPT_TIMEOUT_MS and retrying once when an attempt times out or
|
||||
* the connection fails. A second failure throws, which the proxy answers
|
||||
* the connection fails. An oversized body is not retried: it would fail again
|
||||
* and spend more credits. A second failure throws, which the proxy answers
|
||||
* from its stale cache or the regional fallback.
|
||||
* @param {Record<string, string>} headers Request headers.
|
||||
* @returns {Promise<{upstream: Response, body: string}>}
|
||||
@@ -41,13 +65,27 @@ const OPENSKY_STATES_URL =
|
||||
async function fetchOpenSkyStates(headers) {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
const upstream = await fetch(OPENSKY_STATES_URL, {
|
||||
headers,
|
||||
signal: AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS),
|
||||
});
|
||||
return { upstream, body: await upstream.text() };
|
||||
const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
|
||||
const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
|
||||
if (upstream.status === 429) {
|
||||
// Start the cooldown from the headers and drop the rejected body
|
||||
// unread, so a stalled or oversized 429 body cannot skip it.
|
||||
_openskyCooldownUntil =
|
||||
Date.now() + openSkyRateLimitCooldownMs(upstream);
|
||||
void upstream.body?.cancel().catch(() => {});
|
||||
return {
|
||||
upstream,
|
||||
body: JSON.stringify({ error: 'OpenSky rate limited.' }),
|
||||
};
|
||||
}
|
||||
const body = await readResponseTextCapped(
|
||||
upstream,
|
||||
OPENSKY_STATES_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
return { upstream, body };
|
||||
} catch (error) {
|
||||
if (attempt >= 2) throw error;
|
||||
if (attempt >= 2 || error?.code === 'RESPONSE_TOO_LARGE') throw error;
|
||||
console.warn(
|
||||
'[OpenSky] global snapshot attempt failed, retrying:',
|
||||
error?.message || error,
|
||||
@@ -544,19 +582,9 @@ export function openSkyProxy() {
|
||||
}
|
||||
if (upstream.status === 429) {
|
||||
reason = 'rate_limited';
|
||||
// Credit governor: honor OpenSky's retry-after (bounded 30 s … 30 min;
|
||||
// 2 min when the header is absent) — no upstream attempts until then.
|
||||
const retryAfterSec = Number(
|
||||
upstream.headers.get('x-rate-limit-retry-after-seconds'),
|
||||
);
|
||||
const cooldownMs = Math.min(
|
||||
Math.max(
|
||||
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
|
||||
30_000,
|
||||
),
|
||||
30 * 60_000,
|
||||
);
|
||||
_openskyCooldownUntil = now + cooldownMs;
|
||||
// Credit governor: fetchOpenSkyStates already started the cooldown
|
||||
// from OpenSky's retry-after — no upstream attempts until then.
|
||||
const cooldownMs = openSkyRateLimitCooldownMs(upstream);
|
||||
// Serve the last-good body instead of the 429 when we have one —
|
||||
// the layer keeps rendering (STALE-cued) instead of dying.
|
||||
if (_openskyCacheBody && _openskyCacheStatus === 200) {
|
||||
|
||||
@@ -28,7 +28,7 @@ export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
|
||||
* GET /api/cctv/sources — list all registered camera sources
|
||||
* GET /api/cctv/health — per-camera health/status report
|
||||
* GET /api/cctv/stream/:id — stream info (feedType, URLs) for a camera
|
||||
* GET /api/cctv/media/:id — proxy live video/image media from upstream
|
||||
* GET /api/cctv/media/:id — proxy live video media from upstream
|
||||
* GET /api/cctv/frame/:id — single frame with fallback chain
|
||||
*
|
||||
* @returns {import('vite').Plugin}
|
||||
@@ -164,6 +164,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
sourceKind:
|
||||
source.sourceKind || (source.url ? 'configured' : 'fallback'),
|
||||
poseSource: source.poseSource,
|
||||
model: source.model || '',
|
||||
manufacturer: source.manufacturer || '',
|
||||
license: source.license,
|
||||
credit: source.credit || '',
|
||||
code: source.code || '',
|
||||
@@ -214,6 +216,21 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
|
||||
const source = sourceById.get(cameraId);
|
||||
const mediaUrl = source?.url || '';
|
||||
const feedType = normalizeFeedType(source?.feedType || 'image');
|
||||
// Live feeds only. A still camera is served by /frame, whose fetch
|
||||
// keeps redirects on the registered origin, accepts only image/* and
|
||||
// caps the body; relaying it here would bypass all three.
|
||||
if (!isVideoFeedType(feedType) && feedType !== 'mjpeg') {
|
||||
res.writeHead(404, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: 'No live media for this camera; use the frame endpoint',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const leaseId = url.searchParams.get('lease');
|
||||
if (feedType === 'hls' && !/^[a-f0-9-]{36}$/i.test(leaseId || '')) {
|
||||
res.writeHead(400);
|
||||
|
||||
@@ -4,6 +4,7 @@ import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js';
|
||||
import { allocateSourceCap, resolveCatalogCap } from './cap.js';
|
||||
import { loadGroundHeights, joinGroundHeights } from './groundHeights.js';
|
||||
import { cameraRegion, normalizeSourceItem } from './normalize.js';
|
||||
import { loadLithuaniaSources } from './lithuania.js';
|
||||
import {
|
||||
loadAustinSourcesFromOpenData,
|
||||
loadCaltransSourcesFromOpenData,
|
||||
@@ -17,19 +18,30 @@ import {
|
||||
loadWarendorfSourcesFromCatalog,
|
||||
loadNswSourcesFromOpenData,
|
||||
loadCalgarySourcesFromOpenData,
|
||||
loadCataloniaSourcesFromOpenData,
|
||||
loadDelDOTSourcesFromOpenData,
|
||||
loadKingCountySourcesFromOpenData,
|
||||
loadWsdotSourcesFromOpenData,
|
||||
loadVegvesenSourcesFromOpenData,
|
||||
loadNycSourcesFromOpenData,
|
||||
loadTorontoSourcesFromOpenData,
|
||||
} from './sources.js';
|
||||
|
||||
/** Env kill switch: unset or anything but "0" means enabled. */
|
||||
const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0';
|
||||
/** Env opt-in: only "1" enables. For packs whose feed access needs an
|
||||
* operator agreement or whose terms are unconfirmed, so an unset flag never
|
||||
* reaches the upstream. */
|
||||
const envOptIn = (name) => String(process.env[name] || '').trim() === '1';
|
||||
|
||||
/**
|
||||
* Live open-data packs, in merge order. Adding a region is one entry here
|
||||
* plus its loader in sources.js; the catalog cap is shared across entries
|
||||
* round-robin (cap.js), so a new pack never silently evicts an older one.
|
||||
* plus its loader (in sources.js or its own module); the catalog cap is
|
||||
* shared across entries round-robin (cap.js), so a new pack never silently
|
||||
* evicts an older one.
|
||||
* Each pack fails independently (allSettled) and is gated by its own env
|
||||
* kill switch.
|
||||
* kill switch, or by an opt-in flag where access needs an agreement or the
|
||||
* terms are unconfirmed.
|
||||
*/
|
||||
const LIVE_PACKS = [
|
||||
{ name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData },
|
||||
@@ -88,16 +100,50 @@ const LIVE_PACKS = [
|
||||
enabled: () => envEnabled('CCTV_CALGARY_ENABLED'),
|
||||
load: loadCalgarySourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'catalonia',
|
||||
enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'),
|
||||
load: loadCataloniaSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'deldot',
|
||||
enabled: () => envEnabled('CCTV_DELDOT_ENABLED'),
|
||||
load: loadDelDOTSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'kingcounty',
|
||||
enabled: () => envEnabled('CCTV_KINGCOUNTY_ENABLED'),
|
||||
load: loadKingCountySourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'wsdot',
|
||||
enabled: () => envEnabled('CCTV_WSDOT_ENABLED'),
|
||||
load: loadWsdotSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'vegvesen',
|
||||
enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'),
|
||||
load: loadVegvesenSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
// NYC DOT requires a data-sharing agreement for this feed, so the pack is
|
||||
// off unless the operator opts in (see DATA_SOURCES.md).
|
||||
name: 'nyc',
|
||||
enabled: () => envOptIn('CCTV_NYC_ENABLED'),
|
||||
load: loadNycSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
name: 'toronto',
|
||||
enabled: () => envEnabled('CCTV_TORONTO_ENABLED'),
|
||||
load: loadTorontoSourcesFromOpenData,
|
||||
},
|
||||
{
|
||||
// Opt-in until the Via Lietuva open-data terms are confirmed to cover the
|
||||
// camera photos (see DATA_SOURCES.md).
|
||||
name: 'lithuania',
|
||||
enabled: () => envOptIn('CCTV_LITHUANIA_ENABLED'),
|
||||
load: loadLithuaniaSources,
|
||||
},
|
||||
];
|
||||
/**
|
||||
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
|
||||
|
||||
@@ -8,8 +8,8 @@ export const DEFAULT_AUSTIN_MAX_SOURCES = 250;
|
||||
* Catalog-wide safety ceiling on served cameras. Each pack already caps
|
||||
* itself (nearest-to-anchor first); this bound only matters when the packs
|
||||
* together exceed it, and it is then filled round-robin across packs (see
|
||||
* cap.js) so no region is silently dropped. Sized above the sum of the
|
||||
* default per-pack caps so a default install never trims.
|
||||
* cap.js) so no region is silently dropped. The default per-pack caps
|
||||
* together can exceed it, so a default install may be thinned evenly here.
|
||||
*/
|
||||
export const DEFAULT_CCTV_MAX_SOURCES = 4000;
|
||||
/** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */
|
||||
@@ -236,6 +236,56 @@ export const CALGARY_DOWNTOWN = { lat: 51.0461, lon: -114.0626 };
|
||||
* body cannot be buffered without limit. */
|
||||
export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Catalonia traffic cameras (Servei Català de Trànsit): one keyless WFS/GML
|
||||
* XML list (verified 2026-09-15, hourly catalog refresh), fetched over https
|
||||
* (verified 2026-09-27: same 200 body, no redirect). The feed carries a
|
||||
* `font` field naming the actual publisher per camera — SCT's own highway
|
||||
* cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras are
|
||||
* kept, each confirmed to return a real frame. Frame URLs are checked
|
||||
* against a host allowlist rather than one pinned origin, since each
|
||||
* publisher serves from its own host.
|
||||
*/
|
||||
export const CATALONIA_CAMERAS_URL =
|
||||
'https://www.gencat.cat/transit/opendata/cameres.xml';
|
||||
/** Byte ceiling for the Catalonia catalog read (~90 KB observed 2026-09-27),
|
||||
* so a runaway body cannot be buffered without limit. */
|
||||
export const CATALONIA_MAX_CATALOG_BYTES = 1024 * 1024;
|
||||
/** Allowed image hosts, one per kept `font` publisher in the feed, each
|
||||
* mapped to the one directory its frame links live in (every link in the
|
||||
* 2026-10 feed shares it); a link must name a single file directly inside. */
|
||||
export const CATALONIA_IMAGE_HOSTS = Object.freeze(
|
||||
new Map([
|
||||
['mct.gencat.cat', '/mct2bo/'], // SCT
|
||||
['www.bcn.cat', '/transit/imatges/'], // IMI (Ajuntament de Barcelona)
|
||||
['emap.terrassa.cat', '/it_terrassa/'], // Ajuntament de Terrassa
|
||||
]),
|
||||
);
|
||||
/** SCT `sctidcam` values: a bare image filename, which also keys the camera
|
||||
* id (the feed's `fid` embeds an upstream server-start stamp). */
|
||||
export const CATALONIA_SCT_CAMERA_FILE_RE = /^[a-z0-9]+\.(gif|jpe?g)$/i;
|
||||
/** Per-`font` partner credit, shown beside the provider in the CCTV panel
|
||||
* (same pattern as DriveBC's partner-supplied cameras). SCT is the feed's
|
||||
* own publisher and carries no separate credit. */
|
||||
export const CATALONIA_FONT_CREDIT = Object.freeze({
|
||||
IMI: 'Ajuntament de Barcelona',
|
||||
Terrassa: 'Ajuntament de Terrassa',
|
||||
});
|
||||
/** Ground-elevation priors in metres, by `font`: the feed carries no
|
||||
* elevation, and this pack spans sea level (Barcelona) to Terrassa's plateau
|
||||
* (~270 m). The client's ground snap corrects these on 3D-tile stacks. */
|
||||
export const CATALONIA_FONT_ELEVATION_M = Object.freeze({
|
||||
SCT: 80,
|
||||
IMI: 20,
|
||||
Terrassa: 270,
|
||||
});
|
||||
export const DEFAULT_CATALONIA_MAX_SOURCES = 150;
|
||||
/** Prioritization anchors: Barcelona and Terrassa. */
|
||||
export const CATALONIA_ANCHORS = [
|
||||
{ lat: 41.3874, lon: 2.1686 }, // Barcelona
|
||||
{ lat: 41.5636, lon: 2.0111 }, // Terrassa
|
||||
];
|
||||
|
||||
/** Statens vegvesen (Norway) road cameras: the keyless OGC API Features view
|
||||
* of the official DATEX 3.1 CCTV site table, one GeoJSON request for the whole
|
||||
* country (~900 cameras). The raw DATEX node needs an account; this view does
|
||||
@@ -249,9 +299,9 @@ export const VEGVESEN_IMAGE_ORIGIN =
|
||||
* per-camera manifest path is registered. */
|
||||
export const VEGVESEN_VIDEO_URL = (cameraId) =>
|
||||
`https://kamera.vegvesen.no/public/${cameraId}/manifest.m3u8`;
|
||||
/** Above the ~850 working cameras so a default install keeps them all, while
|
||||
* staying close to them: the per-pack defaults are meant to sum to no more
|
||||
* than DEFAULT_CCTV_MAX_SOURCES. */
|
||||
/** Above the ~850 working cameras so the pack keeps them all on its own. The
|
||||
* catalog-wide DEFAULT_CCTV_MAX_SOURCES cap still applies across packs and is
|
||||
* filled round-robin, so a full default install may thin this pack. */
|
||||
export const DEFAULT_VEGVESEN_MAX_SOURCES = 900;
|
||||
/** Hard ceiling on the catalog body. The whole country is ~800 KB. */
|
||||
export const VEGVESEN_MAX_CATALOG_BYTES = 8 * 1024 * 1024;
|
||||
@@ -267,6 +317,36 @@ export const NORWAY_ANCHORS = [
|
||||
{ lat: 69.6492, lon: 18.9553 }, // Tromsø
|
||||
];
|
||||
|
||||
/** NYC DOT Traffic Management Center cameras: one keyless JSON list for the
|
||||
* five boroughs (~980 cameras). Frames are stills synthesized from each
|
||||
* camera UUID on the city's webcam host — never taken from the payload. */
|
||||
export const NYC_CAMERAS_URL = 'https://webcams.nyctmc.org/api/cameras';
|
||||
/** The only origin NYC camera frames may come from. The catalog's `imageUrl`
|
||||
* is ignored; the registered path is always this origin plus a validated
|
||||
* UUID plus `/image`. */
|
||||
export const NYC_IMAGE_ORIGIN = 'https://webcams.nyctmc.org/api/cameras/';
|
||||
export const NYC_IMAGE_URL = (id) => `${NYC_IMAGE_ORIGIN}${id}/image`;
|
||||
export const NYC_CAMERA_ID_RE =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
export const DEFAULT_NYC_MAX_SOURCES = 250;
|
||||
/** Hard ceiling on the NYC catalog body. The live list is ~980 rows / ~250 KB;
|
||||
* this only exists so an upstream that streams an unbounded body cannot be
|
||||
* buffered without limit. */
|
||||
export const NYC_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
/** Ground-elevation prior in metres. Matches the New York CITY_POIS prior;
|
||||
* the client's one-shot ground snap corrects it wherever 3D tiles load. */
|
||||
export const NYC_GROUND_ELEVATION_M = 10;
|
||||
/** Prioritization anchors: one per borough so a cap keeps citywide coverage
|
||||
* rather than Midtown alone. */
|
||||
export const NYC_ANCHORS = [
|
||||
{ lat: 40.758, lon: -73.9855 }, // Manhattan (Times Square)
|
||||
{ lat: 40.7128, lon: -74.006 }, // Lower Manhattan
|
||||
{ lat: 40.6782, lon: -73.9442 }, // Brooklyn
|
||||
{ lat: 40.7282, lon: -73.7949 }, // Queens
|
||||
{ lat: 40.8448, lon: -73.8648 }, // Bronx
|
||||
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
|
||||
];
|
||||
|
||||
/** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */
|
||||
export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json';
|
||||
export const DEFAULT_DELDOT_MAX_SOURCES = 300;
|
||||
@@ -275,6 +355,83 @@ export const DELDOT_ANCHORS = [
|
||||
{ lat: 39.1582, lon: -75.5244 }, // Dover (Kent)
|
||||
{ lat: 38.6906, lon: -75.3877 }, // Georgetown (Sussex)
|
||||
];
|
||||
|
||||
/** King County, WA road cameras: keyless public ArcGIS FeatureServer layer
|
||||
* ("King County and WSDOT Traffic Cameras", King County DOT Road Services).
|
||||
* GeoJSON with WGS84 point geometry. `outFields` is an explicit list on
|
||||
* purpose: the layer also carries asset-administration columns (including
|
||||
* staff names in `ModifiedBy`) that must never be fetched. Uniquely among the
|
||||
* packs, the county publishes each camera's hardware `Manufacturer`/`Model`
|
||||
* (Cohu 3950/3965, Bosch AUTODOME, Axis Q6135-LE, …), carried through as the
|
||||
* source's `model`. */
|
||||
export const KING_COUNTY_CAMERAS_URL =
|
||||
'https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,ImageURL,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson';
|
||||
/** Frames come only from the county's own image handler host. */
|
||||
export const KING_COUNTY_IMAGE_ORIGIN = 'https://info.kingcounty.gov/';
|
||||
/** The whole catalog is ~125 rows (~59 live county cameras); no thinning by
|
||||
* default. */
|
||||
export const DEFAULT_KING_COUNTY_MAX_SOURCES = 125;
|
||||
/** Prioritization anchor: Renton, roughly central to the county road network. */
|
||||
export const KING_COUNTY_ANCHORS = [{ lat: 47.4829, lon: -122.2171 }];
|
||||
/** Hard ceiling on the King County catalog body. The whole layer is ~125 rows
|
||||
* and well under 100 KB; this only exists so an upstream that streams an
|
||||
* unbounded body cannot be buffered without limit. */
|
||||
export const KING_COUNTY_MAX_CATALOG_BYTES = 1024 * 1024;
|
||||
/**
|
||||
* City of Toronto RESCU traffic cameras: the municipal network the province's
|
||||
* 511 pack does not cover — downtown arterials plus the Gardiner, DVP, Allen
|
||||
* and Lake Shore. One keyless list for the whole city (~336 cameras), from the
|
||||
* City's open data portal. It is fetched on every catalog refresh, but the
|
||||
* upstream file is not updated often and may lag the live camera network.
|
||||
*
|
||||
* The list is served as JSONP (`jsonTMCEarthCamerasCallback({...})`); the
|
||||
* portal publishes the same data as .xml and .csv beside it.
|
||||
*/
|
||||
export const TORONTO_RESCU_LIST_URL =
|
||||
'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/Data/tmcearthcameras.json';
|
||||
/**
|
||||
* The only origin Toronto camera frames may come from. The list carries no URL
|
||||
* field: frame URLs are BUILT from this origin and a strictly-digits camera
|
||||
* number (`loc####.jpg`, the convention the dataset documents), so no upstream
|
||||
* field can steer the frame proxy off-host.
|
||||
*/
|
||||
export const TORONTO_RESCU_IMAGE_ORIGIN =
|
||||
'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/CameraImages/';
|
||||
/** The whole city fits under this; the cap exists to stay tunable, not to trim. */
|
||||
export const DEFAULT_TORONTO_MAX_SOURCES = 336;
|
||||
/** Yonge / Queen: the prioritization anchor. */
|
||||
export const TORONTO_DOWNTOWN = { lat: 43.6532, lon: -79.3832 };
|
||||
/** The city climbs from 76 m at the lake to 209 m at Steeles, so no single
|
||||
* number fits: this is the mean across the 336 camera sites. The ground-height
|
||||
* sidecar has no Toronto entries yet, so this prior seeds placement; a camera
|
||||
* that gains a sidecar entry uses it instead, and the client's ground snap
|
||||
* corrects the prior wherever 3D tiles are loaded. */
|
||||
export const TORONTO_GROUND_ELEVATION_M = 116;
|
||||
/** Hard ceiling on the RESCU list body. The whole city is ~336 rows and under
|
||||
* 60 KB; this only exists so an upstream that streams an unbounded body cannot
|
||||
* be buffered without limit. */
|
||||
export const TORONTO_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
/** WSDOT Travel Center cameras (Washington State): keyless Esri JSON on the
|
||||
* official open-data host — ~1,700 point features in EPSG:3857 (Web Mercator),
|
||||
* ETag/Last-Modified served. WSDOT's metadata marks the feed "low volume" use;
|
||||
* the catalog is fetched once per CCTV_SOURCE_CACHE_MS refresh. */
|
||||
export const WSDOT_CAMERAS_URL =
|
||||
'https://data.wsdot.wa.gov/travelcenter/Cameras.json';
|
||||
/** Only frames on WSDOT's own image host are registered; the catalog also
|
||||
* lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports)
|
||||
* whose own terms apply. */
|
||||
export const WSDOT_IMAGE_ORIGIN = 'https://images.wsdot.wa.gov/';
|
||||
export const DEFAULT_WSDOT_MAX_SOURCES = 250;
|
||||
/** Hard ceiling on the WSDOT catalog body. The live layer is ~400 KB; this
|
||||
* only exists so an upstream that streams an unbounded body cannot be
|
||||
* buffered without limit. */
|
||||
export const WSDOT_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
/** Prioritization anchors: downtown Seattle and Spokane. */
|
||||
export const WSDOT_ANCHORS = [
|
||||
{ lat: 47.6062, lon: -122.3321 }, // Seattle
|
||||
{ lat: 47.6588, lon: -117.426 }, // Spokane
|
||||
];
|
||||
|
||||
/** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */
|
||||
export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000;
|
||||
/** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js';
|
||||
import {
|
||||
cameraDisplayCode,
|
||||
fallbackHeadingFromId,
|
||||
prioritizeSources,
|
||||
} from './normalize.js';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
|
||||
const BASE = 'https://eismoinfo.lt/eismoinfo-backend';
|
||||
export const LITHUANIA_CAMERAS_URL = `${BASE}/camera-info-table`;
|
||||
// Without lks=true this endpoint returns [latitude, longitude]. The camera
|
||||
// table's x/y fields use LKS-94 metres and must never be treated as degrees.
|
||||
export const LITHUANIA_LOCATIONS_URL = `${BASE}/layer-static-features/VKR`;
|
||||
export const LITHUANIA_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
|
||||
const FRAME_PATH = '/eismoinfo-backend/image-provider/camera/last';
|
||||
const ANCHORS = [
|
||||
{ lat: 54.6872, lon: 25.2797 }, // Vilnius
|
||||
{ lat: 54.8985, lon: 23.9036 }, // Kaunas
|
||||
{ lat: 55.7033, lon: 21.1443 }, // Klaipeda
|
||||
{ lat: 55.9349, lon: 23.3137 }, // Siauliai
|
||||
{ lat: 55.7348, lon: 24.3575 }, // Panevezys
|
||||
];
|
||||
|
||||
/** Join public catalogs by ID; register only the official still endpoint. */
|
||||
export function lithuaniaCameraToSource(row, feature) {
|
||||
const id = String(row?.id ?? '');
|
||||
if (!/^[1-9]\d{0,8}$/.test(id) || String(feature?.id) !== id) return null;
|
||||
const point = feature?.points?.[0]?.point;
|
||||
if (!Array.isArray(point) || point.length !== 2) return null;
|
||||
const [lat, lon] = point;
|
||||
if (
|
||||
!Number.isFinite(lat) ||
|
||||
!Number.isFinite(lon) ||
|
||||
lat < 53.8 ||
|
||||
lat > 56.5 ||
|
||||
lon < 20.8 ||
|
||||
lon > 26.9
|
||||
)
|
||||
return null;
|
||||
try {
|
||||
const url = new URL(row.image);
|
||||
if (
|
||||
url.origin !== 'https://eismoinfo.lt' ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
url.pathname !== FRAME_PATH ||
|
||||
url.searchParams.get('id') !== id
|
||||
)
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
// Reconstruct so extra query parameters cannot alter the request.
|
||||
const imageUrl = `https://eismoinfo.lt${FRAME_PATH}?id=${id}`;
|
||||
const cameraId = `lithuania-${id}`;
|
||||
const name = String(row.name || feature.name || `Lietuva ${id}`)
|
||||
.trim()
|
||||
.slice(0, 160);
|
||||
return {
|
||||
id: cameraId,
|
||||
name,
|
||||
city: 'Lithuania / Lietuva',
|
||||
cityId: 'lithuania',
|
||||
provider: 'Via Lietuva / Eismoinfo',
|
||||
lat,
|
||||
lon,
|
||||
// No calibrated pose is published; use the app's low-confidence prior.
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'eismoinfo-public',
|
||||
credit:
|
||||
'Via Lietuva / eismoinfo.lt — periodically updated road-camera stills',
|
||||
code: cameraDisplayCode(name.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
async function readCatalog(url, sibling) {
|
||||
const signal = AbortSignal.any([
|
||||
AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
sibling.signal,
|
||||
]);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'manual',
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
throw new Error(
|
||||
response.status >= 300 && response.status < 400
|
||||
? 'catalog redirected; redirects are not followed'
|
||||
: `Catalog HTTP ${response.status}`,
|
||||
);
|
||||
}
|
||||
return await readResponseJsonCapped(
|
||||
response,
|
||||
LITHUANIA_MAX_CATALOG_BYTES,
|
||||
signal,
|
||||
);
|
||||
} catch (error) {
|
||||
// The join needs both catalogs; release the other request at once
|
||||
// rather than leaving its body open until the timeout.
|
||||
sibling.abort();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Two bounded requests per shared catalog refresh, no API key required. */
|
||||
export async function loadLithuaniaSources() {
|
||||
try {
|
||||
const sibling = new AbortController();
|
||||
const [rows, layers] = await Promise.all([
|
||||
readCatalog(LITHUANIA_CAMERAS_URL, sibling),
|
||||
readCatalog(LITHUANIA_LOCATIONS_URL, sibling),
|
||||
]);
|
||||
if (!Array.isArray(rows) || !Array.isArray(layers)) return [];
|
||||
const features = layers.find((layer) => layer?.layer === 'VKR')?.features;
|
||||
if (!Array.isArray(features)) return [];
|
||||
const locations = new Map(
|
||||
features.filter(Boolean).map((feature) => [String(feature.id), feature]),
|
||||
);
|
||||
const cameras = new Map();
|
||||
for (const row of rows) {
|
||||
const camera = lithuaniaCameraToSource(
|
||||
row,
|
||||
locations.get(String(row?.id)),
|
||||
);
|
||||
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
|
||||
}
|
||||
const rawCap = Number(process.env.CCTV_LITHUANIA_MAX_SOURCES || 400);
|
||||
const cap = Number.isFinite(rawCap)
|
||||
? Math.max(1, Math.min(1000, Math.floor(rawCap)))
|
||||
: 400;
|
||||
const sources = prioritizeSources([...cameras.values()], cap, ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded Lithuania camera sources: ${cameras.size} (using ${sources.length})`,
|
||||
);
|
||||
return sources;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] Lithuania camera download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -367,6 +367,17 @@ export function isLikelyCalgaryCoordinate(lat, lon) {
|
||||
);
|
||||
}
|
||||
|
||||
/** New York City's five boroughs, with slack for the harbor and city line. */
|
||||
export function isLikelyNycCoordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 40.45 &&
|
||||
lat <= 40.95 &&
|
||||
lon >= -74.3 &&
|
||||
lon <= -73.65
|
||||
);
|
||||
}
|
||||
|
||||
/** Mainland Norway, with slack for the coast and the Finnmark border. */
|
||||
export function isLikelyNorwayCoordinate(lat, lon) {
|
||||
return (
|
||||
@@ -383,6 +394,44 @@ export function isLikelyFinlandCoordinate(lat, lon) {
|
||||
return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32;
|
||||
}
|
||||
|
||||
/** Catalonia bounding box (mainland is roughly 40.5..42.9 N, 0.15..3.4 E). */
|
||||
export function isLikelyCataloniaCoordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 40.3 &&
|
||||
lat <= 42.9 &&
|
||||
lon >= 0.0 &&
|
||||
lon <= 3.4
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode numeric HTML character references, decimal ("í") and hex
|
||||
* ("í") alike, the entity form the Catalonia feed uses for accented
|
||||
* place names, plus the handful of named entities XML/HTML text commonly
|
||||
* carries. A reference that names no valid character (NUL, a surrogate, or
|
||||
* anything past U+10FFFF) is left as written rather than thrown on, so one
|
||||
* bad field cannot fail a whole catalog.
|
||||
*
|
||||
* @param {string} text
|
||||
* @returns {string}
|
||||
*/
|
||||
export function decodeNumericEntities(text) {
|
||||
return String(text || '')
|
||||
.replace(/&#(?:x([0-9a-f]+)|(\d+));/gi, (match, hex, dec) => {
|
||||
const code = hex ? Number.parseInt(hex, 16) : Number(dec);
|
||||
const valid =
|
||||
code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff);
|
||||
return valid ? String.fromCodePoint(code) : match;
|
||||
})
|
||||
.replace(/&/g, '&')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Human label for one Fintraffic preset (camera view).
|
||||
*
|
||||
@@ -551,5 +600,30 @@ export function normalizeSourceItem(item) {
|
||||
// badge can distinguish them from raw automated priors (e.g. Austin Open
|
||||
// Data, which never sets this field). Passed through as-is to the client.
|
||||
poseSource: item.poseSource === 'curated' ? 'curated' : undefined,
|
||||
// Optional hardware-model passthrough (additive, like poseSource): packs
|
||||
// whose catalog publishes the camera's hardware model — King County's
|
||||
// layer carries Manufacturer/Model per camera — pass it through so
|
||||
// identified-hardware consumers can use it. Absent everywhere else.
|
||||
model:
|
||||
typeof item.model === 'string' && item.model.trim()
|
||||
? item.model.trim()
|
||||
: undefined,
|
||||
// Bare model strings collide across vendors (e.g. "3950"), so the
|
||||
// publishing pack's manufacturer rides along for disambiguation.
|
||||
manufacturer:
|
||||
typeof item.manufacturer === 'string' && item.manufacturer.trim()
|
||||
? item.manufacturer.trim()
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** City of Toronto bounding box (Steeles to the lake, Etobicoke to Scarborough). */
|
||||
export function isLikelyTorontoCoordinate(lat, lon) {
|
||||
return (
|
||||
isPlausibleLatLon(lat, lon) &&
|
||||
lat >= 43.55 &&
|
||||
lat <= 43.9 &&
|
||||
lon >= -79.7 &&
|
||||
lon <= -79.1
|
||||
);
|
||||
}
|
||||
@@ -55,16 +55,47 @@ import {
|
||||
DEFAULT_CALGARY_MAX_SOURCES,
|
||||
CALGARY_DOWNTOWN,
|
||||
CALGARY_MAX_CATALOG_BYTES,
|
||||
CATALONIA_CAMERAS_URL,
|
||||
CATALONIA_MAX_CATALOG_BYTES,
|
||||
CATALONIA_IMAGE_HOSTS,
|
||||
CATALONIA_SCT_CAMERA_FILE_RE,
|
||||
CATALONIA_FONT_CREDIT,
|
||||
CATALONIA_FONT_ELEVATION_M,
|
||||
DEFAULT_CATALONIA_MAX_SOURCES,
|
||||
CATALONIA_ANCHORS,
|
||||
TORONTO_RESCU_LIST_URL,
|
||||
TORONTO_RESCU_IMAGE_ORIGIN,
|
||||
DEFAULT_TORONTO_MAX_SOURCES,
|
||||
TORONTO_DOWNTOWN,
|
||||
TORONTO_GROUND_ELEVATION_M,
|
||||
TORONTO_MAX_CATALOG_BYTES,
|
||||
CCTV_SOURCE_FETCH_TIMEOUT_MS,
|
||||
DELDOT_CCTV_URL,
|
||||
DEFAULT_DELDOT_MAX_SOURCES,
|
||||
DELDOT_ANCHORS,
|
||||
KING_COUNTY_CAMERAS_URL,
|
||||
KING_COUNTY_IMAGE_ORIGIN,
|
||||
DEFAULT_KING_COUNTY_MAX_SOURCES,
|
||||
KING_COUNTY_ANCHORS,
|
||||
KING_COUNTY_MAX_CATALOG_BYTES,
|
||||
WSDOT_CAMERAS_URL,
|
||||
WSDOT_IMAGE_ORIGIN,
|
||||
DEFAULT_WSDOT_MAX_SOURCES,
|
||||
WSDOT_ANCHORS,
|
||||
WSDOT_MAX_CATALOG_BYTES,
|
||||
DEFAULT_VEGVESEN_CCTV_URL,
|
||||
VEGVESEN_IMAGE_ORIGIN,
|
||||
VEGVESEN_VIDEO_URL,
|
||||
DEFAULT_VEGVESEN_MAX_SOURCES,
|
||||
VEGVESEN_MAX_CATALOG_BYTES,
|
||||
NORWAY_ANCHORS,
|
||||
NYC_CAMERAS_URL,
|
||||
NYC_IMAGE_URL,
|
||||
NYC_CAMERA_ID_RE,
|
||||
DEFAULT_NYC_MAX_SOURCES,
|
||||
NYC_MAX_CATALOG_BYTES,
|
||||
NYC_GROUND_ELEVATION_M,
|
||||
NYC_ANCHORS,
|
||||
} from './constants.js';
|
||||
import {
|
||||
toFiniteNumber,
|
||||
@@ -82,13 +113,21 @@ import {
|
||||
isLikelyTexasCoordinate,
|
||||
isLikelyNswCoordinate,
|
||||
isLikelyCalgaryCoordinate,
|
||||
isLikelyCataloniaCoordinate,
|
||||
decodeNumericEntities,
|
||||
isLikelyNycCoordinate,
|
||||
isLikelyNorwayCoordinate,
|
||||
isLikelyTorontoCoordinate,
|
||||
cameraDisplayCode,
|
||||
rowArrayToObject,
|
||||
prioritizeSources,
|
||||
} from './normalize.js';
|
||||
import { directionToHeading } from '../../../src/data/directionText.js';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
import {
|
||||
readResponseBytesCapped,
|
||||
readResponseJsonCapped,
|
||||
readResponseTextCapped,
|
||||
} from '../common/http.js';
|
||||
/**
|
||||
* Fetch and parse Austin traffic camera records from the city Open Data portal.
|
||||
*
|
||||
@@ -1605,6 +1644,261 @@ export async function loadCalgarySourcesFromOpenData() {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the Catalonia (`cameres.xml`) WFS/GML feed into raw feature records.
|
||||
* Regex-based rather than a full XML parser — the feed is small (~160
|
||||
* features) and this avoids a new dependency for one field set per feature.
|
||||
*
|
||||
* @param {string} xml
|
||||
* @returns {Array<{fid:string, lat:number, lon:number, carretera:string, municipi:string, pk:string, link:string, font:string}>}
|
||||
*/
|
||||
export function parseCataloniaXml(xml) {
|
||||
const out = [];
|
||||
const blockRe = /<gml:featureMember>([\s\S]*?)<\/gml:featureMember>/g;
|
||||
const tag = (block, name) => {
|
||||
const match = new RegExp(`<cite:${name}>([^<]*)<\\/cite:${name}>`).exec(
|
||||
block,
|
||||
);
|
||||
return match ? decodeNumericEntities(match[1]) : '';
|
||||
};
|
||||
let match;
|
||||
while ((match = blockRe.exec(String(xml || ''))) !== null) {
|
||||
const block = match[1];
|
||||
const fidMatch = /<cite:cameres\s+fid="([^"]+)"/.exec(block);
|
||||
const coordsMatch = /<gml:coordinates[^>]*>([^<]+)<\/gml:coordinates>/.exec(
|
||||
block,
|
||||
);
|
||||
if (!fidMatch || !coordsMatch) continue;
|
||||
const [lonRaw, latRaw] = coordsMatch[1].split(',');
|
||||
out.push({
|
||||
fid: fidMatch[1],
|
||||
lon: toFiniteNumber(lonRaw),
|
||||
lat: toFiniteNumber(latRaw),
|
||||
carretera: tag(block, 'carretera'),
|
||||
municipi: tag(block, 'municipi'),
|
||||
pk: tag(block, 'pk'),
|
||||
link: tag(block, 'link'),
|
||||
font: tag(block, 'font'),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and normalize one Catalonia feed `link` against the publisher
|
||||
* host allowlist (CATALONIA_IMAGE_HOSTS), upgrading to https. Rejects
|
||||
* anything else, including a host that merely resembles an allowed one, any
|
||||
* URL carrying userinfo (`https://user:[email protected]/...`) or an explicit
|
||||
* port — the hostname check alone would pass either, letting catalog data
|
||||
* inject credentials or steer the server-side frame request to another
|
||||
* service on that host — and any path outside the host's frame directory.
|
||||
*
|
||||
* SCT's own `RenderService` frame link is a special case, for two stacked
|
||||
* reasons (both verified 2026-09-15):
|
||||
* 1. It always 302s to `TransitCamera`, on a Location the upstream
|
||||
* hardcodes as plain http even when RenderService itself was requested
|
||||
* over https. fetchWithinHost (media.js) only follows same-origin
|
||||
* redirects — same scheme included, by design, to stop an upstream
|
||||
* steering a host-pinned request into a plaintext downgrade — so that
|
||||
* hop is refused and the camera would render as a placeholder.
|
||||
* 2. mct.gencat.cat's TLS setup itself does not complete a handshake with
|
||||
* Node's fetch/OpenSSL (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though it
|
||||
* is reachable over https from other clients (e.g. curl) — an upstream
|
||||
* server misconfiguration this proxy cannot route around.
|
||||
* TransitCamera answers the frame directly over **http**, no redirect, and
|
||||
* that is the only transport this host actually serves to this runtime — so
|
||||
* this builds that URL itself from RenderService's `sctidcam` id instead of
|
||||
* registering the (https, then-redirecting) RenderService link. The
|
||||
* `sctidcam` id must be a bare image filename (CATALONIA_SCT_CAMERA_FILE_RE).
|
||||
*
|
||||
* @param {string} rawUrl
|
||||
* @returns {string} Normalized URL, or '' if not accepted.
|
||||
*/
|
||||
export function normalizeCataloniaImageUrl(rawUrl) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = new URL(String(rawUrl || '').trim());
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return '';
|
||||
if (parsed.username || parsed.password || parsed.port) return '';
|
||||
const dir = CATALONIA_IMAGE_HOSTS.get(parsed.hostname);
|
||||
if (!dir || !parsed.pathname.startsWith(dir)) return '';
|
||||
const file = parsed.pathname.slice(dir.length);
|
||||
if (!file || file.includes('/')) return '';
|
||||
|
||||
if (parsed.hostname === 'mct.gencat.cat') {
|
||||
const camId = parsed.searchParams.get('sctidcam');
|
||||
if (file !== 'RenderService') return '';
|
||||
if (!camId || !CATALONIA_SCT_CAMERA_FILE_RE.test(camId)) return '';
|
||||
const direct = new URL('http://mct.gencat.cat/mct2bo/TransitCamera');
|
||||
direct.searchParams.set('nom', camId);
|
||||
direct.searchParams.set('visualitzacio', 'imatge');
|
||||
return direct.toString();
|
||||
}
|
||||
|
||||
parsed.protocol = 'https:';
|
||||
return parsed.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable camera id for one normalized Catalonia frame URL (the output of
|
||||
* normalizeCataloniaImageUrl). The feed's `fid` is not used: it embeds an
|
||||
* upstream server-start stamp and changes on every upstream restart. SCT
|
||||
* cameras key on the `sctidcam` filename stem (`cat-sct-nc87`), partner
|
||||
* cameras on the frame host and path (`cat-www-bcn-cat-transit-imatges-…`).
|
||||
*
|
||||
* @param {string} imageUrl - A URL normalizeCataloniaImageUrl accepted.
|
||||
* @returns {string}
|
||||
*/
|
||||
export function cataloniaCameraId(imageUrl) {
|
||||
const parsed = new URL(imageUrl);
|
||||
const slug = (text) =>
|
||||
text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '');
|
||||
if (parsed.hostname === 'mct.gencat.cat') {
|
||||
const file = parsed.searchParams.get('nom') || '';
|
||||
return `cat-sct-${slug(file.replace(/\.[^.]*$/, ''))}`;
|
||||
}
|
||||
return `cat-${slug(parsed.hostname)}-${slug(parsed.pathname)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Human label for one Catalonia camera: "<road/place> (<municipality>)",
|
||||
* skipping the parenthetical when the municipality is already named in the
|
||||
* road/place text or when one half is missing.
|
||||
*
|
||||
* @param {string} carretera
|
||||
* @param {string} municipi
|
||||
* @returns {string}
|
||||
*/
|
||||
export function cataloniaCameraName(carretera, municipi) {
|
||||
const road = String(carretera || '').trim();
|
||||
const city = String(municipi || '').trim();
|
||||
if (road && city) {
|
||||
return road.toLowerCase().includes(city.toLowerCase())
|
||||
? road
|
||||
: `${road} (${city})`;
|
||||
}
|
||||
return road || city || 'Càmera de trànsit';
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Catalonia traffic cameras (Servei Català de Trànsit), keyless: one
|
||||
* WFS/GML XML list. The feed carries a `font` field naming the real
|
||||
* publisher per camera; only SCT's own highway cameras and the hotlinked
|
||||
* Barcelona (IMI) and Terrassa municipal cameras are kept — the publishers
|
||||
* whose frame links were confirmed to serve images. Frame URLs are checked
|
||||
* against a host allowlist (CATALONIA_IMAGE_HOSTS), so a row from any other
|
||||
* publisher is dropped. The feed carries no camera status, so a camera its
|
||||
* publisher has taken offline still registers and serves that publisher's
|
||||
* own "image unavailable" still.
|
||||
* Each non-SCT camera carries its publisher as a `credit` alongside the
|
||||
* shared `provider`. The feed carries no heading, so every camera takes the
|
||||
* id-hash fallback, low-confidence pose (same as TfL and Fintraffic).
|
||||
*
|
||||
* Ids come from the validated frame URL (cataloniaCameraId), not the feed's
|
||||
* restart-dependent `fid`, and rows sharing one are collapsed before the cap
|
||||
* (the live feed repeats a few SCT cameras).
|
||||
*
|
||||
* The list decides ids, coordinates, labels and which allowlisted frame path
|
||||
* is proxied, so it is fetched over https, redirects are refused (the list
|
||||
* host cannot be steered or downgraded) and the body read is capped at
|
||||
* CATALONIA_MAX_CATALOG_BYTES.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadCataloniaSourcesFromOpenData() {
|
||||
try {
|
||||
const resp = await fetch(CATALONIA_CAMERAS_URL, {
|
||||
headers: { Accept: 'application/xml,text/xml,*/*' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
// A response this loader will not read still owns its transport until the
|
||||
// body is released, so every rejection path cancels before returning.
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn(
|
||||
'[CCTV] Catalonia catalog redirected; redirects are not followed',
|
||||
);
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] Catalonia camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const xml = await readResponseTextCapped(resp, CATALONIA_MAX_CATALOG_BYTES);
|
||||
const rows = parseCataloniaXml(xml);
|
||||
if (!rows.length) return [];
|
||||
|
||||
const cameras = [];
|
||||
for (const row of rows) {
|
||||
if (!isLikelyCataloniaCoordinate(row.lat, row.lon)) continue;
|
||||
const imageUrl = normalizeCataloniaImageUrl(row.link);
|
||||
if (!imageUrl) continue;
|
||||
|
||||
const font = row.font.trim();
|
||||
const cameraId = cataloniaCameraId(imageUrl);
|
||||
|
||||
cameras.push({
|
||||
id: cameraId,
|
||||
name: cataloniaCameraName(row.carretera, row.municipi),
|
||||
city: row.municipi.trim() || 'Catalunya',
|
||||
cityId: 'catalunya',
|
||||
provider: 'Servei Català de Trànsit',
|
||||
lat: row.lat,
|
||||
lon: row.lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: CATALONIA_FONT_ELEVATION_M[font] ?? 80,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'catalonia-open-data',
|
||||
license:
|
||||
"Generalitat de Catalunya — Llicència oberta d'ús d'informació",
|
||||
credit: CATALONIA_FONT_CREDIT[font] || '',
|
||||
});
|
||||
}
|
||||
|
||||
const unique = Array.from(
|
||||
new Map(cameras.map((camera) => [camera.id, camera])).values(),
|
||||
);
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_CATALONIA_MAX_SOURCES || DEFAULT_CATALONIA_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(600, Math.floor(maxRaw)))
|
||||
: DEFAULT_CATALONIA_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(unique, maxCount, CATALONIA_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded Catalonia camera sources: ${unique.length} (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] Catalonia camera download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadDelDOTSourcesFromOpenData() {
|
||||
try {
|
||||
const resp = await fetch(DELDOT_CCTV_URL, {
|
||||
@@ -1709,6 +2003,319 @@ export async function loadDelDOTSourcesFromOpenData() {
|
||||
}
|
||||
}
|
||||
|
||||
/** Rough King County, WA bounding box. */
|
||||
function isLikelyKingCountyCoordinate(lat, lon) {
|
||||
return (
|
||||
Number.isFinite(lat) &&
|
||||
Number.isFinite(lon) &&
|
||||
lat >= 47.0 &&
|
||||
lat <= 48.0 &&
|
||||
lon >= -123.1 &&
|
||||
lon <= -121.0
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A King County `Model` value, whitespace-collapsed, or '' when it is a
|
||||
* vendor-generic string ("PTZ", "PTZ IP", "BOSCH/PTZ") that names no model.
|
||||
*/
|
||||
function kingCountyModel(raw) {
|
||||
const model = String(raw || '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
return /^(?:[a-z]+\/)?ptz(?: ip)?$/i.test(model) ? '' : model;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch King County road cameras (Washington State). Keyless: one public
|
||||
* ArcGIS FeatureServer GeoJSON query (King County DOT Road Services), with an
|
||||
* explicit `outFields` list so the layer's asset-administration columns are
|
||||
* never fetched. Only cameras that are live and active (`Live`,
|
||||
* `CurrentStatus`) with a positive integer id, finite in-county coordinates,
|
||||
* and a frame on the county's own image handler are kept. WSDOT-owned rows in
|
||||
* the layer are skipped because their frames are WSDOT-hosted, outside the
|
||||
* pinned image origin, while partner-city cameras hosted on the county image
|
||||
* handler (e.g. Newcastle) name their owner as the per-camera credit,
|
||||
* DriveBC-style. The layer publishes no facing, so every heading is the id
|
||||
* hash at low confidence.
|
||||
*
|
||||
* Uniquely among the packs, the county publishes each camera's hardware
|
||||
* `Manufacturer`/`Model`; the model string (whitespace-collapsed — the feed
|
||||
* carries stray newlines) rides along as the source's `model` for
|
||||
* identified-hardware consumers. Vendor-generic strings ("PTZ", "PTZ IP",
|
||||
* "BOSCH/PTZ") name no model and are treated as absent.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadKingCountySourcesFromOpenData() {
|
||||
try {
|
||||
const resp = await fetch(KING_COUNTY_CAMERAS_URL, {
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
// A response this loader will not read still owns its transport until the
|
||||
// body is released, so every rejection path cancels before returning.
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn(
|
||||
'[CCTV] King County catalog redirected; redirects are not followed',
|
||||
);
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] King County camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const payload = await readResponseJsonCapped(
|
||||
resp,
|
||||
KING_COUNTY_MAX_CATALOG_BYTES,
|
||||
);
|
||||
const rows = Array.isArray(payload?.features) ? payload.features : [];
|
||||
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const row of rows) {
|
||||
const attrs = row?.properties;
|
||||
const rawId = attrs?.AssetID;
|
||||
if (!Number.isSafeInteger(rawId) || rawId <= 0) continue;
|
||||
if (attrs.Live !== 1) continue;
|
||||
if (String(attrs.CurrentStatus || '').trim() !== 'Active') continue;
|
||||
const owner = String(attrs.Owner || '').trim();
|
||||
// WSDOT-owned rows serve WSDOT-hosted frames, outside the pinned
|
||||
// county image origin, so they are skipped.
|
||||
if (owner === 'WSDOT') continue;
|
||||
// The handler is served over HTTPS; the feed still writes http://.
|
||||
const imageUrl = String(attrs.ImageURL || '')
|
||||
.trim()
|
||||
.replace(/^http:\/\/info\.kingcounty\.gov\//, KING_COUNTY_IMAGE_ORIGIN);
|
||||
if (!imageUrl.startsWith(KING_COUNTY_IMAGE_ORIGIN)) continue;
|
||||
const [lon, lat] = Array.isArray(row?.geometry?.coordinates)
|
||||
? row.geometry.coordinates
|
||||
: [];
|
||||
if (!isLikelyKingCountyCoordinate(lat, lon)) continue;
|
||||
|
||||
const cameraId = `kingcounty-${rawId}`;
|
||||
if (seen.has(cameraId)) continue;
|
||||
seen.add(cameraId);
|
||||
const model = kingCountyModel(attrs.Model);
|
||||
const manufacturer = String(attrs.Manufacturer || '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
const camera = {
|
||||
id: cameraId,
|
||||
name:
|
||||
String(attrs.Location || '').trim() || `King County camera ${rawId}`,
|
||||
city: 'King County',
|
||||
cityId: 'king-county',
|
||||
provider: 'King County Road Services',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
// The low-confidence pose personality shared by the other packs.
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'kingcounty-open-data',
|
||||
license: 'King County GIS open data (KCGIS Center terms of use)',
|
||||
credit: owner && owner !== 'King County' ? owner : '',
|
||||
};
|
||||
if (model) camera.model = model;
|
||||
if (model && manufacturer) camera.manufacturer = manufacturer;
|
||||
cameras.push(camera);
|
||||
}
|
||||
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_KINGCOUNTY_MAX_SOURCES ||
|
||||
DEFAULT_KING_COUNTY_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(1200, Math.floor(maxRaw)))
|
||||
: DEFAULT_KING_COUNTY_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(
|
||||
cameras,
|
||||
maxCount,
|
||||
KING_COUNTY_ANCHORS,
|
||||
);
|
||||
console.log(
|
||||
`[CCTV] Loaded King County camera sources: ${cameras.length} published (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] King County camera download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** WSDOT `CompassDirection` codes as headings in degrees. The live feed sends
|
||||
* single cardinals (N/E/S/W), `B` (both directions — most rows), `O`, and
|
||||
* null; the diagonals are mapped in case they appear. `B`, `O`, null, and any
|
||||
* other code carry no single facing and fall through to the id hash. */
|
||||
const WSDOT_COMPASS_HEADINGS = Object.freeze({
|
||||
N: 0,
|
||||
NE: 45,
|
||||
E: 90,
|
||||
SE: 135,
|
||||
S: 180,
|
||||
SW: 225,
|
||||
W: 270,
|
||||
NW: 315,
|
||||
});
|
||||
|
||||
/**
|
||||
* Converts an EPSG:3857 (Web Mercator) coordinate pair to WGS84 degrees.
|
||||
* The WSDOT camera layer publishes point geometry in wkid 102100.
|
||||
* @param {number} x - Web Mercator metres east.
|
||||
* @param {number} y - Web Mercator metres north.
|
||||
* @returns {{lat: number, lon: number}} WGS84 degrees.
|
||||
*/
|
||||
export function webMercatorToWgs84(x, y) {
|
||||
const R = 6378137;
|
||||
const lon = (x / R) * (180 / Math.PI);
|
||||
const lat = (2 * Math.atan(Math.exp(y / R)) - Math.PI / 2) * (180 / Math.PI);
|
||||
return { lat, lon };
|
||||
}
|
||||
|
||||
/** Rough Washington State bounding box (with the Columbia River border cams). */
|
||||
function isLikelyWashingtonCoordinate(lat, lon) {
|
||||
return (
|
||||
Number.isFinite(lat) &&
|
||||
Number.isFinite(lon) &&
|
||||
lat >= 45.3 &&
|
||||
lat <= 49.3 &&
|
||||
lon >= -125 &&
|
||||
lon <= -116.5
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch WSDOT highway cameras (Washington State). Keyless: one Esri JSON
|
||||
* layer on WSDOT's official open-data host (the same layer the WSDOT Travel
|
||||
* Center map consumes; a keyed Traveler Information API serves the identical
|
||||
* population for higher-volume use). Only cameras with a positive integer id,
|
||||
* finite in-state coordinates, and a frame on WSDOT's own image host are
|
||||
* kept — the catalog's ~70 partner cameras (Oregon DOT, City of Seattle,
|
||||
* lodges, airports) are skipped rather than proxied under someone else's
|
||||
* terms. A single-facing `CompassDirection` (a cardinal) gives a
|
||||
* high-confidence heading; `B` (both directions), `O`, and null fall back to
|
||||
* the id hash.
|
||||
* The feed carries Windows-1252 bytes (en-dashes in titles) with no charset
|
||||
* declaration, so the body is decoded explicitly before parsing.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadWsdotSourcesFromOpenData() {
|
||||
try {
|
||||
const resp = await fetch(WSDOT_CAMERAS_URL, {
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn(
|
||||
'[CCTV] WSDOT catalog redirected; redirects are not followed',
|
||||
);
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] WSDOT camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const payload = JSON.parse(
|
||||
new TextDecoder('windows-1252').decode(
|
||||
await readResponseBytesCapped(resp, WSDOT_MAX_CATALOG_BYTES),
|
||||
),
|
||||
);
|
||||
const rows = Array.isArray(payload?.features) ? payload.features : [];
|
||||
|
||||
const cameras = [];
|
||||
for (const row of rows) {
|
||||
const attrs = row?.attributes;
|
||||
const rawId = attrs?.CameraID;
|
||||
if (!Number.isSafeInteger(rawId) || rawId <= 0) continue;
|
||||
const imageUrl = String(attrs.ImageURL || '').trim();
|
||||
if (!imageUrl.startsWith(WSDOT_IMAGE_ORIGIN)) continue;
|
||||
const x = row?.geometry?.x;
|
||||
const y = row?.geometry?.y;
|
||||
if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
|
||||
const { lat, lon } = webMercatorToWgs84(x, y);
|
||||
if (!isLikelyWashingtonCoordinate(lat, lon)) continue;
|
||||
|
||||
const cameraId = `wsdot-${rawId}`;
|
||||
const heading =
|
||||
WSDOT_COMPASS_HEADINGS[
|
||||
String(attrs.CompassDirection || '')
|
||||
.trim()
|
||||
.toUpperCase()
|
||||
];
|
||||
const hasHeading = Number.isFinite(heading);
|
||||
cameras.push({
|
||||
id: cameraId,
|
||||
name: String(attrs.CameraTitle || '').trim() || `WSDOT camera ${rawId}`,
|
||||
city: 'Washington',
|
||||
cityId: 'washington-state',
|
||||
provider: 'WSDOT',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: hasHeading ? 'high' : 'low',
|
||||
// Same two pose personalities as the other packs: raw priors that the
|
||||
// client's ground snap and manual calibration refine.
|
||||
pitchDeg: hasHeading ? -24 : -18,
|
||||
fovDeg: hasHeading ? 56 : 44,
|
||||
rangeM: hasHeading ? 210 : 145,
|
||||
mountHeightM: hasHeading ? 10 : 8,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'wsdot-open-data',
|
||||
license: 'WSDOT Travel Center camera data (public low-volume feed)',
|
||||
});
|
||||
}
|
||||
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_WSDOT_MAX_SOURCES || DEFAULT_WSDOT_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(1200, Math.floor(maxRaw)))
|
||||
: DEFAULT_WSDOT_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(cameras, maxCount, WSDOT_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded WSDOT camera sources: ${cameras.length} published (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] WSDOT camera download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One Statens vegvesen `CctvSimple` feature -> one catalog source, or null.
|
||||
*
|
||||
@@ -1863,3 +2470,317 @@ export async function loadVegvesenSourcesFromOpenData() {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
const NYC_BOROUGHS = new Set([
|
||||
'Manhattan',
|
||||
'Brooklyn',
|
||||
'Queens',
|
||||
'Bronx',
|
||||
'Staten Island',
|
||||
]);
|
||||
|
||||
/** NYC TMC publishes `isOnline` as the strings "true"/"false". */
|
||||
export function nycCameraIsOnline(value) {
|
||||
return (
|
||||
value === true ||
|
||||
String(value ?? '')
|
||||
.trim()
|
||||
.toLowerCase() === 'true'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Label for one NYC camera: the intersection an operator recognises
|
||||
* ("Central Park West @ 86 St"). The catalog's `area` is the borough, not
|
||||
* a facing, and is not part of the name.
|
||||
*
|
||||
* @param {object} record - Raw TMC camera row.
|
||||
* @param {string} cameraId - Derived stable id.
|
||||
* @returns {string}
|
||||
*/
|
||||
export function nycCameraName(record, cameraId) {
|
||||
const name = String(record?.name ?? '').trim();
|
||||
if (name) return name;
|
||||
const short = String(cameraId || '')
|
||||
.replace(/^nyc-/, '')
|
||||
.slice(0, 8);
|
||||
return short ? `NYC Camera ${short}` : 'NYC Camera';
|
||||
}
|
||||
|
||||
/**
|
||||
* One NYC DOT TMC camera row -> one catalog source, or null.
|
||||
*
|
||||
* Frame URLs are synthesized from a validated UUID on the pinned TMC image
|
||||
* origin. The payload's `imageUrl` is ignored so a catalog edit cannot steer
|
||||
* the frame proxy off-host. There is no heading in the feed; headings use
|
||||
* the shared id-hash fallback at low confidence, like Calgary and Fintraffic.
|
||||
*
|
||||
* @param {object} record - Raw TMC camera row.
|
||||
* @returns {?object}
|
||||
*/
|
||||
export function nycCameraToSource(record) {
|
||||
if (!record || typeof record !== 'object') return null;
|
||||
if (!nycCameraIsOnline(record.isOnline)) return null;
|
||||
|
||||
const lat = toFiniteNumber(record.latitude);
|
||||
const lon = toFiniteNumber(record.longitude);
|
||||
if (!isLikelyNycCoordinate(lat, lon)) return null;
|
||||
|
||||
const uuid = String(record.id ?? '').trim();
|
||||
if (!NYC_CAMERA_ID_RE.test(uuid)) return null;
|
||||
const cameraId = `nyc-${uuid.toLowerCase()}`;
|
||||
const imageUrl = NYC_IMAGE_URL(uuid);
|
||||
const name = nycCameraName(record, cameraId);
|
||||
const area = String(record.area ?? '').trim();
|
||||
const city = NYC_BOROUGHS.has(area) ? area : 'New York';
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name,
|
||||
city,
|
||||
cityId: 'nyc',
|
||||
provider: 'NYC DOT Traffic Management Center',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: NYC_GROUND_ELEVATION_M,
|
||||
feedType: 'image',
|
||||
url: imageUrl,
|
||||
snapshotUrl: imageUrl,
|
||||
sourceKind: 'nyc-dot-tmc',
|
||||
license: 'Public NYC DOT Traffic Management Center traffic camera',
|
||||
code: cameraDisplayCode(name.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch NYC DOT Traffic Management Center cameras from the keyless
|
||||
* `webcams.nyctmc.org` catalog. Frames are stills on that same host,
|
||||
* registered from a validated UUID rather than from the payload URL.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadNycSourcesFromOpenData() {
|
||||
try {
|
||||
const endpoint = process.env.CCTV_NYC_CAMERAS_URL || NYC_CAMERAS_URL;
|
||||
const resp = await fetch(endpoint, {
|
||||
headers: { Accept: 'application/json' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn('[CCTV] NYC catalog redirected; redirects are not followed');
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] NYC camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const rows = await readResponseJsonCapped(resp, NYC_MAX_CATALOG_BYTES);
|
||||
if (!Array.isArray(rows)) return [];
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const record of rows) {
|
||||
const camera = nycCameraToSource(record);
|
||||
if (!camera || seen.has(camera.id)) continue;
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_NYC_MAX_SOURCES || DEFAULT_NYC_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(400, Math.floor(maxRaw)))
|
||||
: DEFAULT_NYC_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(cameras, maxCount, NYC_ANCHORS);
|
||||
console.log(
|
||||
`[CCTV] Loaded NYC camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn('[CCTV] NYC camera download error:', error?.message || error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The JSONP envelope the City of Toronto wraps its camera list in. Anchored at
|
||||
* both ends, so only that exact callback is unwrapped — a payload that merely
|
||||
* contains the name somewhere is not treated as the list.
|
||||
*/
|
||||
const RESCU_JSONP = /^jsonTMCEarthCamerasCallback\(([\s\S]*)\)\s*;?$/;
|
||||
|
||||
/**
|
||||
* Unwrap the RESCU camera list. The portal serves the list as JSONP; should it
|
||||
* ever drop the wrapper, the bare JSON document still parses. Anything else —
|
||||
* a foreign callback, a truncated body, an error page — yields no cameras
|
||||
* rather than a thrown loader.
|
||||
*
|
||||
* @param {string} text - Raw response body.
|
||||
* @returns {Array<object>} The `Data` rows, or [] when the body is unusable.
|
||||
*/
|
||||
export function parseRescuCameraList(text) {
|
||||
const body = String(text ?? '').trim();
|
||||
if (!body) return [];
|
||||
const unwrapped = RESCU_JSONP.exec(body)?.[1] ?? body;
|
||||
try {
|
||||
const parsed = JSON.parse(unwrapped);
|
||||
const rows = parsed?.Data ?? parsed?.data;
|
||||
return Array.isArray(rows) ? rows : [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert one RESCU list row into a camera source.
|
||||
*
|
||||
* The row carries no URL: the frame URL is built from the documented
|
||||
* `loc####.jpg` convention and a camera number validated as plain digits, so
|
||||
* the frame proxy is pinned to the city origin by construction and no upstream
|
||||
* field — present or planted — can steer it.
|
||||
*
|
||||
* No compass heading exists in this dataset. `D1`-`D4` are the directions of
|
||||
* the *comparison* images (static reference stills captured in 2016, served
|
||||
* from a different path); the live frame is one pan-tilt-zoom camera pointing
|
||||
* wherever the operator left it. Treating those letters as a facing would put
|
||||
* a confident, wrong bearing on every camera in the city, so every row takes
|
||||
* the id-hash fallback and the low-confidence pose, as headingless Austin,
|
||||
* TfL, Fintraffic and Calgary cameras do.
|
||||
*
|
||||
* @param {object} row - One entry from the RESCU `Data` array.
|
||||
* @returns {object|null} Normalized camera source, or null when unusable.
|
||||
*/
|
||||
export function torontoCameraToSource(row) {
|
||||
if (!row || typeof row !== 'object') return null;
|
||||
const number = String(row.Number ?? row.number ?? '').trim();
|
||||
// Plain digits only: a number is about to become a URL path segment.
|
||||
if (!/^\d{1,6}$/.test(number)) return null;
|
||||
|
||||
const lat = toFiniteNumber(row.Latitude ?? row.latitude);
|
||||
const lon = toFiniteNumber(row.Longitude ?? row.longitude);
|
||||
if (!isLikelyTorontoCoordinate(lat, lon)) return null;
|
||||
|
||||
const cameraId = `toronto-${number}`;
|
||||
// The feed's own label, verbatim. It is upper-case ("MCCOWAN RD", "ST
|
||||
// HELEN'S AVE"); title-casing it would mangle more names than it fixed.
|
||||
const name =
|
||||
String(row.Name ?? row.name ?? '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim() || `Toronto Camera ${number}`;
|
||||
|
||||
return {
|
||||
id: cameraId,
|
||||
name,
|
||||
city: 'Toronto',
|
||||
cityId: 'toronto',
|
||||
provider: 'City of Toronto',
|
||||
lat,
|
||||
lon,
|
||||
headingDeg: fallbackHeadingFromId(cameraId),
|
||||
headingConfidence: 'low',
|
||||
pitchDeg: -18,
|
||||
fovDeg: 44,
|
||||
rangeM: 145,
|
||||
mountHeightM: 8,
|
||||
groundElevationM: TORONTO_GROUND_ELEVATION_M,
|
||||
feedType: 'image',
|
||||
url: `${TORONTO_RESCU_IMAGE_ORIGIN}loc${number}.jpg`,
|
||||
snapshotUrl: `${TORONTO_RESCU_IMAGE_ORIGIN}loc${number}.jpg`,
|
||||
sourceKind: 'toronto-rescu-open-data',
|
||||
license:
|
||||
'Contains information licensed under the Open Government Licence – Toronto',
|
||||
// Unselected-label code: the intersection, so a camera at rest reads as a
|
||||
// place rather than as its id.
|
||||
code: cameraDisplayCode(name.toUpperCase()),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch City of Toronto RESCU traffic cameras from the City's open data
|
||||
* portal, keyless. One JSONP list for the whole city; frames are stills on the
|
||||
* same portal origin.
|
||||
*
|
||||
* The list fetch refuses redirects (`redirect: 'manual'`) so the list host
|
||||
* cannot be steered, and caps the body it will buffer.
|
||||
*
|
||||
* Attribution: Open Government Licence – Toronto, registered in
|
||||
* src/data/dataCredits.js.
|
||||
*
|
||||
* @returns {Promise<Array<object>>} Normalized camera source objects.
|
||||
*/
|
||||
export async function loadTorontoSourcesFromOpenData() {
|
||||
try {
|
||||
const endpoint =
|
||||
process.env.CCTV_TORONTO_LIST_URL || TORONTO_RESCU_LIST_URL;
|
||||
const resp = await fetch(endpoint, {
|
||||
headers: { Accept: 'application/json, text/javascript' },
|
||||
redirect: 'manual',
|
||||
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
|
||||
});
|
||||
// A response this loader will not read still owns its transport until the
|
||||
// body is released, so every rejection path cancels before returning.
|
||||
const discard = async () => {
|
||||
try {
|
||||
await resp.body?.cancel();
|
||||
} catch {
|
||||
/* no-op */
|
||||
}
|
||||
return [];
|
||||
};
|
||||
if (resp.status >= 300 && resp.status < 400) {
|
||||
console.warn(
|
||||
'[CCTV] Toronto camera list redirected; redirects are not followed',
|
||||
);
|
||||
return discard();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
console.warn('[CCTV] Toronto camera download failed:', resp.status);
|
||||
return discard();
|
||||
}
|
||||
const text = await readResponseTextCapped(resp, TORONTO_MAX_CATALOG_BYTES);
|
||||
const rows = parseRescuCameraList(text);
|
||||
if (!rows.length) return [];
|
||||
|
||||
const cameras = [];
|
||||
const seen = new Set();
|
||||
for (const record of rows) {
|
||||
const camera = torontoCameraToSource(record);
|
||||
if (!camera || seen.has(camera.id)) continue;
|
||||
seen.add(camera.id);
|
||||
cameras.push(camera);
|
||||
}
|
||||
const maxRaw = Number(
|
||||
process.env.CCTV_TORONTO_MAX_SOURCES || DEFAULT_TORONTO_MAX_SOURCES,
|
||||
);
|
||||
const maxCount = Number.isFinite(maxRaw)
|
||||
? Math.max(8, Math.min(500, Math.floor(maxRaw)))
|
||||
: DEFAULT_TORONTO_MAX_SOURCES;
|
||||
const prioritized = prioritizeSources(cameras, maxCount, [
|
||||
TORONTO_DOWNTOWN,
|
||||
]);
|
||||
console.log(
|
||||
`[CCTV] Loaded Toronto camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
|
||||
);
|
||||
return prioritized;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[CCTV] Toronto camera download error:',
|
||||
error?.message || error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,9 @@ function number(value, min, max) {
|
||||
const n = Number(value);
|
||||
return Number.isFinite(n) && n >= min && n <= max ? n : null;
|
||||
}
|
||||
// NHC stamps an advisory with its nominal time (e.g. 03:00Z) and publishes it
|
||||
// up to about half an hour before; allow that, but not a clock far ahead.
|
||||
const EARLY_ADVISORY_MS = 90 * 60_000;
|
||||
function iso(value, now) {
|
||||
if (
|
||||
typeof value !== 'string' ||
|
||||
@@ -50,7 +53,7 @@ function iso(value, now) {
|
||||
const n = Date.parse(value);
|
||||
if (
|
||||
!Number.isFinite(n) ||
|
||||
n > now + 5 * 60_000 ||
|
||||
n > now + EARLY_ADVISORY_MS ||
|
||||
now - n > 12 * HOUR ||
|
||||
new Date(n).toISOString().replace('.000Z', 'Z') !==
|
||||
value.replace('.000Z', 'Z')
|
||||
|
||||
@@ -2,6 +2,7 @@ import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
|
||||
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
|
||||
import { readResponseTextCapped } from './common/http.js';
|
||||
|
||||
/**
|
||||
* NASA FIRMS live active-fire proxy with a memory + disk cache.
|
||||
@@ -35,6 +36,12 @@ export function firmsProxy() {
|
||||
'VIIRS_SNPP_NRT',
|
||||
'MODIS_NRT',
|
||||
];
|
||||
/**
|
||||
* Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
|
||||
* 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
|
||||
* bound, not a budget.
|
||||
*/
|
||||
const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024;
|
||||
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
|
||||
const CACHE_PATH = path.join(CACHE_DIR, 'firms.json');
|
||||
|
||||
@@ -83,9 +90,12 @@ export function firmsProxy() {
|
||||
*/
|
||||
async function fetchSource(key, source) {
|
||||
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
|
||||
const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
|
||||
const signal = AbortSignal.timeout(60_000);
|
||||
const res = await fetch(url, { signal });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const records = parseFirmsCsv(await res.text());
|
||||
const records = parseFirmsCsv(
|
||||
await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
|
||||
);
|
||||
if (records === null) throw new Error('non-CSV upstream response');
|
||||
return records;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
isAllowedGbfsHost,
|
||||
isAllowedGbfsPath,
|
||||
gbfsCacheControl,
|
||||
gbfsUpstreamHeaders,
|
||||
} from '../../src/data/gbfsSource.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -11,6 +12,9 @@ import {
|
||||
/** Upstream fetch timeout for GBFS requests (ms). */
|
||||
const GBFS_PROXY_TIMEOUT_MS = 12000;
|
||||
|
||||
/** Value of the Client-Identifier header sent with every upstream request. */
|
||||
export const GBFS_CLIENT_IDENTIFIER = 'gods-eye-view';
|
||||
|
||||
export const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB
|
||||
|
||||
function gbfsRedirectHost(location, requestUrl) {
|
||||
@@ -60,6 +64,10 @@ export async function fetchGbfsUpstream(
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': 'gods-eye-view-gbfs-proxy/1.0',
|
||||
// Urban Sharing (the Norwegian city-bike feeds) asks every client to
|
||||
// name itself in this header; other operators ignore it.
|
||||
'Client-Identifier': GBFS_CLIENT_IDENTIFIER,
|
||||
...gbfsUpstreamHeaders(new URL(url).hostname),
|
||||
},
|
||||
redirect: 'manual',
|
||||
signal: controller.signal,
|
||||
@@ -181,7 +189,7 @@ export function gbfsProxy() {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isAllowedGbfsPath(upstreamUrl.pathname)) {
|
||||
if (!isAllowedGbfsPath(upstreamUrl.pathname, upstreamUrl.hostname)) {
|
||||
res.writeHead(400, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import {
|
||||
GNSS_CLASSIFIER,
|
||||
GNSS_WINDOW_MS,
|
||||
normalizeGnssAircraft,
|
||||
} from '../../src/layers/gnss/records.js';
|
||||
import { adsbLolFallbackAnchor } from './aircraft/opensky.js';
|
||||
import { coalesceProxyRequest, readResponseJsonCapped } from './common/http.js';
|
||||
import { makeRateLimiter, clientKey } from './common/rate-limit.js';
|
||||
|
||||
// adsb.lol regional snapshot (keyless, ODbL). Only the integrity fields the
|
||||
// GNSS layer needs leave this proxy, so the browser never sees the full feed.
|
||||
// This is a separate read from the flights fallback on purpose: that point
|
||||
// cache keeps only the fields the flights layer renders (no nic / nac_p), is
|
||||
// keyed on 0.25° anchors and lives for 12 s, while integrity cells need
|
||||
// whole-degree anchors held for a minute.
|
||||
const RADIUS_NM = 250;
|
||||
const CACHE_MS = 60_000;
|
||||
const CACHE_MAX = 64;
|
||||
/** Default 429 cooldown when upstream sends no usable Retry-After. */
|
||||
const COOLDOWN_MS = 60_000;
|
||||
/** Bound sane upstream deadlines to one hour; never shorten a deadline within that range. */
|
||||
const COOLDOWN_MIN_MS = 5_000;
|
||||
const COOLDOWN_MAX_MS = 60 * 60_000;
|
||||
const FAILURE_COOLDOWN_MS = 15_000;
|
||||
const MAX_IN_FLIGHT = 8;
|
||||
const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
const TIMEOUT_MS = 10_000;
|
||||
|
||||
/** Snap a view anchor to a whole degree so nearby views share one upstream read. */
|
||||
export function gnssIntegrityAnchor(url) {
|
||||
const anchor = adsbLolFallbackAnchor({ url });
|
||||
if (!anchor) return null;
|
||||
return {
|
||||
lat: Math.round(anchor.latitude),
|
||||
lon: Math.round(anchor.longitude),
|
||||
};
|
||||
}
|
||||
|
||||
/** Cooldown (ms) an upstream 429 earns, honouring Retry-After when sane. */
|
||||
export function gnssRetryCooldownMs(raw, nowMs) {
|
||||
if (raw) {
|
||||
const seconds = /^\d+(?:\.\d+)?$/.test(raw.trim()) ? Number(raw) : NaN;
|
||||
const ms = seconds * 1000;
|
||||
if (Number.isFinite(seconds) && seconds > 0)
|
||||
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, ms));
|
||||
const at =
|
||||
Number.isNaN(seconds) && /[a-z]/i.test(raw) ? Date.parse(raw) : NaN;
|
||||
if (Number.isFinite(at) && at > nowMs)
|
||||
return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, at - nowMs));
|
||||
}
|
||||
return COOLDOWN_MS;
|
||||
}
|
||||
|
||||
/** Fixed-origin, bounded adsb.lol integrity route for dev and preview. */
|
||||
export function gnssIntegrityProxy({
|
||||
fetchImpl = (...args) => globalThis.fetch(...args),
|
||||
now = () => Date.now(),
|
||||
monotonicNow = () => performance.now(),
|
||||
warn = (...args) => console.warn(...args),
|
||||
} = {}) {
|
||||
const cache = new Map();
|
||||
const inFlight = new Map();
|
||||
const allow = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 600 });
|
||||
let cooldownUntil = 0;
|
||||
let cooldownStatus = 429;
|
||||
|
||||
async function load({ lat, lon }) {
|
||||
if (monotonicNow() < cooldownUntil)
|
||||
throw Object.assign(new Error('cooling_down'), {
|
||||
status: cooldownStatus,
|
||||
});
|
||||
const signal = AbortSignal.timeout(TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetchImpl(
|
||||
`https://api.adsb.lol/v2/lat/${lat}/lon/${lon}/dist/${RADIUS_NM}`,
|
||||
{
|
||||
signal,
|
||||
redirect: 'error',
|
||||
headers: {
|
||||
Accept: 'application/json',
|
||||
'User-Agent': 'gods-eye-view-gnss-integrity/1.0',
|
||||
},
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
if (response.status === 429) {
|
||||
const at = monotonicNow();
|
||||
const raw = response.headers.get('retry-after');
|
||||
const delay = gnssRetryCooldownMs(raw, now());
|
||||
if (delay === COOLDOWN_MAX_MS)
|
||||
warn('[gnss-integrity] upstream cooldown capped at one hour');
|
||||
cooldownStatus = 429;
|
||||
cooldownUntil = Math.max(cooldownUntil, at + delay);
|
||||
throw Object.assign(new Error('upstream_rate_limited'), {
|
||||
status: 429,
|
||||
});
|
||||
}
|
||||
throw new Error('upstream_unavailable');
|
||||
}
|
||||
const payload = await readResponseJsonCapped(
|
||||
response,
|
||||
MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
const rows = normalizeGnssAircraft(payload);
|
||||
if (!rows) throw new Error('invalid_snapshot');
|
||||
const sourceNow = payload.now;
|
||||
if (!Number.isFinite(sourceNow) || sourceNow <= 0)
|
||||
throw new Error('undated_snapshot');
|
||||
const fetchedAt = Math.min(
|
||||
now(),
|
||||
sourceNow > 10_000_000_000 ? sourceNow : sourceNow * 1000,
|
||||
);
|
||||
if (now() - fetchedAt > GNSS_WINDOW_MS)
|
||||
throw new Error('expired_snapshot');
|
||||
return {
|
||||
fetchedAt,
|
||||
anchor: { lat, lon },
|
||||
radiusNm: RADIUS_NM,
|
||||
// `degraded` on each row is this classifier's verdict, not gpsjam's.
|
||||
classifier: GNSS_CLASSIFIER.id,
|
||||
rows,
|
||||
};
|
||||
} catch (error) {
|
||||
if (error.status !== 429) {
|
||||
const until = monotonicNow() + FAILURE_COOLDOWN_MS;
|
||||
if (until > cooldownUntil) {
|
||||
cooldownUntil = until;
|
||||
cooldownStatus = 502;
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function age(entry) {
|
||||
if (!entry) return Infinity;
|
||||
const elapsed = monotonicNow() - entry.savedAt;
|
||||
// A backwards injected clock cannot rejuvenate a snapshot.
|
||||
return elapsed < 0 ? Infinity : entry.ageAtReceipt + elapsed;
|
||||
}
|
||||
|
||||
async function acquire(anchor, peer) {
|
||||
const key = `${anchor.lat},${anchor.lon}`;
|
||||
const previous = cache.get(key);
|
||||
if (age(previous) < CACHE_MS)
|
||||
return { value: previous.value, ageMs: age(previous), stale: false };
|
||||
try {
|
||||
if (!inFlight.has(key) && inFlight.size >= MAX_IN_FLIGHT)
|
||||
throw Object.assign(new Error('busy'), { status: 429, local: true });
|
||||
const { promise } = coalesceProxyRequest(inFlight, key, async () => {
|
||||
if (monotonicNow() >= cooldownUntil && !allow(peer))
|
||||
throw Object.assign(new Error('rate_limited'), {
|
||||
status: 429,
|
||||
local: true,
|
||||
});
|
||||
const value = await load(anchor);
|
||||
const entry = {
|
||||
value,
|
||||
savedAt: monotonicNow(),
|
||||
ageAtReceipt: Math.max(0, now() - value.fetchedAt),
|
||||
};
|
||||
cache.delete(key);
|
||||
cache.set(key, entry);
|
||||
while (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value);
|
||||
return entry;
|
||||
});
|
||||
const entry = await promise;
|
||||
const ageMs = age(entry);
|
||||
return { value: entry.value, ageMs, stale: ageMs >= CACHE_MS };
|
||||
} catch (error) {
|
||||
if (age(previous) <= GNSS_WINDOW_MS)
|
||||
return { value: previous.value, ageMs: age(previous), stale: true };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
const json = (status, value, stale = false, retryAfterS = 60) => {
|
||||
if (res.destroyed) return;
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Cache-Control': 'no-store',
|
||||
...(status === 405 ? { Allow: 'GET' } : {}),
|
||||
...([429, 502].includes(status)
|
||||
? { 'Retry-After': String(retryAfterS) }
|
||||
: {}),
|
||||
...(stale ? { 'X-Data-Stale': 'true' } : {}),
|
||||
});
|
||||
res.end(JSON.stringify(value));
|
||||
};
|
||||
if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' });
|
||||
const path = (req.url || '/').split('?')[0];
|
||||
if (path !== '/' && path !== '')
|
||||
return json(404, { error: 'unknown_route' });
|
||||
const anchor = gnssIntegrityAnchor(req.url);
|
||||
if (!anchor) return json(400, { error: 'invalid_anchor' });
|
||||
try {
|
||||
const { value, stale, ageMs } = await acquire(anchor, clientKey(req));
|
||||
// ageMs is measured on this server's clock, so the browser can date
|
||||
// the rows without comparing its clock to ours.
|
||||
json(200, { ...value, ageMs, ...(stale ? { stale: true } : {}) }, stale);
|
||||
} catch (error) {
|
||||
if (error.status === 429) {
|
||||
// Tell the browser how long the shared upstream cooldown has left.
|
||||
const retryAfterS = Math.max(
|
||||
1,
|
||||
error.local ? 60 : Math.ceil((cooldownUntil - monotonicNow()) / 1000),
|
||||
);
|
||||
return json(
|
||||
429,
|
||||
{
|
||||
error: error.local ? error.message : 'gnss_integrity_rate_limited',
|
||||
},
|
||||
false,
|
||||
retryAfterS,
|
||||
);
|
||||
}
|
||||
json(
|
||||
502,
|
||||
{ error: 'gnss_integrity_unavailable' },
|
||||
false,
|
||||
Math.max(1, Math.ceil((cooldownUntil - monotonicNow()) / 1000)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: 'gnss-integrity',
|
||||
configureServer({ middlewares }) {
|
||||
middlewares.use('/api/gnss-integrity', handler);
|
||||
},
|
||||
configurePreviewServer({ middlewares }) {
|
||||
middlewares.use('/api/gnss-integrity', handler);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import { mapillaryProxy } from './mapillary.js';
|
||||
import { keySetupEndpoint } from '../standalone/key-setup.js';
|
||||
import { weatherProxy } from './weather.js';
|
||||
import { firePerimetersProxy } from './firePerimeters.js';
|
||||
import { gnssIntegrityProxy } from './gnssIntegrity.js';
|
||||
import { cycloneProxy } from './cyclones.js';
|
||||
import { windProxy } from './wind.js';
|
||||
|
||||
@@ -60,6 +61,7 @@ function localProviderPlugins({ realtime } = {}) {
|
||||
cycloneProxy(),
|
||||
mapillaryProxy(),
|
||||
firePerimetersProxy(),
|
||||
gnssIntegrityProxy(),
|
||||
keySetupEndpoint(),
|
||||
];
|
||||
}
|
||||
@@ -81,6 +83,9 @@ export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js';
|
||||
export { googlePlacesContextProxy } from './places.js';
|
||||
export { googleServerApiKey } from './places.js';
|
||||
export { keylessGooglePlacesResponse } from './places.js';
|
||||
export { keylessGoogleGeocodeResponse } from './places.js';
|
||||
export { validateGeocodeAddress } from './places.js';
|
||||
export { geocodeBounds } from './places.js';
|
||||
export { adsbLolFallbackAnchor } from './aircraft/opensky.js';
|
||||
export { readResponseTextCapped } from './common/http.js';
|
||||
export { readResponseJsonCapped } from './common/http.js';
|
||||
|
||||
@@ -9,7 +9,10 @@ import {
|
||||
fetchOverpassPayload,
|
||||
overpassNotConfigured,
|
||||
} from './overpass/transport.js';
|
||||
import { resolveOverpassUpstreams } from './overpass/constants.js';
|
||||
import {
|
||||
resolveOverpassUpstreams,
|
||||
resolveOverpassUpstreamTag,
|
||||
} from './overpass/constants.js';
|
||||
import {
|
||||
_militaryInstallationCache,
|
||||
trimMilitaryInstallationCache,
|
||||
@@ -74,7 +77,13 @@ function militaryInstallationsProxy() {
|
||||
retrievedAt: new Date().toISOString(),
|
||||
status: 'ready',
|
||||
};
|
||||
const entry = { payload, cachedAt: Date.now() };
|
||||
// Tagged with the configured upstream set so the disk layer does not serve
|
||||
// this answer back after the operator repoints (see overpass/cache.js).
|
||||
const entry = {
|
||||
payload,
|
||||
cachedAt: Date.now(),
|
||||
upstreams: resolveOverpassUpstreamTag(),
|
||||
};
|
||||
_militaryInstallationCache.set(key, entry);
|
||||
trimMilitaryInstallationCache();
|
||||
writeMilitaryInstallationDisk(key, entry);
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
MILITARY_INSTALLATION_DISK_DIR,
|
||||
MILITARY_INSTALLATION_MAX_CACHE,
|
||||
} from './constants.js';
|
||||
import { cachedByConfiguredUpstream } from '../overpass/constants.js';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
@@ -114,6 +115,9 @@ async function readMilitaryInstallationDisk(
|
||||
const entry = JSON.parse(
|
||||
await fsp.readFile(militaryInstallationDiskPath(cacheKey, dir), 'utf8'),
|
||||
);
|
||||
// Same reason as the Overpass disk cache: the key is the bbox alone, so an
|
||||
// entry written against a different OVERPASS_UPSTREAMS is otherwise a hit.
|
||||
if (!cachedByConfiguredUpstream(entry)) return null;
|
||||
if (!militaryInstallationDiskFresh(entry, maxAgeMs)) return null;
|
||||
return migrateMilitaryInstallationEntry(entry);
|
||||
} catch {
|
||||
|
||||
@@ -179,7 +179,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
get_current_view_state: {
|
||||
description:
|
||||
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action.',
|
||||
'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action. Tracked aircraft emergency status is a crowd-sourced transponder broadcast, not a confirmed incident: quote emergencyBroadcast and never infer a hijack or an incident ending from a code change.',
|
||||
$position: 1,
|
||||
},
|
||||
set_hud: {
|
||||
@@ -543,7 +543,7 @@ export const ACTION_DESCRIPTIONS = {
|
||||
},
|
||||
analyst_query: {
|
||||
description:
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values).',
|
||||
'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.',
|
||||
$position: 1,
|
||||
parameters: {
|
||||
properties: {
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
OVERPASS_DISK_DIR,
|
||||
OVERPASS_CACHE_MS,
|
||||
OVERPASS_CACHE_MAX_ENTRIES,
|
||||
cachedByConfiguredUpstream,
|
||||
} from './constants.js';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
@@ -47,6 +48,11 @@ async function readOverpassDisk(cacheKey, maxAgeMs) {
|
||||
// Older versions persisted 4xx refusals with normal data TTLs. Ignore
|
||||
// them on both fresh and stale reads so an upgrade can recover immediately.
|
||||
if (!overpassPayloadIsData(payload)) return null;
|
||||
// The key is the query alone, so a file written by a different upstream —
|
||||
// including the public instances that were the default before #742 — is a
|
||||
// hit for the same query. Refuse it: the operator's configured instance
|
||||
// must be what answers, on the stale path too.
|
||||
if (!cachedByConfiguredUpstream(payload)) return null;
|
||||
if (Date.now() - payload.cachedAt > maxAgeMs) return null;
|
||||
return payload;
|
||||
} catch {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Overpass API proxy constants and cache state
|
||||
@@ -27,16 +28,68 @@ function parseOverpassUpstreams(raw) {
|
||||
return endpoints.slice(0, 8);
|
||||
}
|
||||
|
||||
let upstreamMemo = { raw: null, endpoints: [] };
|
||||
let upstreamMemo = { raw: null, endpoints: [], tag: 'none' };
|
||||
|
||||
/**
|
||||
* Short, stable identifier for the configured upstream set. Cached responses
|
||||
* carry it so an answer produced by a different instance is not served after
|
||||
* the operator repoints `OVERPASS_UPSTREAMS` — the disk cache outlives the
|
||||
* configuration change by 7 days, a month for boundary queries, and forever on
|
||||
* the serve-stale path. It is a hash, never the URLs, and it is only ever
|
||||
* compared on read: it is not sent to clients and does not appear in errors.
|
||||
*/
|
||||
function upstreamTag(endpoints) {
|
||||
if (!endpoints.length) return 'none';
|
||||
// Sorted, so reordering the same instances for failover keeps the cache.
|
||||
return createHash('sha256')
|
||||
.update([...endpoints].sort().join('\n'))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
/** Resolve after environment loading. Public Overpass instances are not used by default. */
|
||||
function resolveOverpassUpstreams() {
|
||||
const raw = process.env.OVERPASS_UPSTREAMS || '';
|
||||
if (upstreamMemo.raw !== raw)
|
||||
upstreamMemo = { raw, endpoints: parseOverpassUpstreams(raw) };
|
||||
readUpstreamMemo();
|
||||
return [...upstreamMemo.endpoints];
|
||||
}
|
||||
|
||||
/** The configured-upstream tag stored alongside, and checked against, cache entries. */
|
||||
function resolveOverpassUpstreamTag() {
|
||||
return readUpstreamMemo().tag;
|
||||
}
|
||||
|
||||
let reportedForeignCache = false;
|
||||
|
||||
/**
|
||||
* Was this cached entry produced by the upstream set configured now?
|
||||
*
|
||||
* With nothing configured there is no instance to contradict, and serving
|
||||
* last-good data at any age is the point of the cache (the proxy's
|
||||
* not-configured path depends on it), so any entry is accepted. Once an
|
||||
* operator does configure an instance, only its own answers are served: an
|
||||
* entry with no tag, or one from a different instance, is refused rather than
|
||||
* matched on the query alone. Shared by both caches fed by these upstreams.
|
||||
*/
|
||||
function cachedByConfiguredUpstream(entry) {
|
||||
const tag = resolveOverpassUpstreamTag();
|
||||
if (tag === 'none' || entry?.upstreams === tag) return true;
|
||||
if (!reportedForeignCache) {
|
||||
reportedForeignCache = true;
|
||||
console.warn(
|
||||
'[Overpass] ignoring cached responses produced by a different OVERPASS_UPSTREAMS; they will be refetched from the configured instance.',
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function readUpstreamMemo() {
|
||||
const raw = process.env.OVERPASS_UPSTREAMS || '';
|
||||
if (upstreamMemo.raw !== raw) {
|
||||
const endpoints = parseOverpassUpstreams(raw);
|
||||
upstreamMemo = { raw, endpoints, tag: upstreamTag(endpoints) };
|
||||
}
|
||||
return upstreamMemo;
|
||||
}
|
||||
|
||||
/**
|
||||
* TTL for FRESH cached Overpass responses (ms). Road geometry is static for
|
||||
* months — the original 45 s TTL forced a public-mirror round-trip on nearly
|
||||
@@ -153,6 +206,8 @@ export {
|
||||
OVERPASS_MAX_RESPONSE_BYTES,
|
||||
parseOverpassUpstreams,
|
||||
resolveOverpassUpstreams,
|
||||
resolveOverpassUpstreamTag,
|
||||
cachedByConfiguredUpstream,
|
||||
OVERPASS_USER_AGENT,
|
||||
OVERPASS_TIMEOUT_MS,
|
||||
};
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
OVERPASS_MAX_RESPONSE_BYTES,
|
||||
resolveOverpassUpstreams,
|
||||
resolveOverpassUpstreamTag,
|
||||
OVERPASS_USER_AGENT,
|
||||
OVERPASS_TIMEOUT_MS,
|
||||
} from './constants.js';
|
||||
@@ -187,6 +188,9 @@ async function fetchOverpassPayload(
|
||||
contentType: 'application/json',
|
||||
// Never retain a secret-bearing endpoint in cache or response metadata.
|
||||
endpoint: 'configured',
|
||||
// Which configured instance produced this, as a hash rather than a URL,
|
||||
// so a cached answer is not reused after the operator repoints.
|
||||
upstreams: resolveOverpassUpstreamTag(),
|
||||
rateLimited: false,
|
||||
};
|
||||
} catch {
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
/** Place-search and route middleware for Node development servers. */
|
||||
export { googlePlacesContextProxy } from './places/google.js';
|
||||
export {
|
||||
googlePlacesContextProxy,
|
||||
validateGeocodeAddress,
|
||||
geocodeBounds,
|
||||
} from './places/google.js';
|
||||
export {
|
||||
googleServerApiKey,
|
||||
keylessGooglePlacesResponse,
|
||||
keylessGoogleGeocodeResponse,
|
||||
} from './places/google-key.js';
|
||||
export { installRouteMiddleware } from './places/routes.js';
|
||||
export {
|
||||
|
||||
@@ -14,12 +14,34 @@ export function keylessGooglePlacesResponse(apiKey) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Google API key for the SERVER-SIDE calls (Places nearby/text search, the
|
||||
* CCTV Street View fallback). These never reach the browser, so this key can
|
||||
* be restricted by server IP and scoped to Places API + Street View Static
|
||||
* API — while GOOGLE_MAPS_API_KEY stays referrer-restricted to Map Tiles +
|
||||
* Geocoding for the browser (#33). Splitting them is opt-in: unset, this
|
||||
* falls back to the shared browser key and nothing changes.
|
||||
* Geocoding speaks Google's own response shape rather than the `places: []`
|
||||
* contract its two siblings use, because the browser adapters read `status`
|
||||
* and `results` directly. The keyless capability reads the same way: a status
|
||||
* that is not `OK` and no results, which the adapters already treat as "this
|
||||
* provider did not answer" and pass to the next geocoder in the chain.
|
||||
*/
|
||||
export function keylessGoogleGeocodeResponse(apiKey) {
|
||||
if (String(apiKey ?? '').trim()) return null;
|
||||
return {
|
||||
statusCode: 200,
|
||||
payload: {
|
||||
configured: false,
|
||||
status: 'REQUEST_DENIED',
|
||||
results: [],
|
||||
error: null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Google API key for the SERVER-SIDE calls (Places nearby/text search,
|
||||
* geocoding, the CCTV Street View fallback). These never reach the browser, so
|
||||
* this key can be restricted by server IP and scoped to Places API + Geocoding
|
||||
* API + Street View Static API — while GOOGLE_MAPS_API_KEY stays
|
||||
* referrer-restricted to Map Tiles + Maps JavaScript API for the browser
|
||||
* (#33, #363). Splitting them is opt-in: unset, this falls back to the shared
|
||||
* browser key, which then has to carry every API both sides use and cannot
|
||||
* take a referrer restriction.
|
||||
*/
|
||||
export function googleServerApiKey() {
|
||||
return resolveGoogleServerKey(process.env);
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import {
|
||||
googleServerApiKey,
|
||||
keylessGooglePlacesResponse,
|
||||
keylessGoogleGeocodeResponse,
|
||||
} from './google-key.js';
|
||||
import { makeCostRateLimiter, clientKey } from '../common/rate-limit.js';
|
||||
import { admitSameSite } from '../common/same-site.js';
|
||||
import { readResponseJsonCapped } from '../common/http.js';
|
||||
import {
|
||||
projectNearbyPlaces,
|
||||
projectTextSearchPlaces,
|
||||
@@ -25,7 +27,7 @@ let _googleRateLimiter;
|
||||
*/
|
||||
export const GOOGLE_DEFAULT_PER_MIN = 120;
|
||||
|
||||
/** Google cost endpoints (nearby-places + text-search). Null only when set to 0. */
|
||||
/** Google cost endpoints (nearby-places, text-search, geocoding). Null only when set to 0. */
|
||||
function googleRateLimiter() {
|
||||
if (_googleRateLimiter === undefined)
|
||||
_googleRateLimiter = makeCostRateLimiter(
|
||||
@@ -56,6 +58,51 @@ export function validatePlacesCoordinates(searchParams) {
|
||||
return { ok: true, latitude, longitude };
|
||||
}
|
||||
|
||||
/** Longest accepted geocoder query. Real place names are far shorter. */
|
||||
const GEOCODE_MAX_QUERY = 200;
|
||||
|
||||
/**
|
||||
* Upstream deadline and body ceiling for the geocode routes. The deadline stays
|
||||
* armed through body consumption, so a stalled upstream cannot hold a handler
|
||||
* open, and the cap bounds a chunked or length-omitted reply. A geocode answer
|
||||
* is a handful of results; 256 KiB is far above any honest one.
|
||||
*/
|
||||
const GEOCODE_UPSTREAM_TIMEOUT_MS = 10_000;
|
||||
|
||||
const GEOCODE_MAX_RESPONSE_BYTES = 256 * 1024;
|
||||
|
||||
/** Validate a free-text geocoder query before consuming request quota. */
|
||||
export function validateGeocodeAddress(searchParams) {
|
||||
const address = String(searchParams.get('address') || '').trim();
|
||||
if (!address) return { ok: false, error: 'address is required' };
|
||||
if (address.length > GEOCODE_MAX_QUERY)
|
||||
return { ok: false, error: 'address is too long' };
|
||||
return { ok: true, address };
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate the optional viewport bias, which Google reads as
|
||||
* `sw_lat,sw_lng|ne_lat,ne_lng`. An unparsed value is dropped rather than
|
||||
* refused: the bias only ranks results, so a malformed one costs relevance,
|
||||
* not an answer.
|
||||
*/
|
||||
export function geocodeBounds(searchParams) {
|
||||
const raw = String(searchParams.get('bounds') || '').trim();
|
||||
if (!raw) return null;
|
||||
const corners = raw.split('|');
|
||||
if (corners.length !== 2) return null;
|
||||
const parts = corners.flatMap((corner) => corner.split(','));
|
||||
// A blank component must be refused before Number() turns it into 0 and a
|
||||
// half-written corner like `30.4,` reads as the equator.
|
||||
if (parts.length !== 4 || parts.some((part) => !part.trim())) return null;
|
||||
const numbers = parts.map(Number);
|
||||
if (!numbers.every(Number.isFinite)) return null;
|
||||
const [swLat, swLon, neLat, neLon] = numbers;
|
||||
if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null;
|
||||
if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null;
|
||||
return raw;
|
||||
}
|
||||
|
||||
/** Nearby place labels and view-biased text search, with request-time key resolution. */
|
||||
export function googlePlacesContextProxy({
|
||||
resolveApiKey = googleServerApiKey,
|
||||
@@ -293,6 +340,149 @@ export function googlePlacesContextProxy({
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Geocoding belongs on the server, not in the page. Google's Geocoding web
|
||||
* service refuses referrer-restricted keys, so calling it from the browser
|
||||
* forces GOOGLE_MAPS_API_KEY — which ships in the bundle by design — to be
|
||||
* left unrestricted (#363). Here the request carries the server key, and
|
||||
* the browser key no longer needs the Geocoding API at all.
|
||||
*
|
||||
* Unlike the two Places routes above, these answer in Google's own shape
|
||||
* (`status` + `results`) rather than the `places: []` contract: the browser
|
||||
* normalizers read that shape directly, and keeping it means the proxy adds
|
||||
* a hop without adding a translation nobody asked for.
|
||||
*/
|
||||
function installGeocodeRoute(path, readQuery) {
|
||||
middlewares.use(path, async (req, res) => {
|
||||
const refuse = (statusCode, error) => {
|
||||
res.statusCode = statusCode;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.end(
|
||||
JSON.stringify({ status: 'REQUEST_DENIED', results: [], error }),
|
||||
);
|
||||
};
|
||||
|
||||
// Gate first, like the Places routes above: a cross-site caller learns
|
||||
// nothing about this endpoint's method surface.
|
||||
if (admitSameSite(req, res)) return;
|
||||
if (req.method !== 'GET') {
|
||||
refuse(405, 'Method not allowed');
|
||||
return;
|
||||
}
|
||||
|
||||
// A keyless geocode has no provider cost, so it resolves before the
|
||||
// paid-endpoint limiter can consume or exhaust quota (mirrors the
|
||||
// Places routes above).
|
||||
const apiKey = resolveApiKey();
|
||||
const keyless = keylessGoogleGeocodeResponse(apiKey);
|
||||
if (keyless) {
|
||||
res.statusCode = keyless.statusCode;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.setHeader('Cache-Control', 'no-store');
|
||||
res.end(JSON.stringify(keyless.payload));
|
||||
return;
|
||||
}
|
||||
|
||||
const requestUrl = new URL(req.url || '', 'http://localhost');
|
||||
const query = readQuery(requestUrl.searchParams);
|
||||
if (!query.ok) {
|
||||
refuse(400, query.error);
|
||||
return;
|
||||
}
|
||||
|
||||
// Per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN), shared with the
|
||||
// Places routes because both spend the same Google budget.
|
||||
const _grl = googleRateLimiter();
|
||||
if (_grl && !_grl(clientKey(req))) {
|
||||
res.statusCode = 429;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.setHeader('Retry-After', '5');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
status: 'OVER_QUERY_LIMIT',
|
||||
results: [],
|
||||
error: 'Rate limit exceeded',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const upstream = new URL(
|
||||
endpoints.geocode ||
|
||||
'https://maps.googleapis.com/maps/api/geocode/json',
|
||||
);
|
||||
for (const [name, value] of Object.entries(query.params))
|
||||
upstream.searchParams.set(name, value);
|
||||
upstream.searchParams.set('key', apiKey);
|
||||
const deadline = AbortSignal.timeout(GEOCODE_UPSTREAM_TIMEOUT_MS);
|
||||
const response = await fetchImpl(upstream.toString(), {
|
||||
redirect: 'error',
|
||||
signal: deadline,
|
||||
});
|
||||
// An unreadable, oversized or timed-out body is not an answer: it
|
||||
// falls through to the same shaped failure as a refusal below.
|
||||
const data = await readResponseJsonCapped(
|
||||
response,
|
||||
GEOCODE_MAX_RESPONSE_BYTES,
|
||||
deadline,
|
||||
).catch(() => null);
|
||||
|
||||
// A body we could not read is an upstream failure, not an answer with
|
||||
// a status of its own, so it does not borrow the upstream's code.
|
||||
if (!data) {
|
||||
res.statusCode = 502;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
status: 'UNKNOWN_ERROR',
|
||||
results: [],
|
||||
error: 'Google Geocoding request failed',
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
res.statusCode = response.ok ? 200 : response.status;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.setHeader('Cache-Control', 'private, max-age=300');
|
||||
res.end(JSON.stringify(data));
|
||||
} catch {
|
||||
// Fixed public text: a transport, URL or runtime message would
|
||||
// describe the server to the browser (SECURITY.md, sanitized errors).
|
||||
res.statusCode = 502;
|
||||
res.setHeader('Content-Type', 'application/json; charset=utf-8');
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
status: 'UNKNOWN_ERROR',
|
||||
results: [],
|
||||
error: 'Google Geocoding request failed',
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
installGeocodeRoute('/api/google/geocode', (searchParams) => {
|
||||
const address = validateGeocodeAddress(searchParams);
|
||||
if (!address.ok) return address;
|
||||
const bounds = geocodeBounds(searchParams);
|
||||
return {
|
||||
ok: true,
|
||||
params: {
|
||||
address: address.address,
|
||||
...(bounds ? { bounds } : {}),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
installGeocodeRoute('/api/google/reverse-geocode', (searchParams) => {
|
||||
const coordinates = validatePlacesCoordinates(searchParams);
|
||||
if (!coordinates.ok) return coordinates;
|
||||
const { latitude, longitude } = coordinates;
|
||||
return { ok: true, params: { latlng: `${latitude},${longitude}` } };
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import https from 'node:https';
|
||||
import { Readable } from 'node:stream';
|
||||
import { isNonGlobalIpv4 } from './stations.js';
|
||||
|
||||
const NULL_BODY_STATUSES = new Set([204, 205, 304]);
|
||||
|
||||
export function radioMirrorOrigin(value) {
|
||||
const hostname = String(value ?? '')
|
||||
.toLowerCase()
|
||||
@@ -124,19 +127,33 @@ export function fetchPinnedRadioResponse(url, options, addresses) {
|
||||
},
|
||||
},
|
||||
(response) => {
|
||||
const headers = new Headers();
|
||||
for (const [name, value] of Object.entries(response.headers)) {
|
||||
if (Array.isArray(value))
|
||||
value.forEach((item) => headers.append(name, item));
|
||||
else if (value !== undefined) headers.set(name, String(value));
|
||||
// Nothing thrown here may escape: this callback runs outside the
|
||||
// promise, so an exception would be uncaught and crash the server.
|
||||
try {
|
||||
const status = response.statusCode || 500;
|
||||
// A `Response` only represents statuses up to 599; Node never hands
|
||||
// this callback a 1xx.
|
||||
if (status > 599)
|
||||
throw new Error(
|
||||
`Radio Browser returned unsupported status ${status}`,
|
||||
);
|
||||
const headers = new Headers();
|
||||
for (const [name, value] of Object.entries(response.headers)) {
|
||||
if (Array.isArray(value))
|
||||
value.forEach((item) => headers.append(name, item));
|
||||
else if (value !== undefined) headers.set(name, String(value));
|
||||
}
|
||||
const nullBody = NULL_BODY_STATUSES.has(status);
|
||||
const upstream = new Response(
|
||||
nullBody ? null : Readable.toWeb(response),
|
||||
{ status, statusText: response.statusMessage || '', headers },
|
||||
);
|
||||
if (nullBody) response.resume();
|
||||
resolve(upstream);
|
||||
} catch (error) {
|
||||
response.destroy();
|
||||
reject(error);
|
||||
}
|
||||
resolve(
|
||||
new Response(Readable.toWeb(response), {
|
||||
status: response.statusCode || 500,
|
||||
statusText: response.statusMessage || '',
|
||||
headers,
|
||||
}),
|
||||
);
|
||||
},
|
||||
);
|
||||
request.on('error', reject);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
|
||||
import { readResponseTextCapped } from '../common/http.js';
|
||||
|
||||
/**
|
||||
* Vite plugin: CelesTrak TLE proxy.
|
||||
@@ -48,10 +49,14 @@ export function celestrakProxy() {
|
||||
}
|
||||
}
|
||||
|
||||
/** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */
|
||||
const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
async function fetchUpstream(group) {
|
||||
const url = celestrakTleUrl(group);
|
||||
const signal = AbortSignal.timeout(20000);
|
||||
const res = await fetch(url.toString(), {
|
||||
signal: AbortSignal.timeout(20000),
|
||||
signal,
|
||||
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
|
||||
// descriptive User-Agent with a contact point.
|
||||
headers: {
|
||||
@@ -60,7 +65,11 @@ export function celestrakProxy() {
|
||||
},
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const body = await res.text();
|
||||
const body = await readResponseTextCapped(
|
||||
res,
|
||||
CELESTRAK_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
|
||||
if (!/^1 /m.test(body)) throw new Error('no TLE lines in response');
|
||||
return { at: Date.now(), body };
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import path from 'node:path';
|
||||
import { promises as fsp } from 'node:fs';
|
||||
|
||||
import { readResponseBytesCapped } from './common/http.js';
|
||||
import {
|
||||
isValidTileCoord as isValidTomTomTile,
|
||||
utcDayKey as tomtomUtcDayKey,
|
||||
@@ -48,6 +49,8 @@ export function tomtomProxy() {
|
||||
const DEFAULT_DAILY_BUDGET = 6000;
|
||||
const MEM_MAX_ENTRIES = 256;
|
||||
const UPSTREAM_TIMEOUT_MS = 15000;
|
||||
/** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */
|
||||
const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/** @type {Map<string, {at:number, buf:Buffer}>} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */
|
||||
const mem = new Map();
|
||||
@@ -143,11 +146,15 @@ export function tomtomProxy() {
|
||||
'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' +
|
||||
`${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`;
|
||||
recordUpstreamFetch(); // attempts count — upstream bills the request either way
|
||||
const res = await fetch(url, {
|
||||
signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
|
||||
});
|
||||
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
|
||||
const res = await fetch(url, { signal });
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
const bytes = await readResponseBytesCapped(
|
||||
res,
|
||||
TOMTOM_TILE_MAX_RESPONSE_BYTES,
|
||||
signal,
|
||||
);
|
||||
const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
if (buf.length === 0) throw new Error('empty tile body');
|
||||
return buf;
|
||||
}
|
||||
|
||||
@@ -142,9 +142,11 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
|
||||
const rows = Array.from({ length: 15000 }, (_, i) =>
|
||||
row(now, `vehicle-${i}`),
|
||||
);
|
||||
const start = performance.now();
|
||||
// Main-thread CPU: excludes scheduler waits and parallel GC helper threads.
|
||||
const start = process.threadCpuUsage();
|
||||
h.ingest(feed, rows, now);
|
||||
times.push(performance.now() - start);
|
||||
const usage = process.threadCpuUsage(start);
|
||||
times.push((usage.user + usage.system) / 1000);
|
||||
}
|
||||
assert.equal(h.diagnostics().keys, 15000);
|
||||
for (let i = 0; i < 15000; i++) {
|
||||
@@ -154,12 +156,12 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde
|
||||
}
|
||||
assert.ok(h.diagnostics().allocatedBytes < 16 * 1024 * 1024);
|
||||
t.diagnostic(
|
||||
`15000-vehicle ingestion ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
|
||||
`15000-vehicle ingestion CPU ms: ${times.map((n) => n.toFixed(1)).join('/')}`,
|
||||
);
|
||||
// A ceiling that says "not quadratic", not a benchmark: 15,000 rows in
|
||||
// well under two seconds on a loaded laptop or a slow CI runner. The
|
||||
// per-run figure above is the diagnostic to read.
|
||||
assert.ok(Math.max(...times) < 2000, `ingestion ms: ${times}`);
|
||||
// CPU time measures ingestion work without counting scheduler delays on
|
||||
// loaded runners. Keep the two-second ceiling as a guard against quadratic
|
||||
// work, not a benchmark; the per-poll figures above show any slowdown.
|
||||
assert.ok(Math.max(...times) < 2000, `ingestion CPU ms: ${times}`);
|
||||
});
|
||||
|
||||
test('snapshot capacity pressure preserves members not yet visited and discloses recreated history', (t) => {
|
||||
@@ -177,3 +179,26 @@ test('snapshot capacity pressure preserves members not yet visited and discloses
|
||||
assert.equal(recreated.fixes.length, 1);
|
||||
assert.equal(recreated.truncated, true);
|
||||
});
|
||||
test('a feed fleet-number mode outranks its route mode in the epoch context', (t) => {
|
||||
const h = history(t);
|
||||
const ttc = {
|
||||
id: 'ttc',
|
||||
historyRetention: true,
|
||||
defaultMode: 'bus',
|
||||
routeMode: (route) => (/^5\d\d$/.test(route ?? '') ? 'tram' : 'bus'),
|
||||
vehicleMode: (id) =>
|
||||
/^4[4-6]\d\d$/.test(id) ? 'tram' : /^\d{4}$/.test(id) ? 'bus' : 'unknown',
|
||||
};
|
||||
h.ingest(
|
||||
ttc,
|
||||
[
|
||||
row(100000, '8515', { routeId: '506' }),
|
||||
row(100000, '4507', { routeId: null }),
|
||||
row(100000, 'x1', { routeId: '501' }),
|
||||
],
|
||||
100000,
|
||||
);
|
||||
assert.equal(h.get('ttc', '8515', 100000).epochs[0].mode, 'bus');
|
||||
assert.equal(h.get('ttc', '4507', 100000).epochs[0].mode, 'tram');
|
||||
assert.equal(h.get('ttc', 'x1', 100000).epochs[0].mode, 'tram');
|
||||
});
|
||||
Reference in new issue
Block a user