benmichael/gods-eye-view
1
0 Fork 0
mirror of https://github.com/bilawalsidhu/gods-eye-view.git synced 2026-11-05 10:53:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into HEAD

Browse Source 
# Conflicts:
#	docs/CURRENT-STATE.md
#	server/providers/cctv/sources.js
This commit is contained in:
Trygve Hemminghytt committed 2026-10-16 07:21:30 +02:00
2 parents e2dacd92ea e2274bbf0c
commit 2bec3d521b
154 files changed
+17718 -1059
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

server/providers/aircraft/adsb-lol.js
+18 -2
View File
@@ -1,3 +1,5 @@
import { readResponseTextCapped } from '../common/http.js';
/**
* Vite plugin: adsb.lol military aircraft proxy with 12 s response cache.
*
@@ -25,6 +27,10 @@ export function adsbLolProxy() {
let _cooldownStatus = 0;
/** Response cache TTL (ms). */
const CACHE_MS = 12000;
/** Upstream deadline (ms); the fetch previously had none and could hang a poll. */
const UPSTREAM_TIMEOUT_MS = 12000;
/** /v2/mil measures ~67 KB; a relayed error page is bounded by the same cap. */
const ADSBLOL_MIL_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
/** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */
const RATE_LIMIT_COOLDOWN_MS = 30000;
/** Cooldown after a 5xx (ms). */
@@ -98,11 +104,17 @@ export function adsbLolProxy() {
);
return;
}
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const upstream = await fetch('https://api.adsb.lol/v2/mil', {
headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' },
signal,
});
if (upstream.ok) {
const body = await upstream.text();
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
_cache = body;
_cacheAt = Date.now();
_cooldownUntil = 0;
@@ -129,7 +141,11 @@ export function adsbLolProxy() {
return;
}
}
const body = await upstream.text();
const body = await readResponseTextCapped(
upstream,
ADSBLOL_MIL_MAX_RESPONSE_BYTES,
signal,
);
serve(
res,
upstream.status,
server/providers/aircraft/enrichment.js
+12 -4
View File
@@ -1,5 +1,9 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { readResponseJsonCapped } from '../common/http.js';
/** Route and aircraft lookups are a few hundred bytes; 1 MiB is a hard bound. */
const ADSBDB_MAX_RESPONSE_BYTES = 1024 * 1024;
/**
* adsbdb.com enrichment proxy: callsign → route (airline + origin/destination
* airports) and hex → aircraft type/registration. Free community API — cached
@@ -80,12 +84,16 @@ export function adsbdbProxy() {
kind === 'route'
? `https://api.adsbdb.com/v0/callsign/${encodeURIComponent(key)}`
: `https://api.adsbdb.com/v0/aircraft/${encodeURIComponent(key)}`;
const res = await fetch(url, { signal: AbortSignal.timeout(8000) });
const signal = AbortSignal.timeout(8000);
const res = await fetch(url, { signal });
if (res.ok) {
const payload = await readResponseJsonCapped(
res,
ADSBDB_MAX_RESPONSE_BYTES,
signal,
);
const data =
kind === 'route'
? parseRoute(await res.json())
: parseAircraft(await res.json());
kind === 'route' ? parseRoute(payload) : parseAircraft(payload);
store[key] = { at: Date.now(), data }; // data may be null — negative cache
dirty = true;
return data;
server/providers/aircraft/opensky.js
+48 -20
View File
@@ -2,6 +2,7 @@ import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback
import {
coalesceProxyRequest,
readResponseJsonCapped,
readResponseTextCapped,
} from '../common/http.js';
import { requiredFiniteQueryNumber } from '../common/query.js';
// ---------------------------------------------------------------------------
@@ -29,11 +30,34 @@ const OPENSKY_CACHE_MS = 9000;
const OPENSKY_ATTEMPT_TIMEOUT_MS = 10_000;
const OPENSKY_STATES_URL =
'https://opensky-network.org/api/states/all?extended=1';
/** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */
const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
/**
* Credit-governor cooldown for a 429: OpenSky's retry-after bounded to
* 30 s … 30 min, or 2 min when the header is absent.
* @param {Response} upstream Rate-limited response.
* @returns {number} Cooldown (ms).
*/
function openSkyRateLimitCooldownMs(upstream) {
// headers.get() gives null when absent, and Number(null) is 0, not NaN.
const header = upstream.headers.get('x-rate-limit-retry-after-seconds');
const retryAfterSec =
header == null || header.trim() === '' ? NaN : Number(header);
return Math.min(
Math.max(
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
30_000,
),
30 * 60_000,
);
}
/**
* Fetch and read the global snapshot, giving each attempt
* OPENSKY_ATTEMPT_TIMEOUT_MS and retrying once when an attempt times out or
* the connection fails. A second failure throws, which the proxy answers
* the connection fails. An oversized body is not retried: it would fail again
* and spend more credits. A second failure throws, which the proxy answers
* from its stale cache or the regional fallback.
* @param {Record<string, string>} headers Request headers.
* @returns {Promise<{upstream: Response, body: string}>}
@@ -41,13 +65,27 @@ const OPENSKY_STATES_URL =
async function fetchOpenSkyStates(headers) {
for (let attempt = 1; ; attempt++) {
try {
const upstream = await fetch(OPENSKY_STATES_URL, {
headers,
signal: AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS),
});
return { upstream, body: await upstream.text() };
const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS);
const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal });
if (upstream.status === 429) {
// Start the cooldown from the headers and drop the rejected body
// unread, so a stalled or oversized 429 body cannot skip it.
_openskyCooldownUntil =
Date.now() + openSkyRateLimitCooldownMs(upstream);
void upstream.body?.cancel().catch(() => {});
return {
upstream,
body: JSON.stringify({ error: 'OpenSky rate limited.' }),
};
}
const body = await readResponseTextCapped(
upstream,
OPENSKY_STATES_MAX_RESPONSE_BYTES,
signal,
);
return { upstream, body };
} catch (error) {
if (attempt >= 2) throw error;
if (attempt >= 2 || error?.code === 'RESPONSE_TOO_LARGE') throw error;
console.warn(
'[OpenSky] global snapshot attempt failed, retrying:',
error?.message || error,
@@ -544,19 +582,9 @@ export function openSkyProxy() {
}
if (upstream.status === 429) {
reason = 'rate_limited';
// Credit governor: honor OpenSky's retry-after (bounded 30 s … 30 min;
// 2 min when the header is absent) — no upstream attempts until then.
const retryAfterSec = Number(
upstream.headers.get('x-rate-limit-retry-after-seconds'),
);
const cooldownMs = Math.min(
Math.max(
Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000,
30_000,
),
30 * 60_000,
);
_openskyCooldownUntil = now + cooldownMs;
// Credit governor: fetchOpenSkyStates already started the cooldown
// from OpenSky's retry-after — no upstream attempts until then.
const cooldownMs = openSkyRateLimitCooldownMs(upstream);
// Serve the last-good body instead of the 429 when we have one —
// the layer keeps rendering (STALE-cued) instead of dying.
if (_openskyCacheBody && _openskyCacheStatus === 200) {
server/providers/cctv.js
+18 -1
View File
@@ -28,7 +28,7 @@ export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream };
* GET /api/cctv/sources — list all registered camera sources
* GET /api/cctv/health — per-camera health/status report
* GET /api/cctv/stream/:id — stream info (feedType, URLs) for a camera
* GET /api/cctv/media/:id — proxy live video/image media from upstream
* GET /api/cctv/media/:id — proxy live video media from upstream
* GET /api/cctv/frame/:id — single frame with fallback chain
*
* @returns {import('vite').Plugin}
@@ -164,6 +164,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
sourceKind:
source.sourceKind || (source.url ? 'configured' : 'fallback'),
poseSource: source.poseSource,
model: source.model || '',
manufacturer: source.manufacturer || '',
license: source.license,
credit: source.credit || '',
code: source.code || '',
@@ -214,6 +216,21 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) {
const source = sourceById.get(cameraId);
const mediaUrl = source?.url || '';
const feedType = normalizeFeedType(source?.feedType || 'image');
// Live feeds only. A still camera is served by /frame, whose fetch
// keeps redirects on the registered origin, accepts only image/* and
// caps the body; relaying it here would bypass all three.
if (!isVideoFeedType(feedType) && feedType !== 'mjpeg') {
res.writeHead(404, {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
});
res.end(
JSON.stringify({
error: 'No live media for this camera; use the frame endpoint',
}),
);
return;
}
const leaseId = url.searchParams.get('lease');
if (feedType === 'hls' && !/^[a-f0-9-]{36}$/i.test(leaseId || '')) {
res.writeHead(400);
server/providers/cctv/catalog.js
+39 -4
View File
@@ -4,6 +4,7 @@ import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js';
import { allocateSourceCap, resolveCatalogCap } from './cap.js';
import { loadGroundHeights, joinGroundHeights } from './groundHeights.js';
import { cameraRegion, normalizeSourceItem } from './normalize.js';
import { loadLithuaniaSources } from './lithuania.js';
import {
loadAustinSourcesFromOpenData,
loadCaltransSourcesFromOpenData,
@@ -17,23 +18,30 @@ import {
loadWarendorfSourcesFromCatalog,
loadNswSourcesFromOpenData,
loadCalgarySourcesFromOpenData,
loadCataloniaSourcesFromOpenData,
loadDelDOTSourcesFromOpenData,
loadKingCountySourcesFromOpenData,
loadWsdotSourcesFromOpenData,
loadVegvesenSourcesFromOpenData,
loadNycSourcesFromOpenData,
loadTorontoSourcesFromOpenData,
} from './sources.js';
/** Env kill switch: unset or anything but "0" means enabled. */
const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0';
/** Env opt-in: only "1" enables. For packs whose feed access needs an
* operator agreement, so an unset flag never reaches the upstream. */
* operator agreement or whose terms are unconfirmed, so an unset flag never
* reaches the upstream. */
const envOptIn = (name) => String(process.env[name] || '').trim() === '1';
/**
* Live open-data packs, in merge order. Adding a region is one entry here
* plus its loader in sources.js; the catalog cap is shared across entries
* round-robin (cap.js), so a new pack never silently evicts an older one.
* plus its loader (in sources.js or its own module); the catalog cap is
* shared across entries round-robin (cap.js), so a new pack never silently
* evicts an older one.
* Each pack fails independently (allSettled) and is gated by its own env
* kill switch, or by an opt-in flag where access needs an agreement.
* kill switch, or by an opt-in flag where access needs an agreement or the
* terms are unconfirmed.
*/
const LIVE_PACKS = [
{ name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData },
@@ -92,11 +100,26 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_CALGARY_ENABLED'),
load: loadCalgarySourcesFromOpenData,
},
{
name: 'catalonia',
enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'),
load: loadCataloniaSourcesFromOpenData,
},
{
name: 'deldot',
enabled: () => envEnabled('CCTV_DELDOT_ENABLED'),
load: loadDelDOTSourcesFromOpenData,
},
{
name: 'kingcounty',
enabled: () => envEnabled('CCTV_KINGCOUNTY_ENABLED'),
load: loadKingCountySourcesFromOpenData,
},
{
name: 'wsdot',
enabled: () => envEnabled('CCTV_WSDOT_ENABLED'),
load: loadWsdotSourcesFromOpenData,
},
{
name: 'vegvesen',
enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'),
@@ -109,6 +132,18 @@ const LIVE_PACKS = [
enabled: () => envOptIn('CCTV_NYC_ENABLED'),
load: loadNycSourcesFromOpenData,
},
{
name: 'toronto',
enabled: () => envEnabled('CCTV_TORONTO_ENABLED'),
load: loadTorontoSourcesFromOpenData,
},
{
// Opt-in until the Via Lietuva open-data terms are confirmed to cover the
// camera photos (see DATA_SOURCES.md).
name: 'lithuania',
enabled: () => envOptIn('CCTV_LITHUANIA_ENABLED'),
load: loadLithuaniaSources,
},
];
/**
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
server/providers/cctv/constants.js
+132 -5
View File
@@ -8,8 +8,8 @@ export const DEFAULT_AUSTIN_MAX_SOURCES = 250;
* Catalog-wide safety ceiling on served cameras. Each pack already caps
* itself (nearest-to-anchor first); this bound only matters when the packs
* together exceed it, and it is then filled round-robin across packs (see
* cap.js) so no region is silently dropped. Sized above the sum of the
* default per-pack caps so a default install never trims.
* cap.js) so no region is silently dropped. The default per-pack caps
* together can exceed it, so a default install may be thinned evenly here.
*/
export const DEFAULT_CCTV_MAX_SOURCES = 4000;
/** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */
@@ -236,6 +236,56 @@ export const CALGARY_DOWNTOWN = { lat: 51.0461, lon: -114.0626 };
* body cannot be buffered without limit. */
export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/**
* Catalonia traffic cameras (Servei Català de Trànsit): one keyless WFS/GML
* XML list (verified 2026-09-15, hourly catalog refresh), fetched over https
* (verified 2026-09-27: same 200 body, no redirect). The feed carries a
* `font` field naming the actual publisher per camera — SCT's own highway
* cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras are
* kept, each confirmed to return a real frame. Frame URLs are checked
* against a host allowlist rather than one pinned origin, since each
* publisher serves from its own host.
*/
export const CATALONIA_CAMERAS_URL =
'https://www.gencat.cat/transit/opendata/cameres.xml';
/** Byte ceiling for the Catalonia catalog read (~90 KB observed 2026-09-27),
* so a runaway body cannot be buffered without limit. */
export const CATALONIA_MAX_CATALOG_BYTES = 1024 * 1024;
/** Allowed image hosts, one per kept `font` publisher in the feed, each
* mapped to the one directory its frame links live in (every link in the
* 2026-10 feed shares it); a link must name a single file directly inside. */
export const CATALONIA_IMAGE_HOSTS = Object.freeze(
new Map([
['mct.gencat.cat', '/mct2bo/'], // SCT
['www.bcn.cat', '/transit/imatges/'], // IMI (Ajuntament de Barcelona)
['emap.terrassa.cat', '/it_terrassa/'], // Ajuntament de Terrassa
]),
);
/** SCT `sctidcam` values: a bare image filename, which also keys the camera
* id (the feed's `fid` embeds an upstream server-start stamp). */
export const CATALONIA_SCT_CAMERA_FILE_RE = /^[a-z0-9]+\.(gif|jpe?g)$/i;
/** Per-`font` partner credit, shown beside the provider in the CCTV panel
* (same pattern as DriveBC's partner-supplied cameras). SCT is the feed's
* own publisher and carries no separate credit. */
export const CATALONIA_FONT_CREDIT = Object.freeze({
IMI: 'Ajuntament de Barcelona',
Terrassa: 'Ajuntament de Terrassa',
});
/** Ground-elevation priors in metres, by `font`: the feed carries no
* elevation, and this pack spans sea level (Barcelona) to Terrassa's plateau
* (~270 m). The client's ground snap corrects these on 3D-tile stacks. */
export const CATALONIA_FONT_ELEVATION_M = Object.freeze({
SCT: 80,
IMI: 20,
Terrassa: 270,
});
export const DEFAULT_CATALONIA_MAX_SOURCES = 150;
/** Prioritization anchors: Barcelona and Terrassa. */
export const CATALONIA_ANCHORS = [
{ lat: 41.3874, lon: 2.1686 }, // Barcelona
{ lat: 41.5636, lon: 2.0111 }, // Terrassa
];
/** Statens vegvesen (Norway) road cameras: the keyless OGC API Features view
* of the official DATEX 3.1 CCTV site table, one GeoJSON request for the whole
* country (~900 cameras). The raw DATEX node needs an account; this view does
@@ -249,9 +299,9 @@ export const VEGVESEN_IMAGE_ORIGIN =
* per-camera manifest path is registered. */
export const VEGVESEN_VIDEO_URL = (cameraId) =>
`https://kamera.vegvesen.no/public/${cameraId}/manifest.m3u8`;
/** Above the ~850 working cameras so a default install keeps them all, while
* staying close to them: the per-pack defaults are meant to sum to no more
* than DEFAULT_CCTV_MAX_SOURCES. */
/** Above the ~850 working cameras so the pack keeps them all on its own. The
* catalog-wide DEFAULT_CCTV_MAX_SOURCES cap still applies across packs and is
* filled round-robin, so a full default install may thin this pack. */
export const DEFAULT_VEGVESEN_MAX_SOURCES = 900;
/** Hard ceiling on the catalog body. The whole country is ~800 KB. */
export const VEGVESEN_MAX_CATALOG_BYTES = 8 * 1024 * 1024;
@@ -338,6 +388,83 @@ export const DELDOT_ANCHORS = [
{ lat: 39.1582, lon: -75.5244 }, // Dover (Kent)
{ lat: 38.6906, lon: -75.3877 }, // Georgetown (Sussex)
];
/** King County, WA road cameras: keyless public ArcGIS FeatureServer layer
* ("King County and WSDOT Traffic Cameras", King County DOT Road Services).
* GeoJSON with WGS84 point geometry. `outFields` is an explicit list on
* purpose: the layer also carries asset-administration columns (including
* staff names in `ModifiedBy`) that must never be fetched. Uniquely among the
* packs, the county publishes each camera's hardware `Manufacturer`/`Model`
* (Cohu 3950/3965, Bosch AUTODOME, Axis Q6135-LE, …), carried through as the
* source's `model`. */
export const KING_COUNTY_CAMERAS_URL =
'https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,ImageURL,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson';
/** Frames come only from the county's own image handler host. */
export const KING_COUNTY_IMAGE_ORIGIN = 'https://info.kingcounty.gov/';
/** The whole catalog is ~125 rows (~59 live county cameras); no thinning by
* default. */
export const DEFAULT_KING_COUNTY_MAX_SOURCES = 125;
/** Prioritization anchor: Renton, roughly central to the county road network. */
export const KING_COUNTY_ANCHORS = [{ lat: 47.4829, lon: -122.2171 }];
/** Hard ceiling on the King County catalog body. The whole layer is ~125 rows
* and well under 100 KB; this only exists so an upstream that streams an
* unbounded body cannot be buffered without limit. */
export const KING_COUNTY_MAX_CATALOG_BYTES = 1024 * 1024;
/**
* City of Toronto RESCU traffic cameras: the municipal network the province's
* 511 pack does not cover — downtown arterials plus the Gardiner, DVP, Allen
* and Lake Shore. One keyless list for the whole city (~336 cameras), from the
* City's open data portal. It is fetched on every catalog refresh, but the
* upstream file is not updated often and may lag the live camera network.
*
* The list is served as JSONP (`jsonTMCEarthCamerasCallback({...})`); the
* portal publishes the same data as .xml and .csv beside it.
*/
export const TORONTO_RESCU_LIST_URL =
'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/Data/tmcearthcameras.json';
/**
* The only origin Toronto camera frames may come from. The list carries no URL
* field: frame URLs are BUILT from this origin and a strictly-digits camera
* number (`loc####.jpg`, the convention the dataset documents), so no upstream
* field can steer the frame proxy off-host.
*/
export const TORONTO_RESCU_IMAGE_ORIGIN =
'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/CameraImages/';
/** The whole city fits under this; the cap exists to stay tunable, not to trim. */
export const DEFAULT_TORONTO_MAX_SOURCES = 336;
/** Yonge / Queen: the prioritization anchor. */
export const TORONTO_DOWNTOWN = { lat: 43.6532, lon: -79.3832 };
/** The city climbs from 76 m at the lake to 209 m at Steeles, so no single
* number fits: this is the mean across the 336 camera sites. The ground-height
* sidecar has no Toronto entries yet, so this prior seeds placement; a camera
* that gains a sidecar entry uses it instead, and the client's ground snap
* corrects the prior wherever 3D tiles are loaded. */
export const TORONTO_GROUND_ELEVATION_M = 116;
/** Hard ceiling on the RESCU list body. The whole city is ~336 rows and under
* 60 KB; this only exists so an upstream that streams an unbounded body cannot
* be buffered without limit. */
export const TORONTO_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** WSDOT Travel Center cameras (Washington State): keyless Esri JSON on the
* official open-data host — ~1,700 point features in EPSG:3857 (Web Mercator),
* ETag/Last-Modified served. WSDOT's metadata marks the feed "low volume" use;
* the catalog is fetched once per CCTV_SOURCE_CACHE_MS refresh. */
export const WSDOT_CAMERAS_URL =
'https://data.wsdot.wa.gov/travelcenter/Cameras.json';
/** Only frames on WSDOT's own image host are registered; the catalog also
* lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports)
* whose own terms apply. */
export const WSDOT_IMAGE_ORIGIN = 'https://images.wsdot.wa.gov/';
export const DEFAULT_WSDOT_MAX_SOURCES = 250;
/** Hard ceiling on the WSDOT catalog body. The live layer is ~400 KB; this
* only exists so an upstream that streams an unbounded body cannot be
* buffered without limit. */
export const WSDOT_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Prioritization anchors: downtown Seattle and Spokane. */
export const WSDOT_ANCHORS = [
{ lat: 47.6062, lon: -122.3321 }, // Seattle
{ lat: 47.6588, lon: -117.426 }, // Spokane
];
/** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */
export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000;
/** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one
server/providers/cctv/lithuania.js
+154
View File
@@ -0,0 +1,154 @@
import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js';
import {
cameraDisplayCode,
fallbackHeadingFromId,
prioritizeSources,
} from './normalize.js';
import { readResponseJsonCapped } from '../common/http.js';
const BASE = 'https://eismoinfo.lt/eismoinfo-backend';
export const LITHUANIA_CAMERAS_URL = `${BASE}/camera-info-table`;
// Without lks=true this endpoint returns [latitude, longitude]. The camera
// table's x/y fields use LKS-94 metres and must never be treated as degrees.
export const LITHUANIA_LOCATIONS_URL = `${BASE}/layer-static-features/VKR`;
export const LITHUANIA_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
const FRAME_PATH = '/eismoinfo-backend/image-provider/camera/last';
const ANCHORS = [
{ lat: 54.6872, lon: 25.2797 }, // Vilnius
{ lat: 54.8985, lon: 23.9036 }, // Kaunas
{ lat: 55.7033, lon: 21.1443 }, // Klaipeda
{ lat: 55.9349, lon: 23.3137 }, // Siauliai
{ lat: 55.7348, lon: 24.3575 }, // Panevezys
];
/** Join public catalogs by ID; register only the official still endpoint. */
export function lithuaniaCameraToSource(row, feature) {
const id = String(row?.id ?? '');
if (!/^[1-9]\d{0,8}$/.test(id) || String(feature?.id) !== id) return null;
const point = feature?.points?.[0]?.point;
if (!Array.isArray(point) || point.length !== 2) return null;
const [lat, lon] = point;
if (
!Number.isFinite(lat) ||
!Number.isFinite(lon) ||
lat < 53.8 ||
lat > 56.5 ||
lon < 20.8 ||
lon > 26.9
)
return null;
try {
const url = new URL(row.image);
if (
url.origin !== 'https://eismoinfo.lt' ||
url.username ||
url.password ||
url.pathname !== FRAME_PATH ||
url.searchParams.get('id') !== id
)
return null;
} catch {
return null;
}
// Reconstruct so extra query parameters cannot alter the request.
const imageUrl = `https://eismoinfo.lt${FRAME_PATH}?id=${id}`;
const cameraId = `lithuania-${id}`;
const name = String(row.name || feature.name || `Lietuva ${id}`)
.trim()
.slice(0, 160);
return {
id: cameraId,
name,
city: 'Lithuania / Lietuva',
cityId: 'lithuania',
provider: 'Via Lietuva / Eismoinfo',
lat,
lon,
// No calibrated pose is published; use the app's low-confidence prior.
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'eismoinfo-public',
credit:
'Via Lietuva / eismoinfo.lt — periodically updated road-camera stills',
code: cameraDisplayCode(name.toUpperCase()),
};
}
async function readCatalog(url, sibling) {
const signal = AbortSignal.any([
AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
sibling.signal,
]);
try {
const response = await fetch(url, {
headers: { Accept: 'application/json' },
redirect: 'manual',
signal,
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(
response.status >= 300 && response.status < 400
? 'catalog redirected; redirects are not followed'
: `Catalog HTTP ${response.status}`,
);
}
return await readResponseJsonCapped(
response,
LITHUANIA_MAX_CATALOG_BYTES,
signal,
);
} catch (error) {
// The join needs both catalogs; release the other request at once
// rather than leaving its body open until the timeout.
sibling.abort();
throw error;
}
}
/** Two bounded requests per shared catalog refresh, no API key required. */
export async function loadLithuaniaSources() {
try {
const sibling = new AbortController();
const [rows, layers] = await Promise.all([
readCatalog(LITHUANIA_CAMERAS_URL, sibling),
readCatalog(LITHUANIA_LOCATIONS_URL, sibling),
]);
if (!Array.isArray(rows) || !Array.isArray(layers)) return [];
const features = layers.find((layer) => layer?.layer === 'VKR')?.features;
if (!Array.isArray(features)) return [];
const locations = new Map(
features.filter(Boolean).map((feature) => [String(feature.id), feature]),
);
const cameras = new Map();
for (const row of rows) {
const camera = lithuaniaCameraToSource(
row,
locations.get(String(row?.id)),
);
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
}
const rawCap = Number(process.env.CCTV_LITHUANIA_MAX_SOURCES || 400);
const cap = Number.isFinite(rawCap)
? Math.max(1, Math.min(1000, Math.floor(rawCap)))
: 400;
const sources = prioritizeSources([...cameras.values()], cap, ANCHORS);
console.log(
`[CCTV] Loaded Lithuania camera sources: ${cameras.size} (using ${sources.length})`,
);
return sources;
} catch (error) {
console.warn(
'[CCTV] Lithuania camera download error:',
error?.message || error,
);
return [];
}
}
server/providers/cctv/normalize.js
+63
View File
@@ -394,6 +394,44 @@ export function isLikelyFinlandCoordinate(lat, lon) {
return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32;
}
/** Catalonia bounding box (mainland is roughly 40.5..42.9 N, 0.15..3.4 E). */
export function isLikelyCataloniaCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 40.3 &&
lat <= 42.9 &&
lon >= 0.0 &&
lon <= 3.4
);
}
/**
* Decode numeric HTML character references, decimal ("&#237;") and hex
* ("&#xED;") alike, the entity form the Catalonia feed uses for accented
* place names, plus the handful of named entities XML/HTML text commonly
* carries. A reference that names no valid character (NUL, a surrogate, or
* anything past U+10FFFF) is left as written rather than thrown on, so one
* bad field cannot fail a whole catalog.
*
* @param {string} text
* @returns {string}
*/
export function decodeNumericEntities(text) {
return String(text || '')
.replace(/&#(?:x([0-9a-f]+)|(\d+));/gi, (match, hex, dec) => {
const code = hex ? Number.parseInt(hex, 16) : Number(dec);
const valid =
code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff);
return valid ? String.fromCodePoint(code) : match;
})
.replace(/&amp;/g, '&')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.trim();
}
/**
* Human label for one Fintraffic preset (camera view).
*
@@ -562,5 +600,30 @@ export function normalizeSourceItem(item) {
// badge can distinguish them from raw automated priors (e.g. Austin Open
// Data, which never sets this field). Passed through as-is to the client.
poseSource: item.poseSource === 'curated' ? 'curated' : undefined,
// Optional hardware-model passthrough (additive, like poseSource): packs
// whose catalog publishes the camera's hardware model — King County's
// layer carries Manufacturer/Model per camera — pass it through so
// identified-hardware consumers can use it. Absent everywhere else.
model:
typeof item.model === 'string' && item.model.trim()
? item.model.trim()
: undefined,
// Bare model strings collide across vendors (e.g. "3950"), so the
// publishing pack's manufacturer rides along for disambiguation.
manufacturer:
typeof item.manufacturer === 'string' && item.manufacturer.trim()
? item.manufacturer.trim()
: undefined,
};
}
/** City of Toronto bounding box (Steeles to the lake, Etobicoke to Scarborough). */
export function isLikelyTorontoCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 43.55 &&
lat <= 43.9 &&
lon >= -79.7 &&
lon <= -79.1
);
}
server/providers/cctv/sources.js
+770 -2
View File
@@ -55,10 +55,34 @@ import {
DEFAULT_CALGARY_MAX_SOURCES,
CALGARY_DOWNTOWN,
CALGARY_MAX_CATALOG_BYTES,
CATALONIA_CAMERAS_URL,
CATALONIA_MAX_CATALOG_BYTES,
CATALONIA_IMAGE_HOSTS,
CATALONIA_SCT_CAMERA_FILE_RE,
CATALONIA_FONT_CREDIT,
CATALONIA_FONT_ELEVATION_M,
DEFAULT_CATALONIA_MAX_SOURCES,
CATALONIA_ANCHORS,
TORONTO_RESCU_LIST_URL,
TORONTO_RESCU_IMAGE_ORIGIN,
DEFAULT_TORONTO_MAX_SOURCES,
TORONTO_DOWNTOWN,
TORONTO_GROUND_ELEVATION_M,
TORONTO_MAX_CATALOG_BYTES,
CCTV_SOURCE_FETCH_TIMEOUT_MS,
DELDOT_CCTV_URL,
DEFAULT_DELDOT_MAX_SOURCES,
DELDOT_ANCHORS,
KING_COUNTY_CAMERAS_URL,
KING_COUNTY_IMAGE_ORIGIN,
DEFAULT_KING_COUNTY_MAX_SOURCES,
KING_COUNTY_ANCHORS,
KING_COUNTY_MAX_CATALOG_BYTES,
WSDOT_CAMERAS_URL,
WSDOT_IMAGE_ORIGIN,
DEFAULT_WSDOT_MAX_SOURCES,
WSDOT_ANCHORS,
WSDOT_MAX_CATALOG_BYTES,
DEFAULT_VEGVESEN_CCTV_URL,
VEGVESEN_IMAGE_ORIGIN,
VEGVESEN_VIDEO_URL,
@@ -100,14 +124,21 @@ import {
isLikelyTexasCoordinate,
isLikelyNswCoordinate,
isLikelyCalgaryCoordinate,
isLikelyNorwayCoordinate,
isLikelyCataloniaCoordinate,
decodeNumericEntities,
isLikelyNycCoordinate,
isLikelyNorwayCoordinate,
isLikelyTorontoCoordinate,
cameraDisplayCode,
rowArrayToObject,
prioritizeSources,
} from './normalize.js';
import { directionToHeading } from '../../../src/data/directionText.js';
import { readResponseJsonCapped } from '../common/http.js';
import {
readResponseBytesCapped,
readResponseJsonCapped,
readResponseTextCapped,
} from '../common/http.js';
/**
* Fetch and parse Austin traffic camera records from the city Open Data portal.
*
@@ -1624,6 +1655,261 @@ export async function loadCalgarySourcesFromOpenData() {
}
}
/**
* Parse the Catalonia (`cameres.xml`) WFS/GML feed into raw feature records.
* Regex-based rather than a full XML parser — the feed is small (~160
* features) and this avoids a new dependency for one field set per feature.
*
* @param {string} xml
* @returns {Array<{fid:string, lat:number, lon:number, carretera:string, municipi:string, pk:string, link:string, font:string}>}
*/
export function parseCataloniaXml(xml) {
const out = [];
const blockRe = /<gml:featureMember>([\s\S]*?)<\/gml:featureMember>/g;
const tag = (block, name) => {
const match = new RegExp(`<cite:${name}>([^<]*)<\\/cite:${name}>`).exec(
block,
);
return match ? decodeNumericEntities(match[1]) : '';
};
let match;
while ((match = blockRe.exec(String(xml || ''))) !== null) {
const block = match[1];
const fidMatch = /<cite:cameres\s+fid="([^"]+)"/.exec(block);
const coordsMatch = /<gml:coordinates[^>]*>([^<]+)<\/gml:coordinates>/.exec(
block,
);
if (!fidMatch || !coordsMatch) continue;
const [lonRaw, latRaw] = coordsMatch[1].split(',');
out.push({
fid: fidMatch[1],
lon: toFiniteNumber(lonRaw),
lat: toFiniteNumber(latRaw),
carretera: tag(block, 'carretera'),
municipi: tag(block, 'municipi'),
pk: tag(block, 'pk'),
link: tag(block, 'link'),
font: tag(block, 'font'),
});
}
return out;
}
/**
* Validate and normalize one Catalonia feed `link` against the publisher
* host allowlist (CATALONIA_IMAGE_HOSTS), upgrading to https. Rejects
* anything else, including a host that merely resembles an allowed one, any
* URL carrying userinfo (`https://user:[email protected]/...`) or an explicit
* port — the hostname check alone would pass either, letting catalog data
* inject credentials or steer the server-side frame request to another
* service on that host — and any path outside the host's frame directory.
*
* SCT's own `RenderService` frame link is a special case, for two stacked
* reasons (both verified 2026-09-15):
* 1. It always 302s to `TransitCamera`, on a Location the upstream
* hardcodes as plain http even when RenderService itself was requested
* over https. fetchWithinHost (media.js) only follows same-origin
* redirects — same scheme included, by design, to stop an upstream
* steering a host-pinned request into a plaintext downgrade — so that
* hop is refused and the camera would render as a placeholder.
* 2. mct.gencat.cat's TLS setup itself does not complete a handshake with
* Node's fetch/OpenSSL (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though it
* is reachable over https from other clients (e.g. curl) — an upstream
* server misconfiguration this proxy cannot route around.
* TransitCamera answers the frame directly over **http**, no redirect, and
* that is the only transport this host actually serves to this runtime — so
* this builds that URL itself from RenderService's `sctidcam` id instead of
* registering the (https, then-redirecting) RenderService link. The
* `sctidcam` id must be a bare image filename (CATALONIA_SCT_CAMERA_FILE_RE).
*
* @param {string} rawUrl
* @returns {string} Normalized URL, or '' if not accepted.
*/
export function normalizeCataloniaImageUrl(rawUrl) {
let parsed;
try {
parsed = new URL(String(rawUrl || '').trim());
} catch {
return '';
}
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return '';
if (parsed.username || parsed.password || parsed.port) return '';
const dir = CATALONIA_IMAGE_HOSTS.get(parsed.hostname);
if (!dir || !parsed.pathname.startsWith(dir)) return '';
const file = parsed.pathname.slice(dir.length);
if (!file || file.includes('/')) return '';
if (parsed.hostname === 'mct.gencat.cat') {
const camId = parsed.searchParams.get('sctidcam');
if (file !== 'RenderService') return '';
if (!camId || !CATALONIA_SCT_CAMERA_FILE_RE.test(camId)) return '';
const direct = new URL('http://mct.gencat.cat/mct2bo/TransitCamera');
direct.searchParams.set('nom', camId);
direct.searchParams.set('visualitzacio', 'imatge');
return direct.toString();
}
parsed.protocol = 'https:';
return parsed.toString();
}
/**
* Stable camera id for one normalized Catalonia frame URL (the output of
* normalizeCataloniaImageUrl). The feed's `fid` is not used: it embeds an
* upstream server-start stamp and changes on every upstream restart. SCT
* cameras key on the `sctidcam` filename stem (`cat-sct-nc87`), partner
* cameras on the frame host and path (`cat-www-bcn-cat-transit-imatges-…`).
*
* @param {string} imageUrl - A URL normalizeCataloniaImageUrl accepted.
* @returns {string}
*/
export function cataloniaCameraId(imageUrl) {
const parsed = new URL(imageUrl);
const slug = (text) =>
text
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '');
if (parsed.hostname === 'mct.gencat.cat') {
const file = parsed.searchParams.get('nom') || '';
return `cat-sct-${slug(file.replace(/\.[^.]*$/, ''))}`;
}
return `cat-${slug(parsed.hostname)}-${slug(parsed.pathname)}`;
}
/**
* Human label for one Catalonia camera: "<road/place> (<municipality>)",
* skipping the parenthetical when the municipality is already named in the
* road/place text or when one half is missing.
*
* @param {string} carretera
* @param {string} municipi
* @returns {string}
*/
export function cataloniaCameraName(carretera, municipi) {
const road = String(carretera || '').trim();
const city = String(municipi || '').trim();
if (road && city) {
return road.toLowerCase().includes(city.toLowerCase())
? road
: `${road} (${city})`;
}
return road || city || 'Càmera de trànsit';
}
/**
* Fetch Catalonia traffic cameras (Servei Català de Trànsit), keyless: one
* WFS/GML XML list. The feed carries a `font` field naming the real
* publisher per camera; only SCT's own highway cameras and the hotlinked
* Barcelona (IMI) and Terrassa municipal cameras are kept — the publishers
* whose frame links were confirmed to serve images. Frame URLs are checked
* against a host allowlist (CATALONIA_IMAGE_HOSTS), so a row from any other
* publisher is dropped. The feed carries no camera status, so a camera its
* publisher has taken offline still registers and serves that publisher's
* own "image unavailable" still.
* Each non-SCT camera carries its publisher as a `credit` alongside the
* shared `provider`. The feed carries no heading, so every camera takes the
* id-hash fallback, low-confidence pose (same as TfL and Fintraffic).
*
* Ids come from the validated frame URL (cataloniaCameraId), not the feed's
* restart-dependent `fid`, and rows sharing one are collapsed before the cap
* (the live feed repeats a few SCT cameras).
*
* The list decides ids, coordinates, labels and which allowlisted frame path
* is proxied, so it is fetched over https, redirects are refused (the list
* host cannot be steered or downgraded) and the body read is capped at
* CATALONIA_MAX_CATALOG_BYTES.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadCataloniaSourcesFromOpenData() {
try {
const resp = await fetch(CATALONIA_CAMERAS_URL, {
headers: { Accept: 'application/xml,text/xml,*/*' },
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
// A response this loader will not read still owns its transport until the
// body is released, so every rejection path cancels before returning.
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn(
'[CCTV] Catalonia catalog redirected; redirects are not followed',
);
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] Catalonia camera download failed:', resp.status);
return discard();
}
const xml = await readResponseTextCapped(resp, CATALONIA_MAX_CATALOG_BYTES);
const rows = parseCataloniaXml(xml);
if (!rows.length) return [];
const cameras = [];
for (const row of rows) {
if (!isLikelyCataloniaCoordinate(row.lat, row.lon)) continue;
const imageUrl = normalizeCataloniaImageUrl(row.link);
if (!imageUrl) continue;
const font = row.font.trim();
const cameraId = cataloniaCameraId(imageUrl);
cameras.push({
id: cameraId,
name: cataloniaCameraName(row.carretera, row.municipi),
city: row.municipi.trim() || 'Catalunya',
cityId: 'catalunya',
provider: 'Servei Català de Trànsit',
lat: row.lat,
lon: row.lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
groundElevationM: CATALONIA_FONT_ELEVATION_M[font] ?? 80,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'catalonia-open-data',
license:
"Generalitat de Catalunya — Llicència oberta d'ús d'informació",
credit: CATALONIA_FONT_CREDIT[font] || '',
});
}
const unique = Array.from(
new Map(cameras.map((camera) => [camera.id, camera])).values(),
);
const maxRaw = Number(
process.env.CCTV_CATALONIA_MAX_SOURCES || DEFAULT_CATALONIA_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(600, Math.floor(maxRaw)))
: DEFAULT_CATALONIA_MAX_SOURCES;
const prioritized = prioritizeSources(unique, maxCount, CATALONIA_ANCHORS);
console.log(
`[CCTV] Loaded Catalonia camera sources: ${unique.length} (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] Catalonia camera download error:',
error?.message || error,
);
return [];
}
}
export async function loadDelDOTSourcesFromOpenData() {
try {
const resp = await fetch(DELDOT_CCTV_URL, {
@@ -1728,6 +2014,319 @@ export async function loadDelDOTSourcesFromOpenData() {
}
}
/** Rough King County, WA bounding box. */
function isLikelyKingCountyCoordinate(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
lat >= 47.0 &&
lat <= 48.0 &&
lon >= -123.1 &&
lon <= -121.0
);
}
/**
* A King County `Model` value, whitespace-collapsed, or '' when it is a
* vendor-generic string ("PTZ", "PTZ IP", "BOSCH/PTZ") that names no model.
*/
function kingCountyModel(raw) {
const model = String(raw || '')
.replace(/\s+/g, ' ')
.trim();
return /^(?:[a-z]+\/)?ptz(?: ip)?$/i.test(model) ? '' : model;
}
/**
* Fetch King County road cameras (Washington State). Keyless: one public
* ArcGIS FeatureServer GeoJSON query (King County DOT Road Services), with an
* explicit `outFields` list so the layer's asset-administration columns are
* never fetched. Only cameras that are live and active (`Live`,
* `CurrentStatus`) with a positive integer id, finite in-county coordinates,
* and a frame on the county's own image handler are kept. WSDOT-owned rows in
* the layer are skipped because their frames are WSDOT-hosted, outside the
* pinned image origin, while partner-city cameras hosted on the county image
* handler (e.g. Newcastle) name their owner as the per-camera credit,
* DriveBC-style. The layer publishes no facing, so every heading is the id
* hash at low confidence.
*
* Uniquely among the packs, the county publishes each camera's hardware
* `Manufacturer`/`Model`; the model string (whitespace-collapsed — the feed
* carries stray newlines) rides along as the source's `model` for
* identified-hardware consumers. Vendor-generic strings ("PTZ", "PTZ IP",
* "BOSCH/PTZ") name no model and are treated as absent.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadKingCountySourcesFromOpenData() {
try {
const resp = await fetch(KING_COUNTY_CAMERAS_URL, {
headers: { Accept: 'application/json' },
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
// A response this loader will not read still owns its transport until the
// body is released, so every rejection path cancels before returning.
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn(
'[CCTV] King County catalog redirected; redirects are not followed',
);
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] King County camera download failed:', resp.status);
return discard();
}
const payload = await readResponseJsonCapped(
resp,
KING_COUNTY_MAX_CATALOG_BYTES,
);
const rows = Array.isArray(payload?.features) ? payload.features : [];
const cameras = [];
const seen = new Set();
for (const row of rows) {
const attrs = row?.properties;
const rawId = attrs?.AssetID;
if (!Number.isSafeInteger(rawId) || rawId <= 0) continue;
if (attrs.Live !== 1) continue;
if (String(attrs.CurrentStatus || '').trim() !== 'Active') continue;
const owner = String(attrs.Owner || '').trim();
// WSDOT-owned rows serve WSDOT-hosted frames, outside the pinned
// county image origin, so they are skipped.
if (owner === 'WSDOT') continue;
// The handler is served over HTTPS; the feed still writes http://.
const imageUrl = String(attrs.ImageURL || '')
.trim()
.replace(/^http:\/\/info\.kingcounty\.gov\//, KING_COUNTY_IMAGE_ORIGIN);
if (!imageUrl.startsWith(KING_COUNTY_IMAGE_ORIGIN)) continue;
const [lon, lat] = Array.isArray(row?.geometry?.coordinates)
? row.geometry.coordinates
: [];
if (!isLikelyKingCountyCoordinate(lat, lon)) continue;
const cameraId = `kingcounty-${rawId}`;
if (seen.has(cameraId)) continue;
seen.add(cameraId);
const model = kingCountyModel(attrs.Model);
const manufacturer = String(attrs.Manufacturer || '')
.replace(/\s+/g, ' ')
.trim();
const camera = {
id: cameraId,
name:
String(attrs.Location || '').trim() || `King County camera ${rawId}`,
city: 'King County',
cityId: 'king-county',
provider: 'King County Road Services',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
// The low-confidence pose personality shared by the other packs.
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'kingcounty-open-data',
license: 'King County GIS open data (KCGIS Center terms of use)',
credit: owner && owner !== 'King County' ? owner : '',
};
if (model) camera.model = model;
if (model && manufacturer) camera.manufacturer = manufacturer;
cameras.push(camera);
}
const maxRaw = Number(
process.env.CCTV_KINGCOUNTY_MAX_SOURCES ||
DEFAULT_KING_COUNTY_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(1200, Math.floor(maxRaw)))
: DEFAULT_KING_COUNTY_MAX_SOURCES;
const prioritized = prioritizeSources(
cameras,
maxCount,
KING_COUNTY_ANCHORS,
);
console.log(
`[CCTV] Loaded King County camera sources: ${cameras.length} published (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] King County camera download error:',
error?.message || error,
);
return [];
}
}
/** WSDOT `CompassDirection` codes as headings in degrees. The live feed sends
* single cardinals (N/E/S/W), `B` (both directions — most rows), `O`, and
* null; the diagonals are mapped in case they appear. `B`, `O`, null, and any
* other code carry no single facing and fall through to the id hash. */
const WSDOT_COMPASS_HEADINGS = Object.freeze({
N: 0,
NE: 45,
E: 90,
SE: 135,
S: 180,
SW: 225,
W: 270,
NW: 315,
});
/**
* Converts an EPSG:3857 (Web Mercator) coordinate pair to WGS84 degrees.
* The WSDOT camera layer publishes point geometry in wkid 102100.
* @param {number} x - Web Mercator metres east.
* @param {number} y - Web Mercator metres north.
* @returns {{lat: number, lon: number}} WGS84 degrees.
*/
export function webMercatorToWgs84(x, y) {
const R = 6378137;
const lon = (x / R) * (180 / Math.PI);
const lat = (2 * Math.atan(Math.exp(y / R)) - Math.PI / 2) * (180 / Math.PI);
return { lat, lon };
}
/** Rough Washington State bounding box (with the Columbia River border cams). */
function isLikelyWashingtonCoordinate(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
lat >= 45.3 &&
lat <= 49.3 &&
lon >= -125 &&
lon <= -116.5
);
}
/**
* Fetch WSDOT highway cameras (Washington State). Keyless: one Esri JSON
* layer on WSDOT's official open-data host (the same layer the WSDOT Travel
* Center map consumes; a keyed Traveler Information API serves the identical
* population for higher-volume use). Only cameras with a positive integer id,
* finite in-state coordinates, and a frame on WSDOT's own image host are
* kept — the catalog's ~70 partner cameras (Oregon DOT, City of Seattle,
* lodges, airports) are skipped rather than proxied under someone else's
* terms. A single-facing `CompassDirection` (a cardinal) gives a
* high-confidence heading; `B` (both directions), `O`, and null fall back to
* the id hash.
* The feed carries Windows-1252 bytes (en-dashes in titles) with no charset
* declaration, so the body is decoded explicitly before parsing.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadWsdotSourcesFromOpenData() {
try {
const resp = await fetch(WSDOT_CAMERAS_URL, {
headers: { Accept: 'application/json' },
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn(
'[CCTV] WSDOT catalog redirected; redirects are not followed',
);
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] WSDOT camera download failed:', resp.status);
return discard();
}
const payload = JSON.parse(
new TextDecoder('windows-1252').decode(
await readResponseBytesCapped(resp, WSDOT_MAX_CATALOG_BYTES),
),
);
const rows = Array.isArray(payload?.features) ? payload.features : [];
const cameras = [];
for (const row of rows) {
const attrs = row?.attributes;
const rawId = attrs?.CameraID;
if (!Number.isSafeInteger(rawId) || rawId <= 0) continue;
const imageUrl = String(attrs.ImageURL || '').trim();
if (!imageUrl.startsWith(WSDOT_IMAGE_ORIGIN)) continue;
const x = row?.geometry?.x;
const y = row?.geometry?.y;
if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
const { lat, lon } = webMercatorToWgs84(x, y);
if (!isLikelyWashingtonCoordinate(lat, lon)) continue;
const cameraId = `wsdot-${rawId}`;
const heading =
WSDOT_COMPASS_HEADINGS[
String(attrs.CompassDirection || '')
.trim()
.toUpperCase()
];
const hasHeading = Number.isFinite(heading);
cameras.push({
id: cameraId,
name: String(attrs.CameraTitle || '').trim() || `WSDOT camera ${rawId}`,
city: 'Washington',
cityId: 'washington-state',
provider: 'WSDOT',
lat,
lon,
headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId),
headingConfidence: hasHeading ? 'high' : 'low',
// Same two pose personalities as the other packs: raw priors that the
// client's ground snap and manual calibration refine.
pitchDeg: hasHeading ? -24 : -18,
fovDeg: hasHeading ? 56 : 44,
rangeM: hasHeading ? 210 : 145,
mountHeightM: hasHeading ? 10 : 8,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'wsdot-open-data',
license: 'WSDOT Travel Center camera data (public low-volume feed)',
});
}
const maxRaw = Number(
process.env.CCTV_WSDOT_MAX_SOURCES || DEFAULT_WSDOT_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(1200, Math.floor(maxRaw)))
: DEFAULT_WSDOT_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, WSDOT_ANCHORS);
console.log(
`[CCTV] Loaded WSDOT camera sources: ${cameras.length} published (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] WSDOT camera download error:',
error?.message || error,
);
return [];
}
}
/**
* One Statens vegvesen `CctvSimple` feature -> one catalog source, or null.
*
@@ -2296,3 +2895,172 @@ export async function loadNycSourcesFromOpenData() {
return [];
}
}
/**
* The JSONP envelope the City of Toronto wraps its camera list in. Anchored at
* both ends, so only that exact callback is unwrapped — a payload that merely
* contains the name somewhere is not treated as the list.
*/
const RESCU_JSONP = /^jsonTMCEarthCamerasCallback\(([\s\S]*)\)\s*;?$/;
/**
* Unwrap the RESCU camera list. The portal serves the list as JSONP; should it
* ever drop the wrapper, the bare JSON document still parses. Anything else —
* a foreign callback, a truncated body, an error page — yields no cameras
* rather than a thrown loader.
*
* @param {string} text - Raw response body.
* @returns {Array<object>} The `Data` rows, or [] when the body is unusable.
*/
export function parseRescuCameraList(text) {
const body = String(text ?? '').trim();
if (!body) return [];
const unwrapped = RESCU_JSONP.exec(body)?.[1] ?? body;
try {
const parsed = JSON.parse(unwrapped);
const rows = parsed?.Data ?? parsed?.data;
return Array.isArray(rows) ? rows : [];
} catch {
return [];
}
}
/**
* Convert one RESCU list row into a camera source.
*
* The row carries no URL: the frame URL is built from the documented
* `loc####.jpg` convention and a camera number validated as plain digits, so
* the frame proxy is pinned to the city origin by construction and no upstream
* field — present or planted — can steer it.
*
* No compass heading exists in this dataset. `D1`-`D4` are the directions of
* the *comparison* images (static reference stills captured in 2016, served
* from a different path); the live frame is one pan-tilt-zoom camera pointing
* wherever the operator left it. Treating those letters as a facing would put
* a confident, wrong bearing on every camera in the city, so every row takes
* the id-hash fallback and the low-confidence pose, as headingless Austin,
* TfL, Fintraffic and Calgary cameras do.
*
* @param {object} row - One entry from the RESCU `Data` array.
* @returns {object|null} Normalized camera source, or null when unusable.
*/
export function torontoCameraToSource(row) {
if (!row || typeof row !== 'object') return null;
const number = String(row.Number ?? row.number ?? '').trim();
// Plain digits only: a number is about to become a URL path segment.
if (!/^\d{1,6}$/.test(number)) return null;
const lat = toFiniteNumber(row.Latitude ?? row.latitude);
const lon = toFiniteNumber(row.Longitude ?? row.longitude);
if (!isLikelyTorontoCoordinate(lat, lon)) return null;
const cameraId = `toronto-${number}`;
// The feed's own label, verbatim. It is upper-case ("MCCOWAN RD", "ST
// HELEN'S AVE"); title-casing it would mangle more names than it fixed.
const name =
String(row.Name ?? row.name ?? '')
.replace(/\s+/g, ' ')
.trim() || `Toronto Camera ${number}`;
return {
id: cameraId,
name,
city: 'Toronto',
cityId: 'toronto',
provider: 'City of Toronto',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
groundElevationM: TORONTO_GROUND_ELEVATION_M,
feedType: 'image',
url: `${TORONTO_RESCU_IMAGE_ORIGIN}loc${number}.jpg`,
snapshotUrl: `${TORONTO_RESCU_IMAGE_ORIGIN}loc${number}.jpg`,
sourceKind: 'toronto-rescu-open-data',
license:
'Contains information licensed under the Open Government Licence – Toronto',
// Unselected-label code: the intersection, so a camera at rest reads as a
// place rather than as its id.
code: cameraDisplayCode(name.toUpperCase()),
};
}
/**
* Fetch City of Toronto RESCU traffic cameras from the City's open data
* portal, keyless. One JSONP list for the whole city; frames are stills on the
* same portal origin.
*
* The list fetch refuses redirects (`redirect: 'manual'`) so the list host
* cannot be steered, and caps the body it will buffer.
*
* Attribution: Open Government Licence – Toronto, registered in
* src/data/dataCredits.js.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadTorontoSourcesFromOpenData() {
try {
const endpoint =
process.env.CCTV_TORONTO_LIST_URL || TORONTO_RESCU_LIST_URL;
const resp = await fetch(endpoint, {
headers: { Accept: 'application/json, text/javascript' },
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
// A response this loader will not read still owns its transport until the
// body is released, so every rejection path cancels before returning.
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn(
'[CCTV] Toronto camera list redirected; redirects are not followed',
);
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] Toronto camera download failed:', resp.status);
return discard();
}
const text = await readResponseTextCapped(resp, TORONTO_MAX_CATALOG_BYTES);
const rows = parseRescuCameraList(text);
if (!rows.length) return [];
const cameras = [];
const seen = new Set();
for (const record of rows) {
const camera = torontoCameraToSource(record);
if (!camera || seen.has(camera.id)) continue;
seen.add(camera.id);
cameras.push(camera);
}
const maxRaw = Number(
process.env.CCTV_TORONTO_MAX_SOURCES || DEFAULT_TORONTO_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(500, Math.floor(maxRaw)))
: DEFAULT_TORONTO_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, [
TORONTO_DOWNTOWN,
]);
console.log(
`[CCTV] Loaded Toronto camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] Toronto camera download error:',
error?.message || error,
);
return [];
}
}
server/providers/cyclones.js
+4 -1
View File
@@ -41,6 +41,9 @@ function number(value, min, max) {
const n = Number(value);
return Number.isFinite(n) && n >= min && n <= max ? n : null;
}
// NHC stamps an advisory with its nominal time (e.g. 03:00Z) and publishes it
// up to about half an hour before; allow that, but not a clock far ahead.
const EARLY_ADVISORY_MS = 90 * 60_000;
function iso(value, now) {
if (
typeof value !== 'string' ||
@@ -50,7 +53,7 @@ function iso(value, now) {
const n = Date.parse(value);
if (
!Number.isFinite(n) ||
n > now + 5 * 60_000 ||
n > now + EARLY_ADVISORY_MS ||
now - n > 12 * HOUR ||
new Date(n).toISOString().replace('.000Z', 'Z') !==
value.replace('.000Z', 'Z')
server/providers/firms.js
+12 -2
View File
@@ -2,6 +2,7 @@ import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js';
import { readResponseTextCapped } from './common/http.js';
/**
* NASA FIRMS live active-fire proxy with a memory + disk cache.
@@ -35,6 +36,12 @@ export function firmsProxy() {
'VIIRS_SNPP_NRT',
'MODIS_NRT',
];
/**
* Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about
* 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a
* bound, not a budget.
*/
const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024;
const CACHE_DIR = path.join(process.cwd(), '.gev-cache');
const CACHE_PATH = path.join(CACHE_DIR, 'firms.json');
@@ -83,9 +90,12 @@ export function firmsProxy() {
*/
async function fetchSource(key, source) {
const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`;
const res = await fetch(url, { signal: AbortSignal.timeout(60_000) });
const signal = AbortSignal.timeout(60_000);
const res = await fetch(url, { signal });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const records = parseFirmsCsv(await res.text());
const records = parseFirmsCsv(
await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal),
);
if (records === null) throw new Error('non-CSV upstream response');
return records;
}
server/providers/space/celestrak.js
+11 -2
View File
@@ -1,6 +1,7 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js';
import { readResponseTextCapped } from '../common/http.js';
/**
* Vite plugin: CelesTrak TLE proxy.
@@ -48,10 +49,14 @@ export function celestrakProxy() {
}
}
/** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */
const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
async function fetchUpstream(group) {
const url = celestrakTleUrl(group);
const signal = AbortSignal.timeout(20000);
const res = await fetch(url.toString(), {
signal: AbortSignal.timeout(20000),
signal,
// CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a
// descriptive User-Agent with a contact point.
headers: {
@@ -60,7 +65,11 @@ export function celestrakProxy() {
},
});
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const body = await res.text();
const body = await readResponseTextCapped(
res,
CELESTRAK_MAX_RESPONSE_BYTES,
signal,
);
// An upstream error page parses to zero TLEs — treat as failure, keep cache.
if (!/^1 /m.test(body)) throw new Error('no TLE lines in response');
return { at: Date.now(), body };
server/providers/traffic.js
+11 -4
View File
@@ -1,6 +1,7 @@
import path from 'node:path';
import { promises as fsp } from 'node:fs';
import { readResponseBytesCapped } from './common/http.js';
import {
isValidTileCoord as isValidTomTomTile,
utcDayKey as tomtomUtcDayKey,
@@ -48,6 +49,8 @@ export function tomtomProxy() {
const DEFAULT_DAILY_BUDGET = 6000;
const MEM_MAX_ENTRIES = 256;
const UPSTREAM_TIMEOUT_MS = 15000;
/** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */
const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
/** @type {Map<string, {at:number, buf:Buffer}>} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */
const mem = new Map();
@@ -143,11 +146,15 @@ export function tomtomProxy() {
'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' +
`${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`;
recordUpstreamFetch(); // attempts count — upstream bills the request either way
const res = await fetch(url, {
signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS),
});
const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS);
const res = await fetch(url, { signal });
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const buf = Buffer.from(await res.arrayBuffer());
const bytes = await readResponseBytesCapped(
res,
TOMTOM_TILE_MAX_RESPONSE_BYTES,
signal,
);
const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
if (buf.length === 0) throw new Error('empty tile body');
return buf;
}
server/providers/transitHistory.test.mjs
+23
View File
@@ -179,3 +179,26 @@ test('snapshot capacity pressure preserves members not yet visited and discloses
assert.equal(recreated.fixes.length, 1);
assert.equal(recreated.truncated, true);
});
test('a feed fleet-number mode outranks its route mode in the epoch context', (t) => {
const h = history(t);
const ttc = {
id: 'ttc',
historyRetention: true,
defaultMode: 'bus',
routeMode: (route) => (/^5\d\d$/.test(route ?? '') ? 'tram' : 'bus'),
vehicleMode: (id) =>
/^4[4-6]\d\d$/.test(id) ? 'tram' : /^\d{4}$/.test(id) ? 'bus' : 'unknown',
};
h.ingest(
ttc,
[
row(100000, '8515', { routeId: '506' }),
row(100000, '4507', { routeId: null }),
row(100000, 'x1', { routeId: '501' }),
],
100000,
);
assert.equal(h.get('ttc', '8515', 100000).epochs[0].mode, 'bus');
assert.equal(h.get('ttc', '4507', 100000).epochs[0].mode, 'tram');
assert.equal(h.get('ttc', 'x1', 100000).epochs[0].mode, 'tram');
});