Merge upstream/main into feat/cctv-a22-autobrennero

Resolve conflicts with the NYC, Toronto, Lithuania, Florida, Taiwan and
Kartverket work on main. The CHANGELOG entry moves to
changelog.d/956-a22-brennero-webcams.md, as main now asks.
This commit is contained in:
Said Turksever committed 2026-10-30 14:50:41 +00:00
commit 38302a27a4
1369 files changed
+189595 -25591

No files matched your search

+108 -15
View File
@@ -4,6 +4,8 @@ import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js';
import { allocateSourceCap, resolveCatalogCap } from './cap.js';
import { loadGroundHeights, joinGroundHeights } from './groundHeights.js';
import { cameraRegion, normalizeSourceItem } from './normalize.js';
import { loadLithuaniaSources } from './lithuania.js';
import { loadTaiwanSources } from './taiwan.js';
import {
loadAustinSourcesFromOpenData,
loadCaltransSourcesFromOpenData,
@@ -17,20 +19,33 @@ import {
loadWarendorfSourcesFromCatalog,
loadNswSourcesFromOpenData,
loadCalgarySourcesFromOpenData,
loadCataloniaSourcesFromOpenData,
loadHkSourcesFromOpenData,
loadDelDOTSourcesFromOpenData,
loadKingCountySourcesFromOpenData,
loadWsdotSourcesFromOpenData,
loadVegvesenSourcesFromOpenData,
loadNycSourcesFromOpenData,
loadTorontoSourcesFromOpenData,
loadFloridaSourcesFromOpenData,
loadA22SourcesFromOpenDataHub,
} from './sources.js';
/** Env kill switch: unset or anything but "0" means enabled. */
const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0';
/** Env opt-in: only "1" enables. For packs whose feed access needs an
* operator agreement or whose terms are unconfirmed, so an unset flag never
* reaches the upstream. */
const envOptIn = (name) => String(process.env[name] || '').trim() === '1';
/**
* Live open-data packs, in merge order. Adding a region is one entry here
* plus its loader in sources.js; the catalog cap is shared across entries
* round-robin (cap.js), so a new pack never silently evicts an older one.
* plus its loader (in sources.js or its own module); the catalog cap is
* shared across entries round-robin (cap.js), so a new pack never silently
* evicts an older one.
* Each pack fails independently (allSettled) and is gated by its own env
* kill switch.
* kill switch, or by an opt-in flag where access needs an agreement or the
* terms are unconfirmed.
*/
const LIVE_PACKS = [
{ name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData },
@@ -89,16 +104,68 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_CALGARY_ENABLED'),
load: loadCalgarySourcesFromOpenData,
},
{
name: 'catalonia',
enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'),
load: loadCataloniaSourcesFromOpenData,
},
{
name: 'hk',
enabled: () => envEnabled('CCTV_HK_ENABLED'),
load: loadHkSourcesFromOpenData,
},
{
name: 'deldot',
enabled: () => envEnabled('CCTV_DELDOT_ENABLED'),
load: loadDelDOTSourcesFromOpenData,
},
{
name: 'kingcounty',
enabled: () => envEnabled('CCTV_KINGCOUNTY_ENABLED'),
load: loadKingCountySourcesFromOpenData,
},
{
name: 'wsdot',
enabled: () => envEnabled('CCTV_WSDOT_ENABLED'),
load: loadWsdotSourcesFromOpenData,
},
{
name: 'vegvesen',
enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'),
load: loadVegvesenSourcesFromOpenData,
},
{
// NYC DOT requires a data-sharing agreement for this feed, so the pack is
// off unless the operator opts in (see DATA_SOURCES.md).
name: 'nyc',
enabled: () => envOptIn('CCTV_NYC_ENABLED'),
load: loadNycSourcesFromOpenData,
},
{
name: 'toronto',
enabled: () => envEnabled('CCTV_TORONTO_ENABLED'),
load: loadTorontoSourcesFromOpenData,
},
{
// Opt-in until the Via Lietuva open-data terms are confirmed to cover the
// camera photos (see DATA_SOURCES.md).
name: 'lithuania',
enabled: () => envOptIn('CCTV_LITHUANIA_ENABLED'),
load: loadLithuaniaSources,
},
{
// FL511's terms limit its content to individual, personal use (other use
// needs FDOT's written consent), so the pack is off unless the operator
// opts in (see DATA_SOURCES.md).
name: 'florida',
enabled: () => envOptIn('CCTV_FLORIDA_ENABLED'),
load: loadFloridaSourcesFromOpenData,
},
{
name: 'taiwan',
enabled: () => envEnabled('CCTV_TAIWAN_ENABLED'),
load: loadTaiwanSources,
},
{
name: 'a22',
enabled: () => envEnabled('CCTV_A22_ENABLED'),
@@ -155,16 +222,19 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
/** @type {Promise<Array<object>>|null} In-flight refresh, shared by concurrent
* callers so a post-TTL burst launches ONE refetch, not one per request. */
let _cctvSourceInflight = null;
/** @type {Array<{pack: string, available: number, served: number}>} Packs
* that serve fewer cameras than they offer. */
let _cctvTrimmedPacks = [];
/** @type {WeakMap<Array<object>, Array<{pack: string, available: number, served: number}>>}
* Packs that serve fewer cameras than they offer, keyed by the source list
* they describe, so a request served the old list during a background
* refresh never pairs it with the new list's trim notes. */
const _cctvTrimmedPacks = new WeakMap();
/**
* Assemble and cache the merged CCTV source list.
*
* Merges every source pack (live open-data packs, local file, env
* variable), deduplicates by ID, shares the catalog cap fairly across
* packs, and caches for CCTV_SOURCE_CACHE_MS.
* packs, and caches for CCTV_SOURCE_CACHE_MS. Past that, the cached list is
* returned while a background refresh replaces it; only a first load waits.
*
* @returns {Promise<Array<object>>} Deduplicated, capped source list.
*/
@@ -179,11 +249,31 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
// Single-flight: a burst of requests arriving past the TTL shares ONE refresh
// instead of each launching the full multi-provider refetch. The `.finally`
// clears the ref so the next post-TTL cycle starts fresh.
if (_cctvSourceInflight) return _cctvSourceInflight;
_cctvSourceInflight = refreshCctvSources().finally(() => {
_cctvSourceInflight = null;
});
return _cctvSourceInflight;
if (!_cctvSourceInflight) {
_cctvSourceInflight = refreshCctvSources().finally(() => {
_cctvSourceInflight = null;
});
// A background refresh has no caller waiting on it: report its failure
// here, once per refresh rather than once per request it serves. Keep
// the old catalog for another TTL, as an empty refresh does, so a refresh
// that keeps failing is retried once per TTL, not back to back.
if (_cctvSourceCache.length) {
_cctvSourceInflight.catch((error) => {
_cctvSourceCacheAt = Date.now();
console.warn(
'[CCTV] background catalog refresh failed; serving',
_cctvSourceCache.length,
'cameras from the last catalog:',
error?.message || error,
);
});
}
}
// With a catalog in hand, answer with it while the refresh runs: the
// refresh lasts as long as the slowest pack (Tarktee's DATEX feeds take up
// to its 30 s timeout), and every /api/cctv request would otherwise wait
// for it. Only the first load, with nothing to serve yet, waits.
return _cctvSourceCache.length ? _cctvSourceCache : _cctvSourceInflight;
}
/**
@@ -273,7 +363,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
}
if (capped.length > 0 || _cctvSourceCache.length === 0) {
_cctvSourceCache = capped;
_cctvTrimmedPacks = trimmedPacks;
_cctvTrimmedPacks.set(capped, trimmedPacks);
} else {
// Every source came back empty (all live packs timed out / upstream outage)
// but a good catalog is already cached — serve it stale rather than blanking
@@ -287,7 +377,10 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
return _cctvSourceCache;
}
/** Packs serving fewer cameras than they offer, as of the last refresh. */
getCctvSources.trimmedPacks = () => _cctvTrimmedPacks;
/** Packs serving fewer cameras than they offer in `sources` (by default the
* current catalog). Pass the list a request was served to keep the two in
* step while a background refresh swaps the catalog. */
getCctvSources.trimmedPacks = (sources = _cctvSourceCache) =>
_cctvTrimmedPacks.get(sources) ?? [];
return getCctvSources;
}
+264 -7
View File
@@ -8,8 +8,8 @@ export const DEFAULT_AUSTIN_MAX_SOURCES = 250;
* Catalog-wide safety ceiling on served cameras. Each pack already caps
* itself (nearest-to-anchor first); this bound only matters when the packs
* together exceed it, and it is then filled round-robin across packs (see
* cap.js) so no region is silently dropped. Sized above the sum of the
* default per-pack caps so a default install never trims.
* cap.js) so no region is silently dropped. The default per-pack caps
* together can exceed it, so a default install may be thinned evenly here.
*/
export const DEFAULT_CCTV_MAX_SOURCES = 4000;
/** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */
@@ -18,7 +18,7 @@ export const CCTV_MAX_SOURCES_CEILING = 5000;
export const AUSTIN_DOWNTOWN = { lat: 30.2672, lon: -97.7431 };
/** Caltrans CCTV: one JSON feed per district, identical schema statewide. */
/** TxDOT ITS: one keyless JSON catalog per district (25 districts statewide). */
export const TXDOT_ORIGIN = 'https://its.txdot.gov';
const TXDOT_ORIGIN = 'https://its.txdot.gov';
export const TXDOT_CCTV_STATUS_URL = (district) =>
`${TXDOT_ORIGIN}/its/DistrictIts/GetCctvStatusListByDistrict?districtCode=${encodeURIComponent(district)}`;
/** Per-camera frame. Returns JSON `{snippet:<base64 jpeg>}`, not an image body;
@@ -113,13 +113,34 @@ export const CALTRANS_ANCHORS = [
{ lat: 32.7157, lon: -117.1611 }, // San Diego
{ lat: 38.5816, lon: -121.4944 }, // Sacramento
];
export const FLORIDA_CCTV_URL =
'https://services.arcgis.com/3wFbqsFPLeKqOlIK/arcgis/rest/services/FL511_Traffic_Cameras/FeatureServer/0/query';
export const FLORIDA_IMAGE_HOSTS = [
'images-dis.divas.cloud',
'snapshots.divas.cloud',
];
export const DEFAULT_FLORIDA_MAX_SOURCES = 500;
/** Byte cap per catalog page (2,000 rows, about 650 KiB measured). */
export const FLORIDA_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
/** Catalog page ceiling: 40 pages covers 4,000+ cameras even at 100 rows a page. */
export const FLORIDA_MAX_CATALOG_PAGES = 40;
export const FLORIDA_ANCHORS = [
{ lat: 25.7617, lon: -80.1918 }, // Miami
{ lat: 28.5384, lon: -81.3789 }, // Orlando
{ lat: 27.9506, lon: -82.4572 }, // Tampa
{ lat: 30.3322, lon: -81.6557 }, // Jacksonville
{ lat: 30.4383, lon: -84.2807 }, // Tallahassee
{ lat: 26.1224, lon: -80.1373 }, // Fort Lauderdale
{ lat: 26.7153, lon: -80.0534 }, // West Palm Beach
{ lat: 30.4213, lon: -87.2169 }, // Pensacola (panhandle)
];
/** TfL JamCams: one keyless list endpoint; frames live on a public S3 bucket. */
export const TFL_JAMCAM_URL = 'https://api.tfl.gov.uk/Place/Type/JamCam';
export const TFL_IMAGE_ORIGIN =
'https://s3-eu-west-1.amazonaws.com/jamcams.tfl.gov.uk/';
export const DEFAULT_TFL_MAX_SOURCES = 250;
export const LONDON_CENTER = { lat: 51.5074, lon: -0.1278 };
/** Ontario 511: keyless CARS/511 camera catalog; frame URLs are still images. */
/** Ontario 511: CARS/511 camera catalog, which needs a developer key (ONTARIO_511_API_KEY); frame URLs are still images. */
export const ONTARIO_511_CAMERAS_URL =
'https://511on.ca/api/v2/get/cameras?format=json&lang=en';
export const ONTARIO_511_IMAGE_ORIGIN = 'https://511on.ca/map/Cctv/';
@@ -188,6 +209,8 @@ export const TARKTEE_IMAGES_URL =
'https://tarktee.transpordiamet.ee/api/v1/datex/roadCameraImages';
export const TARKTEE_IMAGE_ORIGIN = 'https://tarktee.transpordiamet.ee/images/';
export const DEFAULT_TARKTEE_MAX_SOURCES = 179;
/** Hard ceiling per Tarktee DATEX XML document (~120–165 KB live). */
export const TARKTEE_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
export const TARKTEE_ANCHORS = [
{ lat: 59.437, lon: 24.753 }, // Tallinn
{ lat: 58.378, lon: 26.729 }, // Tartu
@@ -236,6 +259,86 @@ export const CALGARY_DOWNTOWN = { lat: 51.0461, lon: -114.0626 };
* body cannot be buffered without limit. */
export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/**
* Catalonia traffic cameras (Servei Català de Trànsit): one keyless WFS/GML
* XML list (verified 2026-09-15, hourly catalog refresh), fetched over https
* (verified 2026-09-27: same 200 body, no redirect). The feed carries a
* `font` field naming the actual publisher per camera — SCT's own highway
* cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras are
* kept, each confirmed to return a real frame. Frame URLs are checked
* against a host allowlist rather than one pinned origin, since each
* publisher serves from its own host.
*/
export const CATALONIA_CAMERAS_URL =
'https://www.gencat.cat/transit/opendata/cameres.xml';
/** Byte ceiling for the Catalonia catalog read (~90 KB observed 2026-09-27),
* so a runaway body cannot be buffered without limit. */
export const CATALONIA_MAX_CATALOG_BYTES = 1024 * 1024;
/** Allowed image hosts, one per kept `font` publisher in the feed, each
* mapped to the one directory its frame links live in (every link in the
* 2026-10 feed shares it); a link must name a single file directly inside. */
export const CATALONIA_IMAGE_HOSTS = Object.freeze(
new Map([
['mct.gencat.cat', '/mct2bo/'], // SCT
['www.bcn.cat', '/transit/imatges/'], // IMI (Ajuntament de Barcelona)
['emap.terrassa.cat', '/it_terrassa/'], // Ajuntament de Terrassa
]),
);
/** SCT `sctidcam` values: a bare image filename, which also keys the camera
* id (the feed's `fid` embeds an upstream server-start stamp). */
export const CATALONIA_SCT_CAMERA_FILE_RE = /^[a-z0-9]+\.(gif|jpe?g)$/i;
/** Per-`font` partner credit, shown beside the provider in the CCTV panel
* (same pattern as DriveBC's partner-supplied cameras). SCT is the feed's
* own publisher and carries no separate credit. */
export const CATALONIA_FONT_CREDIT = Object.freeze({
IMI: 'Ajuntament de Barcelona',
Terrassa: 'Ajuntament de Terrassa',
});
/** Ground-elevation priors in metres, by `font`: the feed carries no
* elevation, and this pack spans sea level (Barcelona) to Terrassa's plateau
* (~270 m). The client's ground snap corrects these on 3D-tile stacks. */
export const CATALONIA_FONT_ELEVATION_M = Object.freeze({
SCT: 80,
IMI: 20,
Terrassa: 270,
});
export const DEFAULT_CATALONIA_MAX_SOURCES = 150;
/** Prioritization anchors: Barcelona and Terrassa. */
export const CATALONIA_ANCHORS = [
{ lat: 41.3874, lon: 2.1686 }, // Barcelona
{ lat: 41.5636, lon: 2.0111 }, // Terrassa
];
/** Hong Kong Transport Department traffic snapshot cameras: keyless English
* locations XML on DATA.GOV.HK; frames are stills on the TD CCTV host. */
export const DEFAULT_HK_CAMERAS_URL =
'https://static.data.gov.hk/td/traffic-snapshot-images/code/Traffic_Camera_Locations_En.xml';
/** The only origin HK camera frames may come from. Verified 2026-10-11: still
* URLs answer 200 on this host with no redirect hop. */
export const HK_IMAGE_ORIGIN = 'https://tdcctv.data.one.gov.hk/';
/** Above the full TD English catalog (~1,013 cameras) so a default install
* keeps nearly all of them. Catalog-wide round-robin still applies when the
* sum of packs exceeds DEFAULT_CCTV_MAX_SOURCES (Norway-style). */
export const DEFAULT_HK_MAX_SOURCES = 1100;
/** Prioritization anchors across Hong Kong Island, Kowloon and the New
* Territories so a lowered cap keeps territory-wide coverage rather than
* only the harbour corridor. */
export const HK_ANCHORS = [
{ lat: 22.2819, lon: 114.158 }, // Central
{ lat: 22.2976, lon: 114.1722 }, // Tsim Sha Tsui
{ lat: 22.3827, lon: 114.19 }, // Sha Tin
{ lat: 22.3701, lon: 114.114 }, // Tsuen Wan
{ lat: 22.4503, lon: 114.1688 }, // Tai Po
{ lat: 22.493, lon: 114.139 }, // Fanling
{ lat: 22.391, lon: 113.977 }, // Tuen Mun
{ lat: 22.4445, lon: 114.022 }, // Yuen Long
{ lat: 22.307, lon: 114.26 }, // Tseung Kwan O
];
/** Hard ceiling on the locations XML body. The live English list is ~400 KB
* for ~1,000 cameras; this only exists so an unbounded upstream cannot be
* buffered without limit. */
export const HK_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Statens vegvesen (Norway) road cameras: the keyless OGC API Features view
* of the official DATEX 3.1 CCTV site table, one GeoJSON request for the whole
* country (~900 cameras). The raw DATEX node needs an account; this view does
@@ -249,9 +352,9 @@ export const VEGVESEN_IMAGE_ORIGIN =
* per-camera manifest path is registered. */
export const VEGVESEN_VIDEO_URL = (cameraId) =>
`https://kamera.vegvesen.no/public/${cameraId}/manifest.m3u8`;
/** Above the ~850 working cameras so a default install keeps them all, while
* staying close to them: the per-pack defaults are meant to sum to no more
* than DEFAULT_CCTV_MAX_SOURCES. */
/** Above the ~850 working cameras so the pack keeps them all on its own. The
* catalog-wide DEFAULT_CCTV_MAX_SOURCES cap still applies across packs and is
* filled round-robin, so a full default install may thin this pack. */
export const DEFAULT_VEGVESEN_MAX_SOURCES = 900;
/** Hard ceiling on the catalog body. The whole country is ~800 KB. */
export const VEGVESEN_MAX_CATALOG_BYTES = 8 * 1024 * 1024;
@@ -267,6 +370,79 @@ export const NORWAY_ANCHORS = [
{ lat: 69.6492, lon: 18.9553 }, // Tromsø
];
/** Kartverket's keyless point-height API (CC BY 4.0): terrain height above
* sea level (NN2000), used as the Vegvesen cameras' ground prior because the
* camera feed's points are 2D. */
export const KARTVERKET_HEIGHT_URL =
'https://ws.geonorge.no/hoydedata/v1/punkt';
/** Points per request. The API takes up to 50, but above ~20 it answers the
* extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not
* at all; at 20 nearly every camera resolves from the 1 m terrain model. */
export const KARTVERKET_MAX_POINTS = 20;
/** Requests in flight at once. The whole country is ~36 requests of
* 0.6-9.5 s each, so a cold lookup takes ~6-13 s. */
export const KARTVERKET_CONCURRENCY = 8;
/** Deadline for one complete lookup, shared by every request. A lookup cut
* short by it is resumed on the next refresh, without a backoff. */
export const KARTVERKET_LOOKUP_TIMEOUT_MS = 30 * 1000;
/** One request that takes longer than this counts as a failing service. */
export const KARTVERKET_REQUEST_TIMEOUT_MS = 10 * 1000;
/** The Vegvesen lane's budget, measured from the start of the loader (catalog
* fetch included), for waiting on heights. Under the 15 s per-provider fetch
* timeout so this pack is never the catalog's slowest lane. A lookup still
* running after it keeps filling the cache for the next refresh. */
export const KARTVERKET_CATALOG_WAIT_MS = 12 * 1000;
/** The lane only waits when more than this share of its cameras has no
* answer yet (a cold cache). A few new cameras or expired misses are looked
* up in the background instead of holding the refresh. */
export const KARTVERKET_WAIT_MISSING_SHARE = 0.1;
/** A point Kartverket has no height for (some ferry quays and bridges) is not
* asked about again for this long. */
export const KARTVERKET_NULL_TTL_MS = 6 * 60 * 60 * 1000;
/** After a failing request (408, 429, 5xx, a network error, a request
* timeout or an unreadable body), no lookup starts for this long, or for the
* Retry-After the service sent, bounded to [1 min, 1 h]. Longer than the
* 15-minute catalog refresh (CCTV_SOURCE_CACHE_MS), so a failing service
* costs one attempt per backoff, not one per refresh. */
export const KARTVERKET_FAILURE_BACKOFF_MS = 30 * 60 * 1000;
export const KARTVERKET_MAX_BACKOFF_MS = 60 * 60 * 1000;
export const KARTVERKET_MAX_RESPONSE_BYTES = 256 * 1024;
/** Plausible Norwegian terrain: below sea level only at the shore, and no road
* camera sits above ~1,500 m. Anything outside is treated as no height. */
export const KARTVERKET_HEIGHT_RANGE_M = Object.freeze({ min: -20, max: 2500 });
/** Same identity the other CCTV upstream requests send. */
export const KARTVERKET_USER_AGENT = 'gods-eye-view-cctv-proxy/1.0';
/** NYC DOT Traffic Management Center cameras: one keyless JSON list for the
* five boroughs (~980 cameras). Frames are stills synthesized from each
* camera UUID on the city's webcam host — never taken from the payload. */
export const NYC_CAMERAS_URL = 'https://webcams.nyctmc.org/api/cameras';
/** The only origin NYC camera frames may come from. The catalog's `imageUrl`
* is ignored; the registered path is always this origin plus a validated
* UUID plus `/image`. */
export const NYC_IMAGE_ORIGIN = 'https://webcams.nyctmc.org/api/cameras/';
export const NYC_IMAGE_URL = (id) => `${NYC_IMAGE_ORIGIN}${id}/image`;
export const NYC_CAMERA_ID_RE =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
export const DEFAULT_NYC_MAX_SOURCES = 250;
/** Hard ceiling on the NYC catalog body. The live list is ~980 rows / ~250 KB;
* this only exists so an upstream that streams an unbounded body cannot be
* buffered without limit. */
export const NYC_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Ground-elevation prior in metres. Matches the New York CITY_POIS prior;
* the client's one-shot ground snap corrects it wherever 3D tiles load. */
export const NYC_GROUND_ELEVATION_M = 10;
/** Prioritization anchors: one per borough so a cap keeps citywide coverage
* rather than Midtown alone. */
export const NYC_ANCHORS = [
{ lat: 40.758, lon: -73.9855 }, // Manhattan (Times Square)
{ lat: 40.7128, lon: -74.006 }, // Lower Manhattan
{ lat: 40.6782, lon: -73.9442 }, // Brooklyn
{ lat: 40.7282, lon: -73.7949 }, // Queens
{ lat: 40.8448, lon: -73.8648 }, // Bronx
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
];
/** Autostrada del Brennero (A22, Brenner–Modena) webcams: the keyless Open
* Data Hub (NOI Techpark, South Tyrol) tourism API republishes Autobrennero's
* own camera list with coordinates, one request for the whole motorway
@@ -308,6 +484,83 @@ export const DELDOT_ANCHORS = [
{ lat: 39.1582, lon: -75.5244 }, // Dover (Kent)
{ lat: 38.6906, lon: -75.3877 }, // Georgetown (Sussex)
];
/** King County, WA road cameras: keyless public ArcGIS FeatureServer layer
* ("King County and WSDOT Traffic Cameras", King County DOT Road Services).
* GeoJSON with WGS84 point geometry. `outFields` is an explicit list on
* purpose: the layer also carries asset-administration columns (including
* staff names in `ModifiedBy`) that must never be fetched. Uniquely among the
* packs, the county publishes each camera's hardware `Manufacturer`/`Model`
* (Cohu 3950/3965, Bosch AUTODOME, Axis Q6135-LE, …), carried through as the
* source's `model`. */
export const KING_COUNTY_CAMERAS_URL =
'https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,ImageURL,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson';
/** Frames come only from the county's own image handler host. */
export const KING_COUNTY_IMAGE_ORIGIN = 'https://info.kingcounty.gov/';
/** The whole catalog is ~125 rows (~59 live county cameras); no thinning by
* default. */
export const DEFAULT_KING_COUNTY_MAX_SOURCES = 125;
/** Prioritization anchor: Renton, roughly central to the county road network. */
export const KING_COUNTY_ANCHORS = [{ lat: 47.4829, lon: -122.2171 }];
/** Hard ceiling on the King County catalog body. The whole layer is ~125 rows
* and well under 100 KB; this only exists so an upstream that streams an
* unbounded body cannot be buffered without limit. */
export const KING_COUNTY_MAX_CATALOG_BYTES = 1024 * 1024;
/**
* City of Toronto RESCU traffic cameras: the municipal network the province's
* 511 pack does not cover — downtown arterials plus the Gardiner, DVP, Allen
* and Lake Shore. One keyless list for the whole city (~336 cameras), from the
* City's open data portal. It is fetched on every catalog refresh, but the
* upstream file is not updated often and may lag the live camera network.
*
* The list is served as JSONP (`jsonTMCEarthCamerasCallback({...})`); the
* portal publishes the same data as .xml and .csv beside it.
*/
export const TORONTO_RESCU_LIST_URL =
'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/Data/tmcearthcameras.json';
/**
* The only origin Toronto camera frames may come from. The list carries no URL
* field: frame URLs are BUILT from this origin and a strictly-digits camera
* number (`loc####.jpg`, the convention the dataset documents), so no upstream
* field can steer the frame proxy off-host.
*/
export const TORONTO_RESCU_IMAGE_ORIGIN =
'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/CameraImages/';
/** The whole city fits under this; the cap exists to stay tunable, not to trim. */
export const DEFAULT_TORONTO_MAX_SOURCES = 336;
/** Yonge / Queen: the prioritization anchor. */
export const TORONTO_DOWNTOWN = { lat: 43.6532, lon: -79.3832 };
/** The city climbs from 76 m at the lake to 209 m at Steeles, so no single
* number fits: this is the mean across the 336 camera sites. The ground-height
* sidecar has no Toronto entries yet, so this prior seeds placement; a camera
* that gains a sidecar entry uses it instead, and the client's ground snap
* corrects the prior wherever 3D tiles are loaded. */
export const TORONTO_GROUND_ELEVATION_M = 116;
/** Hard ceiling on the RESCU list body. The whole city is ~336 rows and under
* 60 KB; this only exists so an upstream that streams an unbounded body cannot
* be buffered without limit. */
export const TORONTO_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** WSDOT Travel Center cameras (Washington State): keyless Esri JSON on the
* official open-data host — ~1,700 point features in EPSG:3857 (Web Mercator),
* ETag/Last-Modified served. WSDOT's metadata marks the feed "low volume" use;
* the catalog is fetched once per CCTV_SOURCE_CACHE_MS refresh. */
export const WSDOT_CAMERAS_URL =
'https://data.wsdot.wa.gov/travelcenter/Cameras.json';
/** Only frames on WSDOT's own image host are registered; the catalog also
* lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports)
* whose own terms apply. */
export const WSDOT_IMAGE_ORIGIN = 'https://images.wsdot.wa.gov/';
export const DEFAULT_WSDOT_MAX_SOURCES = 250;
/** Hard ceiling on the WSDOT catalog body. The live layer is ~400 KB; this
* only exists so an upstream that streams an unbounded body cannot be
* buffered without limit. */
export const WSDOT_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Prioritization anchors: downtown Seattle and Spokane. */
export const WSDOT_ANCHORS = [
{ lat: 47.6062, lon: -122.3321 }, // Seattle
{ lat: 47.6588, lon: -117.426 }, // Spokane
];
/** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */
export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000;
/** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one
@@ -315,6 +568,10 @@ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000;
* pending forever — a hung fetch aborts, the loader returns [], and
* serve-stale/other packs take over. */
export const CCTV_SOURCE_FETCH_TIMEOUT_MS = 15 * 1000;
/** Tarktee's two DATEX feeds (120 and 160 KB) answered in 2 to 16.6 s across
* ten requests on 2026-10-18, so the shared 15 s timeout dropped the Estonian
* pack on about a third of loads. Both must arrive for any camera to load. */
export const TARKTEE_FETCH_TIMEOUT_MS = 30 * 1000;
/** Individual CCTV image fetches must settle before the active 10-second
* client refresh cadence. A bounded miss can fall through to Street View or
* the synthetic frame instead of leaving the browser preview pending. */
+1 -1
View File
@@ -3,7 +3,7 @@ import path from 'node:path';
import { poseHash } from '../../../src/data/cctvFootprint.js';
/** Sidecar written by scripts/precompute-cctv-heights.mjs. */
export const DEFAULT_GROUND_HEIGHTS_FILE =
const DEFAULT_GROUND_HEIGHTS_FILE =
'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json';
/** @type {{path:string, mtimeMs:number, cameras:object}|null} */
+154
View File
@@ -0,0 +1,154 @@
import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js';
import {
cameraDisplayCode,
fallbackHeadingFromId,
prioritizeSources,
} from './normalize.js';
import { readResponseJsonCapped } from '../common/http.js';
const BASE = 'https://eismoinfo.lt/eismoinfo-backend';
export const LITHUANIA_CAMERAS_URL = `${BASE}/camera-info-table`;
// Without lks=true this endpoint returns [latitude, longitude]. The camera
// table's x/y fields use LKS-94 metres and must never be treated as degrees.
export const LITHUANIA_LOCATIONS_URL = `${BASE}/layer-static-features/VKR`;
export const LITHUANIA_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
const FRAME_PATH = '/eismoinfo-backend/image-provider/camera/last';
const ANCHORS = [
{ lat: 54.6872, lon: 25.2797 }, // Vilnius
{ lat: 54.8985, lon: 23.9036 }, // Kaunas
{ lat: 55.7033, lon: 21.1443 }, // Klaipeda
{ lat: 55.9349, lon: 23.3137 }, // Siauliai
{ lat: 55.7348, lon: 24.3575 }, // Panevezys
];
/** Join public catalogs by ID; register only the official still endpoint. */
export function lithuaniaCameraToSource(row, feature) {
const id = String(row?.id ?? '');
if (!/^[1-9]\d{0,8}$/.test(id) || String(feature?.id) !== id) return null;
const point = feature?.points?.[0]?.point;
if (!Array.isArray(point) || point.length !== 2) return null;
const [lat, lon] = point;
if (
!Number.isFinite(lat) ||
!Number.isFinite(lon) ||
lat < 53.8 ||
lat > 56.5 ||
lon < 20.8 ||
lon > 26.9
)
return null;
try {
const url = new URL(row.image);
if (
url.origin !== 'https://eismoinfo.lt' ||
url.username ||
url.password ||
url.pathname !== FRAME_PATH ||
url.searchParams.get('id') !== id
)
return null;
} catch {
return null;
}
// Reconstruct so extra query parameters cannot alter the request.
const imageUrl = `https://eismoinfo.lt${FRAME_PATH}?id=${id}`;
const cameraId = `lithuania-${id}`;
const name = String(row.name || feature.name || `Lietuva ${id}`)
.trim()
.slice(0, 160);
return {
id: cameraId,
name,
city: 'Lithuania / Lietuva',
cityId: 'lithuania',
provider: 'Via Lietuva / Eismoinfo',
lat,
lon,
// No calibrated pose is published; use the app's low-confidence prior.
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'eismoinfo-public',
credit:
'Via Lietuva / eismoinfo.lt — periodically updated road-camera stills',
code: cameraDisplayCode(name.toUpperCase()),
};
}
async function readCatalog(url, sibling) {
const signal = AbortSignal.any([
AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
sibling.signal,
]);
try {
const response = await fetch(url, {
headers: { Accept: 'application/json' },
redirect: 'manual',
signal,
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(
response.status >= 300 && response.status < 400
? 'catalog redirected; redirects are not followed'
: `Catalog HTTP ${response.status}`,
);
}
return await readResponseJsonCapped(
response,
LITHUANIA_MAX_CATALOG_BYTES,
signal,
);
} catch (error) {
// The join needs both catalogs; release the other request at once
// rather than leaving its body open until the timeout.
sibling.abort();
throw error;
}
}
/** Two bounded requests per shared catalog refresh, no API key required. */
export async function loadLithuaniaSources() {
try {
const sibling = new AbortController();
const [rows, layers] = await Promise.all([
readCatalog(LITHUANIA_CAMERAS_URL, sibling),
readCatalog(LITHUANIA_LOCATIONS_URL, sibling),
]);
if (!Array.isArray(rows) || !Array.isArray(layers)) return [];
const features = layers.find((layer) => layer?.layer === 'VKR')?.features;
if (!Array.isArray(features)) return [];
const locations = new Map(
features.filter(Boolean).map((feature) => [String(feature.id), feature]),
);
const cameras = new Map();
for (const row of rows) {
const camera = lithuaniaCameraToSource(
row,
locations.get(String(row?.id)),
);
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
}
const rawCap = Number(process.env.CCTV_LITHUANIA_MAX_SOURCES || 400);
const cap = Number.isFinite(rawCap)
? Math.max(1, Math.min(1000, Math.floor(rawCap)))
: 400;
const sources = prioritizeSources([...cameras.values()], cap, ANCHORS);
console.log(
`[CCTV] Loaded Lithuania camera sources: ${cameras.size} (using ${sources.length})`,
);
return sources;
} catch (error) {
console.warn(
'[CCTV] Lithuania camera download error:',
error?.message || error,
);
return [];
}
}
+131 -10
View File
@@ -1,13 +1,21 @@
import { Readable } from 'node:stream';
import { hashSeed, escapeXml } from './normalize.js';
import { hashSeed, escapeXml, pinnedCameraUrl } from './normalize.js';
import {
CALGARY_IMAGE_ORIGIN,
CCTV_FRAME_FETCH_TIMEOUT_MS,
CCTV_FRAME_MAX_BODY_BYTES,
CCTV_MEDIA_FETCH_TIMEOUT_MS,
CCTV_MEDIA_IDLE_TIMEOUT_MS,
CCTV_MEDIA_MAX_BODY_BYTES,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
NSW_IMAGE_ORIGIN,
NSW_IMAGE_USER_AGENT,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
TFL_IMAGE_ORIGIN,
WARENDORF_IMAGE_ORIGINS,
WSDOT_IMAGE_ORIGIN,
} from './constants.js';
/**
* Generate a synthetic SVG billboard image for a CCTV camera placeholder.
@@ -84,7 +92,7 @@ export function buildSyntheticCctvSvg({ cameraId, label, city, status }) {
* @param {ReadableStream|NodeJS.ReadableStream|null} body
* @returns {import('stream').Readable|null}
*/
export function toReadable(body) {
function toReadable(body) {
if (!body) return null;
if (typeof body.pipe === 'function') return body;
if (typeof body.getReader === 'function') {
@@ -93,6 +101,73 @@ export function toReadable(body) {
return null;
}
/**
* Split a Content-Type value into its lower-case essence and parameters.
*
* @param {string|null|undefined} value
* @returns {{essence:string,params:Map<string,string>}|null} Null when the
* value is missing or is not a `type/subtype` media type.
*/
function parseMediaType(value) {
const [head, ...rest] = String(value ?? '').split(';');
const essence = head.trim().toLowerCase();
if (!/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/.test(essence)) return null;
const params = new Map();
for (const part of rest) {
const eq = part.indexOf('=');
if (eq <= 0) continue;
const name = part.slice(0, eq).trim().toLowerCase();
let paramValue = part.slice(eq + 1).trim();
if (/^".*"$/.test(paramValue)) paramValue = paramValue.slice(1, -1);
if (name && !params.has(name)) params.set(name, paramValue);
}
return { essence, params };
}
/** Still-image types a camera frame may be served as. */
const CCTV_RASTER_IMAGE_TYPES = new Set([
'image/jpeg',
'image/png',
'image/gif',
'image/webp',
'image/avif',
]);
/**
* The canonical type of a raster camera frame, or null for any other type.
*
* @param {string|null|undefined} contentType
* @returns {string|null}
*/
export function cctvRasterImageType(contentType) {
const essence = parseMediaType(contentType)?.essence;
return CCTV_RASTER_IMAGE_TYPES.has(essence) ? essence : null;
}
/**
* The type to relay a live feed as, or null when the upstream type does not
* match the feed kind. MJPEG keeps its part boundary, without which the body
* cannot be split into frames.
*
* @param {string} feedType - Normalized feed type (mp4, webm or mjpeg).
* @param {string|null|undefined} contentType
* @returns {string|null}
*/
export function cctvLiveMediaType(feedType, contentType) {
const parsed = parseMediaType(contentType);
if (!parsed) return null;
if (feedType === 'mp4' || feedType === 'webm')
return parsed.essence === `video/${feedType}` ? parsed.essence : null;
if (feedType !== 'mjpeg' || parsed.essence !== 'multipart/x-mixed-replace')
return null;
const boundary = parsed.params.get('boundary') || '';
if (!/^[0-9A-Za-z'()+_,./:=? -]{0,69}[0-9A-Za-z'()+_,./:=?-]$/.test(boundary))
return null;
return /^[0-9A-Za-z'+_.-]+$/.test(boundary)
? `multipart/x-mixed-replace; boundary=${boundary}`
: `multipart/x-mixed-replace; boundary="${boundary}"`;
}
/**
* Pipe an upstream fetch Response (image or video) to the client HTTP response.
*
@@ -104,6 +179,8 @@ export function toReadable(body) {
* @param {Response} upstream - fetch() Response object.
* @param {object} [opts]
* @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header.
* @param {string} [opts.contentType] - Validated type to send instead of the
* upstream's own.
* @param {number} [opts.idleTimeoutMs=CCTV_MEDIA_IDLE_TIMEOUT_MS] - Silence
* allowed between upstream chunks before the stream is released. Injectable
* only to keep the deadline unit-testable.
@@ -114,10 +191,10 @@ export async function proxyMediaResponse(
{
sourceHeader = 'upstream',
idleTimeoutMs = CCTV_MEDIA_IDLE_TIMEOUT_MS,
contentType = upstream.headers.get('content-type') ||
'application/octet-stream',
} = {},
) {
const contentType =
upstream.headers.get('content-type') || 'application/octet-stream';
const cacheControl = upstream.headers.get('cache-control') || 'no-store';
const contentLength = upstream.headers.get('content-length');
const contentRange = upstream.headers.get('content-range');
@@ -340,7 +417,14 @@ export async function fetchCctvMediaUpstream(
if (downstream?.aborted) controller.abort();
else downstream?.addEventListener?.('abort', onDownstreamAbort);
try {
return await fetchImpl(url, { headers, signal: controller.signal });
// Redirects stay on the registered origin, as for frames.
const upstream = await fetchWithinHost(
url,
{ headers, signal: controller.signal },
fetchImpl,
);
if (!upstream) throw new Error('Media redirect refused');
return upstream;
} finally {
clearTimeout(timeoutId);
downstream?.removeEventListener?.('abort', onDownstreamAbort);
@@ -428,9 +512,40 @@ export async function fetchTxdotSnapshot(
/** Redirect hops the frame path will follow, and only within the same host. */
const MAX_SAME_HOST_REDIRECTS = 2;
/**
* Directories camera catalogs are pinned to, by host. A redirect on one of
* these hosts must stay inside the pinned directory too, so a same-origin
* redirect cannot leave it (another bucket on a shared storage host).
*/
const CCTV_PINNED_PREFIXES_BY_HOST = new Map();
for (const prefix of [
TFL_IMAGE_ORIGIN,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
...WARENDORF_IMAGE_ORIGINS,
NSW_IMAGE_ORIGIN,
CALGARY_IMAGE_ORIGIN,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
WSDOT_IMAGE_ORIGIN,
]) {
const host = new URL(prefix).hostname;
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
...(CCTV_PINNED_PREFIXES_BY_HOST.get(host) || []),
prefix,
]);
}
/** Whether a redirect target stays inside its host's pinned directory. */
function withinPinnedPrefix(url) {
const prefixes = CCTV_PINNED_PREFIXES_BY_HOST.get(url.hostname);
return !prefixes || prefixes.some((p) => pinnedCameraUrl(url.href, p));
}
/**
* Fetch a registered frame URL following redirects ONLY within the original
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops).
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops) and,
* on a pinned camera host, within its pinned directory.
* Default redirect-following would let an upstream steer a host-pinned
* request, and its host-specific headers, to any origin, another port, or a
* plaintext downgrade.
@@ -471,7 +586,7 @@ export async function fetchWithinHost(url, init, fetchImpl = fetch) {
} catch {
return null;
}
if (next.origin !== origin) return null;
if (next.origin !== origin || !withinPinnedPrefix(next)) return null;
current = next;
}
return null;
@@ -543,9 +658,15 @@ export async function fetchCctvImageFromUpstream(
fetchImpl,
);
if (!upstream) return null;
const contentType = upstream.headers.get('content-type') || '';
if (!upstream.ok || !contentType.startsWith('image/')) {
controller.abort();
const contentType = cctvRasterImageType(
upstream.headers.get('content-type'),
);
if (!upstream.ok || !contentType) {
try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return null;
}
const body = await readCappedResponseBytes(upstream, maxBytes);
+144 -3
View File
@@ -39,11 +39,21 @@ export function escapeXml(text) {
/**
* Coerce a value to a finite number, returning fallback if NaN/Infinity.
*
* Absent, blank and boolean values take the fallback too: `Number` reads
* null, '' and false as 0, which would turn a missing coordinate, heading or
* elevation into a real zero.
*
* @param {*} value
* @param {number} [fallback=NaN]
* @returns {number}
*/
export function toFiniteNumber(value, fallback = NaN) {
if (
value == null ||
typeof value === 'boolean' ||
(typeof value === 'string' && !value.trim())
)
return fallback;
const num = Number(value);
return Number.isFinite(num) ? num : fallback;
}
@@ -54,7 +64,7 @@ export function toFiniteNumber(value, fallback = NaN) {
* @param {string} text
* @returns {string}
*/
export function normalizeKey(text) {
function normalizeKey(text) {
return String(text || '')
.trim()
.toLowerCase()
@@ -70,7 +80,7 @@ export function normalizeKey(text) {
* @param {string} value
* @returns {{lat:number, lon:number}}
*/
export function parsePointString(value) {
function parsePointString(value) {
const match = String(value || '').match(
/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i,
);
@@ -90,7 +100,7 @@ export function parsePointString(value) {
* @param {string|object|null} value
* @returns {{lat:number, lon:number}}
*/
export function coerceLatLon(value) {
function coerceLatLon(value) {
if (!value) return { lat: NaN, lon: NaN };
if (typeof value === 'string') {
@@ -367,6 +377,39 @@ export function isLikelyCalgaryCoordinate(lat, lon) {
);
}
/** Florida statewide, Key West to Georgia state line, with slack at each edge */
export function isLikelyFloridaCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 24.3 &&
lat <= 31.1 &&
lon >= -87.7 &&
lon <= -79.8
);
}
/** New York City's five boroughs, with slack for the harbor and city line. */
export function isLikelyNycCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 40.45 &&
lat <= 40.95 &&
lon >= -74.3 &&
lon <= -73.65
);
}
/** Hong Kong SAR extent, with slack for the outlying islands. */
export function isLikelyHongKongCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 22.15 &&
lat <= 22.6 &&
lon >= 113.8 &&
lon <= 114.45
);
}
/** Mainland Norway, with slack for the coast and the Finnmark border. */
export function isLikelyNorwayCoordinate(lat, lon) {
return (
@@ -394,6 +437,44 @@ export function isLikelyFinlandCoordinate(lat, lon) {
return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32;
}
/** Catalonia bounding box (mainland is roughly 40.5..42.9 N, 0.15..3.4 E). */
export function isLikelyCataloniaCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 40.3 &&
lat <= 42.9 &&
lon >= 0.0 &&
lon <= 3.4
);
}
/**
* Decode numeric HTML character references, decimal ("&#237;") and hex
* ("&#xED;") alike, the entity form the Catalonia feed uses for accented
* place names, plus the handful of named entities XML/HTML text commonly
* carries. A reference that names no valid character (NUL, a surrogate, or
* anything past U+10FFFF) is left as written rather than thrown on, so one
* bad field cannot fail a whole catalog.
*
* @param {string} text
* @returns {string}
*/
export function decodeNumericEntities(text) {
return String(text || '')
.replace(/&#(?:x([0-9a-f]+)|(\d+));/gi, (match, hex, dec) => {
const code = hex ? Number.parseInt(hex, 16) : Number(dec);
const valid =
code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff);
return valid ? String.fromCodePoint(code) : match;
})
.replace(/&amp;/g, '&')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.trim();
}
/**
* Human label for one Fintraffic preset (camera view).
*
@@ -562,5 +643,65 @@ export function normalizeSourceItem(item) {
// badge can distinguish them from raw automated priors (e.g. Austin Open
// Data, which never sets this field). Passed through as-is to the client.
poseSource: item.poseSource === 'curated' ? 'curated' : undefined,
// Optional hardware-model passthrough (additive, like poseSource): packs
// whose catalog publishes the camera's hardware model — King County's
// layer carries Manufacturer/Model per camera — pass it through so
// identified-hardware consumers can use it. Absent everywhere else.
model:
typeof item.model === 'string' && item.model.trim()
? item.model.trim()
: undefined,
// Bare model strings collide across vendors (e.g. "3950"), so the
// publishing pack's manufacturer rides along for disambiguation.
manufacturer:
typeof item.manufacturer === 'string' && item.manufacturer.trim()
? item.manufacturer.trim()
: undefined,
};
}
/** City of Toronto bounding box (Steeles to the lake, Etobicoke to Scarborough). */
export function isLikelyTorontoCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 43.55 &&
lat <= 43.9 &&
lon >= -79.7 &&
lon <= -79.1
);
}
/**
* A camera URL under a pinned directory, by parsed URL rather than string
* prefix: the same scheme and exact host as `prefix`, no credentials, the
* default port, and a normalized path inside the prefix's directory. Dot
* segments (including encoded ones) and backslashes are resolved before the
* path is compared, and an encoded slash or backslash is refused.
*
* @param {string} raw - Camera URL from a catalog or redirect.
* @param {string} prefix - Pinned origin and directory, ending in `/`.
* @returns {string|null} The normalized URL, or null when it is not pinned.
*/
export function pinnedCameraUrl(raw, prefix) {
let url;
let pin;
try {
url = new URL(String(raw ?? '').trim());
pin = new URL(prefix);
} catch {
return null;
}
if (
url.protocol !== pin.protocol ||
url.hostname !== pin.hostname ||
url.username ||
url.password ||
url.port ||
!url.pathname.startsWith(
pin.pathname.endsWith('/') ? pin.pathname : `${pin.pathname}/`,
) ||
/%2f|%5c/i.test(url.pathname)
)
return null;
return url.href;
}
+129
View File
@@ -0,0 +1,129 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
_resetOntarioKeyWarningForTest,
loadOntarioSourcesFromOpenData,
} from './sources.js';
const ROW = {
Id: '42',
Latitude: 43.6532,
Longitude: -79.3832,
Location: 'Hwy 401 near Yonge St',
Direction: 'Eastbound',
Views: [
{
Status: 'Enabled',
Url: 'https://511on.ca/map/Cctv/4201',
Description: 'Eastbound',
},
],
};
function captureConsole(t) {
const lines = [];
const capture = (...args) => lines.push(args.join(' '));
t.mock.method(console, 'warn', capture);
t.mock.method(console, 'log', capture);
return lines;
}
test.beforeEach(() => _resetOntarioKeyWarningForTest());
test('without ONTARIO_511_API_KEY the Ontario pack is skipped, logged once', async (t) => {
const lines = captureConsole(t);
let fetches = 0;
t.mock.method(globalThis, 'fetch', async () => {
fetches += 1;
return Response.json([ROW]);
});
assert.deepEqual(await loadOntarioSourcesFromOpenData({}, {}), []);
assert.deepEqual(
await loadOntarioSourcesFromOpenData({}, { ONTARIO_511_API_KEY: ' ' }),
[],
);
assert.equal(fetches, 0);
assert.equal(lines.length, 1);
assert.match(lines[0], /ONTARIO_511_API_KEY/);
});
test('with a key the camera list is requested with it and parsed', async (t) => {
const lines = captureConsole(t);
const requested = [];
t.mock.method(globalThis, 'fetch', async (url) => {
requested.push(String(url));
return Response.json([ROW]);
});
const cameras = await loadOntarioSourcesFromOpenData(
{ sourceRoot: '/ignored' },
{ ONTARIO_511_API_KEY: ' test-key ' },
);
assert.equal(requested.length, 1);
const url = new URL(requested[0]);
assert.equal(url.host, '511on.ca');
assert.equal(url.searchParams.get('key'), 'test-key');
assert.equal(url.searchParams.get('format'), 'json');
assert.equal(cameras.length, 1);
assert.equal(cameras[0].id, 'on-42');
assert.equal(cameras[0].url, 'https://511on.ca/map/Cctv/4201');
// The records go to the browser through /api/cctv; the key must stay on
// the server-side catalog request.
assert.doesNotMatch(JSON.stringify(cameras), /test-key/);
assert.doesNotMatch(lines.join('\n'), /test-key/);
});
test('the source cap is read from the same environment as the key', async (t) => {
captureConsole(t);
const rows = Array.from({ length: 12 }, (_, i) => ({
...ROW,
Id: String(100 + i),
Latitude: ROW.Latitude + i * 0.01,
Views: [{ ...ROW.Views[0], Url: `https://511on.ca/map/Cctv/${100 + i}` }],
}));
t.mock.method(globalThis, 'fetch', async () => Response.json(rows));
const cameras = await loadOntarioSourcesFromOpenData(
{},
{ ONTARIO_511_API_KEY: 'test-key', CCTV_ONTARIO_MAX_SOURCES: '8' },
);
assert.equal(cameras.length, 8);
});
for (const status of [400, 401, 403]) {
test(`a ${status} points at the key by name, never by value`, async (t) => {
const lines = captureConsole(t);
t.mock.method(globalThis, 'fetch', async () =>
Response.json({ Message: 'Invalid Key' }, { status }),
);
const cameras = await loadOntarioSourcesFromOpenData(
{},
{ ONTARIO_511_API_KEY: 'secret-key-value' },
);
assert.deepEqual(cameras, []);
assert.match(
lines.join('\n'),
new RegExp(`failed: ${status} \\(check ONTARIO_511_API_KEY\\)`),
);
assert.doesNotMatch(lines.join('\n'), /secret-key-value/);
});
}
for (const status of [429, 500, 503]) {
test(`a ${status} is reported without blaming the key`, async (t) => {
const lines = captureConsole(t);
t.mock.method(globalThis, 'fetch', async () =>
Response.json({}, { status }),
);
const cameras = await loadOntarioSourcesFromOpenData(
{},
{ ONTARIO_511_API_KEY: 'secret-key-value' },
);
assert.deepEqual(cameras, []);
assert.equal(lines.length, 1);
assert.equal(
lines[0],
`[CCTV] Ontario 511 camera download failed: ${status}`,
);
assert.doesNotMatch(lines[0], /secret-key-value/);
});
}
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -70,7 +70,7 @@ export async function fetchHlsBytes(
}
}
export function sameOriginHlsUrl(value, base) {
function sameOriginHlsUrl(value, base) {
const url = new URL(value, base);
if (
!['https:', 'http:'].includes(url.protocol) ||
+303
View File
@@ -0,0 +1,303 @@
import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js';
import {
cameraDisplayCode,
fallbackHeadingFromId,
prioritizeSources,
} from './normalize.js';
import { readResponseTextCapped } from '../common/http.js';
/** Freeway Bureau (MOTC) CCTV static table in the national MOTC 2.0 XML
* standard, published keyless and refreshed daily (~1,800 cameras). */
export const TAIWAN_FREEWAY_CCTV_URL =
'https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml';
/** The whole network is ~1.2 MB. */
export const TAIWAN_MAX_CATALOG_BYTES = 6 * 1024 * 1024;
const DEFAULT_TAIWAN_MAX_SOURCES = 600;
/** After a failed download, wait this long before asking again. The catalog
* host does not answer from some networks outside Taiwan, and every attempt
* there holds the shared catalog refresh until the connect timeout. */
export const TAIWAN_RETRY_AFTER_FAILURE_MS = 60 * 60 * 1000;
/** The table is republished daily; a good copy stands in for a failed
* download for up to this long. */
const TAIWAN_LAST_GOOD_MAX_AGE_MS = 24 * 60 * 60 * 1000;
// The table publishes each camera's MJPEG stream. The same camera servers
// answer a single JPEG for the same camera ID on a sibling path, which is what
// the frame proxy can relay; only these exact pairs are ever registered.
const STREAM_TO_FRAME = new Map([
['cctvn.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']],
['cctvn5.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']],
['cctvc.freeway.gov.tw', ['/abs2mjpg/bmjpg', '/abs2mjpg/jpg']],
[
'cctvs.freeway.gov.tw',
['/live-view/mjpg/video.cgi', '/live-view/jpg/image.cgi'],
],
]);
const DIRECTIONS = { N: 'N', S: 'S', E: 'E', W: 'W' };
/** Prioritization anchors, north to south, so a cap keeps the whole island
* rather than only greater Taipei. */
const ANCHORS = [
{ lat: 25.033, lon: 121.5654 }, // Taipei
{ lat: 24.9936, lon: 121.301 }, // Taoyuan
{ lat: 24.8138, lon: 120.9675 }, // Hsinchu
{ lat: 24.7021, lon: 121.7378 }, // Yilan
{ lat: 24.1477, lon: 120.6736 }, // Taichung
{ lat: 22.9999, lon: 120.227 }, // Tainan
{ lat: 22.6273, lon: 120.3014 }, // Kaohsiung
];
/** Taiwan main island with slack for the coast; the freeway network has no
* offshore-island cameras. */
function isLikelyTaiwanCoordinate(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
lat >= 21.8 &&
lat <= 25.4 &&
lon >= 119.9 &&
lon <= 122.1
);
}
const XML_ENTITIES = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
};
function codePoint(value, radix) {
const code = parseInt(value, radix);
return Number.isInteger(code) && code > 0 && code <= 0x10ffff
? String.fromCodePoint(code)
: '';
}
function decodeXml(text) {
return String(text ?? '')
.replace(
/&(?:(amp|lt|gt|quot|apos)|#(\d{1,7})|#x([0-9a-fA-F]{1,6}));/g,
(_, name, dec, hex) =>
name ? XML_ENTITIES[name] : codePoint(dec ?? hex, dec ? 10 : 16),
)
.trim();
}
function tag(block, name) {
const match = new RegExp(`<${name}>([^<]*)</${name}>`).exec(block);
return match ? decodeXml(match[1]) : '';
}
/**
* Parse the MOTC 2.0 `CCTVList` document into flat rows. Regex-based like the
* Catalonia and Tarktee parsers: the schema is flat, and only the fields used
* below are read.
*
* @param {string} xml
* @returns {Array<object>}
*/
export function parseTaiwanCctvXml(xml) {
const rows = [];
const blockRe = /<CCTV>([\s\S]*?)<\/CCTV>/g;
let match;
while ((match = blockRe.exec(String(xml || ''))) !== null) {
const block = match[1];
rows.push({
cctvId: tag(block, 'CCTVID'),
streamUrl: tag(block, 'VideoStreamURL'),
lat: Number(tag(block, 'PositionLat')),
lon: Number(tag(block, 'PositionLon')),
roadName: tag(block, 'RoadName'),
direction: tag(block, 'RoadDirection'),
sectionStart: tag(block, 'Start'),
sectionEnd: tag(block, 'End'),
mile: tag(block, 'LocationMile'),
});
}
return rows;
}
/**
* Published MJPEG stream URL -> the same camera's still, plus the host and
* camera number that identify it, or null.
*
* @param {string} streamUrl
* @returns {?{frameUrl: string, host: string, camera: string}}
*/
export function taiwanFrame(streamUrl) {
let url;
try {
url = new URL(String(streamUrl ?? ''));
} catch {
return null;
}
if (url.protocol !== 'https:' || url.username || url.password || url.port)
return null;
const paths = STREAM_TO_FRAME.get(url.hostname);
if (!paths || url.pathname !== paths[0]) return null;
const camera = url.searchParams.get('camera') || '';
if (!/^[A-Za-z0-9-]{1,40}$/.test(camera)) return null;
return {
// Reconstruct so extra query parameters cannot alter the request.
frameUrl: `https://${url.hostname}${paths[1]}?camera=${camera}`,
host: url.hostname.split('.')[0],
camera,
};
}
/**
* One parsed Freeway Bureau row -> one catalog source, or null.
*
* @param {object} row - Output of parseTaiwanCctvXml.
* @returns {?object}
*/
export function taiwanCameraToSource(row) {
if (!row || typeof row !== 'object') return null;
const { lat, lon } = row;
if (!isLikelyTaiwanCoordinate(lat, lon)) return null;
const frame = taiwanFrame(row.streamUrl);
if (!frame) return null;
const imageUrl = frame.frameUrl;
// CCTVID carries Chinese place names and a few IDs repeat across distinct
// cameras, so the stable key is the camera server plus its camera number.
const rawId = String(row.cctvId ?? '')
.trim()
.slice(0, 80);
const cameraId = `tw-freeway-${frame.host}-${frame.camera.toLowerCase()}`;
const road = String(row.roadName ?? '').trim();
const direction = DIRECTIONS[String(row.direction ?? '').trim()] || '';
const mile = String(row.mile ?? '').trim();
const section = [row.sectionStart, row.sectionEnd]
.map((part) => String(part ?? '').trim())
.filter(Boolean)
.join('→');
// A bare carriageway letter is not a name; fall back to the published ID.
const name =
(road || mile || section
? [road, direction, mile, section].filter(Boolean).join(' ')
: '') ||
rawId ||
`Freeway camera ${frame.camera}`;
return {
id: cameraId,
name: name.slice(0, 160),
// One island-wide category in the camera picker, like Finland and Norway.
city: 'Taiwan / 臺灣',
cityId: 'taiwan',
provider: 'Freeway Bureau, MOTC',
lat,
lon,
// The table gives the carriageway direction, not the camera's bearing,
// so the pose uses the shared low-confidence prior.
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'tw-freeway-motc',
license:
'Freeway Bureau, MOTC open data, Open Government Data License, Taiwan (Version 1.0)',
code: cameraDisplayCode(
`${road} ${mile}`.trim().toUpperCase() || frame.camera,
),
};
}
let lastGood = null;
let lastGoodAt = 0;
let retryAfter = 0;
/** Forget the last good table and any failure backoff (tests only). */
export function resetTaiwanSourcesCache() {
lastGood = null;
lastGoodAt = 0;
retryAfter = 0;
}
function fallbackSources() {
return lastGood && Date.now() - lastGoodAt <= TAIWAN_LAST_GOOD_MAX_AGE_MS
? lastGood
: [];
}
function taiwanSourceCap() {
const rawCap = Number(
process.env.CCTV_TAIWAN_MAX_SOURCES || DEFAULT_TAIWAN_MAX_SOURCES,
);
return Number.isFinite(rawCap)
? Math.max(1, Math.min(2000, Math.floor(rawCap)))
: DEFAULT_TAIWAN_MAX_SOURCES;
}
/**
* Fetch Taiwan Freeway Bureau cameras from the keyless MOTC 2.0 XML table.
* One bounded request per shared catalog refresh. A failed download serves the
* last good table (up to a day old) and is not retried for an hour, so a host
* that cannot reach the Freeway Bureau does not wait on it every refresh.
*
* @returns {Promise<Array<object>>}
*/
export async function loadTaiwanSources() {
if (Date.now() < retryAfter) return fallbackSources();
try {
const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS);
const response = await fetch(
process.env.CCTV_TAIWAN_URL || TAIWAN_FREEWAY_CCTV_URL,
{
headers: { Accept: 'application/xml,text/xml' },
redirect: 'manual',
signal,
},
);
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(
response.status >= 300 && response.status < 400
? 'catalog redirected; redirects are not followed'
: `HTTP ${response.status}`,
);
}
const xml = await readResponseTextCapped(
response,
TAIWAN_MAX_CATALOG_BYTES,
signal,
);
const cameras = new Map();
for (const row of parseTaiwanCctvXml(xml)) {
const camera = taiwanCameraToSource(row);
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
}
if (!cameras.size) throw new Error('no usable cameras in the table');
const sources = prioritizeSources(
[...cameras.values()],
taiwanSourceCap(),
ANCHORS,
);
console.log(
`[CCTV] Loaded Taiwan camera sources: ${cameras.size} (using ${sources.length})`,
);
lastGood = sources;
lastGoodAt = Date.now();
retryAfter = 0;
return sources;
} catch (error) {
retryAfter = Date.now() + TAIWAN_RETRY_AFTER_FAILURE_MS;
const fallback = fallbackSources();
console.warn(
`[CCTV] Taiwan camera download failed (${error?.message || error}); ` +
(fallback.length
? `serving the last good ${fallback.length} cameras`
: 'no Taiwan cameras') +
', next attempt in 1 h',
);
return fallback;
}
}