Merge main into codex/fix-cctv-session-admission

Resolve the conflicts by keeping main's keyless CCTV gate, which already
refuses cross-site requests on every /api/cctv route and keeps the routes
reachable behind a reverse proxy, instead of the strict gate on /media/.
Keep the per-request client id the lease calls use, and both sides of the
CURRENT-STATE paragraphs and the media tests.
This commit is contained in:
German Khokhlov committed 2026-11-19 17:58:14 +04:00
commit 90e464eb5a
4310 files changed
+635935 -70111

No files matched your search

+49 -12
View File
@@ -1,5 +1,6 @@
import {
CCTV_MAX_SOURCES_CEILING,
CCTV_SHARED_SOURCE_SLOTS,
DEFAULT_CCTV_MAX_SOURCES,
} from './constants.js';
@@ -23,16 +24,27 @@ export function resolveCatalogCap(raw) {
* exceed the catalog cap, the cap is filled ROUND-ROBIN — one camera from
* each pack per turn, in each pack's own order — so a lowered
* CCTV_MAX_SOURCES thins every region a little instead of deleting whichever
* pack happened to be appended last. Duplicate ids resolve last-pack-wins so
* pack happened to be appended last. A pack with a `reservedCount` keeps up to
* that many rows only from the capacity ABOVE the shared floor, so a
* reservation adds room and never shrinks the other packs' round-robin
* share; at or below the floor reserved packs are ordinary lanes. Duplicate
* ids resolve last-pack-wins so
* file/env overrides keep replacing live records; the record is then counted
* against the pack that won it.
*
* @param {Array<{name:string, sources:Array<object>}>} packs - Normalized
* packs in merge order (earlier packs lose duplicate ids to later ones).
* @param {number} maxCount - Catalog cap.
* @param {object} [options]
* @param {number} [options.sharedFloor=CCTV_SHARED_SOURCE_SLOTS] - Slots kept
* for the round-robin before reservations apply.
* @returns {{sources:Array<object>, packs:Array<{name:string, offered:number, kept:number}>}}
*/
export function allocateSourceCap(packs, maxCount) {
export function allocateSourceCap(
packs,
maxCount,
{ sharedFloor = CCTV_SHARED_SOURCE_SLOTS } = {},
) {
const owner = new Map();
for (const pack of packs) {
for (const source of pack.sources) {
@@ -40,30 +52,53 @@ export function allocateSourceCap(packs, maxCount) {
owner.set(source.id, { pack: pack.name, source });
}
}
const lanes = packs.map((pack) => ({
name: pack.name,
const lanes = packs.map((pack) => {
// Only the record that WON its id (the last occurrence anywhere, so also
// the last within this pack) takes a lane slot; earlier duplicates never
// count against the cap.
queue: pack.sources.filter(
const queue = pack.sources.filter(
(source) => source?.id && owner.get(source.id)?.source === source,
),
next: 0,
kept: 0,
}));
);
return {
name: pack.name,
queue,
reserved: Math.max(
0,
Math.min(queue.length, Math.floor(Number(pack.reservedCount) || 0)),
),
next: 0,
kept: 0,
};
});
const total = lanes.reduce((sum, lane) => sum + lane.queue.length, 0);
const limit = Math.min(total, Math.max(0, Math.floor(maxCount)));
const sources = [];
const reservedSources = [];
if (limit >= total) {
for (const lane of lanes) {
sources.push(...lane.queue);
lane.kept = lane.queue.length;
}
} else {
while (sources.length < limit) {
// Reserved packs draw only on the capacity above the shared floor, then
// every lane (reserved ones included) shares the rest round-robin.
const reserveBudget = Math.max(0, limit - Math.max(0, sharedFloor));
while (reservedSources.length < reserveBudget) {
let tookReserved = false;
for (const lane of lanes) {
if (sources.length >= limit) break;
if (reservedSources.length >= reserveBudget) break;
if (lane.kept >= lane.reserved) continue;
reservedSources.push(lane.queue[lane.next]);
lane.next += 1;
lane.kept += 1;
tookReserved = true;
}
if (!tookReserved) break;
}
while (sources.length + reservedSources.length < limit) {
for (const lane of lanes) {
if (sources.length + reservedSources.length >= limit) break;
if (lane.next >= lane.queue.length) continue;
sources.push(lane.queue[lane.next]);
lane.next += 1;
@@ -72,7 +107,9 @@ export function allocateSourceCap(packs, maxCount) {
}
}
return {
sources,
// Keep the shared round-robin sequence first so the default active camera
// remains in its established region; guaranteed regional rows follow it.
sources: [...sources, ...reservedSources],
packs: lanes.map((lane) => ({
name: lane.name,
offered: lane.queue.length,
+46 -2
View File
@@ -1,11 +1,18 @@
import fs from 'node:fs';
import path from 'node:path';
import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js';
import {
DEFAULT_CCTV_SOURCE_FILE,
CCTV_SOURCE_CACHE_MS,
WORLD_CCTV_SOURCE_COUNT,
} from './constants.js';
import { allocateSourceCap, resolveCatalogCap } from './cap.js';
import { loadGroundHeights, joinGroundHeights } from './groundHeights.js';
import { cameraRegion, normalizeSourceItem } from './normalize.js';
import { loadLithuaniaSources } from './lithuania.js';
import { loadOttawaSourcesFromOpenData } from './ottawa.js';
import { loadQuebecSources, QUEBEC_CAMERA_SOURCE_COUNT } from './quebec.js';
import { loadTaiwanSources } from './taiwan.js';
import { loadWorldCameraSources } from './worldCameras.js';
import {
loadAustinSourcesFromOpenData,
loadCaltransSourcesFromOpenData,
@@ -20,6 +27,7 @@ import {
loadNswSourcesFromOpenData,
loadCalgarySourcesFromOpenData,
loadCataloniaSourcesFromOpenData,
loadIcelandSourcesFromOpenData,
loadHkSourcesFromOpenData,
loadDelDOTSourcesFromOpenData,
loadKingCountySourcesFromOpenData,
@@ -28,6 +36,8 @@ import {
loadNycSourcesFromOpenData,
loadTorontoSourcesFromOpenData,
loadFloridaSourcesFromOpenData,
loadHoustonSourcesFromOpenData,
loadA22SourcesFromOpenDataHub,
} from './sources.js';
/** Env kill switch: unset or anything but "0" means enabled. */
@@ -73,6 +83,11 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_DRIVEBC_ENABLED'),
load: loadDriveBcSourcesFromOpenData,
},
{
name: 'ottawa',
enabled: () => envEnabled('CCTV_OTTAWA_ENABLED'),
load: loadOttawaSourcesFromOpenData,
},
{
name: 'txdot',
enabled: () => envEnabled('CCTV_TXDOT_ENABLED'),
@@ -108,6 +123,11 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'),
load: loadCataloniaSourcesFromOpenData,
},
{
name: 'iceland',
enabled: () => envEnabled('CCTV_ICELAND_ENABLED'),
load: loadIcelandSourcesFromOpenData,
},
{
name: 'hk',
enabled: () => envEnabled('CCTV_HK_ENABLED'),
@@ -165,6 +185,28 @@ const LIVE_PACKS = [
enabled: () => envEnabled('CCTV_TAIWAN_ENABLED'),
load: loadTaiwanSources,
},
{
name: 'houston',
enabled: () => envOptIn('CCTV_HOUSTON_ENABLED'),
load: loadHoustonSourcesFromOpenData,
},
{
name: 'quebec',
enabled: () => envEnabled('CCTV_QUEBEC_ENABLED'),
load: loadQuebecSources,
reservedCount: QUEBEC_CAMERA_SOURCE_COUNT,
},
{
name: 'world-cameras',
enabled: () => envEnabled('CCTV_WORLD_CAMERAS_ENABLED'),
load: loadWorldCameraSources,
reservedCount: WORLD_CCTV_SOURCE_COUNT,
},
{
name: 'a22',
enabled: () => envEnabled('CCTV_A22_ENABLED'),
load: loadA22SourcesFromOpenDataHub,
},
];
/**
* Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default).
@@ -304,7 +346,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
: [];
// Live packs first so file/env overrides win on duplicate IDs; each pack
// keeps its own priority order and the catalog cap is shared fairly.
const normalizePack = (name, items) => ({
const normalizePack = (name, items, reservedCount = 0) => ({
name,
sources: items
.filter((item) => item && typeof item === 'object')
@@ -314,6 +356,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
? items.available
: items.length,
region: items.region ?? null,
reservedCount,
});
const packs = [
...LIVE_PACKS.map((pack, index) =>
@@ -322,6 +365,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) {
liveResults[index]?.status === 'fulfilled'
? liveResults[index].value
: [],
pack.reservedCount,
),
),
normalizePack('file', fromFile),
+139 -8
View File
@@ -5,20 +5,23 @@ export const DEFAULT_AUSTIN_ROWS_URL =
/** Default cap on Austin cameras after distance-based prioritization. */
export const DEFAULT_AUSTIN_MAX_SOURCES = 250;
/**
* Catalog-wide safety ceiling on served cameras. Each pack already caps
* itself (nearest-to-anchor first); this bound only matters when the packs
* together exceed it, and it is then filled round-robin across packs (see
* cap.js) so no region is silently dropped. The default per-pack caps
* together can exceed it, so a default install may be thinned evenly here.
* Slots every pack shares round-robin before any reservation applies. A
* reserved pack (Québec, World) only takes capacity above this floor, so a
* cap at or below it thins every region evenly (see cap.js).
*/
export const DEFAULT_CCTV_MAX_SOURCES = 4000;
export const CCTV_SHARED_SOURCE_SLOTS = 4000;
/**
* Catalog-wide default: the 4,000 shared slots plus room reserved for the
* 1,120 Québec and 2,230 World image locations.
*/
export const DEFAULT_CCTV_MAX_SOURCES = 7350;
/** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */
export const CCTV_MAX_SOURCES_CEILING = 5000;
export const CCTV_MAX_SOURCES_CEILING = 8000;
/** Reference point for Austin camera prioritization (Congress & 6th). */
export const AUSTIN_DOWNTOWN = { lat: 30.2672, lon: -97.7431 };
/** Caltrans CCTV: one JSON feed per district, identical schema statewide. */
/** TxDOT ITS: one keyless JSON catalog per district (25 districts statewide). */
export const TXDOT_ORIGIN = 'https://its.txdot.gov';
const TXDOT_ORIGIN = 'https://its.txdot.gov';
export const TXDOT_CCTV_STATUS_URL = (district) =>
`${TXDOT_ORIGIN}/its/DistrictIts/GetCctvStatusListByDistrict?districtCode=${encodeURIComponent(district)}`;
/** Per-camera frame. Returns JSON `{snippet:<base64 jpeg>}`, not an image body;
@@ -197,6 +200,39 @@ export const DRIVEBC_ANCHORS = [
{ lat: 49.2827, lon: -123.1207 }, // Vancouver
{ lat: 48.4284, lon: -123.3656 }, // Victoria
];
/** City of Ottawa traffic cameras: documented JSON list (keyless). */
export const OTTAWA_CAMERAS_URL =
'https://traffic.ottawa.ca/map/service/camera';
/**
* Keyless canonical frame URL for one camera number. The city's Traffic Camera
* Images API needs a certificate (OTTAWA_CAMERA_CERTIFICATE), which the frame
* proxy adds at request time so it never appears in a catalog response.
*/
export const OTTAWA_IMAGE_URL = (cameraNumber) =>
`https://traffic.ottawa.ca/opendata/camera?c=${encodeURIComponent(cameraNumber)}`;
/** Pinned frame directory, for redirect containment in the media proxy. */
export const OTTAWA_IMAGE_ORIGIN = 'https://traffic.ottawa.ca/opendata/';
/** The API lets one certificate and user id fetch a camera once a minute. */
export const OTTAWA_FRAME_REFRESH_MS = 60 * 1000;
/** Alphanumeric user id this server sends with its certificate. */
export const OTTAWA_API_USER_ID = 'gev';
export const DEFAULT_OTTAWA_MAX_SOURCES = 250;
/** Prioritization anchor: downtown Ottawa (Parliament Hill). */
export const OTTAWA_ANCHORS = [{ lat: 45.4215, lon: -75.6972 }];
/**
* Taiwan Highway Bureau MJPEG servers. Each camera is one root path segment,
* so the pin is the host root: a redirect must stay on the same server.
*/
export const TAIWAN_HIGHWAY_IMAGE_ORIGINS = Object.freeze(
[1, 2, 3, 4, 5, 6, 7, 8].map((n) => `https://cctv-ss0${n}.thb.gov.tw/`),
);
/** Direct still-image directories from the World cams add-on's eligible packs. */
export const TRIPCHECK_IMAGE_ORIGIN =
'https://www.tripcheck.com/roadcams/cams/';
export const MLIT_RIVER_IMAGE_ORIGIN = 'https://cam.river.go.jp/cam/now/';
export const DEFAULT_WORLD_CCTV_SOURCE_FILE =
'config/cctv_sources.world-image-cameras.json';
export const WORLD_CCTV_SOURCE_COUNT = 2230;
/** Tallinn intersection cameras: curated catalog + public stills on ristmikud.tallinn.ee. */
export const DEFAULT_TALLINN_SOURCE_FILE = 'config/cctv_sources.tallinn.json';
export const DEFAULT_TALLINN_MAX_SOURCES = 255;
@@ -309,6 +345,37 @@ export const CATALONIA_ANCHORS = [
{ lat: 41.5636, lon: 2.0111 }, // Terrassa
];
/**
* Icelandic Road and Coastal Administration (Vegagerðin, IRCA) road cameras:
* one keyless JSON list, one row per camera view (about 500 views at roughly
* 170 stations, verified 2026-11-06). Frames are stills on the IRCA host, each
* rewritten by the publisher every few minutes. Camera images are published
* under CC BY 4.0 (IRCA terms of use, "Open data" section).
*/
export const ICELAND_CAMERAS_URL =
'https://gagnaveita.vegagerdin.is/api/vefmyndavelar2014_1';
/** The only directory Iceland camera frames may come from. */
export const ICELAND_IMAGE_ORIGIN =
'https://www.vegagerdin.is/vgdata/vefmyndavelar/';
/** Byte ceiling for the Iceland catalog read (~150 KB observed 2026-11-06). */
export const ICELAND_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
/** Ground-elevation prior in metres: the feed carries none, and the country
* runs from sea level to highland passes. The client's ground snap corrects
* this on 3D-tile stacks. */
export const ICELAND_GROUND_ELEVATION_M = 60;
/** The feed lists about 500 views; the default keeps half so the shared
* catalog cap still leaves room for denser regions. */
export const DEFAULT_ICELAND_MAX_SOURCES = 250;
/** Prioritization anchors: Reykjavík and the regional centres, so a cap keeps
* every region's nearest cameras instead of only the capital's. */
export const ICELAND_ANCHORS = [
{ lat: 64.1466, lon: -21.9426 }, // Reykjavík
{ lat: 65.6885, lon: -18.1262 }, // Akureyri
{ lat: 65.2639, lon: -14.3948 }, // Egilsstaðir
{ lat: 66.0749, lon: -23.1351 }, // Ísafjörður
{ lat: 63.9331, lon: -20.9971 }, // Selfoss
];
/** Hong Kong Transport Department traffic snapshot cameras: keyless English
* locations XML on DATA.GOV.HK; frames are stills on the TD CCTV host. */
export const DEFAULT_HK_CAMERAS_URL =
@@ -443,6 +510,70 @@ export const NYC_ANCHORS = [
{ lat: 40.5795, lon: -74.1502 }, // Staten Island
];
/** Houston TranStar cameras: one keyless JSON list for the metro (~1,000
* cameras with a numeric id). Frames are stills synthesized from that id on
* the city's snapshot host — never taken from the payload. The catalog is
* served as `application/javascript`; asking for `application/json` gets HTTP
* 406, so the loader accepts any media type. */
export const HOUSTON_CAMERAS_URL =
'https://traffic.houstontranstar.org/data/layers/cctvSnapshots_json.js';
/** The only origin Houston camera frames may come from. The catalog's `url`
* is a `showWin(...)` viewer helper and is ignored; the registered path is
* always this origin plus `/snapshots/cctv/<id>.jpg`. */
export const HOUSTON_IMAGE_ORIGIN = 'https://www.houstontranstar.org/';
export const HOUSTON_IMAGE_URL = (id) =>
`${HOUSTON_IMAGE_ORIGIN}snapshots/cctv/${id}.jpg`;
/** Numeric TranStar snapshot ids: 3–5 digits, no leading zero. Id `0` is a
* different (non-snapshot) camera class in the same list. */
export const HOUSTON_CAMERA_ID_RE = /^[1-9]\d{2,4}$/;
export const DEFAULT_HOUSTON_MAX_SOURCES = 250;
export const HOUSTON_MAX_CATALOG_BYTES = 4 * 1024 * 1024;
/** Ground-elevation prior in metres. The TranStar payload carries none, and
* Houston sits a few metres above sea level. */
export const HOUSTON_GROUND_ELEVATION_M = 15;
/** Downtown Houston (Main @ Texas): the primary prioritization anchor. */
export const HOUSTON_DOWNTOWN = { lat: 29.7604, lon: -95.3698 };
export const HOUSTON_ANCHORS = [
{ lat: 29.7604, lon: -95.3698 }, // Downtown
{ lat: 29.739, lon: -95.461 }, // Uptown / Galleria
{ lat: 29.71, lon: -95.398 }, // Texas Medical Center
{ lat: 29.9902, lon: -95.3368 }, // IAH
{ lat: 29.3013, lon: -94.7977 }, // Galveston
];
/** Autostrada del Brennero (A22, Brenner–Modena) webcams: the keyless Open
* Data Hub (NOI Techpark, South Tyrol) tourism API republishes Autobrennero's
* own camera list with coordinates, one request for the whole motorway
* (13 cameras). Frames are stills on the operator's host, refreshed about
* once a minute. */
export const DEFAULT_A22_CCTV_URL =
'https://tourism.api.opendatahub.com/v1/WebcamInfo?source=a22&pagesize=200&removenullvalues=true';
/** The only origin A22 camera frames may come from: `km<N>.jpg` per camera. */
export const A22_IMAGE_ORIGIN = 'https://www.autobrennero.it/WebCamImg/';
/** Hard ceiling on the catalog body. The whole motorway is ~45 KB. */
export const A22_MAX_CATALOG_BYTES = 2 * 1024 * 1024;
/** Orthometric ground elevation (m) at each camera, keyed by its motorway
* kilometre, from the Copernicus GLO-90 DEM via Open-Meteo's elevation API.
* The feed's coordinates are 2D and the road climbs from the Po valley (~20 m)
* to the Brenner Pass (~1,370 m), so one flat prior would bury the Alpine
* cameras. A camera not listed here uses A22_DEFAULT_GROUND_ELEVATION_M. */
export const A22_GROUND_ELEVATION_BY_KM = Object.freeze({
1: 1365,
68: 375,
77: 301,
100: 219,
129: 206,
138: 204,
159: 175,
205: 188,
228: 72,
240: 38,
272: 16,
285: 19,
313: 40,
});
export const A22_DEFAULT_GROUND_ELEVATION_M = 200;
/** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */
export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json';
export const DEFAULT_DELDOT_MAX_SOURCES = 300;
+1 -1
View File
@@ -3,7 +3,7 @@ import path from 'node:path';
import { poseHash } from '../../../src/data/cctvFootprint.js';
/** Sidecar written by scripts/precompute-cctv-heights.mjs. */
export const DEFAULT_GROUND_HEIGHTS_FILE =
const DEFAULT_GROUND_HEIGHTS_FILE =
'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json';
/** @type {{path:string, mtimeMs:number, cameras:object}|null} */
+372 -12
View File
@@ -1,13 +1,27 @@
import { Readable } from 'node:stream';
import { hashSeed, escapeXml } from './normalize.js';
import { hashSeed, escapeXml, pinnedCameraUrl } from './normalize.js';
import {
A22_IMAGE_ORIGIN,
CALGARY_IMAGE_ORIGIN,
CCTV_FRAME_FETCH_TIMEOUT_MS,
CCTV_FRAME_MAX_BODY_BYTES,
CCTV_MEDIA_FETCH_TIMEOUT_MS,
CCTV_MEDIA_IDLE_TIMEOUT_MS,
CCTV_MEDIA_MAX_BODY_BYTES,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
MLIT_RIVER_IMAGE_ORIGIN,
TRIPCHECK_IMAGE_ORIGIN,
NSW_IMAGE_ORIGIN,
NSW_IMAGE_USER_AGENT,
OTTAWA_IMAGE_ORIGIN,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
ICELAND_IMAGE_ORIGIN,
TAIWAN_HIGHWAY_IMAGE_ORIGINS,
TFL_IMAGE_ORIGIN,
WARENDORF_IMAGE_ORIGINS,
WSDOT_IMAGE_ORIGIN,
} from './constants.js';
/**
* Generate a synthetic SVG billboard image for a CCTV camera placeholder.
@@ -84,7 +98,7 @@ export function buildSyntheticCctvSvg({ cameraId, label, city, status }) {
* @param {ReadableStream|NodeJS.ReadableStream|null} body
* @returns {import('stream').Readable|null}
*/
export function toReadable(body) {
function toReadable(body) {
if (!body) return null;
if (typeof body.pipe === 'function') return body;
if (typeof body.getReader === 'function') {
@@ -93,6 +107,73 @@ export function toReadable(body) {
return null;
}
/**
* Split a Content-Type value into its lower-case essence and parameters.
*
* @param {string|null|undefined} value
* @returns {{essence:string,params:Map<string,string>}|null} Null when the
* value is missing or is not a `type/subtype` media type.
*/
function parseMediaType(value) {
const [head, ...rest] = String(value ?? '').split(';');
const essence = head.trim().toLowerCase();
if (!/^[a-z0-9!#$&^_.+-]+\/[a-z0-9!#$&^_.+-]+$/.test(essence)) return null;
const params = new Map();
for (const part of rest) {
const eq = part.indexOf('=');
if (eq <= 0) continue;
const name = part.slice(0, eq).trim().toLowerCase();
let paramValue = part.slice(eq + 1).trim();
if (/^".*"$/.test(paramValue)) paramValue = paramValue.slice(1, -1);
if (name && !params.has(name)) params.set(name, paramValue);
}
return { essence, params };
}
/** Still-image types a camera frame may be served as. */
const CCTV_RASTER_IMAGE_TYPES = new Set([
'image/jpeg',
'image/png',
'image/gif',
'image/webp',
'image/avif',
]);
/**
* The canonical type of a raster camera frame, or null for any other type.
*
* @param {string|null|undefined} contentType
* @returns {string|null}
*/
export function cctvRasterImageType(contentType) {
const essence = parseMediaType(contentType)?.essence;
return CCTV_RASTER_IMAGE_TYPES.has(essence) ? essence : null;
}
/**
* The type to relay a live feed as, or null when the upstream type does not
* match the feed kind. MJPEG keeps its part boundary, without which the body
* cannot be split into frames.
*
* @param {string} feedType - Normalized feed type (mp4, webm or mjpeg).
* @param {string|null|undefined} contentType
* @returns {string|null}
*/
export function cctvLiveMediaType(feedType, contentType) {
const parsed = parseMediaType(contentType);
if (!parsed) return null;
if (feedType === 'mp4' || feedType === 'webm')
return parsed.essence === `video/${feedType}` ? parsed.essence : null;
if (feedType !== 'mjpeg' || parsed.essence !== 'multipart/x-mixed-replace')
return null;
const boundary = parsed.params.get('boundary') || '';
if (!/^[0-9A-Za-z'()+_,./:=? -]{0,69}[0-9A-Za-z'()+_,./:=?-]$/.test(boundary))
return null;
return /^[0-9A-Za-z'+_.-]+$/.test(boundary)
? `multipart/x-mixed-replace; boundary=${boundary}`
: `multipart/x-mixed-replace; boundary="${boundary}"`;
}
/**
* Pipe an upstream fetch Response (image or video) to the client HTTP response.
*
@@ -104,6 +185,8 @@ export function toReadable(body) {
* @param {Response} upstream - fetch() Response object.
* @param {object} [opts]
* @param {string} [opts.sourceHeader='upstream'] - Value for X-CCTV-Source header.
* @param {string} [opts.contentType] - Validated type to send instead of the
* upstream's own.
* @param {number} [opts.idleTimeoutMs=CCTV_MEDIA_IDLE_TIMEOUT_MS] - Silence
* allowed between upstream chunks before the stream is released. Injectable
* only to keep the deadline unit-testable.
@@ -114,10 +197,10 @@ export async function proxyMediaResponse(
{
sourceHeader = 'upstream',
idleTimeoutMs = CCTV_MEDIA_IDLE_TIMEOUT_MS,
contentType = upstream.headers.get('content-type') ||
'application/octet-stream',
} = {},
) {
const contentType =
upstream.headers.get('content-type') || 'application/octet-stream';
const cacheControl = upstream.headers.get('cache-control') || 'no-store';
const contentLength = upstream.headers.get('content-length');
const contentRange = upstream.headers.get('content-range');
@@ -340,7 +423,14 @@ export async function fetchCctvMediaUpstream(
if (downstream?.aborted) controller.abort();
else downstream?.addEventListener?.('abort', onDownstreamAbort);
try {
return await fetchImpl(url, { headers, signal: controller.signal });
// Redirects stay on the registered origin, as for frames.
const upstream = await fetchWithinHost(
url,
{ headers, signal: controller.signal },
fetchImpl,
);
if (!upstream) throw new Error('Media redirect refused');
return upstream;
} finally {
clearTimeout(timeoutId);
downstream?.removeEventListener?.('abort', onDownstreamAbort);
@@ -428,9 +518,46 @@ export async function fetchTxdotSnapshot(
/** Redirect hops the frame path will follow, and only within the same host. */
const MAX_SAME_HOST_REDIRECTS = 2;
/**
* Directories camera catalogs are pinned to, by host. A redirect on one of
* these hosts must stay inside the pinned directory too, so a same-origin
* redirect cannot leave it (another bucket on a shared storage host).
*/
const CCTV_PINNED_PREFIXES_BY_HOST = new Map();
for (const prefix of [
TFL_IMAGE_ORIGIN,
TALLINN_IMAGE_ORIGIN,
TARKTEE_IMAGE_ORIGIN,
ICELAND_IMAGE_ORIGIN,
...WARENDORF_IMAGE_ORIGINS,
NSW_IMAGE_ORIGIN,
OTTAWA_IMAGE_ORIGIN,
CALGARY_IMAGE_ORIGIN,
HK_IMAGE_ORIGIN,
KING_COUNTY_IMAGE_ORIGIN,
WSDOT_IMAGE_ORIGIN,
A22_IMAGE_ORIGIN,
TRIPCHECK_IMAGE_ORIGIN,
MLIT_RIVER_IMAGE_ORIGIN,
...TAIWAN_HIGHWAY_IMAGE_ORIGINS,
]) {
const host = new URL(prefix).hostname;
CCTV_PINNED_PREFIXES_BY_HOST.set(host, [
...(CCTV_PINNED_PREFIXES_BY_HOST.get(host) || []),
prefix,
]);
}
/** Whether a redirect target stays inside its host's pinned directory. */
function withinPinnedPrefix(url) {
const prefixes = CCTV_PINNED_PREFIXES_BY_HOST.get(url.hostname);
return !prefixes || prefixes.some((p) => pinnedCameraUrl(url.href, p));
}
/**
* Fetch a registered frame URL following redirects ONLY within the original
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops).
* origin (scheme, host and port; at most MAX_SAME_HOST_REDIRECTS hops) and,
* on a pinned camera host, within its pinned directory.
* Default redirect-following would let an upstream steer a host-pinned
* request, and its host-specific headers, to any origin, another port, or a
* plaintext downgrade.
@@ -471,7 +598,7 @@ export async function fetchWithinHost(url, init, fetchImpl = fetch) {
} catch {
return null;
}
if (next.origin !== origin) return null;
if (next.origin !== origin || !withinPinnedPrefix(next)) return null;
current = next;
}
return null;
@@ -504,6 +631,216 @@ export function cctvUpstreamUserAgent(url) {
}
}
/** Largest single JPEG the MJPEG first-frame reader will buffer. */
export const CCTV_MJPEG_FRAME_MAX_BYTES = 2 * 1024 * 1024;
/**
* Read one bounded JPEG part from an open MJPEG response, then stop the stream.
*
* The first part may follow a CRLF preamble, and its delimiter may be written
* as `--<boundary>` or, when the declared boundary already starts with `--`,
* as the boundary itself. A part with Content-Length is read to that length;
* a part without one ends at the next delimiter, so a JPEG end marker inside
* the image (an EXIF thumbnail) never truncates it. Bytes are copied once
* into a preallocated buffer and every search resumes where it stopped, so a
* trickling upstream costs linear time.
*/
async function readFirstMjpegFrame(upstream, maxBytes) {
const mediaType = cctvLiveMediaType(
'mjpeg',
upstream.headers.get('content-type'),
);
const boundary = parseMediaType(mediaType)?.params.get('boundary') || '';
const frameMax = Math.min(maxBytes, CCTV_MJPEG_FRAME_MAX_BYTES);
if (
!boundary ||
!upstream.body ||
!Number.isSafeInteger(frameMax) ||
frameMax < 4
) {
try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return null;
}
const delimiters = [Buffer.from('--' + boundary, 'latin1')];
if (boundary.startsWith('--') && boundary.length > 2)
delimiters.push(Buffer.from(boundary, 'latin1'));
const lineEnd = Buffer.from('\r\n', 'ascii');
const headerTerminator = Buffer.from('\r\n\r\n', 'ascii');
const jpegStart = Buffer.from([0xff, 0xd8]);
const jpegEnd = Buffer.from([0xff, 0xd9]);
const maxPreambleBytes = 64;
const maxHeaderBytes = 8 * 1024;
const maxBufferedBytes =
maxPreambleBytes +
delimiters[0].length +
lineEnd.length +
maxHeaderBytes +
headerTerminator.length +
frameMax +
lineEnd.length +
delimiters[0].length;
const buffer = Buffer.allocUnsafe(maxBufferedBytes);
let length = 0;
let reader;
let headerStart = -1;
let delimiter = null;
let headerScanFrom = 0;
let bodyStart = -1;
let declaredLength = null;
let boundaryScanFrom = -1;
/** Match the opening delimiter after an optional CRLF preamble. */
function findOpening() {
let at = 0;
while (at < length && (buffer[at] === 0x0d || buffer[at] === 0x0a)) at++;
if (at > maxPreambleBytes) return { failed: true };
for (const candidate of delimiters) {
const line = Buffer.concat([candidate, lineEnd]);
const need = at + line.length;
if (length < need) {
const seen = buffer.subarray(at, length);
if (line.subarray(0, seen.length).equals(seen))
return { pending: true };
continue;
}
if (buffer.subarray(at, need).equals(line))
return { start: need, delimiter: candidate };
}
return { failed: true };
}
/** Parse the part headers; false rejects the stream. */
function parseHeaders(headerEnd) {
const headerText = buffer
.subarray(headerStart, headerEnd)
.toString('latin1');
const headers = new Map();
for (const line of headerText.split('\r\n')) {
const colon = line.indexOf(':');
if (colon <= 0) return false;
const name = line.slice(0, colon).trim().toLowerCase();
if (!name || headers.has(name)) return false;
headers.set(name, line.slice(colon + 1).trim());
}
if (cctvRasterImageType(headers.get('content-type')) !== 'image/jpeg')
return false;
const declared = headers.get('content-length');
if (declared !== undefined) {
if (!/^\d+$/.test(declared)) return false;
declaredLength = Number(declared);
if (
!Number.isSafeInteger(declaredLength) ||
declaredLength < jpegStart.length + jpegEnd.length ||
declaredLength > frameMax
)
return false;
}
return true;
}
/** A complete JPEG body, or null. */
function asJpeg(frame) {
return frame.length >= jpegStart.length + jpegEnd.length &&
frame.subarray(0, jpegStart.length).equals(jpegStart) &&
frame.subarray(-jpegEnd.length).equals(jpegEnd)
? Buffer.from(frame)
: null;
}
try {
reader = upstream.body.getReader();
for (;;) {
const { done, value } = await reader.read();
if (done) return null;
const chunk = Buffer.from(value);
const room = maxBufferedBytes - length;
const included = Math.min(room, chunk.length);
chunk.copy(buffer, length, 0, included);
length += included;
const overflow = included < chunk.length;
if (headerStart < 0) {
const opening = findOpening();
if (opening.failed) return null;
if (opening.pending) {
if (overflow) return null;
continue;
}
headerStart = opening.start;
delimiter = opening.delimiter;
headerScanFrom = headerStart;
}
if (bodyStart < 0) {
const headerEnd = buffer
.subarray(0, length)
.indexOf(headerTerminator, headerScanFrom);
if (headerEnd < 0) {
if (length - headerStart > maxHeaderBytes || overflow) return null;
headerScanFrom = Math.max(
headerStart,
length - headerTerminator.length + 1,
);
continue;
}
if (headerEnd - headerStart > maxHeaderBytes) return null;
if (!parseHeaders(headerEnd)) return null;
bodyStart = headerEnd + headerTerminator.length;
boundaryScanFrom = bodyStart;
}
const available = length - bodyStart;
if (available >= jpegStart.length) {
if (
!buffer
.subarray(bodyStart, bodyStart + jpegStart.length)
.equals(jpegStart)
)
return null;
}
const nextDelimiter = Buffer.concat([lineEnd, delimiter]);
const nextPart = buffer
.subarray(0, length)
.indexOf(nextDelimiter, boundaryScanFrom);
if (declaredLength !== null) {
if (nextPart >= bodyStart && nextPart - bodyStart < declaredLength)
return null;
if (available >= declaredLength)
return asJpeg(buffer.subarray(bodyStart, bodyStart + declaredLength));
} else if (nextPart >= bodyStart) {
const frame = buffer.subarray(bodyStart, nextPart);
return frame.length > frameMax ? null : asJpeg(frame);
} else if (available > frameMax) {
return null;
}
if (nextPart < 0)
boundaryScanFrom = Math.max(
bodyStart,
length - nextDelimiter.length + 1,
);
if (overflow) return null;
}
} finally {
if (reader) {
try {
await reader.cancel();
} catch {
/* already closed or aborted */
}
try {
reader.releaseLock();
} catch {
/* already released */
}
}
}
}
/**
* Fetch one upstream CCTV image within the frame-refresh budget.
*
@@ -516,6 +853,10 @@ export function cctvUpstreamUserAgent(url) {
* @param {typeof fetch} [options.fetchImpl=fetch] - Fetch implementation.
* @param {number} [options.timeoutMs=CCTV_FRAME_FETCH_TIMEOUT_MS] - Abort timeout.
* @param {number} [options.maxBytes=CCTV_FRAME_MAX_BODY_BYTES] - Snapshot byte cap.
* @param {boolean} [options.allowMjpeg=false] - Accept a multipart MJPEG
* answer and keep its first JPEG. Only sources registered as MJPEG stills
* (the Taiwan Highway Bureau feeds) set it; any other host answering
* multipart falls through to the next fallback.
* @returns {Promise<{ok:true,body:Buffer,contentType:string}|null>}
*/
export async function fetchCctvImageFromUpstream(
@@ -524,6 +865,7 @@ export async function fetchCctvImageFromUpstream(
fetchImpl = fetch,
timeoutMs = CCTV_FRAME_FETCH_TIMEOUT_MS,
maxBytes = CCTV_FRAME_MAX_BODY_BYTES,
allowMjpeg = false,
} = {},
) {
if (!url || !/^https?:\/\//i.test(url)) return null;
@@ -543,14 +885,32 @@ export async function fetchCctvImageFromUpstream(
fetchImpl,
);
if (!upstream) return null;
const contentType = upstream.headers.get('content-type') || '';
if (!upstream.ok || !contentType.startsWith('image/')) {
controller.abort();
if (!upstream.ok) {
try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return null;
}
const body = await readCappedResponseBytes(upstream, maxBytes);
const upstreamType = upstream.headers.get('content-type');
const contentType = cctvRasterImageType(upstreamType);
if (contentType) {
const body = await readCappedResponseBytes(upstream, maxBytes);
if (!body) return null;
return { ok: true, body, contentType };
}
if (!allowMjpeg || !cctvLiveMediaType('mjpeg', upstreamType)) {
try {
await upstream.body?.cancel();
} catch {
/* no-op */
}
return null;
}
const body = await readFirstMjpegFrame(upstream, maxBytes);
if (!body) return null;
return { ok: true, body, contentType };
return { ok: true, body, contentType: 'image/jpeg' };
} catch {
return null;
} finally {
+76 -3
View File
@@ -64,7 +64,7 @@ export function toFiniteNumber(value, fallback = NaN) {
* @param {string} text
* @returns {string}
*/
export function normalizeKey(text) {
function normalizeKey(text) {
return String(text || '')
.trim()
.toLowerCase()
@@ -80,7 +80,7 @@ export function normalizeKey(text) {
* @param {string} value
* @returns {{lat:number, lon:number}}
*/
export function parsePointString(value) {
function parsePointString(value) {
const match = String(value || '').match(
/POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)/i,
);
@@ -100,7 +100,7 @@ export function parsePointString(value) {
* @param {string|object|null} value
* @returns {{lat:number, lon:number}}
*/
export function coerceLatLon(value) {
function coerceLatLon(value) {
if (!value) return { lat: NaN, lon: NaN };
if (typeof value === 'string') {
@@ -410,6 +410,18 @@ export function isLikelyHongKongCoordinate(lat, lon) {
);
}
/** Greater Houston (the Woodlands to Galveston, Katy to Baytown). Farther
* TranStar partners (Dallas, Columbus) are out of this city pack. */
export function isLikelyHoustonCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 29.2 &&
lat <= 30.35 &&
lon >= -96.05 &&
lon <= -94.7
);
}
/** Mainland Norway, with slack for the coast and the Finnmark border. */
export function isLikelyNorwayCoordinate(lat, lon) {
return (
@@ -421,6 +433,17 @@ export function isLikelyNorwayCoordinate(lat, lon) {
);
}
/** The A22 corridor, Brenner Pass to Modena, with slack either side. */
export function isLikelyA22Coordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 44.5 &&
lat <= 47.1 &&
lon >= 10.5 &&
lon <= 11.8
);
}
export function isLikelyFinlandCoordinate(lat, lon) {
if (!Number.isFinite(lat) || !Number.isFinite(lon)) return false;
return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32;
@@ -437,6 +460,17 @@ export function isLikelyCataloniaCoordinate(lat, lon) {
);
}
/** Iceland bounding box (mainland and nearby islands, with a margin). */
export function isLikelyIcelandCoordinate(lat, lon) {
return (
isPlausibleLatLon(lat, lon) &&
lat >= 63.0 &&
lat <= 66.7 &&
lon >= -25.0 &&
lon <= -13.0
);
}
/**
* Decode numeric HTML character references, decimal ("&#237;") and hex
* ("&#xED;") alike, the entity form the Catalonia feed uses for accented
@@ -616,6 +650,10 @@ export function normalizeSourceItem(item) {
feedType: normalizeFeedType(item.feedType || item.type || ''),
url: typeof item.url === 'string' ? item.url : '',
snapshotUrl: typeof item.snapshotUrl === 'string' ? item.snapshotUrl : '',
frameRefreshMs: toFiniteNumber(item.frameRefreshMs),
sourceUrl: typeof item.sourceUrl === 'string' ? item.sourceUrl : '',
accuracyM: toFiniteNumber(item.accuracyM),
positionSource: String(item.positionSource || ''),
license: String(item.license || item.licenseNote || ''),
// Per-camera attribution for feeds a partner supplies inside a pack
// (DriveBC: TransLink, city cameras). Shown beside the provider.
@@ -659,3 +697,38 @@ export function isLikelyTorontoCoordinate(lat, lon) {
lon <= -79.1
);
}
/**
* A camera URL under a pinned directory, by parsed URL rather than string
* prefix: the same scheme and exact host as `prefix`, no credentials, the
* default port, and a normalized path inside the prefix's directory. Dot
* segments (including encoded ones) and backslashes are resolved before the
* path is compared, and an encoded slash or backslash is refused.
*
* @param {string} raw - Camera URL from a catalog or redirect.
* @param {string} prefix - Pinned origin and directory, ending in `/`.
* @returns {string|null} The normalized URL, or null when it is not pinned.
*/
export function pinnedCameraUrl(raw, prefix) {
let url;
let pin;
try {
url = new URL(String(raw ?? '').trim());
pin = new URL(prefix);
} catch {
return null;
}
if (
url.protocol !== pin.protocol ||
url.hostname !== pin.hostname ||
url.username ||
url.password ||
url.port ||
!url.pathname.startsWith(
pin.pathname.endsWith('/') ? pin.pathname : `${pin.pathname}/`,
) ||
/%2f|%5c/i.test(url.pathname)
)
return null;
return url.href;
}
+185
View File
@@ -0,0 +1,185 @@
import {
CCTV_SOURCE_FETCH_TIMEOUT_MS,
DEFAULT_OTTAWA_MAX_SOURCES,
OTTAWA_ANCHORS,
OTTAWA_API_USER_ID,
OTTAWA_CAMERAS_URL,
OTTAWA_FRAME_REFRESH_MS,
OTTAWA_IMAGE_ORIGIN,
OTTAWA_IMAGE_URL,
} from './constants.js';
import {
fallbackHeadingFromId,
pinnedCameraUrl,
prioritizeSources,
toFiniteNumber,
} from './normalize.js';
let ottawaCertificateMissingLogged = false;
/** Test hook: let the next missing-certificate skip log again. */
export function _resetOttawaCertificateWarningForTest() {
ottawaCertificateMissingLogged = false;
}
/** The operator's Traffic Camera Images API certificate, trimmed. */
function ottawaCertificate(env = process.env) {
return String(env.OTTAWA_CAMERA_CERTIFICATE ?? '').trim();
}
/**
* Add the operator's certificate to a registered keyless Ottawa frame URL.
* Only a URL inside the pinned `/opendata/` directory with a numeric camera
* number is signed, so the certificate never goes to any other path or host.
*
* @param {string} url - Canonical frame URL from the catalog.
* @param {Record<string,string|undefined>} [env]
* @returns {string} The signed URL, or '' when there is no certificate or the
* URL is not a registered Ottawa frame.
*/
export function ottawaAuthorizedFrameUrl(url, env = process.env) {
const certificate = ottawaCertificate(env);
const pinned = pinnedCameraUrl(url, OTTAWA_IMAGE_ORIGIN);
if (!certificate || !pinned) return '';
const parsed = new URL(pinned);
const cameraNumber = parsed.searchParams.get('c') || '';
if (parsed.pathname !== '/opendata/camera' || !/^\d+$/.test(cameraNumber))
return '';
const signed = new URL(OTTAWA_IMAGE_URL(cameraNumber));
signed.searchParams.set('certificate', certificate);
signed.searchParams.set('id', OTTAWA_API_USER_ID);
return signed.href;
}
/**
* City of Ottawa envelope: Kanata to Orléans, Barrhaven to the river, with
* slack for the MTO cameras on the 416/417 that the city feed also carries.
*/
const OTTAWA_BOUNDS = Object.freeze({
minLat: 45.1,
maxLat: 45.65,
minLon: -76.2,
maxLon: -75.1,
});
/** Whether a point is inside the city feed's service area. */
function isOttawaCoordinate(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
lat >= OTTAWA_BOUNDS.minLat &&
lat <= OTTAWA_BOUNDS.maxLat &&
lon >= OTTAWA_BOUNDS.minLon &&
lon <= OTTAWA_BOUNDS.maxLon
);
}
/**
* Load the City of Ottawa's traffic cameras.
*
* Camera images come from the city's Traffic Camera Images API, which needs a
* certificate from the city's registration form (OTTAWA_CAMERA_CERTIFICATE)
* and allows one fetch per camera per minute for a certificate and user id.
* Without a certificate the pack is skipped, with no request and one log line.
* The camera list is the documented `map/service/camera` JSON: a camera
* number, English and French names, coordinates and the owner (CITY or MTO).
* The feed carries no bearing, no elevation and no status field, so headings
* use the same deterministic low-confidence fallback as the other
* heading-less packs and the client's ground snap refines the elevation
* prior.
*
* @param {object} [_options]
* @param {Record<string,string|undefined>} [env]
* @returns {Promise<Array<object>>} Cameras nearest downtown first, or [] on
* a missing certificate or any upstream or payload failure.
*/
export async function loadOttawaSourcesFromOpenData(
_options = {},
env = process.env,
) {
if (!ottawaCertificate(env)) {
if (!ottawaCertificateMissingLogged) {
ottawaCertificateMissingLogged = true;
console.warn(
'[CCTV] Ottawa cameras skipped: the city camera image API needs a certificate (set OTTAWA_CAMERA_CERTIFICATE)',
);
}
return [];
}
try {
const resp = await fetch(OTTAWA_CAMERAS_URL, {
headers: { Accept: 'application/json' },
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
if (!resp.ok) {
console.warn('[CCTV] Ottawa camera download failed:', resp.status);
return [];
}
const payload = await resp.json();
const rows = Array.isArray(payload?.cameras) ? payload.cameras : null;
if (!rows) return [];
const cameras = [];
const seen = new Set();
for (const row of rows) {
const number = toFiniteNumber(row?.camera_number);
if (!Number.isSafeInteger(number) || number <= 0 || seen.has(number))
continue;
const lat = toFiniteNumber(row?.latitude);
const lon = toFiniteNumber(row?.longitude);
if (!isOttawaCoordinate(lat, lon)) continue;
seen.add(number);
const cameraId = `ottawa-${number}`;
const imageUrl = OTTAWA_IMAGE_URL(number);
// The docs name the field camera_owner; the live feed sends cameraOwner.
const owner = String(row?.camera_owner || row?.cameraOwner || '')
.trim()
.toUpperCase();
cameras.push({
id: cameraId,
name: String(row?.name || '').trim() || `Ottawa camera ${number}`,
city: 'Ottawa',
cityId: 'ottawa',
provider: 'City of Ottawa',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
groundElevationM: 0,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
frameRefreshMs: OTTAWA_FRAME_REFRESH_MS,
sourceKind: 'ottawa-traffic',
license: 'City of Ottawa Traffic Camera Images API (registered access)',
credit:
owner === 'MTO'
? 'Ministry of Transportation of Ontario (MTO) camera'
: '',
});
}
const maxRaw = Number(
process.env.CCTV_OTTAWA_MAX_SOURCES || DEFAULT_OTTAWA_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(500, Math.floor(maxRaw)))
: DEFAULT_OTTAWA_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, OTTAWA_ANCHORS);
console.log(
`[CCTV] Loaded Ottawa camera sources: ${cameras.length} published (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] Ottawa camera download error:',
error?.message || error,
);
return [];
}
}
+138
View File
@@ -0,0 +1,138 @@
import fs from 'node:fs';
import path from 'node:path';
import { cameraDisplayCode, fallbackHeadingFromId } from './normalize.js';
const QUEBEC_CATALOG_PATH = 'public/quebec-cameras.json';
export const QUEBEC_CAMERA_SOURCE_COUNT = 1120;
/**
* Whether the operator opted in to the browser-direct Québec 511 video. Off by
* default: the camera locations are CC BY 4.0, but whether Québec 511's
* service terms allow another site to embed its clips is unconfirmed, so an
* unset flag lists the cameras with a link to the 511 viewer page only.
*/
export function quebecBrowserVideoEnabled(env = process.env) {
return String(env.CCTV_QUEBEC_VIDEO_ENABLED || '').trim() === '1';
}
/** Build the direct browser-only MP4 URL for one namespaced MTQ camera id. */
export function quebec511VideoUrl(cameraId) {
const match = /^qc-511-(\d+)$/.exec(String(cameraId || ''));
return match
? `https://www.quebec511.info/Carte/Fenetres/camera.ashx?id=${match[1]}&format=mp4`
: '';
}
function safeViewerUrl(value) {
try {
const url = new URL(String(value || ''));
return url.protocol === 'https:' && !url.username && !url.password
? url.href
: '';
} catch {
return '';
}
}
function sourceId(camera) {
const id = String(camera?.id || '').trim();
if (camera?.kind === 'traffic' && /^\d+$/.test(id)) return `qc-511-${id}`;
if (camera?.kind === 'montreal' && /^mtl-\d+(?:-\d+)*$/.test(id))
return `qc-mtl-${id.slice(4)}`;
if (camera?.kind === 'webcam' && /^web-\d+$/.test(id))
return `qc-web-${id.slice(4)}`;
return '';
}
/**
* Read the checked-in, no-key Québec location catalog and adapt it to the
* regular CCTV source schema. MTQ video stays browser-only; other rows link to
* their operator page because this dataset has no direct image URL for them.
*/
export function loadQuebecSources({ sourceRoot = process.cwd() } = {}) {
const filename = path.resolve(sourceRoot, QUEBEC_CATALOG_PATH);
let cameras;
try {
const payload = JSON.parse(fs.readFileSync(filename, 'utf8'));
cameras = Array.isArray(payload?.cameras) ? payload.cameras : [];
} catch (error) {
console.warn('[CCTV] Québec camera catalog unavailable:', error?.message);
return [];
}
const sources = [];
const seen = new Set();
for (const camera of cameras) {
const id = sourceId(camera);
const lat = Number(camera.lat);
const lon = Number(camera.lon);
if (
!id ||
seen.has(id) ||
!Number.isFinite(lat) ||
!Number.isFinite(lon) ||
lat < 44.9 ||
lat > 62.6 ||
lon < -79.9 ||
lon > -57.1
)
continue;
seen.add(id);
const name = String(camera.nameEn || camera.name || id).trim();
const isTraffic = camera.kind === 'traffic';
const isMontreal = camera.kind === 'montreal';
const sourceKind = isTraffic
? 'quebec-511-browser'
: isMontreal
? 'quebec-montreal-osm'
: 'quebec-webcam-link';
const provider = isTraffic
? 'Québec 511 / MTQ'
: isMontreal
? 'Ville de Montréal / OpenStreetMap'
: String(camera.operator || 'Québec webcam');
sources.push({
id,
name: name.slice(0, 180),
city: String(camera.region || 'Québec'),
cityId: 'quebec',
provider,
lat,
lon,
headingDeg: fallbackHeadingFromId(id),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 74,
rangeM: 400,
mountHeightM: 8,
groundElevationM: 0,
feedType: 'image',
// No frame URL is registered: the server never fetches 511 media. With
// CCTV_QUEBEC_VIDEO_ENABLED=1 the browser plays the 511 MP4 in the DOM
// overlay after a play click; otherwise the camera links to 511.
url: '',
snapshotUrl: '',
sourceKind,
sourceUrl: safeViewerUrl(camera.viewerUrl),
accuracyM: Number.isFinite(Number(camera.accuracyM))
? Number(camera.accuracyM)
: null,
positionSource: String(camera.positionSource || ''),
license: isTraffic
? 'Locations: Données Québec / MTQ, CC BY 4.0. Video: hosted by Québec 511 under its terms of use'
: isMontreal
? 'OpenStreetMap contributors, ODbL 1.0'
: 'Operator terms apply; location provenance shown per camera',
credit: isTraffic
? 'Locations: MTQ · CC BY 4.0 · Video: Québec 511'
: isMontreal
? 'OpenStreetMap contributors · ODbL'
: provider,
code: cameraDisplayCode(name.toUpperCase()),
});
}
return sources;
}
+509 -30
View File
@@ -122,6 +122,24 @@ import {
NYC_MAX_CATALOG_BYTES,
NYC_GROUND_ELEVATION_M,
NYC_ANCHORS,
DEFAULT_A22_CCTV_URL,
A22_IMAGE_ORIGIN,
A22_MAX_CATALOG_BYTES,
A22_GROUND_ELEVATION_BY_KM,
A22_DEFAULT_GROUND_ELEVATION_M,
HOUSTON_CAMERAS_URL,
HOUSTON_IMAGE_URL,
HOUSTON_CAMERA_ID_RE,
DEFAULT_HOUSTON_MAX_SOURCES,
HOUSTON_MAX_CATALOG_BYTES,
HOUSTON_GROUND_ELEVATION_M,
HOUSTON_ANCHORS,
ICELAND_CAMERAS_URL,
ICELAND_IMAGE_ORIGIN,
ICELAND_MAX_CATALOG_BYTES,
ICELAND_GROUND_ELEVATION_M,
DEFAULT_ICELAND_MAX_SOURCES,
ICELAND_ANCHORS,
} from './constants.js';
import {
toFiniteNumber,
@@ -133,7 +151,6 @@ import {
fallbackHeadingFromId,
isLikelyFinlandCoordinate,
fintrafficCameraName,
hashSeed,
isPlausibleLatLon,
isLikelyBcCoordinate,
isLikelyTexasCoordinate,
@@ -141,14 +158,18 @@ import {
isLikelyCalgaryCoordinate,
isLikelyFloridaCoordinate,
isLikelyCataloniaCoordinate,
isLikelyIcelandCoordinate,
decodeNumericEntities,
isLikelyNycCoordinate,
isLikelyHoustonCoordinate,
isLikelyHongKongCoordinate,
isLikelyNorwayCoordinate,
isLikelyTorontoCoordinate,
isLikelyA22Coordinate,
cameraDisplayCode,
rowArrayToObject,
prioritizeSources,
pinnedCameraUrl,
} from './normalize.js';
import { directionToHeading } from '../../../src/data/directionText.js';
import {
@@ -310,9 +331,12 @@ export async function loadCaltransSourcesFromOpenData() {
const lon = toFiniteNumber(loc.longitude);
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
const imageUrl = String(cctv.imageData?.static?.currentImageURL || '');
// Official-host pin (see JSDoc). Also drops records with no still image.
if (!imageUrl.startsWith('https://cwwp2.dot.ca.gov/')) continue;
const imageUrl = pinnedCameraUrl(
cctv.imageData?.static?.currentImageURL,
'https://cwwp2.dot.ca.gov/',
);
if (!imageUrl) continue;
const locationName = String(loc.locationName || '').trim();
// Leading token of locationName is the stable camera code ("TV102 -- I-580 : …").
@@ -417,8 +441,8 @@ export async function loadTflSourcesFromOpenData() {
const lat = toFiniteNumber(place?.lat);
const lon = toFiniteNumber(place?.lon);
if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
const imageUrl = String(props.imageUrl || '');
if (!imageUrl.startsWith(TFL_IMAGE_ORIGIN)) continue; // official-bucket pin
const imageUrl = pinnedCameraUrl(props.imageUrl, TFL_IMAGE_ORIGIN);
if (!imageUrl) continue; // official-bucket pin
// "JamCams_00002.00865" → "tfl-00002.00865" (provider-stable id).
const rawId = String(place?.id || '').replace(/^JamCams_/, '');
@@ -1127,13 +1151,15 @@ export function loadTallinnSourcesFromCatalog({
// Rough Estonia/Tallinn metro sanity (allows nearby suburbs already in the pack).
if (lat < 59.2 || lat > 59.7 || lon < 24.3 || lon > 25.4) continue;
const imageUrl =
const imageUrl = pinnedCameraUrl(
typeof item.url === 'string'
? item.url.trim()
? item.url
: typeof item.snapshotUrl === 'string'
? item.snapshotUrl.trim()
: '';
if (!imageUrl.startsWith(TALLINN_IMAGE_ORIGIN)) continue;
? item.snapshotUrl
: '',
TALLINN_IMAGE_ORIGIN,
);
if (!imageUrl) continue;
const extractedHeading = toFiniteNumber(item.headingDeg, NaN);
const hasHeading = Number.isFinite(extractedHeading);
@@ -1235,8 +1261,8 @@ export function parseTarkteeDatexImages(xml) {
body,
);
if (!refMatch || !urlMatch) continue;
const url = urlMatch[1].trim();
if (!url.startsWith(TARKTEE_IMAGE_ORIGIN)) continue;
const url = pinnedCameraUrl(urlMatch[1], TARKTEE_IMAGE_ORIGIN);
if (!url) continue;
out.set(refMatch[1], url);
}
return out;
@@ -1422,14 +1448,16 @@ export function loadWarendorfSourcesFromCatalog({
for (const item of rows) {
if (!item || typeof item !== 'object') continue;
const id = typeof item.id === 'string' ? item.id.trim() : '';
const url =
const raw =
typeof item.url === 'string'
? item.url.trim()
? item.url
: typeof item.snapshotUrl === 'string'
? item.snapshotUrl.trim()
? item.snapshotUrl
: '';
if (!id || !WARENDORF_IMAGE_ORIGINS.some((o) => url.startsWith(o)))
continue;
const url = WARENDORF_IMAGE_ORIGINS.map((o) =>
pinnedCameraUrl(raw, o),
).find(Boolean);
if (!id || !url) continue;
const lat = typeof item.lat === 'number' ? item.lat : NaN;
const lon = typeof item.lon === 'number' ? item.lon : NaN;
if (!isPlausibleLatLon(lat, lon)) continue;
@@ -1483,8 +1511,8 @@ export function nswCameraToSource(feature) {
const lat = typeof coords?.[1] === 'number' ? coords[1] : NaN;
if (!isLikelyNswCoordinate(lat, lon)) return null;
const props = feature?.properties || {};
const url = String(props.href || '').trim();
if (!url.startsWith(NSW_IMAGE_ORIGIN)) return null;
const url = pinnedCameraUrl(props.href, NSW_IMAGE_ORIGIN);
if (!url) return null;
// "N-E" -> "NE" for the compass lookup.
const direction = String(props.direction || '')
.trim()
@@ -1578,8 +1606,7 @@ export function normalizeCalgaryImageUrl(raw) {
}
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null;
parsed.protocol = 'https:';
const upgraded = parsed.toString();
return upgraded.startsWith(CALGARY_IMAGE_ORIGIN) ? upgraded : null;
return pinnedCameraUrl(parsed.toString(), CALGARY_IMAGE_ORIGIN);
}
/**
@@ -1862,7 +1889,7 @@ export function normalizeCataloniaImageUrl(rawUrl) {
* @param {string} imageUrl - A URL normalizeCataloniaImageUrl accepted.
* @returns {string}
*/
export function cataloniaCameraId(imageUrl) {
function cataloniaCameraId(imageUrl) {
const parsed = new URL(imageUrl);
const slug = (text) =>
text
@@ -2009,6 +2036,174 @@ export async function loadCataloniaSourcesFromOpenData() {
}
}
/**
* Compass headings for Icelandic direction words, as they appear after "til"
* (genitive: "til vesturs"), "í" (accusative: "í vestur") or "séð" ("seen",
* as in "séð austur").
*/
const ICELAND_DIRECTION_HEADINGS = {
norður: 0,
norðaustur: 45,
austur: 90,
suðaustur: 135,
suður: 180,
suðvestur: 225,
vestur: 270,
norðvestur: 315,
};
const ICELAND_DIRECTION_WORD =
'(norðaustur|norðvestur|suðaustur|suðvestur|norður|suður|austur|vestur)s?';
/** A facing, plus a second one joined by "og" ("til austurs og vesturs"). */
const ICELAND_DIRECTION_RE = new RegExp(
`(?<![\\p{L}])(?:til|í|séð)\\s+${ICELAND_DIRECTION_WORD}` +
`(?:\\s+og\\s+(?:til\\s+|í\\s+)?${ICELAND_DIRECTION_WORD})?(?![\\p{L}])`,
'gu',
);
/**
* The compass heading an Icelandic view description states, or NaN. Only an
* explicit "til <átt>", "í <átt>" or "séð <átt>" counts: a bare place name or
* a road direction ("Miklabraut austur") is not a facing. A description naming
* two different directions ("til norðurs og til suðurs", "til austurs og
* vesturs") is ambiguous and yields NaN, and "niður á veg" (down onto the
* road) states none.
*
* @param {string} text - The feed's `Skyring` field.
* @returns {number} Heading in degrees [0..360), or NaN.
*/
export function icelandHeadingFromDescription(text) {
const headings = new Set();
for (const match of String(text || '')
.toLowerCase()
.matchAll(ICELAND_DIRECTION_RE)) {
for (const word of [match[1], match[2]]) {
if (word) headings.add(ICELAND_DIRECTION_HEADINGS[word]);
}
}
return headings.size === 1 ? headings.values().next().value : NaN;
}
/**
* One Vegagerðin camera row -> one catalog source, or null. A row is one camera
* view; a station (`Maelist_nr`) carries several. The id comes from the frame
* file name, which is unique per view in the live feed.
*
* @param {object} row - Element of the `vefmyndavelar2014_1` array.
* @returns {?object}
*/
export function icelandCameraToSource(row) {
const url = pinnedCameraUrl(row?.Slod, ICELAND_IMAGE_ORIGIN);
if (!url || !/\.jpe?g$/i.test(new URL(url).pathname)) return null;
// Numbers only: Number(null) and Number('') are 0.
const lat = typeof row.Breidd === 'number' ? row.Breidd : NaN;
const lon = typeof row.Lengd === 'number' ? row.Lengd : NaN;
if (!isLikelyIcelandCoordinate(lat, lon)) return null;
const stem = new URL(url).pathname
.slice(new URL(ICELAND_IMAGE_ORIGIN).pathname.length)
.replace(/\.jpe?g$/i, '')
.toLowerCase();
if (!/^[a-z0-9_-]+$/.test(stem)) return null;
const cameraId = `is-vegagerdin-${stem}`;
const description = String(row.Skyring || '')
.replace(/\s+/g, ' ')
// Letter lookarounds, not \b: \b is ASCII-only and would join "Þverá
// Árnessýslu" while missing "séð séð".
.replace(/(?<!\p{L})(\p{L}+) \1(?!\p{L})/giu, '$1')
.trim();
const station = String(row.Myndavel || '').trim();
const heading = icelandHeadingFromDescription(description);
const hasHeading = Number.isFinite(heading);
return {
id: cameraId,
name: description || station || `Vegagerðin ${row.Maelist_nr ?? stem}`,
city: 'Iceland',
cityId: 'iceland',
provider: 'Vegagerðin (IRCA)',
lat,
lon,
headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId),
headingConfidence: hasHeading ? 'high' : 'low',
pitchDeg: hasHeading ? -24 : -18,
fovDeg: hasHeading ? 56 : 44,
rangeM: hasHeading ? 210 : 145,
mountHeightM: hasHeading ? 10 : 8,
groundElevationM: ICELAND_GROUND_ELEVATION_M,
feedType: 'image',
url,
snapshotUrl: url,
sourceKind: 'iceland-vegagerdin',
license:
'Icelandic Road and Coastal Administration (IRCA) — road camera images, CC BY 4.0',
code: cameraDisplayCode(station.toUpperCase() || description),
};
}
/**
* Fetch Icelandic road cameras (Vegagerðin / IRCA), keyless: one JSON list.
* Frames are stills on the IRCA host and the loader registers only links
* inside that host's camera directory (ICELAND_IMAGE_ORIGIN). The list decides
* ids, coordinates, labels and which frame path is proxied, so it is fetched
* over https, redirects are refused and the body read is capped at
* ICELAND_MAX_CATALOG_BYTES.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadIcelandSourcesFromOpenData() {
try {
const resp = await fetch(ICELAND_CAMERAS_URL, {
headers: {
Accept: 'application/json',
'User-Agent': 'gods-eye-view-cctv-proxy/1.0',
},
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
// A response this loader will not read still owns its transport until the
// body is released, so every rejection path cancels before returning.
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn(
'[CCTV] Iceland catalog redirected; redirects are not followed',
);
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] Iceland camera download failed:', resp.status);
return discard();
}
const body = await readResponseJsonCapped(resp, ICELAND_MAX_CATALOG_BYTES);
const rows = Array.isArray(body) ? body : [];
const cameras = rows.map(icelandCameraToSource).filter(Boolean);
const unique = Array.from(
new Map(cameras.map((camera) => [camera.id, camera])).values(),
);
const maxRaw = Number(
process.env.CCTV_ICELAND_MAX_SOURCES || DEFAULT_ICELAND_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(600, Math.floor(maxRaw)))
: DEFAULT_ICELAND_MAX_SOURCES;
const prioritized = prioritizeSources(unique, maxCount, ICELAND_ANCHORS);
console.log(
`[CCTV] Loaded Iceland camera sources: ${unique.length} (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] Iceland camera download error:',
error?.message || error,
);
return [];
}
}
/**
* Read one XML child tag's text from an `<image>` block.
*
@@ -2085,8 +2280,7 @@ export function normalizeHkImageUrl(raw) {
}
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return null;
parsed.protocol = 'https:';
const upgraded = parsed.toString();
return upgraded.startsWith(HK_IMAGE_ORIGIN) ? upgraded : null;
return pinnedCameraUrl(parsed.toString(), HK_IMAGE_ORIGIN);
}
/**
@@ -2423,10 +2617,16 @@ export async function loadKingCountySourcesFromOpenData() {
// county image origin, so they are skipped.
if (owner === 'WSDOT') continue;
// The handler is served over HTTPS; the feed still writes http://.
const imageUrl = String(attrs.ImageURL || '')
.trim()
.replace(/^http:\/\/info\.kingcounty\.gov\//, KING_COUNTY_IMAGE_ORIGIN);
if (!imageUrl.startsWith(KING_COUNTY_IMAGE_ORIGIN)) continue;
const imageUrl = pinnedCameraUrl(
String(attrs.ImageURL || '')
.trim()
.replace(
/^http:\/\/info\.kingcounty\.gov\//,
KING_COUNTY_IMAGE_ORIGIN,
),
KING_COUNTY_IMAGE_ORIGIN,
);
if (!imageUrl) continue;
const [lon, lat] = Array.isArray(row?.geometry?.coordinates)
? row.geometry.coordinates
: [];
@@ -2586,8 +2786,8 @@ export async function loadWsdotSourcesFromOpenData() {
const attrs = row?.attributes;
const rawId = attrs?.CameraID;
if (!Number.isSafeInteger(rawId) || rawId <= 0) continue;
const imageUrl = String(attrs.ImageURL || '').trim();
if (!imageUrl.startsWith(WSDOT_IMAGE_ORIGIN)) continue;
const imageUrl = pinnedCameraUrl(attrs.ImageURL, WSDOT_IMAGE_ORIGIN);
if (!imageUrl) continue;
const x = row?.geometry?.x;
const y = row?.geometry?.y;
if (!Number.isFinite(x) || !Number.isFinite(y)) continue;
@@ -3634,3 +3834,282 @@ export async function loadFloridaSourcesFromOpenData() {
return [];
}
}
/**
* One Open Data Hub `WebcamInfo` item (source `a22`) -> one catalog source,
* or null.
*
* The frame must be the operator's own `WebCamImg/km<N>.jpg` for the
* camera's motorway kilometre. The feed names the place and the kilometre but
* no bearing, so the heading uses the shared id-hash fallback at low
* confidence, like Calgary and Vegvesen; the ground elevation comes from the
* per-kilometre DEM table because the road spans ~1,350 m of altitude.
*
* @param {object} item - One element of the API's `Items` array.
* @returns {?object}
*/
export function a22WebcamToSource(item) {
if (!item || typeof item !== 'object') return null;
if (item.Active === false) return null;
if (String(item.Source ?? '').toLowerCase() !== 'a22') return null;
const gps = Array.isArray(item.GpsInfo) ? item.GpsInfo[0] : null;
const lat = toFiniteNumber(gps?.Latitude);
const lon = toFiniteNumber(gps?.Longitude);
if (!isLikelyA22Coordinate(lat, lon)) return null;
const km = String(item.Mapping?.a22?.km ?? '').trim();
if (!/^\d{1,3}$/.test(km)) return null;
const imageUrl = `${A22_IMAGE_ORIGIN}km${Number(km)}.jpg`;
let image;
try {
image = new URL(String(item.Webcamurl ?? ''));
} catch {
return null;
}
if (image.username || image.password || image.href !== imageUrl) return null;
const place =
String(item.Shortname ?? item.Webcamname?.it ?? '')
.replace(/\s+/g, ' ')
.trim() || `Km ${Number(km)}`;
const cameraId = `it-a22-km${Number(km)}`;
const ground = A22_GROUND_ELEVATION_BY_KM[Number(km)];
return {
id: cameraId,
name: `A22 ${place} (km ${Number(km)})`,
// One motorway-wide category in the camera picker; the place is already
// in the name.
city: 'A22 Brennero',
cityId: 'a22',
provider: 'Autostrada del Brennero',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -15,
fovDeg: 50,
rangeM: 160,
mountHeightM: 8,
groundElevationM: Number.isFinite(ground)
? ground
: A22_DEFAULT_GROUND_ELEVATION_M,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'a22-opendatahub',
license:
'Webcam images © Autostrada del Brennero S.p.A., via the Open Data Hub (NOI Techpark)',
code: cameraDisplayCode(place.toUpperCase()),
};
}
/**
* Fetch the Autostrada del Brennero (A22) webcams from the keyless Open Data
* Hub tourism API. Frames are stills on www.autobrennero.it.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadA22SourcesFromOpenDataHub() {
try {
const endpoint = process.env.CCTV_A22_URL || DEFAULT_A22_CCTV_URL;
const resp = await fetch(endpoint, {
headers: { Accept: 'application/json' },
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn('[CCTV] A22 catalog redirected; redirects are not followed');
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] A22 camera download failed:', resp.status);
return discard();
}
const payload = await readResponseJsonCapped(resp, A22_MAX_CATALOG_BYTES);
const items = Array.isArray(payload?.Items) ? payload.Items : [];
const cameras = [];
const seen = new Set();
for (const item of items) {
const camera = a22WebcamToSource(item);
if (!camera || seen.has(camera.id)) continue;
seen.add(camera.id);
cameras.push(camera);
}
// North to south along the motorway, so the picker reads in road order.
cameras.sort((a, b) => b.lat - a.lat);
console.log(`[CCTV] Loaded A22 camera sources: ${cameras.length}`);
return cameras;
} catch (error) {
console.warn('[CCTV] A22 camera download error:', error?.message || error);
return [];
}
}
/**
* Label for one Houston TranStar camera: the interchange an operator
* recognises ("10 EAST @ SAN JACINTO").
*
* @param {object} record - Raw TranStar camera row.
* @param {string} cameraId - Derived stable id.
* @returns {string}
*/
export function houstonCameraName(record, cameraId) {
const name = String(record?.location ?? '').trim();
if (name) return name;
const short = String(cameraId || '').replace(/^houston-/, '');
return short ? `Houston Camera ${short}` : 'Houston Camera';
}
/**
* TranStar `dir` (the camera's side of the freeway) -> the travel direction
* of the nearest lanes, as TranStar's map script `translateCameraDirection`
* reads it. Only these four values are mapped; anything else is unknown.
*/
const HOUSTON_TRAVEL_HEADING_BY_SIDE = Object.freeze({
south: 90,
north: 270,
west: 180,
east: 0,
});
/**
* One Houston TranStar camera row -> one catalog source, or null.
*
* Frame URLs are synthesized from a validated numeric id on the pinned
* snapshot origin. The payload's `url` is a `showWin(...)` viewer helper
* and is ignored so a catalog edit cannot steer the frame proxy off-host.
* TranStar's `dir` is the side of the freeway the camera sits on, not the
* way it faces: TranStar's own map labels it "Traffic closest to the camera
* is traveling …" (South→Eastbound, North→Westbound, West→Southbound,
* East→Northbound). That travel direction is only a low-confidence prior
* for the view, so the heading stays estimated with the low-confidence
* pose; `None`, blank or unknown values fall back to the shared id-hash
* heading.
*
* @param {object} record - Raw TranStar camera row.
* @returns {?object}
*/
export function houstonCameraToSource(record) {
if (!record || typeof record !== 'object') return null;
const lat = toFiniteNumber(record.lat);
const lon = toFiniteNumber(record.lng);
if (!isLikelyHoustonCoordinate(lat, lon)) return null;
const rawId = String(record.id ?? '').trim();
if (!HOUSTON_CAMERA_ID_RE.test(rawId)) return null;
const cameraId = `houston-${rawId}`;
const imageUrl = HOUSTON_IMAGE_URL(rawId);
const name = houstonCameraName(record, cameraId);
const travelHeading =
HOUSTON_TRAVEL_HEADING_BY_SIDE[
String(record.dir ?? '')
.trim()
.toLowerCase()
];
return {
id: cameraId,
name,
city: 'Houston',
cityId: 'houston',
provider: 'Houston TranStar',
lat,
lon,
headingDeg: Number.isFinite(travelHeading)
? travelHeading
: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
groundElevationM: HOUSTON_GROUND_ELEVATION_M,
feedType: 'image',
url: imageUrl,
snapshotUrl: imageUrl,
sourceKind: 'houston-transtar',
license:
'TranStar disclaimer — redistribution and external framing need TxDOT written permission',
code: cameraDisplayCode(name.toUpperCase()),
};
}
/**
* Fetch Houston TranStar cameras from the keyless `cctvSnapshots_json.js`
* catalog. Frames are stills on `www.houstontranstar.org`, registered from
* a validated numeric id rather than from the payload URL. The catalog host
* answers `Accept: application/json` with HTTP 406, so this request accepts
* any media type.
*
* @returns {Promise<Array<object>>} Normalized camera source objects.
*/
export async function loadHoustonSourcesFromOpenData() {
try {
const endpoint =
process.env.CCTV_HOUSTON_CAMERAS_URL || HOUSTON_CAMERAS_URL;
const resp = await fetch(endpoint, {
headers: { Accept: '*/*' },
redirect: 'manual',
signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS),
});
const discard = async () => {
try {
await resp.body?.cancel();
} catch {
/* no-op */
}
return [];
};
if (resp.status >= 300 && resp.status < 400) {
console.warn(
'[CCTV] Houston catalog redirected; redirects are not followed',
);
return discard();
}
if (!resp.ok) {
console.warn('[CCTV] Houston camera download failed:', resp.status);
return discard();
}
const payload = await readResponseJsonCapped(
resp,
HOUSTON_MAX_CATALOG_BYTES,
);
const rows = Array.isArray(payload?.cameras) ? payload.cameras : [];
const cameras = [];
const seen = new Set();
for (const record of rows) {
const camera = houstonCameraToSource(record);
if (!camera || seen.has(camera.id)) continue;
seen.add(camera.id);
cameras.push(camera);
}
const maxRaw = Number(
process.env.CCTV_HOUSTON_MAX_SOURCES || DEFAULT_HOUSTON_MAX_SOURCES,
);
const maxCount = Number.isFinite(maxRaw)
? Math.max(8, Math.min(400, Math.floor(maxRaw)))
: DEFAULT_HOUSTON_MAX_SOURCES;
const prioritized = prioritizeSources(cameras, maxCount, HOUSTON_ANCHORS);
console.log(
`[CCTV] Loaded Houston camera sources: ${cameras.length} (using nearest ${prioritized.length})`,
);
return prioritized;
} catch (error) {
console.warn(
'[CCTV] Houston camera download error:',
error?.message || error,
);
return [];
}
}
+4 -2
View File
@@ -6,7 +6,9 @@ export const HLS_LIMITS = Object.freeze({
leasesPerSession: 8,
leasesPerClient: 8,
playlistBytes: 256 * 1024,
segmentBytes: 4 * 1024 * 1024,
// High-resolution cameras can exceed 4 MiB in a single ten-second TS segment.
// Retained data still shares the independent 24 MiB session budget below.
segmentBytes: 8 * 1024 * 1024,
sessionBytes: 24 * 1024 * 1024,
segments: 12,
pollMs: 2000,
@@ -71,7 +73,7 @@ export async function fetchHlsBytes(
}
}
export function sameOriginHlsUrl(value, base) {
function sameOriginHlsUrl(value, base) {
const url = new URL(value, base);
if (
!['https:', 'http:'].includes(url.protocol) ||
+161 -14
View File
@@ -1,4 +1,7 @@
import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js';
import {
CCTV_SOURCE_FETCH_TIMEOUT_MS,
TAIWAN_HIGHWAY_IMAGE_ORIGINS,
} from './constants.js';
import {
cameraDisplayCode,
fallbackHeadingFromId,
@@ -10,16 +13,18 @@ import { readResponseTextCapped } from '../common/http.js';
* standard, published keyless and refreshed daily (~1,800 cameras). */
export const TAIWAN_FREEWAY_CCTV_URL =
'https://tisvcloud.freeway.gov.tw/history/motc20/CCTV.xml';
export const TAIWAN_HIGHWAY_CCTV_URL =
'https://cctv-maintain.thb.gov.tw/opendataCCTVs.xml';
/** The whole network is ~1.2 MB. */
export const TAIWAN_MAX_CATALOG_BYTES = 6 * 1024 * 1024;
export const DEFAULT_TAIWAN_MAX_SOURCES = 600;
const DEFAULT_TAIWAN_MAX_SOURCES = 600;
/** After a failed download, wait this long before asking again. The catalog
* host does not answer from some networks outside Taiwan, and every attempt
* there holds the shared catalog refresh until the connect timeout. */
export const TAIWAN_RETRY_AFTER_FAILURE_MS = 60 * 60 * 1000;
/** The table is republished daily; a good copy stands in for a failed
* download for up to this long. */
export const TAIWAN_LAST_GOOD_MAX_AGE_MS = 24 * 60 * 60 * 1000;
const TAIWAN_LAST_GOOD_MAX_AGE_MS = 24 * 60 * 60 * 1000;
// The table publishes each camera's MJPEG stream. The same camera servers
// answer a single JPEG for the same camera ID on a sibling path, which is what
@@ -35,6 +40,9 @@ const STREAM_TO_FRAME = new Map([
]);
const DIRECTIONS = { N: 'N', S: 'S', E: 'E', W: 'W' };
const TAIWAN_HIGHWAY_CAMERA_HOSTS = new Set(
TAIWAN_HIGHWAY_IMAGE_ORIGINS.map((origin) => new URL(origin).hostname),
);
/** Prioritization anchors, north to south, so a cap keeps the whole island
* rather than only greater Taipei. */
@@ -50,7 +58,7 @@ const ANCHORS = [
/** Taiwan main island with slack for the coast; the freeway network has no
* offshore-island cameras. */
export function isLikelyTaiwanCoordinate(lat, lon) {
function isLikelyTaiwanCoordinate(lat, lon) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
@@ -211,6 +219,74 @@ export function taiwanCameraToSource(row) {
};
}
/**
* One Highway Bureau row -> a source whose first MJPEG frame is served as a
* snapshot by the existing camera-frame proxy.
*
* @param {object} row - Output of parseTaiwanCctvXml.
* @returns {?object}
*/
export function taiwanHighwayCameraToSource(row) {
if (!row || typeof row !== 'object') return null;
const { lat, lon } = row;
if (!isLikelyTaiwanCoordinate(lat, lon)) return null;
let url;
try {
url = new URL(String(row.streamUrl ?? ''));
} catch {
return null;
}
if (
url.protocol !== 'https:' ||
url.username ||
url.password ||
url.port ||
url.search ||
url.hash ||
!TAIWAN_HIGHWAY_CAMERA_HOSTS.has(url.hostname) ||
!/^\/[A-Za-z0-9._+~-]{1,100}$/.test(url.pathname) ||
url.pathname.includes('..')
)
return null;
const rawId = String(row.cctvId ?? '').trim();
if (!/^[A-Za-z0-9-]{1,80}$/.test(rawId)) return null;
const cameraId = 'tw-thb-' + rawId.toLowerCase();
const road = String(row.roadName ?? '').trim();
const direction = DIRECTIONS[String(row.direction ?? '').trim()] || '';
const mile = String(row.mile ?? '').trim();
const section = [row.sectionStart, row.sectionEnd]
.map((part) => String(part ?? '').trim())
.filter(Boolean)
.join('→');
const name =
(road || mile || section
? [road, direction, mile, section].filter(Boolean).join(' ')
: '') || rawId;
return {
id: cameraId,
name: name.slice(0, 160),
city: 'Taiwan / 臺灣',
cityId: 'taiwan',
provider: 'Highway Bureau, MOTC',
lat,
lon,
headingDeg: fallbackHeadingFromId(cameraId),
headingConfidence: 'low',
pitchDeg: -18,
fovDeg: 44,
rangeM: 145,
mountHeightM: 8,
feedType: 'mjpeg',
url: url.href,
snapshotUrl: url.href,
sourceKind: 'taiwan-highway-motc',
license: 'Highway Bureau, MOTC open data',
code: cameraDisplayCode((road + ' ' + mile).trim().toUpperCase() || rawId),
};
}
let lastGood = null;
let lastGoodAt = 0;
let retryAfter = 0;
@@ -237,6 +313,49 @@ function taiwanSourceCap() {
: DEFAULT_TAIWAN_MAX_SOURCES;
}
/** Fetch the public provincial-road CCTV table as a no-key fallback. */
async function loadTaiwanHighwaySources() {
const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS);
const response = await fetch(TAIWAN_HIGHWAY_CCTV_URL, {
headers: { Accept: 'application/xml,text/xml' },
redirect: 'manual',
signal,
});
if (!response.ok) {
await response.body?.cancel().catch(() => {});
throw new Error(
response.status >= 300 && response.status < 400
? 'Highway Bureau catalog redirected; redirects are not followed'
: 'Highway Bureau catalog HTTP ' + response.status,
);
}
const xml = await readResponseTextCapped(
response,
TAIWAN_MAX_CATALOG_BYTES,
signal,
);
const cameras = new Map();
for (const row of parseTaiwanCctvXml(xml)) {
const camera = taiwanHighwayCameraToSource(row);
if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera);
}
if (!cameras.size)
throw new Error('no usable Highway Bureau cameras in the table');
const sources = prioritizeSources(
[...cameras.values()],
taiwanSourceCap(),
ANCHORS,
);
console.log(
'[CCTV] Loaded Taiwan Highway Bureau fallback: ' +
cameras.size +
' (using ' +
sources.length +
')',
);
return sources;
}
/**
* Fetch Taiwan Freeway Bureau cameras from the keyless MOTC 2.0 XML table.
* One bounded request per shared catalog refresh. A failed download serves the
@@ -262,7 +381,7 @@ export async function loadTaiwanSources() {
throw new Error(
response.status >= 300 && response.status < 400
? 'catalog redirected; redirects are not followed'
: `HTTP ${response.status}`,
: 'HTTP ' + response.status,
);
}
const xml = await readResponseTextCapped(
@@ -282,7 +401,11 @@ export async function loadTaiwanSources() {
ANCHORS,
);
console.log(
`[CCTV] Loaded Taiwan camera sources: ${cameras.size} (using ${sources.length})`,
'[CCTV] Loaded Taiwan camera sources: ' +
cameras.size +
' (using ' +
sources.length +
')',
);
lastGood = sources;
lastGoodAt = Date.now();
@@ -291,13 +414,37 @@ export async function loadTaiwanSources() {
} catch (error) {
retryAfter = Date.now() + TAIWAN_RETRY_AFTER_FAILURE_MS;
const fallback = fallbackSources();
console.warn(
`[CCTV] Taiwan camera download failed (${error?.message || error}); ` +
(fallback.length
? `serving the last good ${fallback.length} cameras`
: 'no Taiwan cameras') +
', next attempt in 1 h',
);
return fallback;
if (fallback.length) {
console.warn(
'[CCTV] Taiwan Freeway camera download failed (' +
(error?.message || error) +
'); serving the last good ' +
fallback.length +
' cameras, next attempt in 1 h',
);
return fallback;
}
try {
const sources = await loadTaiwanHighwaySources();
lastGood = sources;
lastGoodAt = Date.now();
console.warn(
'[CCTV] Taiwan Freeway camera download failed (' +
(error?.message || error) +
'); using ' +
sources.length +
' Highway Bureau cameras as fallback, next attempt in 1 h',
);
return sources;
} catch (fallbackError) {
console.warn(
'[CCTV] Taiwan Freeway camera download failed (' +
(error?.message || error) +
'); Highway Bureau fallback failed (' +
(fallbackError?.message || fallbackError) +
'); no Taiwan cameras, next attempt in 1 h',
);
return [];
}
}
}
+132
View File
@@ -0,0 +1,132 @@
import { createHash } from 'node:crypto';
import fs from 'node:fs';
import path from 'node:path';
import {
DEFAULT_WORLD_CCTV_SOURCE_FILE,
MLIT_RIVER_IMAGE_ORIGIN,
TRIPCHECK_IMAGE_ORIGIN,
WORLD_CCTV_SOURCE_COUNT,
} from './constants.js';
import { pinnedCameraUrl } from './normalize.js';
const SOURCE_POLICIES = Object.freeze({
'tripcheck-odot': {
idPrefix: 'world-or-',
origin: TRIPCHECK_IMAGE_ORIGIN,
bounds: [41, 47, -125, -116],
filename: /^\/roadcams\/cams\/[^/]+\.jpe?g$/i,
frameRefreshMs: 5 * 60 * 1000,
},
'mlit-river': {
idPrefix: 'world-jp-river-',
origin: MLIT_RIVER_IMAGE_ORIGIN,
bounds: [24, 46, 122, 154],
filename: /^\/cam\/now\/\d+\.jpg$/i,
// The provider asks clients to avoid frequent repeated image collection.
frameRefreshMs: 10 * 60 * 1000,
},
});
// MLIT currently serves this fixed JPEG on cameras whose images are paused
// for maintenance (verified against two Yamagata cameras on 2026-11-02).
const MLIT_ADJUSTMENT_IMAGE_SHA256 =
'dbd2c3790eb75edf9ca44162f7979f2a3799090e23f2c5142f63211a035cb191';
export function isMlitRiverAdjustmentImage(image) {
return (
image?.ok === true &&
Buffer.isBuffer(image.body) &&
createHash('sha256').update(image.body).digest('hex') ===
MLIT_ADJUSTMENT_IMAGE_SHA256
);
}
function safeFrameUrl(value, policy) {
const normalized = pinnedCameraUrl(value, policy.origin);
if (!normalized) return '';
const url = new URL(normalized);
return !url.search && !url.hash && policy.filename.test(url.pathname)
? url.href
: '';
}
function inBounds(lat, lon, [minLat, maxLat, minLon, maxLon]) {
return (
Number.isFinite(lat) &&
Number.isFinite(lon) &&
lat >= minLat &&
lat <= maxLat &&
lon >= minLon &&
lon <= maxLon
);
}
/**
* Load only the new image-backed networks from the add-on export. Existing
* Canadian networks, Québec entries, JMA placeholder images, and NZTA's
* scraped Journey Planner rows are deliberately excluded. Vegagerðin (Iceland)
* rows are excluded too: the Iceland pack loads the full live list of those
* cameras, so a World copy would draw each one twice.
*/
export function loadWorldCameraSources({ sourceRoot = process.cwd() } = {}) {
const filename = path.resolve(sourceRoot, DEFAULT_WORLD_CCTV_SOURCE_FILE);
let rows;
try {
const payload = JSON.parse(fs.readFileSync(filename, 'utf8'));
rows = Array.isArray(payload) ? payload : [];
} catch (error) {
console.warn('[CCTV] World camera catalog unavailable:', error?.message);
return [];
}
const seen = new Set();
const sources = [];
for (const row of rows) {
const sourceKind = String(row?.sourceKind || '');
const policy = SOURCE_POLICIES[sourceKind];
if (!policy) continue;
const id = String(row.id || '').trim();
const lat = Number(row.lat);
const lon = Number(row.lon);
const url = safeFrameUrl(row.url || row.snapshotUrl, policy);
const snapshotUrl = safeFrameUrl(row.snapshotUrl || row.url, policy);
if (
!id.startsWith(policy.idPrefix) ||
seen.has(id) ||
!inBounds(lat, lon, policy.bounds) ||
!url ||
!snapshotUrl ||
url !== snapshotUrl
)
continue;
seen.add(id);
sources.push({
id,
name: String(row.name || id)
.trim()
.slice(0, 180),
city: String(row.city || row.country || 'World'),
cityId: String(row.cityId || ''),
provider: String(row.provider || 'World traffic camera'),
lat,
lon,
headingDeg: Number.isFinite(Number(row.headingDeg))
? Number(row.headingDeg)
: undefined,
headingConfidence: String(row.headingConfidence || 'low'),
feedType: 'image',
sourceKind,
url,
snapshotUrl,
frameRefreshMs: policy.frameRefreshMs,
license: String(row.license || ''),
credit: String(row.credit || row.provider || ''),
});
}
console.log(
`[CCTV] Loaded World camera image sources: ${sources.length}/${WORLD_CCTV_SOURCE_COUNT}`,
);
return sources;
}