From ae661db7ab61a7ab913856c654a3731ee354d9e3 Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:38:00 +0200 Subject: [PATCH 1/7] =?UTF-8?q?feat(cctv):=20add=20Catalonia=20(Servei=20C?= =?UTF-8?q?atal=C3=A0=20de=20Tr=C3=A0nsit)=20as=20a=20keyless=20CCTV=20sou?= =?UTF-8?q?rce=20pack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One WFS/GML XML feed (~135 cameras) aggregating three publishers under a `font` field: SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras. Frame URLs are checked against a small host allowlist rather than one pinned origin, and every non-SCT camera carries its real owner as a `credit` shown beside the shared provider. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic), and ground elevation is primed per publisher rather than one flat prior. SCT's own frame link needed a workaround: RenderService always 302s to a second endpoint (TransitCamera) on a Location the upstream hardcodes as plain http even when requested over https — refused by fetchWithinHost's same-origin (scheme-included) redirect policy, by design (stops an upstream steering a host-pinned request into a plaintext downgrade). Separately, mct.gencat.cat's TLS setup fails the handshake with Node's fetch outright (ERR_SSL_WRONG_SIGNATURE_TYPE) even though other HTTP clients complete it. The loader builds the TransitCamera URL itself, over plain http, from RenderService's sctidcam id, instead of registering the broken upstream link — the only transport this host actually serves to this runtime. Verified against the live feed and the production fetch path (fetchCctvImageFromUpstream): all 135 cameras return real image bytes, 0 false positives. Ambient-card refresh cadence added to src/data/cctvLod.js (3 minutes), matching the upstream update rate measured empirically (no change inside 64s, changed within ~90-150s) — the same cadence already used for Caltrans/TfL/Ontario/TxDOT. CCTV_CATALONIA_MAX_SOURCES sets the cap and CCTV_CATALONIA_ENABLED=0 turns the pack off. No overlap with any existing pack (checked catalog.js and the git history for Spain/Catalonia/Barcelona coverage: none exists). Verified before this commit: format, check:boundaries, unit suite (3692 tests), and build all green. Co-Authored-By: Claude Sonnet 5 --- .env.example | 2 + CHANGELOG.md | 7 + DATA_SOURCES.md | 2 + docs/CURRENT-STATE.md | 2 +- server/providers/cctv/catalog.js | 6 + server/providers/cctv/constants.js | 41 ++++ server/providers/cctv/normalize.js | 30 +++ server/providers/cctv/sources.js | 203 +++++++++++++++++ src/data/cctvCataloniaSource.test.mjs | 317 ++++++++++++++++++++++++++ src/data/cctvDriveBcSource.test.mjs | 2 + src/data/cctvLod.js | 4 + src/data/cctvLod.test.mjs | 4 + src/data/dataCredits.js | 8 + 13 files changed, 627 insertions(+), 1 deletion(-) create mode 100644 src/data/cctvCataloniaSource.test.mjs diff --git a/.env.example b/.env.example index 5bc02df9c..5d2a268df 100644 --- a/.env.example +++ b/.env.example @@ -171,6 +171,8 @@ AISSTREAM_API_KEY= # CCTV_CALGARY_ENABLED=1 # set 0 to disable the Open Calgary (Alberta) cameras # CCTV_CALGARY_MAX_SOURCES=220 # cap cameras loaded from the Open Calgary catalog # CCTV_CALGARY_ROWS_URL= # override the Open Calgary catalog URL +# CCTV_CATALONIA_ENABLED=1 # set 0 to disable the Servei Català de Trànsit cameras +# CCTV_CATALONIA_MAX_SOURCES=150 # cap cameras loaded from Catalonia (~135 exist today) # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3d01413..0cfff159d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~135 + cameras from a WFS/GML feed that aggregates SCT's own highway cameras with + hotlinked Barcelona and Terrassa municipal cameras, each partner named as a + credit beside the shared provider. Ambient-card refresh paced to 3 minutes, + matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` + sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. + - Separate map-feature acquisition from annotation/search selection. Move road and mapped-installation decoding into source adapters while preserving geometry policy, cancellation, retry outcomes and compatibility exports. - Cancel hidden Nepal provider preloads on Stop, event disable and replacement, and respect drawing-tool pointer ownership for fallback evidence cards. Preserve @manjunath22466’s Nepal scene contribution and source attribution. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 850d27e39..02e7e4272 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -45,6 +45,7 @@ How to read this: | **Fintraffic / Digitraffic (weathercams)** | CCTV camera catalog + frames, Finland | [CC BY 4.0](https://www.digitraffic.fi/en/terms-of-service/) — attribution REQUIRED | "Fintraffic / digitraffic.fi, license CC BY 4.0" | | **DriveBC** (Government of British Columbia) | CCTV camera catalog + frames, British Columbia highways | [Open Government Licence – British Columbia](https://www2.gov.bc.ca/gov/content/data/open-data/open-government-licence-bc) — attribution REQUIRED | "DriveBC. Contains information licensed under the Open Government Licence – British Columbia" | | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | +| **Servei Català de Trànsit** (+ Ajuntament de Barcelona, Ajuntament de Terrassa) | CCTV camera catalog + frames, Catalonia | [Llicència oberta d'ús d'informació - Catalunya](https://web.gencat.cat/ca/generalitat/dades-indicadors/dades-obertes/llicencies) — attribution REQUIRED | "Servei Català de Trànsit"; partner-supplied cameras also name their owner | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | @@ -73,6 +74,7 @@ How to read this: - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. +- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `gencat.cat/transit/opendata/cameres.xml` WFS/GML feed. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin, and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 40e441b25..42ab3f07a 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2433,7 +2433,7 @@ its criteria cannot be silently ignored. | Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | | Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | | Traffic | OSM Overpass (+ optional TomTom live flow) | `src/data/traffic.js` | `/api/overpass` + `/api/tomtom` | viewport-driven | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Catalonia (Servei Català de Trànsit) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index a8519c85c..c681d321c 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -17,6 +17,7 @@ import { loadWarendorfSourcesFromCatalog, loadNswSourcesFromOpenData, loadCalgarySourcesFromOpenData, + loadCataloniaSourcesFromOpenData, } from './sources.js'; /** Env kill switch: unset or anything but "0" means enabled. */ @@ -86,6 +87,11 @@ const LIVE_PACKS = [ enabled: () => envEnabled('CCTV_CALGARY_ENABLED'), load: loadCalgarySourcesFromOpenData, }, + { + name: 'catalonia', + enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'), + load: loadCataloniaSourcesFromOpenData, + }, ]; /** * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index d66806e45..4dfcbbd15 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -236,6 +236,47 @@ export const CALGARY_DOWNTOWN = { lat: 51.0461, lon: -114.0626 }; * body cannot be buffered without limit. */ export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +/** + * Catalonia traffic cameras (Servei Català de Trànsit): one keyless WFS/GML + * XML list (verified 2026-09-15, hourly catalog refresh). The feed carries a + * `font` field naming the actual publisher per camera — SCT's own highway + * cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras are + * kept, each confirmed to return a real frame. Frame URLs are checked + * against a host allowlist rather than one pinned origin, since each + * publisher serves from its own host. + */ +export const CATALONIA_CAMERAS_URL = + 'http://www.gencat.cat/transit/opendata/cameres.xml'; +/** Allowed image hosts, one per kept `font` publisher in the feed. */ +export const CATALONIA_IMAGE_HOSTS = Object.freeze( + new Set([ + 'mct.gencat.cat', // SCT + 'www.bcn.cat', // IMI (Ajuntament de Barcelona) + 'emap.terrassa.cat', // Ajuntament de Terrassa + ]), +); +/** Per-`font` partner credit, shown beside the provider in the CCTV panel + * (same pattern as DriveBC's partner-supplied cameras). SCT is the feed's + * own publisher and carries no separate credit. */ +export const CATALONIA_FONT_CREDIT = Object.freeze({ + IMI: 'Ajuntament de Barcelona', + Terrassa: 'Ajuntament de Terrassa', +}); +/** Ground-elevation priors in metres, by `font`: the feed carries no + * elevation, and this pack spans sea level (Barcelona) to Terrassa's plateau + * (~270 m). The client's ground snap corrects these on 3D-tile stacks. */ +export const CATALONIA_FONT_ELEVATION_M = Object.freeze({ + SCT: 80, + IMI: 20, + Terrassa: 270, +}); +export const DEFAULT_CATALONIA_MAX_SOURCES = 150; +/** Prioritization anchors: Barcelona and Terrassa. */ +export const CATALONIA_ANCHORS = [ + { lat: 41.3874, lon: 2.1686 }, // Barcelona + { lat: 41.5636, lon: 2.0111 }, // Terrassa +]; + /** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; /** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index b564ef440..2469bee76 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -372,6 +372,36 @@ export function isLikelyFinlandCoordinate(lat, lon) { return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32; } +/** Catalonia bounding box (mainland is roughly 40.5..42.9 N, 0.15..3.4 E). */ +export function isLikelyCataloniaCoordinate(lat, lon) { + return ( + isPlausibleLatLon(lat, lon) && + lat >= 40.3 && + lat <= 42.9 && + lon >= 0.0 && + lon <= 3.4 + ); +} + +/** + * Decode decimal numeric HTML character references ("í" -> "í"), the + * entity form the Catalonia feed uses for accented place names, plus the + * handful of named entities XML/HTML text commonly carries. + * + * @param {string} text + * @returns {string} + */ +export function decodeNumericEntities(text) { + return String(text || '') + .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/</g, '<') + .replace(/>/g, '>') + .trim(); +} + /** * Human label for one Fintraffic preset (camera view). * diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 41471f5ba..3f73e70bc 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -55,6 +55,12 @@ import { DEFAULT_CALGARY_MAX_SOURCES, CALGARY_DOWNTOWN, CALGARY_MAX_CATALOG_BYTES, + CATALONIA_CAMERAS_URL, + CATALONIA_IMAGE_HOSTS, + CATALONIA_FONT_CREDIT, + CATALONIA_FONT_ELEVATION_M, + DEFAULT_CATALONIA_MAX_SOURCES, + CATALONIA_ANCHORS, CCTV_SOURCE_FETCH_TIMEOUT_MS, } from './constants.js'; import { @@ -73,6 +79,8 @@ import { isLikelyTexasCoordinate, isLikelyNswCoordinate, isLikelyCalgaryCoordinate, + isLikelyCataloniaCoordinate, + decodeNumericEntities, cameraDisplayCode, rowArrayToObject, prioritizeSources, @@ -1594,3 +1602,198 @@ export async function loadCalgarySourcesFromOpenData() { return []; } } + +/** + * Parse the Catalonia (`cameres.xml`) WFS/GML feed into raw feature records. + * Regex-based rather than a full XML parser — the feed is small (~160 + * features) and this avoids a new dependency for one field set per feature. + * + * @param {string} xml + * @returns {Array<{fid:string, lat:number, lon:number, carretera:string, municipi:string, pk:string, link:string, font:string}>} + */ +export function parseCataloniaXml(xml) { + const out = []; + const blockRe = /([\s\S]*?)<\/gml:featureMember>/g; + const tag = (block, name) => { + const match = new RegExp(`([^<]*)<\\/cite:${name}>`).exec( + block, + ); + return match ? decodeNumericEntities(match[1]) : ''; + }; + let match; + while ((match = blockRe.exec(String(xml || ''))) !== null) { + const block = match[1]; + const fidMatch = /]*>([^<]+)<\/gml:coordinates>/.exec( + block, + ); + if (!fidMatch || !coordsMatch) continue; + const [lonRaw, latRaw] = coordsMatch[1].split(','); + out.push({ + fid: fidMatch[1], + lon: toFiniteNumber(lonRaw), + lat: toFiniteNumber(latRaw), + carretera: tag(block, 'carretera'), + municipi: tag(block, 'municipi'), + pk: tag(block, 'pk'), + link: tag(block, 'link'), + font: tag(block, 'font'), + }); + } + return out; +} + +/** + * Validate and normalize one Catalonia feed `link` against the publisher + * host allowlist (CATALONIA_IMAGE_HOSTS), upgrading to https. Rejects + * anything else, including a host that merely resembles an allowed one. + * + * SCT's own `RenderService` frame link is a special case, for two stacked + * reasons (both verified 2026-09-15): + * 1. It always 302s to `TransitCamera`, on a Location the upstream + * hardcodes as plain http even when RenderService itself was requested + * over https. fetchWithinHost (media.js) only follows same-origin + * redirects — same scheme included, by design, to stop an upstream + * steering a host-pinned request into a plaintext downgrade — so that + * hop is refused and the camera would render as a placeholder. + * 2. mct.gencat.cat's TLS setup itself does not complete a handshake with + * Node's fetch/OpenSSL (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though it + * is reachable over https from other clients (e.g. curl) — an upstream + * server misconfiguration this proxy cannot route around. + * TransitCamera answers the frame directly over **http**, no redirect, and + * that is the only transport this host actually serves to this runtime — so + * this builds that URL itself from RenderService's `sctidcam` id instead of + * registering the (https, then-redirecting) RenderService link. + * + * @param {string} rawUrl + * @returns {string} Normalized URL, or '' if not accepted. + */ +export function normalizeCataloniaImageUrl(rawUrl) { + let parsed; + try { + parsed = new URL(String(rawUrl || '').trim()); + } catch { + return ''; + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return ''; + if (!CATALONIA_IMAGE_HOSTS.has(parsed.hostname)) return ''; + + if (parsed.hostname === 'mct.gencat.cat') { + const camId = parsed.searchParams.get('sctidcam'); + if (!camId) return ''; + const direct = new URL('http://mct.gencat.cat/mct2bo/TransitCamera'); + direct.searchParams.set('nom', camId); + direct.searchParams.set('visualitzacio', 'imatge'); + return direct.toString(); + } + + parsed.protocol = 'https:'; + return parsed.toString(); +} + +/** + * Human label for one Catalonia camera: " ()", + * skipping the parenthetical when the municipality is already named in the + * road/place text or when one half is missing. + * + * @param {string} carretera + * @param {string} municipi + * @returns {string} + */ +export function cataloniaCameraName(carretera, municipi) { + const road = String(carretera || '').trim(); + const city = String(municipi || '').trim(); + if (road && city) { + return road.toLowerCase().includes(city.toLowerCase()) + ? road + : `${road} (${city})`; + } + return road || city || 'Càmera de trànsit'; +} + +/** + * Fetch Catalonia traffic cameras (Servei Català de Trànsit), keyless: one + * WFS/GML XML list. The feed carries a `font` field naming the real + * publisher per camera; only SCT's own highway cameras and the hotlinked + * Barcelona (IMI) and Terrassa municipal cameras are kept, each confirmed to + * return a real frame — frame URLs are checked against a host allowlist + * (CATALONIA_IMAGE_HOSTS), so a row from any other publisher is dropped. + * Each non-SCT camera carries its publisher as a `credit` alongside the + * shared `provider`. The feed carries no heading, so every camera takes the + * id-hash fallback, low-confidence pose (same as TfL and Fintraffic). + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadCataloniaSourcesFromOpenData() { + try { + const resp = await fetch(CATALONIA_CAMERAS_URL, { + headers: { Accept: 'application/xml,text/xml,*/*' }, + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + if (!resp.ok) { + console.warn('[CCTV] Catalonia camera download failed:', resp.status); + return []; + } + const xml = await resp.text(); + const rows = parseCataloniaXml(xml); + if (!rows.length) return []; + + const cameras = []; + for (const row of rows) { + if (!row.fid) continue; + if (!isLikelyCataloniaCoordinate(row.lat, row.lon)) continue; + const imageUrl = normalizeCataloniaImageUrl(row.link); + if (!imageUrl) continue; + + const font = row.font.trim(); + const cameraId = `cat-${row.fid + .replace(/^cameres\.fid-/, '') + .replace(/[^a-zA-Z0-9]/g, '')}`; + + cameras.push({ + id: cameraId, + name: cataloniaCameraName(row.carretera, row.municipi), + city: row.municipi.trim() || 'Catalunya', + cityId: 'catalunya', + provider: 'Servei Català de Trànsit', + lat: row.lat, + lon: row.lon, + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + groundElevationM: CATALONIA_FONT_ELEVATION_M[font] ?? 80, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'catalonia-open-data', + license: + "Generalitat de Catalunya — Llicència oberta d'ús d'informació", + credit: CATALONIA_FONT_CREDIT[font] || '', + }); + } + + const unique = Array.from( + new Map(cameras.map((camera) => [camera.id, camera])).values(), + ); + const maxRaw = Number( + process.env.CCTV_CATALONIA_MAX_SOURCES || DEFAULT_CATALONIA_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(600, Math.floor(maxRaw))) + : DEFAULT_CATALONIA_MAX_SOURCES; + const prioritized = prioritizeSources(unique, maxCount, CATALONIA_ANCHORS); + console.log( + `[CCTV] Loaded Catalonia camera sources: ${unique.length} (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn( + '[CCTV] Catalonia camera download error:', + error?.message || error, + ); + return []; + } +} diff --git a/src/data/cctvCataloniaSource.test.mjs b/src/data/cctvCataloniaSource.test.mjs new file mode 100644 index 000000000..d9b9f3456 --- /dev/null +++ b/src/data/cctvCataloniaSource.test.mjs @@ -0,0 +1,317 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; +import { CATALONIA_CAMERAS_URL } from '../../server/providers/cctv/constants.js'; +import { + parseCataloniaXml, + normalizeCataloniaImageUrl, + cataloniaCameraName, + loadCataloniaSourcesFromOpenData, +} from '../../server/providers/cctv/sources.js'; + +/** Set (or, for `undefined`, delete) environment variables for one test. */ +function withEnv(t, env) { + for (const [name, value] of Object.entries(env)) { + const previous = process.env[name]; + t.after(() => { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + }); + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +} + +/** Silence the loaders' progress and failure logging. */ +function quiet(t) { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +} + +/** One `` block matching the real feed's shape. */ +function cataloniaFeature({ + fid, + lon, + lat, + carretera = '', + municipi = '', + pk = '', + link = '', + font = '', +}) { + const pkTag = pk ? `${pk}` : ''; + return ( + `` + + `` + + `${lon},${lat}` + + `` + + `${carretera}` + + `${municipi}${pkTag}` + + `${link}` + + `${font}` + + `` + ); +} + +function cataloniaFeed(features) { + return ( + '' + + features.join('') + + '' + ); +} + +test('parseCataloniaXml extracts every field from a feature block', () => { + const xml = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-52bf6eca_1a0a06bea1c_386f', + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + municipi: 'Nus Trinitat', + pk: '0.50', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]); + const rows = parseCataloniaXml(xml); + assert.equal(rows.length, 1); + assert.deepEqual(rows[0], { + fid: 'cameres.fid-52bf6eca_1a0a06bea1c_386f', + lon: 2.1849528, + lat: 41.45989301, + carretera: 'C-58', + municipi: 'Nus Trinitat', + pk: '0.50', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }); +}); + +test('normalizeCataloniaImageUrl rewrites SCT RenderService links to the direct http TransitCamera URL', () => { + // RenderService always 302s to a hardcoded http:// Location (which + // fetchWithinHost refuses as a scheme downgrade), and mct.gencat.cat's TLS + // setup fails the handshake with Node's fetch regardless — TransitCamera + // over plain http is the only transport that actually works here, so we + // build that URL from RenderService's sctidcam id instead of registering + // the redirector. + assert.equal( + normalizeCataloniaImageUrl( + 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + ), + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=nc87.gif&visualitzacio=imatge', + ); + assert.equal( + normalizeCataloniaImageUrl( + 'https://mct.gencat.cat/mct2bo/RenderService?sctidcam=sc64.gif', + ), + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=sc64.gif&visualitzacio=imatge', + 'an already-https RenderService link is rewritten the same way', + ); + assert.equal( + normalizeCataloniaImageUrl('https://mct.gencat.cat/mct2bo/RenderService'), + '', + 'a RenderService link with no sctidcam id is rejected', + ); +}); + +test('normalizeCataloniaImageUrl accepts the other registered hosts, upgraded to https', () => { + assert.equal( + normalizeCataloniaImageUrl('http://www.bcn.cat/transit/imatges/a.gif'), + 'https://www.bcn.cat/transit/imatges/a.gif', + ); + assert.equal( + normalizeCataloniaImageUrl( + 'https://emap.terrassa.cat/it_terrassa/cam02.jpeg', + ), + 'https://emap.terrassa.cat/it_terrassa/cam02.jpeg', + ); + assert.equal( + normalizeCataloniaImageUrl('https://evil.example.com/mct.gencat.cat.gif'), + '', + 'a look-alike host is rejected', + ); + assert.equal( + normalizeCataloniaImageUrl('https://unlisted-publisher.example/frame.gif'), + '', + 'any host outside the allowlist is rejected', + ); + assert.equal(normalizeCataloniaImageUrl(''), ''); + assert.equal(normalizeCataloniaImageUrl('not a url'), ''); +}); + +test('cataloniaCameraName joins road and municipality, skipping a redundant municipality', () => { + assert.equal( + cataloniaCameraName('C-58', 'Nus Trinitat'), + 'C-58 (Nus Trinitat)', + ); + assert.equal( + cataloniaCameraName('Plaça Urquinaona', 'Barcelona'), + 'Plaça Urquinaona (Barcelona)', + ); + assert.equal( + cataloniaCameraName('Barcelona Ronda Litoral', 'Barcelona'), + 'Barcelona Ronda Litoral', + ); + assert.equal(cataloniaCameraName('', ''), 'Càmera de trànsit'); + assert.equal(cataloniaCameraName('C-58', ''), 'C-58'); +}); + +test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publisher credit', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + const xml = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-1', + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + municipi: 'Nus Trinitat', + pk: '0.50', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: 'cameres.fid-2', + lon: '2.17228212', + lat: '41.38890733', + carretera: 'Plaça Urquinaona', + municipi: 'Barcelona', + link: 'http://www.bcn.cat/transit/imatges/PlUrquinaona.gif?a=1', + font: 'IMI', + }), + cataloniaFeature({ + fid: 'cameres.fid-3', + lon: '2.0111', + lat: '41.5636', + carretera: 'Cam02', + municipi: 'Terrassa', + link: 'https://emap.terrassa.cat/it_terrassa/cam02.jpeg?a=1', + font: 'Terrassa', + }), + // Rejected: link host not in the allowlist. + cataloniaFeature({ + fid: 'cameres.fid-4', + lon: '2.0', + lat: '41.5', + link: 'https://example.com/fake.gif', + font: 'SCT', + }), + // Rejected: outside the Catalonia bounding box. + cataloniaFeature({ + fid: 'cameres.fid-5', + lon: '-3.7', + lat: '40.4', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=x.gif', + font: 'SCT', + }), + ]); + return new Response(xml, { + status: 200, + headers: { 'Content-Type': 'application/xml' }, + }); + }); + + const cameras = await loadCataloniaSourcesFromOpenData(); + + assert.deepEqual(requested, [CATALONIA_CAMERAS_URL]); + assert.deepEqual( + cameras.map((camera) => camera.id).sort(), + ['cat-1', 'cat-2', 'cat-3'], + 'the off-allowlist link (fid-4) and the out-of-bbox row (fid-5) are excluded', + ); + + const sct = cameras.find((c) => c.id === 'cat-1'); + assert.equal(sct.name, 'C-58 (Nus Trinitat)'); + assert.equal(sct.provider, 'Servei Català de Trànsit'); + assert.equal(sct.credit, '', 'SCT is the publisher, no separate credit'); + assert.equal( + sct.url, + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=nc87.gif&visualitzacio=imatge', + 'registers the direct, non-redirecting http frame URL, not the RenderService redirector', + ); + assert.equal(sct.groundElevationM, 80); + assert.equal(sct.headingConfidence, 'low', 'the feed carries no heading'); + + const bcn = cameras.find((c) => c.id === 'cat-2'); + assert.equal(bcn.credit, 'Ajuntament de Barcelona'); + assert.equal(bcn.city, 'Barcelona'); + assert.equal(bcn.groundElevationM, 20); + + const terrassa = cameras.find((c) => c.id === 'cat-3'); + assert.equal(terrassa.credit, 'Ajuntament de Terrassa'); + assert.equal(terrassa.groundElevationM, 270); +}); + +test('Catalonia loader fails soft on HTTP errors, empty payloads and network errors', async (t) => { + quiet(t); + let respond; + t.mock.method(globalThis, 'fetch', (...args) => respond(...args)); + for (respond of [ + async () => new Response('unavailable', { status: 503 }), + async () => new Response(''), + async () => { + throw new TypeError('fetch failed'); + }, + ]) { + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + } +}); + +test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips the request', async (t) => { + quiet(t); + const sourceRoot = fs.mkdtempSync( + path.join(os.tmpdir(), 'gev-cctv-catalog-catalonia-'), + ); + t.after(() => fs.rmSync(sourceRoot, { recursive: true, force: true })); + withEnv(t, { + CCTV_SOURCES_FILE: undefined, + CCTV_SOURCES_JSON: undefined, + CCTV_FORCE_AUSTIN: undefined, + CCTV_PREFER_AUSTIN: undefined, + CCTV_MAX_SOURCES: undefined, + CCTV_CALTRANS_DISTRICTS: undefined, + CCTV_TFL_ENABLED: undefined, + CCTV_CATALONIA_ENABLED: undefined, + CCTV_CATALONIA_MAX_SOURCES: undefined, + }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + if (String(url) !== CATALONIA_CAMERAS_URL) { + return new Response('unavailable', { status: 503 }); + } + const xml = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-1', + lon: '2.1849528', + lat: '41.45989301', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]); + return new Response(xml, { status: 200 }); + }); + + const sources = await createCctvCatalog({ sourceRoot })(); + assert.deepEqual( + sources.map((source) => source.id), + ['cat-1'], + ); + + process.env.CCTV_CATALONIA_ENABLED = '0'; + requested.length = 0; + assert.deepEqual(await createCctvCatalog({ sourceRoot })(), []); + assert.ok(requested.length > 0, 'the other live packs still load'); + assert.equal( + requested.includes(CATALONIA_CAMERAS_URL), + false, + 'a disabled pack makes no request', + ); +}); diff --git a/src/data/cctvDriveBcSource.test.mjs b/src/data/cctvDriveBcSource.test.mjs index 617bb1350..a4934af3c 100644 --- a/src/data/cctvDriveBcSource.test.mjs +++ b/src/data/cctvDriveBcSource.test.mjs @@ -17,6 +17,7 @@ import { DEFAULT_TALLINN_MAX_SOURCES, DEFAULT_TARKTEE_MAX_SOURCES, DEFAULT_NSW_MAX_SOURCES, + DEFAULT_CATALONIA_MAX_SOURCES, DRIVEBC_WEBCAMS_URL, } from '../../server/providers/cctv/constants.js'; import { @@ -234,6 +235,7 @@ test('default per-pack camera caps fit inside the default catalog cap', () => { DEFAULT_TALLINN_MAX_SOURCES + DEFAULT_TARKTEE_MAX_SOURCES + DEFAULT_NSW_MAX_SOURCES + + DEFAULT_CATALONIA_MAX_SOURCES + 1; assert.ok( packs <= DEFAULT_CCTV_MAX_SOURCES, diff --git a/src/data/cctvLod.js b/src/data/cctvLod.js index 65ddc2e12..44b2e8ba3 100644 --- a/src/data/cctvLod.js +++ b/src/data/cctvLod.js @@ -38,6 +38,10 @@ const PROVIDER_STATIC_REFRESH_MS = Object.freeze({ // collectionInterval, 600 s for every station sampled; polling faster only // re-fetches the same JPEG. fintraffic: 10 * 60 * 1000, + // Measured empirically (2026-09-15): polling the same SCT/Terrassa frame + // every 20s showed no change inside 64s but a new frame by ~90-150s later — + // matches the other highway packs' 3-minute cadence. + 'servei català de trànsit': 3 * 60 * 1000, }); /** diff --git a/src/data/cctvLod.test.mjs b/src/data/cctvLod.test.mjs index c6f6e4923..4f5982496 100644 --- a/src/data/cctvLod.test.mjs +++ b/src/data/cctvLod.test.mjs @@ -641,6 +641,10 @@ test('staticFrameRefreshMs follows known pack cadences and bounds explicit value assert.equal(staticFrameRefreshMs({ provider: 'TxDOT' }), 180_000); // Digitraffic publishes one weathercam frame per 600 s collection interval. assert.equal(staticFrameRefreshMs({ provider: 'Fintraffic' }), 600_000); + assert.equal( + staticFrameRefreshMs({ provider: 'Servei Català de Trànsit' }), + 180_000, + ); assert.equal(staticFrameRefreshMs({ frameRefreshMs: 5_000 }), 60_000); assert.equal(staticFrameRefreshMs({ frameRefreshMs: 3_000_000 }), 1_200_000); assert.equal(staticFrameRefreshMs({ provider: 'Unknown Provider' }), 300_000); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 76029ace8..c44ac3ca6 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -169,6 +169,14 @@ export const DATA_CREDITS = [ 'Traffic cameras (Calgary): contains information licensed under the ' + 'Open Government Licence – City of Calgary', }, + { + key: 'catalonia-cctv', + html: + 'CCTV cameras & frames (Catalonia): ' + + 'Servei Català de Trànsit ' + + '(Llicència oberta d’ús d’informació - Catalunya); some cameras are supplied by partners ' + + '(Ajuntament de Barcelona, Ajuntament de Terrassa), each named in the CCTV panel', + }, { key: 'gbfs', html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)', From ffbe98b4ac9e269fea098f26eadd01d28a0b4333 Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:38:04 +0200 Subject: [PATCH 2/7] test(format-scope): register cctvCataloniaSource.test.mjs for format:check Mechanical addition per CONTRIBUTING.md, following the existing per-pack test-file entries (cctvCalgary.test.mjs, cctvNswSource.test.mjs, etc.). Co-Authored-By: Claude Sonnet 5 --- scripts/format-scope.json | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index b86632108..05fe8327a 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -489,6 +489,7 @@ "src/data/cctvEstonia.test.mjs", "src/data/cctvWarendorf.test.mjs", "src/data/cctvCalgary.test.mjs", + "src/data/cctvCataloniaSource.test.mjs", "src/data/cctvNswSource.test.mjs", "src/data/cctvTxdotSource.test.mjs", "src/layers/cctv/source.test.mjs", From 320a1d6bfd65b9cd96cad39f90e7aa48c93d2a67 Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:01:17 +0200 Subject: [PATCH 3/7] fix(cctv): harden the Catalonia catalog fetch and frame-link allowlist Address review feedback on #595: - Reject frame links carrying userinfo (user:pass@) in normalizeCataloniaImageUrl, so catalog data cannot inject credentials into the server-side frame request on an allowlisted host. - Fetch cameres.xml over https (verified: same 200 body, no redirect), refuse redirects with the unread body released, and read the body through readResponseTextCapped with a 1 MiB CATALONIA_MAX_CATALOG_BYTES cap instead of an unbounded resp.text(). - Add regression tests for userinfo links on every allowed host, the refused redirect, and declared and streamed oversize bodies. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- DATA_SOURCES.md | 2 +- server/providers/cctv/constants.js | 8 +- server/providers/cctv/sources.js | 38 +++++++- src/data/cctvCataloniaSource.test.mjs | 125 ++++++++++++++++++++++++-- 5 files changed, 163 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d48a26e..1ad4538e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -295,7 +295,9 @@ Analyst records for loaded satellites, datacenters and dams, with explicit bound hotlinked Barcelona and Terrassa municipal cameras, each partner named as a credit beside the shared provider. Ambient-card refresh paced to 3 minutes, matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` - sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. + sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. The catalog + is fetched over https with redirects refused and a 1 MiB body cap, and frame + links carrying userinfo are rejected even on an allowlisted host. - Distinguish PARTIAL vessel snapshots from STALE data in the layer panel, with accepted-record counts and unchanged retention, freshness and outage safeguards. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 824cb95be..5cc0be53d 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -90,7 +90,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `gencat.cat/transit/opendata/cameres.xml` WFS/GML feed. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin, and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. +- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo is rejected even on an allowed host), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index a64b06cd0..391c35230 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -238,7 +238,8 @@ export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024; /** * Catalonia traffic cameras (Servei Català de Trànsit): one keyless WFS/GML - * XML list (verified 2026-09-15, hourly catalog refresh). The feed carries a + * XML list (verified 2026-09-15, hourly catalog refresh), fetched over https + * (verified 2026-09-27: same 200 body, no redirect). The feed carries a * `font` field naming the actual publisher per camera — SCT's own highway * cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras are * kept, each confirmed to return a real frame. Frame URLs are checked @@ -246,7 +247,10 @@ export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024; * publisher serves from its own host. */ export const CATALONIA_CAMERAS_URL = - 'http://www.gencat.cat/transit/opendata/cameres.xml'; + 'https://www.gencat.cat/transit/opendata/cameres.xml'; +/** Byte ceiling for the Catalonia catalog read (~90 KB observed 2026-09-27), + * so a runaway body cannot be buffered without limit. */ +export const CATALONIA_MAX_CATALOG_BYTES = 1024 * 1024; /** Allowed image hosts, one per kept `font` publisher in the feed. */ export const CATALONIA_IMAGE_HOSTS = Object.freeze( new Set([ diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index f32b928a2..ee83f284e 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -56,6 +56,7 @@ import { CALGARY_DOWNTOWN, CALGARY_MAX_CATALOG_BYTES, CATALONIA_CAMERAS_URL, + CATALONIA_MAX_CATALOG_BYTES, CATALONIA_IMAGE_HOSTS, CATALONIA_FONT_CREDIT, CATALONIA_FONT_ELEVATION_M, @@ -89,7 +90,10 @@ import { prioritizeSources, } from './normalize.js'; import { directionToHeading } from '../../../src/data/directionText.js'; -import { readResponseJsonCapped } from '../common/http.js'; +import { + readResponseJsonCapped, + readResponseTextCapped, +} from '../common/http.js'; /** * Fetch and parse Austin traffic camera records from the city Open Data portal. * @@ -1649,7 +1653,10 @@ export function parseCataloniaXml(xml) { /** * Validate and normalize one Catalonia feed `link` against the publisher * host allowlist (CATALONIA_IMAGE_HOSTS), upgrading to https. Rejects - * anything else, including a host that merely resembles an allowed one. + * anything else, including a host that merely resembles an allowed one, and + * any URL carrying userinfo (`https://user:pass@www.bcn.cat/...`): the + * hostname check alone would pass it, letting catalog data inject + * credentials into the server-side frame request. * * SCT's own `RenderService` frame link is a special case, for two stacked * reasons (both verified 2026-09-15): @@ -1679,6 +1686,7 @@ export function normalizeCataloniaImageUrl(rawUrl) { return ''; } if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return ''; + if (parsed.username || parsed.password) return ''; if (!CATALONIA_IMAGE_HOSTS.has(parsed.hostname)) return ''; if (parsed.hostname === 'mct.gencat.cat') { @@ -1725,19 +1733,41 @@ export function cataloniaCameraName(carretera, municipi) { * shared `provider`. The feed carries no heading, so every camera takes the * id-hash fallback, low-confidence pose (same as TfL and Fintraffic). * + * The list decides ids, coordinates, labels and which allowlisted frame path + * is proxied, so it is fetched over https, redirects are refused (the list + * host cannot be steered or downgraded) and the body read is capped at + * CATALONIA_MAX_CATALOG_BYTES. + * * @returns {Promise>} Normalized camera source objects. */ export async function loadCataloniaSourcesFromOpenData() { try { const resp = await fetch(CATALONIA_CAMERAS_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, + redirect: 'manual', signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), }); + // A response this loader will not read still owns its transport until the + // body is released, so every rejection path cancels before returning. + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn( + '[CCTV] Catalonia catalog redirected; redirects are not followed', + ); + return discard(); + } if (!resp.ok) { console.warn('[CCTV] Catalonia camera download failed:', resp.status); - return []; + return discard(); } - const xml = await resp.text(); + const xml = await readResponseTextCapped(resp, CATALONIA_MAX_CATALOG_BYTES); const rows = parseCataloniaXml(xml); if (!rows.length) return []; diff --git a/src/data/cctvCataloniaSource.test.mjs b/src/data/cctvCataloniaSource.test.mjs index d9b9f3456..a2b1ed673 100644 --- a/src/data/cctvCataloniaSource.test.mjs +++ b/src/data/cctvCataloniaSource.test.mjs @@ -4,7 +4,10 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; -import { CATALONIA_CAMERAS_URL } from '../../server/providers/cctv/constants.js'; +import { + CATALONIA_CAMERAS_URL, + CATALONIA_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/constants.js'; import { parseCataloniaXml, normalizeCataloniaImageUrl, @@ -31,6 +34,24 @@ function quiet(t) { t.mock.method(console, 'warn', () => {}); } +/** + * A response whose body is a live stream, plus a flag that flips when the + * stream is cancelled. A rejection path that returns without cancelling holds + * the transport open, so the flag is what the refusal tests actually assert. + */ +const streamingResponse = (init = {}) => { + const state = { cancelled: false }; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('` block matching the real feed's shape. */ function cataloniaFeature({ fid, @@ -143,6 +164,19 @@ test('normalizeCataloniaImageUrl accepts the other registered hosts, upgraded to assert.equal(normalizeCataloniaImageUrl('not a url'), ''); }); +test('normalizeCataloniaImageUrl rejects credential-bearing URLs on every allowed host', () => { + // The hostname matches the allowlist, so only an explicit userinfo check + // stops catalog data injecting credentials into the frame request. + for (const link of [ + 'https://user:pass@www.bcn.cat/transit/imatges/a.gif', + 'http://user@www.bcn.cat/transit/imatges/a.gif', + 'https://:pass@emap.terrassa.cat/it_terrassa/cam02.jpeg', + 'http://user:pass@mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + ]) { + assert.equal(normalizeCataloniaImageUrl(link), '', link); + } +}); + test('cataloniaCameraName joins road and municipality, skipping a redundant municipality', () => { assert.equal( cataloniaCameraName('C-58', 'Nus Trinitat'), @@ -164,8 +198,8 @@ test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publis quiet(t); withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); const requested = []; - t.mock.method(globalThis, 'fetch', async (url) => { - requested.push(String(url)); + t.mock.method(globalThis, 'fetch', async (url, init) => { + requested.push([String(url), init.redirect]); const xml = cataloniaFeed([ cataloniaFeature({ fid: 'cameres.fid-1', @@ -220,7 +254,11 @@ test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publis const cameras = await loadCataloniaSourcesFromOpenData(); - assert.deepEqual(requested, [CATALONIA_CAMERAS_URL]); + assert.deepEqual(requested, [[CATALONIA_CAMERAS_URL, 'manual']]); + assert.ok( + CATALONIA_CAMERAS_URL.startsWith('https://'), + 'the catalog is fetched over https', + ); assert.deepEqual( cameras.map((camera) => camera.id).sort(), ['cat-1', 'cat-2', 'cat-3'], @@ -264,6 +302,81 @@ test('Catalonia loader fails soft on HTTP errors, empty payloads and network err } }); +test('the Catalonia catalog fetch refuses redirects and oversized bodies', async (t) => { + t.mock.method(console, 'warn', () => {}); + // A redirect is never followed: the list host cannot be steered or + // downgraded. Its body is a live stream, so the test also proves the + // refusal releases the transport. + const seen = []; + const redirected = streamingResponse({ + status: 302, + headers: { location: 'http://www.gencat.cat/transit/opendata/cameres.xml' }, + }); + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init.redirect]); + return redirected.response; + }); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + assert.deepEqual(seen, [[CATALONIA_CAMERAS_URL, 'manual']]); + assert.equal( + redirected.state.cancelled, + true, + 'the redirect body is cancelled', + ); + + // An HTTP error body is released too. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + const failed = streamingResponse({ status: 503 }); + t.mock.method(globalThis, 'fetch', async () => failed.response); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + assert.equal(failed.state.cancelled, true, 'the error body is cancelled'); + + const validFeed = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-1', + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]); + + // A body that declares a size over the cap is refused rather than buffered, + // even though its actual content would parse. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + return new Response(validFeed, { + headers: { + 'Content-Type': 'application/xml', + 'content-length': String(CATALONIA_MAX_CATALOG_BYTES + 1), + }, + }); + }); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + + // So is a body with no declared size that only turns out too long while + // streaming — a valid feed padded past the cap. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + const oversized = + validFeed + ' '.repeat(CATALONIA_MAX_CATALOG_BYTES + 1024); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(oversized)); + controller.close(); + }, + }), + { headers: { 'Content-Type': 'application/xml' } }, + ); + }); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); +}); + test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips the request', async (t) => { quiet(t); const sourceRoot = fs.mkdtempSync( @@ -282,8 +395,8 @@ test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips t CCTV_CATALONIA_MAX_SOURCES: undefined, }); const requested = []; - t.mock.method(globalThis, 'fetch', async (url) => { - requested.push(String(url)); + t.mock.method(globalThis, 'fetch', async (url, init) => { + requested.push([String(url), init.redirect]); if (String(url) !== CATALONIA_CAMERAS_URL) { return new Response('unavailable', { status: 503 }); } From 97492e28fe259802fea1eba885f80031a5aa8f06 Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:04:55 +0200 Subject: [PATCH 4/7] docs: list the Catalonia pack under Unreleased and in the README The changelog entry sat below the 0.2.0/0.2.1 release headings after those were cut; move it to [Unreleased] beside the Norway pack. Name Catalonia in the README's CCTV Mesh row the way the other packs are listed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 18 ++++++++--------- README.md | 56 ++++++++++++++++++++++++++-------------------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7862af2c8..55ebdb1bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,15 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. +- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~135 + cameras from a WFS/GML feed that aggregates SCT's own highway cameras with + hotlinked Barcelona and Terrassa municipal cameras, each partner named as a + credit beside the shared provider. Ambient-card refresh paced to 3 minutes, + matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` + sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. The catalog + is fetched over https with redirects refused and a 1 MiB body cap, and frame + links carrying userinfo are rejected even on an allowlisted host. + ## [0.2.1] — 2026-10-02 — God's Eye View through Your Agent (with security fixes and other features) - Ship each bundled data pack once. The region, marine, admin-boundary, @@ -648,15 +657,6 @@ Analyst records for loaded satellites, datacenters and dams, with explicit bound "~50k/day" free-tier figure in the proxy, `.env.example` and `DATA_SOURCES.md`. Still an application-side ceiling, not a billing cap. -- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~135 - cameras from a WFS/GML feed that aggregates SCT's own highway cameras with - hotlinked Barcelona and Terrassa municipal cameras, each partner named as a - credit beside the shared provider. Ambient-card refresh paced to 3 minutes, - matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` - sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. The catalog - is fetched over https with redirects refused and a 1 MiB body cap, and frame - links carrying userinfo are rejected even on an allowlisted host. - - Distinguish PARTIAL vessel snapshots from STALE data in the layer panel, with accepted-record counts and unchanged retention, freshness and outage safeguards. diff --git a/README.md b/README.md index 5d97dfb2f..bb191d9e7 100644 --- a/README.md +++ b/README.md @@ -291,28 +291,28 @@ _Ask for radio near anywhere and the globe starts broadcasting — every station Twenty layers and map sources. **Seventeen have a keyless path.** Some offer additional capabilities with a provider key. (🟢 no key · 🟡 free key · 🔴 metered.) -| Layer | What you get | Source | Auth | -| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------- | -| 🗺️ **Map Stack** | Esri satellite imagery, Google Photorealistic 3D, OSM, plus additional ion-hosted stacks | Esri / Google / Ion / OSM | 🟢 Esri satellite + OSM · 🟡 ion-hosted Google 3D + world terrain · 🔴 direct Google + place search | -| ✈️ **Live Flights** | 11,000+ live aircraft + route history | OpenSky + adsb.lol | 🟢 (🟡 optional for more polling credits) | -| 🎖️ **Military Flights** | ADS-B military traffic in amber | adsb.lol | 🟢 | -| 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | -| 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | -| 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | -| 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | -| 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | -| 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary first; Panoramax, KartaView and Google Street View planned | Mapillary | 🟡 Mapillary client token | -| 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | -| 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | -| 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | -| 🧭 **Directions** | Click A and B on the globe for a street-following drive, walk or cycle route draped on the terrain with turn-by-turn steps — then FLY the camera along it. No key, no geocoder, no mic | OSRM on FOSSGIS servers (OpenStreetMap) | 🟢 | -| 🔥 **Active Fires** | Live NASA FIRMS detections, trailing 24h | NASA FIRMS | 🟡 | -| 🚀 **Space Missions** | Rolling 30-day launches with payload, stage, and recovery detail | Launch Library 2 | 🟢 (🟡 optional token raises the allowance) | -| 🎖️ **Mapped Installations** | Viewport-bounded military-site context from community mapping — incomplete by nature, and labeled that way | OpenStreetMap | 🟢 | -| 🌬️ **Wind** | Animated 10 m forecast wind with an optional speed, temperature or pressure color field and a reading at map center | NOAA GFS / ECMWF IFS | 🟢 | -| 🌦️ **Observed Weather** | Rain radar (contiguous US), Satellite clouds (GOES regional or global infrared, Clouds only or Full image) and Lightning density (Americas and Pacific) on one history timeline | NOAA nowCOAST | 🟢 | -| 🌀 **Cyclones** | Advisory positions, forecast tracks and uncertainty cones for the Atlantic and eastern/central North Pacific | NOAA NHC / CPHC | 🟢 | +| Layer | What you get | Source | Auth | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------- | +| 🗺️ **Map Stack** | Esri satellite imagery, Google Photorealistic 3D, OSM, plus additional ion-hosted stacks | Esri / Google / Ion / OSM | 🟢 Esri satellite + OSM · 🟡 ion-hosted Google 3D + world terrain · 🔴 direct Google + place search | +| ✈️ **Live Flights** | 11,000+ live aircraft + route history | OpenSky + adsb.lol | 🟢 (🟡 optional for more polling credits) | +| 🎖️ **Military Flights** | ADS-B military traffic in amber | adsb.lol | 🟢 | +| 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | +| 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | +| 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | +| 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | +| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Catalonia (Servei Català de Trànsit). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | +| 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary first; Panoramax, KartaView and Google Street View planned | Mapillary | 🟡 Mapillary client token | +| 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | +| 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | +| 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | +| 🧭 **Directions** | Click A and B on the globe for a street-following drive, walk or cycle route draped on the terrain with turn-by-turn steps — then FLY the camera along it. No key, no geocoder, no mic | OSRM on FOSSGIS servers (OpenStreetMap) | 🟢 | +| 🔥 **Active Fires** | Live NASA FIRMS detections, trailing 24h | NASA FIRMS | 🟡 | +| 🚀 **Space Missions** | Rolling 30-day launches with payload, stage, and recovery detail | Launch Library 2 | 🟢 (🟡 optional token raises the allowance) | +| 🎖️ **Mapped Installations** | Viewport-bounded military-site context from community mapping — incomplete by nature, and labeled that way | OpenStreetMap | 🟢 | +| 🌬️ **Wind** | Animated 10 m forecast wind with an optional speed, temperature or pressure color field and a reading at map center | NOAA GFS / ECMWF IFS | 🟢 | +| 🌦️ **Observed Weather** | Rain radar (contiguous US), Satellite clouds (GOES regional or global infrared, Clouds only or Full image) and Lightning density (Americas and Pacific) on one history timeline | NOAA nowCOAST | 🟢 | +| 🌀 **Cyclones** | Advisory positions, forecast tracks and uncertainty cones for the Atlantic and eastern/central North Pacific | NOAA NHC / CPHC | 🟢 | **Weather, keyless.** **Wind** animates 10 m forecast flow from NOAA GFS or ECMWF IFS, with an optional color field and a reading at map center. **Rain radar**, **Satellite clouds** and **Lightning density** are observations on one history timeline in the **WEATHER** panel (step back, **Play**, **Latest**); **Cyclone advisories** draw NHC and CPHC positions, forecast tracks and cones. On Google 3D Tiles the observed layers float above the city as translucent shells, sharpened around your view. @@ -435,7 +435,7 @@ Seven keys. Five have a free tier, and the two 🔴 ones are metered: | --- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 🟡 | **Cesium ion** | 🗺️ Google Photorealistic 3D, world terrain, and additional ion-hosted imagery stacks. The free Community plan is for eligible individual, personal/non-commercial use and has quotas | [cesium.com/ion](https://cesium.com/ion) — use a public `assets:read` token and check current [pricing/eligibility](https://cesium.com/platform/cesium-ion/pricing/) | | 🔴 | **Google Maps** | Direct Google Photorealistic 3D + Google place search ([Map Tiles API](https://developers.google.com/maps/documentation/tile)) | [Google Cloud Console](https://console.cloud.google.com/) — URL-restrict it | -| 🔴 | **OpenAI** | 🎙️ Voice + AI HUD summary through an API key. Provider Settings also offers **USE CHATGPT OAUTH** for cloud voice without removing the API-key option | [platform.openai.com](https://platform.openai.com) — API-key usage is metered, see costs below | +| 🔴 | **OpenAI** | 🎙️ Voice + AI HUD summary through an API key. Provider Settings also offers **USE CHATGPT OAUTH** for cloud voice without removing the API-key option | [platform.openai.com](https://platform.openai.com) — API-key usage is metered, see costs below | | 🟡 | **AISStream** | 🚢 Live global ships | [aisstream.io](https://aisstream.io) — free signup | | 🟡 | **NASA FIRMS** | 🔥 Live active fires | [firms.modaps.eosdis.nasa.gov](https://firms.modaps.eosdis.nasa.gov/api/map_key/) — free | | 🟡 | **TomTom** | 🚦 Live flow speeds and congestion colors for the simulated traffic layer | [my.tomtom.com](https://my.tomtom.com/keys) — free tier available | @@ -509,11 +509,11 @@ RTMP-only sources are not supported by this integration. Honest numbers, roughly, as of mid-2026 — always check the provider pricing pages: -| | Cost reality | -| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **🟢 Most layers** | **$0, no signup.** OpenSky anon, USGS, CelesTrak, adsb.lol, city CCTV, Radio Browser, GBFS, Launch Library 2, transit feeds, OSRM routing, NOAA and ECMWF weather, OpenStreetMap ALPR mapping, bundled datasets. | -| **🟡 The free-key tier** | **$0 with a signup.** AISStream, FIRMS, TomTom, OpenSky, plus Cesium ion for eligible personal/non-commercial use. Provider quotas and eligibility still apply. | -| **🗺️ Google 3D tiles** | **Free through an eligible Cesium ion Community account within its quota; metered through a direct Google key.** Use the direct route for Google place search or commercial deployment, verify current provider terms, and set budget alerts where billing is enabled. | +| | Cost reality | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **🟢 Most layers** | **$0, no signup.** OpenSky anon, USGS, CelesTrak, adsb.lol, city CCTV, Radio Browser, GBFS, Launch Library 2, transit feeds, OSRM routing, NOAA and ECMWF weather, OpenStreetMap ALPR mapping, bundled datasets. | +| **🟡 The free-key tier** | **$0 with a signup.** AISStream, FIRMS, TomTom, OpenSky, plus Cesium ion for eligible personal/non-commercial use. Provider quotas and eligibility still apply. | +| **🗺️ Google 3D tiles** | **Free through an eligible Cesium ion Community account within its quota; metered through a direct Google key.** Use the direct route for Google place search or commercial deployment, verify current provider terms, and set budget alerts where billing is enabled. | | **🔴 OpenAI voice** | API-key sessions keep the live spend readout, STD/MINI model toggle, $2 warning, and **$5 hard cap**. ChatGPT OAuth is an explicit choice in Provider Settings. Its usage is tracked without an API-dollar estimate or spend cap. The voice context window stays deliberately short. | Google's direct 3D route is surprisingly generous: the first 1,000 Photorealistic From 492ecb484918bb5000a973e566ab71148e70e5be Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:07:54 +0200 Subject: [PATCH 5/7] docs(cctv): state that offline Catalonia cameras show the publisher's still The Catalonia feed has no camera status field, so an offline camera keeps its catalog entry and SCT or Barcelona answer its frame with their own "image unavailable" still (HTTP 200). Say so in DATA_SOURCES.md and stop the loader comment from implying every camera returns a live frame. Co-Authored-By: Claude Opus 5.5 --- DATA_SOURCES.md | 2 +- server/providers/cctv/sources.js | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index cdadba27e..1ec77f705 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -91,7 +91,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo is rejected even on an allowed host), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. +- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo is rejected even on an allowed host), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. The feed carries no camera status field, so a camera its publisher has taken offline still registers and shows the publisher's own "image unavailable" still (SCT and Barcelona both serve one with HTTP 200), the same as other packs without a status field. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 53017a2fe..d9a1c3ac3 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -1733,9 +1733,12 @@ export function cataloniaCameraName(carretera, municipi) { * Fetch Catalonia traffic cameras (Servei Català de Trànsit), keyless: one * WFS/GML XML list. The feed carries a `font` field naming the real * publisher per camera; only SCT's own highway cameras and the hotlinked - * Barcelona (IMI) and Terrassa municipal cameras are kept, each confirmed to - * return a real frame — frame URLs are checked against a host allowlist - * (CATALONIA_IMAGE_HOSTS), so a row from any other publisher is dropped. + * Barcelona (IMI) and Terrassa municipal cameras are kept — the publishers + * whose frame links were confirmed to serve images. Frame URLs are checked + * against a host allowlist (CATALONIA_IMAGE_HOSTS), so a row from any other + * publisher is dropped. The feed carries no camera status, so a camera its + * publisher has taken offline still registers and serves that publisher's + * own "image unavailable" still. * Each non-SCT camera carries its publisher as a `credit` alongside the * shared `provider`. The feed carries no heading, so every camera takes the * id-hash fallback, low-confidence pose (same as TfL and Fintraffic). From 3f239f11730f7c837fccd932d82f398100162013 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 15:21:34 -0700 Subject: [PATCH 6/7] fix(cctv): give Catalonia cameras stable ids and pin their frame paths Ids were built from the feed's fid, which embeds an upstream server-start stamp, so every upstream restart renamed all cameras (it already had since the PR fixture). Ids now come from the validated frame URL (cat-sct-, or host and path for partner cameras), which also collapses the five duplicate rows. Frame links must sit directly in each host's frame directory with no explicit port, and sctidcam must be a bare image filename. Numeric entity decoding handles hex references and leaves invalid code points as written, so one bad name no longer empties the pack. Counts in the docs are now ~130. --- .env.example | 2 +- CHANGELOG.md | 7 +- DATA_SOURCES.md | 2 +- README.md | 2 +- server/providers/cctv/constants.js | 15 +- server/providers/cctv/normalize.js | 16 +- server/providers/cctv/sources.js | 56 ++++-- src/data/cctvCataloniaSource.test.mjs | 265 +++++++++++++++++++++++++- 8 files changed, 333 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index be3e14c15..62056f0c6 100644 --- a/.env.example +++ b/.env.example @@ -220,7 +220,7 @@ AISSTREAM_API_KEY= # CCTV_CALGARY_MAX_SOURCES=220 # cap cameras loaded from the Open Calgary catalog # CCTV_CALGARY_ROWS_URL= # override the Open Calgary catalog URL # CCTV_CATALONIA_ENABLED=1 # set 0 to disable the Servei Català de Trànsit cameras -# CCTV_CATALONIA_MAX_SOURCES=150 # cap cameras loaded from Catalonia (~135 exist today) +# CCTV_CATALONIA_MAX_SOURCES=150 # cap cameras loaded from Catalonia (~130 exist today) # CCTV_VEGVESEN_ENABLED=1 # set 0 to disable the Statens vegvesen (Norway) road cameras # CCTV_VEGVESEN_MAX_SOURCES=900 # cap cameras loaded from Statens vegvesen (default loads all ~850 live; max 1000) # CCTV_VEGVESEN_VIDEO=1 # set 0 to show stills instead of live HLS for Statens vegvesen video cameras diff --git a/CHANGELOG.md b/CHANGELOG.md index b132d0794..3a9332ca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -351,14 +351,17 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. -- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~135 +- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~130 cameras from a WFS/GML feed that aggregates SCT's own highway cameras with hotlinked Barcelona and Terrassa municipal cameras, each partner named as a credit beside the shared provider. Ambient-card refresh paced to 3 minutes, matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. The catalog is fetched over https with redirects refused and a 1 MiB body cap, and frame - links carrying userinfo are rejected even on an allowlisted host. + links carrying userinfo or an explicit port, or outside the publisher's frame + directory, are rejected even on an allowlisted host. Camera ids come from + each frame's SCT camera file or image path rather than the feed's + restart-dependent feature id, and rows the feed repeats collapse to one. - Import your own GeoJSON, KML, KMZ and GPX files from DISPLAY or by dropping them onto the app. Show, hide, fly to or remove each file for the session. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index ae6709501..694ed9c19 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -143,7 +143,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo is rejected even on an allowed host), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. The feed carries no camera status field, so a camera its publisher has taken offline still registers and shows the publisher's own "image unavailable" still (SCT and Barcelona both serve one with HTTP 200), the same as other packs without a status field. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. +- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~130 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo or an explicit port, or pointing outside that host's frame directory, is rejected even on an allowed host; an SCT `sctidcam` must be a bare `.gif`/`.jpg` filename), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Camera ids are built from the SCT camera file or the partner image path, not the feed's `fid` (which embeds an upstream server-start stamp and changes on every upstream restart), and rows the feed lists more than once collapse to one camera. The feed carries no camera status field, so a camera its publisher has taken offline still registers and shows the publisher's own "image unavailable" still (SCT and Barcelona both serve one with HTTP 200), the same as other packs without a status field. Frames are fetched at request time and never stored or redistributed. The feed itself states no licence; the pack relies on the Generalitat de Catalunya open-data portal's terms for this dataset (the licence and attribution above). Disable with `CCTV_CATALONIA_ENABLED=0`. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. diff --git a/README.md b/README.md index c50b195f7..3f7dcb69e 100644 --- a/README.md +++ b/README.md @@ -377,7 +377,7 @@ Twenty layers and map sources. **Eighteen have a keyless path.** Some offer addi | 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | | ◍ **Air quality** | Live AQHI on the published 1–10+ health-risk scale, with a Weather-panel card showing the band legend and the highest readings in view. **Canada only** — ECCC is the first registered provider | ECCC MSC GeoMet | 🟢 | | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Catalonia (Servei Català de Trànsit), plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📹 **CCTV Mesh** | Up to 4,000 public cameras (the shared catalog cap) projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Catalonia (Servei Català de Trànsit), plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | | 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary, Google Street View, KartaView and Panoramax | Mapillary · Google · KartaView · Panoramax | 🟢 KartaView, Panoramax · 🟡 Mapillary token · 🔴 Google Maps key | | 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 02687a33c..ae339d5e2 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -251,14 +251,19 @@ export const CATALONIA_CAMERAS_URL = /** Byte ceiling for the Catalonia catalog read (~90 KB observed 2026-09-27), * so a runaway body cannot be buffered without limit. */ export const CATALONIA_MAX_CATALOG_BYTES = 1024 * 1024; -/** Allowed image hosts, one per kept `font` publisher in the feed. */ +/** Allowed image hosts, one per kept `font` publisher in the feed, each + * mapped to the one directory its frame links live in (every link in the + * 2026-10 feed shares it); a link must name a single file directly inside. */ export const CATALONIA_IMAGE_HOSTS = Object.freeze( - new Set([ - 'mct.gencat.cat', // SCT - 'www.bcn.cat', // IMI (Ajuntament de Barcelona) - 'emap.terrassa.cat', // Ajuntament de Terrassa + new Map([ + ['mct.gencat.cat', '/mct2bo/'], // SCT + ['www.bcn.cat', '/transit/imatges/'], // IMI (Ajuntament de Barcelona) + ['emap.terrassa.cat', '/it_terrassa/'], // Ajuntament de Terrassa ]), ); +/** SCT `sctidcam` values: a bare image filename, which also keys the camera + * id (the feed's `fid` embeds an upstream server-start stamp). */ +export const CATALONIA_SCT_CAMERA_FILE_RE = /^[a-z0-9]+\.(gif|jpe?g)$/i; /** Per-`font` partner credit, shown beside the provider in the CCTV panel * (same pattern as DriveBC's partner-supplied cameras). SCT is the feed's * own publisher and carries no separate credit. */ diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index 0bec05f39..3af4f74a9 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -406,16 +406,24 @@ export function isLikelyCataloniaCoordinate(lat, lon) { } /** - * Decode decimal numeric HTML character references ("í" -> "í"), the - * entity form the Catalonia feed uses for accented place names, plus the - * handful of named entities XML/HTML text commonly carries. + * Decode numeric HTML character references, decimal ("í") and hex + * ("í") alike, the entity form the Catalonia feed uses for accented + * place names, plus the handful of named entities XML/HTML text commonly + * carries. A reference that names no valid character (NUL, a surrogate, or + * anything past U+10FFFF) is left as written rather than thrown on, so one + * bad field cannot fail a whole catalog. * * @param {string} text * @returns {string} */ export function decodeNumericEntities(text) { return String(text || '') - .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) + .replace(/&#(?:x([0-9a-f]+)|(\d+));/gi, (match, hex, dec) => { + const code = hex ? Number.parseInt(hex, 16) : Number(dec); + const valid = + code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff); + return valid ? String.fromCodePoint(code) : match; + }) .replace(/&/g, '&') .replace(/"/g, '"') .replace(/'/g, "'") diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 51cef87a0..7587f4cbe 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -58,6 +58,7 @@ import { CATALONIA_CAMERAS_URL, CATALONIA_MAX_CATALOG_BYTES, CATALONIA_IMAGE_HOSTS, + CATALONIA_SCT_CAMERA_FILE_RE, CATALONIA_FONT_CREDIT, CATALONIA_FONT_ELEVATION_M, DEFAULT_CATALONIA_MAX_SOURCES, @@ -1668,10 +1669,11 @@ export function parseCataloniaXml(xml) { /** * Validate and normalize one Catalonia feed `link` against the publisher * host allowlist (CATALONIA_IMAGE_HOSTS), upgrading to https. Rejects - * anything else, including a host that merely resembles an allowed one, and - * any URL carrying userinfo (`https://user:pass@www.bcn.cat/...`): the - * hostname check alone would pass it, letting catalog data inject - * credentials into the server-side frame request. + * anything else, including a host that merely resembles an allowed one, any + * URL carrying userinfo (`https://user:pass@www.bcn.cat/...`) or an explicit + * port — the hostname check alone would pass either, letting catalog data + * inject credentials or steer the server-side frame request to another + * service on that host — and any path outside the host's frame directory. * * SCT's own `RenderService` frame link is a special case, for two stacked * reasons (both verified 2026-09-15): @@ -1688,7 +1690,8 @@ export function parseCataloniaXml(xml) { * TransitCamera answers the frame directly over **http**, no redirect, and * that is the only transport this host actually serves to this runtime — so * this builds that URL itself from RenderService's `sctidcam` id instead of - * registering the (https, then-redirecting) RenderService link. + * registering the (https, then-redirecting) RenderService link. The + * `sctidcam` id must be a bare image filename (CATALONIA_SCT_CAMERA_FILE_RE). * * @param {string} rawUrl * @returns {string} Normalized URL, or '' if not accepted. @@ -1701,12 +1704,16 @@ export function normalizeCataloniaImageUrl(rawUrl) { return ''; } if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return ''; - if (parsed.username || parsed.password) return ''; - if (!CATALONIA_IMAGE_HOSTS.has(parsed.hostname)) return ''; + if (parsed.username || parsed.password || parsed.port) return ''; + const dir = CATALONIA_IMAGE_HOSTS.get(parsed.hostname); + if (!dir || !parsed.pathname.startsWith(dir)) return ''; + const file = parsed.pathname.slice(dir.length); + if (!file || file.includes('/')) return ''; if (parsed.hostname === 'mct.gencat.cat') { const camId = parsed.searchParams.get('sctidcam'); - if (!camId) return ''; + if (file !== 'RenderService') return ''; + if (!camId || !CATALONIA_SCT_CAMERA_FILE_RE.test(camId)) return ''; const direct = new URL('http://mct.gencat.cat/mct2bo/TransitCamera'); direct.searchParams.set('nom', camId); direct.searchParams.set('visualitzacio', 'imatge'); @@ -1717,6 +1724,30 @@ export function normalizeCataloniaImageUrl(rawUrl) { return parsed.toString(); } +/** + * Stable camera id for one normalized Catalonia frame URL (the output of + * normalizeCataloniaImageUrl). The feed's `fid` is not used: it embeds an + * upstream server-start stamp and changes on every upstream restart. SCT + * cameras key on the `sctidcam` filename stem (`cat-sct-nc87`), partner + * cameras on the frame host and path (`cat-www-bcn-cat-transit-imatges-…`). + * + * @param {string} imageUrl - A URL normalizeCataloniaImageUrl accepted. + * @returns {string} + */ +export function cataloniaCameraId(imageUrl) { + const parsed = new URL(imageUrl); + const slug = (text) => + text + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + if (parsed.hostname === 'mct.gencat.cat') { + const file = parsed.searchParams.get('nom') || ''; + return `cat-sct-${slug(file.replace(/\.[^.]*$/, ''))}`; + } + return `cat-${slug(parsed.hostname)}-${slug(parsed.pathname)}`; +} + /** * Human label for one Catalonia camera: " ()", * skipping the parenthetical when the municipality is already named in the @@ -1751,6 +1782,10 @@ export function cataloniaCameraName(carretera, municipi) { * shared `provider`. The feed carries no heading, so every camera takes the * id-hash fallback, low-confidence pose (same as TfL and Fintraffic). * + * Ids come from the validated frame URL (cataloniaCameraId), not the feed's + * restart-dependent `fid`, and rows sharing one are collapsed before the cap + * (the live feed repeats a few SCT cameras). + * * The list decides ids, coordinates, labels and which allowlisted frame path * is proxied, so it is fetched over https, redirects are refused (the list * host cannot be steered or downgraded) and the body read is capped at @@ -1791,15 +1826,12 @@ export async function loadCataloniaSourcesFromOpenData() { const cameras = []; for (const row of rows) { - if (!row.fid) continue; if (!isLikelyCataloniaCoordinate(row.lat, row.lon)) continue; const imageUrl = normalizeCataloniaImageUrl(row.link); if (!imageUrl) continue; const font = row.font.trim(); - const cameraId = `cat-${row.fid - .replace(/^cameres\.fid-/, '') - .replace(/[^a-zA-Z0-9]/g, '')}`; + const cameraId = cataloniaCameraId(imageUrl); cameras.push({ id: cameraId, diff --git a/src/data/cctvCataloniaSource.test.mjs b/src/data/cctvCataloniaSource.test.mjs index a2b1ed673..be472784f 100644 --- a/src/data/cctvCataloniaSource.test.mjs +++ b/src/data/cctvCataloniaSource.test.mjs @@ -8,6 +8,7 @@ import { CATALONIA_CAMERAS_URL, CATALONIA_MAX_CATALOG_BYTES, } from '../../server/providers/cctv/constants.js'; +import { decodeNumericEntities } from '../../server/providers/cctv/normalize.js'; import { parseCataloniaXml, normalizeCataloniaImageUrl, @@ -177,6 +178,62 @@ test('normalizeCataloniaImageUrl rejects credential-bearing URLs on every allowe } }); +test("normalizeCataloniaImageUrl rejects explicit ports and paths outside each host's frame directory", () => { + // An allowlisted hostname alone would let catalog data aim the frame + // request at another service or path on that host. + for (const link of [ + 'http://www.bcn.cat:8080/transit/imatges/a.gif', + 'https://emap.terrassa.cat:8443/it_terrassa/cam02.jpeg', + 'http://mct.gencat.cat:8080/mct2bo/RenderService?sctidcam=nc87.gif', + 'http://www.bcn.cat/admin/a.gif', + 'http://www.bcn.cat/transit/imatges/', + 'http://www.bcn.cat/transit/imatges/sub/a.gif', + 'http://www.bcn.cat/transit/imatges/../../admin/a.gif', + 'https://emap.terrassa.cat/other/cam02.jpeg', + 'http://mct.gencat.cat/mct2bo/OtherService?sctidcam=nc87.gif', + ]) { + assert.equal(normalizeCataloniaImageUrl(link), '', link); + } +}); + +test('normalizeCataloniaImageUrl accepts only a bare image filename as the SCT sctidcam', () => { + for (const camId of [ + '../../etc/passwd', + '..%2F..%2Fetc%2Fpasswd', + 'nc87.png', + 'nc87', + 'nc87.gif/../x.gif', + 'nc87.gif&nom=other.gif', + 'nc-87.gif', + ]) { + const link = `http://mct.gencat.cat/mct2bo/RenderService?sctidcam=${encodeURIComponent(camId)}`; + assert.equal(normalizeCataloniaImageUrl(link), '', camId); + } + assert.equal( + normalizeCataloniaImageUrl( + 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=CB10.JPEG', + ), + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=CB10.JPEG&visualitzacio=imatge', + ); +}); + +test('decodeNumericEntities decodes decimal and hex references and leaves invalid ones as written', () => { + assert.equal(decodeNumericEntities('Lluís'), 'Lluís'); + assert.equal(decodeNumericEntities('Lluís 🚦'), 'Lluís 🚦'); + assert.equal(decodeNumericEntities('Lluís'), 'Lluís'); + for (const raw of [ + 'a�b', + 'a�b', + 'a�b', + 'a�b', + 'a�b', + 'a�b', + 'a�b', + ]) { + assert.equal(decodeNumericEntities(raw), raw, raw); + } +}); + test('cataloniaCameraName joins road and municipality, skipping a redundant municipality', () => { assert.equal( cataloniaCameraName('C-58', 'Nus Trinitat'), @@ -261,11 +318,15 @@ test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publis ); assert.deepEqual( cameras.map((camera) => camera.id).sort(), - ['cat-1', 'cat-2', 'cat-3'], + [ + 'cat-emap-terrassa-cat-it-terrassa-cam02-jpeg', + 'cat-sct-nc87', + 'cat-www-bcn-cat-transit-imatges-plurquinaona-gif', + ], 'the off-allowlist link (fid-4) and the out-of-bbox row (fid-5) are excluded', ); - const sct = cameras.find((c) => c.id === 'cat-1'); + const sct = cameras.find((c) => c.id === 'cat-sct-nc87'); assert.equal(sct.name, 'C-58 (Nus Trinitat)'); assert.equal(sct.provider, 'Servei Català de Trànsit'); assert.equal(sct.credit, '', 'SCT is the publisher, no separate credit'); @@ -277,12 +338,16 @@ test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publis assert.equal(sct.groundElevationM, 80); assert.equal(sct.headingConfidence, 'low', 'the feed carries no heading'); - const bcn = cameras.find((c) => c.id === 'cat-2'); + const bcn = cameras.find( + (c) => c.id === 'cat-www-bcn-cat-transit-imatges-plurquinaona-gif', + ); assert.equal(bcn.credit, 'Ajuntament de Barcelona'); assert.equal(bcn.city, 'Barcelona'); assert.equal(bcn.groundElevationM, 20); - const terrassa = cameras.find((c) => c.id === 'cat-3'); + const terrassa = cameras.find( + (c) => c.id === 'cat-emap-terrassa-cat-it-terrassa-cam02-jpeg', + ); assert.equal(terrassa.credit, 'Ajuntament de Terrassa'); assert.equal(terrassa.groundElevationM, 270); }); @@ -415,7 +480,7 @@ test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips t const sources = await createCctvCatalog({ sourceRoot })(); assert.deepEqual( sources.map((source) => source.id), - ['cat-1'], + ['cat-sct-nc87'], ); process.env.CCTV_CATALONIA_ENABLED = '0'; @@ -423,8 +488,196 @@ test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips t assert.deepEqual(await createCctvCatalog({ sourceRoot })(), []); assert.ok(requested.length > 0, 'the other live packs still load'); assert.equal( - requested.includes(CATALONIA_CAMERAS_URL), + requested.some(([url]) => url === CATALONIA_CAMERAS_URL), false, 'a disabled pack makes no request', ); }); + +/** The two SCT, one Barcelona and one Terrassa rows of a real feed snapshot, + * under one upstream run's `fid` prefix. */ +function snapshotFeed(fidPrefix) { + return cataloniaFeed([ + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_386f`, + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_b48`, + lon: '2.18', + lat: '41.46', + carretera: 'C-31', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc85.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_1c2`, + lon: '2.17228212', + lat: '41.38890733', + carretera: 'Plaça Urquinaona', + municipi: 'Barcelona', + link: 'http://www.bcn.cat/transit/imatges/PlUrquinaona.gif?a=1', + font: 'IMI', + }), + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_1c3`, + lon: '2.0111', + lat: '41.5636', + municipi: 'Terrassa', + link: 'https://emap.terrassa.cat/it_terrassa/cam01.jpeg?a=1 ', + font: 'Terrassa', + }), + ]); +} + +async function loadFromFeed(t, xml) { + t.mock.method(globalThis, 'fetch', async () => { + return new Response(xml, { + status: 200, + headers: { 'Content-Type': 'application/xml' }, + }); + }); + try { + return await loadCataloniaSourcesFromOpenData(); + } finally { + t.mock.restoreAll(); + } +} + +test('Catalonia camera ids survive an upstream restart that changes every fid', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + // The fid embeds the upstream server-start stamp: the PR fixture's + // 1a0a06bea1c run and a later 1a140311953 run name the same cameras. + const before = await loadFromFeed(t, snapshotFeed('52bf6eca_1a0a06bea1c')); + const after = await loadFromFeed(t, snapshotFeed('52bf6eca_1a140311953')); + const ids = (cameras) => cameras.map((camera) => camera.id).sort(); + assert.deepEqual(ids(before), [ + 'cat-emap-terrassa-cat-it-terrassa-cam01-jpeg', + 'cat-sct-nc85', + 'cat-sct-nc87', + 'cat-www-bcn-cat-transit-imatges-plurquinaona-gif', + ]); + assert.deepEqual(ids(after), ids(before)); +}); + +test('Catalonia loader collapses rows the feed repeats for one camera', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + // The live feed lists nc85, cb5, cb10, cb25 and cd17 twice each, under + // different fids. + const row = (fid, camId) => + cataloniaFeature({ + fid, + lon: '2.18', + lat: '41.46', + carretera: 'C-31', + link: `http://mct.gencat.cat/mct2bo/RenderService?sctidcam=${camId}`, + font: 'SCT', + }); + const cameras = await loadFromFeed( + t, + cataloniaFeed([ + row('cameres.fid-x_1', 'nc85.gif'), + row('cameres.fid-x_2', 'cb5.gif'), + row('cameres.fid-x_3', 'nc85.gif'), + row('cameres.fid-x_4', 'cb5.gif'), + row('cameres.fid-x_5', 'cb10.gif'), + ]), + ); + assert.deepEqual(cameras.map((camera) => camera.id).sort(), [ + 'cat-sct-cb10', + 'cat-sct-cb5', + 'cat-sct-nc85', + ]); +}); + +test('Catalonia loader drops ported and bad-sctidcam links but keeps the rest', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + const cameras = await loadFromFeed( + t, + cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-x_1', + lon: '2.17', + lat: '41.39', + link: 'http://www.bcn.cat:8080/transit/imatges/PlUrquinaona.gif?a=1', + font: 'IMI', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_2', + lon: '2.0111', + lat: '41.5636', + link: 'https://emap.terrassa.cat:8443/it_terrassa/cam02.jpeg?a=1', + font: 'Terrassa', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_3', + lon: '2.18', + lat: '41.46', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=../../etc/passwd', + font: 'SCT', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_4', + lon: '2.18', + lat: '41.46', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]), + ); + assert.deepEqual( + cameras.map((camera) => [camera.id, camera.url]), + [ + [ + 'cat-sct-nc87', + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=nc87.gif&visualitzacio=imatge', + ], + ], + ); +}); + +test('a malformed numeric entity in one row neither throws nor empties the Catalonia pack', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + const cameras = await loadFromFeed( + t, + cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-x_1', + lon: '2.18', + lat: '41.46', + carretera: 'C-31 �', + municipi: 'Bad�', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc85.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_2', + lon: '2.18', + lat: '41.46', + carretera: 'Plaça Catalunya', + municipi: 'Llinars del Vallès', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]), + ); + const byId = new Map(cameras.map((camera) => [camera.id, camera])); + assert.equal(cameras.length, 2); + assert.equal( + byId.get('cat-sct-nc85').name, + 'C-31 � (Bad�)', + 'an invalid reference is kept as written', + ); + assert.equal( + byId.get('cat-sct-nc87').name, + 'Plaça Catalunya (Llinars del Vallès)', + ); +}); From 248a658713431dbd2ab8a9d2245270960a44af44 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 15:17:20 -0700 Subject: [PATCH 7/7] fix(cctv): serve only live feeds on /media /api/cctv/media relayed still-image cameras with an ordinary fetch: it followed redirects to any host, passed through any content type and had no body cap. Stills are served by /api/cctv/frame, which keeps redirects on the registered origin, accepts only image/* and caps the body, and no client requests a still through /media. /media now answers 404 for anything but HLS, MP4/WebM and MJPEG, before any upstream request. --- CHANGELOG.md | 10 ++ docs/CURRENT-STATE.md | 3 + server/providers/cctv.js | 17 +- src/data/cctvMediaFeedType.test.mjs | 270 ++++++++++++++++++++++++++++ 4 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 src/data/cctvMediaFeedType.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a9332ca6..8d7d4fdc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +- Refuse still-image cameras on the CCTV media relay. `/api/cctv/media` is the + live-feed route, but it also relayed any registered still camera through an + ordinary fetch that followed redirects to any host, passed through whatever + content type came back and had no cap on the body, bypassing every protection + `/api/cctv/frame` applies to the same URL. Only HLS, MP4, WebM and MJPEG + feeds are relayed now; a still camera, or an unknown id, gets a JSON 404 + without an upstream request and is served only through `/api/cctv/frame`. + The client already loaded stills through `/frame`, so nothing it requests + changes. + - Stop one retired bikeshare feed from reporting the whole layer as failed. City feeds are fetched independently, but failure was recorded in a single flag that only a successful *activation* cleared — and an already-loaded city diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 4774b5c80..d1b4c3a3b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1162,6 +1162,9 @@ still waiting to drain reschedules that deadline, so a slow client does not read as a stalled upstream. Error responses are cancelled. Buffered snapshots have a 16 MiB streaming cap; an oversized image remains an upstream miss and uses the normal fallback chain. The existing declared media size ceiling remains 64 MiB. +`/api/cctv/media` relays live feeds (HLS, MP4, WebM, MJPEG) only; a still-image +camera is refused there with a JSON 404 and is served solely through +`/api/cctv/frame` and its same-origin, `image/*`-only, size-capped fetch. ## GBFS upstream bounds diff --git a/server/providers/cctv.js b/server/providers/cctv.js index 9cf3b0c47..b4c8673f1 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -28,7 +28,7 @@ export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream }; * GET /api/cctv/sources — list all registered camera sources * GET /api/cctv/health — per-camera health/status report * GET /api/cctv/stream/:id — stream info (feedType, URLs) for a camera - * GET /api/cctv/media/:id — proxy live video/image media from upstream + * GET /api/cctv/media/:id — proxy live video media from upstream * GET /api/cctv/frame/:id — single frame with fallback chain * * @returns {import('vite').Plugin} @@ -214,6 +214,21 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { const source = sourceById.get(cameraId); const mediaUrl = source?.url || ''; const feedType = normalizeFeedType(source?.feedType || 'image'); + // Live feeds only. A still camera is served by /frame, whose fetch + // keeps redirects on the registered origin, accepts only image/* and + // caps the body; relaying it here would bypass all three. + if (!isVideoFeedType(feedType) && feedType !== 'mjpeg') { + res.writeHead(404, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); + res.end( + JSON.stringify({ + error: 'No live media for this camera; use the frame endpoint', + }), + ); + return; + } const leaseId = url.searchParams.get('lease'); if (feedType === 'hls' && !/^[a-f0-9-]{36}$/i.test(leaseId || '')) { res.writeHead(400); diff --git a/src/data/cctvMediaFeedType.test.mjs b/src/data/cctvMediaFeedType.test.mjs new file mode 100644 index 000000000..5791e433b --- /dev/null +++ b/src/data/cctvMediaFeedType.test.mjs @@ -0,0 +1,270 @@ +// /api/cctv/media is the live-feed relay. A still camera used to be relayed +// through it as well, by an ordinary redirect-following fetch with no content +// type check and no body cap, which side-stepped everything /api/cctv/frame +// enforces for the same URL. These pin that a still is refused there and only +// reachable through /frame, while live feeds keep their relay. +// +// Run with: npm test (node --test) +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { Readable } from 'node:stream'; +import { cctvProxy } from '../../server/providers/cctv.js'; + +const STILL = { + id: 'still-fixture', + name: 'Still fixture', + lat: 30.27, + lon: -97.74, + feedType: 'image', + url: 'https://cams.upstream.invalid/still.jpg', +}; +// 'jpg' normalizes to 'image'; the alias must not reopen the relay. +const STILL_ALIAS = { + ...STILL, + id: 'still-alias', + feedType: 'jpg', + url: 'https://cams.upstream.invalid/alias.jpg', +}; +const VIDEO = { + id: 'video-fixture', + name: 'Video fixture', + lat: 30.27, + lon: -97.74, + feedType: 'video', + url: 'https://cams.upstream.invalid/live.mp4', +}; +const MJPEG = { + id: 'mjpeg-fixture', + name: 'MJPEG fixture', + lat: 30.27, + lon: -97.74, + feedType: 'mjpeg', + url: 'https://cams.upstream.invalid/live.mjpg', +}; + +/** A minimal upstream Response with a streaming body. */ +function upstreamResponse({ status = 200, headers = {}, body = 'payload' }) { + const store = new Map( + Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]), + ); + return { + ok: status >= 200 && status < 300, + status, + headers: { get: (name) => store.get(String(name).toLowerCase()) ?? null }, + body: Readable.toWeb(Readable.from([Buffer.from(String(body))])), + }; +} + +/** Collects what a handler wrote, the way a Node ServerResponse would. */ +function recordingResponse() { + const chunks = []; + let finished; + const done = new Promise((resolve) => { + finished = resolve; + }); + const res = { + statusCode: 0, + headers: {}, + writableEnded: false, + body: '', + writeHead(status, headers) { + res.statusCode = status; + res.headers = headers || {}; + }, + write(chunk) { + chunks.push(Buffer.from(chunk)); + return true; + }, + end(chunk) { + if (chunk) chunks.push(Buffer.from(chunk)); + res.writableEnded = true; + res.body = Buffer.concat(chunks).toString('utf8'); + finished(); + }, + on() {}, + once() {}, + emit() {}, + destroy() {}, + }; + return { res, done }; +} + +/** + * Mount the real provider over the fixture cameras and answer every upstream + * request from the supplied function. + */ +function mount(t, respond) { + const before = { + json: process.env.CCTV_SOURCES_JSON, + file: process.env.CCTV_SOURCES_FILE, + austin: process.env.CCTV_FORCE_AUSTIN, + serverKey: process.env.GOOGLE_MAPS_SERVER_API_KEY, + browserKey: process.env.GOOGLE_MAPS_API_KEY, + }; + process.env.CCTV_SOURCES_JSON = JSON.stringify([ + STILL, + STILL_ALIAS, + VIDEO, + MJPEG, + ]); + process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; + process.env.CCTV_FORCE_AUSTIN = '0'; + // No Street View key: a /frame miss lands on the synthetic billboard rather + // than a network call this file does not model. + delete process.env.GOOGLE_MAPS_SERVER_API_KEY; + delete process.env.GOOGLE_MAPS_API_KEY; + const nativeFetch = globalThis.fetch; + const requests = []; + globalThis.fetch = async (url, init = {}) => { + requests.push({ url: String(url), redirect: init.redirect }); + return respond({ url: String(url), init }); + }; + t.after(() => { + globalThis.fetch = nativeFetch; + for (const [name, value] of [ + ['CCTV_SOURCES_JSON', before.json], + ['CCTV_SOURCES_FILE', before.file], + ['CCTV_FORCE_AUSTIN', before.austin], + ['GOOGLE_MAPS_SERVER_API_KEY', before.serverKey], + ['GOOGLE_MAPS_API_KEY', before.browserKey], + ]) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + let handler = null; + cctvProxy().configureServer({ + middlewares: { + use: (_route, fn) => { + handler = fn; + }, + }, + }); + assert.ok(handler, 'the CCTV provider must mount a middleware'); + + const call = async (path) => { + const { res, done } = recordingResponse(); + await handler({ url: path, headers: {}, method: 'GET', on() {} }, res); + await done; + return res; + }; + const health = async (id) => { + const res = await call('/health'); + return JSON.parse(res.body).cameras.find((entry) => entry.id === id); + }; + return { call, health, requests }; +} + +test('a still camera is refused by /media without any upstream request', async (t) => { + // Everything the old relay would have done with this camera: redirect to a + // private address, then answer with HTML. + const app = mount(t, ({ url }) => + url.includes('169.254.169.254') + ? upstreamResponse({ + headers: { 'content-type': 'text/html' }, + body: 'metadata', + }) + : upstreamResponse({ + status: 302, + headers: { location: 'http://169.254.169.254/latest/meta-data/' }, + body: '', + }), + ); + for (const camera of [STILL, STILL_ALIAS]) { + const res = await app.call(`/media/${camera.id}`); + assert.equal(res.statusCode, 404, camera.id); + assert.equal(res.headers['Content-Type'], 'application/json'); + assert.deepEqual(JSON.parse(res.body), { + error: 'No live media for this camera; use the frame endpoint', + }); + assert.doesNotMatch(res.body, /metadata/); + // Being asked on the wrong route is not a camera fault. + assert.equal(await app.health(camera.id), undefined); + } + assert.deepEqual(app.requests, [], 'a still camera reached its upstream'); +}); + +test('an unregistered camera id is refused by /media the same way', async (t) => { + const app = mount(t, () => { + throw new Error('an unknown camera must not fetch'); + }); + const res = await app.call('/media/not-a-camera'); + assert.equal(res.statusCode, 404); + assert.equal(app.requests.length, 0); +}); + +test('a video camera is still relayed by /media', async (t) => { + const app = mount(t, () => + upstreamResponse({ + headers: { 'content-type': 'video/mp4', 'content-length': '5' }, + body: 'video', + }), + ); + const res = await app.call(`/media/${VIDEO.id}`); + assert.equal(res.statusCode, 200); + assert.equal(res.headers['Content-Type'], 'video/mp4'); + assert.equal(res.headers['X-CCTV-Source'], 'live-media'); + assert.equal(res.body, 'video'); + assert.deepEqual( + app.requests.map(({ url }) => url), + [VIDEO.url], + ); + assert.equal((await app.health(VIDEO.id)).status, 'ok'); +}); + +test('an MJPEG camera is still relayed by /media', async (t) => { + const app = mount(t, () => + upstreamResponse({ + headers: { 'content-type': 'multipart/x-mixed-replace; boundary=frame' }, + body: '--frame', + }), + ); + const res = await app.call(`/media/${MJPEG.id}`); + assert.equal(res.statusCode, 200); + assert.match(res.headers['Content-Type'], /^multipart\/x-mixed-replace/); + assert.equal(res.body, '--frame'); + assert.deepEqual( + app.requests.map(({ url }) => url), + [MJPEG.url], + ); +}); + +test('a still camera is still served through /frame with its bounded fetch', async (t) => { + const app = mount(t, () => + upstreamResponse({ + headers: { 'content-type': 'image/jpeg' }, + body: 'jpeg-bytes', + }), + ); + const res = await app.call(`/frame/${STILL.id}`); + assert.equal(res.statusCode, 200); + assert.equal(res.headers['Content-Type'], 'image/jpeg'); + assert.equal(res.headers['X-CCTV-Source'], 'upstream-image'); + assert.equal(res.body, 'jpeg-bytes'); + assert.deepEqual(app.requests, [{ url: STILL.url, redirect: 'manual' }]); +}); + +test('/frame does not follow a still camera off its origin', async (t) => { + // The protection /media used to bypass, shown on the route that keeps it. + const app = mount(t, ({ url }) => + url === STILL.url + ? upstreamResponse({ + status: 302, + headers: { location: 'http://169.254.169.254/latest/meta-data/' }, + body: '', + }) + : upstreamResponse({ + headers: { 'content-type': 'image/jpeg' }, + body: 'private', + }), + ); + const res = await app.call(`/frame/${STILL.id}`); + assert.equal(res.statusCode, 200); + assert.equal(res.headers['X-CCTV-Source'], 'synthetic'); + assert.doesNotMatch(res.body, /private/); + assert.deepEqual( + app.requests.map(({ url }) => url), + [STILL.url], + ); +});