From 4887162d761fb47ffbd59eb799165a0b47c422d6 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 15:17:11 -0700 Subject: [PATCH] fix(cctv): harden the King County catalog fetch and dedupe before the cap The catalog fetch now refuses redirects, cancels rejected bodies and reads through the shared capped JSON reader. Duplicate ids are dropped before prioritization so they cannot use up the per-pack cap. Vendor-generic model strings (PTZ, PTZ IP, BOSCH/PTZ) are treated as no model. Comments and credits say why WSDOT rows are skipped (their frames are WSDOT-hosted, outside the pinned origin) and name Newcastle as the partner city that appears. --- DATA_SOURCES.md | 2 +- server/providers/cctv/constants.js | 6 +- server/providers/cctv/sources.js | 56 +++++++++++--- src/data/cctvKingCountySource.test.mjs | 102 +++++++++++++++++++++++-- src/data/dataCredits.js | 2 +- 5 files changed, 148 insertions(+), 20 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 1273ddcce..45de1c92a 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -143,7 +143,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped so a camera never appears twice when a statewide WSDOT pack is present; the few city-owned cameras the county hosts (Redmond, Sammamish, Newcastle, Snoqualmie) carry their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); both are carried on the source (and served by `/api/cctv/sources`) as `model` and `manufacturer` for identified-hardware consumers. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. +- **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped because their frames are WSDOT-hosted, outside the pinned image origin; partner-city cameras hosted on the county image handler (e.g. Newcastle) name their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); both are carried on the source (and served by `/api/cctv/sources`) as `model` and `manufacturer` for identified-hardware consumers (vendor-generic strings such as `PTZ` name no model and are omitted). Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 7fd6ef950..313a3338d 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -315,7 +315,7 @@ export const DELDOT_ANCHORS = [ * (Cohu 3950/3965, Bosch AUTODOME, Axis Q6135-LE, …), carried through as the * source's `model`. */ export const KING_COUNTY_CAMERAS_URL = - 'https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,Description,ImageURL,CamRegion,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson'; + 'https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,ImageURL,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson'; /** Frames come only from the county's own image handler host. */ export const KING_COUNTY_IMAGE_ORIGIN = 'https://info.kingcounty.gov/'; /** The whole catalog is ~125 rows (~59 live county cameras); no thinning by @@ -323,6 +323,10 @@ export const KING_COUNTY_IMAGE_ORIGIN = 'https://info.kingcounty.gov/'; export const DEFAULT_KING_COUNTY_MAX_SOURCES = 125; /** Prioritization anchor: Renton, roughly central to the county road network. */ export const KING_COUNTY_ANCHORS = [{ lat: 47.4829, lon: -122.2171 }]; +/** Hard ceiling on the King County catalog body. The whole layer is ~125 rows + * and well under 100 KB; this only exists so an upstream that streams an + * unbounded body cannot be buffered without limit. */ +export const KING_COUNTY_MAX_CATALOG_BYTES = 1024 * 1024; /** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 4955be563..84d694c70 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -63,6 +63,7 @@ import { KING_COUNTY_IMAGE_ORIGIN, DEFAULT_KING_COUNTY_MAX_SOURCES, KING_COUNTY_ANCHORS, + KING_COUNTY_MAX_CATALOG_BYTES, DEFAULT_VEGVESEN_CCTV_URL, VEGVESEN_IMAGE_ORIGIN, VEGVESEN_VIDEO_URL, @@ -1733,6 +1734,17 @@ function isLikelyKingCountyCoordinate(lat, lon) { ); } +/** + * A King County `Model` value, whitespace-collapsed, or '' when it is a + * vendor-generic string ("PTZ", "PTZ IP", "BOSCH/PTZ") that names no model. + */ +function kingCountyModel(raw) { + const model = String(raw || '') + .replace(/\s+/g, ' ') + .trim(); + return /^(?:[a-z]+\/)?ptz(?: ip)?$/i.test(model) ? '' : model; +} + /** * Fetch King County road cameras (Washington State). Keyless: one public * ArcGIS FeatureServer GeoJSON query (King County DOT Road Services), with an @@ -1740,16 +1752,17 @@ function isLikelyKingCountyCoordinate(lat, lon) { * never fetched. Only cameras that are live and active (`Live`, * `CurrentStatus`) with a positive integer id, finite in-county coordinates, * and a frame on the county's own image handler are kept. WSDOT-owned rows in - * the layer are skipped — the statewide WSDOT pack is their home — while the - * handful of city-owned cameras the county hosts (Redmond, Sammamish, - * Newcastle, Snoqualmie) keep their owner as the per-camera credit, + * the layer are skipped because their frames are WSDOT-hosted, outside the + * pinned image origin, while partner-city cameras hosted on the county image + * handler (e.g. Newcastle) name their owner as the per-camera credit, * DriveBC-style. The layer publishes no facing, so every heading is the id * hash at low confidence. * * Uniquely among the packs, the county publishes each camera's hardware * `Manufacturer`/`Model`; the model string (whitespace-collapsed — the feed * carries stray newlines) rides along as the source's `model` for - * identified-hardware consumers. + * identified-hardware consumers. Vendor-generic strings ("PTZ", "PTZ IP", + * "BOSCH/PTZ") name no model and are treated as absent. * * @returns {Promise>} Normalized camera source objects. */ @@ -1757,16 +1770,37 @@ export async function loadKingCountySourcesFromOpenData() { try { const resp = await fetch(KING_COUNTY_CAMERAS_URL, { headers: { Accept: 'application/json' }, + redirect: 'manual', signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), }); + // A response this loader will not read still owns its transport until the + // body is released, so every rejection path cancels before returning. + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn( + '[CCTV] King County catalog redirected; redirects are not followed', + ); + return discard(); + } if (!resp.ok) { console.warn('[CCTV] King County camera download failed:', resp.status); - return []; + return discard(); } - const payload = await resp.json(); + const payload = await readResponseJsonCapped( + resp, + KING_COUNTY_MAX_CATALOG_BYTES, + ); const rows = Array.isArray(payload?.features) ? payload.features : []; const cameras = []; + const seen = new Set(); for (const row of rows) { const attrs = row?.properties; const rawId = attrs?.AssetID; @@ -1774,8 +1808,8 @@ export async function loadKingCountySourcesFromOpenData() { if (attrs.Live !== 1) continue; if (String(attrs.CurrentStatus || '').trim() !== 'Active') continue; const owner = String(attrs.Owner || '').trim(); - // WSDOT-owned rows belong to the statewide WSDOT pack; skipping them - // here keeps one camera from appearing twice on the globe. + // WSDOT-owned rows serve WSDOT-hosted frames, outside the pinned + // county image origin, so they are skipped. if (owner === 'WSDOT') continue; // The handler is served over HTTPS; the feed still writes http://. const imageUrl = String(attrs.ImageURL || '') @@ -1788,9 +1822,9 @@ export async function loadKingCountySourcesFromOpenData() { if (!isLikelyKingCountyCoordinate(lat, lon)) continue; const cameraId = `kingcounty-${rawId}`; - const model = String(attrs.Model || '') - .replace(/\s+/g, ' ') - .trim(); + if (seen.has(cameraId)) continue; + seen.add(cameraId); + const model = kingCountyModel(attrs.Model); const manufacturer = String(attrs.Manufacturer || '') .replace(/\s+/g, ' ') .trim(); diff --git a/src/data/cctvKingCountySource.test.mjs b/src/data/cctvKingCountySource.test.mjs index a0f765446..5da0ae708 100644 --- a/src/data/cctvKingCountySource.test.mjs +++ b/src/data/cctvKingCountySource.test.mjs @@ -6,6 +6,7 @@ import path from 'node:path'; import { DEFAULT_KING_COUNTY_MAX_SOURCES, KING_COUNTY_CAMERAS_URL, + KING_COUNTY_MAX_CATALOG_BYTES, } from '../../server/providers/cctv/constants.js'; import { loadKingCountySourcesFromOpenData } from '../../server/providers/cctv/sources.js'; import { normalizeSourceItem } from '../../server/providers/cctv/normalize.js'; @@ -73,9 +74,9 @@ test('King County loader keeps live county cameras and upgrades frame URLs to HT }), // City-owned camera the county hosts: kept, owner becomes the credit. kingCountyFeature(90, 47.674, -122.119, { - properties: { Owner: 'City of Redmond', Model: null }, + properties: { Owner: 'City of Newcastle', Model: null }, }), - // WSDOT-owned row: the statewide WSDOT pack is its home. + // WSDOT-owned row: its frames are WSDOT-hosted, so it is skipped. kingCountyFeature(70, 47.6, -122.3, { properties: { Owner: 'WSDOT' } }), // Off-host frame, not live, retired, bad id, bad geometry, out of county. kingCountyFeature(71, 47.6, -122.3, { @@ -118,11 +119,11 @@ test('King County loader keeps live county cameras and upgrades frame URLs to HT assert.equal(novelty.fovDeg, 44); assert.ok(novelty.headingDeg >= 0 && novelty.headingDeg < 360); - const redmond = cameras.find((camera) => camera.id === 'kingcounty-90'); - assert.equal(redmond.credit, 'City of Redmond'); - assert.equal(redmond.model, undefined); + const newcastle = cameras.find((camera) => camera.id === 'kingcounty-90'); + assert.equal(newcastle.credit, 'City of Newcastle'); + assert.equal(newcastle.model, undefined); // A manufacturer without a model identifies nothing, so it is dropped too. - assert.equal(redmond.manufacturer, undefined); + assert.equal(newcastle.manufacturer, undefined); }); test('normalizeSourceItem passes a hardware model through and omits it otherwise', () => { @@ -275,3 +276,92 @@ test('King County loader defaults its cap and survives upstream failure', async }); assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); }); + +test('King County loader does not follow a redirected catalog', async (t) => { + quiet(t); + let init; + t.mock.method(globalThis, 'fetch', async (_url, options) => { + init = options; + return new Response( + JSON.stringify({ + type: 'FeatureCollection', + features: [kingCountyFeature(56, 47.70059, -122.02443)], + }), + { status: 302, headers: { Location: 'https://example.com/cams' } }, + ); + }); + + assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); + assert.equal(init.redirect, 'manual'); +}); + +test('King County loader rejects an oversized catalog', async (t) => { + quiet(t); + const features = [kingCountyFeature(56, 47.70059, -122.02443)]; + const body = JSON.stringify({ + type: 'FeatureCollection', + features, + padding: 'x'.repeat(KING_COUNTY_MAX_CATALOG_BYTES), + }); + t.mock.method(globalThis, 'fetch', async () => new Response(body)); + + assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); +}); + +test('King County loader drops duplicate ids before the per-pack cap', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: '8' }); + // Repeats of the camera nearest the anchor would fill an 8-camera cap + // before the second, farther camera if duplicates reached prioritization. + const features = []; + for (let i = 0; i < 10; i += 1) { + features.push(kingCountyFeature(56, 47.4829, -122.2171)); + } + features.push(kingCountyFeature(57, 47.70059, -122.02443)); + t.mock.method(globalThis, 'fetch', async () => + Response.json({ type: 'FeatureCollection', features }), + ); + + const cameras = await loadKingCountySourcesFromOpenData(); + assert.deepEqual(cameras.map((camera) => camera.id).sort(), [ + 'kingcounty-56', + 'kingcounty-57', + ]); +}); + +test('King County loader omits vendor-generic model strings', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: undefined }); + t.mock.method(globalThis, 'fetch', async () => + Response.json({ + type: 'FeatureCollection', + features: [ + kingCountyFeature(61, 47.5, -122.2, { + properties: { Model: 'PTZ', Manufacturer: 'Bosch' }, + }), + kingCountyFeature(62, 47.51, -122.2, { + properties: { Model: 'PTZ IP', Manufacturer: 'Cohu' }, + }), + kingCountyFeature(63, 47.52, -122.2, { + properties: { Model: 'BOSCH/PTZ', Manufacturer: 'Bosch' }, + }), + kingCountyFeature(64, 47.53, -122.2, { + properties: { Model: 'AUTODOME IP 5000', Manufacturer: 'Bosch' }, + }), + ], + }), + ); + + const byId = new Map( + (await loadKingCountySourcesFromOpenData()).map((camera) => [ + camera.id, + camera, + ]), + ); + for (const id of ['kingcounty-61', 'kingcounty-62', 'kingcounty-63']) { + assert.equal(byId.get(id).model, undefined, id); + assert.equal(byId.get(id).manufacturer, undefined, id); + } + assert.equal(byId.get('kingcounty-64').model, 'AUTODOME IP 5000'); + assert.equal(byId.get('kingcounty-64').manufacturer, 'Bosch'); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 267e1d6bd..d1bba9a85 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -261,7 +261,7 @@ export const DATA_CREDITS = [ 'kingcounty.gov, ' + 'per the ' + 'KCGIS Center terms. ' + - 'A few cameras are supplied by partner cities (Redmond, Sammamish, Newcastle, Snoqualmie); each names its owner in the CCTV panel.', + 'Partner-city cameras hosted on the county image handler (e.g. Newcastle) name their owner in the CCTV panel.', }, { key: 'vegvesen-cctv',