diff --git a/.env.example b/.env.example index c89680b30..0b9a0037a 100644 --- a/.env.example +++ b/.env.example @@ -215,6 +215,9 @@ AISSTREAM_API_KEY= # CCTV_VEGVESEN_MAX_SOURCES=900 # cap cameras loaded from Statens vegvesen (default loads all ~850 live; max 1000) # CCTV_VEGVESEN_VIDEO=1 # set 0 to show stills instead of live HLS for Statens vegvesen video cameras # CCTV_VEGVESEN_URL= # override the Statens vegvesen catalog URL +# CCTV_NYC_ENABLED=1 # set 0 to disable the NYC DOT Traffic Management Center cameras +# CCTV_NYC_MAX_SOURCES=250 # cap cameras loaded from the NYC TMC catalog (nearest borough cores first) +# CCTV_NYC_CAMERAS_URL= # override the NYC TMC catalog URL # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/CHANGELOG.md b/CHANGELOG.md index b24ac95ed..0b12ee132 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +- Add New York City traffic cameras from the NYC DOT Traffic Management Center + as a CCTV pack. The camera list is one keyless JSON request to + `webcams.nyctmc.org/api/cameras`; frames are stills synthesized from each + camera UUID on that same host, so a catalog field cannot steer the proxy + off-host. Offline cameras are dropped. By default the 250 cameras nearest + the five borough cores load. `CCTV_NYC_ENABLED=0` turns the pack off and + `CCTV_NYC_MAX_SOURCES` changes the cap. The catalog-wide ceiling is + unchanged; extra cameras share it round-robin with the other packs. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 0b8eac358..dbb07f0bf 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -56,6 +56,7 @@ How to read this: | **DriveBC** (Government of British Columbia) | CCTV camera catalog + frames, British Columbia highways | [Open Government Licence – British Columbia](https://www2.gov.bc.ca/gov/content/data/open-data/open-government-licence-bc) — attribution REQUIRED | "DriveBC. Contains information licensed under the Open Government Licence – British Columbia" | | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | +| **NYC DOT Traffic Management Center** (`webcams.nyctmc.org`) | CCTV camera catalog + frames, New York City five boroughs | Public NYC DOT traffic camera stills | "NYC DOT Traffic Management Center" (courtesy) | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | @@ -91,6 +92,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. +- **NYC DOT Traffic Management Center cameras.** Keyless. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. By default the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap and `CCTV_NYC_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/README.md b/README.md index 5d97dfb2f..38f7a4913 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,7 @@ Twenty layers and map sources. **Seventeen have a keyless path.** Some offer add | 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | | 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📹 **CCTV Mesh** | ~4,100 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · New York City (NYC DOT TMC). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | | 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary first; Panoramax, KartaView and Google Street View planned | Mapillary | 🟡 Mapillary client token | | 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 26d56e66b..38bfe3a20 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -914,6 +914,21 @@ owners. Tuner calculations have a pure entry, with existing layer exports preserved. Destruction removes listeners and state subscriptions before ending tuning; a delayed Enable result cannot reveal or refocus removed controls. +## NYC DOT TMC CCTV source pack + +The CCTV catalog adds New York City traffic cameras from the NYC DOT Traffic +Management Center alongside the other live packs. The server reads the keyless +camera list at `https://webcams.nyctmc.org/api/cameras` and keeps cameras the +feed marks online. Frame URLs are built from each camera UUID under +`https://webcams.nyctmc.org/api/cameras//image`; they are never taken +from the payload. There is no heading in the feed, so gizmos use the shared +id-hash fallback at low confidence. + +By default the 250 cameras nearest the five borough cores load. +`CCTV_NYC_MAX_SOURCES` sets the pack cap (8–400) and `CCTV_NYC_ENABLED=0` +turns the pack off. The catalog-wide ceiling stays at 4,000 and is shared +round-robin, so adding this pack does not silently drop another region. + ## DriveBC CCTV source pack The CCTV catalog adds DriveBC highway cameras for British Columbia alongside the @@ -3058,7 +3073,7 @@ its criteria cannot be silently ignored. | Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | | Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | | Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + NYC DOT TMC + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | | Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 6b5c9b544..388028925 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -532,6 +532,7 @@ "src/data/cctvWarendorf.test.mjs", "src/data/cctvCalgary.test.mjs", "src/data/cctvVegvesen.test.mjs", + "src/data/cctvNyc.test.mjs", "src/data/cctvNswSource.test.mjs", "src/data/cctvTxdotSource.test.mjs", "src/layers/cctv/source.test.mjs", diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 9bc3b70ec..0939a71da 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -19,6 +19,7 @@ import { loadCalgarySourcesFromOpenData, loadDelDOTSourcesFromOpenData, loadVegvesenSourcesFromOpenData, + loadNycSourcesFromOpenData, } from './sources.js'; /** Env kill switch: unset or anything but "0" means enabled. */ @@ -98,6 +99,11 @@ const LIVE_PACKS = [ enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'), load: loadVegvesenSourcesFromOpenData, }, + { + name: 'nyc', + enabled: () => envEnabled('CCTV_NYC_ENABLED'), + load: loadNycSourcesFromOpenData, + }, ]; /** * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 6298e86a0..ae80904a5 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -267,6 +267,36 @@ export const NORWAY_ANCHORS = [ { lat: 69.6492, lon: 18.9553 }, // Tromsø ]; +/** NYC DOT Traffic Management Center cameras: one keyless JSON list for the + * five boroughs (~980 cameras). Frames are stills synthesized from each + * camera UUID on the city's webcam host — never taken from the payload. */ +export const NYC_CAMERAS_URL = 'https://webcams.nyctmc.org/api/cameras'; +/** The only origin NYC camera frames may come from. The catalog's `imageUrl` + * is ignored; the registered path is always this origin plus a validated + * UUID plus `/image`. */ +export const NYC_IMAGE_ORIGIN = 'https://webcams.nyctmc.org/api/cameras/'; +export const NYC_IMAGE_URL = (id) => `${NYC_IMAGE_ORIGIN}${id}/image`; +export const NYC_CAMERA_ID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +export const DEFAULT_NYC_MAX_SOURCES = 250; +/** Hard ceiling on the NYC catalog body. The live list is ~980 rows / ~250 KB; + * this only exists so an upstream that streams an unbounded body cannot be + * buffered without limit. */ +export const NYC_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +/** Ground-elevation prior in metres. Matches the New York CITY_POIS prior; + * the client's one-shot ground snap corrects it wherever 3D tiles load. */ +export const NYC_GROUND_ELEVATION_M = 10; +/** Prioritization anchors: one per borough so a cap keeps citywide coverage + * rather than Midtown alone. */ +export const NYC_ANCHORS = [ + { lat: 40.758, lon: -73.9855 }, // Manhattan (Times Square) + { lat: 40.7128, lon: -74.006 }, // Lower Manhattan + { lat: 40.6782, lon: -73.9442 }, // Brooklyn + { lat: 40.7282, lon: -73.7949 }, // Queens + { lat: 40.8448, lon: -73.8648 }, // Bronx + { lat: 40.5795, lon: -74.1502 }, // Staten Island +]; + /** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */ export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json'; export const DEFAULT_DELDOT_MAX_SOURCES = 300; diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index e62ba33f3..99b685f44 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -367,6 +367,17 @@ export function isLikelyCalgaryCoordinate(lat, lon) { ); } +/** New York City's five boroughs, with slack for the harbor and city line. */ +export function isLikelyNycCoordinate(lat, lon) { + return ( + isPlausibleLatLon(lat, lon) && + lat >= 40.45 && + lat <= 40.95 && + lon >= -74.3 && + lon <= -73.65 + ); +} + /** Mainland Norway, with slack for the coast and the Finnmark border. */ export function isLikelyNorwayCoordinate(lat, lon) { return ( diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 7ecd3434d..43ba30994 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -65,6 +65,13 @@ import { DEFAULT_VEGVESEN_MAX_SOURCES, VEGVESEN_MAX_CATALOG_BYTES, NORWAY_ANCHORS, + NYC_CAMERAS_URL, + NYC_IMAGE_URL, + NYC_CAMERA_ID_RE, + DEFAULT_NYC_MAX_SOURCES, + NYC_MAX_CATALOG_BYTES, + NYC_GROUND_ELEVATION_M, + NYC_ANCHORS, } from './constants.js'; import { toFiniteNumber, @@ -82,6 +89,7 @@ import { isLikelyTexasCoordinate, isLikelyNswCoordinate, isLikelyCalgaryCoordinate, + isLikelyNycCoordinate, isLikelyNorwayCoordinate, cameraDisplayCode, rowArrayToObject, @@ -1863,3 +1871,148 @@ export async function loadVegvesenSourcesFromOpenData() { return []; } } + +const NYC_BOROUGHS = new Set([ + 'Manhattan', + 'Brooklyn', + 'Queens', + 'Bronx', + 'Staten Island', +]); + +/** NYC TMC publishes `isOnline` as the strings "true"/"false". */ +export function nycCameraIsOnline(value) { + return ( + value === true || + String(value ?? '') + .trim() + .toLowerCase() === 'true' + ); +} + +/** + * Label for one NYC camera: the intersection an operator recognises + * ("Central Park West @ 86 St"). The catalog's `area` is the borough, not + * a facing, and is not part of the name. + * + * @param {object} record - Raw TMC camera row. + * @param {string} cameraId - Derived stable id. + * @returns {string} + */ +export function nycCameraName(record, cameraId) { + const name = String(record?.name ?? '').trim(); + if (name) return name; + const short = String(cameraId || '') + .replace(/^nyc-/, '') + .slice(0, 8); + return short ? `NYC Camera ${short}` : 'NYC Camera'; +} + +/** + * One NYC DOT TMC camera row -> one catalog source, or null. + * + * Frame URLs are synthesized from a validated UUID on the pinned TMC image + * origin. The payload's `imageUrl` is ignored so a catalog edit cannot steer + * the frame proxy off-host. There is no heading in the feed; headings use + * the shared id-hash fallback at low confidence, like Calgary and Fintraffic. + * + * @param {object} record - Raw TMC camera row. + * @returns {?object} + */ +export function nycCameraToSource(record) { + if (!record || typeof record !== 'object') return null; + if (!nycCameraIsOnline(record.isOnline)) return null; + + const lat = toFiniteNumber(record.latitude); + const lon = toFiniteNumber(record.longitude); + if (!isLikelyNycCoordinate(lat, lon)) return null; + + const uuid = String(record.id ?? '').trim(); + if (!NYC_CAMERA_ID_RE.test(uuid)) return null; + const cameraId = `nyc-${uuid.toLowerCase()}`; + const imageUrl = NYC_IMAGE_URL(uuid); + const name = nycCameraName(record, cameraId); + const area = String(record.area ?? '').trim(); + const city = NYC_BOROUGHS.has(area) ? area : 'New York'; + + return { + id: cameraId, + name, + city, + cityId: 'nyc', + provider: 'NYC DOT Traffic Management Center', + lat, + lon, + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + groundElevationM: NYC_GROUND_ELEVATION_M, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'nyc-dot-tmc', + license: 'Public NYC DOT Traffic Management Center traffic camera', + code: cameraDisplayCode(name.toUpperCase()), + }; +} + +/** + * Fetch NYC DOT Traffic Management Center cameras from the keyless + * `webcams.nyctmc.org` catalog. Frames are stills on that same host, + * registered from a validated UUID rather than from the payload URL. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadNycSourcesFromOpenData() { + try { + const endpoint = process.env.CCTV_NYC_CAMERAS_URL || NYC_CAMERAS_URL; + const resp = await fetch(endpoint, { + headers: { Accept: 'application/json' }, + redirect: 'manual', + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn('[CCTV] NYC catalog redirected; redirects are not followed'); + return discard(); + } + if (!resp.ok) { + console.warn('[CCTV] NYC camera download failed:', resp.status); + return discard(); + } + const rows = await readResponseJsonCapped(resp, NYC_MAX_CATALOG_BYTES); + if (!Array.isArray(rows)) return []; + const cameras = []; + const seen = new Set(); + for (const record of rows) { + const camera = nycCameraToSource(record); + if (!camera || seen.has(camera.id)) continue; + seen.add(camera.id); + cameras.push(camera); + } + const maxRaw = Number( + process.env.CCTV_NYC_MAX_SOURCES || DEFAULT_NYC_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(400, Math.floor(maxRaw))) + : DEFAULT_NYC_MAX_SOURCES; + const prioritized = prioritizeSources(cameras, maxCount, NYC_ANCHORS); + console.log( + `[CCTV] Loaded NYC camera sources: ${cameras.length} (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn('[CCTV] NYC camera download error:', error?.message || error); + return []; + } +} diff --git a/src/data/cctvNyc.test.mjs b/src/data/cctvNyc.test.mjs new file mode 100644 index 000000000..d67ffa002 --- /dev/null +++ b/src/data/cctvNyc.test.mjs @@ -0,0 +1,385 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + loadNycSourcesFromOpenData, + nycCameraIsOnline, + nycCameraName, + nycCameraToSource, +} from '../../server/providers/cctv/sources.js'; +import { + DEFAULT_CCTV_MAX_SOURCES, + DEFAULT_NYC_MAX_SOURCES, + NYC_CAMERAS_URL, + NYC_GROUND_ELEVATION_M, + NYC_IMAGE_ORIGIN, + NYC_IMAGE_URL, + NYC_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/constants.js'; +import { + CAMERA_CODE_MAX_CHARS, + isLikelyNycCoordinate, +} from '../../server/providers/cctv/normalize.js'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; + +/** + * A response whose body is a live stream, plus a flag that flips when the + * stream is cancelled. A rejection path that returns without cancelling holds + * the transport open, so the flag is what the refusal tests actually assert. + */ +const streamingResponse = (init = {}) => { + const state = { cancelled: false }; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('[')); + }, + cancel() { + state.cancelled = true; + }, + }); + return { response: new Response(body, init), state }; +}; + +const MIDTOWN_ID = '8a6bc417-4877-4ebe-8052-88c1b261baf1'; +const BROOKLYN_ID = '11111111-2222-3333-4444-555555555555'; + +/** One TMC camera row, shaped like the live `webcams.nyctmc.org` payload. */ +const row = (overrides = {}) => ({ + id: MIDTOWN_ID, + name: 'Central Park West @ 86 St', + latitude: 40.785302, + longitude: -73.969353, + area: 'Manhattan', + isOnline: 'true', + imageUrl: NYC_IMAGE_URL(MIDTOWN_ID), + ...overrides, +}); + +test('a NYC row maps to a source on the pinned HTTPS frame host', () => { + const source = nycCameraToSource(row()); + assert.equal(source.id, `nyc-${MIDTOWN_ID}`); + assert.equal(source.name, 'Central Park West @ 86 St'); + assert.equal(source.city, 'Manhattan'); + assert.equal(source.cityId, 'nyc'); + assert.equal(source.provider, 'NYC DOT Traffic Management Center'); + assert.equal(source.lat, 40.785302); + assert.equal(source.lon, -73.969353); + assert.equal(source.feedType, 'image'); + assert.equal(source.sourceKind, 'nyc-dot-tmc'); + assert.equal( + source.license, + 'Public NYC DOT Traffic Management Center traffic camera', + ); + assert.equal(source.groundElevationM, NYC_GROUND_ELEVATION_M); + assert.equal(source.url, NYC_IMAGE_URL(MIDTOWN_ID)); + assert.equal(source.url, source.snapshotUrl); + assert.ok(source.url.startsWith(NYC_IMAGE_ORIGIN)); + assert.equal(source.headingConfidence, 'low'); + assert.ok(Number.isFinite(source.headingDeg)); +}); + +test('frame URLs are synthesized from the UUID; the payload URL is ignored', () => { + const source = nycCameraToSource( + row({ imageUrl: 'https://evil.example/cameras/x/image' }), + ); + assert.equal(source.url, NYC_IMAGE_URL(MIDTOWN_ID)); + assert.equal(source.snapshotUrl, source.url); +}); + +test('offline cameras, off-city geometry and unusable ids are dropped', () => { + assert.equal(nycCameraToSource(row({ isOnline: 'false' })), null); + assert.equal(nycCameraToSource(row({ isOnline: false })), null); + assert.equal( + nycCameraToSource(row({ isOnline: 'TRUE' })).id, + `nyc-${MIDTOWN_ID}`, + ); + assert.equal( + nycCameraToSource(row({ isOnline: true })).id, + `nyc-${MIDTOWN_ID}`, + ); + // Austin: a plausible lat/lon, but not a New York camera. + assert.equal( + nycCameraToSource(row({ latitude: 30.2672, longitude: -97.7431 })), + null, + ); + assert.equal(nycCameraToSource(row({ latitude: 0, longitude: 0 })), null); + assert.equal(nycCameraToSource(row({ id: '../etc/passwd' })), null); + assert.equal(nycCameraToSource(row({ id: 'not-a-uuid' })), null); + assert.equal(nycCameraToSource(row({ id: `${MIDTOWN_ID}/../x` })), null); + assert.equal(nycCameraToSource(null), null); +}); + +test('ids lower-case the UUID and unknown boroughs fall back to New York', () => { + const mixed = nycCameraToSource( + row({ id: MIDTOWN_ID.toUpperCase(), area: 'Long Island' }), + ); + assert.equal(mixed.id, `nyc-${MIDTOWN_ID}`); + assert.equal(mixed.city, 'New York'); + assert.equal( + mixed.url, + NYC_IMAGE_URL(MIDTOWN_ID.toUpperCase()), + 'the registered frame path keeps the catalog UUID spelling', + ); +}); + +test('a nameless row still gets a label', () => { + assert.equal( + nycCameraName({ name: ' FDR Drive @ 23 St ' }, 'nyc-x'), + 'FDR Drive @ 23 St', + ); + assert.equal(nycCameraName({}, `nyc-${MIDTOWN_ID}`), 'NYC Camera 8a6bc417'); + assert.equal(nycCameraIsOnline('true'), true); + assert.equal(nycCameraIsOnline('false'), false); +}); + +test('the unselected label is the intersection, trimmed to the code width', () => { + assert.equal(nycCameraToSource(row()).code, 'CENTRAL PARK WEST @ 86 ST'); + const long = nycCameraToSource( + row({ + name: 'Brooklyn Queens Expressway @ Atlantic Avenue / Flatbush Avenue Extension', + }), + ); + assert.ok(long.code.length <= CAMERA_CODE_MAX_CHARS); + assert.ok(long.code.endsWith('…')); + assert.ok(long.code.startsWith('BROOKLYN')); +}); + +test('the loader reads the keyless catalog and collapses duplicate ids', async (t) => { + t.mock.method(console, 'log', () => {}); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url, init) => { + requested.push([String(url), init.redirect]); + return Response.json([ + row({ latitude: 40.758, longitude: -73.9855 }), + row({ + name: 'Central Park West @ 86 St (duplicate)', + latitude: 40.758, + longitude: -73.9855, + }), + row({ + id: BROOKLYN_ID, + name: 'Atlantic Ave @ Flatbush Ave', + latitude: 40.65, + longitude: -73.96, + area: 'Brooklyn', + imageUrl: NYC_IMAGE_URL(BROOKLYN_ID), + }), + row({ isOnline: 'false', name: 'Offline camera' }), + row({ id: 'https://evil.example/x' }), + ]); + }); + const cameras = await loadNycSourcesFromOpenData(); + assert.deepEqual(requested, [[NYC_CAMERAS_URL, 'manual']]); + assert.deepEqual( + cameras.map((camera) => camera.id), + // Nearest to a borough anchor first: the Times Square camera before Brooklyn. + [`nyc-${MIDTOWN_ID}`, `nyc-${BROOKLYN_ID}`], + ); + assert.equal(cameras[0].name, 'Central Park West @ 86 St'); + assert.equal(cameras[1].city, 'Brooklyn'); +}); + +test('an upstream failure yields an empty pack and releases the response', async (t) => { + t.mock.method(console, 'warn', () => {}); + const failed = streamingResponse({ status: 503 }); + t.mock.method(globalThis, 'fetch', async () => failed.response); + assert.deepEqual(await loadNycSourcesFromOpenData(), []); + assert.equal(failed.state.cancelled, true, 'the failed body is cancelled'); + + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + throw new Error('network down'); + }); + assert.deepEqual(await loadNycSourcesFromOpenData(), []); +}); + +test('the catalog fetch refuses redirects and oversized bodies', async (t) => { + t.mock.method(console, 'warn', () => {}); + const seen = []; + const redirected = streamingResponse({ + status: 302, + headers: { location: 'https://evil.example/cameras.json' }, + }); + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init.redirect]); + return redirected.response; + }); + assert.deepEqual(await loadNycSourcesFromOpenData(), []); + assert.deepEqual(seen, [[NYC_CAMERAS_URL, 'manual']]); + assert.equal( + redirected.state.cancelled, + true, + 'the redirect body is cancelled', + ); + + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + const body = JSON.stringify([row()]); + return new Response(body, { + headers: { + 'Content-Type': 'application/json', + 'content-length': String(NYC_MAX_CATALOG_BYTES + 1), + }, + }); + }); + assert.deepEqual(await loadNycSourcesFromOpenData(), []); + + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + const oversized = 'x'.repeat(NYC_MAX_CATALOG_BYTES + 1024); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(oversized)); + controller.close(); + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ); + }); + assert.deepEqual(await loadNycSourcesFromOpenData(), []); +}); + +test('a reduced pack cap keeps cameras nearest the borough anchors', async (t) => { + t.mock.method(console, 'log', () => {}); + const saved = process.env.CCTV_NYC_MAX_SOURCES; + t.after(() => { + if (saved === undefined) delete process.env.CCTV_NYC_MAX_SOURCES; + else process.env.CCTV_NYC_MAX_SOURCES = saved; + }); + process.env.CCTV_NYC_MAX_SOURCES = '8'; + const rows = [ + // Far southern Staten Island, listed first so only distance can keep them. + ...[40.5, 40.505, 40.51, 40.515, 40.52, 40.525].map((lat, i) => + row({ + id: `aaaaaaaa-bbbb-cccc-dddd-${String(i).padStart(12, '0')}`, + latitude: lat, + longitude: -74.24, + area: 'Staten Island', + name: `SI ${i}`, + }), + ), + ...[40.758, 40.757, 40.756, 40.755, 40.754, 40.753].map((lat, i) => + row({ + id: `bbbbbbbb-cccc-dddd-eeee-${String(i).padStart(12, '0')}`, + latitude: lat, + longitude: -73.9855, + area: 'Manhattan', + name: `Midtown ${i}`, + }), + ), + ]; + t.mock.method(globalThis, 'fetch', async () => Response.json(rows)); + const cameras = await loadNycSourcesFromOpenData(); + assert.equal(cameras.length, 8); + assert.deepEqual( + cameras.map((camera) => camera.city), + [ + 'Manhattan', + 'Manhattan', + 'Manhattan', + 'Manhattan', + 'Manhattan', + 'Manhattan', + 'Staten Island', + 'Staten Island', + ], + 'Midtown fills first; the Staten Island anchor still keeps a share', + ); +}); + +/** + * Serve the NYC catalog to the NYC endpoint and an empty payload to every + * other pack, so one catalog refresh exercises the registration without + * reaching the network. + */ +const runCatalogWithMockedUpstreams = async (t) => { + const requested = []; + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async (url) => { + const href = String(url); + requested.push(href); + if (href === NYC_CAMERAS_URL) { + return Response.json([ + row({ latitude: 40.758, longitude: -73.9855 }), + row({ + id: BROOKLYN_ID, + name: 'Atlantic Ave @ Flatbush Ave', + latitude: 40.65, + longitude: -73.96, + area: 'Brooklyn', + }), + ]); + } + return Response.json([]); + }); + const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); + return { requested, sources }; +}; + +test('the NYC lane is wired into the catalog and its loader runs', async (t) => { + const saved = { ...process.env }; + try { + delete process.env.CCTV_SOURCES_FILE; + delete process.env.CCTV_SOURCES_JSON; + delete process.env.CCTV_NYC_ENABLED; + const { requested, sources } = await runCatalogWithMockedUpstreams(t); + assert.ok( + requested.includes(NYC_CAMERAS_URL), + 'the catalog refresh invokes the NYC loader', + ); + assert.deepEqual( + sources.filter((s) => s.cityId === 'nyc').map((s) => s.id), + [`nyc-${MIDTOWN_ID}`, `nyc-${BROOKLYN_ID}`], + 'NYC cameras reach the served catalog through the registered lane', + ); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + } +}); + +test('CCTV_NYC_ENABLED=0 keeps the lane from being loaded at all', async (t) => { + const saved = { ...process.env }; + try { + delete process.env.CCTV_SOURCES_FILE; + delete process.env.CCTV_SOURCES_JSON; + process.env.CCTV_NYC_ENABLED = '0'; + const { requested, sources } = await runCatalogWithMockedUpstreams(t); + assert.equal( + requested.includes(NYC_CAMERAS_URL), + false, + 'the disabled lane never reaches its upstream', + ); + assert.deepEqual( + sources.filter((s) => s.cityId === 'nyc'), + [], + ); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + } +}); + +test('the shipped catalog ceiling is not raised to make room for this pack', () => { + assert.equal(DEFAULT_CCTV_MAX_SOURCES, 4000); + assert.equal(DEFAULT_NYC_MAX_SOURCES, 250); +}); + +test('isLikelyNycCoordinate spans the five boroughs and rejects the rest', () => { + assert.equal(isLikelyNycCoordinate(40.758, -73.9855), true); // Times Square + assert.equal(isLikelyNycCoordinate(40.5795, -74.1502), true); // Staten Island + assert.equal(isLikelyNycCoordinate(40.8448, -73.8648), true); // Bronx + assert.equal(isLikelyNycCoordinate(40.5077, -74.2304), true); // southern SI slack + assert.equal(isLikelyNycCoordinate(30.2672, -97.7431), false); // Austin + assert.equal(isLikelyNycCoordinate(0, 0), false); + assert.equal(isLikelyNycCoordinate(NaN, -73.98), false); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 2967f0a2b..891cd21e2 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -247,6 +247,12 @@ export const DATA_CREDITS = [ 'Norwegian licence for Open Government data (NLOD) ' + 'distributed by Statens vegvesen', }, + { + key: 'nyc-cctv', + html: + 'CCTV cameras & frames (New York City): ' + + 'NYC DOT Traffic Management Center', + }, { key: 'gbfs', html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)',