diff --git a/.env.example b/.env.example index f2fa1a98c..a78c4485b 100644 --- a/.env.example +++ b/.env.example @@ -229,6 +229,7 @@ AISSTREAM_API_KEY= # CCTV_VEGVESEN_ENABLED=1 # set 0 to disable the Statens vegvesen (Norway) road cameras # CCTV_VEGVESEN_MAX_SOURCES=900 # cap cameras loaded from Statens vegvesen (default loads all ~850 live; max 1000) # CCTV_VEGVESEN_VIDEO=1 # set 0 to show stills instead of live HLS for Statens vegvesen video cameras +# CCTV_VEGVESEN_HEIGHTS=1 # set 0 to skip Kartverket ground heights for Statens vegvesen cameras (flat prior instead) # CCTV_VEGVESEN_URL= # override the Statens vegvesen catalog URL # CCTV_NYC_ENABLED= # off by default; set 1 only if you hold a NYC DOT data-sharing agreement (tmcdot@dot.nyc.gov) for the camera feed # CCTV_NYC_MAX_SOURCES=250 # cap cameras loaded from the NYC TMC catalog (nearest borough cores first) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index f4da7f433..00967a88a 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -63,6 +63,7 @@ How to read this: | **King County Road Services** (`services.arcgis.com`, King County GIS) | CCTV camera catalog + frames, unincorporated King County WA road network | [KCGIS Center terms](https://kingcounty.gov/en/dept/kcit/data-information-services/gis-center/about/terms-conditions-copyrights) — attribution required; public ArcGIS layer | "King County Road Services" (partner-city cameras name their owner) | | **WSDOT** (`data.wsdot.wa.gov`) | CCTV camera catalog + frames, Washington State highways | [WSDOT travel information disclaimer](https://wsdot.wa.gov/about/policies/travel-information-disclaimer) — public "low volume" feed; catalog cached, frames on WSDOT's own image host only | "Washington State Department of Transportation (Travel Center)" (courtesy) | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | +| **Kartverket** (`ws.geonorge.no/hoydedata/v1/punkt`) | Terrain heights (NN2000) used as the ground prior for Statens vegvesen cameras | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) — attribution REQUIRED | "© Kartverket" | | **NYC DOT Traffic Management Center** (`webcams.nyctmc.org`) | CCTV camera catalog + frames, New York City five boroughs | NYC DOT requires developers to sign a [data-sharing agreement](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) for the camera feed. **Off by default**; enable only under your own agreement | "NYC DOT Traffic Management Center" + NYC DOT's required application disclaimer (in the Data attribution popover) | | **City of Toronto Open Data** (`opendata.toronto.ca`) | CCTV camera catalog + frames, Toronto city streets | [Open Government Licence – Toronto](https://www.toronto.ca/city-government/data-research-maps/open-data/open-data-licence/) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – Toronto" | | **GBFS (Lyft / BCycle / Social Bicycles / PBSC)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | @@ -166,6 +167,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped because their frames are WSDOT-hosted, outside the pinned image origin; partner-city cameras hosted on the county image handler (e.g. Newcastle) name their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); both are carried on the source (and served by `/api/cctv/sources`) as `model` and `manufacturer` for identified-hardware consumers (vendor-generic strings such as `PTZ` name no model and are omitted). Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. - **WSDOT highway cameras.** Keyless. The camera list is the Esri JSON layer `data.wsdot.wa.gov/travelcenter/Cameras.json` — the same layer WSDOT's own Travel Center map consumes (~1,700 point features, EPSG:3857 geometry converted to WGS84 on load; the body carries Windows-1252 bytes and is decoded as such). Only frames on WSDOT's own `images.wsdot.wa.gov` host are registered — the catalog also lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) whose own terms apply, and those are skipped rather than proxied. WSDOT's layer metadata marks the feed for "low volume" use (see WSDOT's [travel information disclaimer](https://wsdot.wa.gov/about/policies/travel-information-disclaimer)). The catalog is fetched once per CCTV catalog refresh (every 15 minutes), and a camera's frame is fetched only when that camera is viewed; frames are never stored or redistributed. The live feed's `CompassDirection` is a single cardinal (N/E/S/W), `B` (both directions — most rows), `O`, or null. A single cardinal gives a high-confidence heading (the diagonals NE/SE/SW/NW are mapped too should they appear); `B`, `O`, null, and any other code carry no single facing and use the shared id-hash fallback at low confidence. By default the 250 cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` changes the cap (8–1,200) and `CCTV_WSDOT_ENABLED=0` turns the pack off. A keyed WSDOT Traveler Information API serves the same population for higher-volume use and is deliberately not required here. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. +- **Kartverket ground heights for Statens vegvesen cameras.** The camera feed's points are 2D, so the loader asks Kartverket's keyless point-height API (`ws.geonorge.no/hoydedata/v1/punkt`, CC BY 4.0) for the terrain height at each camera. Requests carry 20 points: the API accepts 50, but above ~20 it answers the extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not at all. Eight requests run at a time, about 36 for the whole country: a cold lookup took 6–13 s in testing (single requests 0.6–9.5 s). Answers are matched by the coordinates the API echoes and recorded only for points that request asked about. Over water the API returns the sea-floor depth (`dybdekurver` or `Havflate`), which is replaced by 0 m, the water surface; for the ~14 bridge cameras that is below the deck, but much closer than the sea floor or the flat prior. Heights are cached in memory by position for the life of the server, and points with no height (a few ferry quays and bridges) for 6 h, so the 15-minute catalog refresh only asks about new or moved cameras. The first 429, 5xx or network failure stops the lookup, with no retries, and no new lookup starts until the backoff ends (Retry-After when sent, else 10 min; bounded to 1 min–1 h). The lane waits at most 12 s, counted from the start of the loader; a slower lookup keeps filling the cache, and the cameras it had not reached keep the flat prior until the next 15-minute refresh (about 190 of 865 on one slow first run). Only coordinates are sent, with the same User-Agent as the other CCTV requests. `CCTV_VEGVESEN_HEIGHTS=0` turns the lookup off. The Kartverket credit is shown beside the Statens vegvesen entry in the Data attribution popover. - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. diff --git a/changelog.d/945-cctv-vegvesen-kartverket-heights.md b/changelog.d/945-cctv-vegvesen-kartverket-heights.md new file mode 100644 index 000000000..c6c4b41d9 --- /dev/null +++ b/changelog.d/945-cctv-vegvesen-kartverket-heights.md @@ -0,0 +1,15 @@ +- Give each Statens vegvesen camera its own ground height from Kartverket's + keyless point-height API (CC BY 4.0) instead of one flat 150 m prior, so + mountain-pass cameras no longer sit hundreds of metres below the terrain + where neither the point-height prior nor the ground snap resolves. About 850 + of ~855 cameras get a height; the rest keep the old prior. Requests carry 20 + points, the most the API answers from its 1 m terrain model, eight at a + time; a cold lookup takes about 6–13 s. Cameras over water use the water + surface rather than the sea-floor depth the API returns there; for bridge + cameras that is still below the deck. Heights are cached for good and misses + for 6 h, so a refresh only asks about new or moved cameras. After a 429, a + 5xx or a network failure the lookup stops and backs off (Retry-After, else + 10 min). The lane waits at most 12 s from its start; a slower first lookup + leaves the cameras it had not reached on the flat prior until the next + 15-minute refresh, and keeps filling the cache meanwhile. + `CCTV_VEGVESEN_HEIGHTS=0` turns it off. Follow-up to #929 (trygvehhh, #945). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index dfd14c959..dba0518fc 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3407,7 +3407,7 @@ its criteria cannot be silently ignored. | Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | | Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | | Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + King County (WA) Open Data + WSDOT (WA) Open Data + NYC DOT TMC (opt-in) + City of Toronto RESCU + Catalonia (Servei Català de Trànsit) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS, Kartverket ground heights) + King County (WA) Open Data + WSDOT (WA) Open Data + NYC DOT TMC (opt-in) + City of Toronto RESCU + Catalonia (Servei Català de Trànsit) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | | Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto, Bike Itaú (Brazil), Bicing (Barcelona) and seven other Spanish cities + Urban Sharing, Entur, JCDecaux (Lillestrøm, Valencia, Seville) + nextbike (AMBici)) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index e2bf2722a..42a43f5cd 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -319,6 +319,39 @@ export const NORWAY_ANCHORS = [ { lat: 69.6492, lon: 18.9553 }, // Tromsø ]; +/** Kartverket's keyless point-height API (CC BY 4.0): terrain height above + * sea level (NN2000), used as the Vegvesen cameras' ground prior because the + * camera feed's points are 2D. */ +export const KARTVERKET_HEIGHT_URL = + 'https://ws.geonorge.no/hoydedata/v1/punkt'; +/** Points per request. The API takes up to 50, but above ~20 it answers the + * extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not + * at all; at 20 nearly every camera resolves from the 1 m terrain model. */ +export const KARTVERKET_MAX_POINTS = 20; +/** Requests in flight at once. The whole country is ~36 requests of ~1-2 s, + * so a cold lookup takes ~6 s. */ +export const KARTVERKET_CONCURRENCY = 8; +/** Deadline for one complete lookup, shared by every request. */ +export const KARTVERKET_LOOKUP_TIMEOUT_MS = 30 * 1000; +/** The Vegvesen lane's budget, measured from the start of the loader (catalog + * fetch included), for waiting on heights. Under the 15 s per-provider fetch + * timeout so this pack is never the catalog's slowest lane. A lookup still + * running after it keeps filling the cache for the next refresh. */ +export const KARTVERKET_CATALOG_WAIT_MS = 12 * 1000; +/** A point Kartverket has no height for (some ferry quays and bridges) is not + * asked about again for this long. */ +export const KARTVERKET_NULL_TTL_MS = 6 * 60 * 60 * 1000; +/** After a 429, a 5xx or a network failure, no lookup starts for this long, + * or for the Retry-After the service sent, bounded to [1 min, 1 h]. */ +export const KARTVERKET_FAILURE_BACKOFF_MS = 10 * 60 * 1000; +export const KARTVERKET_MAX_BACKOFF_MS = 60 * 60 * 1000; +export const KARTVERKET_MAX_RESPONSE_BYTES = 256 * 1024; +/** Plausible Norwegian terrain: below sea level only at the shore, and no road + * camera sits above ~1,500 m. Anything outside is treated as no height. */ +export const KARTVERKET_HEIGHT_RANGE_M = Object.freeze({ min: -20, max: 2500 }); +/** Same identity the other CCTV upstream requests send. */ +export const KARTVERKET_USER_AGENT = 'gods-eye-view-cctv-proxy/1.0'; + /** NYC DOT Traffic Management Center cameras: one keyless JSON list for the * five boroughs (~980 cameras). Frames are stills synthesized from each * camera UUID on the city's webcam host — never taken from the payload. */ diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 760997df6..71589acdd 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -91,6 +91,17 @@ import { DEFAULT_VEGVESEN_MAX_SOURCES, VEGVESEN_MAX_CATALOG_BYTES, NORWAY_ANCHORS, + KARTVERKET_HEIGHT_URL, + KARTVERKET_MAX_POINTS, + KARTVERKET_CONCURRENCY, + KARTVERKET_LOOKUP_TIMEOUT_MS, + KARTVERKET_CATALOG_WAIT_MS, + KARTVERKET_NULL_TTL_MS, + KARTVERKET_FAILURE_BACKOFF_MS, + KARTVERKET_MAX_BACKOFF_MS, + KARTVERKET_USER_AGENT, + KARTVERKET_MAX_RESPONSE_BYTES, + KARTVERKET_HEIGHT_RANGE_M, NYC_CAMERAS_URL, NYC_IMAGE_URL, NYC_CAMERA_ID_RE, @@ -2457,14 +2468,11 @@ export function vegvesenCameraToSource(feature) { fovDeg: 44, rangeM: 145, mountHeightM: 8, - // KNOWN LIMITATION: the feed's coordinates are 2D, so this is one flat - // prior for the whole country, while cameras run from sea level to - // mountain passes near 1,000 m (Haukelifjell, Sjonfjellet, Rugeldalen). - // The point-height prior and the client's one-shot ground snap correct it - // where they resolve; on a stack where neither does (no 3D tiles), a pass - // camera stays hundreds of metres below the terrain, the same risk the - // Caltrans pack documents. A per-camera height (e.g. Kartverket's keyless - // point-height API) would remove it. + // The feed's coordinates are 2D. The loader replaces this flat prior with + // Kartverket's terrain height for each camera (applyVegvesenGroundHeights); + // it only survives when that lookup fails, and then a mountain-pass camera + // can sit below the terrain on a stack without 3D tiles, the same risk the + // Caltrans pack documents. groundElevationM: 150, feedType: videoUrl ? 'hls' : 'image', url: videoUrl || imageUrl, @@ -2484,6 +2492,7 @@ export function vegvesenCameraToSource(feature) { * @returns {Promise>} Normalized camera source objects. */ export async function loadVegvesenSourcesFromOpenData() { + const startedAt = Date.now(); try { const endpoint = process.env.CCTV_VEGVESEN_URL || DEFAULT_VEGVESEN_CCTV_URL; const resp = await fetch(endpoint, { @@ -2529,6 +2538,12 @@ export async function loadVegvesenSourcesFromOpenData() { ? Math.max(8, Math.min(1000, Math.floor(maxRaw))) : DEFAULT_VEGVESEN_MAX_SOURCES; const prioritized = prioritizeSources(cameras, maxCount, NORWAY_ANCHORS); + await applyVegvesenGroundHeights(prioritized, { + waitMs: Math.max( + 0, + KARTVERKET_CATALOG_WAIT_MS - (Date.now() - startedAt), + ), + }); console.log( `[CCTV] Loaded Vegvesen camera sources: ${cameras.length} (using nearest ${prioritized.length})`, ); @@ -2542,6 +2557,271 @@ export async function loadVegvesenSourcesFromOpenData() { } } +/** + * Kartverket answers by rounded position: `{z, expires}`, where z is metres or + * null for "no height there". Heights never expire (camera positions barely + * change); nulls expire after KARTVERKET_NULL_TTL_MS. The 15-minute catalog + * refresh therefore only asks about new or moved cameras. Bounded well above + * the ~900 cameras the pack can load. + */ +const kartverketHeightCache = new Map(); +const KARTVERKET_CACHE_LIMIT = 5000; +/** The lookup in flight, so overlapping catalog loads never start a second. */ +let kartverketPending = null; +/** No lookup starts before this time (epoch ms) after a transport failure. */ +let kartverketBackoffUntil = 0; + +/** Cache key for one position, at the ~0.1 m precision the request sends. */ +const kartverketKey = (lon, lat) => `${lon.toFixed(6)},${lat.toFixed(6)}`; + +/** Forget cached heights, the backoff and any lookup in flight. Tests only. */ +export function clearKartverketHeightCache() { + kartverketHeightCache.clear(); + kartverketPending = null; + kartverketBackoffUntil = 0; +} + +/** + * How long to back off after a failed request: the Retry-After the service + * sent (seconds or an HTTP date), else the default, bounded to [1 min, 1 h]. + * + * @param {?string} retryAfter - The Retry-After header, if any. + * @param {number} now - Epoch ms. + * @returns {number} Milliseconds. + */ +export function kartverketBackoffMs(retryAfter, now = Date.now()) { + let ms = KARTVERKET_FAILURE_BACKOFF_MS; + const text = String(retryAfter ?? '').trim(); + if (/^\d+$/.test(text)) ms = Number(text) * 1000; + else if (text && Number.isFinite(Date.parse(text))) { + ms = Date.parse(text) - now; + } + return Math.max(60 * 1000, Math.min(KARTVERKET_MAX_BACKOFF_MS, ms)); +} + +/** + * Terrain heights above sea level (NN2000) from Kartverket's keyless + * point-height API, KARTVERKET_MAX_POINTS points per request. Each answer is + * matched back by the coordinates the API echoes, and only for points that + * request asked about, then reported through `into.set(key, z)` as its + * request lands, so a caller that stops waiting keeps what has arrived. `z` is + * metres, or null when the API has no usable height there. + * + * Over water `/punkt` returns the depth of the sea floor (`datakilde: + * dybdekurver`, `terreng: Havflate`; −547 m under Nordhordlandsbrua). Either + * marker counts as 0 m, the water surface. For the ~14 bridge cameras that is + * below the deck, but far closer than the sea floor or a flat prior. + * + * The first transport failure (a non-200 answer, a network error or the + * shared deadline) stops the lookup: no further requests start, and the + * failure is returned so the caller can back off. Points in failed requests + * are not reported, so they are asked again after the backoff. + * + * @param {Array<{lon:number, lat:number}>} points + * @param {{into?: {set: Function}, timeoutMs?: number}} [options] + * @returns {Promise<{heights: Map, failure: ?{status: ?number, retryAfter: ?string}}>} + * `heights` is `into` when it is a Map, else a Map of what was reported. + */ +export async function fetchKartverketHeights( + points, + { into = new Map(), timeoutMs = KARTVERKET_LOOKUP_TIMEOUT_MS } = {}, +) { + const heights = into instanceof Map ? into : new Map(); + const report = (key, z) => { + if (heights !== into) heights.set(key, z); + into.set(key, z); + }; + const unique = Array.from( + new Map( + (points || []) + .filter((p) => isLikelyNorwayCoordinate(p?.lat, p?.lon)) + .map((p) => [kartverketKey(p.lon, p.lat), p]), + ).values(), + ); + let failure = null; + if (!unique.length) return { heights, failure }; + const batches = []; + for (let i = 0; i < unique.length; i += KARTVERKET_MAX_POINTS) { + batches.push(unique.slice(i, i + KARTVERKET_MAX_POINTS)); + } + const signal = AbortSignal.timeout(timeoutMs); + const fetchBatch = async (batch) => { + const asked = new Set(batch.map((p) => kartverketKey(p.lon, p.lat))); + const url = new URL(KARTVERKET_HEIGHT_URL); + url.searchParams.set( + 'punkter', + JSON.stringify( + batch.map((p) => [Number(p.lon.toFixed(6)), Number(p.lat.toFixed(6))]), + ), + ); + url.searchParams.set('koordsys', '4258'); + const resp = await fetch(url, { + headers: { + Accept: 'application/json', + 'User-Agent': KARTVERKET_USER_AGENT, + }, + redirect: 'error', + signal, + }); + if (!resp.ok) { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + const error = new Error(`HTTP ${resp.status}`); + error.status = resp.status; + error.retryAfter = resp.headers.get('retry-after'); + throw error; + } + const payload = await readResponseJsonCapped( + resp, + KARTVERKET_MAX_RESPONSE_BYTES, + ); + for (const point of Array.isArray(payload?.punkter) + ? payload.punkter + : []) { + const x = toFiniteNumber(point?.x); + const y = toFiniteNumber(point?.y); + if (!Number.isFinite(x) || !Number.isFinite(y)) continue; + const key = kartverketKey(x, y); + // Only the points this request asked about: an odd answer can neither + // grow the cache nor overwrite another camera. + if (!asked.has(key)) continue; + asked.delete(key); + // `z: null` means no height there; Number(null) would read 0. + const raw = point?.z; + let z = + raw === null || raw === undefined || raw === '' + ? NaN + : toFiniteNumber(raw); + if ( + Number.isFinite(z) && + (point?.datakilde === 'dybdekurver' || point?.terreng === 'Havflate') + ) { + z = 0; + } + const usable = + Number.isFinite(z) && + z >= KARTVERKET_HEIGHT_RANGE_M.min && + z <= KARTVERKET_HEIGHT_RANGE_M.max; + report(key, usable ? z : null); + } + }; + let next = 0; + const worker = async () => { + while (next < batches.length && !failure) { + const batch = batches[next++]; + try { + await fetchBatch(batch); + } catch (error) { + if (!failure) { + failure = { + status: Number.isFinite(error?.status) ? error.status : null, + retryAfter: error?.retryAfter ?? null, + }; + console.warn( + '[CCTV] Kartverket height lookup failed:', + error?.message || error, + ); + } + } + } + }; + await Promise.all( + Array.from( + { length: Math.min(KARTVERKET_CONCURRENCY, batches.length) }, + worker, + ), + ); + return { heights, failure }; +} + +/** + * Give each Vegvesen camera its own terrain height as the ground prior, in + * place. Waits at most `waitMs` for positions it has no answer for; a lookup + * still running then keeps filling the cache for the next catalog refresh. + * Cameras without a height keep the flat prior. After a transport failure no + * lookup starts until the backoff ends, so a failing Kartverket costs one + * attempt, not one per refresh. Never throws. CCTV_VEGVESEN_HEIGHTS=0 turns it + * off. + * + * @param {Array} cameras - Sources from vegvesenCameraToSource(). + * @param {{waitMs?: number, now?: () => number}} [options] + * @returns {Promise} How many cameras got a terrain height. + */ +export async function applyVegvesenGroundHeights( + cameras, + { waitMs = KARTVERKET_CATALOG_WAIT_MS, now = Date.now } = {}, +) { + if (String(process.env.CCTV_VEGVESEN_HEIGHTS || '1').trim() === '0') return 0; + const list = Array.isArray(cameras) ? cameras : []; + const startedAt = now(); + const missing = list.filter((c) => { + const entry = kartverketHeightCache.get(kartverketKey(c.lon, c.lat)); + return !entry || entry.expires <= startedAt; + }); + if ( + missing.length && + !kartverketPending && + startedAt >= kartverketBackoffUntil + ) { + if (kartverketHeightCache.size + missing.length > KARTVERKET_CACHE_LIMIT) { + kartverketHeightCache.clear(); + } + const sink = { + set: (key, z) => + kartverketHeightCache.set(key, { + z, + expires: z === null ? now() + KARTVERKET_NULL_TTL_MS : Infinity, + }), + }; + kartverketPending = fetchKartverketHeights(missing, { into: sink }) + .then(({ failure }) => { + if (failure) { + kartverketBackoffUntil = + now() + kartverketBackoffMs(failure.retryAfter, now()); + } + }) + .catch((error) => { + kartverketBackoffUntil = now() + KARTVERKET_FAILURE_BACKOFF_MS; + console.warn( + '[CCTV] Kartverket height lookup error:', + error?.message || error, + ); + }) + .finally(() => { + kartverketPending = null; + }); + } + if (missing.length && kartverketPending && waitMs > 0) { + let timer; + await Promise.race([ + kartverketPending, + new Promise((resolve) => { + timer = setTimeout(resolve, waitMs); + timer.unref?.(); + }), + ]); + clearTimeout(timer); + } + let applied = 0; + for (const camera of list) { + const z = kartverketHeightCache.get( + kartverketKey(camera.lon, camera.lat), + )?.z; + if (!Number.isFinite(z)) continue; + camera.groundElevationM = Math.round(z * 10) / 10; + applied += 1; + } + if (list.length) { + console.log( + `[CCTV] Kartverket ground heights: ${applied}/${list.length} Vegvesen cameras`, + ); + } + return applied; +} + const NYC_BOROUGHS = new Set([ 'Manhattan', 'Brooklyn', diff --git a/src/data/cctvVegvesen.test.mjs b/src/data/cctvVegvesen.test.mjs index 457fecbda..47ae732d1 100644 --- a/src/data/cctvVegvesen.test.mjs +++ b/src/data/cctvVegvesen.test.mjs @@ -2,11 +2,21 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { + applyVegvesenGroundHeights, + clearKartverketHeightCache, + fetchKartverketHeights, + kartverketBackoffMs, loadVegvesenSourcesFromOpenData, vegvesenCameraToSource, } from '../../server/providers/cctv/sources.js'; import { DEFAULT_VEGVESEN_CCTV_URL, + KARTVERKET_HEIGHT_URL, + KARTVERKET_CONCURRENCY, + KARTVERKET_FAILURE_BACKOFF_MS, + KARTVERKET_MAX_POINTS, + KARTVERKET_NULL_TTL_MS, + KARTVERKET_USER_AGENT, VEGVESEN_IMAGE_ORIGIN, } from '../../server/providers/cctv/constants.js'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; @@ -104,6 +114,8 @@ test('faulty cameras, off-host frames and bad geometry are dropped', () => { }); test('the loader dedupes, and failures degrade to an empty pack', async (t) => { + clearKartverketHeightCache(); + t.after(clearKartverketHeightCache); t.mock.method(console, 'log', () => {}); t.mock.method(console, 'warn', () => {}); const requested = []; @@ -127,7 +139,11 @@ test('the loader dedupes, and failures degrade to an empty pack', async (t) => { }); }); const cameras = await loadVegvesenSourcesFromOpenData(); - assert.deepEqual(requested, [DEFAULT_VEGVESEN_CCTV_URL]); + // The catalog first; the rest is the Kartverket height lookup. + assert.equal(requested[0], DEFAULT_VEGVESEN_CCTV_URL); + assert.ok( + requested.slice(1).every((url) => url.startsWith(KARTVERKET_HEIGHT_URL)), + ); // Nearest-to-anchor first: Oslo leads. assert.deepEqual( cameras.map((c) => c.id), @@ -200,3 +216,328 @@ test('CCTV_VEGVESEN_ENABLED=0 keeps the lane from being loaded', async (t) => { ); }); }); + +/** Answer a Kartverket request the way the live API does: echo every point, + * with the height `heightFor(x, y)` returns (null = outside coverage). */ +const kartverketResponse = (url, heightFor) => { + const points = JSON.parse(new URL(url).searchParams.get('punkter')); + return Response.json({ + koordsys: 4258, + punkter: points.map(([x, y]) => ({ x, y, z: heightFor(x, y) })), + }); +}; + +/** Cameras at distinct Norwegian positions. */ +const cameraAt = (i) => ({ + id: `no-vegvesen-${i}`, + lat: 60 + i * 0.001, + lon: 7 + i * 0.001, + groundElevationM: 150, +}); + +const withHeightsEnv = (t) => { + const saved = process.env.CCTV_VEGVESEN_HEIGHTS; + delete process.env.CCTV_VEGVESEN_HEIGHTS; + clearKartverketHeightCache(); + t.after(() => { + clearKartverketHeightCache(); + if (saved === undefined) delete process.env.CCTV_VEGVESEN_HEIGHTS; + else process.env.CCTV_VEGVESEN_HEIGHTS = saved; + }); + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +}; + +const sentPoints = (url) => + JSON.parse(new URL(url).searchParams.get('punkter')); + +test('Kartverket heights come in batches of 20, matched by echoed coordinates', async (t) => { + withHeightsEnv(t); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url, init) => { + requested.push(String(url)); + assert.ok(String(url).startsWith(KARTVERKET_HEIGHT_URL)); + assert.equal(init.headers['User-Agent'], KARTVERKET_USER_AGENT); + // Reply in reverse order: matching must not depend on list position. + const response = kartverketResponse(url, (x) => + x > 7.04 ? null : Math.round((x - 7) * 10000), + ); + const body = await response.json(); + body.punkter.reverse(); + return Response.json(body); + }); + const { heights, failure } = await fetchKartverketHeights( + Array.from({ length: 50 }, (_, i) => cameraAt(i)), + ); + assert.equal(failure, null); + // One pass, no retries: 20 + 20 + 10. + assert.deepEqual( + requested.map((url) => sentPoints(url).length), + [KARTVERKET_MAX_POINTS, 20, 10], + ); + for (const url of requested) { + assert.equal(new URL(url).searchParams.get('koordsys'), '4258'); + } + assert.equal(heights.size, 50); + assert.equal(heights.get('7.010000,60.010000'), 100); + // Past x = 7.04 the API has no height: recorded as null, never as 0. + assert.equal(heights.get('7.049000,60.049000'), null); +}); + +test('only the points a request asked about are recorded', async (t) => { + withHeightsEnv(t); + t.mock.method(globalThis, 'fetch', async (url) => { + const points = sentPoints(url); + return Response.json({ + koordsys: 4258, + punkter: [ + ...points.map(([x, y]) => ({ x, y, z: 10 })), + { x: 7.5, y: 60.5, z: 999 }, // never asked + ], + }); + }); + const { heights } = await fetchKartverketHeights([cameraAt(0)]); + assert.deepEqual([...heights.entries()], [['7.000000,60.000000', 10]]); +}); + +test('either water marker counts as the surface', async (t) => { + withHeightsEnv(t); + t.mock.method(globalThis, 'fetch', async (url) => { + const [a, b, c] = sentPoints(url); + return Response.json({ + koordsys: 4258, + punkter: [ + { + x: a[0], + y: a[1], + z: -547.1, + datakilde: 'dybdekurver', + terreng: null, + }, + { x: b[0], y: b[1], z: -9.2, datakilde: 'dtm1', terreng: 'Havflate' }, + { + x: c[0], + y: c[1], + z: -0.2, + datakilde: 'dtm1', + terreng: 'ÅpentOmråde', + }, + ], + }); + }); + const { heights } = await fetchKartverketHeights([ + cameraAt(0), + cameraAt(1), + cameraAt(2), + ]); + assert.equal(heights.get('7.000000,60.000000'), 0); + assert.equal(heights.get('7.001000,60.001000'), 0); + // A quay just below the datum on land data stays as measured. + assert.equal(heights.get('7.002000,60.002000'), -0.2); +}); + +test('heights outside -20..2500 m and points outside Norway are not used', async (t) => { + withHeightsEnv(t); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + const zs = [-20, -20.5, 2500, 2500.5]; + const points = sentPoints(url); + return Response.json({ + koordsys: 4258, + punkter: points.map(([x, y], i) => ({ x, y, z: zs[i] })), + }); + }); + const { heights } = await fetchKartverketHeights([ + cameraAt(0), + cameraAt(1), + cameraAt(2), + cameraAt(3), + { lat: 55.68, lon: 12.57 }, // Copenhagen: never sent + ]); + assert.equal(requested.length, 1); + assert.equal(sentPoints(requested[0]).length, 4); + assert.deepEqual( + [0, 1, 2, 3].map((i) => + heights.get( + `${(7 + i * 0.001).toFixed(6)},${(60 + i * 0.001).toFixed(6)}`, + ), + ), + [-20, null, 2500, null], + ); +}); + +test('no more than eight requests are in flight', async (t) => { + withHeightsEnv(t); + let inFlight = 0; + let peak = 0; + t.mock.method(globalThis, 'fetch', async (url) => { + inFlight += 1; + peak = Math.max(peak, inFlight); + await new Promise((resolve) => setTimeout(resolve, 5)); + inFlight -= 1; + return kartverketResponse(url, () => 1); + }); + const { heights } = await fetchKartverketHeights( + Array.from({ length: 400 }, (_, i) => cameraAt(i)), + ); + assert.equal(heights.size, 400); + assert.equal(peak, KARTVERKET_CONCURRENCY); +}); + +test('a transport failure stops the lookup: no retries, no new requests', async (t) => { + withHeightsEnv(t); + let calls = 0; + t.mock.method(globalThis, 'fetch', async () => { + calls += 1; + return new Response('busy', { + status: 429, + headers: { 'Retry-After': '120' }, + }); + }); + const { heights, failure } = await fetchKartverketHeights( + Array.from({ length: 400 }, (_, i) => cameraAt(i)), + ); + // The first wave is already in flight when the first failure lands; nothing + // after it starts, and no point is resent. + assert.equal(calls, KARTVERKET_CONCURRENCY); + assert.equal(heights.size, 0); + assert.deepEqual(failure, { status: 429, retryAfter: '120' }); +}); + +test('the lookup deadline bounds a hanging service', async (t) => { + withHeightsEnv(t); + t.mock.method( + globalThis, + 'fetch', + (url, init) => + new Promise((_, reject) => { + init.signal.addEventListener('abort', () => reject(init.signal.reason)); + }), + ); + const started = Date.now(); + const { heights, failure } = await fetchKartverketHeights([cameraAt(0)], { + timeoutMs: 30, + }); + assert.ok(Date.now() - started < 2000); + assert.equal(heights.size, 0); + assert.equal(failure.status, null); +}); + +test('Retry-After sets the backoff, bounded to [1 min, 1 h]', () => { + const now = Date.parse('2026-10-06T12:00:00Z'); + assert.equal(kartverketBackoffMs('120', now), 120_000); + assert.equal(kartverketBackoffMs('5', now), 60_000); + assert.equal(kartverketBackoffMs('86400', now), 3_600_000); + assert.equal( + kartverketBackoffMs('Mon, 06 Oct 2026 12:05:00 GMT', now), + 300_000, + ); + assert.equal(kartverketBackoffMs(null, now), KARTVERKET_FAILURE_BACKOFF_MS); + assert.equal(kartverketBackoffMs('soon', now), KARTVERKET_FAILURE_BACKOFF_MS); +}); + +test('heights are cached, and misses are not asked again until they expire', async (t) => { + withHeightsEnv(t); + let clock = 1_000_000; + const now = () => clock; + const asked = []; + t.mock.method(globalThis, 'fetch', async (url) => { + asked.push(sentPoints(url).length); + return kartverketResponse(url, (x) => (x < 7.002 ? 1043.26 : null)); + }); + const cameras = [cameraAt(0), cameraAt(1), cameraAt(5)]; + assert.equal(await applyVegvesenGroundHeights(cameras, { now }), 2); + assert.deepEqual( + cameras.map((c) => c.groundElevationM), + [1043.3, 1043.3, 150], + ); + // The next refresh asks nothing: heights and the miss are both cached. + assert.equal( + await applyVegvesenGroundHeights([cameraAt(0), cameraAt(1), cameraAt(5)], { + now, + }), + 2, + ); + assert.deepEqual(asked, [3]); + // Once the miss expires, only that point is asked again. + clock += KARTVERKET_NULL_TTL_MS + 1; + await applyVegvesenGroundHeights([cameraAt(0), cameraAt(5)], { now }); + assert.deepEqual(asked, [3, 1]); +}); + +test('after a failure no lookup starts until the backoff ends', async (t) => { + withHeightsEnv(t); + let clock = 1_000_000; + const now = () => clock; + const fetchMock = t.mock.method(globalThis, 'fetch', async () => { + throw new Error('offline'); + }); + const cameras = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(cameras, { now }), 0); + assert.equal(cameras[0].groundElevationM, 150); + assert.equal(fetchMock.mock.callCount(), 1); + + // Refreshes inside the backoff cost nothing. + clock += KARTVERKET_FAILURE_BACKOFF_MS - 1; + assert.equal(await applyVegvesenGroundHeights([cameraAt(0)], { now }), 0); + assert.equal(fetchMock.mock.callCount(), 1); + + // After it, the failed point is asked again. + clock += 2; + fetchMock.mock.mockImplementation(async (url) => + kartverketResponse(url, () => 321), + ); + const later = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(later, { now }), 1); + assert.equal(later[0].groundElevationM, 321); + assert.equal(fetchMock.mock.callCount(), 2); +}); + +test('CCTV_VEGVESEN_HEIGHTS=0 skips the lookup', async (t) => { + withHeightsEnv(t); + const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => + kartverketResponse(url, () => 5), + ); + process.env.CCTV_VEGVESEN_HEIGHTS = '0'; + const cameras = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(cameras), 0); + assert.equal(cameras[0].groundElevationM, 150); + assert.equal(fetchMock.mock.callCount(), 0); +}); + +test('a slow lookup does not hold the catalog; the next refresh applies it', async (t) => { + withHeightsEnv(t); + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + t.mock.method(globalThis, 'fetch', async (url) => { + await gate; + return kartverketResponse(url, () => 812); + }); + const first = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(first, { waitMs: 20 }), 0); + assert.equal(first[0].groundElevationM, 150); + release(); + // The lookup kept running and filled the cache for the next refresh. + await new Promise((resolve) => setTimeout(resolve, 20)); + const next = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(next, { waitMs: 20 }), 1); + assert.equal(next[0].groundElevationM, 812); +}); + +test('the loader applies Kartverket heights to the cameras it returns', async (t) => { + withHeightsEnv(t); + t.mock.method(globalThis, 'fetch', async (url) => { + if (String(url).startsWith(KARTVERKET_HEIGHT_URL)) { + return kartverketResponse(url, () => 987.6); + } + return Response.json({ type: 'FeatureCollection', features: [feature()] }); + }); + const cameras = await loadVegvesenSourcesFromOpenData(); + assert.deepEqual( + cameras.map((c) => [c.id, c.groundElevationM]), + [['no-vegvesen-3000063_1', 987.6]], + ); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 931452068..23f317e66 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -286,7 +286,9 @@ export const DATA_CREDITS = [ html: 'Road cameras (Norway): contains data under the ' + 'Norwegian licence for Open Government data (NLOD) ' + - 'distributed by Statens vegvesen', + 'distributed by Statens vegvesen · camera ground heights © ' + + 'Kartverket ' + + '(CC BY 4.0)', }, { key: 'nyc-cctv',