From c43f58dc3f41f881f4dcbeae63ca86159448696c Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:46:20 +0200 Subject: [PATCH 01/25] feat: Kartverket ground heights for Statens vegvesen cameras The Vegvesen camera feed is 2D, so every camera shared one flat 150 m ground prior and mountain-pass cameras could sit hundreds of metres below the terrain where neither the point-height prior nor the ground snap resolves. Each camera now gets its terrain height from Kartverket's keyless point-height API (CC BY 4.0): ~840 of ~845 cameras. - 50 points per request, six in flight; points a full batch answers null for are retried once in batches of 10 (the API drops points from large batches). - Sea-floor depths (dybdekurver / Havflate) under bridges and quays count as the water surface, 0 m. - Heights are cached by position, so the 15-minute refresh only asks about new or moved cameras. A catalog load waits at most 12 s; a slower lookup keeps filling the cache for the next refresh. - CCTV_VEGVESEN_HEIGHTS=0 turns it off; Kartverket is credited beside Statens vegvesen in the attribution popover. Follow-up to #929. Co-Authored-By: Claude Opus 5.5 --- .env.example | 1 + CHANGELOG.md | 10 ++ DATA_SOURCES.md | 2 + docs/CURRENT-STATE.md | 2 +- server/providers/cctv/constants.js | 24 +++ server/providers/cctv/sources.js | 225 ++++++++++++++++++++++++++++- src/data/cctvVegvesen.test.mjs | 223 +++++++++++++++++++++++++++- src/data/dataCredits.js | 3 +- 8 files changed, 479 insertions(+), 11 deletions(-) diff --git a/.env.example b/.env.example index ee7620be9..99a3ff88b 100644 --- a/.env.example +++ b/.env.example @@ -195,6 +195,7 @@ AISSTREAM_API_KEY= # CCTV_VEGVESEN_ENABLED=1 # set 0 to disable the Statens vegvesen (Norway) road cameras # CCTV_VEGVESEN_MAX_SOURCES=900 # cap cameras loaded from Statens vegvesen (default loads all ~850 live; max 1000) # CCTV_VEGVESEN_VIDEO=1 # set 0 to show stills instead of live HLS for Statens vegvesen video cameras +# CCTV_VEGVESEN_HEIGHTS=1 # set 0 to skip Kartverket ground heights for Statens vegvesen cameras (flat prior instead) # CCTV_VEGVESEN_URL= # override the Statens vegvesen catalog URL # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry diff --git a/CHANGELOG.md b/CHANGELOG.md index b433f8c06..86cda068c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## [Unreleased] +- Give each Statens vegvesen camera its own ground height from Kartverket's + keyless point-height API (CC BY 4.0) instead of one flat 150 m prior, so + mountain-pass cameras no longer sit hundreds of metres below the terrain + where neither the point-height prior nor the ground snap resolves. About + 840 of ~845 cameras get a height; the rest keep the old prior. Bridge and + quay cameras over water use the sea surface, not the sea-floor depth the + API returns there. A catalog load waits at most 12 s for heights, and a + lookup still running then fills a cache the next refresh applies. + `CCTV_VEGVESEN_HEIGHTS=0` turns the lookup off. Follow-up to #929. + - Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera list is one keyless GeoJSON request to the agency's OGC API view of its DATEX 3.1 CCTV table, so no DATEX account is needed. Every working camera diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 734e35a69..4959c1c75 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -55,6 +55,7 @@ How to read this: | **DriveBC** (Government of British Columbia) | CCTV camera catalog + frames, British Columbia highways | [Open Government Licence – British Columbia](https://www2.gov.bc.ca/gov/content/data/open-data/open-government-licence-bc) — attribution REQUIRED | "DriveBC. Contains information licensed under the Open Government Licence – British Columbia" | | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | +| **Kartverket** (`ws.geonorge.no/hoydedata/v1/punkt`) | Terrain heights (NN2000) used as the ground prior for Statens vegvesen cameras | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) — attribution REQUIRED | "© Kartverket" | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | @@ -90,6 +91,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. +- **Kartverket ground heights for Statens vegvesen cameras.** The camera feed's points are 2D, so the loader asks Kartverket's keyless point-height API (`ws.geonorge.no/hoydedata/v1/punkt`, CC BY 4.0) for the terrain height at each camera, 50 points per request with up to six in flight (about 17 requests, ~5 s, for the whole country). Points a full batch answers `null` for are retried once in batches of 10, since the API sometimes drops points from large batches; points still without data keep the flat 150 m prior. Over water the API returns the sea-floor depth (`dybdekurver` / `Havflate`), which is replaced by 0 m, the water surface. Heights are cached in memory by position, so the 15-minute catalog refresh only asks about new or moved cameras. A catalog load waits at most 12 s; a slower lookup keeps filling the cache for the next refresh. Only coordinates are sent. `CCTV_VEGVESEN_HEIGHTS=0` turns the lookup off. The Kartverket credit is shown beside the Statens vegvesen entry in the Data attribution popover. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d4fe022a8..c5bf9df9e 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2919,7 +2919,7 @@ its criteria cannot be silently ignored. | Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | | Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | | Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS, Kartverket ground heights) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | | Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 6298e86a0..7609a3fc5 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -267,6 +267,30 @@ export const NORWAY_ANCHORS = [ { lat: 69.6492, lon: 18.9553 }, // Tromsø ]; +/** Kartverket's keyless point-height API (CC BY 4.0): terrain height above + * sea level (NN2000) for up to 50 points per request, used as the Vegvesen + * cameras' ground prior because the camera feed's points are 2D. */ +export const KARTVERKET_HEIGHT_URL = + 'https://ws.geonorge.no/hoydedata/v1/punkt'; +export const KARTVERKET_MAX_POINTS = 50; +/** Batch size for the one retry of points a full batch answered null for. */ +export const KARTVERKET_RETRY_POINTS = 10; +/** Batches in flight at once. The whole country is ~17 batches of ~2 s each + * (spread-out points are slow to resolve), so six keep a cold lookup ~6 s. */ +export const KARTVERKET_CONCURRENCY = 6; +/** Deadline for one complete lookup, shared by every batch. */ +export const KARTVERKET_LOOKUP_TIMEOUT_MS = 30 * 1000; +/** How long a catalog load waits for heights. Kept under the per-provider + * fetch timeout so this pack is never the catalog's slowest lane; a lookup + * still running after this keeps filling the cache, and the next refresh + * applies the rest. At app start, with every pack loading at once, a cold + * lookup takes ~10 s. */ +export const KARTVERKET_CATALOG_WAIT_MS = 12 * 1000; +export const KARTVERKET_MAX_RESPONSE_BYTES = 256 * 1024; +/** Plausible Norwegian terrain: below sea level only at the shore, and no road + * camera sits above ~1,500 m. Anything outside is treated as no answer. */ +export const KARTVERKET_HEIGHT_RANGE_M = Object.freeze({ min: -20, max: 2500 }); + /** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */ export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json'; export const DEFAULT_DELDOT_MAX_SOURCES = 300; diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 7ecd3434d..281ddde85 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -65,6 +65,14 @@ import { DEFAULT_VEGVESEN_MAX_SOURCES, VEGVESEN_MAX_CATALOG_BYTES, NORWAY_ANCHORS, + KARTVERKET_HEIGHT_URL, + KARTVERKET_MAX_POINTS, + KARTVERKET_RETRY_POINTS, + KARTVERKET_CONCURRENCY, + KARTVERKET_LOOKUP_TIMEOUT_MS, + KARTVERKET_CATALOG_WAIT_MS, + KARTVERKET_MAX_RESPONSE_BYTES, + KARTVERKET_HEIGHT_RANGE_M, } from './constants.js'; import { toFiniteNumber, @@ -1779,14 +1787,11 @@ export function vegvesenCameraToSource(feature) { fovDeg: 44, rangeM: 145, mountHeightM: 8, - // KNOWN LIMITATION: the feed's coordinates are 2D, so this is one flat - // prior for the whole country, while cameras run from sea level to - // mountain passes near 1,000 m (Haukelifjell, Sjonfjellet, Rugeldalen). - // The point-height prior and the client's one-shot ground snap correct it - // where they resolve; on a stack where neither does (no 3D tiles), a pass - // camera stays hundreds of metres below the terrain, the same risk the - // Caltrans pack documents. A per-camera height (e.g. Kartverket's keyless - // point-height API) would remove it. + // The feed's coordinates are 2D. The loader replaces this flat prior with + // Kartverket's terrain height for each camera (applyVegvesenGroundHeights); + // it only survives when that lookup fails, and then a mountain-pass camera + // can sit below the terrain on a stack without 3D tiles, the same risk the + // Caltrans pack documents. groundElevationM: 150, feedType: videoUrl ? 'hls' : 'image', url: videoUrl || imageUrl, @@ -1851,6 +1856,7 @@ export async function loadVegvesenSourcesFromOpenData() { ? Math.max(8, Math.min(1000, Math.floor(maxRaw))) : DEFAULT_VEGVESEN_MAX_SOURCES; const prioritized = prioritizeSources(cameras, maxCount, NORWAY_ANCHORS); + await applyVegvesenGroundHeights(prioritized); console.log( `[CCTV] Loaded Vegvesen camera sources: ${cameras.length} (using nearest ${prioritized.length})`, ); @@ -1863,3 +1869,206 @@ export async function loadVegvesenSourcesFromOpenData() { return []; } } + +/** + * Terrain heights already resolved, keyed by rounded position. Camera + * positions rarely change, so after the first catalog load a 15-minute refresh + * asks Kartverket only for new or moved cameras. Bounded well above the ~900 + * cameras the pack can load. + */ +const kartverketHeightCache = new Map(); +const KARTVERKET_CACHE_LIMIT = 5000; +/** The lookup in flight, so overlapping catalog loads never start a second. */ +let kartverketPending = null; + +/** Cache key for one position, at the ~0.1 m precision the request sends. */ +const kartverketKey = (lon, lat) => `${lon.toFixed(6)},${lat.toFixed(6)}`; + +/** Forget cached heights and any lookup in flight. Tests only. */ +export function clearKartverketHeightCache() { + kartverketHeightCache.clear(); + kartverketPending = null; +} + +/** + * Terrain heights above sea level (NN2000) from Kartverket's keyless + * point-height API, in batches of up to 50 points. Answers are matched back by + * the coordinates the API echoes, not by position in the list. Each answer is + * written to `into` as its batch lands, so a caller that stops waiting still + * keeps what has arrived. One deadline covers every request. + * + * Two quirks of the live API are handled here: + * - A large batch sometimes answers `z: null` for a point it resolves when + * asked again (Valdresflye: null in a 50-point batch, 1,390.7 m alone), so + * every point still missing after the first pass is retried once in batches + * of KARTVERKET_RETRY_POINTS. A point still null then has no data. + * - Over water `/punkt` returns the depth of the sea floor (`dybdekurver`, + * terrain `Havflate`; −547 m under Nordhordlandsbrua). A camera on a bridge + * or quay stands over the water surface, so those answers count as 0 m. + * + * @param {Array<{lon:number, lat:number}>} points + * @param {{into?: Map, timeoutMs?: number}} [options] + * @returns {Promise>} kartverketKey(lon, lat) → metres. + */ +export async function fetchKartverketHeights( + points, + { into = new Map(), timeoutMs = KARTVERKET_LOOKUP_TIMEOUT_MS } = {}, +) { + const unique = Array.from( + new Map( + (points || []) + .filter((p) => isLikelyNorwayCoordinate(p?.lat, p?.lon)) + .map((p) => [kartverketKey(p.lon, p.lat), p]), + ).values(), + ); + if (!unique.length) return into; + const signal = AbortSignal.timeout(timeoutMs); + const fetchBatch = async (batch) => { + const url = new URL(KARTVERKET_HEIGHT_URL); + url.searchParams.set( + 'punkter', + JSON.stringify( + batch.map((p) => [Number(p.lon.toFixed(6)), Number(p.lat.toFixed(6))]), + ), + ); + url.searchParams.set('koordsys', '4258'); + const resp = await fetch(url, { + headers: { Accept: 'application/json' }, + redirect: 'error', + signal, + }); + if (!resp.ok) { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + throw new Error(`HTTP ${resp.status}`); + } + const payload = await readResponseJsonCapped( + resp, + KARTVERKET_MAX_RESPONSE_BYTES, + ); + for (const point of Array.isArray(payload?.punkter) + ? payload.punkter + : []) { + // `z: null` means no answer for that point; Number(null) would read 0. + if (point?.z === null || point?.z === undefined || point?.z === '') + continue; + const x = toFiniteNumber(point?.x); + const y = toFiniteNumber(point?.y); + let z = toFiniteNumber(point?.z); + if (![x, y, z].every(Number.isFinite)) continue; + if (point?.datakilde === 'dybdekurver' || point?.terreng === 'Havflate') + z = 0; + if ( + z < KARTVERKET_HEIGHT_RANGE_M.min || + z > KARTVERKET_HEIGHT_RANGE_M.max + ) + continue; + into.set(kartverketKey(x, y), z); + } + }; + let failed = 0; + const runPass = async (list, size) => { + const batches = []; + for (let i = 0; i < list.length; i += size) { + batches.push(list.slice(i, i + size)); + } + let next = 0; + const worker = async () => { + while (next < batches.length && !signal.aborted) { + const batch = batches[next++]; + try { + await fetchBatch(batch); + } catch (error) { + failed += 1; + if (failed === 1) { + console.warn( + '[CCTV] Kartverket height lookup failed:', + error?.message || error, + ); + } + } + } + }; + await Promise.all( + Array.from( + { length: Math.min(KARTVERKET_CONCURRENCY, batches.length) }, + worker, + ), + ); + }; + await runPass(unique, KARTVERKET_MAX_POINTS); + const missing = unique.filter((p) => !into.has(kartverketKey(p.lon, p.lat))); + if (missing.length && !signal.aborted) { + await runPass(missing, KARTVERKET_RETRY_POINTS); + } + return into; +} + +/** + * Give each Vegvesen camera its own terrain height as the ground prior, in + * place. Waits at most `waitMs` for heights it does not have cached; a lookup + * still running then keeps filling the cache for the next catalog refresh. + * Cameras without a height keep the flat prior. Never throws: a failed lookup + * only means the old behavior. CCTV_VEGVESEN_HEIGHTS=0 turns it off. + * + * @param {Array} cameras - Sources from vegvesenCameraToSource(). + * @param {{waitMs?: number}} [options] + * @returns {Promise} How many cameras got a terrain height. + */ +export async function applyVegvesenGroundHeights( + cameras, + { waitMs = KARTVERKET_CATALOG_WAIT_MS } = {}, +) { + if (String(process.env.CCTV_VEGVESEN_HEIGHTS || '1').trim() === '0') return 0; + const list = Array.isArray(cameras) ? cameras : []; + const missing = list.filter( + (c) => !kartverketHeightCache.has(kartverketKey(c.lon, c.lat)), + ); + if (missing.length) { + if (!kartverketPending) { + if ( + kartverketHeightCache.size + missing.length > + KARTVERKET_CACHE_LIMIT + ) { + kartverketHeightCache.clear(); + } + kartverketPending = fetchKartverketHeights(missing, { + into: kartverketHeightCache, + }) + .catch((error) => { + console.warn( + '[CCTV] Kartverket height lookup error:', + error?.message || error, + ); + }) + .finally(() => { + kartverketPending = null; + }); + } + let timer; + await Promise.race([ + kartverketPending, + new Promise((resolve) => { + timer = setTimeout(resolve, waitMs); + timer.unref?.(); + }), + ]); + clearTimeout(timer); + } + let applied = 0; + for (const camera of list) { + const z = kartverketHeightCache.get(kartverketKey(camera.lon, camera.lat)); + if (!Number.isFinite(z)) continue; + camera.groundElevationM = Math.round(z * 10) / 10; + applied += 1; + } + if (list.length) { + console.log( + `[CCTV] Kartverket ground heights: ${applied}/${list.length} Vegvesen cameras`, + ); + } + return applied; +} diff --git a/src/data/cctvVegvesen.test.mjs b/src/data/cctvVegvesen.test.mjs index 457fecbda..ffb25aec3 100644 --- a/src/data/cctvVegvesen.test.mjs +++ b/src/data/cctvVegvesen.test.mjs @@ -2,11 +2,17 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { + applyVegvesenGroundHeights, + clearKartverketHeightCache, + fetchKartverketHeights, loadVegvesenSourcesFromOpenData, vegvesenCameraToSource, } from '../../server/providers/cctv/sources.js'; import { DEFAULT_VEGVESEN_CCTV_URL, + KARTVERKET_HEIGHT_URL, + KARTVERKET_MAX_POINTS, + KARTVERKET_RETRY_POINTS, VEGVESEN_IMAGE_ORIGIN, } from '../../server/providers/cctv/constants.js'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; @@ -104,6 +110,8 @@ test('faulty cameras, off-host frames and bad geometry are dropped', () => { }); test('the loader dedupes, and failures degrade to an empty pack', async (t) => { + clearKartverketHeightCache(); + t.after(clearKartverketHeightCache); t.mock.method(console, 'log', () => {}); t.mock.method(console, 'warn', () => {}); const requested = []; @@ -127,7 +135,11 @@ test('the loader dedupes, and failures degrade to an empty pack', async (t) => { }); }); const cameras = await loadVegvesenSourcesFromOpenData(); - assert.deepEqual(requested, [DEFAULT_VEGVESEN_CCTV_URL]); + // The catalog first; the rest is the Kartverket height lookup. + assert.equal(requested[0], DEFAULT_VEGVESEN_CCTV_URL); + assert.ok( + requested.slice(1).every((url) => url.startsWith(KARTVERKET_HEIGHT_URL)), + ); // Nearest-to-anchor first: Oslo leads. assert.deepEqual( cameras.map((c) => c.id), @@ -200,3 +212,212 @@ test('CCTV_VEGVESEN_ENABLED=0 keeps the lane from being loaded', async (t) => { ); }); }); + +/** Answer a Kartverket request the way the live API does: echo every point, + * with the height `heightFor(x, y)` returns (null = outside coverage). */ +const kartverketResponse = (url, heightFor) => { + const points = JSON.parse(new URL(url).searchParams.get('punkter')); + return Response.json({ + koordsys: 4258, + punkter: points.map(([x, y]) => ({ x, y, z: heightFor(x, y) })), + }); +}; + +/** Cameras at distinct Norwegian positions. */ +const cameraAt = (i) => ({ + id: `no-vegvesen-${i}`, + lat: 60 + i * 0.001, + lon: 7 + i * 0.001, + groundElevationM: 150, +}); + +const withHeightsEnv = (t) => { + const saved = process.env.CCTV_VEGVESEN_HEIGHTS; + delete process.env.CCTV_VEGVESEN_HEIGHTS; + clearKartverketHeightCache(); + t.after(() => { + clearKartverketHeightCache(); + if (saved === undefined) delete process.env.CCTV_VEGVESEN_HEIGHTS; + else process.env.CCTV_VEGVESEN_HEIGHTS = saved; + }); + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +}; + +test('Kartverket heights come in batches of 50, matched by echoed coordinates', async (t) => { + withHeightsEnv(t); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + assert.ok(String(url).startsWith(KARTVERKET_HEIGHT_URL)); + // Reply in reverse order: matching must not depend on list position. + const response = kartverketResponse(url, (x) => + x > 7.05 ? null : Math.round((x - 7) * 10000), + ); + const body = await response.json(); + body.punkter.reverse(); + return Response.json(body); + }); + const points = Array.from({ length: 120 }, (_, i) => cameraAt(i)); + const heights = await fetchKartverketHeights(points); + const sizes = requested.map( + (url) => JSON.parse(new URL(url).searchParams.get('punkter')).length, + ); + // Three full batches, then one retry pass over the 69 points with no height. + assert.deepEqual(sizes.slice(0, 3), [KARTVERKET_MAX_POINTS, 50, 20]); + assert.equal( + sizes.slice(3).reduce((a, b) => a + b, 0), + 69, + ); + assert.ok(sizes.slice(3).every((n) => n <= KARTVERKET_RETRY_POINTS)); + for (const url of requested) { + assert.equal(new URL(url).searchParams.get('koordsys'), '4258'); + } + // Points past x = 7.05 have no height (outside coverage) and are left out. + assert.equal(heights.size, 51); + assert.equal(heights.get('7.010000,60.010000'), 100); +}); + +test('points a full batch answers null for are retried once in small batches', async (t) => { + withHeightsEnv(t); + const sizes = []; + t.mock.method(globalThis, 'fetch', async (url) => { + const points = JSON.parse(new URL(url).searchParams.get('punkter')); + sizes.push(points.length); + // A full batch drops the first two points; a small batch answers them, + // except the one point that has no data at all. + const full = points.length > KARTVERKET_RETRY_POINTS; + return kartverketResponse(url, (x) => { + if (Math.abs(x - 7.002) < 1e-9) return null; + if (full && x < 7.002) return null; + return 500; + }); + }); + const heights = await fetchKartverketHeights( + Array.from({ length: 60 }, (_, i) => cameraAt(i)), + ); + assert.deepEqual(sizes, [50, 10, 3]); + assert.equal(heights.size, 59); + assert.equal(heights.get('7.000000,60.000000'), 500); + assert.equal(heights.has('7.002000,60.002000'), false); +}); + +test('a sea-floor depth counts as the water surface', async (t) => { + withHeightsEnv(t); + t.mock.method(globalThis, 'fetch', async (url) => { + const points = JSON.parse(new URL(url).searchParams.get('punkter')); + return Response.json({ + koordsys: 4258, + punkter: points.map(([x, y], i) => + i === 0 + ? { x, y, z: -547.1, datakilde: 'dybdekurver', terreng: 'Havflate' } + : { x, y, z: -0.2, datakilde: 'dtm1', terreng: 'ÅpentOmråde' }, + ), + }); + }); + const heights = await fetchKartverketHeights([cameraAt(0), cameraAt(1)]); + // Under a bridge: the surface, not the sea floor. A quay just below the + // datum on land data stays as measured. + assert.equal(heights.get('7.000000,60.000000'), 0); + assert.equal(heights.get('7.001000,60.001000'), -0.2); +}); + +test('implausible heights and points outside Norway are ignored', async (t) => { + withHeightsEnv(t); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + return kartverketResponse(url, () => 99999); + }); + const heights = await fetchKartverketHeights([ + cameraAt(1), + { lat: 55.68, lon: 12.57 }, // Copenhagen: never sent + ]); + assert.equal(heights.size, 0); + // The first pass and its one retry, each carrying only the Norway point. + assert.equal(requested.length, 2); + for (const url of requested) { + assert.equal( + JSON.parse(new URL(url).searchParams.get('punkter')).length, + 1, + ); + } +}); + +test('cameras get their terrain height, and a refresh reuses the cache', async (t) => { + withHeightsEnv(t); + let calls = 0; + t.mock.method(globalThis, 'fetch', async (url) => { + calls += 1; + return kartverketResponse(url, (x) => (x < 7.002 ? 1043.26 : null)); + }); + const cameras = [cameraAt(0), cameraAt(1), cameraAt(5)]; + assert.equal(await applyVegvesenGroundHeights(cameras), 2); + assert.deepEqual( + cameras.map((c) => c.groundElevationM), + [1043.3, 1043.3, 150], + ); + // One full pass plus one retry for the point with no height. + assert.equal(calls, 2); + // Same positions on the next catalog refresh: answered from the cache. + const again = [cameraAt(0), cameraAt(1)]; + assert.equal(await applyVegvesenGroundHeights(again), 2); + assert.equal(calls, 2); +}); + +test('a failed or disabled lookup keeps the flat prior', async (t) => { + withHeightsEnv(t); + const fetchMock = t.mock.method(globalThis, 'fetch', async () => { + throw new Error('offline'); + }); + const cameras = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(cameras), 0); + assert.equal(cameras[0].groundElevationM, 150); + + fetchMock.mock.mockImplementation( + async () => new Response('nope', { status: 503 }), + ); + assert.equal(await applyVegvesenGroundHeights(cameras), 0); + assert.equal(cameras[0].groundElevationM, 150); + + process.env.CCTV_VEGVESEN_HEIGHTS = '0'; + fetchMock.mock.resetCalls(); + assert.equal(await applyVegvesenGroundHeights(cameras), 0); + assert.equal(fetchMock.mock.callCount(), 0); +}); + +test('a slow lookup does not hold the catalog; the next refresh applies it', async (t) => { + withHeightsEnv(t); + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + t.mock.method(globalThis, 'fetch', async (url) => { + await gate; + return kartverketResponse(url, () => 812); + }); + const first = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(first, { waitMs: 20 }), 0); + assert.equal(first[0].groundElevationM, 150); + release(); + // The lookup kept running and filled the cache for the next refresh. + await new Promise((resolve) => setTimeout(resolve, 20)); + const next = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(next, { waitMs: 20 }), 1); + assert.equal(next[0].groundElevationM, 812); +}); + +test('the loader applies Kartverket heights to the cameras it returns', async (t) => { + withHeightsEnv(t); + t.mock.method(globalThis, 'fetch', async (url) => { + if (String(url).startsWith(KARTVERKET_HEIGHT_URL)) { + return kartverketResponse(url, () => 987.6); + } + return Response.json({ type: 'FeatureCollection', features: [feature()] }); + }); + const cameras = await loadVegvesenSourcesFromOpenData(); + assert.deepEqual( + cameras.map((c) => [c.id, c.groundElevationM]), + [['no-vegvesen-3000063_1', 987.6]], + ); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 2967f0a2b..8ad585eca 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -245,7 +245,8 @@ export const DATA_CREDITS = [ html: 'Road cameras (Norway): contains data under the ' + 'Norwegian licence for Open Government data (NLOD) ' + - 'distributed by Statens vegvesen', + 'distributed by Statens vegvesen · camera ground heights © ' + + 'Kartverket (CC BY 4.0)', }, { key: 'gbfs', From 34d53edc9188c996259b84e8c62ebb4d3697af08 Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:21:27 +0200 Subject: [PATCH 02/25] fix: address review on Kartverket ground heights - Send 20 points per request, not 50: above ~20 Kartverket answers the extra points from contour lines (hoydekurver) or not at all. At 20, nearly every camera resolves from dtm1, so the null-retry pass is gone. Eight requests in flight keep a cold lookup ~6 s. - Cache misses: positions with no height are not asked again for 6 h. - Transport failures: the first 429, 5xx or network error stops the lookup with no retries, and no lookup starts until the backoff ends (Retry-After when sent, else 10 min; bounded to 1 min-1 h). - Budget the 12 s wait from the start of the loader, catalog fetch included. - Record answers only for the points each request asked about. - Send the CCTV User-Agent; link CC BY 4.0 in the credit. - Docs: bridge cameras sit at the water surface, below the deck. - Tests: batching, 8-in-flight cap, request count on failure, the shared deadline, Retry-After bounds, null TTL, backoff, the -20 m bound and each water marker on its own. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 15 +- DATA_SOURCES.md | 2 +- server/providers/cctv/constants.js | 39 ++-- server/providers/cctv/sources.js | 267 +++++++++++++++++---------- src/data/cctvVegvesen.test.mjs | 282 ++++++++++++++++++++--------- src/data/dataCredits.js | 3 +- 6 files changed, 407 insertions(+), 201 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86cda068c..152279ea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,16 @@ keyless point-height API (CC BY 4.0) instead of one flat 150 m prior, so mountain-pass cameras no longer sit hundreds of metres below the terrain where neither the point-height prior nor the ground snap resolves. About - 840 of ~845 cameras get a height; the rest keep the old prior. Bridge and - quay cameras over water use the sea surface, not the sea-floor depth the - API returns there. A catalog load waits at most 12 s for heights, and a - lookup still running then fills a cache the next refresh applies. - `CCTV_VEGVESEN_HEIGHTS=0` turns the lookup off. Follow-up to #929. + 850 of ~855 cameras get a height; the rest keep the old prior. Requests + carry 20 points, the most the API answers from its 1 m terrain model, eight + at a time (~6 s cold). Cameras over water use the water surface rather than + the sea-floor depth the API returns there; for bridge cameras that is still + below the deck. Heights are cached for good and misses for 6 h, so a + refresh only asks about new or moved cameras. After a 429, a 5xx or a + network failure the lookup stops and backs off (Retry-After, else 10 min). + The lane waits at most 12 s from its start; a slower lookup fills the cache + for the next refresh. `CCTV_VEGVESEN_HEIGHTS=0` turns it off. Follow-up to + #929. - Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera list is one keyless GeoJSON request to the agency's OGC API view of its diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 4959c1c75..fb53776c0 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -91,7 +91,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. -- **Kartverket ground heights for Statens vegvesen cameras.** The camera feed's points are 2D, so the loader asks Kartverket's keyless point-height API (`ws.geonorge.no/hoydedata/v1/punkt`, CC BY 4.0) for the terrain height at each camera, 50 points per request with up to six in flight (about 17 requests, ~5 s, for the whole country). Points a full batch answers `null` for are retried once in batches of 10, since the API sometimes drops points from large batches; points still without data keep the flat 150 m prior. Over water the API returns the sea-floor depth (`dybdekurver` / `Havflate`), which is replaced by 0 m, the water surface. Heights are cached in memory by position, so the 15-minute catalog refresh only asks about new or moved cameras. A catalog load waits at most 12 s; a slower lookup keeps filling the cache for the next refresh. Only coordinates are sent. `CCTV_VEGVESEN_HEIGHTS=0` turns the lookup off. The Kartverket credit is shown beside the Statens vegvesen entry in the Data attribution popover. +- **Kartverket ground heights for Statens vegvesen cameras.** The camera feed's points are 2D, so the loader asks Kartverket's keyless point-height API (`ws.geonorge.no/hoydedata/v1/punkt`, CC BY 4.0) for the terrain height at each camera. Requests carry 20 points: the API accepts 50, but above ~20 it answers the extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not at all. Eight requests run at a time, about 36 for the whole country (~6 s). Answers are matched by the coordinates the API echoes and recorded only for points that request asked about. Over water the API returns the sea-floor depth (`dybdekurver` or `Havflate`), which is replaced by 0 m, the water surface; for the ~14 bridge cameras that is below the deck, but much closer than the sea floor or the flat prior. Heights are cached in memory by position for the life of the server, and points with no height (a few ferry quays and bridges) for 6 h, so the 15-minute catalog refresh only asks about new or moved cameras. The first 429, 5xx or network failure stops the lookup, with no retries, and no new lookup starts until the backoff ends (Retry-After when sent, else 10 min; bounded to 1 min–1 h). The lane waits at most 12 s, counted from the start of the loader; a slower lookup keeps filling the cache for the next refresh. Only coordinates are sent, with the same User-Agent as the other CCTV requests. `CCTV_VEGVESEN_HEIGHTS=0` turns the lookup off. The Kartverket credit is shown beside the Statens vegvesen entry in the Data attribution popover. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 7609a3fc5..6e9c6cd04 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -268,28 +268,37 @@ export const NORWAY_ANCHORS = [ ]; /** Kartverket's keyless point-height API (CC BY 4.0): terrain height above - * sea level (NN2000) for up to 50 points per request, used as the Vegvesen - * cameras' ground prior because the camera feed's points are 2D. */ + * sea level (NN2000), used as the Vegvesen cameras' ground prior because the + * camera feed's points are 2D. */ export const KARTVERKET_HEIGHT_URL = 'https://ws.geonorge.no/hoydedata/v1/punkt'; -export const KARTVERKET_MAX_POINTS = 50; -/** Batch size for the one retry of points a full batch answered null for. */ -export const KARTVERKET_RETRY_POINTS = 10; -/** Batches in flight at once. The whole country is ~17 batches of ~2 s each - * (spread-out points are slow to resolve), so six keep a cold lookup ~6 s. */ -export const KARTVERKET_CONCURRENCY = 6; -/** Deadline for one complete lookup, shared by every batch. */ +/** Points per request. The API takes up to 50, but above ~20 it answers the + * extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not + * at all; at 20 nearly every camera resolves from the 1 m terrain model. */ +export const KARTVERKET_MAX_POINTS = 20; +/** Requests in flight at once. The whole country is ~36 requests of ~1-2 s, + * so a cold lookup takes ~6 s. */ +export const KARTVERKET_CONCURRENCY = 8; +/** Deadline for one complete lookup, shared by every request. */ export const KARTVERKET_LOOKUP_TIMEOUT_MS = 30 * 1000; -/** How long a catalog load waits for heights. Kept under the per-provider - * fetch timeout so this pack is never the catalog's slowest lane; a lookup - * still running after this keeps filling the cache, and the next refresh - * applies the rest. At app start, with every pack loading at once, a cold - * lookup takes ~10 s. */ +/** The Vegvesen lane's budget, measured from the start of the loader (catalog + * fetch included), for waiting on heights. Under the 15 s per-provider fetch + * timeout so this pack is never the catalog's slowest lane. A lookup still + * running after it keeps filling the cache for the next refresh. */ export const KARTVERKET_CATALOG_WAIT_MS = 12 * 1000; +/** A point Kartverket has no height for (some ferry quays and bridges) is not + * asked about again for this long. */ +export const KARTVERKET_NULL_TTL_MS = 6 * 60 * 60 * 1000; +/** After a 429, a 5xx or a network failure, no lookup starts for this long, + * or for the Retry-After the service sent, bounded to [1 min, 1 h]. */ +export const KARTVERKET_FAILURE_BACKOFF_MS = 10 * 60 * 1000; +export const KARTVERKET_MAX_BACKOFF_MS = 60 * 60 * 1000; export const KARTVERKET_MAX_RESPONSE_BYTES = 256 * 1024; /** Plausible Norwegian terrain: below sea level only at the shore, and no road - * camera sits above ~1,500 m. Anything outside is treated as no answer. */ + * camera sits above ~1,500 m. Anything outside is treated as no height. */ export const KARTVERKET_HEIGHT_RANGE_M = Object.freeze({ min: -20, max: 2500 }); +/** Same identity the other CCTV upstream requests send. */ +export const KARTVERKET_USER_AGENT = 'gods-eye-view-cctv-proxy/1.0'; /** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */ export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json'; diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 281ddde85..15f6df74b 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -67,10 +67,13 @@ import { NORWAY_ANCHORS, KARTVERKET_HEIGHT_URL, KARTVERKET_MAX_POINTS, - KARTVERKET_RETRY_POINTS, KARTVERKET_CONCURRENCY, KARTVERKET_LOOKUP_TIMEOUT_MS, KARTVERKET_CATALOG_WAIT_MS, + KARTVERKET_NULL_TTL_MS, + KARTVERKET_FAILURE_BACKOFF_MS, + KARTVERKET_MAX_BACKOFF_MS, + KARTVERKET_USER_AGENT, KARTVERKET_MAX_RESPONSE_BYTES, KARTVERKET_HEIGHT_RANGE_M, } from './constants.js'; @@ -1811,6 +1814,7 @@ export function vegvesenCameraToSource(feature) { * @returns {Promise>} Normalized camera source objects. */ export async function loadVegvesenSourcesFromOpenData() { + const startedAt = Date.now(); try { const endpoint = process.env.CCTV_VEGVESEN_URL || DEFAULT_VEGVESEN_CCTV_URL; const resp = await fetch(endpoint, { @@ -1856,7 +1860,12 @@ export async function loadVegvesenSourcesFromOpenData() { ? Math.max(8, Math.min(1000, Math.floor(maxRaw))) : DEFAULT_VEGVESEN_MAX_SOURCES; const prioritized = prioritizeSources(cameras, maxCount, NORWAY_ANCHORS); - await applyVegvesenGroundHeights(prioritized); + await applyVegvesenGroundHeights(prioritized, { + waitMs: Math.max( + 0, + KARTVERKET_CATALOG_WAIT_MS - (Date.now() - startedAt), + ), + }); console.log( `[CCTV] Loaded Vegvesen camera sources: ${cameras.length} (using nearest ${prioritized.length})`, ); @@ -1871,49 +1880,79 @@ export async function loadVegvesenSourcesFromOpenData() { } /** - * Terrain heights already resolved, keyed by rounded position. Camera - * positions rarely change, so after the first catalog load a 15-minute refresh - * asks Kartverket only for new or moved cameras. Bounded well above the ~900 - * cameras the pack can load. + * Kartverket answers by rounded position: `{z, expires}`, where z is metres or + * null for "no height there". Heights never expire (camera positions barely + * change); nulls expire after KARTVERKET_NULL_TTL_MS. The 15-minute catalog + * refresh therefore only asks about new or moved cameras. Bounded well above + * the ~900 cameras the pack can load. */ const kartverketHeightCache = new Map(); const KARTVERKET_CACHE_LIMIT = 5000; /** The lookup in flight, so overlapping catalog loads never start a second. */ let kartverketPending = null; +/** No lookup starts before this time (epoch ms) after a transport failure. */ +let kartverketBackoffUntil = 0; /** Cache key for one position, at the ~0.1 m precision the request sends. */ const kartverketKey = (lon, lat) => `${lon.toFixed(6)},${lat.toFixed(6)}`; -/** Forget cached heights and any lookup in flight. Tests only. */ +/** Forget cached heights, the backoff and any lookup in flight. Tests only. */ export function clearKartverketHeightCache() { kartverketHeightCache.clear(); kartverketPending = null; + kartverketBackoffUntil = 0; +} + +/** + * How long to back off after a failed request: the Retry-After the service + * sent (seconds or an HTTP date), else the default, bounded to [1 min, 1 h]. + * + * @param {?string} retryAfter - The Retry-After header, if any. + * @param {number} now - Epoch ms. + * @returns {number} Milliseconds. + */ +export function kartverketBackoffMs(retryAfter, now = Date.now()) { + let ms = KARTVERKET_FAILURE_BACKOFF_MS; + const text = String(retryAfter ?? '').trim(); + if (/^\d+$/.test(text)) ms = Number(text) * 1000; + else if (text && Number.isFinite(Date.parse(text))) { + ms = Date.parse(text) - now; + } + return Math.max(60 * 1000, Math.min(KARTVERKET_MAX_BACKOFF_MS, ms)); } /** * Terrain heights above sea level (NN2000) from Kartverket's keyless - * point-height API, in batches of up to 50 points. Answers are matched back by - * the coordinates the API echoes, not by position in the list. Each answer is - * written to `into` as its batch lands, so a caller that stops waiting still - * keeps what has arrived. One deadline covers every request. + * point-height API, KARTVERKET_MAX_POINTS points per request. Each answer is + * matched back by the coordinates the API echoes, and only for points that + * request asked about, then reported through `into.set(key, z)` as its + * request lands, so a caller that stops waiting keeps what has arrived. `z` is + * metres, or null when the API has no usable height there. * - * Two quirks of the live API are handled here: - * - A large batch sometimes answers `z: null` for a point it resolves when - * asked again (Valdresflye: null in a 50-point batch, 1,390.7 m alone), so - * every point still missing after the first pass is retried once in batches - * of KARTVERKET_RETRY_POINTS. A point still null then has no data. - * - Over water `/punkt` returns the depth of the sea floor (`dybdekurver`, - * terrain `Havflate`; −547 m under Nordhordlandsbrua). A camera on a bridge - * or quay stands over the water surface, so those answers count as 0 m. + * Over water `/punkt` returns the depth of the sea floor (`datakilde: + * dybdekurver`, `terreng: Havflate`; −547 m under Nordhordlandsbrua). Either + * marker counts as 0 m, the water surface. For the ~14 bridge cameras that is + * below the deck, but far closer than the sea floor or a flat prior. + * + * The first transport failure (a non-200 answer, a network error or the + * shared deadline) stops the lookup: no further requests start, and the + * failure is returned so the caller can back off. Points in failed requests + * are not reported, so they are asked again after the backoff. * * @param {Array<{lon:number, lat:number}>} points - * @param {{into?: Map, timeoutMs?: number}} [options] - * @returns {Promise>} kartverketKey(lon, lat) → metres. + * @param {{into?: {set: Function}, timeoutMs?: number}} [options] + * @returns {Promise<{heights: Map, failure: ?{status: ?number, retryAfter: ?string}}>} + * `heights` is `into` when it is a Map, else a Map of what was reported. */ export async function fetchKartverketHeights( points, { into = new Map(), timeoutMs = KARTVERKET_LOOKUP_TIMEOUT_MS } = {}, ) { + const heights = into instanceof Map ? into : new Map(); + const report = (key, z) => { + if (heights !== into) heights.set(key, z); + into.set(key, z); + }; const unique = Array.from( new Map( (points || []) @@ -1921,9 +1960,15 @@ export async function fetchKartverketHeights( .map((p) => [kartverketKey(p.lon, p.lat), p]), ).values(), ); - if (!unique.length) return into; + let failure = null; + if (!unique.length) return { heights, failure }; + const batches = []; + for (let i = 0; i < unique.length; i += KARTVERKET_MAX_POINTS) { + batches.push(unique.slice(i, i + KARTVERKET_MAX_POINTS)); + } const signal = AbortSignal.timeout(timeoutMs); const fetchBatch = async (batch) => { + const asked = new Set(batch.map((p) => kartverketKey(p.lon, p.lat))); const url = new URL(KARTVERKET_HEIGHT_URL); url.searchParams.set( 'punkter', @@ -1933,7 +1978,10 @@ export async function fetchKartverketHeights( ); url.searchParams.set('koordsys', '4258'); const resp = await fetch(url, { - headers: { Accept: 'application/json' }, + headers: { + Accept: 'application/json', + 'User-Agent': KARTVERKET_USER_AGENT, + }, redirect: 'error', signal, }); @@ -1943,7 +1991,10 @@ export async function fetchKartverketHeights( } catch { /* no-op */ } - throw new Error(`HTTP ${resp.status}`); + const error = new Error(`HTTP ${resp.status}`); + error.status = resp.status; + error.retryAfter = resp.headers.get('retry-after'); + throw error; } const payload = await readResponseJsonCapped( resp, @@ -1952,102 +2003,120 @@ export async function fetchKartverketHeights( for (const point of Array.isArray(payload?.punkter) ? payload.punkter : []) { - // `z: null` means no answer for that point; Number(null) would read 0. - if (point?.z === null || point?.z === undefined || point?.z === '') - continue; const x = toFiniteNumber(point?.x); const y = toFiniteNumber(point?.y); - let z = toFiniteNumber(point?.z); - if (![x, y, z].every(Number.isFinite)) continue; - if (point?.datakilde === 'dybdekurver' || point?.terreng === 'Havflate') - z = 0; + if (!Number.isFinite(x) || !Number.isFinite(y)) continue; + const key = kartverketKey(x, y); + // Only the points this request asked about: an odd answer can neither + // grow the cache nor overwrite another camera. + if (!asked.has(key)) continue; + asked.delete(key); + // `z: null` means no height there; Number(null) would read 0. + const raw = point?.z; + let z = + raw === null || raw === undefined || raw === '' + ? NaN + : toFiniteNumber(raw); if ( - z < KARTVERKET_HEIGHT_RANGE_M.min || - z > KARTVERKET_HEIGHT_RANGE_M.max - ) - continue; - into.set(kartverketKey(x, y), z); + Number.isFinite(z) && + (point?.datakilde === 'dybdekurver' || point?.terreng === 'Havflate') + ) { + z = 0; + } + const usable = + Number.isFinite(z) && + z >= KARTVERKET_HEIGHT_RANGE_M.min && + z <= KARTVERKET_HEIGHT_RANGE_M.max; + report(key, usable ? z : null); } }; - let failed = 0; - const runPass = async (list, size) => { - const batches = []; - for (let i = 0; i < list.length; i += size) { - batches.push(list.slice(i, i + size)); - } - let next = 0; - const worker = async () => { - while (next < batches.length && !signal.aborted) { - const batch = batches[next++]; - try { - await fetchBatch(batch); - } catch (error) { - failed += 1; - if (failed === 1) { - console.warn( - '[CCTV] Kartverket height lookup failed:', - error?.message || error, - ); - } + let next = 0; + const worker = async () => { + while (next < batches.length && !failure) { + const batch = batches[next++]; + try { + await fetchBatch(batch); + } catch (error) { + if (!failure) { + failure = { + status: Number.isFinite(error?.status) ? error.status : null, + retryAfter: error?.retryAfter ?? null, + }; + console.warn( + '[CCTV] Kartverket height lookup failed:', + error?.message || error, + ); } } - }; - await Promise.all( - Array.from( - { length: Math.min(KARTVERKET_CONCURRENCY, batches.length) }, - worker, - ), - ); + } }; - await runPass(unique, KARTVERKET_MAX_POINTS); - const missing = unique.filter((p) => !into.has(kartverketKey(p.lon, p.lat))); - if (missing.length && !signal.aborted) { - await runPass(missing, KARTVERKET_RETRY_POINTS); - } - return into; + await Promise.all( + Array.from( + { length: Math.min(KARTVERKET_CONCURRENCY, batches.length) }, + worker, + ), + ); + return { heights, failure }; } /** * Give each Vegvesen camera its own terrain height as the ground prior, in - * place. Waits at most `waitMs` for heights it does not have cached; a lookup + * place. Waits at most `waitMs` for positions it has no answer for; a lookup * still running then keeps filling the cache for the next catalog refresh. - * Cameras without a height keep the flat prior. Never throws: a failed lookup - * only means the old behavior. CCTV_VEGVESEN_HEIGHTS=0 turns it off. + * Cameras without a height keep the flat prior. After a transport failure no + * lookup starts until the backoff ends, so a failing Kartverket costs one + * attempt, not one per refresh. Never throws. CCTV_VEGVESEN_HEIGHTS=0 turns it + * off. * * @param {Array} cameras - Sources from vegvesenCameraToSource(). - * @param {{waitMs?: number}} [options] + * @param {{waitMs?: number, now?: () => number}} [options] * @returns {Promise} How many cameras got a terrain height. */ export async function applyVegvesenGroundHeights( cameras, - { waitMs = KARTVERKET_CATALOG_WAIT_MS } = {}, + { waitMs = KARTVERKET_CATALOG_WAIT_MS, now = Date.now } = {}, ) { if (String(process.env.CCTV_VEGVESEN_HEIGHTS || '1').trim() === '0') return 0; const list = Array.isArray(cameras) ? cameras : []; - const missing = list.filter( - (c) => !kartverketHeightCache.has(kartverketKey(c.lon, c.lat)), - ); - if (missing.length) { - if (!kartverketPending) { - if ( - kartverketHeightCache.size + missing.length > - KARTVERKET_CACHE_LIMIT - ) { - kartverketHeightCache.clear(); - } - kartverketPending = fetchKartverketHeights(missing, { - into: kartverketHeightCache, - }) - .catch((error) => { - console.warn( - '[CCTV] Kartverket height lookup error:', - error?.message || error, - ); - }) - .finally(() => { - kartverketPending = null; - }); + const startedAt = now(); + const missing = list.filter((c) => { + const entry = kartverketHeightCache.get(kartverketKey(c.lon, c.lat)); + return !entry || entry.expires <= startedAt; + }); + if ( + missing.length && + !kartverketPending && + startedAt >= kartverketBackoffUntil + ) { + if (kartverketHeightCache.size + missing.length > KARTVERKET_CACHE_LIMIT) { + kartverketHeightCache.clear(); } + const sink = { + set: (key, z) => + kartverketHeightCache.set(key, { + z, + expires: z === null ? now() + KARTVERKET_NULL_TTL_MS : Infinity, + }), + }; + kartverketPending = fetchKartverketHeights(missing, { into: sink }) + .then(({ failure }) => { + if (failure) { + kartverketBackoffUntil = + now() + kartverketBackoffMs(failure.retryAfter, now()); + } + }) + .catch((error) => { + kartverketBackoffUntil = now() + KARTVERKET_FAILURE_BACKOFF_MS; + console.warn( + '[CCTV] Kartverket height lookup error:', + error?.message || error, + ); + }) + .finally(() => { + kartverketPending = null; + }); + } + if (missing.length && kartverketPending && waitMs > 0) { let timer; await Promise.race([ kartverketPending, @@ -2060,7 +2129,9 @@ export async function applyVegvesenGroundHeights( } let applied = 0; for (const camera of list) { - const z = kartverketHeightCache.get(kartverketKey(camera.lon, camera.lat)); + const z = kartverketHeightCache.get( + kartverketKey(camera.lon, camera.lat), + )?.z; if (!Number.isFinite(z)) continue; camera.groundElevationM = Math.round(z * 10) / 10; applied += 1; diff --git a/src/data/cctvVegvesen.test.mjs b/src/data/cctvVegvesen.test.mjs index ffb25aec3..47ae732d1 100644 --- a/src/data/cctvVegvesen.test.mjs +++ b/src/data/cctvVegvesen.test.mjs @@ -5,14 +5,18 @@ import { applyVegvesenGroundHeights, clearKartverketHeightCache, fetchKartverketHeights, + kartverketBackoffMs, loadVegvesenSourcesFromOpenData, vegvesenCameraToSource, } from '../../server/providers/cctv/sources.js'; import { DEFAULT_VEGVESEN_CCTV_URL, KARTVERKET_HEIGHT_URL, + KARTVERKET_CONCURRENCY, + KARTVERKET_FAILURE_BACKOFF_MS, KARTVERKET_MAX_POINTS, - KARTVERKET_RETRY_POINTS, + KARTVERKET_NULL_TTL_MS, + KARTVERKET_USER_AGENT, VEGVESEN_IMAGE_ORIGIN, } from '../../server/providers/cctv/constants.js'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; @@ -244,145 +248,261 @@ const withHeightsEnv = (t) => { t.mock.method(console, 'warn', () => {}); }; -test('Kartverket heights come in batches of 50, matched by echoed coordinates', async (t) => { +const sentPoints = (url) => + JSON.parse(new URL(url).searchParams.get('punkter')); + +test('Kartverket heights come in batches of 20, matched by echoed coordinates', async (t) => { withHeightsEnv(t); const requested = []; - t.mock.method(globalThis, 'fetch', async (url) => { + t.mock.method(globalThis, 'fetch', async (url, init) => { requested.push(String(url)); assert.ok(String(url).startsWith(KARTVERKET_HEIGHT_URL)); + assert.equal(init.headers['User-Agent'], KARTVERKET_USER_AGENT); // Reply in reverse order: matching must not depend on list position. const response = kartverketResponse(url, (x) => - x > 7.05 ? null : Math.round((x - 7) * 10000), + x > 7.04 ? null : Math.round((x - 7) * 10000), ); const body = await response.json(); body.punkter.reverse(); return Response.json(body); }); - const points = Array.from({ length: 120 }, (_, i) => cameraAt(i)); - const heights = await fetchKartverketHeights(points); - const sizes = requested.map( - (url) => JSON.parse(new URL(url).searchParams.get('punkter')).length, + const { heights, failure } = await fetchKartverketHeights( + Array.from({ length: 50 }, (_, i) => cameraAt(i)), ); - // Three full batches, then one retry pass over the 69 points with no height. - assert.deepEqual(sizes.slice(0, 3), [KARTVERKET_MAX_POINTS, 50, 20]); - assert.equal( - sizes.slice(3).reduce((a, b) => a + b, 0), - 69, + assert.equal(failure, null); + // One pass, no retries: 20 + 20 + 10. + assert.deepEqual( + requested.map((url) => sentPoints(url).length), + [KARTVERKET_MAX_POINTS, 20, 10], ); - assert.ok(sizes.slice(3).every((n) => n <= KARTVERKET_RETRY_POINTS)); for (const url of requested) { assert.equal(new URL(url).searchParams.get('koordsys'), '4258'); } - // Points past x = 7.05 have no height (outside coverage) and are left out. - assert.equal(heights.size, 51); + assert.equal(heights.size, 50); assert.equal(heights.get('7.010000,60.010000'), 100); + // Past x = 7.04 the API has no height: recorded as null, never as 0. + assert.equal(heights.get('7.049000,60.049000'), null); }); -test('points a full batch answers null for are retried once in small batches', async (t) => { - withHeightsEnv(t); - const sizes = []; - t.mock.method(globalThis, 'fetch', async (url) => { - const points = JSON.parse(new URL(url).searchParams.get('punkter')); - sizes.push(points.length); - // A full batch drops the first two points; a small batch answers them, - // except the one point that has no data at all. - const full = points.length > KARTVERKET_RETRY_POINTS; - return kartverketResponse(url, (x) => { - if (Math.abs(x - 7.002) < 1e-9) return null; - if (full && x < 7.002) return null; - return 500; - }); - }); - const heights = await fetchKartverketHeights( - Array.from({ length: 60 }, (_, i) => cameraAt(i)), - ); - assert.deepEqual(sizes, [50, 10, 3]); - assert.equal(heights.size, 59); - assert.equal(heights.get('7.000000,60.000000'), 500); - assert.equal(heights.has('7.002000,60.002000'), false); -}); - -test('a sea-floor depth counts as the water surface', async (t) => { +test('only the points a request asked about are recorded', async (t) => { withHeightsEnv(t); t.mock.method(globalThis, 'fetch', async (url) => { - const points = JSON.parse(new URL(url).searchParams.get('punkter')); + const points = sentPoints(url); return Response.json({ koordsys: 4258, - punkter: points.map(([x, y], i) => - i === 0 - ? { x, y, z: -547.1, datakilde: 'dybdekurver', terreng: 'Havflate' } - : { x, y, z: -0.2, datakilde: 'dtm1', terreng: 'ÅpentOmråde' }, - ), + punkter: [ + ...points.map(([x, y]) => ({ x, y, z: 10 })), + { x: 7.5, y: 60.5, z: 999 }, // never asked + ], }); }); - const heights = await fetchKartverketHeights([cameraAt(0), cameraAt(1)]); - // Under a bridge: the surface, not the sea floor. A quay just below the - // datum on land data stays as measured. - assert.equal(heights.get('7.000000,60.000000'), 0); - assert.equal(heights.get('7.001000,60.001000'), -0.2); + const { heights } = await fetchKartverketHeights([cameraAt(0)]); + assert.deepEqual([...heights.entries()], [['7.000000,60.000000', 10]]); }); -test('implausible heights and points outside Norway are ignored', async (t) => { +test('either water marker counts as the surface', async (t) => { + withHeightsEnv(t); + t.mock.method(globalThis, 'fetch', async (url) => { + const [a, b, c] = sentPoints(url); + return Response.json({ + koordsys: 4258, + punkter: [ + { + x: a[0], + y: a[1], + z: -547.1, + datakilde: 'dybdekurver', + terreng: null, + }, + { x: b[0], y: b[1], z: -9.2, datakilde: 'dtm1', terreng: 'Havflate' }, + { + x: c[0], + y: c[1], + z: -0.2, + datakilde: 'dtm1', + terreng: 'ÅpentOmråde', + }, + ], + }); + }); + const { heights } = await fetchKartverketHeights([ + cameraAt(0), + cameraAt(1), + cameraAt(2), + ]); + assert.equal(heights.get('7.000000,60.000000'), 0); + assert.equal(heights.get('7.001000,60.001000'), 0); + // A quay just below the datum on land data stays as measured. + assert.equal(heights.get('7.002000,60.002000'), -0.2); +}); + +test('heights outside -20..2500 m and points outside Norway are not used', async (t) => { withHeightsEnv(t); const requested = []; t.mock.method(globalThis, 'fetch', async (url) => { requested.push(String(url)); - return kartverketResponse(url, () => 99999); + const zs = [-20, -20.5, 2500, 2500.5]; + const points = sentPoints(url); + return Response.json({ + koordsys: 4258, + punkter: points.map(([x, y], i) => ({ x, y, z: zs[i] })), + }); }); - const heights = await fetchKartverketHeights([ + const { heights } = await fetchKartverketHeights([ + cameraAt(0), cameraAt(1), + cameraAt(2), + cameraAt(3), { lat: 55.68, lon: 12.57 }, // Copenhagen: never sent ]); - assert.equal(heights.size, 0); - // The first pass and its one retry, each carrying only the Norway point. - assert.equal(requested.length, 2); - for (const url of requested) { - assert.equal( - JSON.parse(new URL(url).searchParams.get('punkter')).length, - 1, - ); - } + assert.equal(requested.length, 1); + assert.equal(sentPoints(requested[0]).length, 4); + assert.deepEqual( + [0, 1, 2, 3].map((i) => + heights.get( + `${(7 + i * 0.001).toFixed(6)},${(60 + i * 0.001).toFixed(6)}`, + ), + ), + [-20, null, 2500, null], + ); }); -test('cameras get their terrain height, and a refresh reuses the cache', async (t) => { +test('no more than eight requests are in flight', async (t) => { + withHeightsEnv(t); + let inFlight = 0; + let peak = 0; + t.mock.method(globalThis, 'fetch', async (url) => { + inFlight += 1; + peak = Math.max(peak, inFlight); + await new Promise((resolve) => setTimeout(resolve, 5)); + inFlight -= 1; + return kartverketResponse(url, () => 1); + }); + const { heights } = await fetchKartverketHeights( + Array.from({ length: 400 }, (_, i) => cameraAt(i)), + ); + assert.equal(heights.size, 400); + assert.equal(peak, KARTVERKET_CONCURRENCY); +}); + +test('a transport failure stops the lookup: no retries, no new requests', async (t) => { withHeightsEnv(t); let calls = 0; - t.mock.method(globalThis, 'fetch', async (url) => { + t.mock.method(globalThis, 'fetch', async () => { calls += 1; + return new Response('busy', { + status: 429, + headers: { 'Retry-After': '120' }, + }); + }); + const { heights, failure } = await fetchKartverketHeights( + Array.from({ length: 400 }, (_, i) => cameraAt(i)), + ); + // The first wave is already in flight when the first failure lands; nothing + // after it starts, and no point is resent. + assert.equal(calls, KARTVERKET_CONCURRENCY); + assert.equal(heights.size, 0); + assert.deepEqual(failure, { status: 429, retryAfter: '120' }); +}); + +test('the lookup deadline bounds a hanging service', async (t) => { + withHeightsEnv(t); + t.mock.method( + globalThis, + 'fetch', + (url, init) => + new Promise((_, reject) => { + init.signal.addEventListener('abort', () => reject(init.signal.reason)); + }), + ); + const started = Date.now(); + const { heights, failure } = await fetchKartverketHeights([cameraAt(0)], { + timeoutMs: 30, + }); + assert.ok(Date.now() - started < 2000); + assert.equal(heights.size, 0); + assert.equal(failure.status, null); +}); + +test('Retry-After sets the backoff, bounded to [1 min, 1 h]', () => { + const now = Date.parse('2026-10-06T12:00:00Z'); + assert.equal(kartverketBackoffMs('120', now), 120_000); + assert.equal(kartverketBackoffMs('5', now), 60_000); + assert.equal(kartverketBackoffMs('86400', now), 3_600_000); + assert.equal( + kartverketBackoffMs('Mon, 06 Oct 2026 12:05:00 GMT', now), + 300_000, + ); + assert.equal(kartverketBackoffMs(null, now), KARTVERKET_FAILURE_BACKOFF_MS); + assert.equal(kartverketBackoffMs('soon', now), KARTVERKET_FAILURE_BACKOFF_MS); +}); + +test('heights are cached, and misses are not asked again until they expire', async (t) => { + withHeightsEnv(t); + let clock = 1_000_000; + const now = () => clock; + const asked = []; + t.mock.method(globalThis, 'fetch', async (url) => { + asked.push(sentPoints(url).length); return kartverketResponse(url, (x) => (x < 7.002 ? 1043.26 : null)); }); const cameras = [cameraAt(0), cameraAt(1), cameraAt(5)]; - assert.equal(await applyVegvesenGroundHeights(cameras), 2); + assert.equal(await applyVegvesenGroundHeights(cameras, { now }), 2); assert.deepEqual( cameras.map((c) => c.groundElevationM), [1043.3, 1043.3, 150], ); - // One full pass plus one retry for the point with no height. - assert.equal(calls, 2); - // Same positions on the next catalog refresh: answered from the cache. - const again = [cameraAt(0), cameraAt(1)]; - assert.equal(await applyVegvesenGroundHeights(again), 2); - assert.equal(calls, 2); + // The next refresh asks nothing: heights and the miss are both cached. + assert.equal( + await applyVegvesenGroundHeights([cameraAt(0), cameraAt(1), cameraAt(5)], { + now, + }), + 2, + ); + assert.deepEqual(asked, [3]); + // Once the miss expires, only that point is asked again. + clock += KARTVERKET_NULL_TTL_MS + 1; + await applyVegvesenGroundHeights([cameraAt(0), cameraAt(5)], { now }); + assert.deepEqual(asked, [3, 1]); }); -test('a failed or disabled lookup keeps the flat prior', async (t) => { +test('after a failure no lookup starts until the backoff ends', async (t) => { withHeightsEnv(t); + let clock = 1_000_000; + const now = () => clock; const fetchMock = t.mock.method(globalThis, 'fetch', async () => { throw new Error('offline'); }); const cameras = [cameraAt(0)]; - assert.equal(await applyVegvesenGroundHeights(cameras), 0); + assert.equal(await applyVegvesenGroundHeights(cameras, { now }), 0); assert.equal(cameras[0].groundElevationM, 150); + assert.equal(fetchMock.mock.callCount(), 1); - fetchMock.mock.mockImplementation( - async () => new Response('nope', { status: 503 }), + // Refreshes inside the backoff cost nothing. + clock += KARTVERKET_FAILURE_BACKOFF_MS - 1; + assert.equal(await applyVegvesenGroundHeights([cameraAt(0)], { now }), 0); + assert.equal(fetchMock.mock.callCount(), 1); + + // After it, the failed point is asked again. + clock += 2; + fetchMock.mock.mockImplementation(async (url) => + kartverketResponse(url, () => 321), ); + const later = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(later, { now }), 1); + assert.equal(later[0].groundElevationM, 321); + assert.equal(fetchMock.mock.callCount(), 2); +}); + +test('CCTV_VEGVESEN_HEIGHTS=0 skips the lookup', async (t) => { + withHeightsEnv(t); + const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => + kartverketResponse(url, () => 5), + ); + process.env.CCTV_VEGVESEN_HEIGHTS = '0'; + const cameras = [cameraAt(0)]; assert.equal(await applyVegvesenGroundHeights(cameras), 0); assert.equal(cameras[0].groundElevationM, 150); - - process.env.CCTV_VEGVESEN_HEIGHTS = '0'; - fetchMock.mock.resetCalls(); - assert.equal(await applyVegvesenGroundHeights(cameras), 0); assert.equal(fetchMock.mock.callCount(), 0); }); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 8ad585eca..aa8e8f489 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -246,7 +246,8 @@ export const DATA_CREDITS = [ 'Road cameras (Norway): contains data under the ' + 'Norwegian licence for Open Government data (NLOD) ' + 'distributed by Statens vegvesen · camera ground heights © ' + - 'Kartverket (CC BY 4.0)', + 'Kartverket ' + + '(CC BY 4.0)', }, { key: 'gbfs', From 7688e53424217ce2b807fea06f061b14898ce8a7 Mon Sep 17 00:00:00 2001 From: manjunath Date: Sun, 18 Oct 2026 16:15:06 +0530 Subject: [PATCH 03/25] fix(voice): skip replies for rejected typed sends --- docs/CURRENT-STATE.md | 2 +- src/voice/realtimeTurns.js | 2 +- src/voice/realtimeTypedSend.test.mjs | 268 +++++++++++++++++++++++++++ 3 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 src/voice/realtimeTypedSend.test.mjs diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 8cb7b6d17..a1acaa7e5 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4684,7 +4684,7 @@ and unreachable upstream (502/504) separately from road geometry. - **Every selectable layer writes that one slot** (2026-08-21). The tracking layers — flights, military, satellites — publish selection on their own awareness lane (`gev:awareness-subject-selected`, consumed by the readout card and the Contacts panel) and for the life of the voice tools never wrote the shared slot, so `scope:'selected'` silently answered `in_view` with a contact plainly selected on screen. They now call `selectTrackedSubjectContext` / `refreshTrackedSubjectContext` / `clearTrackedSubjectContext` (`src/data/contextStore.js`) on select / poll / deselect. **Invariants: (a)** that write path must NOT dispatch `gev:entity-selected` — tracking layers already own an event lane and a second one makes two surfaces fight over one subject; **(b)** exactly one record per tracking layer, because a frozen snapshot of a moved contact must never reach the visible-entity scan; **(c)** precedence is recency, not layer — one slot, last selection wins, so clicking an overlay entity supersedes a tracked plane as the voice subject while the plane stays tracked. Satellites refresh on the 1 s propagation beat, not per frame. - **Context-mode vocabulary is symmetric** (2026-08-21). `set_context_mode` accepts `contacts`; the internal id is `flights`. Every model-readable field (`mode`, `entering`, `priorMode`, nested `context`/`contextRollback`, and the transition diagnostic text) is reported in the accepted vocabulary, with the internal id preserved as `Internal`; an absent secondary mode stays `null` rather than claiming to be `off`. Mapping lives in `src/contextModePolicy.js` so UI text and voice payloads cannot drift. Reporting the internal id made the model read `mode:'flights'` as "Contacts is off" and refuse to answer from the `contactsWindow` counts in the same payload. - **Analyst → track handoff carries a key, not just a label** (2026-08-21). `analyst_query` items include `icao24`/`mmsi` alongside the display `id`, and contact lookup uses the shared tiered ranking in `src/data/contactMatch.js`: hex exact → callsign exact → registration exact → callsign prefix → registration prefix → callsign substring → registration substring. The tiers keep an exact callsign ahead of a colliding registration regardless of feed order, and registrations compare separator-insensitively (`G-ABCD`/`GABCD`, `05-8152`/`058152`, `N123AB`/`N-123AB`). - - **A typed command supersedes the turn it interrupts** (2026-08-21). `sendTextCommand` defers its `response.create` behind an active response instead of colliding with it, marks that response superseded so a late function call from it is refused rather than dispatched, and drops the old turn's queued follow-up. A refused call is still ANSWERED — a terminal `{ok:false, superseded:true}` `function_call_output` — because an unanswered `function_call` strands a pending call and deadlocks the model; the refusal creates no response of its own. A burst of typed commands coalesces into one response while keeping both conversation items. + - **A typed command supersedes the turn it interrupts** (2026-08-21; hardened 2026-10-17). `sendTextCommand` defers its `response.create` behind an active response instead of colliding with it, marks that response superseded so a late function call from it is refused rather than dispatched, and drops the old turn's queued follow-up. A refused call is still ANSWERED — a terminal `{ok:false, superseded:true}` `function_call_output` — because an unanswered `function_call` strands a pending call and deadlocks the model; the refusal creates no response of its own. A burst of typed commands coalesces into one response while keeping both conversation items. If the local channel rejects the user item, that command requests no response and adds no deferred answer; earlier accepted deferred input remains queued. Interruption and pointer preparation still happen before the send attempt. - **Subject reconciliation across satellite catalog rebuilds** (2026-08-21). A dense↔core toggle or TLE refresh clears and repopulates the catalog, so the published subject is re-resolved against the new satrec; a subject that did NOT survive releases the slot. The per-frame refresh cannot do this itself — `_getTrackedFramePosition` returns early once the satellite has no catalog entry — so the reconcile runs at rebuild completion. An empty catalog is a rebuild in flight, not a disappearance. - **One aircraft-proximity snapshot** (2026-08-22; hardened 2026-09-30). `collectAircraftProximityWindow` (`src/data/militaryAwareness.js`) computes the Contacts panel cohort once. The voice analyst reads the exact retained snapshot and evaluation timestamp instead of re-scanning live layers, so the panel readout and spoken count for one centre cannot cross snapshots. **Invariant: do not re-derive a proximity count anywhere else.** Each layer retains 20,000 rows and samples one row beyond the cap; a saturated cohort reports `complete:false`, `truncated:true`, and is spoken as a lower bound. They diverged before because the panel read live billboard positions while the analyst used last-fix coordinates over a 2,000-record slice — 111 on screen, 15 spoken. Explicit regions, arbitrary points, and a radius different from the panel radius deliberately keep the general record engine. Entity-centred results carry `window: {engine:'contacts-window', centeredOn, radiusKm, flights, military, aircraft, complete, evaluatedAt}`. - **Centre precedence for nearby asks**: explicit place in the question > Contacts subject (a selected non-contact entity never silently becomes the centre) > an entity the user names > the current view, said aloud. Contacts active with no subject uses the view rather than reading an empty panel. diff --git a/src/voice/realtimeTurns.js b/src/voice/realtimeTurns.js index 6905cc17e..cbc8d5dbc 100644 --- a/src/voice/realtimeTurns.js +++ b/src/voice/realtimeTurns.js @@ -137,7 +137,7 @@ export class RealtimeTurns { content: [{ type: 'input_text', text: cleanText }], }, }; - this.sendRealtimeEvent(itemEvent, 'client.user_text'); + if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return; this.requestUserTextResponse(); } diff --git a/src/voice/realtimeTypedSend.test.mjs b/src/voice/realtimeTypedSend.test.mjs new file mode 100644 index 000000000..6a32172c0 --- /dev/null +++ b/src/voice/realtimeTypedSend.test.mjs @@ -0,0 +1,268 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { GevRealtimeController } from './realtimeController.js'; + +function fixture({ pointer = null, onUserSend = () => {} } = {}) { + const sent = []; + const executed = []; + const rejectedTexts = new Set(); + let rejectResponse = false; + const controller = new GevRealtimeController({ + runner: async (name) => { + executed.push(name); + return { ok: true }; + }, + backend: {}, + debugSink: null, + pointer, + ui: { + root: { + dataset: {}, + classList: { remove() {} }, + querySelectorAll: () => [], + }, + status: {}, + detail: {}, + }, + }); + controller.dc = { + readyState: 'open', + send(body) { + const value = JSON.parse(body); + const text = value.item?.role === 'user' && value.item.content?.[0]?.text; + if (value.item?.role === 'user') onUserSend(); + if ( + rejectedTexts.has(text) || + (rejectResponse && value.type === 'response.create') + ) + throw new Error('Synthetic channel rejection'); + sent.push(value); + }, + close() { + this.readyState = 'closed'; + }, + }; + controller.status = 'listening'; + controller._turns.narration = null; + const event = (payload) => + controller.handleRealtimeEvent({ data: JSON.stringify(payload) }); + const created = (id) => event({ type: 'response.created', response: { id } }); + const done = (id) => + event({ + type: 'response.done', + response: { id, status: 'completed', output: [] }, + }); + const creates = () => + sent.filter( + (x) => + x.type === 'response.create' && + x.response?.metadata?.gev !== 'progress', + ); + const texts = () => + sent + .filter((x) => x.item?.role === 'user') + .map((x) => x.item.content[0].text); + return { + controller, + sent, + executed, + rejectedTexts, + event, + created, + done, + creates, + texts, + rejectResponses(value) { + rejectResponse = value; + }, + }; +} + +test('rejected idle text schedules no answer, and a successful successor answers once', async () => { + const f = fixture(); + try { + f.rejectedTexts.add('rejected'); + assert.equal(f.controller.sendTextCommand('rejected'), undefined); + assert.equal(f.creates().length, 0); + assert.equal(f.controller.pendingUserTextResponse, false); + assert.equal(f.controller.responseCreatePending, false); + assert.equal(f.controller.sendTextCommand('successor'), undefined); + assert.deepEqual(f.texts(), ['successor']); + assert.equal(f.creates().length, 1); + await f.created('successor'); + await f.done('successor'); + assert.equal(f.creates().length, 1); + } finally { + f.controller.stop(); + } +}); + +test('rejected text behind an active response cannot queue an answer', async () => { + const f = fixture(); + try { + await f.created('active'); + f.controller.queueResponseCreate('Old tool follow-up'); + assert.equal( + f.controller.pendingResponseInstructions, + 'Old tool follow-up', + ); + const runningTool = new AbortController(); + f.controller._turns.activeToolAbortControllers.add(runningTool); + f.rejectedTexts.add('rejected'); + f.controller.sendTextCommand('rejected'); + assert.equal(f.controller.pendingUserTextResponse, false); + assert.equal(f.controller.pendingResponseInstructions, null); + assert.equal(f.controller._turns.isSupersededResponse('active'), true); + assert.equal(runningTool.signal.aborted, true); + const staleCall = { + type: 'response.function_call_arguments.done', + response_id: 'active', + call_id: 'stale-call', + name: 'fly_to_location', + arguments: '{"query":"Paris"}', + }; + await f.event(staleCall); + await f.event(staleCall); + assert.deepEqual(f.executed, []); + const outputs = f.sent.filter( + (x) => x.item?.type === 'function_call_output', + ); + assert.equal(outputs.length, 1); + assert.equal(outputs[0].item.call_id, 'stale-call'); + assert.equal(JSON.parse(outputs[0].item.output).superseded, true); + await f.done('active'); + assert.equal(f.creates().length, 0); + } finally { + f.controller.stop(); + } +}); + +test('rejected text still prepares narration and pointer context before sending', () => { + const steps = []; + const f = fixture({ + pointer: { + beginTurn() { + steps.push('pointer'); + return null; + }, + clear() {}, + }, + onUserSend() { + steps.push('send'); + }, + }); + f.controller._turns.narration = { + cancel() { + steps.push('cancel'); + }, + userTurnEnded() { + steps.push('user-turn-ended'); + }, + }; + try { + f.rejectedTexts.add('rejected'); + f.controller.sendTextCommand('rejected'); + assert.deepEqual(steps, ['cancel', 'user-turn-ended', 'pointer', 'send']); + assert.equal(f.creates().length, 0); + } finally { + f.controller.stop(); + } +}); + +test('earlier accepted deferred input survives a newer failed send and answers once', async () => { + const f = fixture(); + try { + await f.created('active'); + f.controller.sendTextCommand('accepted'); + f.rejectedTexts.add('rejected'); + f.controller.sendTextCommand('rejected'); + assert.deepEqual(f.texts(), ['accepted']); + assert.equal(f.controller.pendingUserTextResponse, true); + assert.equal(f.creates().length, 0); + await f.done('active'); + assert.equal(f.creates().length, 1); + await f.done('active'); + assert.equal(f.creates().length, 1); + } finally { + f.controller.stop(); + } +}); + +test('rejected text preserves an outstanding create without adding deferred input', async () => { + const f = fixture(); + try { + f.controller.sendTextCommand('accepted'); + f.rejectedTexts.add('rejected'); + f.controller.sendTextCommand('rejected'); + assert.equal(f.controller.responseCreatePending, true); + assert.equal(f.controller.pendingUserTextResponse, false); + await f.created('accepted'); + await f.done('accepted'); + assert.equal(f.creates().length, 1); + } finally { + f.controller.stop(); + } +}); + +for (const acceptedFirst of [false, true]) { + test(`progress completion answers only accepted deferred input (accepted first=${acceptedFirst})`, async () => { + const f = fixture(); + try { + assert.equal( + f.controller._turns.speakProgressLine('Checking the map.'), + true, + ); + await f.event({ + type: 'response.created', + response: { id: 'progress', metadata: { gev: 'progress' } }, + }); + if (acceptedFirst) f.controller.sendTextCommand('accepted'); + f.rejectedTexts.add('rejected'); + f.controller.sendTextCommand('rejected'); + assert.equal(f.controller.pendingUserTextResponse, acceptedFirst); + await f.event({ + type: 'response.done', + response: { + id: 'progress', + status: 'cancelled', + metadata: { gev: 'progress' }, + }, + }); + assert.equal(f.creates().length, acceptedFirst ? 1 : 0); + } finally { + f.controller.stop(); + } + }); +} + +test('accepted text with a rejected response.create releases the pending latch for a successor', async () => { + const f = fixture(); + try { + f.rejectResponses(true); + f.controller.sendTextCommand('accepted without response'); + assert.equal(f.controller.responseCreatePending, false); + assert.equal(f.creates().length, 0); + f.rejectResponses(false); + f.controller.sendTextCommand('successor'); + assert.deepEqual(f.texts(), ['accepted without response', 'successor']); + assert.equal(f.creates().length, 1); + } finally { + f.controller.stop(); + } +}); + +test('successful deferred typed burst still retains both items and coalesces one answer', async () => { + const f = fixture(); + try { + await f.created('active'); + f.controller.sendTextCommand('first'); + f.controller.sendTextCommand('second'); + assert.deepEqual(f.texts(), ['first', 'second']); + assert.equal(f.creates().length, 0); + await f.done('active'); + await f.done('active'); + assert.equal(f.creates().length, 1); + } finally { + f.controller.stop(); + } +}); From 8d3c9297406d86df6413ee35620dba6ab7f446d0 Mon Sep 17 00:00:00 2001 From: manjunath Date: Sun, 18 Oct 2026 16:16:39 +0530 Subject: [PATCH 04/25] docs: add numbered changelog for typed send rejection --- changelog.d/1220-openai-typed-send.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/1220-openai-typed-send.md diff --git a/changelog.d/1220-openai-typed-send.md b/changelog.d/1220-openai-typed-send.md new file mode 100644 index 000000000..c75e0c99c --- /dev/null +++ b/changelog.d/1220-openai-typed-send.md @@ -0,0 +1,3 @@ +- avoid starting or queuing a reply when the local channel rejects a typed OpenAI + command, while preserving an earlier accepted deferred input + (@manjunath22466, #1220) From 15208b5c40323e62523a411b0035f1cf4e4e3d2a Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:03:24 +0200 Subject: [PATCH 05/25] docs: move the changelog entry to changelog.d --- CHANGELOG.md | 14 -------------- .../945-cctv-vegvesen-kartverket-heights.md | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) create mode 100644 changelog.d/945-cctv-vegvesen-kartverket-heights.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 58e2f1798..98d32365f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -435,20 +435,6 @@ panel, resize it from any edge or corner, and double-click the header to dock it again. The position persists across reloads. -- Give each Statens vegvesen camera its own ground height from Kartverket's - keyless point-height API (CC BY 4.0) instead of one flat 150 m prior, so - mountain-pass cameras no longer sit hundreds of metres below the terrain - where neither the point-height prior nor the ground snap resolves. About - 850 of ~855 cameras get a height; the rest keep the old prior. Requests - carry 20 points, the most the API answers from its 1 m terrain model, eight - at a time (~6 s cold). Cameras over water use the water surface rather than - the sea-floor depth the API returns there; for bridge cameras that is still - below the deck. Heights are cached for good and misses for 6 h, so a - refresh only asks about new or moved cameras. After a 429, a 5xx or a - network failure the lookup stops and backs off (Retry-After, else 10 min). - The lane waits at most 12 s from its start; a slower lookup fills the cache - for the next refresh. `CCTV_VEGVESEN_HEIGHTS=0` turns it off. Follow-up to - #929. - Add Norway's city bikes to the Bike Share layer: Oslo, Bergen and Trondheim Bysykkel (Urban Sharing), Kolumbus Bysykkel in Stavanger and Farte Bysykkel in Skien/Porsgrunn (Entur Mobility), and Bysykkel diff --git a/changelog.d/945-cctv-vegvesen-kartverket-heights.md b/changelog.d/945-cctv-vegvesen-kartverket-heights.md new file mode 100644 index 000000000..e300b702c --- /dev/null +++ b/changelog.d/945-cctv-vegvesen-kartverket-heights.md @@ -0,0 +1,14 @@ +- Give each Statens vegvesen camera its own ground height from Kartverket's + keyless point-height API (CC BY 4.0) instead of one flat 150 m prior, so + mountain-pass cameras no longer sit hundreds of metres below the terrain + where neither the point-height prior nor the ground snap resolves. About 850 + of ~855 cameras get a height; the rest keep the old prior. Requests carry 20 + points, the most the API answers from its 1 m terrain model, eight at a time + (~6 s cold). Cameras over water use the water surface rather than the + sea-floor depth the API returns there; for bridge cameras that is still + below the deck. Heights are cached for good and misses for 6 h, so a refresh + only asks about new or moved cameras. After a 429, a 5xx or a network + failure the lookup stops and backs off (Retry-After, else 10 min). The lane + waits at most 12 s from its start; a slower lookup fills the cache for the + next refresh. `CCTV_VEGVESEN_HEIGHTS=0` turns it off. Follow-up to #929 + (trygvehhh, #945). From 59db4ac22798a89e84a157ba2bf4665f6229f488 Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:22:21 +0200 Subject: [PATCH 06/25] docs(cctv): state the measured cold lookup time and what a slow first run leaves --- DATA_SOURCES.md | 2 +- .../945-cctv-vegvesen-kartverket-heights.md | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index b77ee5678..f2b9f8fb9 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -167,7 +167,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped because their frames are WSDOT-hosted, outside the pinned image origin; partner-city cameras hosted on the county image handler (e.g. Newcastle) name their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); both are carried on the source (and served by `/api/cctv/sources`) as `model` and `manufacturer` for identified-hardware consumers (vendor-generic strings such as `PTZ` name no model and are omitted). Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. - **WSDOT highway cameras.** Keyless. The camera list is the Esri JSON layer `data.wsdot.wa.gov/travelcenter/Cameras.json` — the same layer WSDOT's own Travel Center map consumes (~1,700 point features, EPSG:3857 geometry converted to WGS84 on load; the body carries Windows-1252 bytes and is decoded as such). Only frames on WSDOT's own `images.wsdot.wa.gov` host are registered — the catalog also lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) whose own terms apply, and those are skipped rather than proxied. WSDOT's layer metadata marks the feed for "low volume" use (see WSDOT's [travel information disclaimer](https://wsdot.wa.gov/about/policies/travel-information-disclaimer)). The catalog is fetched once per CCTV catalog refresh (every 15 minutes), and a camera's frame is fetched only when that camera is viewed; frames are never stored or redistributed. The live feed's `CompassDirection` is a single cardinal (N/E/S/W), `B` (both directions — most rows), `O`, or null. A single cardinal gives a high-confidence heading (the diagonals NE/SE/SW/NW are mapped too should they appear); `B`, `O`, null, and any other code carry no single facing and use the shared id-hash fallback at low confidence. By default the 250 cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` changes the cap (8–1,200) and `CCTV_WSDOT_ENABLED=0` turns the pack off. A keyed WSDOT Traveler Information API serves the same population for higher-volume use and is deliberately not required here. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. -- **Kartverket ground heights for Statens vegvesen cameras.** The camera feed's points are 2D, so the loader asks Kartverket's keyless point-height API (`ws.geonorge.no/hoydedata/v1/punkt`, CC BY 4.0) for the terrain height at each camera. Requests carry 20 points: the API accepts 50, but above ~20 it answers the extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not at all. Eight requests run at a time, about 36 for the whole country (~6 s). Answers are matched by the coordinates the API echoes and recorded only for points that request asked about. Over water the API returns the sea-floor depth (`dybdekurver` or `Havflate`), which is replaced by 0 m, the water surface; for the ~14 bridge cameras that is below the deck, but much closer than the sea floor or the flat prior. Heights are cached in memory by position for the life of the server, and points with no height (a few ferry quays and bridges) for 6 h, so the 15-minute catalog refresh only asks about new or moved cameras. The first 429, 5xx or network failure stops the lookup, with no retries, and no new lookup starts until the backoff ends (Retry-After when sent, else 10 min; bounded to 1 min–1 h). The lane waits at most 12 s, counted from the start of the loader; a slower lookup keeps filling the cache for the next refresh. Only coordinates are sent, with the same User-Agent as the other CCTV requests. `CCTV_VEGVESEN_HEIGHTS=0` turns the lookup off. The Kartverket credit is shown beside the Statens vegvesen entry in the Data attribution popover. +- **Kartverket ground heights for Statens vegvesen cameras.** The camera feed's points are 2D, so the loader asks Kartverket's keyless point-height API (`ws.geonorge.no/hoydedata/v1/punkt`, CC BY 4.0) for the terrain height at each camera. Requests carry 20 points: the API accepts 50, but above ~20 it answers the extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not at all. Eight requests run at a time, about 36 for the whole country: a cold lookup took 6–13 s in testing (single requests 0.6–9.5 s). Answers are matched by the coordinates the API echoes and recorded only for points that request asked about. Over water the API returns the sea-floor depth (`dybdekurver` or `Havflate`), which is replaced by 0 m, the water surface; for the ~14 bridge cameras that is below the deck, but much closer than the sea floor or the flat prior. Heights are cached in memory by position for the life of the server, and points with no height (a few ferry quays and bridges) for 6 h, so the 15-minute catalog refresh only asks about new or moved cameras. The first 429, 5xx or network failure stops the lookup, with no retries, and no new lookup starts until the backoff ends (Retry-After when sent, else 10 min; bounded to 1 min–1 h). The lane waits at most 12 s, counted from the start of the loader; a slower lookup keeps filling the cache, and the cameras it had not reached keep the flat prior until the next 15-minute refresh (about 190 of 865 on one slow first run). Only coordinates are sent, with the same User-Agent as the other CCTV requests. `CCTV_VEGVESEN_HEIGHTS=0` turns the lookup off. The Kartverket credit is shown beside the Statens vegvesen entry in the Data attribution popover. - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. diff --git a/changelog.d/945-cctv-vegvesen-kartverket-heights.md b/changelog.d/945-cctv-vegvesen-kartverket-heights.md index e300b702c..c6c4b41d9 100644 --- a/changelog.d/945-cctv-vegvesen-kartverket-heights.md +++ b/changelog.d/945-cctv-vegvesen-kartverket-heights.md @@ -3,12 +3,13 @@ mountain-pass cameras no longer sit hundreds of metres below the terrain where neither the point-height prior nor the ground snap resolves. About 850 of ~855 cameras get a height; the rest keep the old prior. Requests carry 20 - points, the most the API answers from its 1 m terrain model, eight at a time - (~6 s cold). Cameras over water use the water surface rather than the - sea-floor depth the API returns there; for bridge cameras that is still - below the deck. Heights are cached for good and misses for 6 h, so a refresh - only asks about new or moved cameras. After a 429, a 5xx or a network - failure the lookup stops and backs off (Retry-After, else 10 min). The lane - waits at most 12 s from its start; a slower lookup fills the cache for the - next refresh. `CCTV_VEGVESEN_HEIGHTS=0` turns it off. Follow-up to #929 - (trygvehhh, #945). + points, the most the API answers from its 1 m terrain model, eight at a + time; a cold lookup takes about 6–13 s. Cameras over water use the water + surface rather than the sea-floor depth the API returns there; for bridge + cameras that is still below the deck. Heights are cached for good and misses + for 6 h, so a refresh only asks about new or moved cameras. After a 429, a + 5xx or a network failure the lookup stops and backs off (Retry-After, else + 10 min). The lane waits at most 12 s from its start; a slower first lookup + leaves the cameras it had not reached on the flat prior until the next + 15-minute refresh, and keeps filling the cache meanwhile. + `CCTV_VEGVESEN_HEIGHTS=0` turns it off. Follow-up to #929 (trygvehhh, #945). From 6b3f7a3ec11c8d3b5311bbc9635c9eef6a6d2e1a Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:37:59 -0700 Subject: [PATCH 07/25] fix(cctv): give Tarktee's slow DATEX feeds 30 s instead of the shared 15 s The Estonian Transpordiamet feeds answered in 2 to 16.6 s across ten requests (120 and 160 KB), and both must arrive for any camera to load, so the shared 15 s catalog timeout dropped the whole pack on about a third of loads ("Tarktee DATEX download error: The operation was aborted due to timeout"), for the 15 minutes until the next catalog refresh. TARKTEE_FETCH_TIMEOUT_MS gives the two requests 30 s; other packs keep 15 s. Live: 175 cameras in 3.3 s. Co-Authored-By: Claude Opus 5.5 --- server/providers/cctv/constants.js | 4 ++++ server/providers/cctv/sources.js | 5 +++-- src/data/cctvEstonia.test.mjs | 26 ++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 20b963770..b0e768c83 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -439,6 +439,10 @@ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; * pending forever — a hung fetch aborts, the loader returns [], and * serve-stale/other packs take over. */ export const CCTV_SOURCE_FETCH_TIMEOUT_MS = 15 * 1000; +/** Tarktee's two DATEX feeds (120 and 160 KB) answered in 2 to 16.6 s across + * ten requests on 2026-10-18, so the shared 15 s timeout dropped the Estonian + * pack on about a third of loads. Both must arrive for any camera to load. */ +export const TARKTEE_FETCH_TIMEOUT_MS = 30 * 1000; /** Individual CCTV image fetches must settle before the active 10-second * client refresh cadence. A bounded miss can fall through to Street View or * the synthetic frame instead of leaving the browser preview pending. */ diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index c3d11e752..f9711ea0c 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -70,6 +70,7 @@ import { TORONTO_GROUND_ELEVATION_M, TORONTO_MAX_CATALOG_BYTES, CCTV_SOURCE_FETCH_TIMEOUT_MS, + TARKTEE_FETCH_TIMEOUT_MS, DELDOT_CCTV_URL, DEFAULT_DELDOT_MAX_SOURCES, DELDOT_ANCHORS, @@ -1194,11 +1195,11 @@ export async function loadTarkteeSourcesFromDatex() { const [locResp, imgResp] = await Promise.all([ fetch(TARKTEE_LOCATIONS_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, - signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), fetch(TARKTEE_IMAGES_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, - signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), ]); if (!locResp.ok) { diff --git a/src/data/cctvEstonia.test.mjs b/src/data/cctvEstonia.test.mjs index 7c0ebaba1..0d578a26c 100644 --- a/src/data/cctvEstonia.test.mjs +++ b/src/data/cctvEstonia.test.mjs @@ -2,9 +2,11 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { loadTallinnSourcesFromCatalog, + loadTarkteeSourcesFromDatex, parseTarkteeDatexImages, parseTarkteeDatexLocations, } from '../../server/providers/cctv/sources.js'; +import { TARKTEE_FETCH_TIMEOUT_MS } from '../../server/providers/cctv/constants.js'; const SAMPLE_LOCATIONS_XML = ` @@ -84,3 +86,27 @@ test('Tarktee DATEX image parser rejects non-official hosts', () => { const images = parseTarkteeDatexImages(xml); assert.equal(images.size, 0); }); + +test('Tarktee DATEX requests wait 30 s, past the slow answers measured live', async (t) => { + const timeouts = []; + const timeout = AbortSignal.timeout; + t.mock.method(AbortSignal, 'timeout', (ms) => { + timeouts.push(ms); + return timeout.call(AbortSignal, ms); + }); + t.mock.method(console, 'log', () => {}); + t.mock.method( + globalThis, + 'fetch', + async (url) => + new Response( + String(url).includes('roadCameraLocations') + ? SAMPLE_LOCATIONS_XML + : SAMPLE_IMAGES_XML, + ), + ); + const cameras = await loadTarkteeSourcesFromDatex(); + assert.deepEqual(timeouts, [30_000, 30_000]); + assert.equal(TARKTEE_FETCH_TIMEOUT_MS, 30_000); + assert.equal(cameras.length, 1); +}); From f26586bb3e6e0b791b63c26df65777e7276ce36d Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:08:24 -0700 Subject: [PATCH 08/25] fix(cctv): serve the camera catalog while it refreshes An expired catalog made every /api/cctv request (camera list and frames) wait for the full refresh, which lasts as long as the slowest of the live packs: Tarktee's DATEX feeds took 2 to 16.6 s across ten requests. With Tarktee's timeout raised to 30 s, that wait could reach 30 s at each 15-minute refresh. The cached list is now returned while a single-flight refresh replaces it in the background; only a first load, with nothing to serve, waits. A background refresh that throws is logged. Co-Authored-By: Claude Opus 5.5 --- server/providers/cctv/catalog.js | 25 ++++++++--- src/data/cctvCatalogCap.test.mjs | 75 ++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 75e58d2a6..396b62f8b 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -204,7 +204,8 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { * * Merges every source pack (live open-data packs, local file, env * variable), deduplicates by ID, shares the catalog cap fairly across - * packs, and caches for CCTV_SOURCE_CACHE_MS. + * packs, and caches for CCTV_SOURCE_CACHE_MS. Past that, the cached list is + * returned while a background refresh replaces it; only a first load waits. * * @returns {Promise>} Deduplicated, capped source list. */ @@ -219,10 +220,24 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { // Single-flight: a burst of requests arriving past the TTL shares ONE refresh // instead of each launching the full multi-provider refetch. The `.finally` // clears the ref so the next post-TTL cycle starts fresh. - if (_cctvSourceInflight) return _cctvSourceInflight; - _cctvSourceInflight = refreshCctvSources().finally(() => { - _cctvSourceInflight = null; - }); + if (!_cctvSourceInflight) { + _cctvSourceInflight = refreshCctvSources().finally(() => { + _cctvSourceInflight = null; + }); + } + // With a catalog in hand, answer with it while the refresh runs: the + // refresh lasts as long as the slowest pack (Tarktee's DATEX feeds take up + // to its 30 s timeout), and every /api/cctv request would otherwise wait + // for it. Only the first load, with nothing to serve yet, waits. + if (_cctvSourceCache.length) { + _cctvSourceInflight.catch((error) => { + console.warn( + '[CCTV] background catalog refresh failed:', + error?.message || error, + ); + }); + return _cctvSourceCache; + } return _cctvSourceInflight; } diff --git a/src/data/cctvCatalogCap.test.mjs b/src/data/cctvCatalogCap.test.mjs index 9d93778db..46faa43de 100644 --- a/src/data/cctvCatalogCap.test.mjs +++ b/src/data/cctvCatalogCap.test.mjs @@ -6,6 +6,7 @@ import { } from '../../server/providers/cctv/cap.js'; import { CCTV_MAX_SOURCES_CEILING, + CCTV_SOURCE_CACHE_MS, DEFAULT_CCTV_MAX_SOURCES, } from '../../server/providers/cctv/constants.js'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; @@ -160,3 +161,77 @@ test('a pack trimmed to its nearest cameras reports how many it offered', async assert.equal(kept.available, 5); assert.deepEqual(kept.region, { west: 0, south: 0, east: 0, north: 4 }); }); + +test('an expired catalog is served at once while a slow pack refreshes in the background', async (t) => { + const saved = { ...process.env }; + const envSources = (prefix) => + JSON.stringify( + Array.from({ length: 3 }, (_, i) => ({ + id: `${prefix}-${i}`, + name: `${prefix} ${i}`, + lat: 30 + i * 0.01, + lon: -97, + url: `https://example.test/${prefix}-${i}.jpg`, + })), + ); + try { + // Live packs run alongside the env pack, so one slow upstream can hold + // the refresh open. + process.env.CCTV_FORCE_AUSTIN = '1'; + process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json'; + process.env.CCTV_SOURCES_JSON = envSources('old'); + let now = 1_000_000; + t.mock.method(Date, 'now', () => now); + t.mock.method(console, 'warn', () => {}); + t.mock.method(console, 'log', () => {}); + const hung = []; + let hang = false; + t.mock.method(globalThis, 'fetch', () => + hang + ? new Promise((_, reject) => { + hung.push(() => reject(new Error('upstream gave up'))); + }) + : Promise.reject(new Error('offline')), + ); + const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' }); + assert.deepEqual( + (await getSources()).map((s) => s.id), + ['old-0', 'old-1', 'old-2'], + ); + + // Past the TTL, with every live pack's upstream hanging. + now += CCTV_SOURCE_CACHE_MS + 1; + hang = true; + process.env.CCTV_SOURCES_JSON = envSources('new'); + const served = await Promise.race([ + getSources(), + new Promise((resolve) => setTimeout(() => resolve('waited'), 200)), + ]); + assert.notEqual( + served, + 'waited', + 'the request did not wait for the refresh', + ); + assert.deepEqual( + served.map((s) => s.id), + ['old-0', 'old-1', 'old-2'], + ); + + // When the slow pack gives up, the refreshed catalog replaces the old one. + for (const giveUp of hung) giveUp(); + for (let i = 0; i < 50; i += 1) { + const ids = (await getSources()).map((s) => s.id); + if (ids[0] === 'new-0') break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.deepEqual( + (await getSources()).map((s) => s.id), + ['new-0', 'new-1', 'new-2'], + ); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + } +}); From 1b66d284c46c3a4fd97935a9ee2452f4c92f9846 Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:12:13 -0700 Subject: [PATCH 09/25] docs(cctv): CURRENT-STATE describes the background catalog refresh and Tarktee's 30 s Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9562dc9f5..9eb1dd751 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4397,7 +4397,11 @@ silently demoting every later lookup for the session. GATHERING stations carry 2,256 in-collection presets, prioritized to 300 against seven anchors on the main road spine. `CCTV_MAX_SOURCES` is a 4,000 catalog-wide ceiling shared round-robin across packs, so a lower global cap thins every region instead of starving the last pack. - ~1,100 cameras total, all RAW PRIOR poses, stills-first. + ~1,100 cameras total, all RAW PRIOR poses, stills-first. The catalog lives for 15 minutes; once + it expires, `/api/cctv` keeps answering from it while one background refresh reloads every + pack, so only the very first load waits for the slowest pack (a live first load took about + 22 s). Estonia's Tarktee DATEX feeds, which answer in 2 to 16.6 s, get 30 s + (`TARKTEE_FETCH_TIMEOUT_MS`) where other packs get 15 s. - **CCTV v3 UX — viewshed + calibration gizmo** (built 2026-07-05 and field validated 2026-07-21): the COVERAGE toggle is a tri-state cycle `OFF → ON → VIEWSHED`; viewshed mode renders each visible camera's frustum From 3792d7377e2aedb3138b86d313b0512442cfcd47 Mon Sep 17 00:00:00 2001 From: yaka0007 Date: Mon, 19 Oct 2026 01:12:18 -0400 Subject: [PATCH 10/25] fix(cctv): harden Tarktee DATEX catalog fetches and dedupe before cap --- server/providers/cctv/constants.js | 5 ++ server/providers/cctv/sources.js | 58 ++++++++++-- src/data/cctvEstonia.test.mjs | 139 +++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 8 deletions(-) diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 20b963770..9bdb5e511 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -188,6 +188,11 @@ export const TARKTEE_IMAGES_URL = 'https://tarktee.transpordiamet.ee/api/v1/datex/roadCameraImages'; export const TARKTEE_IMAGE_ORIGIN = 'https://tarktee.transpordiamet.ee/images/'; export const DEFAULT_TARKTEE_MAX_SOURCES = 179; +/** Hard ceiling per Tarktee DATEX XML document (~120–165 KB live). */ +export const TARKTEE_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +/** Two DATEX XML documents download concurrently from the same origin (~5.3 s + * baseline) alongside the other packs, so the 8 s default can abort mid-stream. */ +export const TARKTEE_FETCH_TIMEOUT_MS = 15_000; export const TARKTEE_ANCHORS = [ { lat: 59.437, lon: 24.753 }, // Tallinn { lat: 58.378, lon: 26.729 }, // Tartu diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index c3d11e752..cba857565 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -42,6 +42,8 @@ import { TARKTEE_IMAGES_URL, TARKTEE_IMAGE_ORIGIN, DEFAULT_TARKTEE_MAX_SOURCES, + TARKTEE_MAX_CATALOG_BYTES, + TARKTEE_FETCH_TIMEOUT_MS, TARKTEE_ANCHORS, DEFAULT_WARENDORF_SOURCE_FILE, WARENDORF_IMAGE_ORIGINS, @@ -1194,25 +1196,62 @@ export async function loadTarkteeSourcesFromDatex() { const [locResp, imgResp] = await Promise.all([ fetch(TARKTEE_LOCATIONS_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, - signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + redirect: 'manual', + signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), fetch(TARKTEE_IMAGES_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, - signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + redirect: 'manual', + signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), ]); + // Both responses are fetched in parallel; if either is rejected or overflows + // its byte cap, both bodies must be released so neither transport leaks. + const discardBoth = async () => { + await Promise.allSettled([ + locResp?.body?.cancel(), + imgResp?.body?.cancel(), + ]); + return []; + }; + if ( + (locResp.status >= 300 && locResp.status < 400) || + (imgResp.status >= 300 && imgResp.status < 400) + ) { + console.warn( + '[CCTV] Tarktee DATEX catalog redirected; redirects are not followed', + ); + return discardBoth(); + } if (!locResp.ok) { console.warn('[CCTV] Tarktee locations download failed:', locResp.status); - return []; + return discardBoth(); } if (!imgResp.ok) { console.warn('[CCTV] Tarktee images download failed:', imgResp.status); - return []; + return discardBoth(); + } + let locXml; + let imgXml; + const readAbort = new AbortController(); + try { + [locXml, imgXml] = await Promise.all([ + readResponseTextCapped( + locResp, + TARKTEE_MAX_CATALOG_BYTES, + readAbort.signal, + ), + readResponseTextCapped( + imgResp, + TARKTEE_MAX_CATALOG_BYTES, + readAbort.signal, + ), + ]); + } catch (error) { + readAbort.abort(); + await discardBoth(); + throw error; } - const [locXml, imgXml] = await Promise.all([ - locResp.text(), - imgResp.text(), - ]); const locations = parseTarkteeDatexLocations(locXml); const images = parseTarkteeDatexImages(imgXml); if (!locations.size || !images.size) { @@ -1224,6 +1263,7 @@ export async function loadTarkteeSourcesFromDatex() { } const cameras = []; + const seen = new Set(); for (const [locationId, loc] of locations.entries()) { const imageUrl = images.get(locationId); if (!imageUrl) continue; @@ -1235,6 +1275,8 @@ export async function loadTarkteeSourcesFromDatex() { const cameraId = numMatch ? `ee-tarktee-${numMatch[1]}` : `ee-tarktee-${locationId}`; + if (seen.has(cameraId)) continue; + seen.add(cameraId); cameras.push({ id: cameraId, name: loc.name, diff --git a/src/data/cctvEstonia.test.mjs b/src/data/cctvEstonia.test.mjs index 7c0ebaba1..a6385854a 100644 --- a/src/data/cctvEstonia.test.mjs +++ b/src/data/cctvEstonia.test.mjs @@ -1,11 +1,23 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { + TARKTEE_IMAGES_URL, + TARKTEE_LOCATIONS_URL, + TARKTEE_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/constants.js'; import { loadTallinnSourcesFromCatalog, + loadTarkteeSourcesFromDatex, parseTarkteeDatexImages, parseTarkteeDatexLocations, } from '../../server/providers/cctv/sources.js'; +/** Silence the loaders' progress and failure logging. */ +function quiet(t) { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +} + const SAMPLE_LOCATIONS_XML = ` @@ -84,3 +96,130 @@ test('Tarktee DATEX image parser rejects non-official hosts', () => { const images = parseTarkteeDatexImages(xml); assert.equal(images.size, 0); }); + +test('Tarktee loader deduplicates cameras sharing an image number before the cap', async (t) => { + quiet(t); + const dupLocationsXml = ` + + + + + Tallinn Ringtee + 59.437024.7530 + + + Tallinn Ringtee duplicate + 59.437124.7531 + + + +`; + const dupImagesXml = ` + + + https://tarktee.transpordiamet.ee/images/42/42_1.jpg + https://tarktee.transpordiamet.ee/images/42/42_2.jpg + +`; + + const seen = []; + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init?.redirect]); + const body = + String(url) === TARKTEE_LOCATIONS_URL ? dupLocationsXml : dupImagesXml; + return new Response(body, { + status: 200, + headers: { 'Content-Type': 'application/xml' }, + }); + }); + + const cameras = await loadTarkteeSourcesFromDatex(); + assert.deepEqual(seen, [ + [TARKTEE_LOCATIONS_URL, 'manual'], + [TARKTEE_IMAGES_URL, 'manual'], + ]); + assert.equal(cameras.length, 1); + assert.equal(cameras[0].id, 'ee-tarktee-42'); + assert.equal(cameras[0].name, 'Tallinn Ringtee'); +}); + +test('Tarktee loader refuses a redirect and cancels both response bodies', async (t) => { + quiet(t); + const cancelled = { locations: false, images: false }; + t.mock.method(globalThis, 'fetch', async (url) => { + const isLocations = String(url) === TARKTEE_LOCATIONS_URL; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('')); + }, + cancel() { + if (isLocations) cancelled.locations = true; + else cancelled.images = true; + }, + }); + return new Response(body, { + status: isLocations ? 302 : 200, + headers: isLocations ? { location: 'https://evil.example/loc.xml' } : {}, + }); + }); + + assert.deepEqual(await loadTarkteeSourcesFromDatex(), []); + assert.equal(cancelled.locations, true, 'redirected body is cancelled'); + assert.equal(cancelled.images, true, 'sibling 200 body is also cancelled'); +}); + +test('Tarktee loader cancels the sibling body when one feed fails or overflows the byte cap', async (t) => { + quiet(t); + let siblingCancelled = false; + t.mock.method(globalThis, 'fetch', async (url) => { + const isLocations = String(url) === TARKTEE_LOCATIONS_URL; + if (isLocations) { + return new Response('', { status: 503 }); + } + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(SAMPLE_IMAGES_XML)); + }, + cancel() { + siblingCancelled = true; + }, + }); + return new Response(body, { status: 200 }); + }); + + assert.deepEqual(await loadTarkteeSourcesFromDatex(), []); + assert.equal( + siblingCancelled, + true, + 'sibling 200 stream is cancelled on 503', + ); + + let overflowSiblingCancelled = false; + t.mock.method(globalThis, 'fetch', async (url) => { + const isLocations = String(url) === TARKTEE_LOCATIONS_URL; + if (isLocations) { + const padded = + SAMPLE_LOCATIONS_XML + ' '.repeat(TARKTEE_MAX_CATALOG_BYTES); + return new Response(padded, { + status: 200, + headers: { 'Content-Length': String(padded.length) }, + }); + } + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(SAMPLE_IMAGES_XML)); + }, + cancel() { + overflowSiblingCancelled = true; + }, + }); + return new Response(body, { status: 200 }); + }); + + assert.deepEqual(await loadTarkteeSourcesFromDatex(), []); + assert.equal( + overflowSiblingCancelled, + true, + 'sibling stream is cancelled when the other feed exceeds the byte cap', + ); +}); From 9b400622af0cb86d957fefe8d9e642efb6c69ec0 Mon Sep 17 00:00:00 2001 From: yaka0007 Date: Mon, 19 Oct 2026 01:13:30 -0400 Subject: [PATCH 11/25] docs(changelog): add fragment for #1233 --- changelog.d/1233-tarktee-datex-catalog-hardening.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/1233-tarktee-datex-catalog-hardening.md diff --git a/changelog.d/1233-tarktee-datex-catalog-hardening.md b/changelog.d/1233-tarktee-datex-catalog-hardening.md new file mode 100644 index 000000000..11b9641e2 --- /dev/null +++ b/changelog.d/1233-tarktee-datex-catalog-hardening.md @@ -0,0 +1,4 @@ +- Harden the Tarktee (Estonia) DATEX2 camera catalog fetches: refuse redirects, + cancel both response streams when either request fails or overflows the 4 MB + byte cap, extend the dual-XML fetch deadline to 15 s, and deduplicate camera + IDs before per-pack prioritization (yaka0007, #1233). From ca9dbea06000b0cfeeea2f77addfa334ae2f2cfc Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:13:36 -0700 Subject: [PATCH 12/25] docs(changelog): fragment for #1234 Co-Authored-By: Claude Opus 5.5 --- changelog.d/1234-cctv-catalog-background-refresh.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog.d/1234-cctv-catalog-background-refresh.md diff --git a/changelog.d/1234-cctv-catalog-background-refresh.md b/changelog.d/1234-cctv-catalog-background-refresh.md new file mode 100644 index 000000000..2d3706b14 --- /dev/null +++ b/changelog.d/1234-cctv-catalog-background-refresh.md @@ -0,0 +1,5 @@ +- Serve the camera catalog while it refreshes. An expired catalog made every + camera request wait for the slowest pack to reload; it is now answered at + once while one background refresh runs, so only the first load waits. Estonia's + slow Tarktee feeds get 30 s instead of 15 s, so the pack no longer drops out + on about a third of loads (quanticsoul4772, #1234). From 5b12f51e9897c7a85c1a799a01cbf1b6b54d5ef8 Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:28:56 -0700 Subject: [PATCH 13/25] fix(cctv): warn once per failed background refresh; test the single flight Attach the background-refresh warning where the refresh starts, so a failed refresh logs once instead of once per request served meanwhile. The test now checks that requests during a refresh share it, and turns the hang off before releasing the fetches so a retrying loader can't stall it. Co-Authored-By: Claude Opus 5.5 --- server/providers/cctv/catalog.js | 21 +++++++++++---------- src/data/cctvCatalogCap.test.mjs | 11 +++++++++++ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 396b62f8b..840ca9ce1 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -224,21 +224,22 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { _cctvSourceInflight = refreshCctvSources().finally(() => { _cctvSourceInflight = null; }); + // A background refresh has no caller waiting on it: report its failure + // here, once per refresh rather than once per request it serves. + if (_cctvSourceCache.length) { + _cctvSourceInflight.catch((error) => { + console.warn( + '[CCTV] background catalog refresh failed:', + error?.message || error, + ); + }); + } } // With a catalog in hand, answer with it while the refresh runs: the // refresh lasts as long as the slowest pack (Tarktee's DATEX feeds take up // to its 30 s timeout), and every /api/cctv request would otherwise wait // for it. Only the first load, with nothing to serve yet, waits. - if (_cctvSourceCache.length) { - _cctvSourceInflight.catch((error) => { - console.warn( - '[CCTV] background catalog refresh failed:', - error?.message || error, - ); - }); - return _cctvSourceCache; - } - return _cctvSourceInflight; + return _cctvSourceCache.length ? _cctvSourceCache : _cctvSourceInflight; } /** diff --git a/src/data/cctvCatalogCap.test.mjs b/src/data/cctvCatalogCap.test.mjs index 46faa43de..26775cb48 100644 --- a/src/data/cctvCatalogCap.test.mjs +++ b/src/data/cctvCatalogCap.test.mjs @@ -217,7 +217,18 @@ test('an expired catalog is served at once while a slow pack refreshes in the ba ['old-0', 'old-1', 'old-2'], ); + // Requests during the refresh share it rather than starting another. + const pause = () => new Promise((resolve) => setTimeout(resolve, 50)); + await pause(); + const inFlight = hung.length; + assert.ok(inFlight > 0, 'the refresh reached the hanging upstreams'); + await getSources(); + await getSources(); + await pause(); + assert.equal(hung.length, inFlight, 'only one refresh ran'); + // When the slow pack gives up, the refreshed catalog replaces the old one. + hang = false; for (const giveUp of hung) giveUp(); for (let i = 0; i < 50; i += 1) { const ids = (await getSources()).map((s) => s.id); From b66d02df7c30a7562c21b2d6dca51cd82b49276b Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Mon, 19 Oct 2026 17:02:06 +0100 Subject: [PATCH 14/25] fix(cctv): space out failed background refreshes; test the stale paths - A background catalog refresh that rejects now keeps the old catalog for another TTL, as an empty refresh already did, so a refresh that keeps failing is retried once per TTL instead of on every request. - Trim notes are keyed by the source list they describe, and /sources asks for the list it served, so a request answered during a refresh never pairs the old cameras with the new list's notes. - Tests: a failed background refresh (one warning, no unhandled rejection, no retry before the TTL, trim notes stay with their list) and an empty background refresh (old catalog kept, no upstream fetch before the TTL). The race assertion's message now names the failure; its timer is cleared. - The Tarktee test no longer imports the new constant, so on main it fails on its own assertion instead of taking the whole file down. - precompute-cctv-heights waits 60 s for a cold catalog, past Tarktee's 30 s. - CURRENT-STATE notes the 30 s bound and the retry spacing. Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 4 +- scripts/precompute-cctv-heights.mjs | 4 +- server/providers/cctv.js | 2 +- server/providers/cctv/catalog.js | 26 +++-- src/data/cctvCatalogCap.test.mjs | 166 +++++++++++++++++++++++++--- src/data/cctvEstonia.test.mjs | 2 - 6 files changed, 176 insertions(+), 28 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index b9dc02918..269b1c7e9 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4412,7 +4412,9 @@ silently demoting every later lookup for the session. ~1,100 cameras total, all RAW PRIOR poses, stills-first. The catalog lives for 15 minutes; once it expires, `/api/cctv` keeps answering from it while one background refresh reloads every pack, so only the very first load waits for the slowest pack (a live first load took about - 22 s). Estonia's Tarktee DATEX feeds, which answer in 2 to 16.6 s, get 30 s + 22 s, and is bounded by Tarktee's 30 s). A background refresh that fails or comes back + empty keeps the old catalog and is retried after another 15 minutes, not on the next + request. Estonia's Tarktee DATEX feeds, which answer in 2 to 16.6 s, get 30 s (`TARKTEE_FETCH_TIMEOUT_MS`) where other packs get 15 s. - **CCTV v3 UX — viewshed + calibration gizmo** (built 2026-07-05 and field validated 2026-07-21): the COVERAGE toggle is a diff --git a/scripts/precompute-cctv-heights.mjs b/scripts/precompute-cctv-heights.mjs index 648768341..2d55f4168 100644 --- a/scripts/precompute-cctv-heights.mjs +++ b/scripts/precompute-cctv-heights.mjs @@ -326,8 +326,10 @@ async function main() { 'src/data/local_data/cctv_ground_heights/cctv_ground_heights.json', ); let sidecar = await loadSidecar(output); + // A cold catalog waits for its slowest pack (Tarktee gets 30 s), so allow + // more than that. const response = await fetch(`${base}/api/cctv/sources`, { - signal: AbortSignal.timeout(30_000), + signal: AbortSignal.timeout(60_000), }); if (!response.ok) throw new Error(`Catalog HTTP ${response.status}`); const catalog = await response.json(); diff --git a/server/providers/cctv.js b/server/providers/cctv.js index b24f38fa1..e549753e8 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -143,7 +143,7 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { if (url.pathname === '/sources') { const body = { - trimmedPacks: getCctvSources.trimmedPacks?.() ?? [], + trimmedPacks: getCctvSources.trimmedPacks?.(sources) ?? [], sources: sources.map((source) => ({ id: source.id, pack: source.pack, diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 840ca9ce1..5e933bc02 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -195,9 +195,11 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { /** @type {Promise>|null} In-flight refresh, shared by concurrent * callers so a post-TTL burst launches ONE refetch, not one per request. */ let _cctvSourceInflight = null; - /** @type {Array<{pack: string, available: number, served: number}>} Packs - * that serve fewer cameras than they offer. */ - let _cctvTrimmedPacks = []; + /** @type {WeakMap, Array<{pack: string, available: number, served: number}>>} + * Packs that serve fewer cameras than they offer, keyed by the source list + * they describe, so a request served the old list during a background + * refresh never pairs it with the new list's trim notes. */ + const _cctvTrimmedPacks = new WeakMap(); /** * Assemble and cache the merged CCTV source list. @@ -225,11 +227,16 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { _cctvSourceInflight = null; }); // A background refresh has no caller waiting on it: report its failure - // here, once per refresh rather than once per request it serves. + // here, once per refresh rather than once per request it serves. Keep + // the old catalog for another TTL, as an empty refresh does, so a refresh + // that keeps failing is retried once per TTL, not back to back. if (_cctvSourceCache.length) { _cctvSourceInflight.catch((error) => { + _cctvSourceCacheAt = Date.now(); console.warn( - '[CCTV] background catalog refresh failed:', + '[CCTV] background catalog refresh failed; serving', + _cctvSourceCache.length, + 'cameras from the last catalog:', error?.message || error, ); }); @@ -329,7 +336,7 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { } if (capped.length > 0 || _cctvSourceCache.length === 0) { _cctvSourceCache = capped; - _cctvTrimmedPacks = trimmedPacks; + _cctvTrimmedPacks.set(capped, trimmedPacks); } else { // Every source came back empty (all live packs timed out / upstream outage) // but a good catalog is already cached — serve it stale rather than blanking @@ -343,7 +350,10 @@ export function createCctvCatalog({ sourceRoot = process.cwd() } = {}) { return _cctvSourceCache; } - /** Packs serving fewer cameras than they offer, as of the last refresh. */ - getCctvSources.trimmedPacks = () => _cctvTrimmedPacks; + /** Packs serving fewer cameras than they offer in `sources` (by default the + * current catalog). Pass the list a request was served to keep the two in + * step while a background refresh swaps the catalog. */ + getCctvSources.trimmedPacks = (sources = _cctvSourceCache) => + _cctvTrimmedPacks.get(sources) ?? []; return getCctvSources; } diff --git a/src/data/cctvCatalogCap.test.mjs b/src/data/cctvCatalogCap.test.mjs index 26775cb48..7ef4b1f7b 100644 --- a/src/data/cctvCatalogCap.test.mjs +++ b/src/data/cctvCatalogCap.test.mjs @@ -162,18 +162,31 @@ test('a pack trimmed to its nearest cameras reports how many it offered', async assert.deepEqual(kept.region, { west: 0, south: 0, east: 0, north: 4 }); }); +/** A CCTV_SOURCES_JSON value with `count` cameras named `-`. */ +const envSources = (prefix, count = 3) => + JSON.stringify( + Array.from({ length: count }, (_, i) => ({ + id: `${prefix}-${i}`, + name: `${prefix} ${i}`, + lat: 30 + i * 0.01, + lon: -97, + url: `https://example.test/${prefix}-${i}.jpg`, + })), + ); + +/** Let a background refresh that needs no timers run to completion. */ +const settle = () => new Promise((resolve) => setTimeout(resolve, 10)); + +/** Restore process.env to a snapshot taken with `{ ...process.env }`. */ +const restoreEnv = (saved) => { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); +}; + test('an expired catalog is served at once while a slow pack refreshes in the background', async (t) => { const saved = { ...process.env }; - const envSources = (prefix) => - JSON.stringify( - Array.from({ length: 3 }, (_, i) => ({ - id: `${prefix}-${i}`, - name: `${prefix} ${i}`, - lat: 30 + i * 0.01, - lon: -97, - url: `https://example.test/${prefix}-${i}.jpg`, - })), - ); try { // Live packs run alongside the env pack, so one slow upstream can hold // the refresh open. @@ -203,14 +216,18 @@ test('an expired catalog is served at once while a slow pack refreshes in the ba now += CCTV_SOURCE_CACHE_MS + 1; hang = true; process.env.CCTV_SOURCES_JSON = envSources('new'); + let timer; const served = await Promise.race([ getSources(), - new Promise((resolve) => setTimeout(() => resolve('waited'), 200)), + new Promise((resolve) => { + timer = setTimeout(() => resolve('waited'), 200); + }), ]); + clearTimeout(timer); assert.notEqual( served, 'waited', - 'the request did not wait for the refresh', + 'a stale request waited for the background refresh', ); assert.deepEqual( served.map((s) => s.id), @@ -240,9 +257,128 @@ test('an expired catalog is served at once while a slow pack refreshes in the ba ['new-0', 'new-1', 'new-2'], ); } finally { - for (const key of Object.keys(process.env)) { - if (!(key in saved)) delete process.env[key]; + restoreEnv(saved); + } +}); + +test('a failed background refresh keeps the catalog, warns once, and waits a TTL to retry', async (t) => { + const saved = { ...process.env }; + const unhandled = []; + const onUnhandled = (reason) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + try { + process.env.CCTV_PREFER_AUSTIN = '0'; + process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json'; + process.env.CCTV_SOURCES_JSON = envSources('old', 9); + process.env.CCTV_MAX_SOURCES = '8'; + let now = 1_000_000; + t.mock.method(Date, 'now', () => now); + // Nine cameras over a cap of eight make every refresh warn; throwing from + // that warning is how this test makes a refresh reject. + let failRefresh = false; + let failedRefreshes = 0; + const warnings = []; + t.mock.method(console, 'warn', (...args) => { + const text = args.join(' '); + if (failRefresh && text.includes('exceeds cap')) { + failedRefreshes += 1; + throw new Error('refresh broke'); + } + warnings.push(text); + }); + const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' }); + const old = await getSources(); + assert.equal(old.length, 8); + + // Past the TTL the refresh throws: the old catalog is still served. + now += CCTV_SOURCE_CACHE_MS + 1; + failRefresh = true; + assert.equal(await getSources(), old); + await settle(); + assert.equal(failedRefreshes, 1); + assert.equal( + warnings.filter((w) => w.includes('background catalog refresh failed')) + .length, + 1, + ); + + // The failure is not retried on every request until the next TTL. + assert.equal(await getSources(), old); + assert.equal(await getSources(), old); + await settle(); + assert.equal(failedRefreshes, 1, 'a failed refresh was retried at once'); + + // After it, a refresh that works replaces the catalog. With no live packs + // the refresh finishes before the request that started it returns. + now += CCTV_SOURCE_CACHE_MS + 1; + failRefresh = false; + process.env.CCTV_SOURCES_JSON = envSources('new'); + await getSources(); + await settle(); + const fresh = await getSources(); + assert.deepEqual( + fresh.map((s) => s.id), + ['new-0', 'new-1', 'new-2'], + ); + // Trim notes stay with the list they describe, so a request served the + // old list during the refresh never reports the new list's notes. + assert.deepEqual( + getSources.trimmedPacks(old).map((p) => [p.pack, p.available, p.served]), + [['env', 9, 8]], + ); + assert.deepEqual(getSources.trimmedPacks(fresh), []); + assert.deepEqual(getSources.trimmedPacks(), []); + assert.deepEqual(unhandled, []); + } finally { + process.off('unhandledRejection', onUnhandled); + restoreEnv(saved); + } +}); + +test('an empty background refresh keeps the catalog and waits a TTL before fetching again', async (t) => { + const saved = { ...process.env }; + try { + process.env.CCTV_FORCE_AUSTIN = '1'; + process.env.CCTV_SOURCES_FILE = '/nonexistent/cctv.json'; + process.env.CCTV_SOURCES_JSON = envSources('old'); + let now = 1_000_000; + t.mock.method(Date, 'now', () => now); + t.mock.method(console, 'log', () => {}); + const warnings = []; + t.mock.method(console, 'warn', (...args) => warnings.push(args.join(' '))); + let fetches = 0; + t.mock.method(globalThis, 'fetch', () => { + fetches += 1; + return Promise.reject(new Error('offline')); + }); + const getSources = createCctvCatalog({ sourceRoot: '/nonexistent' }); + const old = await getSources(); + + // Past the TTL every pack comes back empty, as in an upstream outage. + now += CCTV_SOURCE_CACHE_MS + 1; + process.env.CCTV_SOURCES_JSON = '[]'; + assert.equal(await getSources(), old); + for (let i = 0; i < 50; i += 1) { + if (warnings.some((w) => w.includes('source refresh returned empty'))) + break; + await settle(); } - Object.assign(process.env, saved); + assert.ok( + warnings.some((w) => w.includes('serving 3 stale cameras')), + 'the empty refresh kept the old catalog', + ); + + // Until the next TTL, requests are served without touching any upstream. + const fetchesAfterRefresh = fetches; + assert.equal(await getSources(), old); + assert.equal(await getSources(), old); + await settle(); + assert.equal( + fetches, + fetchesAfterRefresh, + 'an empty refresh reran at once', + ); + } finally { + restoreEnv(saved); } }); diff --git a/src/data/cctvEstonia.test.mjs b/src/data/cctvEstonia.test.mjs index 0d578a26c..b46f9eb06 100644 --- a/src/data/cctvEstonia.test.mjs +++ b/src/data/cctvEstonia.test.mjs @@ -6,7 +6,6 @@ import { parseTarkteeDatexImages, parseTarkteeDatexLocations, } from '../../server/providers/cctv/sources.js'; -import { TARKTEE_FETCH_TIMEOUT_MS } from '../../server/providers/cctv/constants.js'; const SAMPLE_LOCATIONS_XML = ` @@ -107,6 +106,5 @@ test('Tarktee DATEX requests wait 30 s, past the slow answers measured live', as ); const cameras = await loadTarkteeSourcesFromDatex(); assert.deepEqual(timeouts, [30_000, 30_000]); - assert.equal(TARKTEE_FETCH_TIMEOUT_MS, 30_000); assert.equal(cameras.length, 1); }); From c808bfe609b491e33fea9da4a0ed22f72cb8f7a3 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Mon, 19 Oct 2026 17:03:50 +0100 Subject: [PATCH 15/25] fix(cctv): harden the Kartverket height lookup for Vegvesen cameras - Default backoff is 30 min, longer than the 15-minute catalog refresh, so a failing Kartverket costs one attempt per backoff, not per refresh. - A point a 200 answer leaves out is cached as no height (6 h TTL) instead of being asked again, and waited on, every refresh. - Only 408, 429 and 5xx stop the lookup; any other 4xx marks that request's points as no height and the lookup carries on. - Each request has its own 10 s timeout inside the shared 30 s deadline; a lookup cut short only by the deadline resumes without a backoff. - The lane waits only on a cold cache (>10% of cameras missing); a few new cameras or expired misses are looked up in the background. - applyVegvesenGroundHeights skips cameras without finite coordinates, so it really never throws. - clearKartverketHeightCache bumps a generation, so a lookup started before a clear cannot write to the cache, backoff or pending slot. - Credit and DATA_SOURCES note the modification (water set to 0 m, rounded to 0.1 m) as CC BY 4.0 asks; docs match the stop rules. - Tests for the deadline being shared, per-request timeouts, 503, oversized and non-JSON bodies, 4xx, left-out points, Retry-After end to end, overlapping refreshes, the warm-cache wait, bad coordinates, the cache-clear generation and the loader's wait budget. Co-Authored-By: Claude Opus 5.5 --- DATA_SOURCES.md | 4 +- .../945-cctv-vegvesen-kartverket-heights.md | 12 +- server/providers/cctv/constants.js | 22 +- server/providers/cctv/sources.js | 101 ++++-- src/data/cctvVegvesen.test.mjs | 311 +++++++++++++++++- src/data/dataCredits.js | 3 +- 6 files changed, 399 insertions(+), 54 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 43d92b4ca..c99ec28d1 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -64,7 +64,7 @@ How to read this: | **King County Road Services** (`services.arcgis.com`, King County GIS) | CCTV camera catalog + frames, unincorporated King County WA road network | [KCGIS Center terms](https://kingcounty.gov/en/dept/kcit/data-information-services/gis-center/about/terms-conditions-copyrights) — attribution required; public ArcGIS layer | "King County Road Services" (partner-city cameras name their owner) | | **WSDOT** (`data.wsdot.wa.gov`) | CCTV camera catalog + frames, Washington State highways | [WSDOT travel information disclaimer](https://wsdot.wa.gov/about/policies/travel-information-disclaimer) — public "low volume" feed; catalog cached, frames on WSDOT's own image host only | "Washington State Department of Transportation (Travel Center)" (courtesy) | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | -| **Kartverket** (`ws.geonorge.no/hoydedata/v1/punkt`) | Terrain heights (NN2000) used as the ground prior for Statens vegvesen cameras | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) — attribution REQUIRED | "© Kartverket" | +| **Kartverket** (`ws.geonorge.no/hoydedata/v1/punkt`) | Terrain heights (NN2000) used as the ground prior for Statens vegvesen cameras | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/) — attribution REQUIRED | "© Kartverket", modified (heights over water set to 0 m, rounded to 0.1 m) | | **NYC DOT Traffic Management Center** (`webcams.nyctmc.org`) | CCTV camera catalog + frames, New York City five boroughs | NYC DOT requires developers to sign a [data-sharing agreement](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) for the camera feed. **Off by default**; enable only under your own agreement | "NYC DOT Traffic Management Center" + NYC DOT's required application disclaimer (in the Data attribution popover) | | **City of Toronto Open Data** (`opendata.toronto.ca`) | CCTV camera catalog + frames, Toronto city streets | [Open Government Licence – Toronto](https://www.toronto.ca/city-government/data-research-maps/open-data/open-data-licence/) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – Toronto" | | **GBFS (Lyft / BCycle / Social Bicycles / PBSC)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | @@ -169,7 +169,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped because their frames are WSDOT-hosted, outside the pinned image origin; partner-city cameras hosted on the county image handler (e.g. Newcastle) name their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); both are carried on the source (and served by `/api/cctv/sources`) as `model` and `manufacturer` for identified-hardware consumers (vendor-generic strings such as `PTZ` name no model and are omitted). Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. - **WSDOT highway cameras.** Keyless. The camera list is the Esri JSON layer `data.wsdot.wa.gov/travelcenter/Cameras.json` — the same layer WSDOT's own Travel Center map consumes (~1,700 point features, EPSG:3857 geometry converted to WGS84 on load; the body carries Windows-1252 bytes and is decoded as such). Only frames on WSDOT's own `images.wsdot.wa.gov` host are registered — the catalog also lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) whose own terms apply, and those are skipped rather than proxied. WSDOT's layer metadata marks the feed for "low volume" use (see WSDOT's [travel information disclaimer](https://wsdot.wa.gov/about/policies/travel-information-disclaimer)). The catalog is fetched once per CCTV catalog refresh (every 15 minutes), and a camera's frame is fetched only when that camera is viewed; frames are never stored or redistributed. The live feed's `CompassDirection` is a single cardinal (N/E/S/W), `B` (both directions — most rows), `O`, or null. A single cardinal gives a high-confidence heading (the diagonals NE/SE/SW/NW are mapped too should they appear); `B`, `O`, null, and any other code carry no single facing and use the shared id-hash fallback at low confidence. By default the 250 cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` changes the cap (8–1,200) and `CCTV_WSDOT_ENABLED=0` turns the pack off. A keyed WSDOT Traveler Information API serves the same population for higher-volume use and is deliberately not required here. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. -- **Kartverket ground heights for Statens vegvesen cameras.** The camera feed's points are 2D, so the loader asks Kartverket's keyless point-height API (`ws.geonorge.no/hoydedata/v1/punkt`, CC BY 4.0) for the terrain height at each camera. Requests carry 20 points: the API accepts 50, but above ~20 it answers the extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not at all. Eight requests run at a time, about 36 for the whole country: a cold lookup took 6–13 s in testing (single requests 0.6–9.5 s). Answers are matched by the coordinates the API echoes and recorded only for points that request asked about. Over water the API returns the sea-floor depth (`dybdekurver` or `Havflate`), which is replaced by 0 m, the water surface; for the ~14 bridge cameras that is below the deck, but much closer than the sea floor or the flat prior. Heights are cached in memory by position for the life of the server, and points with no height (a few ferry quays and bridges) for 6 h, so the 15-minute catalog refresh only asks about new or moved cameras. The first 429, 5xx or network failure stops the lookup, with no retries, and no new lookup starts until the backoff ends (Retry-After when sent, else 10 min; bounded to 1 min–1 h). The lane waits at most 12 s, counted from the start of the loader; a slower lookup keeps filling the cache, and the cameras it had not reached keep the flat prior until the next 15-minute refresh (about 190 of 865 on one slow first run). Only coordinates are sent, with the same User-Agent as the other CCTV requests. `CCTV_VEGVESEN_HEIGHTS=0` turns the lookup off. The Kartverket credit is shown beside the Statens vegvesen entry in the Data attribution popover. +- **Kartverket ground heights for Statens vegvesen cameras.** The camera feed's points are 2D, so the loader asks Kartverket's keyless point-height API (`ws.geonorge.no/hoydedata/v1/punkt`, CC BY 4.0) for the terrain height at each camera. Requests carry 20 points: the API accepts 50, but above ~20 it answers the extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not at all. Eight requests run at a time, about 36 for the whole country: a cold lookup took 6–13 s in testing (single requests 0.6–9.5 s). Answers are matched by the coordinates the API echoes and recorded only for points that request asked about. Over water the API returns the sea-floor depth (`dybdekurver` or `Havflate`), which is replaced by 0 m, the water surface; for the ~14 bridge cameras that is below the deck, but much closer than the sea floor or the flat prior. Heights are cached in memory by position for the life of the server, and points with no height (a few ferry quays and bridges, or a point a 200 answer leaves out) for 6 h, so the 15-minute catalog refresh only asks about new or moved cameras. The first failing request (a 408, 429 or 5xx, a network error, a request over 10 s, or an unreadable or oversized body) stops the lookup, with no retries, and no new lookup starts until the backoff ends (Retry-After when sent, else 30 min, longer than the refresh; bounded to 1 min–1 h). Any other 4xx marks that request's points as having no height and the lookup carries on. The whole lookup has a 30 s deadline; a lookup cut short by it resumes on the next refresh without a backoff. The lane waits at most 12 s, counted from the start of the loader, and only when more than 10% of the cameras have no answer yet; a few new cameras or expired misses are looked up in the background. A slower lookup keeps filling the cache, and the cameras it had not reached keep the flat prior until the next 15-minute refresh (about 190 of 865 on one slow first run). Only coordinates are sent, with the same User-Agent as the other CCTV requests. `CCTV_VEGVESEN_HEIGHTS=0` turns the lookup off. The Kartverket credit, with a note that heights over water are set to 0 m and heights are rounded to 0.1 m, is shown beside the Statens vegvesen entry in the Data attribution popover. - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. diff --git a/changelog.d/945-cctv-vegvesen-kartverket-heights.md b/changelog.d/945-cctv-vegvesen-kartverket-heights.md index c6c4b41d9..1572c0316 100644 --- a/changelog.d/945-cctv-vegvesen-kartverket-heights.md +++ b/changelog.d/945-cctv-vegvesen-kartverket-heights.md @@ -7,9 +7,11 @@ time; a cold lookup takes about 6–13 s. Cameras over water use the water surface rather than the sea-floor depth the API returns there; for bridge cameras that is still below the deck. Heights are cached for good and misses - for 6 h, so a refresh only asks about new or moved cameras. After a 429, a - 5xx or a network failure the lookup stops and backs off (Retry-After, else - 10 min). The lane waits at most 12 s from its start; a slower first lookup - leaves the cameras it had not reached on the flat prior until the next - 15-minute refresh, and keeps filling the cache meanwhile. + for 6 h, so a refresh only asks about new or moved cameras. After a + failing request (a 408, 429 or 5xx, a network error, a request over 10 s or + an unreadable body) the lookup stops and backs off (Retry-After, else + 30 min, longer than the refresh). The lane waits at most 12 s from its + start, and only on a cold cache; a slower first lookup leaves the cameras it + had not reached on the flat prior until the next 15-minute refresh, and + keeps filling the cache meanwhile. `CCTV_VEGVESEN_HEIGHTS=0` turns it off. Follow-up to #929 (trygvehhh, #945). diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 553ff849f..8922e8938 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -347,22 +347,32 @@ export const KARTVERKET_HEIGHT_URL = * extra points from contour lines (`hoydekurver`, metres to ~20 m off) or not * at all; at 20 nearly every camera resolves from the 1 m terrain model. */ export const KARTVERKET_MAX_POINTS = 20; -/** Requests in flight at once. The whole country is ~36 requests of ~1-2 s, - * so a cold lookup takes ~6 s. */ +/** Requests in flight at once. The whole country is ~36 requests of + * 0.6-9.5 s each, so a cold lookup takes ~6-13 s. */ export const KARTVERKET_CONCURRENCY = 8; -/** Deadline for one complete lookup, shared by every request. */ +/** Deadline for one complete lookup, shared by every request. A lookup cut + * short by it is resumed on the next refresh, without a backoff. */ export const KARTVERKET_LOOKUP_TIMEOUT_MS = 30 * 1000; +/** One request that takes longer than this counts as a failing service. */ +export const KARTVERKET_REQUEST_TIMEOUT_MS = 10 * 1000; /** The Vegvesen lane's budget, measured from the start of the loader (catalog * fetch included), for waiting on heights. Under the 15 s per-provider fetch * timeout so this pack is never the catalog's slowest lane. A lookup still * running after it keeps filling the cache for the next refresh. */ export const KARTVERKET_CATALOG_WAIT_MS = 12 * 1000; +/** The lane only waits when more than this share of its cameras has no + * answer yet (a cold cache). A few new cameras or expired misses are looked + * up in the background instead of holding the refresh. */ +export const KARTVERKET_WAIT_MISSING_SHARE = 0.1; /** A point Kartverket has no height for (some ferry quays and bridges) is not * asked about again for this long. */ export const KARTVERKET_NULL_TTL_MS = 6 * 60 * 60 * 1000; -/** After a 429, a 5xx or a network failure, no lookup starts for this long, - * or for the Retry-After the service sent, bounded to [1 min, 1 h]. */ -export const KARTVERKET_FAILURE_BACKOFF_MS = 10 * 60 * 1000; +/** After a failing request (408, 429, 5xx, a network error, a request + * timeout or an unreadable body), no lookup starts for this long, or for the + * Retry-After the service sent, bounded to [1 min, 1 h]. Longer than the + * 15-minute catalog refresh (CCTV_SOURCE_CACHE_MS), so a failing service + * costs one attempt per backoff, not one per refresh. */ +export const KARTVERKET_FAILURE_BACKOFF_MS = 30 * 60 * 1000; export const KARTVERKET_MAX_BACKOFF_MS = 60 * 60 * 1000; export const KARTVERKET_MAX_RESPONSE_BYTES = 256 * 1024; /** Plausible Norwegian terrain: below sea level only at the shore, and no road diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 38888c7db..4964f2aa0 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -99,6 +99,8 @@ import { KARTVERKET_MAX_POINTS, KARTVERKET_CONCURRENCY, KARTVERKET_LOOKUP_TIMEOUT_MS, + KARTVERKET_REQUEST_TIMEOUT_MS, + KARTVERKET_WAIT_MISSING_SHARE, KARTVERKET_CATALOG_WAIT_MS, KARTVERKET_NULL_TTL_MS, KARTVERKET_FAILURE_BACKOFF_MS, @@ -2506,12 +2508,20 @@ const KARTVERKET_CACHE_LIMIT = 5000; let kartverketPending = null; /** No lookup starts before this time (epoch ms) after a transport failure. */ let kartverketBackoffUntil = 0; +/** Bumped by clearKartverketHeightCache, so a lookup started before a clear + * can no longer write to the cache, the backoff or the pending slot. */ +let kartverketGeneration = 0; /** Cache key for one position, at the ~0.1 m precision the request sends. */ const kartverketKey = (lon, lat) => `${lon.toFixed(6)},${lat.toFixed(6)}`; +/** Statuses that say the service, not the request, is failing. */ +const isKartverketTransportStatus = (status) => + status === 408 || status === 429 || status >= 500; + /** Forget cached heights, the backoff and any lookup in flight. Tests only. */ export function clearKartverketHeightCache() { + kartverketGeneration += 1; kartverketHeightCache.clear(); kartverketPending = null; kartverketBackoffUntil = 0; @@ -2541,26 +2551,36 @@ export function kartverketBackoffMs(retryAfter, now = Date.now()) { * matched back by the coordinates the API echoes, and only for points that * request asked about, then reported through `into.set(key, z)` as its * request lands, so a caller that stops waiting keeps what has arrived. `z` is - * metres, or null when the API has no usable height there. + * metres, or null when the API has no usable height there. A point a 200 + * answer leaves out is reported as null too, so it is not asked again on + * every refresh. * * Over water `/punkt` returns the depth of the sea floor (`datakilde: * dybdekurver`, `terreng: Havflate`; −547 m under Nordhordlandsbrua). Either * marker counts as 0 m, the water surface. For the ~14 bridge cameras that is * below the deck, but far closer than the sea floor or a flat prior. * - * The first transport failure (a non-200 answer, a network error or the + * The first transport failure (a 408, 429 or 5xx, a network error, a request + * past KARTVERKET_REQUEST_TIMEOUT_MS, an unreadable or oversized body, or the * shared deadline) stops the lookup: no further requests start, and the * failure is returned so the caller can back off. Points in failed requests - * are not reported, so they are asked again after the backoff. + * are not reported, so they are asked again later. Any other 4xx rejects that + * request's points, not the service: they are reported as null and the lookup + * carries on. * * @param {Array<{lon:number, lat:number}>} points - * @param {{into?: {set: Function}, timeoutMs?: number}} [options] - * @returns {Promise<{heights: Map, failure: ?{status: ?number, retryAfter: ?string}}>} + * @param {{into?: {set: Function}, timeoutMs?: number, requestTimeoutMs?: number}} [options] + * @returns {Promise<{heights: Map, failure: ?{status: ?number, retryAfter: ?string, deadline: boolean}}>} * `heights` is `into` when it is a Map, else a Map of what was reported. + * `failure.deadline` is true when only the shared deadline ran out. */ export async function fetchKartverketHeights( points, - { into = new Map(), timeoutMs = KARTVERKET_LOOKUP_TIMEOUT_MS } = {}, + { + into = new Map(), + timeoutMs = KARTVERKET_LOOKUP_TIMEOUT_MS, + requestTimeoutMs = KARTVERKET_REQUEST_TIMEOUT_MS, + } = {}, ) { const heights = into instanceof Map ? into : new Map(); const report = (key, z) => { @@ -2580,7 +2600,7 @@ export async function fetchKartverketHeights( for (let i = 0; i < unique.length; i += KARTVERKET_MAX_POINTS) { batches.push(unique.slice(i, i + KARTVERKET_MAX_POINTS)); } - const signal = AbortSignal.timeout(timeoutMs); + const deadline = AbortSignal.timeout(timeoutMs); const fetchBatch = async (batch) => { const asked = new Set(batch.map((p) => kartverketKey(p.lon, p.lat))); const url = new URL(KARTVERKET_HEIGHT_URL); @@ -2597,7 +2617,10 @@ export async function fetchKartverketHeights( 'User-Agent': KARTVERKET_USER_AGENT, }, redirect: 'error', - signal, + signal: AbortSignal.any([ + deadline, + AbortSignal.timeout(requestTimeoutMs), + ]), }); if (!resp.ok) { try { @@ -2605,6 +2628,13 @@ export async function fetchKartverketHeights( } catch { /* no-op */ } + if (!isKartverketTransportStatus(resp.status)) { + console.warn( + `[CCTV] Kartverket rejected ${asked.size} points: HTTP ${resp.status}`, + ); + for (const key of asked) report(key, null); + return; + } const error = new Error(`HTTP ${resp.status}`); error.status = resp.status; error.retryAfter = resp.headers.get('retry-after'); @@ -2643,6 +2673,8 @@ export async function fetchKartverketHeights( z <= KARTVERKET_HEIGHT_RANGE_M.max; report(key, usable ? z : null); } + // A 200 that leaves a point out has no height for it either. + for (const key of asked) report(key, null); }; let next = 0; const worker = async () => { @@ -2655,6 +2687,7 @@ export async function fetchKartverketHeights( failure = { status: Number.isFinite(error?.status) ? error.status : null, retryAfter: error?.retryAfter ?? null, + deadline: deadline.aborted, }; console.warn( '[CCTV] Kartverket height lookup failed:', @@ -2675,23 +2708,35 @@ export async function fetchKartverketHeights( /** * Give each Vegvesen camera its own terrain height as the ground prior, in - * place. Waits at most `waitMs` for positions it has no answer for; a lookup - * still running then keeps filling the cache for the next catalog refresh. - * Cameras without a height keep the flat prior. After a transport failure no - * lookup starts until the backoff ends, so a failing Kartverket costs one - * attempt, not one per refresh. Never throws. CCTV_VEGVESEN_HEIGHTS=0 turns it - * off. + * place. Waits at most `waitMs` for positions it has no answer for, and only + * when more than KARTVERKET_WAIT_MISSING_SHARE of the cameras are missing + * (a cold cache); a few new cameras or expired misses are looked up in the + * background. A lookup still running when the wait ends keeps filling the + * cache for the next catalog refresh. Cameras without a height keep the flat + * prior. After a transport failure no lookup starts until the backoff ends, + * which outlasts the 15-minute catalog refresh, so a failing Kartverket costs + * one attempt per backoff, not one per refresh. A lookup cut short only by its + * shared deadline is resumed on the next refresh without a backoff. Cameras + * without finite coordinates are skipped. Never throws. + * CCTV_VEGVESEN_HEIGHTS=0 turns it off. * * @param {Array} cameras - Sources from vegvesenCameraToSource(). - * @param {{waitMs?: number, now?: () => number}} [options] + * @param {{waitMs?: number, now?: () => number, timeoutMs?: number}} [options] + * `timeoutMs` is the lookup's shared deadline. * @returns {Promise} How many cameras got a terrain height. */ export async function applyVegvesenGroundHeights( cameras, - { waitMs = KARTVERKET_CATALOG_WAIT_MS, now = Date.now } = {}, + { + waitMs = KARTVERKET_CATALOG_WAIT_MS, + now = Date.now, + timeoutMs = KARTVERKET_LOOKUP_TIMEOUT_MS, + } = {}, ) { if (String(process.env.CCTV_VEGVESEN_HEIGHTS || '1').trim() === '0') return 0; - const list = Array.isArray(cameras) ? cameras : []; + const list = (Array.isArray(cameras) ? cameras : []).filter( + (c) => Number.isFinite(c?.lon) && Number.isFinite(c?.lat), + ); const startedAt = now(); const missing = list.filter((c) => { const entry = kartverketHeightCache.get(kartverketKey(c.lon, c.lat)); @@ -2705,32 +2750,40 @@ export async function applyVegvesenGroundHeights( if (kartverketHeightCache.size + missing.length > KARTVERKET_CACHE_LIMIT) { kartverketHeightCache.clear(); } + const generation = kartverketGeneration; + const current = () => generation === kartverketGeneration; const sink = { - set: (key, z) => + set: (key, z) => { + if (!current()) return; kartverketHeightCache.set(key, { z, expires: z === null ? now() + KARTVERKET_NULL_TTL_MS : Infinity, - }), + }); + }, }; - kartverketPending = fetchKartverketHeights(missing, { into: sink }) + const pending = fetchKartverketHeights(missing, { into: sink, timeoutMs }) .then(({ failure }) => { - if (failure) { + if (failure && !failure.deadline && current()) { kartverketBackoffUntil = now() + kartverketBackoffMs(failure.retryAfter, now()); } }) .catch((error) => { - kartverketBackoffUntil = now() + KARTVERKET_FAILURE_BACKOFF_MS; + if (current()) { + kartverketBackoffUntil = now() + KARTVERKET_FAILURE_BACKOFF_MS; + } console.warn( '[CCTV] Kartverket height lookup error:', error?.message || error, ); }) .finally(() => { - kartverketPending = null; + if (kartverketPending === pending) kartverketPending = null; }); + kartverketPending = pending; } - if (missing.length && kartverketPending && waitMs > 0) { + const cold = missing.length > list.length * KARTVERKET_WAIT_MISSING_SHARE; + if (cold && kartverketPending && waitMs > 0) { let timer; await Promise.race([ kartverketPending, diff --git a/src/data/cctvVegvesen.test.mjs b/src/data/cctvVegvesen.test.mjs index 47ae732d1..3025f0d91 100644 --- a/src/data/cctvVegvesen.test.mjs +++ b/src/data/cctvVegvesen.test.mjs @@ -12,11 +12,14 @@ import { import { DEFAULT_VEGVESEN_CCTV_URL, KARTVERKET_HEIGHT_URL, + KARTVERKET_CATALOG_WAIT_MS, KARTVERKET_CONCURRENCY, KARTVERKET_FAILURE_BACKOFF_MS, KARTVERKET_MAX_POINTS, + KARTVERKET_MAX_RESPONSE_BYTES, KARTVERKET_NULL_TTL_MS, KARTVERKET_USER_AGENT, + CCTV_SOURCE_CACHE_MS, VEGVESEN_IMAGE_ORIGIN, } from '../../server/providers/cctv/constants.js'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; @@ -141,9 +144,9 @@ test('the loader dedupes, and failures degrade to an empty pack', async (t) => { const cameras = await loadVegvesenSourcesFromOpenData(); // The catalog first; the rest is the Kartverket height lookup. assert.equal(requested[0], DEFAULT_VEGVESEN_CCTV_URL); - assert.ok( - requested.slice(1).every((url) => url.startsWith(KARTVERKET_HEIGHT_URL)), - ); + // Two unique positions: one Kartverket request. + assert.equal(requested.length, 2); + assert.ok(requested[1].startsWith(KARTVERKET_HEIGHT_URL)); // Nearest-to-anchor first: Oslo leads. assert.deepEqual( cameras.map((c) => c.id), @@ -258,6 +261,8 @@ test('Kartverket heights come in batches of 20, matched by echoed coordinates', requested.push(String(url)); assert.ok(String(url).startsWith(KARTVERKET_HEIGHT_URL)); assert.equal(init.headers['User-Agent'], KARTVERKET_USER_AGENT); + // Redirects are refused, so a request can never leave the fixed host. + assert.equal(init.redirect, 'error'); // Reply in reverse order: matching must not depend on list position. const response = kartverketResponse(url, (x) => x > 7.04 ? null : Math.round((x - 7) * 10000), @@ -402,26 +407,138 @@ test('a transport failure stops the lookup: no retries, no new requests', async // after it starts, and no point is resent. assert.equal(calls, KARTVERKET_CONCURRENCY); assert.equal(heights.size, 0); - assert.deepEqual(failure, { status: 429, retryAfter: '120' }); + assert.deepEqual(failure, { + status: 429, + retryAfter: '120', + deadline: false, + }); }); +/** A fetch that never answers, and rejects only when its signal aborts. */ +const hangingFetch = (url, init) => { + assert.ok(init.signal instanceof AbortSignal); + return new Promise((_, reject) => { + init.signal.addEventListener('abort', () => reject(init.signal.reason)); + }); +}; + test('the lookup deadline bounds a hanging service', async (t) => { withHeightsEnv(t); - t.mock.method( - globalThis, - 'fetch', - (url, init) => - new Promise((_, reject) => { - init.signal.addEventListener('abort', () => reject(init.signal.reason)); - }), - ); + t.mock.method(globalThis, 'fetch', hangingFetch); const started = Date.now(); const { heights, failure } = await fetchKartverketHeights([cameraAt(0)], { timeoutMs: 30, }); - assert.ok(Date.now() - started < 2000); + const elapsed = Date.now() - started; + assert.ok(elapsed >= 25 && elapsed < 2000, `elapsed ${elapsed} ms`); assert.equal(heights.size, 0); - assert.equal(failure.status, null); + assert.deepEqual(failure, { status: null, retryAfter: null, deadline: true }); +}); + +test('one hanging request times out on its own, as a failing service', async (t) => { + withHeightsEnv(t); + t.mock.method(globalThis, 'fetch', hangingFetch); + const { failure } = await fetchKartverketHeights([cameraAt(0)], { + timeoutMs: 60_000, + requestTimeoutMs: 20, + }); + assert.deepEqual(failure, { + status: null, + retryAfter: null, + deadline: false, + }); +}); + +test('the deadline is shared: later requests do not get a fresh one', async (t) => { + withHeightsEnv(t); + let calls = 0; + t.mock.method(globalThis, 'fetch', (url, init) => { + calls += 1; + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => resolve(kartverketResponse(url, () => 1)), + 40, + ); + init.signal.addEventListener('abort', () => { + clearTimeout(timer); + reject(init.signal.reason); + }); + }); + }); + // Five waves of 40 ms against a 100 ms deadline for the whole lookup. + const { heights, failure } = await fetchKartverketHeights( + Array.from({ length: 5 * KARTVERKET_CONCURRENCY * 20 }, (_, i) => + cameraAt(i), + ), + { timeoutMs: 100 }, + ); + assert.equal(failure?.deadline, true); + assert.ok(heights.size < 5 * KARTVERKET_CONCURRENCY * 20); + assert.ok(calls < 5 * KARTVERKET_CONCURRENCY, `calls ${calls}`); +}); + +test('a 503, an unreadable or an oversized body stops the lookup', async (t) => { + withHeightsEnv(t); + const answers = { + 503: () => new Response('down', { status: 503 }), + 'not json': () => + new Response('maintenance', { + headers: { 'Content-Type': 'text/html' }, + }), + oversized: () => + Response.json({ + punkter: [], + pad: 'x'.repeat(KARTVERKET_MAX_RESPONSE_BYTES + 1), + }), + }; + for (const [label, answer] of Object.entries(answers)) { + clearKartverketHeightCache(); + let calls = 0; + t.mock.method(globalThis, 'fetch', async () => { + calls += 1; + return answer(); + }); + const { heights, failure } = await fetchKartverketHeights( + Array.from({ length: 400 }, (_, i) => cameraAt(i)), + ); + assert.equal(calls, KARTVERKET_CONCURRENCY, label); + assert.equal(heights.size, 0, label); + assert.equal(failure?.deadline, false, label); + assert.equal(failure?.status, label === '503' ? 503 : null, label); + } +}); + +test('a 4xx rejects that request only: its points get no height, the rest go on', async (t) => { + withHeightsEnv(t); + t.mock.method(globalThis, 'fetch', async (url) => + sentPoints(url)[0][0] === 7 + ? new Response('bad point', { status: 400 }) + : kartverketResponse(url, () => 42), + ); + const { heights, failure } = await fetchKartverketHeights( + Array.from({ length: 40 }, (_, i) => cameraAt(i)), + ); + assert.equal(failure, null); + assert.equal(heights.size, 40); + assert.equal(heights.get('7.000000,60.000000'), null); + assert.equal(heights.get('7.020000,60.020000'), 42); +}); + +test('a point a 200 answer leaves out is cached as no height', async (t) => { + withHeightsEnv(t); + const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => { + const points = sentPoints(url).slice(1); // drop the first point + return Response.json({ + koordsys: 4258, + punkter: points.map(([x, y]) => ({ x, y, z: 7 })), + }); + }); + const cameras = [cameraAt(0), cameraAt(1)]; + assert.equal(await applyVegvesenGroundHeights(cameras), 1); + assert.equal(cameras[0].groundElevationM, 150); + // The next refresh asks nothing: the left-out point is a cached miss. + assert.equal(await applyVegvesenGroundHeights([cameraAt(0), cameraAt(1)]), 1); + assert.equal(fetchMock.mock.callCount(), 1); }); test('Retry-After sets the backoff, bounded to [1 min, 1 h]', () => { @@ -478,8 +595,12 @@ test('after a failure no lookup starts until the backoff ends', async (t) => { assert.equal(cameras[0].groundElevationM, 150); assert.equal(fetchMock.mock.callCount(), 1); - // Refreshes inside the backoff cost nothing. - clock += KARTVERKET_FAILURE_BACKOFF_MS - 1; + // The default backoff outlasts the catalog refresh, so the next refresh + // and every one inside the backoff cost nothing. + assert.ok(KARTVERKET_FAILURE_BACKOFF_MS > CCTV_SOURCE_CACHE_MS); + clock += CCTV_SOURCE_CACHE_MS; + assert.equal(await applyVegvesenGroundHeights([cameraAt(0)], { now }), 0); + clock += KARTVERKET_FAILURE_BACKOFF_MS - CCTV_SOURCE_CACHE_MS - 1; assert.equal(await applyVegvesenGroundHeights([cameraAt(0)], { now }), 0); assert.equal(fetchMock.mock.callCount(), 1); @@ -494,6 +615,143 @@ test('after a failure no lookup starts until the backoff ends', async (t) => { assert.equal(fetchMock.mock.callCount(), 2); }); +test('a 429 backs off for its Retry-After, not the default', async (t) => { + withHeightsEnv(t); + let clock = 1_000_000; + const now = () => clock; + const retryAfterS = 50 * 60; + const fetchMock = t.mock.method( + globalThis, + 'fetch', + async () => + new Response('busy', { + status: 429, + headers: { 'Retry-After': String(retryAfterS) }, + }), + ); + await applyVegvesenGroundHeights([cameraAt(0)], { now }); + assert.equal(fetchMock.mock.callCount(), 1); + clock += KARTVERKET_FAILURE_BACKOFF_MS + 1; + await applyVegvesenGroundHeights([cameraAt(0)], { now }); + assert.equal(fetchMock.mock.callCount(), 1); + clock += retryAfterS * 1000 - KARTVERKET_FAILURE_BACKOFF_MS; + await applyVegvesenGroundHeights([cameraAt(0)], { now }); + assert.equal(fetchMock.mock.callCount(), 2); +}); + +test('a lookup cut short by its deadline resumes on the next refresh', async (t) => { + withHeightsEnv(t); + let clock = 1_000_000; + const now = () => clock; + t.mock.method(globalThis, 'fetch', hangingFetch); + // The lookup runs out of time; that is not a failing service. + assert.equal( + await applyVegvesenGroundHeights([cameraAt(0)], { + now, + timeoutMs: 10, + waitMs: 1000, + }), + 0, + ); + const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => + kartverketResponse(url, () => 64), + ); + clock += CCTV_SOURCE_CACHE_MS; + const cameras = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(cameras, { now }), 1); + assert.equal(cameras[0].groundElevationM, 64); + assert.equal(fetchMock.mock.callCount(), 1); +}); + +test('overlapping refreshes share one lookup', async (t) => { + withHeightsEnv(t); + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => { + await gate; + return kartverketResponse(url, () => 3); + }); + await Promise.all([ + applyVegvesenGroundHeights([cameraAt(0)], { waitMs: 5 }), + applyVegvesenGroundHeights([cameraAt(0)], { waitMs: 5 }), + ]); + assert.equal(fetchMock.mock.callCount(), 1); + release(); +}); + +test('a warm cache does not wait for a few new cameras', async (t) => { + withHeightsEnv(t); + const cameras = Array.from({ length: 20 }, (_, i) => cameraAt(i)); + t.mock.method(globalThis, 'fetch', async (url) => + kartverketResponse(url, () => 500), + ); + assert.equal(await applyVegvesenGroundHeights(cameras), 20); + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => { + await gate; + return kartverketResponse(url, () => 600); + }); + const refresh = [...cameras, cameraAt(20)]; + const started = Date.now(); + assert.equal( + await applyVegvesenGroundHeights(refresh, { waitMs: 5_000 }), + 20, + ); + assert.ok(Date.now() - started < 1000); + assert.equal(fetchMock.mock.callCount(), 1); + // The background lookup fills the cache for the next refresh. + release(); + await new Promise((resolve) => setTimeout(resolve, 20)); + const next = [...cameras, cameraAt(20)]; + assert.equal(await applyVegvesenGroundHeights(next), 21); + assert.equal(next[20].groundElevationM, 600); +}); + +test('cameras without numeric coordinates are skipped, never thrown on', async (t) => { + withHeightsEnv(t); + t.mock.method(globalThis, 'fetch', async (url) => + kartverketResponse(url, () => 9), + ); + const cameras = [ + { id: 'no-coordinates' }, + null, + { id: 'strings', lat: '60', lon: '7' }, + cameraAt(0), + ]; + assert.equal(await applyVegvesenGroundHeights(cameras), 1); + assert.equal(cameras[3].groundElevationM, 9); +}); + +test('a lookup started before a cache clear cannot write after it', async (t) => { + withHeightsEnv(t); + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + t.mock.method(globalThis, 'fetch', async (url) => { + await gate; + return kartverketResponse(url, () => 111); + }); + await applyVegvesenGroundHeights([cameraAt(0)], { waitMs: 5 }); + clearKartverketHeightCache(); + t.mock.method(globalThis, 'fetch', async (url) => + kartverketResponse(url, () => 222), + ); + const fresh = [cameraAt(0)]; + assert.equal(await applyVegvesenGroundHeights(fresh), 1); + assert.equal(fresh[0].groundElevationM, 222); + release(); + await new Promise((resolve) => setTimeout(resolve, 20)); + const later = [cameraAt(0)]; + await applyVegvesenGroundHeights(later, { waitMs: 0 }); + assert.equal(later[0].groundElevationM, 222); +}); + test('CCTV_VEGVESEN_HEIGHTS=0 skips the lookup', async (t) => { withHeightsEnv(t); const fetchMock = t.mock.method(globalThis, 'fetch', async (url) => @@ -541,3 +799,24 @@ test('the loader applies Kartverket heights to the cameras it returns', async (t [['no-vegvesen-3000063_1', 987.6]], ); }); + +test('the loader budgets the height wait from its own start', async (t) => { + withHeightsEnv(t); + const realNow = Date.now.bind(Date); + let offset = 0; + t.mock.method(Date, 'now', () => realNow() + offset); + t.mock.method(globalThis, 'fetch', (url, init) => { + if (String(url).startsWith(KARTVERKET_HEIGHT_URL)) { + return hangingFetch(url, init); + } + // The catalog fetch uses up all but 10 ms of the lane's budget. + offset += KARTVERKET_CATALOG_WAIT_MS - 10; + return Promise.resolve( + Response.json({ type: 'FeatureCollection', features: [feature()] }), + ); + }); + const started = performance.now(); + const cameras = await loadVegvesenSourcesFromOpenData(); + assert.ok(performance.now() - started < 1000); + assert.equal(cameras[0].groundElevationM, 150); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index f1aea1b74..b01492d9b 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -295,7 +295,8 @@ export const DATA_CREDITS = [ 'Norwegian licence for Open Government data (NLOD) ' + 'distributed by Statens vegvesen · camera ground heights © ' + 'Kartverket ' + - '(CC BY 4.0)', + '(CC BY 4.0); ' + + 'modified: heights over water set to 0 m and rounded to 0.1 m', }, { key: 'nyc-cctv', From 194a547955816aed49d2dcc45511bb6c5a5f45e9 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Mon, 19 Oct 2026 13:51:07 -0700 Subject: [PATCH 16/25] docs(changelog): match fragment style for #1220 --- changelog.d/1220-openai-typed-send.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/1220-openai-typed-send.md b/changelog.d/1220-openai-typed-send.md index c75e0c99c..cd1b5b470 100644 --- a/changelog.d/1220-openai-typed-send.md +++ b/changelog.d/1220-openai-typed-send.md @@ -1,3 +1,3 @@ -- avoid starting or queuing a reply when the local channel rejects a typed OpenAI +- Avoid starting or queuing a reply when the local channel rejects a typed OpenAI command, while preserving an earlier accepted deferred input - (@manjunath22466, #1220) + (Manjunath, #1220). From 05f668dfa7c6aea521f0b38d60f8158006c33bb0 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Wed, 14 Oct 2026 13:08:35 +0100 Subject: [PATCH 17/25] fix(qa-cctv-v2): keep long headful waits under puppeteer's protocolTimeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The headful drain budget is uncapped (camCount * 800 + 30000 ms, ~50 min at N≈3,700), but each page.waitForFunction is one CDP call that puppeteer aborts after its 180 s default protocolTimeout. The .catch then recorded "timed out waiting for drain on the real-GPU sign-off path" after ~3 min. - Poll the drain in <=120 s slices (waitForFunctionSliced) so no single CDP call outlives protocolTimeout, whatever the catalog size. - Launch with protocolTimeout 240 s, headroom above the largest slice and the 60 s fixed waits. - Bound the sibling-owner 3-frame postRender wait at 10 s and report the frame count, so a renderer that stops producing frames fails with evidence instead of hanging the evaluate. - List --disable-backgrounding-occluded-windows explicitly. Puppeteer 25.10 already passes it by default, so this documents the headful contract rather than changing behaviour. Co-Authored-By: Claude Opus 5.5 --- scripts/qa-cctv-v2.mjs | 76 +++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 16 deletions(-) diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index 08995603f..9234093be 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -139,6 +139,41 @@ function record(name, ok, detail) { const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +// Every page.waitForFunction / page.evaluate is a single CDP call that +// puppeteer aborts after `protocolTimeout` (default 180s). The headful drain +// budget is uncapped and reaches ~50 minutes on the full ~3,900-camera +// catalog, so long waits are polled in slices of at most WAIT_SLICE_MS and +// protocolTimeout is set with headroom above the largest slice. +const WAIT_SLICE_MS = 120000; +const PROTOCOL_TIMEOUT_MS = 2 * WAIT_SLICE_MS; + +/** + * page.waitForFunction with an overall budget that may exceed + * protocolTimeout: waits in WAIT_SLICE_MS slices until the predicate holds or + * the budget runs out. + * @param {import('puppeteer').Page} page + * @param {Function} predicate - In-page predicate. + * @param {number} budgetMs - Overall budget. + * @returns {Promise} True if the predicate held within the budget. + */ +async function waitForFunctionSliced(page, predicate, budgetMs) { + const deadline = Date.now() + budgetMs; + for (;;) { + const remaining = deadline - Date.now(); + if (remaining <= 0) return false; + const ok = await page + .waitForFunction(predicate, { + timeout: Math.min(remaining, WAIT_SLICE_MS), + }) + .then(() => true) + .catch((err) => { + if (err?.name === 'TimeoutError') return false; + throw err; + }); + if (ok) return true; + } +} + // SwiftShader can spend tens of seconds servicing the hover drillPick emitted // by page.mouse.move(), before the LEFT_DOWN under test even runs. Position the // pointer while ADJUST is disabled, then re-enable it and explicitly render a @@ -364,6 +399,7 @@ async function main() { const chromeExecutable = findChromeExecutable(); const browser = await puppeteer.launch({ headless: HEADFUL ? false : 'new', + protocolTimeout: PROTOCOL_TIMEOUT_MS, ...(chromeExecutable ? { executablePath: chromeExecutable } : {}), args: [ '--no-sandbox', @@ -373,6 +409,9 @@ async function main() { '--disable-web-security', '--disable-background-timer-throttling', '--disable-renderer-backgrounding', + // Puppeteer's defaults already include this; listed so the headful + // run keeps rendering when covered even if those defaults change. + '--disable-backgrounding-occluded-windows', '--window-size=1280,800', ], }); @@ -475,18 +514,15 @@ async function main() { console.log( `Waiting for geometry-load queue to drain (N=${camCount}, budget ${Math.round(drainBudgetMs / 1000)}s)...`, ); - const drained = await page - .waitForFunction( - () => { - const mod = - window.__godsEyeView.dataManager.layers.get('cctv').module; - const ui = mod.getUIState(); - return ui.loading && ui.loading.active === false; - }, - { timeout: drainBudgetMs }, - ) - .then(() => true) - .catch(() => false); + const drained = await waitForFunctionSliced( + page, + () => { + const mod = window.__godsEyeView.dataManager.layers.get('cctv').module; + const ui = mod.getUIState(); + return ui.loading && ui.loading.active === false; + }, + drainBudgetMs, + ); record( `geometry-load queue drains within ${Math.round(drainBudgetMs / 1000)}s (N=${camCount})`, drained ? true : HEADFUL ? false : null, @@ -1358,16 +1394,23 @@ async function main() { }, }); window.__qaCctvSiblingOwner = owner; - await new Promise((resolve) => { + // Bounded: if the browser stops producing frames (headful window + // asleep or hidden), report it instead of hanging the CDP call. + const framesRendered = await new Promise((resolve) => { let frames = 0; + const finish = () => { + clearTimeout(timer); + remove(); + resolve(frames); + }; const remove = scene.postRender.addEventListener(() => { if (++frames < 3) { scene.requestRender(); return; } - remove(); - resolve(); + finish(); }); + const timer = setTimeout(finish, 10000); scene.requestRender(); }); const rect = scene.canvas.getBoundingClientRect(); @@ -1382,6 +1425,7 @@ async function main() { typeof pickedId === 'string' ? pickedId : (pickedId?.id ?? null), ownerId: owner.id, ownsExactObject: picked?.id === owner, + framesRendered, projectionDelta: Math.hypot( projected.x - canvasPoint.x, projected.y - canvasPoint.y, @@ -1395,7 +1439,7 @@ async function main() { siblingTarget.ownsExactObject : null, siblingTarget - ? `picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta}` + ? `picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta} frames=${siblingTarget.framesRendered}/3` : 'no previously verified empty canvas ray was available', ); if ( From 474d44c4db534f0133a7189745d100b2b10e352c Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Mon, 19 Oct 2026 16:57:06 +0100 Subject: [PATCH 18/25] fix(qa-cctv-v2): report a stalled renderer apart from a missed sibling pick When the sibling-owner pick misses after fewer than 3 rendered frames, the cause is a stalled renderer, not a pick regression. Record it as a failure on the headful sign-off path and as inconclusive under headless GL, with "renderer stalled: N/3 frames" in the detail. A missed pick after 3 frames still fails in both modes. Also corrects the catalog-size comment (~3,700). Co-Authored-By: Claude Opus 5.5 --- scripts/qa-cctv-v2.mjs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index 9234093be..b291c54f4 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -141,7 +141,7 @@ const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); // Every page.waitForFunction / page.evaluate is a single CDP call that // puppeteer aborts after `protocolTimeout` (default 180s). The headful drain -// budget is uncapped and reaches ~50 minutes on the full ~3,900-camera +// budget is uncapped and reaches ~50 minutes on the full ~3,700-camera // catalog, so long waits are polled in slices of at most WAIT_SLICE_MS and // protocolTimeout is set with headroom above the largest slice. const WAIT_SLICE_MS = 120000; @@ -1432,20 +1432,28 @@ async function main() { ), }; }, emptyClickPoint); + const siblingOwnsPick = + siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && + siblingTarget.ownsExactObject; + // A missed pick after fewer than 3 frames means the renderer stalled, not + // that picking regressed: fail it on the real-GPU sign-off path, leave it + // inconclusive under headless GL. + const siblingRendererStalled = + siblingTarget && !siblingOwnsPick && siblingTarget.framesRendered < 3; record( 'sibling test object owns its canvas pick', + !siblingTarget + ? null + : siblingRendererStalled + ? HEADFUL + ? false + : null + : siblingOwnsPick, siblingTarget - ? siblingTarget.pickedId === 'qa-cctv-sibling-owner' && - siblingTarget.ownsExactObject - : null, - siblingTarget - ? `picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta} frames=${siblingTarget.framesRendered}/3` + ? `${siblingRendererStalled ? `renderer stalled: ${siblingTarget.framesRendered}/3 frames in 10s; ` : ''}picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta} frames=${siblingTarget.framesRendered}/3` : 'no previously verified empty canvas ray was available', ); - if ( - siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && - siblingTarget.ownsExactObject - ) { + if (siblingOwnsPick) { await page.mouse.click(siblingTarget.x, siblingTarget.y); await sleep(250); } @@ -1470,8 +1478,7 @@ async function main() { }); record( 'sibling canvas click passes through without CCTV selection or deselection', - siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && - siblingTarget.ownsExactObject + siblingOwnsPick ? siblingClick.activeId === clickEvidenceSetup.activeId && siblingClick.transitions === siblingClick.baselineTransitions && siblingClick.focusEvents === 0 && From 12b8fd0c1d25ad4d402ee7a4004f745dff7c730f Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Mon, 19 Oct 2026 18:59:51 +0100 Subject: [PATCH 19/25] fix(qa-cctv-v2): poll waits from Node so a hidden window can't stall them page.waitForFunction polls on requestAnimationFrame by default. In a minimised or hidden headful window rAF stops while timers keep running, so each 120 s slice only checked the predicate when it started: a drain that finished in the last slice was recorded as a FAIL on the sign-off path. - waitForPredicate replaces waitForFunctionSliced and the other three waitForFunction calls: one short page.evaluate per tick (1 s for the drain, 250 ms elsewhere), a monotonic performance.now() deadline, retry across a context swap, and a throwing predicate fails the run instead of silently spending the budget. - One in-page window.__qaRenderFrames(count, capMs) helper serves both the gizmo-drag and the sibling-owner frame waits. - The sibling pick counts as a stalled renderer only at 0 of 3 frames, so a slow SwiftShader frame still reports a real miss; the record flattens and prints the frame count once. - Review screenshots log and continue when a capture fails. - Launch with pipe: true (no localhost CDP port) and protocolTimeout 300_000 like the other harnesses; drop the occlusion flag that puppeteer already passes by default. Co-Authored-By: Claude Opus 5.5 --- scripts/qa-cctv-v2.mjs | 252 ++++++++++++++++++++++------------------- 1 file changed, 135 insertions(+), 117 deletions(-) diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index b291c54f4..2c121e0d6 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -139,38 +139,53 @@ function record(name, ok, detail) { const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); -// Every page.waitForFunction / page.evaluate is a single CDP call that -// puppeteer aborts after `protocolTimeout` (default 180s). The headful drain -// budget is uncapped and reaches ~50 minutes on the full ~3,700-camera -// catalog, so long waits are polled in slices of at most WAIT_SLICE_MS and -// protocolTimeout is set with headroom above the largest slice. -const WAIT_SLICE_MS = 120000; -const PROTOCOL_TIMEOUT_MS = 2 * WAIT_SLICE_MS; +// Each page.evaluate is bounded by puppeteer's `protocolTimeout` (default +// 180s). Waits never come close: they poll from Node (below). The headroom is +// for the few long evaluates, such as enabling the full ~3,700-camera catalog +// on a real GPU, and matches the other qa-*.mjs harnesses. +const PROTOCOL_TIMEOUT_MS = 300_000; /** - * page.waitForFunction with an overall budget that may exceed - * protocolTimeout: waits in WAIT_SLICE_MS slices until the predicate holds or - * the budget runs out. + * Polls an in-page predicate from Node, one short page.evaluate per tick, + * until it holds or the budget runs out. Used instead of page.waitForFunction + * for two reasons: its default rAF polling stops in a minimised or hidden + * headful window (timers keep running, rAF does not), and its result is one + * CDP call that protocolTimeout cuts off, while the headful drain budget + * reaches ~50 minutes. A predicate that throws fails the run instead of + * silently spending the budget. * @param {import('puppeteer').Page} page * @param {Function} predicate - In-page predicate. - * @param {number} budgetMs - Overall budget. + * @param {object} opts + * @param {number} opts.timeoutMs - Overall budget. + * @param {number} [opts.intervalMs=250] - Delay between checks. + * @param {unknown[]} [opts.args=[]] - Arguments passed to the predicate. + * The predicate must return a boolean: its result crosses CDP by value. * @returns {Promise} True if the predicate held within the budget. */ -async function waitForFunctionSliced(page, predicate, budgetMs) { - const deadline = Date.now() + budgetMs; +async function waitForPredicate( + page, + predicate, + { timeoutMs, intervalMs = 250, args = [] }, +) { + // performance.now() is monotonic and doesn't advance while the machine + // sleeps, so neither a clock change nor a laptop sleep eats the budget. + const deadline = performance.now() + timeoutMs; for (;;) { - const remaining = deadline - Date.now(); - if (remaining <= 0) return false; - const ok = await page - .waitForFunction(predicate, { - timeout: Math.min(remaining, WAIT_SLICE_MS), - }) - .then(() => true) - .catch((err) => { - if (err?.name === 'TimeoutError') return false; + try { + if (await page.evaluate(predicate, ...args)) return true; + } catch (err) { + // A navigation or Vite reload replaces the execution context: poll again. + if ( + !/Execution context was destroyed|Cannot find context/.test( + err?.message ?? '', + ) + ) { throw err; - }); - if (ok) return true; + } + } + const remaining = deadline - performance.now(); + if (remaining <= 0) return false; + await sleep(Math.min(intervalMs, remaining)); } } @@ -190,29 +205,58 @@ async function positionPointerForGizmoDrag(page, point, frameCount = 3) { gev.dataManager.layers .get('cctv') .module.setParams({ calibrationMode: true }); - const scene = gev.viewer.scene; - let rendered = 0; - return new Promise((resolve) => { - let settled = false; - const finish = (complete) => { - if (settled) return; - settled = true; - clearTimeout(timer); - remove(); - resolve(complete); - }; - const remove = scene.postRender.addEventListener(() => { - rendered += 1; - if (rendered >= count) { - finish(true); - return; - } + return (await window.__qaRenderFrames(count, 5000)) >= count; + }, frameCount); +} + +/** + * Saves a screenshot for human review. These are never a gate, so a capture + * that fails (for example when the renderer stalls in a hidden headful window) + * is logged instead of ending the run. + * @param {import('puppeteer').Page} page + * @param {string} file + * @param {string} label + */ +async function saveReviewShot(page, file, label) { + try { + await page.screenshot({ path: file }); + console.log(` screenshot (${label}) → ${path.relative(REPO_ROOT, file)}`); + } catch (err) { + console.warn( + ` screenshot skipped (${label}): ${path.relative(REPO_ROOT, file)}: ${err?.message ?? err}`, + ); + } +} + +/** + * Installs window.__qaRenderFrames(count, capMs) in the page: requests renders + * until `count` frames have passed scene.postRender or capMs runs out, and + * resolves with the number of frames rendered. The cap keeps a stalled + * renderer (headful window asleep or hidden) from hanging the CDP call. + * @param {import('puppeteer').Page} page + */ +function installRenderFramesHelper(page) { + return page.evaluate(() => { + window.__qaRenderFrames = (count, capMs) => + new Promise((resolve) => { + const scene = window.__godsEyeView.viewer.scene; + let frames = 0; + const finish = () => { + clearTimeout(timer); + remove(); + resolve(frames); + }; + const remove = scene.postRender.addEventListener(() => { + if (++frames >= count) { + finish(); + return; + } + scene.requestRender(); + }); + const timer = setTimeout(finish, capMs); scene.requestRender(); }); - const timer = setTimeout(() => finish(false), 5000); - scene.requestRender(); - }); - }, frameCount); + }); } // In-page helper (stringified and re-defined inside each evaluate that needs @@ -315,23 +359,21 @@ const BEARING_SHIFT_SRC = ` * @returns {Promise} True if tilesLoaded was observed within the timeout. */ function waitForTilesLoaded(page, timeoutMs = 15000) { - return page - .waitForFunction( - () => { - const scene = window.__godsEyeView.viewer.scene; - const prims = scene.primitives; - for (let i = 0; i < prims.length; i++) { - const p = prims.get(i); - if (p && p.constructor && p.constructor.name === 'Cesium3DTileset') { - return p.tilesLoaded === true; - } + return waitForPredicate( + page, + () => { + const scene = window.__godsEyeView.viewer.scene; + const prims = scene.primitives; + for (let i = 0; i < prims.length; i++) { + const p = prims.get(i); + if (p && p.constructor && p.constructor.name === 'Cesium3DTileset') { + return p.tilesLoaded === true; } - return true; // no tileset in the scene — nothing to wait for - }, - { timeout: timeoutMs }, - ) - .then(() => true) - .catch(() => false); + } + return true; // no tileset in the scene — nothing to wait for + }, + { timeoutMs }, + ); } /** @@ -400,6 +442,9 @@ async function main() { const browser = await puppeteer.launch({ headless: HEADFUL ? false : 'new', protocolTimeout: PROTOCOL_TIMEOUT_MS, + // Talk CDP over a pipe, so no debugging port stays open on localhost for + // the length of a headful run. + pipe: true, ...(chromeExecutable ? { executablePath: chromeExecutable } : {}), args: [ '--no-sandbox', @@ -409,9 +454,6 @@ async function main() { '--disable-web-security', '--disable-background-timer-throttling', '--disable-renderer-backgrounding', - // Puppeteer's defaults already include this; listed so the headful - // run keeps rendering when covered even if those defaults change. - '--disable-backgrounding-occluded-windows', '--window-size=1280,800', ], }); @@ -436,13 +478,18 @@ async function main() { }); console.log('Loading app...'); await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); - await page.waitForFunction( + const appReady = await waitForPredicate( + page, () => - window.__godsEyeView && - window.__godsEyeView.viewer && - window.__godsEyeView.dataManager, - { timeout: 60000 }, + Boolean( + window.__godsEyeView && + window.__godsEyeView.viewer && + window.__godsEyeView.dataManager, + ), + { timeoutMs: 60000 }, ); + if (!appReady) throw new Error('app did not initialise within 60s'); + await installRenderFramesHelper(page); // Let the initial fly-to Austin and first tiles settle. await sleep(4000); @@ -514,14 +561,14 @@ async function main() { console.log( `Waiting for geometry-load queue to drain (N=${camCount}, budget ${Math.round(drainBudgetMs / 1000)}s)...`, ); - const drained = await waitForFunctionSliced( + const drained = await waitForPredicate( page, () => { const mod = window.__godsEyeView.dataManager.layers.get('cctv').module; const ui = mod.getUIState(); - return ui.loading && ui.loading.active === false; + return Boolean(ui.loading) && ui.loading.active === false; }, - drainBudgetMs, + { timeoutMs: drainBudgetMs, intervalMs: 1000 }, ); record( `geometry-load queue drains within ${Math.round(drainBudgetMs / 1000)}s (N=${camCount})`, @@ -615,14 +662,11 @@ async function main() { const tilesSeenLoaded = await waitForTilesLoaded(page, 45000); let sampleFloorOk = false; if (tilesSeenLoaded) { - sampleFloorOk = await page - .waitForFunction( - (base) => window.__qaCounters.sampleHeight - base >= 1, - { timeout: 30000 }, - c0.sampleHeight, - ) - .then(() => true) - .catch(() => false); + sampleFloorOk = await waitForPredicate( + page, + (base) => window.__qaCounters.sampleHeight - base >= 1, + { timeoutMs: 30000, args: [c0.sampleHeight] }, + ); } const cFloor = await readCounters(); record( @@ -1394,25 +1438,7 @@ async function main() { }, }); window.__qaCctvSiblingOwner = owner; - // Bounded: if the browser stops producing frames (headful window - // asleep or hidden), report it instead of hanging the CDP call. - const framesRendered = await new Promise((resolve) => { - let frames = 0; - const finish = () => { - clearTimeout(timer); - remove(); - resolve(frames); - }; - const remove = scene.postRender.addEventListener(() => { - if (++frames < 3) { - scene.requestRender(); - return; - } - finish(); - }); - const timer = setTimeout(finish, 10000); - scene.requestRender(); - }); + const framesRendered = await window.__qaRenderFrames(3, 10000); const rect = scene.canvas.getBoundingClientRect(); const picked = scene.pick(canvasPoint); const pickedId = picked?.id?.id ?? picked?.id ?? null; @@ -1435,22 +1461,20 @@ async function main() { const siblingOwnsPick = siblingTarget?.pickedId === 'qa-cctv-sibling-owner' && siblingTarget.ownsExactObject; - // A missed pick after fewer than 3 frames means the renderer stalled, not - // that picking regressed: fail it on the real-GPU sign-off path, leave it - // inconclusive under headless GL. + // No frame at all within the 10s cap points at a stalled renderer rather + // than at picking, though the harness can't rule out both. Fail it on the + // real-GPU sign-off path; leave it inconclusive under headless GL. A slow + // SwiftShader frame (1-2 of 3) still counts as a real miss. const siblingRendererStalled = - siblingTarget && !siblingOwnsPick && siblingTarget.framesRendered < 3; + siblingTarget && !siblingOwnsPick && siblingTarget.framesRendered === 0; record( 'sibling test object owns its canvas pick', !siblingTarget ? null - : siblingRendererStalled - ? HEADFUL - ? false - : null - : siblingOwnsPick, + : siblingOwnsPick || + (siblingRendererStalled && !HEADFUL ? null : false), siblingTarget - ? `${siblingRendererStalled ? `renderer stalled: ${siblingTarget.framesRendered}/3 frames in 10s; ` : ''}picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta} frames=${siblingTarget.framesRendered}/3` + ? `${siblingRendererStalled ? 'renderer stalled: ' : ''}frames=${siblingTarget.framesRendered}/3 picked=${siblingTarget.pickedId} exactOwner=${siblingTarget.ownsExactObject} projectionΔ=${siblingTarget.projectionDelta}` : 'no previously verified empty canvas ray was available', ); if (siblingOwnsPick) { @@ -1677,10 +1701,7 @@ async function main() { ); const viewshedShot = path.join(SHOTS_DIR, 'cctv-v3-viewshed.png'); - await page.screenshot({ path: viewshedShot }); - console.log( - ` screenshot (visual review only) → ${path.relative(REPO_ROOT, viewshedShot)}`, - ); + await saveReviewShot(page, viewshedShot, 'visual review only'); // ========================================================================= // Group 7: calibration gizmo (v3 design §3c — ADJUST mode + drag) @@ -2014,10 +2035,7 @@ async function main() { // ----------------------------------------------------------------------- await sleep(1000); const shot = path.join(SHOTS_DIR, 'cctv-v2-active.png'); - await page.screenshot({ path: shot }); - console.log( - ` screenshot (visual review only, not a gate) → ${path.relative(REPO_ROOT, shot)}`, - ); + await saveReviewShot(page, shot, 'visual review only, not a gate'); // No console errors during the whole exercise. record( From 97630a598a544c647efa49dcbae8ca333f449c20 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Mon, 19 Oct 2026 23:15:43 +0100 Subject: [PATCH 20/25] fix(cctv): release Tarktee sibling on fetch rejection --- docs/CURRENT-STATE.md | 5 ++++ server/providers/cctv/sources.js | 10 +++++++- src/data/cctvEstonia.test.mjs | 40 ++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 60937dd05..4b68e7030 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4457,6 +4457,11 @@ silently demoting every later lookup for the session. empty keeps the old catalog and is retried after another 15 minutes, not on the next request. Estonia's Tarktee DATEX feeds, which answer in 2 to 16.6 s, get 30 s (`TARKTEE_FETCH_TIMEOUT_MS`) where other packs get 15 s. + Both Tarktee XML catalogs refuse redirects and have a 4 MiB per-document + limit, enforced against declared length and streamed bytes. A rejected fetch, + unsuccessful response, or body-read failure releases both available response + bodies; read failures also cancel an active sibling reader. Cameras sharing + an image number are deduplicated before the Tarktee pack cap is applied. - **CCTV v3 UX — viewshed + calibration gizmo** (built 2026-07-05 and field validated 2026-07-21): the COVERAGE toggle is a tri-state cycle `OFF → ON → VIEWSHED`; viewshed mode renders each visible camera's frustum diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 7ccbf3aad..1e13df302 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -1200,7 +1200,7 @@ export function parseTarkteeDatexImages(xml) { */ export async function loadTarkteeSourcesFromDatex() { try { - const [locResp, imgResp] = await Promise.all([ + const responses = await Promise.allSettled([ fetch(TARKTEE_LOCATIONS_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, redirect: 'manual', @@ -1212,6 +1212,9 @@ export async function loadTarkteeSourcesFromDatex() { signal: AbortSignal.timeout(TARKTEE_FETCH_TIMEOUT_MS), }), ]); + const [locResp, imgResp] = responses.map((result) => + result.status === 'fulfilled' ? result.value : undefined, + ); // Both responses are fetched in parallel; if either is rejected or overflows // its byte cap, both bodies must be released so neither transport leaks. const discardBoth = async () => { @@ -1221,6 +1224,11 @@ export async function loadTarkteeSourcesFromDatex() { ]); return []; }; + const rejected = responses.find((result) => result.status === 'rejected'); + if (rejected) { + await discardBoth(); + throw rejected.reason; + } if ( (locResp.status >= 300 && locResp.status < 400) || (imgResp.status >= 300 && imgResp.status < 400) diff --git a/src/data/cctvEstonia.test.mjs b/src/data/cctvEstonia.test.mjs index 4c2a0f686..670432315 100644 --- a/src/data/cctvEstonia.test.mjs +++ b/src/data/cctvEstonia.test.mjs @@ -168,6 +168,46 @@ test('Tarktee loader refuses a redirect and cancels both response bodies', async assert.equal(cancelled.images, true, 'sibling 200 body is also cancelled'); }); +test('Tarktee loader cancels fulfilled bodies when either fetch rejects', async (t) => { + for (const failedUrl of [TARKTEE_LOCATIONS_URL, TARKTEE_IMAGES_URL]) { + for (const delayedSibling of [false, true]) { + await t.test( + `${failedUrl}, delayed sibling: ${delayedSibling}`, + async (t) => { + quiet(t); + let cancelled = false; + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('')); + }, + cancel() { + cancelled = true; + }, + }), + ); + t.mock.method(globalThis, 'fetch', async (url) => { + if (String(url) === failedUrl) throw new TypeError('fetch failed'); + if (delayedSibling) await new Promise(setImmediate); + return response; + }); + + try { + assert.deepEqual(await loadTarkteeSourcesFromDatex(), []); + assert.equal( + cancelled, + true, + 'fulfilled sibling body is cancelled', + ); + } finally { + await response.body.cancel(); + } + }, + ); + } + } +}); + test('Tarktee loader cancels the sibling body when one feed fails or overflows the byte cap', async (t) => { quiet(t); let siblingCancelled = false; From f0e0ea42ce6025d18af9e8eb195f93444d368d12 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Mon, 19 Oct 2026 23:31:52 +0100 Subject: [PATCH 21/25] test(cctv): verify DEM placement in the keyless globe gate --- scripts/qa-cctv-v2.mjs | 101 +++++++++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 18 deletions(-) diff --git a/scripts/qa-cctv-v2.mjs b/scripts/qa-cctv-v2.mjs index 2c121e0d6..88e0a5a2c 100644 --- a/scripts/qa-cctv-v2.mjs +++ b/scripts/qa-cctv-v2.mjs @@ -11,8 +11,10 @@ * exactly once per camera ACTIVATION (the §9.1 obstruction probe) and * never again in steady state; shared mesh-floor sampling fires at most * once per coarse cell during the staggered geometry-load drain, and - * actually COMPLETES once tiles load (>=1 real sample — the - * ceiling alone would pass trivially at Δ=0 if the snap never finished), + * actually COMPLETES once tiles load in photoreal mode (>=1 real sample + * — the ceiling alone would pass trivially at Δ=0 if the snap never + * finished). Globe mode instead proves the rendered mount uses a real + * terrain height with zero mesh samples. Sampling * stays flat while idle, and stays flat for non-positional pose edits. * 2. Geometry contract: the 5-polyline frustum wireframe + far-cap plane * exist for the active camera, the plane's corners coincide with the @@ -658,24 +660,87 @@ async function main() { // (existence, not just ceiling). The wait below tolerates the update-tick // latency of the completion pass (update interval is 10s). // ----------------------------------------------------------------------- - console.log('Waiting for tiles + the one-shot completion pass...'); - const tilesSeenLoaded = await waitForTilesLoaded(page, 45000); - let sampleFloorOk = false; - if (tilesSeenLoaded) { - sampleFloorOk = await waitForPredicate( - page, - (base) => window.__qaCounters.sampleHeight - base >= 1, - { timeoutMs: 30000, args: [c0.sampleHeight] }, + const globeMode = await page.evaluate( + () => window.__godsEyeView.viewer.scene.globe?.show !== false, + ); + if (globeMode) { + // Globe stacks intentionally perform no scene.sampleHeight query. + // Require independently fetched DEM heights and the actual rendered + // mount, so a catalog/geoid fallback cannot turn zero samples into a pass. + console.log( + 'Checking terrain-globe completion against live DEM heights...', + ); + const terrain = await page.evaluate(async () => { + const camera = window.__godsEyeView.dataManager.layers + .get('cctv') + .module.getUIState().activeCamera; + const exact = `${camera.lon.toFixed(5)},${camera.lat.toFixed(5)}`; + const cell = `${camera.lon.toFixed(3)},${camera.lat.toFixed(3)}`; + const response = await fetch( + `/api/terrain/heights?points=${encodeURIComponent(`${exact};${cell}`)}`, + { signal: AbortSignal.timeout(30000) }, + ); + const body = await response.json(); + return { ok: response.ok, results: body.results }; + }); + const heights = terrain.results?.map((result) => result?.ellipsoid); + const terrainReady = + terrain.ok && heights?.length === 2 && heights.every(Number.isFinite); + const mountResolved = + terrainReady && + (await waitForPredicate( + page, + (expected, base) => { + const { viewer, dataManager } = window.__godsEyeView; + const camera = dataManager.layers + .get('cctv') + .module.getUIState().activeCamera; + const ray = viewer.entities.getById(`cctv-${camera.id}-ray-tl`); + const mount = ray?.polyline?.positions.getValue( + viewer.clock.currentTime, + )?.[0]; + const height = + mount && + viewer.scene.globe.ellipsoid.cartesianToCartographic(mount) + .height; + return ( + Number.isFinite(camera.groundPriorM) && + Math.abs(camera.groundPriorM - expected[0]) < 0.01 && + Number.isFinite(height) && + expected.some( + (ground) => + Math.abs(height - ground - camera.mountHeightM) < 0.05, + ) && + window.__qaCounters.sampleHeight === base + ); + }, + { timeoutMs: 30000, args: [heights, c0.sampleHeight] }, + )); + record( + 'terrain-globe completion uses a real DEM height at the rendered mount without mesh samples', + Boolean(mountResolved), + `terrain=${JSON.stringify(heights)}, mesh sample Δ=${(await readCounters()).sampleHeight - c0.sampleHeight}`, + ); + } else { + console.log('Waiting for tiles + the one-shot completion pass...'); + const tilesSeenLoaded = await waitForTilesLoaded(page, 45000); + let sampleFloorOk = false; + if (tilesSeenLoaded) { + sampleFloorOk = await waitForPredicate( + page, + (base) => window.__qaCounters.sampleHeight - base >= 1, + { timeoutMs: 30000, args: [c0.sampleHeight] }, + ); + } + const cFloor = await readCounters(); + record( + '>=1 real ground sample once tilesLoaded observed (one-shot snap completed)', + tilesSeenLoaded ? sampleFloorOk : null, + tilesSeenLoaded + ? `total real samples since enable=${cFloor.sampleHeight - c0.sampleHeight}` + : 'tilesLoaded never observed within 45s — floor not assessable this run', ); } - const cFloor = await readCounters(); - record( - '>=1 real ground sample once tilesLoaded observed (one-shot snap completed)', - tilesSeenLoaded ? sampleFloorOk : null, - tilesSeenLoaded - ? `total real samples since enable=${cFloor.sampleHeight - c0.sampleHeight}` - : 'tilesLoaded never observed within 45s — floor not assessable this run', - ); // Let the completion pass finish before the steady-state window: require // the sampleHeight counter quiet for 12s (> the 10s update interval, so From b43970eecc319dae2a0b9d2e5f7dbd4d903a194d Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Sat, 17 Oct 2026 15:18:23 -0400 Subject: [PATCH 22/25] fix(satellites): retain cached catalog provenance --- docs/CURRENT-STATE.md | 6 +- scripts/format-scope.json | 1 + src/layers/satellites/catalog.js | 2 + src/layers/satellites/controls.js | 7 +- src/layers/satellites/ingestion.js | 15 +- src/layers/satellites/lifecycle.js | 4 + src/layers/satellites/stale.test.mjs | 279 +++++++++++++++++++++++++++ src/layers/satellites/state.js | 4 + 8 files changed, 313 insertions(+), 5 deletions(-) create mode 100644 src/layers/satellites/stale.test.mjs diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 55b7a63ab..1a4952093 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -817,7 +817,11 @@ Viewport limits, saturation retry, placement, query caps and controls are retain Satellite and mission layers expose separate factories through `./layers/satellites` and `./layers/launches`. Each owns its catalog, render state, tracking, interaction -and teardown. Source adapters retain the existing CelesTrak and Launch Library +and teardown. Satellite core and dense catalogs retain upstream stale-cache +provenance only for admitted records; duplicate or invalid entries cannot taint +a fresh winning record. Dense removal clears only its own provenance, and a +fresh core rebuild or lifecycle reset replaces the corresponding ownership. +Source adapters retain the existing CelesTrak and Launch Library endpoints; standalone entries provide scene services and the satellite dependency. The mission factory separates paths, replay, camera, panel, overlays and placement. Launch Library pad coordinates retain unknown values: missing fields use the diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 543b0f9a4..8812790e2 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -446,6 +446,7 @@ "src/data/satellites.js", "src/data/rocketLaunches.js", "src/layers/satellites/source.test.mjs", + "src/layers/satellites/stale.test.mjs", "src/layers/launches/source.test.mjs", "src/layers/launches/ingestion.test.mjs", "src/layers/installations/lifecycle.js", diff --git a/src/layers/satellites/catalog.js b/src/layers/satellites/catalog.js index fd136c82d..097d9e3be 100644 --- a/src/layers/satellites/catalog.js +++ b/src/layers/satellites/catalog.js @@ -139,6 +139,7 @@ export function createCatalog({ state: layerState, services, parts, source }) { }); layerState._points.set(noradId, point); layerState._denseIds.push(noradId); + layerState._denseStale ||= Boolean(res.stale); added++; } // Yield to the event loop between chunks. @@ -226,6 +227,7 @@ export function createCatalog({ state: layerState, services, parts, source }) { layerState._catalog.delete(noradId); } layerState._denseIds = []; + layerState._denseStale = false; layerState._denseCursor = 0; layerState._count = layerState._points.size; layerState._catalogRevision++; diff --git a/src/layers/satellites/controls.js b/src/layers/satellites/controls.js index 9a56daac3..f3327b332 100644 --- a/src/layers/satellites/controls.js +++ b/src/layers/satellites/controls.js @@ -545,16 +545,19 @@ export function createControls({ state: layerState, services, parts, source }) { }, getStats() { + const stale = Boolean(layerState._coreStale || layerState._denseStale); return { count: layerState._count, lastUpdate: layerState._lastUpdate, - stale: false, + stale, status: layerState._lastError === 'CelesTrak unreachable' ? 'unavailable' : layerState._lastError ? 'degraded' - : 'nominal', + : stale + ? 'stale' + : 'nominal', error: layerState._lastError, }; }, diff --git a/src/layers/satellites/ingestion.js b/src/layers/satellites/ingestion.js index f8ca1db56..01653611c 100644 --- a/src/layers/satellites/ingestion.js +++ b/src/layers/satellites/ingestion.js @@ -36,7 +36,12 @@ export function createIngestion({ if (!res.ok) return { ...groupDef, entries: [], ok: false }; const entries = parts.orbits.parseTLE(res.text); updateSignal.throwIfAborted(); - return { ...groupDef, entries, ok: entries.length > 0 }; + return { + ...groupDef, + entries, + ok: entries.length > 0, + stale: Boolean(res.stale), + }; } catch (error) { if (updateSignal.aborted || error?.name === 'AbortError') throw error; @@ -91,6 +96,7 @@ export function createIngestion({ // catalog or those labels stay stale for the life of the session. layerState._detectionObjects.clear(); layerState._denseIds = []; + layerState._denseStale = false; layerState._denseCursor = 0; layerState._denseLoadController?.abort(); layerState._denseLoadController = null; @@ -102,11 +108,13 @@ export function createIngestion({ // Process all TLE entries in CATALOG_GROUPS order const allEntries = []; for (const r of results) { - for (const e of r.entries) allEntries.push({ ...e, group: r.tag }); + for (const e of r.entries) + allEntries.push({ ...e, group: r.tag, stale: r.stale }); } // Deduplicate by NORAD ID — first (most specific) group tag wins const seen = new Set(); + let coreStale = false; for (const entry of allEntries) { const satrec = twoline2satrec(entry.line1, entry.line2); @@ -123,6 +131,8 @@ export function createIngestion({ group: entry.group, }); + coreStale ||= entry.stale; + // Propagate initial position const pos = parts.orbits.propagatePosition(satrec, now); if (!pos) continue; @@ -157,6 +167,7 @@ export function createIngestion({ parts.labels._syncIssOverlay(); } + layerState._coreStale = coreStale; layerState._count = layerState._points.size; layerState._catalogRevision++; layerState._lastUpdate = Date.now(); diff --git a/src/layers/satellites/lifecycle.js b/src/layers/satellites/lifecycle.js index 7c0d36670..642a6879f 100644 --- a/src/layers/satellites/lifecycle.js +++ b/src/layers/satellites/lifecycle.js @@ -24,6 +24,8 @@ export function createLifecycle({ layerState._orbitPaths = new Map(); layerState._count = 0; layerState._lastUpdate = null; + layerState._coreStale = false; + layerState._denseStale = false; layerState._trackedNorad = null; parts.tracking._cancelPendingTrackingRestore(); layerState._trackedEntity = null; @@ -162,6 +164,8 @@ export function createLifecycle({ layerState._rowControlsListener = null; layerState._count = 0; layerState._lastUpdate = null; + layerState._coreStale = false; + layerState._denseStale = false; layerState._lastError = null; layerState._lastFocusUpdate = 0; layerState._activeFocusCount = 0; diff --git a/src/layers/satellites/stale.test.mjs b/src/layers/satellites/stale.test.mjs new file mode 100644 index 000000000..89b4d2dbf --- /dev/null +++ b/src/layers/satellites/stale.test.mjs @@ -0,0 +1,279 @@ +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { EventEmitter, once } from 'node:events'; +import test from 'node:test'; +import * as Cesium from 'cesium'; +import { createSatelliteSource } from './source.js'; +import { createState } from './state.js'; +import { createControls } from './controls.js'; +import { createOrbits } from './orbits.js'; +import { createLabels } from './labels.js'; +import { createTracking } from './tracking.js'; +import { createRendering } from './rendering.js'; +import { createCatalog } from './catalog.js'; +import { createLifecycle } from './lifecycle.js'; +import { createInteraction } from './interaction.js'; +import { createIngestion } from './ingestion.js'; +import { layerFeedState } from '../../data/feedState.js'; + +const GROUPS = [ + 'stations', + 'visual', + 'gps-ops', + 'glo-ops', + 'galileo', + 'geo', + 'starlink', +]; + +function tle(id, invalid = false) { + // Owned synthetic catalog IDs over one valid element set; never live requests. + const motion = invalid ? ' 0.00000000' : '15.48899044'; + return `OWNED ORBIT FIXTURE ${id}\n1 ${id}U 98067A 26289.78971908 .00007803 00000+0 15012-3 0 9995\n2 ${id} 51.6326 58.3278 0006748 271.9059 88.1156 ${motion}590607`; +} + +async function fixture(t) { + const policy = { + stale: new Set(), + invalid: new Set(), + ids: {}, + status: 200, + hold: false, + }; + const held = new EventEmitter(); + let heldCount = 0; + const server = http.createServer((req, res) => { + const group = req.url.split('/').at(-1); + if (policy.hold) { + held.emit('request', ++heldCount); + return; + } + res.writeHead(policy.status, { + 'Content-Type': 'text/plain', + 'X-TLE-Cache': policy.stale.has(group) ? 'STALE-ERROR' : 'MISS', + }); + res.end( + tle( + policy.ids[group] ?? 25545 + GROUPS.indexOf(group), + policy.invalid.has(group), + ), + ); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const source = createSatelliteSource({ + fetchImpl: (path, options) => + fetch(`http://127.0.0.1:${server.address().port}${path}`, options), + }); + // Application host services and native EventTargets; actual Cesium collections, + // input handler, events, SGP4, ingestion and lifecycle run without a WebGL scene. + const noop = () => {}; + const services = { + overlays: { + setOverlayEntries: noop, + setOverlaySourceVisible: noop, + clearOverlaySource: noop, + }, + layerState: { isExplicitLayerStateOrigin: () => false }, + focus: { + clearFocusTarget: noop, + publishFocusTargetFromCachedPosition: noop, + getFocusTarget: () => null, + }, + context: { + clearTrackedSubjectContext: noop, + getContextStore: () => ({ entities: new Map() }), + refreshTrackedSubjectContext: noop, + selectTrackedSubjectContext: noop, + }, + readout: { refreshTrackedReadout: noop }, + render: { holdContinuousRender: noop, releaseContinuousRender: noop }, + picking: { registerPickOwner: noop, unregisterPickOwner: noop }, + }; + const previousDocument = globalThis.document; + globalThis.document = new EventTarget(); + const state = createState({ services }); + const parts = {}; + const context = { state, services, parts, source }; + for (const [name, factory] of Object.entries({ + controls: createControls, + orbits: createOrbits, + labels: createLabels, + tracking: createTracking, + rendering: createRendering, + catalog: createCatalog, + interaction: createInteraction, + lifecycle: createLifecycle, + ingestion: createIngestion, + })) { + parts[name] = factory(context); + } + const viewer = { + scene: { + canvas: new EventTarget(), + primitives: new Cesium.PrimitiveCollection(), + preRender: new Cesium.Event(), + }, + trackedEntityChanged: new Cesium.Event(), + }; + await parts.lifecycle.methods.init(viewer); + t.after(async () => { + if (state._pointCollection) parts.lifecycle.methods.destroy(viewer); + viewer.scene.primitives.destroy(); + if (previousDocument === undefined) delete globalThis.document; + else globalThis.document = previousDocument; + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + }); + return { + state, + parts, + viewer, + policy, + update: (options) => parts.ingestion.methods.update(viewer, options), + stats: () => parts.controls.methods.getStats(), + feed: () => layerFeedState(parts.controls.methods.getStats()), + async waitHeld(count) { + while (heldCount < count) + await once(held, 'request', { signal: AbortSignal.timeout(3000) }); + }, + }; +} + +test('stale core records surface through actual HTTP, ingestion and feed state, then recover', async (t) => { + const f = await fixture(t); + await f.update(); + assert.equal(f.feed(), 'nominal'); + assert.equal(f.stats().count, 6); + f.policy.stale = new Set(GROUPS.slice(0, 6)); + await f.update(); + assert.equal(f.stats().count, 6); + assert.equal(f.stats().stale, true); + assert.equal(f.feed(), 'stale'); + f.policy.stale = new Set(['visual']); + await f.update(); + assert.equal(f.feed(), 'stale'); + f.policy.stale.clear(); + await f.update(); + assert.equal(f.stats().stale, false); + assert.equal(f.feed(), 'nominal'); +}); + +test('a stale duplicate cannot taint its earlier fresh winning record', async (t) => { + const f = await fixture(t); + f.policy.stale.add('visual'); + f.policy.ids.visual = 25545; + await f.update(); + assert.equal(f.stats().count, 5); + assert.equal(f.state._catalog.get(25545).group, 'stations'); + assert.equal(f.stats().stale, false); + assert.equal(f.feed(), 'nominal'); + f.policy.stale = new Set(['stations']); + await f.update(); + assert.equal(f.stats().count, 5); + assert.equal( + f.stats().stale, + true, + 'a later fresh duplicate does not replace the stale winner', + ); +}); + +test('invalid stale orbital elements are not admitted as provenance', async (t) => { + const f = await fixture(t); + f.policy.stale.add('visual'); + f.policy.invalid.add('visual'); + await f.update(); + assert.equal(f.stats().count, 5); + assert.equal(f.state._catalog.has(25546), false); + assert.equal(f.stats().stale, false); +}); + +test('failed and aborted core refreshes keep provenance of retained records', async (t) => { + const f = await fixture(t); + f.policy.stale.add('stations'); + await f.update(); + const priorTime = f.stats().lastUpdate; + f.policy.status = 503; + await f.update(); + assert.equal(f.stats().count, 6); + assert.equal(f.stats().stale, true); + assert.equal(f.stats().status, 'unavailable'); + assert.equal(f.stats().lastUpdate, priorTime); + f.policy.hold = true; + const controller = new AbortController(); + const work = f.update({ signal: controller.signal }); + await f.waitHeld(6); + controller.abort(); + await assert.rejects(work, { name: 'AbortError' }); + assert.equal(f.stats().count, 6); + assert.equal(f.stats().stale, true); + assert.equal(f.stats().lastUpdate, priorTime); + assert.equal(f.state._activeUpdateControllers.size, 0); +}); + +test('dense provenance belongs to its admitted records and clears on removal', async (t) => { + const f = await fixture(t); + await f.update(); + f.policy.stale.add('starlink'); + f.state._params.catalog = 'dense'; + assert.equal((await f.parts.catalog._loadDenseCatalog()).status, 'ready'); + assert.equal(f.stats().count, 7); + assert.equal(f.feed(), 'stale'); + f.state._params.catalog = 'core'; + f.parts.catalog._removeDenseCatalog(); + assert.equal(f.stats().count, 6); + assert.equal(f.feed(), 'nominal'); + f.policy.stale = new Set(['stations']); + await f.update(); + f.state._params.catalog = 'dense'; + await f.parts.catalog._loadDenseCatalog(); + f.state._params.catalog = 'core'; + f.parts.catalog._removeDenseCatalog(); + assert.equal( + f.feed(), + 'stale', + 'removing dense data must not clear core provenance', + ); +}); + +test('cancelling dense replacement preserves admitted stale extras until rebuild', async (t) => { + const f = await fixture(t); + await f.update(); + f.policy.stale.add('starlink'); + f.state._params.catalog = 'dense'; + await f.parts.catalog._loadDenseCatalog(); + f.policy.hold = true; + const controller = new AbortController(); + const work = f.parts.catalog._loadDenseCatalog({ signal: controller.signal }); + await f.waitHeld(1); + controller.abort(); + assert.equal((await work).status, 'cancelled'); + assert.equal(f.stats().count, 7); + assert.equal(f.feed(), 'stale'); + f.policy.hold = false; + f.policy.stale.clear(); + f.state._params.catalog = 'core'; + await f.update(); + assert.equal(f.stats().count, 6); + assert.equal(f.feed(), 'nominal'); +}); + +test('destroy aborts an in-flight response and initialization starts with new ownership', async (t) => { + const f = await fixture(t); + f.policy.stale.add('stations'); + await f.update(); + f.policy.hold = true; + const work = f.update(); + await f.waitHeld(6); + const rejected = assert.rejects(work, { name: 'AbortError' }); + f.parts.lifecycle.methods.destroy(f.viewer); + assert.equal(f.stats().count, 0); + assert.equal(f.stats().stale, false); + await f.parts.lifecycle.methods.init(f.viewer); + await rejected; + assert.equal(f.stats().stale, false); + f.policy.hold = false; + f.policy.stale.clear(); + await f.update(); + assert.equal(f.stats().count, 6); + assert.equal(f.feed(), 'nominal'); +}); diff --git a/src/layers/satellites/state.js b/src/layers/satellites/state.js index 106802ba6..50a127b54 100644 --- a/src/layers/satellites/state.js +++ b/src/layers/satellites/state.js @@ -40,6 +40,10 @@ export function createState({ services }) { state._lastError = null; + // Cache provenance belongs to admitted core and dense records separately. + state._coreStale = false; + state._denseStale = false; + state._activeUpdateControllers = new Set(); state._denseLoadController = null; From 57983fb4c28859b2f208dd28a053d52e3a6d82ff Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Mon, 19 Oct 2026 22:16:37 -0400 Subject: [PATCH 23/25] docs: add satellite cache provenance changelog --- changelog.d/1283-satellite-cache-provenance.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/1283-satellite-cache-provenance.md diff --git a/changelog.d/1283-satellite-cache-provenance.md b/changelog.d/1283-satellite-cache-provenance.md new file mode 100644 index 000000000..c9b0dd7dc --- /dev/null +++ b/changelog.d/1283-satellite-cache-provenance.md @@ -0,0 +1,3 @@ +- Report cached CelesTrak satellite records as stale in the Data Layers row, + preserving separate core and dense catalog provenance through refresh, + removal and lifecycle cleanup (rudycelekli, #1283). From 876cd5e4a5c55582fc4ca4e2167ddddbc5aa19c8 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Tue, 20 Oct 2026 03:22:18 -0400 Subject: [PATCH 24/25] fix(flights): bound route labels to the scheduled leg --- docs/CURRENT-STATE.md | 7 ++ scripts/format-scope.json | 1 + src/data/routePlausible.js | 29 +++++- src/data/routeSegment.test.mjs | 174 +++++++++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 src/data/routeSegment.test.mjs diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 55b7a63ab..e8345fe34 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2986,6 +2986,13 @@ down | auth-failed` (plus the unchanged `missing-key`/`unsupported`) with > position getter reads only `_trackedDisplayCached`, the same authoritative > Cartesian already consumed by the tracked visual and camera, and never > advances dead reckoning from the host frame. +> - **Scheduled route plausibility:** Aircraft route labels and selected-route +> context reject the distant backwards, forwards and antipodal extensions of +> an airport pair's great circle. The existing 130 km endpoint admission and +> 200 km cross-track tolerance remain; a projected fix outside the finite +> origin-to-destination minor arc also has a generous 200 km extension at +> either end. Missing coordinates and coincident or antipodal endpoint pairs +> retain their loose, unjudgeable behavior; vertical-trend checks are unchanged. > - **Aircraft label convention (both flight layers, 2026-08-18):** every civil > and military label surface resolves **callsign → registration → icao24** > (`_contactLabel()` in `flights.js`; the same chain inline in diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 543b0f9a4..0afad9e74 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -1070,5 +1070,6 @@ "scripts/assemble-changelog.mjs", "src/tooling/changelogFragments.test.mjs", "src/data/gbfsProxyRateLimit.test.mjs", + "src/data/routeSegment.test.mjs", "src/terrainHeightsBounds.test.mjs" ] diff --git a/src/data/routePlausible.js b/src/data/routePlausible.js index 238c2e6cb..d52b44ef8 100644 --- a/src/data/routePlausible.js +++ b/src/data/routePlausible.js @@ -33,6 +33,18 @@ export function crossTrackKm(lat, lon, lat1, lon1, lat2, lon2) { return Math.asin(Math.sin(d13) * Math.sin(b13 - b12)) * EARTH_RADIUS_KM; } +/** Signed distance along the great circle from the origin toward the destination. */ +function alongTrackKm(position, origin, destination) { + const distance = greatCircleKm(origin, position) / EARTH_RADIUS_KM; + const direction = + bearingRad(origin.lat, origin.lon, position.lat, position.lon) - + bearingRad(origin.lat, origin.lon, destination.lat, destination.lon); + return ( + Math.atan2(Math.sin(distance) * Math.cos(direction), Math.cos(distance)) * + EARTH_RADIUS_KM + ); +} + const NEAR_ENDPOINT_KM = 130; const CROSS_TRACK_KM = 200; const LOW_ALT_M = 3700; // ~12 000 ft @@ -66,6 +78,18 @@ export function routePlausible({ const near = (point) => greatCircleKm(position, point) < NEAR_ENDPOINT_KM; let geomOk = (haveO && near(origin)) || (haveD && near(destination)); if (!geomOk && haveO && haveD) { + const routeKm = greatCircleKm(origin, destination); + const alongKm = alongTrackKm(position, origin, destination); + // atan2 returns the principal arc in [-πR, πR]. Near an antipodal + // destination, its allowed forward extension can cross that branch cut. + const segmentAlongKm = + alongKm < -CROSS_TRACK_KM + ? alongKm + 2 * Math.PI * EARTH_RADIUS_KM + : alongKm; + // Coincident or antipodal endpoints do not define one minor arc. Keep the + // existing loose check when the segment cannot be judged reliably. + const segmentKnown = + routeKm > 0.001 && routeKm < Math.PI * EARTH_RADIUS_KM - 0.001; geomOk = Math.abs( crossTrackKm( @@ -76,7 +100,10 @@ export function routePlausible({ destination.lat, destination.lon, ), - ) < CROSS_TRACK_KM; + ) < CROSS_TRACK_KM && + (!segmentKnown || + (segmentAlongKm >= -CROSS_TRACK_KM && + segmentAlongKm <= routeKm + CROSS_TRACK_KM)); } else if (!geomOk) { geomOk = true; // only one endpoint known and not near — cannot judge, allow } diff --git a/src/data/routeSegment.test.mjs b/src/data/routeSegment.test.mjs new file mode 100644 index 000000000..caa02c4ea --- /dev/null +++ b/src/data/routeSegment.test.mjs @@ -0,0 +1,174 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import * as Cesium from 'cesium'; +import { routePlausible } from './routePlausible.js'; +import { createTracking } from '../layers/flights/tracking.js'; + +const SFO = { lat: 37.6188, lon: -122.3754, code: 'SFO' }; +const LAX = { lat: 33.9416, lon: -118.4085, code: 'LAX' }; +const beyond = { lat: -35.818227816411174, lon: -66.16647396367095 }; +const behind = { lat: 35.818227816411174, lon: 113.83352603632906 }; +const antipode = { lat: -SFO.lat, lon: SFO.lon + 180 }; +const midpoint = { lat: 35.8, lon: -120.4 }; + +const plausible = (position, origin = SFO, destination = LAX) => + routePlausible({ + latDeg: position.lat, + lonDeg: position.lon, + altitudeM: 10000, + verticalRateMps: 0, + origin, + destination, + }); + +// Independent spherical oracle: native unit-vector angular distances, without +// the production bearing/cross-track/along-track calculations. +const unit = ({ lat, lon }) => { + const latitude = Cesium.Math.toRadians(lat); + const longitude = Cesium.Math.toRadians(lon); + return new Cesium.Cartesian3( + Math.cos(latitude) * Math.cos(longitude), + Math.cos(latitude) * Math.sin(longitude), + Math.sin(latitude), + ); +}; +const angle = (a, b) => Cesium.Cartesian3.angleBetween(unit(a), unit(b)); +const excess = (position, origin, destination) => + angle(origin, position) + + angle(position, destination) - + angle(origin, destination); + +test('a short airport route cannot describe a plane thousands of kilometres beyond its destination', () => { + assert.ok(excess(beyond, SFO, LAX) > 2.9); + assert.equal(plausible(beyond), false); +}); + +test('backwards and antipodal extensions are not the scheduled airport leg', () => { + for (const position of [behind, antipode]) { + assert.ok(excess(position, SFO, LAX) > 3); + assert.equal(plausible(position), false); + } +}); + +test('ordinary route offsets and generous endpoint extensions stay plausible', () => { + assert.ok(excess(midpoint, SFO, LAX) < 0.001); + for (const position of [midpoint, SFO, LAX, { lat: 36.5, lon: -119.5 }]) { + assert.equal(plausible(position), true); + } + // ~167 km past either endpoint is outside the 130 km endpoint circle, + // but remains inside the 200 km coarse-route tolerance. + const origin = { lat: 0, lon: 0 }; + const destination = { lat: 0, lon: 10 }; + assert.equal(plausible({ lat: 0, lon: -1.5 }, origin, destination), true); + assert.equal(plausible({ lat: 0, lon: 11.5 }, origin, destination), true); +}); + +test('the generous endpoint tolerance still excludes a clearly different leg', () => { + const origin = { lat: 0, lon: 0 }; + const destination = { lat: 0, lon: 10 }; + assert.equal(plausible({ lat: 0, lon: -2.5 }, origin, destination), false); + assert.equal(plausible({ lat: 0, lon: 12.5 }, origin, destination), false); +}); + +test('minor-arc admission works across the antimeridian and near the pole', () => { + const dateline = [ + { lat: 10, lon: 170 }, + { lat: 10, lon: -170 }, + ]; + assert.equal(plausible({ lat: 10.15, lon: 180 }, ...dateline), true); + assert.equal(plausible({ lat: -10.15, lon: 0 }, ...dateline), false); + const polar = [ + { lat: 80, lon: -60 }, + { lat: 80, lon: 60 }, + ]; + const top = { lat: 84.96163122670251, lon: 0 }; + assert.ok(excess(top, ...polar) < 1e-8); + assert.equal(plausible(top, ...polar), true); + assert.equal(plausible({ lat: -top.lat, lon: 180 }, ...polar), false); +}); + +test('the forward tolerance survives the signed-angle branch cut in both directions', () => { + const origin = { lat: 0, lon: 0 }; + for (const direction of [-1, 1]) { + const destination = { lat: 0, lon: direction * 179.9 }; + const within = { lat: 0, lon: direction * -178.8 }; + const endpointGapKm = angle(destination, within) * 6371.0088; + assert.ok(endpointGapKm > 130 && endpointGapKm < 200); + assert.equal(plausible(within, origin, destination), true); + assert.equal( + plausible({ lat: 0, lon: direction * -178 }, origin, destination), + false, + ); + } +}); + +test('unjudgeable endpoint geometry and missing coordinates keep the loose contract', () => { + assert.equal( + plausible({ lat: 0, lon: -90 }, { lat: 0, lon: 0 }, { lat: 0, lon: 180 }), + true, + ); + assert.equal( + plausible({ lat: 45, lon: 0 }, { lat: 0, lon: 0 }, { lat: 0, lon: 0 }), + true, + ); + assert.equal(plausible(beyond, SFO, null), true); + assert.equal(plausible(beyond, null, null), true); +}); + +test('the actual native tracked label hides the wrong leg and keeps a healthy route', (t) => { + const collection = new Cesium.BillboardCollection(); + t.after(() => collection.destroy()); + const flightState = { + records: { data: new Map(), missingPolls: new Map() }, + feed: { _backoff: false }, + _billboards: new Map(), + _scratchCarto: new Cesium.Cartographic(), + }; + // Only application host services are supplied. Route admission, WGS84 + // billboard conversion and tracked-label formatting are the actual owners. + const services = Object.fromEntries( + [ + 'context', + 'trails', + 'geoid', + 'groundFloor', + 'focus', + 'militaryRegistry', + 'readout', + 'camera', + 'picking', + ].map((name) => [name, {}]), + ); + services.aircraftPresentation = { isTr3b: () => false }; + const parts = { queries: { _contactLabel: (_, info) => info.callsign } }; + const tracking = createTracking({ flightState, services, parts }); + for (const [position, expected] of [ + [beyond, false], + [midpoint, true], + ]) { + const id = 'abc001'; + const route = { origin: SFO, destination: LAX }; + flightState.records.data.set(id, { + callsign: 'OWNED001', + altitude: 10000, + verticalRate: 0, + route, + }); + flightState._billboards.set( + id, + collection.add({ + id, + position: Cesium.Cartesian3.fromDegrees( + position.lon, + position.lat, + 10000, + ), + }), + ); + assert.equal( + tracking._trackedLabelText(id).includes('SFO → LAX'), + expected, + ); + assert.equal(tracking._routeIsPlausible(id, route), expected); + } +}); From 140c18c55aa312c88dfdf06e15ab632d985e9f5a Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Tue, 20 Oct 2026 03:35:27 -0400 Subject: [PATCH 25/25] docs: add scheduled route plausibility changelog --- changelog.d/1284-route-plausibility-segment.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/1284-route-plausibility-segment.md diff --git a/changelog.d/1284-route-plausibility-segment.md b/changelog.d/1284-route-plausibility-segment.md new file mode 100644 index 000000000..4ab211957 --- /dev/null +++ b/changelog.d/1284-route-plausibility-segment.md @@ -0,0 +1,3 @@ +- Keep aircraft route labels on the scheduled airport leg instead of accepting + its distant great-circle extensions, preserving generous endpoint tolerance + across the signed-angle branch cut (rudycelekli, #1284).