From 2dd96e1f25f37a1464cf84ddbd162c677e546f87 Mon Sep 17 00:00:00 2001 From: MiguelGFerreira Date: Sat, 12 Sep 2026 14:24:33 -0300 Subject: [PATCH 01/95] =?UTF-8?q?feat(data):=20add=20seven=20Brazilian=20B?= =?UTF-8?q?ike=20Ita=C3=BA=20cities=20to=20the=20GBFS=20registry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit São Paulo, Rio de Janeiro, Salvador, Recife, Porto Alegre, Brasília and Curitiba run on the same PBSC platform as the existing Austin, Honolulu and Chattanooga entries, so they need no new feed handling and no proxy allowlist change — isAllowedGbfsHost already accepts *.publicbikesystem.net. These are the layer's first stations outside the United States: 1,017 stations across the seven systems. --- CHANGELOG.md | 4 ++ DATA_SOURCES.md | 2 +- src/data/dataCredits.js | 2 +- src/layers/bikeshare/registry.js | 84 ++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32f8245b4..3c451b4f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,10 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md ## [Unreleased] +- Add seven Brazilian Bike Itaú (Tembici) cities to the GBFS bikeshare registry: + São Paulo, Rio de Janeiro, Salvador, Recife, Porto Alegre, Brasília and + Curitiba. These are the layer's first stations outside the United States. + - Separate radio directory loading, station selection, globe presentation and playback into composed components with an explicit metadata source. - Separate submarine cable sources and rendering components, and export bundled geography lookup modules. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 650d3fe6b..2dc8ba0d7 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -35,7 +35,7 @@ How to read this: | **City of Austin Open Data** | CCTV camera catalog + frames | City of Austin Open Data Terms of Use | "City of Austin, TX — data.austintexas.gov" | | **Caltrans (cwwp2.dot.ca.gov)** | CCTV camera catalogs + frames, California districts | Public Caltrans traffic camera data | "Caltrans — cwwp2.dot.ca.gov" (courtesy) | | **TfL Open Data (JamCams)** | CCTV camera catalog + frames, London | [TfL Open Data terms](https://tfl.gov.uk/info-for/open-data-users/) — attribution REQUIRED | "Powered by TfL Open Data. Contains OS data © Crown copyright and database rights" | -| **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | +| **GBFS (Lyft / BCycle / PBSC)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle, Bike Itaú/Tembici) + its `license_url` | | **Radio Browser** | Geolocated internet-radio station directory and station-level tags | Public-domain directory data under PDDL 1.0; individual broadcaster stream terms apply | "Radio Browser" plus a link to the selected broadcaster | | **Re:Earth Terrain** (Mapterhorn) | Terrain (keyless globe stacks — OSM etc. — + `/api/terrain/heights` ellipsoidal-height lookups) | Terrain mesh: CC BY 4.0; geoid: EGM2008 (NGA, public domain) | "Terrain (keyless globe stacks): Re:Earth Terrain / Mapterhorn (CC BY 4.0) / EGM2008 (NGA)" | diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 009ff6d01..817261608 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -137,7 +137,7 @@ export const DATA_CREDITS = [ }, { key: 'gbfs', - html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)', + html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle, Bike Itaú/Tembici)', }, { key: 'radio-browser', diff --git a/src/layers/bikeshare/registry.js b/src/layers/bikeshare/registry.js index ff752f664..a5a1ee32c 100644 --- a/src/layers/bikeshare/registry.js +++ b/src/layers/bikeshare/registry.js @@ -167,6 +167,90 @@ const RAW_GBFS_CITY_REGISTRY = [ 'https://chat.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', provider: 'Bike Chattanooga', }, + { + id: 'sao-paulo-bike-itau', + city: 'São Paulo, SP', + centerLat: -23.5505, + centerLon: -46.6333, + loadRadiusKm: 100, + stationInformationUrl: + 'https://saopaulo.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://saopaulo.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Bike Itaú', + }, + { + id: 'rio-de-janeiro-bike-itau', + city: 'Rio de Janeiro, RJ', + centerLat: -22.9068, + centerLon: -43.1729, + loadRadiusKm: 100, + stationInformationUrl: + 'https://riodejaneiro.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://riodejaneiro.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Bike Itaú', + }, + { + id: 'salvador-bike-itau', + city: 'Salvador, BA', + centerLat: -12.9777, + centerLon: -38.5016, + loadRadiusKm: 90, + stationInformationUrl: + 'https://salvador.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://salvador.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Bike Itaú', + }, + { + id: 'recife-bike-itau', + city: 'Recife, PE', + centerLat: -8.0476, + centerLon: -34.877, + loadRadiusKm: 90, + stationInformationUrl: + 'https://recife.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://recife.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Bike Itaú', + }, + { + id: 'porto-alegre-bike-itau', + city: 'Porto Alegre, RS', + centerLat: -30.0346, + centerLon: -51.2177, + loadRadiusKm: 90, + stationInformationUrl: + 'https://portoalegre.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://portoalegre.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Bike Itaú', + }, + { + id: 'brasilia-bike-itau', + city: 'Brasília, DF', + centerLat: -15.7939, + centerLon: -47.8828, + loadRadiusKm: 90, + stationInformationUrl: + 'https://brasilia.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://brasilia.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Bike Itaú', + }, + { + id: 'curitiba-bike-itau', + city: 'Curitiba, PR', + centerLat: -25.4284, + centerLon: -49.2733, + loadRadiusKm: 90, + stationInformationUrl: + 'https://curitiba.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://curitiba.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Bike Itaú', + }, bcycleEntry({ id: 'boulder-bcycle', city: 'Boulder, CO', From ae661db7ab61a7ab913856c654a3731ee354d9e3 Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:38:00 +0200 Subject: [PATCH 02/95] =?UTF-8?q?feat(cctv):=20add=20Catalonia=20(Servei?= =?UTF-8?q?=20Catal=C3=A0=20de=20Tr=C3=A0nsit)=20as=20a=20keyless=20CCTV?= =?UTF-8?q?=20source=20pack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One WFS/GML XML feed (~135 cameras) aggregating three publishers under a `font` field: SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras. Frame URLs are checked against a small host allowlist rather than one pinned origin, and every non-SCT camera carries its real owner as a `credit` shown beside the shared provider. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic), and ground elevation is primed per publisher rather than one flat prior. SCT's own frame link needed a workaround: RenderService always 302s to a second endpoint (TransitCamera) on a Location the upstream hardcodes as plain http even when requested over https — refused by fetchWithinHost's same-origin (scheme-included) redirect policy, by design (stops an upstream steering a host-pinned request into a plaintext downgrade). Separately, mct.gencat.cat's TLS setup fails the handshake with Node's fetch outright (ERR_SSL_WRONG_SIGNATURE_TYPE) even though other HTTP clients complete it. The loader builds the TransitCamera URL itself, over plain http, from RenderService's sctidcam id, instead of registering the broken upstream link — the only transport this host actually serves to this runtime. Verified against the live feed and the production fetch path (fetchCctvImageFromUpstream): all 135 cameras return real image bytes, 0 false positives. Ambient-card refresh cadence added to src/data/cctvLod.js (3 minutes), matching the upstream update rate measured empirically (no change inside 64s, changed within ~90-150s) — the same cadence already used for Caltrans/TfL/Ontario/TxDOT. CCTV_CATALONIA_MAX_SOURCES sets the cap and CCTV_CATALONIA_ENABLED=0 turns the pack off. No overlap with any existing pack (checked catalog.js and the git history for Spain/Catalonia/Barcelona coverage: none exists). Verified before this commit: format, check:boundaries, unit suite (3692 tests), and build all green. Co-Authored-By: Claude Sonnet 5 --- .env.example | 2 + CHANGELOG.md | 7 + DATA_SOURCES.md | 2 + docs/CURRENT-STATE.md | 2 +- server/providers/cctv/catalog.js | 6 + server/providers/cctv/constants.js | 41 ++++ server/providers/cctv/normalize.js | 30 +++ server/providers/cctv/sources.js | 203 +++++++++++++++++ src/data/cctvCataloniaSource.test.mjs | 317 ++++++++++++++++++++++++++ src/data/cctvDriveBcSource.test.mjs | 2 + src/data/cctvLod.js | 4 + src/data/cctvLod.test.mjs | 4 + src/data/dataCredits.js | 8 + 13 files changed, 627 insertions(+), 1 deletion(-) create mode 100644 src/data/cctvCataloniaSource.test.mjs diff --git a/.env.example b/.env.example index 5bc02df9c..5d2a268df 100644 --- a/.env.example +++ b/.env.example @@ -171,6 +171,8 @@ AISSTREAM_API_KEY= # CCTV_CALGARY_ENABLED=1 # set 0 to disable the Open Calgary (Alberta) cameras # CCTV_CALGARY_MAX_SOURCES=220 # cap cameras loaded from the Open Calgary catalog # CCTV_CALGARY_ROWS_URL= # override the Open Calgary catalog URL +# CCTV_CATALONIA_ENABLED=1 # set 0 to disable the Servei Català de Trànsit cameras +# CCTV_CATALONIA_MAX_SOURCES=150 # cap cameras loaded from Catalonia (~135 exist today) # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3d01413..0cfff159d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~135 + cameras from a WFS/GML feed that aggregates SCT's own highway cameras with + hotlinked Barcelona and Terrassa municipal cameras, each partner named as a + credit beside the shared provider. Ambient-card refresh paced to 3 minutes, + matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` + sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. + - Separate map-feature acquisition from annotation/search selection. Move road and mapped-installation decoding into source adapters while preserving geometry policy, cancellation, retry outcomes and compatibility exports. - Cancel hidden Nepal provider preloads on Stop, event disable and replacement, and respect drawing-tool pointer ownership for fallback evidence cards. Preserve @manjunath22466’s Nepal scene contribution and source attribution. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 850d27e39..02e7e4272 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -45,6 +45,7 @@ How to read this: | **Fintraffic / Digitraffic (weathercams)** | CCTV camera catalog + frames, Finland | [CC BY 4.0](https://www.digitraffic.fi/en/terms-of-service/) — attribution REQUIRED | "Fintraffic / digitraffic.fi, license CC BY 4.0" | | **DriveBC** (Government of British Columbia) | CCTV camera catalog + frames, British Columbia highways | [Open Government Licence – British Columbia](https://www2.gov.bc.ca/gov/content/data/open-data/open-government-licence-bc) — attribution REQUIRED | "DriveBC. Contains information licensed under the Open Government Licence – British Columbia" | | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | +| **Servei Català de Trànsit** (+ Ajuntament de Barcelona, Ajuntament de Terrassa) | CCTV camera catalog + frames, Catalonia | [Llicència oberta d'ús d'informació - Catalunya](https://web.gencat.cat/ca/generalitat/dades-indicadors/dades-obertes/llicencies) — attribution REQUIRED | "Servei Català de Trànsit"; partner-supplied cameras also name their owner | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | @@ -73,6 +74,7 @@ How to read this: - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. +- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `gencat.cat/transit/opendata/cameres.xml` WFS/GML feed. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin, and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 40e441b25..42ab3f07a 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2433,7 +2433,7 @@ its criteria cannot be silently ignored. | Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | | Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | | Traffic | OSM Overpass (+ optional TomTom live flow) | `src/data/traffic.js` | `/api/overpass` + `/api/tomtom` | viewport-driven | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Catalonia (Servei Català de Trànsit) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index a8519c85c..c681d321c 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -17,6 +17,7 @@ import { loadWarendorfSourcesFromCatalog, loadNswSourcesFromOpenData, loadCalgarySourcesFromOpenData, + loadCataloniaSourcesFromOpenData, } from './sources.js'; /** Env kill switch: unset or anything but "0" means enabled. */ @@ -86,6 +87,11 @@ const LIVE_PACKS = [ enabled: () => envEnabled('CCTV_CALGARY_ENABLED'), load: loadCalgarySourcesFromOpenData, }, + { + name: 'catalonia', + enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'), + load: loadCataloniaSourcesFromOpenData, + }, ]; /** * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index d66806e45..4dfcbbd15 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -236,6 +236,47 @@ export const CALGARY_DOWNTOWN = { lat: 51.0461, lon: -114.0626 }; * body cannot be buffered without limit. */ export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +/** + * Catalonia traffic cameras (Servei Català de Trànsit): one keyless WFS/GML + * XML list (verified 2026-09-15, hourly catalog refresh). The feed carries a + * `font` field naming the actual publisher per camera — SCT's own highway + * cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras are + * kept, each confirmed to return a real frame. Frame URLs are checked + * against a host allowlist rather than one pinned origin, since each + * publisher serves from its own host. + */ +export const CATALONIA_CAMERAS_URL = + 'http://www.gencat.cat/transit/opendata/cameres.xml'; +/** Allowed image hosts, one per kept `font` publisher in the feed. */ +export const CATALONIA_IMAGE_HOSTS = Object.freeze( + new Set([ + 'mct.gencat.cat', // SCT + 'www.bcn.cat', // IMI (Ajuntament de Barcelona) + 'emap.terrassa.cat', // Ajuntament de Terrassa + ]), +); +/** Per-`font` partner credit, shown beside the provider in the CCTV panel + * (same pattern as DriveBC's partner-supplied cameras). SCT is the feed's + * own publisher and carries no separate credit. */ +export const CATALONIA_FONT_CREDIT = Object.freeze({ + IMI: 'Ajuntament de Barcelona', + Terrassa: 'Ajuntament de Terrassa', +}); +/** Ground-elevation priors in metres, by `font`: the feed carries no + * elevation, and this pack spans sea level (Barcelona) to Terrassa's plateau + * (~270 m). The client's ground snap corrects these on 3D-tile stacks. */ +export const CATALONIA_FONT_ELEVATION_M = Object.freeze({ + SCT: 80, + IMI: 20, + Terrassa: 270, +}); +export const DEFAULT_CATALONIA_MAX_SOURCES = 150; +/** Prioritization anchors: Barcelona and Terrassa. */ +export const CATALONIA_ANCHORS = [ + { lat: 41.3874, lon: 2.1686 }, // Barcelona + { lat: 41.5636, lon: 2.0111 }, // Terrassa +]; + /** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; /** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index b564ef440..2469bee76 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -372,6 +372,36 @@ export function isLikelyFinlandCoordinate(lat, lon) { return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32; } +/** Catalonia bounding box (mainland is roughly 40.5..42.9 N, 0.15..3.4 E). */ +export function isLikelyCataloniaCoordinate(lat, lon) { + return ( + isPlausibleLatLon(lat, lon) && + lat >= 40.3 && + lat <= 42.9 && + lon >= 0.0 && + lon <= 3.4 + ); +} + +/** + * Decode decimal numeric HTML character references ("í" -> "í"), the + * entity form the Catalonia feed uses for accented place names, plus the + * handful of named entities XML/HTML text commonly carries. + * + * @param {string} text + * @returns {string} + */ +export function decodeNumericEntities(text) { + return String(text || '') + .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/</g, '<') + .replace(/>/g, '>') + .trim(); +} + /** * Human label for one Fintraffic preset (camera view). * diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 41471f5ba..3f73e70bc 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -55,6 +55,12 @@ import { DEFAULT_CALGARY_MAX_SOURCES, CALGARY_DOWNTOWN, CALGARY_MAX_CATALOG_BYTES, + CATALONIA_CAMERAS_URL, + CATALONIA_IMAGE_HOSTS, + CATALONIA_FONT_CREDIT, + CATALONIA_FONT_ELEVATION_M, + DEFAULT_CATALONIA_MAX_SOURCES, + CATALONIA_ANCHORS, CCTV_SOURCE_FETCH_TIMEOUT_MS, } from './constants.js'; import { @@ -73,6 +79,8 @@ import { isLikelyTexasCoordinate, isLikelyNswCoordinate, isLikelyCalgaryCoordinate, + isLikelyCataloniaCoordinate, + decodeNumericEntities, cameraDisplayCode, rowArrayToObject, prioritizeSources, @@ -1594,3 +1602,198 @@ export async function loadCalgarySourcesFromOpenData() { return []; } } + +/** + * Parse the Catalonia (`cameres.xml`) WFS/GML feed into raw feature records. + * Regex-based rather than a full XML parser — the feed is small (~160 + * features) and this avoids a new dependency for one field set per feature. + * + * @param {string} xml + * @returns {Array<{fid:string, lat:number, lon:number, carretera:string, municipi:string, pk:string, link:string, font:string}>} + */ +export function parseCataloniaXml(xml) { + const out = []; + const blockRe = /([\s\S]*?)<\/gml:featureMember>/g; + const tag = (block, name) => { + const match = new RegExp(`([^<]*)<\\/cite:${name}>`).exec( + block, + ); + return match ? decodeNumericEntities(match[1]) : ''; + }; + let match; + while ((match = blockRe.exec(String(xml || ''))) !== null) { + const block = match[1]; + const fidMatch = /]*>([^<]+)<\/gml:coordinates>/.exec( + block, + ); + if (!fidMatch || !coordsMatch) continue; + const [lonRaw, latRaw] = coordsMatch[1].split(','); + out.push({ + fid: fidMatch[1], + lon: toFiniteNumber(lonRaw), + lat: toFiniteNumber(latRaw), + carretera: tag(block, 'carretera'), + municipi: tag(block, 'municipi'), + pk: tag(block, 'pk'), + link: tag(block, 'link'), + font: tag(block, 'font'), + }); + } + return out; +} + +/** + * Validate and normalize one Catalonia feed `link` against the publisher + * host allowlist (CATALONIA_IMAGE_HOSTS), upgrading to https. Rejects + * anything else, including a host that merely resembles an allowed one. + * + * SCT's own `RenderService` frame link is a special case, for two stacked + * reasons (both verified 2026-09-15): + * 1. It always 302s to `TransitCamera`, on a Location the upstream + * hardcodes as plain http even when RenderService itself was requested + * over https. fetchWithinHost (media.js) only follows same-origin + * redirects — same scheme included, by design, to stop an upstream + * steering a host-pinned request into a plaintext downgrade — so that + * hop is refused and the camera would render as a placeholder. + * 2. mct.gencat.cat's TLS setup itself does not complete a handshake with + * Node's fetch/OpenSSL (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though it + * is reachable over https from other clients (e.g. curl) — an upstream + * server misconfiguration this proxy cannot route around. + * TransitCamera answers the frame directly over **http**, no redirect, and + * that is the only transport this host actually serves to this runtime — so + * this builds that URL itself from RenderService's `sctidcam` id instead of + * registering the (https, then-redirecting) RenderService link. + * + * @param {string} rawUrl + * @returns {string} Normalized URL, or '' if not accepted. + */ +export function normalizeCataloniaImageUrl(rawUrl) { + let parsed; + try { + parsed = new URL(String(rawUrl || '').trim()); + } catch { + return ''; + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return ''; + if (!CATALONIA_IMAGE_HOSTS.has(parsed.hostname)) return ''; + + if (parsed.hostname === 'mct.gencat.cat') { + const camId = parsed.searchParams.get('sctidcam'); + if (!camId) return ''; + const direct = new URL('http://mct.gencat.cat/mct2bo/TransitCamera'); + direct.searchParams.set('nom', camId); + direct.searchParams.set('visualitzacio', 'imatge'); + return direct.toString(); + } + + parsed.protocol = 'https:'; + return parsed.toString(); +} + +/** + * Human label for one Catalonia camera: " ()", + * skipping the parenthetical when the municipality is already named in the + * road/place text or when one half is missing. + * + * @param {string} carretera + * @param {string} municipi + * @returns {string} + */ +export function cataloniaCameraName(carretera, municipi) { + const road = String(carretera || '').trim(); + const city = String(municipi || '').trim(); + if (road && city) { + return road.toLowerCase().includes(city.toLowerCase()) + ? road + : `${road} (${city})`; + } + return road || city || 'Càmera de trànsit'; +} + +/** + * Fetch Catalonia traffic cameras (Servei Català de Trànsit), keyless: one + * WFS/GML XML list. The feed carries a `font` field naming the real + * publisher per camera; only SCT's own highway cameras and the hotlinked + * Barcelona (IMI) and Terrassa municipal cameras are kept, each confirmed to + * return a real frame — frame URLs are checked against a host allowlist + * (CATALONIA_IMAGE_HOSTS), so a row from any other publisher is dropped. + * Each non-SCT camera carries its publisher as a `credit` alongside the + * shared `provider`. The feed carries no heading, so every camera takes the + * id-hash fallback, low-confidence pose (same as TfL and Fintraffic). + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadCataloniaSourcesFromOpenData() { + try { + const resp = await fetch(CATALONIA_CAMERAS_URL, { + headers: { Accept: 'application/xml,text/xml,*/*' }, + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + if (!resp.ok) { + console.warn('[CCTV] Catalonia camera download failed:', resp.status); + return []; + } + const xml = await resp.text(); + const rows = parseCataloniaXml(xml); + if (!rows.length) return []; + + const cameras = []; + for (const row of rows) { + if (!row.fid) continue; + if (!isLikelyCataloniaCoordinate(row.lat, row.lon)) continue; + const imageUrl = normalizeCataloniaImageUrl(row.link); + if (!imageUrl) continue; + + const font = row.font.trim(); + const cameraId = `cat-${row.fid + .replace(/^cameres\.fid-/, '') + .replace(/[^a-zA-Z0-9]/g, '')}`; + + cameras.push({ + id: cameraId, + name: cataloniaCameraName(row.carretera, row.municipi), + city: row.municipi.trim() || 'Catalunya', + cityId: 'catalunya', + provider: 'Servei Català de Trànsit', + lat: row.lat, + lon: row.lon, + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + groundElevationM: CATALONIA_FONT_ELEVATION_M[font] ?? 80, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'catalonia-open-data', + license: + "Generalitat de Catalunya — Llicència oberta d'ús d'informació", + credit: CATALONIA_FONT_CREDIT[font] || '', + }); + } + + const unique = Array.from( + new Map(cameras.map((camera) => [camera.id, camera])).values(), + ); + const maxRaw = Number( + process.env.CCTV_CATALONIA_MAX_SOURCES || DEFAULT_CATALONIA_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(600, Math.floor(maxRaw))) + : DEFAULT_CATALONIA_MAX_SOURCES; + const prioritized = prioritizeSources(unique, maxCount, CATALONIA_ANCHORS); + console.log( + `[CCTV] Loaded Catalonia camera sources: ${unique.length} (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn( + '[CCTV] Catalonia camera download error:', + error?.message || error, + ); + return []; + } +} diff --git a/src/data/cctvCataloniaSource.test.mjs b/src/data/cctvCataloniaSource.test.mjs new file mode 100644 index 000000000..d9b9f3456 --- /dev/null +++ b/src/data/cctvCataloniaSource.test.mjs @@ -0,0 +1,317 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; +import { CATALONIA_CAMERAS_URL } from '../../server/providers/cctv/constants.js'; +import { + parseCataloniaXml, + normalizeCataloniaImageUrl, + cataloniaCameraName, + loadCataloniaSourcesFromOpenData, +} from '../../server/providers/cctv/sources.js'; + +/** Set (or, for `undefined`, delete) environment variables for one test. */ +function withEnv(t, env) { + for (const [name, value] of Object.entries(env)) { + const previous = process.env[name]; + t.after(() => { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + }); + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +} + +/** Silence the loaders' progress and failure logging. */ +function quiet(t) { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +} + +/** One `` block matching the real feed's shape. */ +function cataloniaFeature({ + fid, + lon, + lat, + carretera = '', + municipi = '', + pk = '', + link = '', + font = '', +}) { + const pkTag = pk ? `${pk}` : ''; + return ( + `` + + `` + + `${lon},${lat}` + + `` + + `${carretera}` + + `${municipi}${pkTag}` + + `${link}` + + `${font}` + + `` + ); +} + +function cataloniaFeed(features) { + return ( + '' + + features.join('') + + '' + ); +} + +test('parseCataloniaXml extracts every field from a feature block', () => { + const xml = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-52bf6eca_1a0a06bea1c_386f', + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + municipi: 'Nus Trinitat', + pk: '0.50', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]); + const rows = parseCataloniaXml(xml); + assert.equal(rows.length, 1); + assert.deepEqual(rows[0], { + fid: 'cameres.fid-52bf6eca_1a0a06bea1c_386f', + lon: 2.1849528, + lat: 41.45989301, + carretera: 'C-58', + municipi: 'Nus Trinitat', + pk: '0.50', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }); +}); + +test('normalizeCataloniaImageUrl rewrites SCT RenderService links to the direct http TransitCamera URL', () => { + // RenderService always 302s to a hardcoded http:// Location (which + // fetchWithinHost refuses as a scheme downgrade), and mct.gencat.cat's TLS + // setup fails the handshake with Node's fetch regardless — TransitCamera + // over plain http is the only transport that actually works here, so we + // build that URL from RenderService's sctidcam id instead of registering + // the redirector. + assert.equal( + normalizeCataloniaImageUrl( + 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + ), + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=nc87.gif&visualitzacio=imatge', + ); + assert.equal( + normalizeCataloniaImageUrl( + 'https://mct.gencat.cat/mct2bo/RenderService?sctidcam=sc64.gif', + ), + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=sc64.gif&visualitzacio=imatge', + 'an already-https RenderService link is rewritten the same way', + ); + assert.equal( + normalizeCataloniaImageUrl('https://mct.gencat.cat/mct2bo/RenderService'), + '', + 'a RenderService link with no sctidcam id is rejected', + ); +}); + +test('normalizeCataloniaImageUrl accepts the other registered hosts, upgraded to https', () => { + assert.equal( + normalizeCataloniaImageUrl('http://www.bcn.cat/transit/imatges/a.gif'), + 'https://www.bcn.cat/transit/imatges/a.gif', + ); + assert.equal( + normalizeCataloniaImageUrl( + 'https://emap.terrassa.cat/it_terrassa/cam02.jpeg', + ), + 'https://emap.terrassa.cat/it_terrassa/cam02.jpeg', + ); + assert.equal( + normalizeCataloniaImageUrl('https://evil.example.com/mct.gencat.cat.gif'), + '', + 'a look-alike host is rejected', + ); + assert.equal( + normalizeCataloniaImageUrl('https://unlisted-publisher.example/frame.gif'), + '', + 'any host outside the allowlist is rejected', + ); + assert.equal(normalizeCataloniaImageUrl(''), ''); + assert.equal(normalizeCataloniaImageUrl('not a url'), ''); +}); + +test('cataloniaCameraName joins road and municipality, skipping a redundant municipality', () => { + assert.equal( + cataloniaCameraName('C-58', 'Nus Trinitat'), + 'C-58 (Nus Trinitat)', + ); + assert.equal( + cataloniaCameraName('Plaça Urquinaona', 'Barcelona'), + 'Plaça Urquinaona (Barcelona)', + ); + assert.equal( + cataloniaCameraName('Barcelona Ronda Litoral', 'Barcelona'), + 'Barcelona Ronda Litoral', + ); + assert.equal(cataloniaCameraName('', ''), 'Càmera de trànsit'); + assert.equal(cataloniaCameraName('C-58', ''), 'C-58'); +}); + +test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publisher credit', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + const xml = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-1', + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + municipi: 'Nus Trinitat', + pk: '0.50', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: 'cameres.fid-2', + lon: '2.17228212', + lat: '41.38890733', + carretera: 'Plaça Urquinaona', + municipi: 'Barcelona', + link: 'http://www.bcn.cat/transit/imatges/PlUrquinaona.gif?a=1', + font: 'IMI', + }), + cataloniaFeature({ + fid: 'cameres.fid-3', + lon: '2.0111', + lat: '41.5636', + carretera: 'Cam02', + municipi: 'Terrassa', + link: 'https://emap.terrassa.cat/it_terrassa/cam02.jpeg?a=1', + font: 'Terrassa', + }), + // Rejected: link host not in the allowlist. + cataloniaFeature({ + fid: 'cameres.fid-4', + lon: '2.0', + lat: '41.5', + link: 'https://example.com/fake.gif', + font: 'SCT', + }), + // Rejected: outside the Catalonia bounding box. + cataloniaFeature({ + fid: 'cameres.fid-5', + lon: '-3.7', + lat: '40.4', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=x.gif', + font: 'SCT', + }), + ]); + return new Response(xml, { + status: 200, + headers: { 'Content-Type': 'application/xml' }, + }); + }); + + const cameras = await loadCataloniaSourcesFromOpenData(); + + assert.deepEqual(requested, [CATALONIA_CAMERAS_URL]); + assert.deepEqual( + cameras.map((camera) => camera.id).sort(), + ['cat-1', 'cat-2', 'cat-3'], + 'the off-allowlist link (fid-4) and the out-of-bbox row (fid-5) are excluded', + ); + + const sct = cameras.find((c) => c.id === 'cat-1'); + assert.equal(sct.name, 'C-58 (Nus Trinitat)'); + assert.equal(sct.provider, 'Servei Català de Trànsit'); + assert.equal(sct.credit, '', 'SCT is the publisher, no separate credit'); + assert.equal( + sct.url, + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=nc87.gif&visualitzacio=imatge', + 'registers the direct, non-redirecting http frame URL, not the RenderService redirector', + ); + assert.equal(sct.groundElevationM, 80); + assert.equal(sct.headingConfidence, 'low', 'the feed carries no heading'); + + const bcn = cameras.find((c) => c.id === 'cat-2'); + assert.equal(bcn.credit, 'Ajuntament de Barcelona'); + assert.equal(bcn.city, 'Barcelona'); + assert.equal(bcn.groundElevationM, 20); + + const terrassa = cameras.find((c) => c.id === 'cat-3'); + assert.equal(terrassa.credit, 'Ajuntament de Terrassa'); + assert.equal(terrassa.groundElevationM, 270); +}); + +test('Catalonia loader fails soft on HTTP errors, empty payloads and network errors', async (t) => { + quiet(t); + let respond; + t.mock.method(globalThis, 'fetch', (...args) => respond(...args)); + for (respond of [ + async () => new Response('unavailable', { status: 503 }), + async () => new Response(''), + async () => { + throw new TypeError('fetch failed'); + }, + ]) { + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + } +}); + +test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips the request', async (t) => { + quiet(t); + const sourceRoot = fs.mkdtempSync( + path.join(os.tmpdir(), 'gev-cctv-catalog-catalonia-'), + ); + t.after(() => fs.rmSync(sourceRoot, { recursive: true, force: true })); + withEnv(t, { + CCTV_SOURCES_FILE: undefined, + CCTV_SOURCES_JSON: undefined, + CCTV_FORCE_AUSTIN: undefined, + CCTV_PREFER_AUSTIN: undefined, + CCTV_MAX_SOURCES: undefined, + CCTV_CALTRANS_DISTRICTS: undefined, + CCTV_TFL_ENABLED: undefined, + CCTV_CATALONIA_ENABLED: undefined, + CCTV_CATALONIA_MAX_SOURCES: undefined, + }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + if (String(url) !== CATALONIA_CAMERAS_URL) { + return new Response('unavailable', { status: 503 }); + } + const xml = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-1', + lon: '2.1849528', + lat: '41.45989301', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]); + return new Response(xml, { status: 200 }); + }); + + const sources = await createCctvCatalog({ sourceRoot })(); + assert.deepEqual( + sources.map((source) => source.id), + ['cat-1'], + ); + + process.env.CCTV_CATALONIA_ENABLED = '0'; + requested.length = 0; + assert.deepEqual(await createCctvCatalog({ sourceRoot })(), []); + assert.ok(requested.length > 0, 'the other live packs still load'); + assert.equal( + requested.includes(CATALONIA_CAMERAS_URL), + false, + 'a disabled pack makes no request', + ); +}); diff --git a/src/data/cctvDriveBcSource.test.mjs b/src/data/cctvDriveBcSource.test.mjs index 617bb1350..a4934af3c 100644 --- a/src/data/cctvDriveBcSource.test.mjs +++ b/src/data/cctvDriveBcSource.test.mjs @@ -17,6 +17,7 @@ import { DEFAULT_TALLINN_MAX_SOURCES, DEFAULT_TARKTEE_MAX_SOURCES, DEFAULT_NSW_MAX_SOURCES, + DEFAULT_CATALONIA_MAX_SOURCES, DRIVEBC_WEBCAMS_URL, } from '../../server/providers/cctv/constants.js'; import { @@ -234,6 +235,7 @@ test('default per-pack camera caps fit inside the default catalog cap', () => { DEFAULT_TALLINN_MAX_SOURCES + DEFAULT_TARKTEE_MAX_SOURCES + DEFAULT_NSW_MAX_SOURCES + + DEFAULT_CATALONIA_MAX_SOURCES + 1; assert.ok( packs <= DEFAULT_CCTV_MAX_SOURCES, diff --git a/src/data/cctvLod.js b/src/data/cctvLod.js index 65ddc2e12..44b2e8ba3 100644 --- a/src/data/cctvLod.js +++ b/src/data/cctvLod.js @@ -38,6 +38,10 @@ const PROVIDER_STATIC_REFRESH_MS = Object.freeze({ // collectionInterval, 600 s for every station sampled; polling faster only // re-fetches the same JPEG. fintraffic: 10 * 60 * 1000, + // Measured empirically (2026-09-15): polling the same SCT/Terrassa frame + // every 20s showed no change inside 64s but a new frame by ~90-150s later — + // matches the other highway packs' 3-minute cadence. + 'servei català de trànsit': 3 * 60 * 1000, }); /** diff --git a/src/data/cctvLod.test.mjs b/src/data/cctvLod.test.mjs index c6f6e4923..4f5982496 100644 --- a/src/data/cctvLod.test.mjs +++ b/src/data/cctvLod.test.mjs @@ -641,6 +641,10 @@ test('staticFrameRefreshMs follows known pack cadences and bounds explicit value assert.equal(staticFrameRefreshMs({ provider: 'TxDOT' }), 180_000); // Digitraffic publishes one weathercam frame per 600 s collection interval. assert.equal(staticFrameRefreshMs({ provider: 'Fintraffic' }), 600_000); + assert.equal( + staticFrameRefreshMs({ provider: 'Servei Català de Trànsit' }), + 180_000, + ); assert.equal(staticFrameRefreshMs({ frameRefreshMs: 5_000 }), 60_000); assert.equal(staticFrameRefreshMs({ frameRefreshMs: 3_000_000 }), 1_200_000); assert.equal(staticFrameRefreshMs({ provider: 'Unknown Provider' }), 300_000); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 76029ace8..c44ac3ca6 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -169,6 +169,14 @@ export const DATA_CREDITS = [ 'Traffic cameras (Calgary): contains information licensed under the ' + 'Open Government Licence – City of Calgary', }, + { + key: 'catalonia-cctv', + html: + 'CCTV cameras & frames (Catalonia): ' + + 'Servei Català de Trànsit ' + + '(Llicència oberta d’ús d’informació - Catalunya); some cameras are supplied by partners ' + + '(Ajuntament de Barcelona, Ajuntament de Terrassa), each named in the CCTV panel', + }, { key: 'gbfs', html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)', From ffbe98b4ac9e269fea098f26eadd01d28a0b4333 Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:38:04 +0200 Subject: [PATCH 03/95] test(format-scope): register cctvCataloniaSource.test.mjs for format:check Mechanical addition per CONTRIBUTING.md, following the existing per-pack test-file entries (cctvCalgary.test.mjs, cctvNswSource.test.mjs, etc.). Co-Authored-By: Claude Sonnet 5 --- scripts/format-scope.json | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index b86632108..05fe8327a 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -489,6 +489,7 @@ "src/data/cctvEstonia.test.mjs", "src/data/cctvWarendorf.test.mjs", "src/data/cctvCalgary.test.mjs", + "src/data/cctvCataloniaSource.test.mjs", "src/data/cctvNswSource.test.mjs", "src/data/cctvTxdotSource.test.mjs", "src/layers/cctv/source.test.mjs", From 976efed6dc2fa207524def3c90e5cf9683a2323b Mon Sep 17 00:00:00 2001 From: Reloded <37674025+Reloded@users.noreply.github.com> Date: Wed, 16 Sep 2026 00:14:46 +0300 Subject: [PATCH 04/95] feat(cctv): add Lithuania Eismoinfo road cameras --- .env.example | 2 + CHANGELOG.md | 2 + DATA_SOURCES.md | 2 + docs/CURRENT-STATE.md | 14 +++ scripts/format-scope.json | 2 + scripts/package-boundaries.json | 1 + server/providers/cctv/catalog.js | 6 ++ server/providers/cctv/lithuania.js | 135 ++++++++++++++++++++++++++++ src/data/cctvLithuania.test.mjs | 138 +++++++++++++++++++++++++++++ src/data/dataCredits.js | 6 ++ 10 files changed, 308 insertions(+) create mode 100644 server/providers/cctv/lithuania.js create mode 100644 src/data/cctvLithuania.test.mjs diff --git a/.env.example b/.env.example index 5bc02df9c..02fdd78b4 100644 --- a/.env.example +++ b/.env.example @@ -171,6 +171,8 @@ AISSTREAM_API_KEY= # CCTV_CALGARY_ENABLED=1 # set 0 to disable the Open Calgary (Alberta) cameras # CCTV_CALGARY_MAX_SOURCES=220 # cap cameras loaded from the Open Calgary catalog # CCTV_CALGARY_ROWS_URL= # override the Open Calgary catalog URL +# CCTV_LITHUANIA_ENABLED=1 # set 0 to disable Via Lietuva / Eismoinfo road cameras +# CCTV_LITHUANIA_MAX_SOURCES=400 # cap Lithuanian cameras; no API key required # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/CHANGELOG.md b/CHANGELOG.md index d3ab7807e..a9a9e8687 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +- Add Lithuania's public Via Lietuva / Eismoinfo road cameras to CCTV, with published coordinates, on-demand snapshots, attribution and bounded catalog requests. No API key is required. + - Keep parked transit vehicles aligned to their world course during camera orbits, fall back to reported bearing, and keep vehicles with no course consistently screen-up. - Separate Realtime connection, response/tool, Radio, input/audio, cost, viewport diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 850d27e39..a2610887b 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -59,6 +59,8 @@ How to read this: ### Notes on the live sources +- **Via Lietuva / Eismoinfo (Lithuania).** Public road-camera catalogs and periodically updated still images from [Eismoinfo](https://eismoinfo.lt/), credited to Via Lietuva / eismoinfo.lt in the app. The provider's [open-data statement](https://eismoinfo.lt/eismoinfo-backend/content?id=44) describes reuse of its public data; no specific Creative Commons licence is asserted here. `eismoinfo-backend/camera-info-table` is joined by ID to `layer-static-features/VKR`. Without `lks=true`, the latter returns WGS84 `[latitude, longitude]`; the table's `x/y` instead use LKS-94 metres. Only the official HTTPS `/eismoinfo-backend/image-provider/camera/last?id=…` image endpoint is registered. Catalogs use the shared 15-minute cache; frames are fetched on demand and never bundled or archived. These are periodic snapshots, not continuous video. Camera headings are low-confidence estimates. No API key is required. Disable with `CCTV_LITHUANIA_ENABLED=0`; `CCTV_LITHUANIA_MAX_SOURCES` defaults to 400. + - **Google Maps Platform.** You supply your own API key and are bound by [Google's ToS](https://cloud.google.com/maps-platform/terms). Google Maps Content (tiles, geocodes, places) **may not be cached, stored, rehosted, or committed** — this app only ever uses it live, which is the compliant pattern. The "Google" attribution is displayed on the globe and must stay visible. Restrict your key (see [SECURITY.md](SECURITY.md)). - **OpenSky Network.** Its license is **non-commercial**, and operational use of the REST API in a live product can require a prior written agreement with OpenSky — even for non-profit/government use. If you deploy this commercially, contact OpenSky for your own terms. The flights layer is a toggle and runs anonymously by default. - **adsb.lol flight fallback.** When OpenSky is unavailable and no last-good OpenSky response exists, the server requests a cached, capped 250 nm adsb.lol point snapshot around the current camera subpoint. This is regional observed context, not worldwide completeness; provenance is exposed in the Flights stats/context row. Military ICAOs remain reconciled through the existing dedicated military registry rather than duplicated. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1b524a961..9d66da384 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -391,6 +391,20 @@ owners. Tuner calculations have a pure entry, with existing layer exports preserved. Destruction removes listeners and state subscriptions before ending tuning; a delayed Enable result cannot reveal or refocus removed controls. +## Lithuania CCTV source pack + +The CCTV catalog includes Via Lietuva / Eismoinfo road cameras by default. +Two keyless public endpoints provide current snapshot URLs and WGS84 camera +locations; records are joined by ID, with invalid coordinates and off-origin +image URLs rejected. Catalog requests refuse redirects, have a 15-second +deadline and a 4 MiB response cap, and use the existing shared 15-minute cache +and fair catalog allocation. An unavailable source contributes an empty pack. +Frames use the registered-image proxy and retain Via Lietuva attribution. +The images are periodically updated road-condition photographs, not continuous +video; camera poses use the existing low-confidence heading prior. +`CCTV_LITHUANIA_ENABLED=0` disables the pack, and +`CCTV_LITHUANIA_MAX_SOURCES` sets its cap (default 400, bounded to 1–1000). + ## DriveBC CCTV source pack The CCTV catalog adds DriveBC highway cameras for British Columbia alongside the diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 7f26e1e71..1bdb1c157 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -85,6 +85,8 @@ "server/providers/cctv/cap.js", "server/providers/cctv/groundHeights.js", "server/providers/cctv/catalog.js", + "server/providers/cctv/lithuania.js", + "src/data/cctvLithuania.test.mjs", "server/providers/cctv/constants.js", "server/providers/cctv/media.js", "server/providers/cctv/normalize.js", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index a5806db67..f40976374 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -156,6 +156,7 @@ "server/providers/cctv/cap.js", "server/providers/cctv/groundHeights.js", "server/providers/cctv/catalog.js", + "server/providers/cctv/lithuania.js", "server/providers/cctv/constants.js", "server/providers/cctv/media.js", "server/providers/cctv/normalize.js", diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index a8519c85c..ca8a849ab 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -4,6 +4,7 @@ import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js'; import { allocateSourceCap, resolveCatalogCap } from './cap.js'; import { loadGroundHeights, joinGroundHeights } from './groundHeights.js'; import { normalizeSourceItem } from './normalize.js'; +import { loadLithuaniaSources } from './lithuania.js'; import { loadAustinSourcesFromOpenData, loadCaltransSourcesFromOpenData, @@ -30,6 +31,11 @@ const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0'; * kill switch. */ const LIVE_PACKS = [ + { + name: 'lithuania', + enabled: () => envEnabled('CCTV_LITHUANIA_ENABLED'), + load: loadLithuaniaSources, + }, { name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData }, { name: 'caltrans', diff --git a/server/providers/cctv/lithuania.js b/server/providers/cctv/lithuania.js new file mode 100644 index 000000000..df4ee154d --- /dev/null +++ b/server/providers/cctv/lithuania.js @@ -0,0 +1,135 @@ +import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js'; +import { + cameraDisplayCode, + fallbackHeadingFromId, + prioritizeSources, +} from './normalize.js'; +import { readResponseJsonCapped } from '../common/http.js'; + +const BASE = 'https://eismoinfo.lt/eismoinfo-backend'; +export const LITHUANIA_CAMERAS_URL = `${BASE}/camera-info-table`; +// Without lks=true this endpoint returns [latitude, longitude]. The camera +// table's x/y fields use LKS-94 metres and must never be treated as degrees. +export const LITHUANIA_LOCATIONS_URL = `${BASE}/layer-static-features/VKR`; +export const LITHUANIA_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +const FRAME_PATH = '/eismoinfo-backend/image-provider/camera/last'; +const ANCHORS = [ + { lat: 54.6872, lon: 25.2797 }, // Vilnius + { lat: 54.8985, lon: 23.9036 }, // Kaunas + { lat: 55.7033, lon: 21.1443 }, // Klaipeda + { lat: 55.9349, lon: 23.3137 }, // Siauliai + { lat: 55.7348, lon: 24.3575 }, // Panevezys +]; + +/** Join public catalogs by ID; register only the official still endpoint. */ +export function lithuaniaCameraToSource(row, feature) { + const id = String(row?.id ?? ''); + if (!/^[1-9]\d{0,8}$/.test(id) || String(feature?.id) !== id) return null; + const point = feature?.points?.[0]?.point; + if (!Array.isArray(point) || point.length !== 2) return null; + const [lat, lon] = point; + if ( + !Number.isFinite(lat) || + !Number.isFinite(lon) || + lat < 53.8 || + lat > 56.5 || + lon < 20.8 || + lon > 26.9 + ) + return null; + try { + const url = new URL(row.image); + if ( + url.origin !== 'https://eismoinfo.lt' || + url.username || + url.password || + url.pathname !== FRAME_PATH || + url.searchParams.get('id') !== id + ) + return null; + } catch { + return null; + } + // Reconstruct so extra query parameters cannot alter the request. + const imageUrl = `https://eismoinfo.lt${FRAME_PATH}?id=${id}`; + const cameraId = `lithuania-${id}`; + const name = String(row.name || feature.name || `Lietuva ${id}`) + .trim() + .slice(0, 160); + return { + id: cameraId, + name, + city: 'Lithuania / Lietuva', + cityId: 'lithuania', + provider: 'Via Lietuva / Eismoinfo', + lat, + lon, + // No calibrated pose is published; use the app's low-confidence prior. + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'eismoinfo-public', + credit: + 'Via Lietuva / eismoinfo.lt — periodically updated road-camera stills', + code: cameraDisplayCode(name.toUpperCase()), + }; +} + +async function readCatalog(url) { + const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS); + const response = await fetch(url, { + headers: { Accept: 'application/json' }, + redirect: 'manual', + signal, + }); + if (!response.ok) { + await response.body?.cancel(); + throw new Error(`Catalog HTTP ${response.status}`); + } + return readResponseJsonCapped(response, LITHUANIA_MAX_CATALOG_BYTES, signal); +} + +/** Two bounded requests per shared catalog refresh, no API key required. */ +export async function loadLithuaniaSources() { + try { + const [rows, layers] = await Promise.all([ + readCatalog(LITHUANIA_CAMERAS_URL), + readCatalog(LITHUANIA_LOCATIONS_URL), + ]); + if (!Array.isArray(rows) || !Array.isArray(layers)) return []; + const features = layers.find((layer) => layer?.layer === 'VKR')?.features; + if (!Array.isArray(features)) return []; + const locations = new Map( + features.filter(Boolean).map((feature) => [String(feature.id), feature]), + ); + const cameras = new Map(); + for (const row of rows) { + const camera = lithuaniaCameraToSource( + row, + locations.get(String(row?.id)), + ); + if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera); + } + const rawCap = Number(process.env.CCTV_LITHUANIA_MAX_SOURCES || 400); + const cap = Number.isFinite(rawCap) + ? Math.max(1, Math.min(1000, Math.floor(rawCap))) + : 400; + const sources = prioritizeSources([...cameras.values()], cap, ANCHORS); + console.log( + `[CCTV] Loaded Lithuania camera sources: ${cameras.size} (using ${sources.length})`, + ); + return sources; + } catch (error) { + console.warn( + '[CCTV] Lithuania camera download error:', + error?.message || error, + ); + return []; + } +} diff --git a/src/data/cctvLithuania.test.mjs b/src/data/cctvLithuania.test.mjs new file mode 100644 index 000000000..675e8d251 --- /dev/null +++ b/src/data/cctvLithuania.test.mjs @@ -0,0 +1,138 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + lithuaniaCameraToSource, + loadLithuaniaSources, + LITHUANIA_CAMERAS_URL, + LITHUANIA_LOCATIONS_URL, + LITHUANIA_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/lithuania.js'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; + +const row = { + id: 72, + name: 'Vilnius A1 10,04', + x: 576154, + y: 6056867, + image: + 'https://eismoinfo.lt/eismoinfo-backend/image-provider/camera/last?id=72', +}; +const feature = { + id: '72', + points: [{ point: [54.642550940833566, 25.17981587877114] }], +}; +const layers = [{ layer: 'VKR', features: [feature] }]; + +test('Lithuania joins by ID and uses WGS84 latitude/longitude, never LKS-94 metres', () => { + const camera = lithuaniaCameraToSource(row, feature); + assert.equal(camera.id, 'lithuania-72'); + assert.equal(camera.lat, 54.642550940833566); + assert.equal(camera.lon, 25.17981587877114); + assert.equal(camera.name, 'Vilnius A1 10,04'); + assert.equal(camera.feedType, 'image'); + assert.equal(camera.headingConfidence, 'low'); + assert.equal(camera.url, row.image); + assert.match(camera.credit, /Via Lietuva/); +}); + +test('invalid joins, coordinates and image destinations cannot enter the proxy registry', () => { + for (const invalid of [ + null, + { ...feature, id: '8' }, + { ...feature, points: [] }, + { ...feature, points: [{ point: [25.18, 54.64] }] }, + { ...feature, points: [{ point: [576154, 6056867] }] }, + { ...feature, points: [{ point: [null, 25] }] }, + ]) { + assert.equal(lithuaniaCameraToSource(row, invalid), null); + } + for (const image of [ + 'http://eismoinfo.lt/eismoinfo-backend/image-provider/camera/last?id=72', + row.image.replace('eismoinfo.lt', 'eismoinfo.lt.evil.test'), + row.image.replace('https://', 'https://user:pass@'), + row.image.replace('id=72', 'id=8'), + 'http://127.0.0.1/private', + '', + null, + ]) { + assert.equal(lithuaniaCameraToSource({ ...row, image }, feature), null); + } + assert.equal( + lithuaniaCameraToSource( + { ...row, image: row.image + '&extra=ignored' }, + feature, + ).url, + row.image, + ); + assert.equal(lithuaniaCameraToSource(null, feature), null); +}); + +function mockCatalogs(t) { + const requested = []; + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async (url, options) => { + requested.push(String(url)); + if (url === LITHUANIA_CAMERAS_URL) { + assert.equal(options.redirect, 'manual'); + return Response.json([row, row, { ...row, id: 999 }]); + } + if (url === LITHUANIA_LOCATIONS_URL) return Response.json(layers); + return Response.json([]); + }); + return requested; +} + +test('loader deduplicates and omits cameras without matching published coordinates', async (t) => { + const requested = mockCatalogs(t); + assert.deepEqual( + (await loadLithuaniaSources()).map((c) => c.id), + ['lithuania-72'], + ); + assert.deepEqual( + requested.sort(), + [LITHUANIA_CAMERAS_URL, LITHUANIA_LOCATIONS_URL].sort(), + ); +}); + +test('catalog includes Lithuania by default and respects the disable switch', async (t) => { + const requested = mockCatalogs(t); + const saved = process.env.CCTV_LITHUANIA_ENABLED; + try { + delete process.env.CCTV_LITHUANIA_ENABLED; + const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); + assert.equal( + sources.find((c) => c.id === 'lithuania-72')?.cityId, + 'lithuania', + ); + requested.length = 0; + process.env.CCTV_LITHUANIA_ENABLED = '0'; + const disabled = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); + assert.ok(!disabled.some((c) => c.cityId === 'lithuania')); + assert.ok(!requested.includes(LITHUANIA_CAMERAS_URL)); + } finally { + if (saved === undefined) delete process.env.CCTV_LITHUANIA_ENABLED; + else process.env.CCTV_LITHUANIA_ENABLED = saved; + } +}); + +test('failed, redirected, malformed and oversized catalogs degrade to an empty pack', async (t) => { + t.mock.method(console, 'warn', () => {}); + for (const makeResponse of [ + () => new Response('', { status: 503 }), + () => + new Response('', { + status: 302, + headers: { Location: 'http://127.0.0.1/' }, + }), + () => new Response('invalid json'), + () => Response.json({ unexpected: true }), + () => + new Response('[]', { + headers: { 'Content-Length': String(LITHUANIA_MAX_CATALOG_BYTES + 1) }, + }), + ]) { + t.mock.method(globalThis, 'fetch', async () => makeResponse()); + assert.deepEqual(await loadLithuaniaSources(), []); + } +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 76029ace8..d5ee02c54 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -163,6 +163,12 @@ export const DATA_CREDITS = [ 'digitraffic.fi, ' + 'license CC BY 4.0', }, + { + key: 'lithuania-cctv', + html: + 'Road-camera stills (Lithuania): ' + + 'Via Lietuva / Eismoinfo; periodically updated images', + }, { key: 'calgary-cctv', html: From c5b1d41cbade33efd598286a104e49b9cd1d2952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Saggese?= Date: Mon, 21 Sep 2026 11:30:49 +0200 Subject: [PATCH 05/95] fix(search): geocode through the server, not the browser - Google's Geocoding web service refuses referrer-restricted keys, so the browser call forced GOOGLE_MAPS_API_KEY to ship unrestricted - add /api/google/geocode and /api/google/reverse-geocode, using the server key like Places and Street View already do - the browser key now needs the Map Tiles API only --- CHANGELOG.md | 9 ++ SECURITY.md | 6 +- docs/CURRENT-STATE.md | 19 ++- server/providers/local.js | 3 + server/providers/places.js | 7 +- server/providers/places/google-key.js | 33 +++- server/providers/places/google.js | 155 ++++++++++++++++++ src/annotations/annotationEngine.test.mjs | 2 +- src/annotations/annotationResolver.test.mjs | 20 +-- src/googleGeocodeProxy.test.mjs | 167 ++++++++++++++++++++ src/googleServerKey.test.mjs | 16 +- src/search/defaults.js | 23 +-- src/search/http.js | 15 +- src/search/placeSearch.test.mjs | 2 +- src/voice/gevActions.test.mjs | 6 +- 15 files changed, 436 insertions(+), 47 deletions(-) create mode 100644 src/googleGeocodeProxy.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 744f4be0a..526a89bc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +- Move Google geocoding behind the local server. Forward and reverse lookups now + go through `/api/google/geocode` and `/api/google/reverse-geocode` with the + server key, joining Places and Street View. Google's Geocoding web service + refuses referrer-restricted keys, so the previous browser call left + `SECURITY.md`'s advice and working search mutually exclusive: the bundled + browser key had to be unrestricted for search to work at all. That key now + needs the Map Tiles API only. Keyless sessions and the Photon/Nominatim + fallback order are unchanged. + - Enable responsive trackpad pinch zoom on the globe. Browser pixel-mode `Ctrl+wheel` pinch gestures now reach Cesium with bounded amplification, while ordinary wheel, line-mode and touch-pinch inputs retain their existing diff --git a/SECURITY.md b/SECURITY.md index a21ea99a4..661e8764f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -20,18 +20,18 @@ The golden rule: **secret-bearing API keys stay on the server side.** The dev/pr | `OPENAI_API_KEY` | Server only | Browser fetches a short-lived **ephemeral** Realtime session token from `/api/realtime/token`; the real key never ships | | `AISSTREAM_API_KEY` | Server only | Server holds the AISStream websocket; browser polls the same-origin `/api/ais-live` cache | | OpenSky OAuth (`OPENSKY_CLIENT_ID/SECRET`) | Server only | Server mints + refreshes the token behind `/api/opensky` | -| `GOOGLE_MAPS_SERVER_API_KEY` (optional, #33) | Server only | Server calls Places (`/api/google/nearby-places`, `/api/google/text-search`) and the Street View fallback with this key; falls back to `GOOGLE_MAPS_API_KEY` when unset | +| `GOOGLE_MAPS_SERVER_API_KEY` (optional, #33) | Server only | Server calls Places (`/api/google/nearby-places`, `/api/google/text-search`), Geocoding (`/api/google/geocode`, `/api/google/reverse-geocode`) and the Street View fallback with this key; falls back to `GOOGLE_MAPS_API_KEY` when unset | ### Two deliberately client-side keys — restrict them These are designed to be used directly in the browser (like a Mapbox public token). They are injected into the client bundle via Vite's `define`, so they **will** be visible in browser devtools. Scope and restrict them rather than trying to hide them: -1. **Google Maps API key** — loads Photorealistic 3D Tiles directly and powers GEV place search. **Restrict it** (HTTP referrer + API restriction to the required Google APIs) in the Google Cloud Console. An unrestricted key in a public deployment can be abused and billed to you. +1. **Google Maps API key** — loads Photorealistic 3D Tiles directly. **Restrict it** (HTTP referrer + API restriction to the required Google APIs) in the Google Cloud Console. An unrestricted key in a public deployment can be abused and billed to you. Restrict it to the **Map Tiles API only**: a referrer restriction applies to the Maps JavaScript API and the client SDKs, not to the web services, so a key carrying Geocoding under one is refused with `API keys with referer restrictions cannot be used with this API`. Search does not need it — geocoding goes through the server proxies above (#363). 2. **Cesium ion token** (`CESIUM_ION_TOKEN`, optional — for ion-hosted Google Photorealistic 3D Tiles, Bing world imagery, and world terrain) — used as `Cesium.Ion.defaultAccessToken` client-side. Use a public **`assets:read`** token with **URL restrictions** for any hosted deployment. The Community plan has eligibility and usage limits; a public token is not a secret, but it can still consume the account's quota. > The explicit browser `define` block in `build/vite.js` controls exactly what reaches the client: only these two keys. Everything else stays server-side. -**Places and Street View never needed to be on that list** (#33): they're called from the server-side proxies in the table above, which use `GOOGLE_MAPS_SERVER_API_KEY` when it's set. Splitting it from the browser-exposed key lets each key's Google Cloud restriction actually match what it does — the browser key referrer-restricted to the APIs the client loads, the server key IP-restricted (never a referrer, since it never leaves your server) to Places + Street View Static — instead of one key that has to be either over-permissioned or broken for one of its two jobs. A single shared `GOOGLE_MAPS_API_KEY` still works if you don't split them; it just has to cover every API both sides use. +**Places, Geocoding and Street View never needed to be on that list** (#33, #363): they're called from the server-side proxies in the table above, which use `GOOGLE_MAPS_SERVER_API_KEY` when it's set. Splitting it from the browser-exposed key lets each key's Google Cloud restriction actually match what it does — the browser key referrer-restricted to Map Tiles, the server key IP-restricted (never a referrer, since it never leaves your server) to Places + Geocoding + Street View Static — instead of one key that has to be either over-permissioned or broken for one of its jobs. A single shared `GOOGLE_MAPS_API_KEY` still works if you don't split them, but then it cannot carry a referrer restriction at all: the web services refuse one, and search would be the half that breaks. Never commit real keys. `.env` is gitignored; only `.env.example` (placeholder names) is tracked. On macOS `dev-fresh.sh` can read keys from the Keychain; plain Vite uses env vars or a local `.env`, and Pinokio uses its ignored app `ENVIRONMENT` file. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9c381ac0c..e038bfe20 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3445,7 +3445,24 @@ Providers normalize coordinates, canonical name, label, place types and optional bounds. Camera framing, nearby landmark recovery and footprint selection remain in their consumers, including the Capitol identity/containment safeguards. Radio keeps localized country names in labels rather than station filters. -Reverse geocoding and nearby/text-search endpoints retain their existing behavior. +Nearby/text-search endpoints retain their existing behavior. + +**Google geocoding is a server route, not a browser call** (#363). Forward and +reverse lookups go to `/api/google/geocode` and `/api/google/reverse-geocode`, +which carry `GOOGLE_MAPS_SERVER_API_KEY` (falling back to `GOOGLE_MAPS_API_KEY`) +and share the Places routes' opt-in `GEV_RATELIMIT_GOOGLE_PER_MIN` throttle. +Google's Geocoding **web service** refuses referrer-restricted keys, so the +previous browser call forced the bundled browser key — visible in devtools by +design — to be left unrestricted; it now needs the Map Tiles API only. Both +routes answer in Google's own `status`/`results` shape rather than the Places +routes' `places: []` contract, because the browser adapters normalize that shape +directly. The browser key still gates the provider client-side, so a keyless +session spends no request to learn it has no Google: the route's own keyless +answer reads as an unanswered provider, which would stop the chain caching the +negative Photon or Nominatim then produces. Its query is capped at 200 +characters, its reverse coordinates are validated against WGS84 bounds before +the request is paid for, and a malformed viewport bias is dropped rather than +refused, since a bias only ranks results. Only valid answers and definitive misses enter bounded caches; malformed replies, HTTP refusals and outages remain retryable. Searches share a 12-second total diff --git a/server/providers/local.js b/server/providers/local.js index 68f61a3cb..07e1a5bb8 100644 --- a/server/providers/local.js +++ b/server/providers/local.js @@ -66,6 +66,9 @@ export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js'; export { googlePlacesContextProxy } from './places.js'; export { googleServerApiKey } from './places.js'; export { keylessGooglePlacesResponse } from './places.js'; +export { keylessGoogleGeocodeResponse } from './places.js'; +export { validateGeocodeAddress } from './places.js'; +export { geocodeBounds } from './places.js'; export { adsbLolFallbackAnchor } from './aircraft/opensky.js'; export { readResponseTextCapped } from './common/http.js'; export { readResponseJsonCapped } from './common/http.js'; diff --git a/server/providers/places.js b/server/providers/places.js index 648be2aa8..6c36627e0 100644 --- a/server/providers/places.js +++ b/server/providers/places.js @@ -1,8 +1,13 @@ /** Place-search and route middleware for Node development servers. */ -export { googlePlacesContextProxy } from './places/google.js'; +export { + googlePlacesContextProxy, + validateGeocodeAddress, + geocodeBounds, +} from './places/google.js'; export { googleServerApiKey, keylessGooglePlacesResponse, + keylessGoogleGeocodeResponse, } from './places/google-key.js'; export { installRouteMiddleware } from './places/routes.js'; export { diff --git a/server/providers/places/google-key.js b/server/providers/places/google-key.js index 353bc9e24..a5c54b302 100644 --- a/server/providers/places/google-key.js +++ b/server/providers/places/google-key.js @@ -14,12 +14,33 @@ export function keylessGooglePlacesResponse(apiKey) { } /** - * Google API key for the SERVER-SIDE calls (Places nearby/text search, the - * CCTV Street View fallback). These never reach the browser, so this key can - * be restricted by server IP and scoped to Places API + Street View Static - * API — while GOOGLE_MAPS_API_KEY stays referrer-restricted to Map Tiles + - * Geocoding for the browser (#33). Splitting them is opt-in: unset, this - * falls back to the shared browser key and nothing changes. + * Geocoding speaks Google's own response shape rather than the `places: []` + * contract its two siblings use, because the browser adapters read `status` + * and `results` directly. The keyless capability reads the same way: a status + * that is not `OK` and no results, which the adapters already treat as "this + * provider did not answer" and pass to the next geocoder in the chain. + */ +export function keylessGoogleGeocodeResponse(apiKey) { + if (String(apiKey ?? '').trim()) return null; + return { + statusCode: 200, + payload: { + configured: false, + status: 'REQUEST_DENIED', + results: [], + error: null, + }, + }; +} + +/** + * Google API key for the SERVER-SIDE calls (Places nearby/text search, + * geocoding, the CCTV Street View fallback). These never reach the browser, so + * this key can be restricted by server IP and scoped to Places API + Geocoding + * API + Street View Static API — while GOOGLE_MAPS_API_KEY stays + * referrer-restricted to Map Tiles for the browser (#33, #363). Splitting them + * is opt-in: unset, this falls back to the shared browser key, which then has + * to carry every API both sides use and cannot take a referrer restriction. */ export function googleServerApiKey() { return resolveGoogleServerKey(process.env); diff --git a/server/providers/places/google.js b/server/providers/places/google.js index 1ee83a892..8895bc19d 100644 --- a/server/providers/places/google.js +++ b/server/providers/places/google.js @@ -1,6 +1,7 @@ import { googleServerApiKey, keylessGooglePlacesResponse, + keylessGoogleGeocodeResponse, } from './google-key.js'; import { makeOptInRateLimiter, clientKey } from '../common/rate-limit.js'; import { @@ -42,6 +43,37 @@ export function validatePlacesCoordinates(searchParams) { return { ok: true, latitude, longitude }; } +/** Longest accepted geocoder query. Real place names are far shorter. */ +const GEOCODE_MAX_QUERY = 200; + +/** Validate a free-text geocoder query before consuming request quota. */ +export function validateGeocodeAddress(searchParams) { + const address = String(searchParams.get('address') || '').trim(); + if (!address) return { ok: false, error: 'address is required' }; + if (address.length > GEOCODE_MAX_QUERY) + return { ok: false, error: 'address is too long' }; + return { ok: true, address }; +} + +/** + * Validate the optional viewport bias, which Google reads as + * `sw_lat,sw_lng|ne_lat,ne_lng`. An unparsed value is dropped rather than + * refused: the bias only ranks results, so a malformed one costs relevance, + * not an answer. + */ +export function geocodeBounds(searchParams) { + const raw = String(searchParams.get('bounds') || '').trim(); + if (!raw) return null; + const corners = raw.split('|'); + if (corners.length !== 2) return null; + const numbers = corners.flatMap((corner) => corner.split(',').map(Number)); + if (numbers.length !== 4 || !numbers.every(Number.isFinite)) return null; + const [swLat, swLon, neLat, neLon] = numbers; + if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null; + if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null; + return raw; +} + /** Nearby place labels and view-biased text search, with request-time key resolution. */ export function googlePlacesContextProxy({ resolveApiKey = googleServerApiKey, @@ -274,6 +306,129 @@ export function googlePlacesContextProxy({ ); } }); + + /** + * Geocoding belongs on the server, not in the page. Google's Geocoding web + * service refuses referrer-restricted keys, so calling it from the browser + * forces GOOGLE_MAPS_API_KEY — which ships in the bundle by design — to be + * left unrestricted (#363). Here the request carries the server key, and + * the browser key no longer needs the Geocoding API at all. + * + * Unlike the two Places routes above, these answer in Google's own shape + * (`status` + `results`) rather than the `places: []` contract: the browser + * normalizers read that shape directly, and keeping it means the proxy adds + * a hop without adding a translation nobody asked for. + */ + function installGeocodeRoute(path, readQuery) { + middlewares.use(path, async (req, res) => { + const refuse = (statusCode, error) => { + res.statusCode = statusCode; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ status: 'REQUEST_DENIED', results: [], error }), + ); + }; + + if (req.method !== 'GET') { + refuse(405, 'Method not allowed'); + return; + } + + // A keyless geocode has no provider cost, so it resolves before the + // paid-endpoint limiter can consume or exhaust quota (mirrors the + // Places routes above). + const apiKey = resolveApiKey(); + const keyless = keylessGoogleGeocodeResponse(apiKey); + if (keyless) { + res.statusCode = keyless.statusCode; + res.setHeader('Content-Type', 'application/json'); + res.setHeader('Cache-Control', 'no-store'); + res.end(JSON.stringify(keyless.payload)); + return; + } + + const requestUrl = new URL(req.url || '', 'http://localhost'); + const query = readQuery(requestUrl.searchParams); + if (!query.ok) { + refuse(400, query.error); + return; + } + + // Opt-in per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN), shared with + // the Places routes because both spend the same Google budget. + const _grl = googleRateLimiter(); + if (_grl && !_grl(clientKey(req))) { + res.statusCode = 429; + res.setHeader('Content-Type', 'application/json'); + res.setHeader('Retry-After', '5'); + res.end( + JSON.stringify({ + status: 'OVER_QUERY_LIMIT', + results: [], + error: 'Rate limit exceeded', + }), + ); + return; + } + + try { + const upstream = new URL( + endpoints.geocode || + 'https://maps.googleapis.com/maps/api/geocode/json', + ); + for (const [name, value] of Object.entries(query.params)) + upstream.searchParams.set(name, value); + upstream.searchParams.set('key', apiKey); + const response = await fetchImpl(upstream.toString(), { + redirect: 'error', + }); + const data = await response.json().catch(() => null); + + res.statusCode = response.ok ? 200 : response.status; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.setHeader('Cache-Control', 'private, max-age=300'); + res.end( + JSON.stringify( + data ?? { + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }, + ), + ); + } catch (error) { + res.statusCode = 502; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end( + JSON.stringify({ + status: 'UNKNOWN_ERROR', + results: [], + error: error?.message || 'Google Geocoding request failed', + }), + ); + } + }); + } + + installGeocodeRoute('/api/google/geocode', (searchParams) => { + const address = validateGeocodeAddress(searchParams); + if (!address.ok) return address; + const bounds = geocodeBounds(searchParams); + return { + ok: true, + params: { + address: address.address, + ...(bounds ? { bounds } : {}), + }, + }; + }); + + installGeocodeRoute('/api/google/reverse-geocode', (searchParams) => { + const coordinates = validatePlacesCoordinates(searchParams); + if (!coordinates.ok) return coordinates; + const { latitude, longitude } = coordinates; + return { ok: true, params: { latlng: `${latitude},${longitude}` } }; + }); } return { diff --git a/src/annotations/annotationEngine.test.mjs b/src/annotations/annotationEngine.test.mjs index 0d26974ae..cc43b612a 100644 --- a/src/annotations/annotationEngine.test.mjs +++ b/src/annotations/annotationEngine.test.mjs @@ -400,7 +400,7 @@ test('outline upgrade updates the rendered element in place without remove/add', }; let overpassCall = 0; globalThis.fetch = async (url) => { - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => ({ status: 'OK', results: [{ diff --git a/src/annotations/annotationResolver.test.mjs b/src/annotations/annotationResolver.test.mjs index 452d6997c..2c51213c1 100644 --- a/src/annotations/annotationResolver.test.mjs +++ b/src/annotations/annotationResolver.test.mjs @@ -200,7 +200,7 @@ test('ask-side admin bypass: "the Texas Capitol" recovers near-view despite a fa const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -235,7 +235,7 @@ test('ask-side admin bypass: explicit "state of Texas" skips recovery and proxim const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - assert.match(String(url), /^https:\/\/maps\.googleapis\.com\/maps\/api\/geocode/); + assert.match(String(url), /^\/api\/google\/geocode\?/); return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -273,7 +273,7 @@ for (const fixture of [ const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: fixture.lat, lon: fixture.lon, @@ -299,7 +299,7 @@ test('ask-side admin bypass: bare "Texas" remains on the guarded recovery path', const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -328,7 +328,7 @@ test('ask-side admin bypass: admin level 2/3 result types never grant a township const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { const query = new URL(String(url)).searchParams.get('address'); return { json: async () => geocodePayload({ lat: 39.7817, @@ -380,7 +380,7 @@ function installCapitolMocks(t, elements, components = [ { long_name: 'Capitol Hill', types: ['neighborhood', 'political'] }, ]) { installGoogleMocks(t, async (url) => { - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => ({ status: 'OK', results: [{ formatted_address: 'Washington, DC 20004, USA', types: ['establishment', 'landmark', 'point_of_interest', 'tourist_attraction'], @@ -458,7 +458,7 @@ test('keyless: a key that geocodes to nothing still anchors the annotation', asy const requests = []; installGoogleMocks(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (String(url).startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; return { ok: true, json: async () => ({ @@ -495,7 +495,7 @@ test('keyless: a key that geocodes to nothing still anchors the annotation', asy high: { latitude: 30.28, longitude: -97.68 }, }); // Google is asked first and exactly once; one Photon call answers it. - assert.equal(requests.filter((url) => url.includes('maps.googleapis.com')).length, 1); + assert.equal(requests.filter((url) => url.includes('/api/google/geocode')).length, 1); assert.equal(requests.filter((url) => url.includes('photon.komoot.io')).length, 1); }); @@ -512,7 +512,7 @@ test('keyless: OSM matching uses the feature\'s canonical name, not the user\'s installGoogleMocks(t, async (url, init) => { const href = String(url); - if (href.startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (href.startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; if (href.startsWith('/api/google/text-search')) return { ok: true, json: async () => ({ places: [] }) }; if (href.startsWith('https://photon.komoot.io/')) { return { @@ -575,7 +575,7 @@ test('keyless: a Photon outage is not remembered as "no such place"', async (t) const requests = []; installGoogleMocks(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (String(url).startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; if (!photonReachable) throw new Error('network down'); return photonUp; }); diff --git a/src/googleGeocodeProxy.test.mjs b/src/googleGeocodeProxy.test.mjs new file mode 100644 index 000000000..2518d02cb --- /dev/null +++ b/src/googleGeocodeProxy.test.mjs @@ -0,0 +1,167 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + googlePlacesContextProxy, + validateGeocodeAddress, + geocodeBounds, +} from '../vite.config.js'; + +const UPSTREAM = 'https://geocode.test/json'; + +function installRoutes({ apiKey = 'server-secret', fetchImpl } = {}) { + const routes = new Map(); + googlePlacesContextProxy({ + resolveApiKey: () => apiKey, + fetchImpl, + endpoints: { geocode: UPSTREAM }, + }).configureServer({ + middlewares: { use: (name, handler) => routes.set(name, handler) }, + }); + return routes; +} + +function invokeRoute(handler, { method = 'GET', url = '/' } = {}) { + return new Promise((resolve, reject) => { + const headers = new Map(); + const res = { + statusCode: 200, + setHeader(name, value) { + headers.set(String(name).toLowerCase(), String(value)); + }, + end(body = '') { + resolve({ + statusCode: this.statusCode, + headers: Object.fromEntries(headers), + body: body ? JSON.parse(String(body)) : null, + }); + }, + }; + Promise.resolve( + handler( + { method, url, headers: {}, socket: { remoteAddress: '127.0.0.1' } }, + res, + ), + ).catch(reject); + }); +} + +test('geocoder query and viewport bias are validated before Google is paid', () => { + assert.deepEqual(validateGeocodeAddress(new URLSearchParams()), { + ok: false, + error: 'address is required', + }); + assert.deepEqual(validateGeocodeAddress(new URLSearchParams({ address: ' ' })), { + ok: false, + error: 'address is required', + }); + assert.deepEqual( + validateGeocodeAddress(new URLSearchParams({ address: 'x'.repeat(201) })), + { ok: false, error: 'address is too long' }, + ); + assert.deepEqual( + validateGeocodeAddress(new URLSearchParams({ address: ' Austin ' })), + { ok: true, address: 'Austin' }, + ); + + // A bias only ranks results, so an unusable one is dropped, not refused. + const bias = '30.1,-97.9|30.4,-97.6'; + assert.equal(geocodeBounds(new URLSearchParams({ bounds: bias })), bias); + for (const bounds of [ + '', + '30.1,-97.9', + '30.1,-97.9|30.4', + 'north,-97.9|30.4,-97.6', + '91,-97.9|30.4,-97.6', + '30.1,-181|30.4,-97.6', + ]) + assert.equal(geocodeBounds(new URLSearchParams({ bounds })), null); +}); + +test('the geocode routes keep the key server-side and answer in Google shape', async () => { + const upstream = []; + const routes = installRoutes({ + fetchImpl: async (url) => { + upstream.push(new URL(String(url))); + return { + ok: true, + json: async () => ({ status: 'OK', results: [{ place_id: 'fixture' }] }), + }; + }, + }); + const forward = routes.get('/api/google/geocode'); + const reverse = routes.get('/api/google/reverse-geocode'); + assert.equal(typeof forward, 'function'); + assert.equal(typeof reverse, 'function'); + + const forwarded = await invokeRoute(forward, { + url: '/?address=Austin&bounds=30.1,-97.9|30.4,-97.6', + }); + assert.equal(forwarded.statusCode, 200); + assert.deepEqual(forwarded.body, { + status: 'OK', + results: [{ place_id: 'fixture' }], + }); + assert.equal(upstream.at(-1).searchParams.get('address'), 'Austin'); + assert.equal(upstream.at(-1).searchParams.get('bounds'), '30.1,-97.9|30.4,-97.6'); + assert.equal(upstream.at(-1).searchParams.get('key'), 'server-secret'); + + const reversed = await invokeRoute(reverse, { url: '/?lat=30.27&lon=-97.74' }); + assert.equal(reversed.statusCode, 200); + assert.equal(upstream.at(-1).searchParams.get('latlng'), '30.27,-97.74'); + assert.equal(upstream.at(-1).searchParams.get('key'), 'server-secret'); + + // The key travels to Google, never back to the page that asked. + assert.ok(!JSON.stringify(forwarded.body).includes('server-secret')); + assert.ok(!JSON.stringify(reversed.body).includes('server-secret')); + + // A malformed bias is dropped rather than forwarded. + await invokeRoute(forward, { url: '/?address=Austin&bounds=north' }); + assert.equal(upstream.at(-1).searchParams.has('bounds'), false); +}); + +test('a bad or keyless geocode request never reaches Google', async () => { + const upstream = []; + const fetchImpl = async (url) => { + upstream.push(String(url)); + return { ok: true, json: async () => ({ status: 'OK', results: [] }) }; + }; + + const routes = installRoutes({ fetchImpl }); + for (const [path, url] of [ + ['/api/google/geocode', '/?address='], + ['/api/google/reverse-geocode', '/?lat=91&lon=-97.74'], + ]) { + const response = await invokeRoute(routes.get(path), { url }); + assert.equal(response.statusCode, 400); + assert.deepEqual(response.body.results, []); + assert.equal(response.body.status, 'REQUEST_DENIED'); + } + + for (const path of ['/api/google/geocode', '/api/google/reverse-geocode']) { + const rejected = await invokeRoute(routes.get(path), { + method: 'POST', + url: '/?address=Austin&lat=30&lon=-97', + }); + assert.equal(rejected.statusCode, 405); + } + + // Keyless is an empty capability, not an outage: a 200 the browser adapters + // read as "this provider did not answer", so the chain moves on. It resolves + // ahead of validation, like the Places routes: with nothing configured there + // is no request to validate and no quota to protect. + const keyless = installRoutes({ apiKey: ' ', fetchImpl }); + for (const path of ['/api/google/geocode', '/api/google/reverse-geocode']) + for (const url of ['/?address=Austin&lat=30&lon=-97', '/?address=&lat=91']) { + const response = await invokeRoute(keyless.get(path), { url }); + assert.equal(response.statusCode, 200); + assert.equal(response.headers['cache-control'], 'no-store'); + assert.deepEqual(response.body, { + configured: false, + status: 'REQUEST_DENIED', + results: [], + error: null, + }); + } + + assert.deepEqual(upstream, []); +}); diff --git a/src/googleServerKey.test.mjs b/src/googleServerKey.test.mjs index 0851afcd5..2789219f5 100644 --- a/src/googleServerKey.test.mjs +++ b/src/googleServerKey.test.mjs @@ -67,11 +67,19 @@ test('the Street View tool resolves per-variable overrides before preferring the } }); -test('both Places routes select the intended key and keep it out of responses', async (t) => { +test('every Google route selects the intended key and keeps it out of responses', async (t) => { const original = { server: process.env.GOOGLE_MAPS_SERVER_API_KEY, browser: process.env.GOOGLE_MAPS_API_KEY, limit: process.env.GEV_RATELIMIT_GOOGLE_PER_MIN }; const calls = []; t.mock.method(globalThis, 'fetch', async (url, options) => { - calls.push({ url, key: options.headers['X-Goog-Api-Key'] }); + // Places sends the key as a header; Geocoding's web service takes it as a + // query parameter. Either way it must be the server key, and never the + // browser's copy of it. + calls.push({ + url, + key: + options?.headers?.['X-Goog-Api-Key'] ?? + new URL(String(url)).searchParams.get('key'), + }); return Response.json({ places: [] }); }); try { @@ -88,12 +96,12 @@ test('both Places routes select the intended key and keep it out of responses', for (const install of ['configureServer', 'configurePreviewServer']) { const routes = new Map(); googlePlacesContextProxy()[install]({ middlewares: { use: (name, handler) => routes.set(name, handler) } }); - assert.equal(routes.size, 2); + assert.equal(routes.size, 4); for (const handler of routes.values()) { const before = calls.length; let body; const res = { setHeader() {}, end(value) { body = value; } }; - await handler({ method: 'GET', url: '/?lat=30&lon=-97&q=capitol', headers: {}, socket: { remoteAddress: '127.0.0.1' } }, res); + await handler({ method: 'GET', url: '/?lat=30&lon=-97&q=capitol&address=capitol', headers: {}, socket: { remoteAddress: '127.0.0.1' } }, res); assert.equal(res.statusCode, 200); assert.ok(!body.includes('server-secret')); if (expected) { diff --git a/src/search/defaults.js b/src/search/defaults.js index 2b0428025..247512405 100644 --- a/src/search/defaults.js +++ b/src/search/defaults.js @@ -40,18 +40,23 @@ export function createDefaultPlaceSearch({ ...(selected ? [selected] : [ + // Google answers through the local proxy, which holds the server + // key: its Geocoding web service refuses referrer-restricted keys, + // so calling it from here would force the bundled browser key to be + // left unrestricted (#363). That key still gates the provider, + // because a keyless session must not spend a request to learn it + // has no Google: the proxy's "not configured" reads as an + // unanswered provider, which stops the chain caching the negative + // Photon or Nominatim then produces. createGoogleGeocoder({ request(query, { bias, signal }) { - const key = resolveApiKey?.(); - if (!key) return null; - const url = new URL( - endpoints.geocode || - 'https://maps.googleapis.com/maps/api/geocode/json', + if (!resolveApiKey?.()) return null; + const params = new URLSearchParams({ address: query }); + if (bias) params.set('bounds', bias); + return fetchImpl( + `${endpoints.geocode || '/api/google/geocode'}?${params}`, + { signal }, ); - url.searchParams.set('address', query); - url.searchParams.set('key', key); - if (bias) url.searchParams.set('bounds', bias); - return fetchImpl(url.toString(), { signal }); }, }), createPhotonGeocoder({ fetchImpl, endpoint: endpoints.photon }), diff --git a/src/search/http.js b/src/search/http.js index 5b1642c67..5afa35b75 100644 --- a/src/search/http.js +++ b/src/search/http.js @@ -53,7 +53,9 @@ export function createHttpGeospatialProvider({ endpoints = {}, } = {}) { const urls = { - reverse: 'https://maps.googleapis.com/maps/api/geocode/json', + // Same-origin like its siblings below: Google's Geocoding web service + // refuses referrer-restricted keys, so the key stays on the server (#363). + reverse: '/api/google/reverse-geocode', textSearch: '/api/google/text-search', nearby: '/api/google/nearby-places', route: '/api/route', @@ -88,14 +90,11 @@ export function createHttpGeospatialProvider({ }, routeProfiles: ['foot', 'car', 'bike'], async reverseGeocode(latitude, longitude, options) { - const key = resolveApiKey?.(); - if (resolveApiKey && !key) return null; + // The browser key gates the provider without travelling with the + // request: the proxy supplies the key Google actually sees. + if (resolveApiKey && !resolveApiKey()) return null; return normalizeGoogleReverse( - await json( - urls.reverse, - { latlng: `${latitude},${longitude}`, ...(key ? { key } : {}) }, - options, - ), + await json(urls.reverse, { lat: latitude, lon: longitude }, options), ); }, async textSearch(query, { latitude, longitude, radiusM = 6000 }, options) { diff --git a/src/search/placeSearch.test.mjs b/src/search/placeSearch.test.mjs index e609f9da9..05e1aaea0 100644 --- a/src/search/placeSearch.test.mjs +++ b/src/search/placeSearch.test.mjs @@ -20,7 +20,7 @@ test('standalone geocoding falls back after Google connection, JSON and refusal for (const fail of [() => { throw new Error('offline'); }, () => new Response('invalid json'), () => Response.json({ status: 'REQUEST_DENIED' })]) { const urls = []; const service = createStandalonePlaceSearch({ resolveApiKey: () => 'fixture', fetchImpl: async (url) => { - urls.push(url); return url.includes('maps.googleapis.com') ? fail() : hit(); + urls.push(url); return url.includes('/api/google/geocode') ? fail() : hit(); } }); const result = await service.geocode('Hanoi'); assert.equal(result.place.name, 'Hà Nội'); diff --git a/src/voice/gevActions.test.mjs b/src/voice/gevActions.test.mjs index ba0ed9cf7..1d8d28c97 100644 --- a/src/voice/gevActions.test.mjs +++ b/src/voice/gevActions.test.mjs @@ -3082,7 +3082,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl const requests = []; installKeyedFetch(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) }; } assert.match(String(url), /^https:\/\/photon\.komoot\.io\/api\/\?/); @@ -3107,7 +3107,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl assert.ok(Math.abs(calls[0].criteria.anchor.lat - 20.9101) < 1e-9); assert.ok(Math.abs(calls[0].criteria.anchor.lon - 107.1839) < 1e-9); // Google is asked first and exactly once; Photon answers unbiased, in one call. - assert.equal(requests.filter((url) => url.includes('maps.googleapis.com')).length, 1); + assert.equal(requests.filter((url) => url.includes('/api/google/geocode')).length, 1); assert.equal(requests.filter((url) => url.includes('photon.komoot.io')).length, 1); assert.match(requests.at(-1), /[?&]q=H%E1%BA%A1\+Long\+Bay/); assert.doesNotMatch(requests.at(-1), /[?&](lat|lon|bbox)=/, 'a named radio location is not viewport-biased'); @@ -3120,7 +3120,7 @@ test('voice Radio: the keyless path applies no country filter the keyed path wou // "No Radio station matched" for exactly the places it just resolved, while a // keyed install placed a station. The label may carry it; the filter may not. const { calls, dataManager } = radioSelectionHarness(); - installKeyedFetch(t, async (url) => (String(url).startsWith('https://maps.googleapis.com/') + installKeyedFetch(t, async (url) => (String(url).startsWith('/api/google/geocode') ? { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) } : { ok: true, From 771c9e5632ae6f0177adc09bc4ab4c34b4698765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Saggese?= Date: Mon, 21 Sep 2026 11:56:33 +0200 Subject: [PATCH 06/95] fix(search): bound and sanitize the geocode proxy - reject blank viewport-bias components before Number() reads them as 0 - give the upstream call a 10s deadline and a 256 KiB capped body read - return fixed public text on failure instead of the upstream message --- server/providers/places/google.js | 60 +++++++++++++++++++++++-------- src/googleGeocodeProxy.test.mjs | 47 +++++++++++++++++++++--- 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/server/providers/places/google.js b/server/providers/places/google.js index 8895bc19d..a4aa48a78 100644 --- a/server/providers/places/google.js +++ b/server/providers/places/google.js @@ -4,6 +4,7 @@ import { keylessGoogleGeocodeResponse, } from './google-key.js'; import { makeOptInRateLimiter, clientKey } from '../common/rate-limit.js'; +import { readResponseJsonCapped } from '../common/http.js'; import { projectNearbyPlaces, projectTextSearchPlaces, @@ -46,6 +47,16 @@ export function validatePlacesCoordinates(searchParams) { /** Longest accepted geocoder query. Real place names are far shorter. */ const GEOCODE_MAX_QUERY = 200; +/** + * Upstream deadline and body ceiling for the geocode routes. The deadline stays + * armed through body consumption, so a stalled upstream cannot hold a handler + * open, and the cap bounds a chunked or length-omitted reply. A geocode answer + * is a handful of results; 256 KiB is far above any honest one. + */ +const GEOCODE_UPSTREAM_TIMEOUT_MS = 10_000; + +const GEOCODE_MAX_RESPONSE_BYTES = 256 * 1024; + /** Validate a free-text geocoder query before consuming request quota. */ export function validateGeocodeAddress(searchParams) { const address = String(searchParams.get('address') || '').trim(); @@ -66,8 +77,12 @@ export function geocodeBounds(searchParams) { if (!raw) return null; const corners = raw.split('|'); if (corners.length !== 2) return null; - const numbers = corners.flatMap((corner) => corner.split(',').map(Number)); - if (numbers.length !== 4 || !numbers.every(Number.isFinite)) return null; + const parts = corners.flatMap((corner) => corner.split(',')); + // A blank component must be refused before Number() turns it into 0 and a + // half-written corner like `30.4,` reads as the equator. + if (parts.length !== 4 || parts.some((part) => !part.trim())) return null; + const numbers = parts.map(Number); + if (!numbers.every(Number.isFinite)) return null; const [swLat, swLon, neLat, neLon] = numbers; if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null; if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null; @@ -379,31 +394,48 @@ export function googlePlacesContextProxy({ for (const [name, value] of Object.entries(query.params)) upstream.searchParams.set(name, value); upstream.searchParams.set('key', apiKey); + const deadline = AbortSignal.timeout(GEOCODE_UPSTREAM_TIMEOUT_MS); const response = await fetchImpl(upstream.toString(), { redirect: 'error', + signal: deadline, }); - const data = await response.json().catch(() => null); + // An unreadable, oversized or timed-out body is not an answer: it + // falls through to the same shaped failure as a refusal below. + const data = await readResponseJsonCapped( + response, + GEOCODE_MAX_RESPONSE_BYTES, + deadline, + ).catch(() => null); + + // A body we could not read is an upstream failure, not an answer with + // a status of its own, so it does not borrow the upstream's code. + if (!data) { + res.statusCode = 502; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end( + JSON.stringify({ + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }), + ); + return; + } res.statusCode = response.ok ? 200 : response.status; res.setHeader('Content-Type', 'application/json; charset=utf-8'); res.setHeader('Cache-Control', 'private, max-age=300'); - res.end( - JSON.stringify( - data ?? { - status: 'UNKNOWN_ERROR', - results: [], - error: 'Google Geocoding request failed', - }, - ), - ); - } catch (error) { + res.end(JSON.stringify(data)); + } catch { + // Fixed public text: a transport, URL or runtime message would + // describe the server to the browser (SECURITY.md, sanitized errors). res.statusCode = 502; res.setHeader('Content-Type', 'application/json; charset=utf-8'); res.end( JSON.stringify({ status: 'UNKNOWN_ERROR', results: [], - error: error?.message || 'Google Geocoding request failed', + error: 'Google Geocoding request failed', }), ); } diff --git a/src/googleGeocodeProxy.test.mjs b/src/googleGeocodeProxy.test.mjs index 2518d02cb..98ffd876e 100644 --- a/src/googleGeocodeProxy.test.mjs +++ b/src/googleGeocodeProxy.test.mjs @@ -73,6 +73,10 @@ test('geocoder query and viewport bias are validated before Google is paid', () 'north,-97.9|30.4,-97.6', '91,-97.9|30.4,-97.6', '30.1,-181|30.4,-97.6', + // A blank component must not read as 0 — that is the equator, not a corner. + '30.1,-97.9|30.4,', + '30.1,|30.4,-97.6', + '30.1,-97.9| ,-97.6', ]) assert.equal(geocodeBounds(new URLSearchParams({ bounds })), null); }); @@ -82,10 +86,7 @@ test('the geocode routes keep the key server-side and answer in Google shape', a const routes = installRoutes({ fetchImpl: async (url) => { upstream.push(new URL(String(url))); - return { - ok: true, - json: async () => ({ status: 'OK', results: [{ place_id: 'fixture' }] }), - }; + return Response.json({ status: 'OK', results: [{ place_id: 'fixture' }] }); }, }); const forward = routes.get('/api/google/geocode'); @@ -123,7 +124,7 @@ test('a bad or keyless geocode request never reaches Google', async () => { const upstream = []; const fetchImpl = async (url) => { upstream.push(String(url)); - return { ok: true, json: async () => ({ status: 'OK', results: [] }) }; + return Response.json({ status: 'OK', results: [] }); }; const routes = installRoutes({ fetchImpl }); @@ -165,3 +166,39 @@ test('a bad or keyless geocode request never reaches Google', async () => { assert.deepEqual(upstream, []); }); + +test('an upstream failure reaches the browser as fixed public text', async () => { + const routes = installRoutes({ + fetchImpl: async () => { + throw new Error('connect ECONNREFUSED 10.0.0.7:443'); + }, + }); + for (const path of ['/api/google/geocode', '/api/google/reverse-geocode']) { + const response = await invokeRoute(routes.get(path), { + url: '/?address=Austin&lat=30&lon=-97', + }); + assert.equal(response.statusCode, 502); + assert.deepEqual(response.body, { + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }); + } +}); + +test('an oversized upstream body is dropped rather than buffered', async () => { + const routes = installRoutes({ + // Two hundred times the 256 KiB ceiling, declared honestly. + fetchImpl: async () => + Response.json({ status: 'OK', results: [{ pad: 'x'.repeat(52_428_800) }] }), + }); + const response = await invokeRoute(routes.get('/api/google/geocode'), { + url: '/?address=Austin', + }); + assert.equal(response.statusCode, 502); + assert.deepEqual(response.body, { + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }); +}); From ee3e3c7a414158387d6334528f56d1cb1af92b0b Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Thu, 17 Sep 2026 10:36:49 +0200 Subject: [PATCH 07/95] feat(cctv): WSDOT highway camera source pack (Washington State) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Washington State's highway cameras as a keyless source pack, from the Esri JSON layer WSDOT's own Travel Center map consumes (data.wsdot.wa.gov/travelcenter/Cameras.json, ~1,700 point features). Closes #624. - Geometry arrives in EPSG:3857; converted to WGS84 on load and bounded to a Washington bounding box. - Only frames on WSDOT's own images.wsdot.wa.gov host are registered — the catalog also lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) whose own terms apply; those are skipped rather than proxied. 1,630 cameras survive the filters against the live feed. - CompassDirection letters (N/E/S/W) give a high-confidence heading; the majority `B` ("both directions") rows carry no single facing and use the shared id-hash fallback at low confidence, with the usual two pose personalities. - The feed body carries Windows-1252 bytes (en-dashes in titles) with no charset declaration, so it is decoded explicitly before parsing. - WSDOT's layer metadata marks the feed "low volume": the 15-minute catalog cache and per-request frame fetches respect that, frames are never stored or redistributed, and the WSDOT credit is registered in the attribution popover. A keyed WSDOT Traveler Information API serves the same population for higher-volume use and is deliberately not required. - Defaults: 250 cameras nearest Seattle and Spokane; CCTV_WSDOT_MAX_SOURCES changes the cap, CCTV_WSDOT_ENABLED=0 turns the pack off. - Tests: Mercator conversion against a real layer feature, official-host and malformed-row filtering, compass mapping and both pose personalities, cp1252 decoding, env cap, upstream-failure fallback. --- DATA_SOURCES.md | 2 + server/providers/cctv/catalog.js | 6 ++ server/providers/cctv/constants.js | 18 ++++ server/providers/cctv/sources.js | 134 +++++++++++++++++++++++ src/data/cctvWsdotSource.test.mjs | 167 +++++++++++++++++++++++++++++ src/data/dataCredits.js | 10 ++ 6 files changed, 337 insertions(+) create mode 100644 src/data/cctvWsdotSource.test.mjs diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index b1439230e..73eaf8a1f 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -55,6 +55,7 @@ How to read this: | **Fintraffic / Digitraffic (weathercams)** | CCTV camera catalog + frames, Finland | [CC BY 4.0](https://www.digitraffic.fi/en/terms-of-service/) — attribution REQUIRED | "Fintraffic / digitraffic.fi, license CC BY 4.0" | | **DriveBC** (Government of British Columbia) | CCTV camera catalog + frames, British Columbia highways | [Open Government Licence – British Columbia](https://www2.gov.bc.ca/gov/content/data/open-data/open-government-licence-bc) — attribution REQUIRED | "DriveBC. Contains information licensed under the Open Government Licence – British Columbia" | | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | +| **WSDOT** (`data.wsdot.wa.gov`) | CCTV camera catalog + frames, Washington State highways | [WSDOT travel information disclaimer](https://wsdot.wa.gov/about/policies/travel-information-disclaimer) — public "low volume" feed; catalog cached, frames on WSDOT's own image host only | "Washington State Department of Transportation (Travel Center)" (courtesy) | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | @@ -89,6 +90,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. +- **WSDOT highway cameras.** Keyless. The camera list is the Esri JSON layer `data.wsdot.wa.gov/travelcenter/Cameras.json` — the same layer WSDOT's own Travel Center map consumes (~1,700 point features, EPSG:3857 geometry converted to WGS84 on load; the body carries Windows-1252 bytes and is decoded as such). Only frames on WSDOT's own `images.wsdot.wa.gov` host are registered — the catalog also lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) whose own terms apply, and those are skipped rather than proxied. WSDOT's layer metadata marks the feed for "low volume" use, which the 15-minute catalog cache and per-request frame fetches respect; frames are never stored or redistributed. `CompassDirection` letters (N/E/S/W) give a high-confidence heading; the majority `B` ("both directions") rows carry no single facing and use the shared id-hash fallback at low confidence. By default the 250 cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` changes the cap and `CCTV_WSDOT_ENABLED=0` turns the pack off. A keyed WSDOT Traveler Information API serves the same population for higher-volume use and is deliberately not required here. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index ae0ab9afa..3241cacd5 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -18,6 +18,7 @@ import { loadNswSourcesFromOpenData, loadCalgarySourcesFromOpenData, loadDelDOTSourcesFromOpenData, + loadWsdotSourcesFromOpenData, } from './sources.js'; /** Env kill switch: unset or anything but "0" means enabled. */ @@ -92,6 +93,11 @@ const LIVE_PACKS = [ enabled: () => envEnabled('CCTV_DELDOT_ENABLED'), load: loadDelDOTSourcesFromOpenData, }, + { + name: 'wsdot', + enabled: () => envEnabled('CCTV_WSDOT_ENABLED'), + load: loadWsdotSourcesFromOpenData, + }, ]; /** * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index da0f2fcda..3ab8ce9cb 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -244,6 +244,24 @@ export const DELDOT_ANCHORS = [ { lat: 39.1582, lon: -75.5244 }, // Dover (Kent) { lat: 38.6906, lon: -75.3877 }, // Georgetown (Sussex) ]; + +/** WSDOT Travel Center cameras (Washington State): keyless Esri JSON on the + * official open-data host — ~1,700 point features in EPSG:3857 (Web Mercator), + * ETag/Last-Modified served. WSDOT's metadata marks the feed "low volume" use, + * which the 15-minute catalog cache respects. */ +export const WSDOT_CAMERAS_URL = + 'https://data.wsdot.wa.gov/travelcenter/Cameras.json'; +/** Only frames on WSDOT's own image host are registered; the catalog also + * lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) + * whose own terms apply. */ +export const WSDOT_IMAGE_ORIGIN = 'https://images.wsdot.wa.gov/'; +export const DEFAULT_WSDOT_MAX_SOURCES = 250; +/** Prioritization anchors: downtown Seattle and Spokane. */ +export const WSDOT_ANCHORS = [ + { lat: 47.6062, lon: -122.3321 }, // Seattle + { lat: 47.6588, lon: -117.426 }, // Spokane +]; + /** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; /** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index ad8a883da..1124b187f 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -59,6 +59,10 @@ import { DELDOT_CCTV_URL, DEFAULT_DELDOT_MAX_SOURCES, DELDOT_ANCHORS, + WSDOT_CAMERAS_URL, + WSDOT_IMAGE_ORIGIN, + DEFAULT_WSDOT_MAX_SOURCES, + WSDOT_ANCHORS, } from './constants.js'; import { toFiniteNumber, @@ -1701,3 +1705,133 @@ export async function loadDelDOTSourcesFromOpenData() { return []; } } + +/** WSDOT compass letters as headings in degrees. `B` (both directions), `O` + * (other) and null carry no single facing and fall through to the id hash. */ +const WSDOT_COMPASS_HEADINGS = Object.freeze({ + N: 0, + E: 90, + S: 180, + W: 270, +}); + +/** + * Converts an EPSG:3857 (Web Mercator) coordinate pair to WGS84 degrees. + * The WSDOT camera layer publishes point geometry in wkid 102100. + * @param {number} x - Web Mercator metres east. + * @param {number} y - Web Mercator metres north. + * @returns {{lat: number, lon: number}} WGS84 degrees. + */ +export function webMercatorToWgs84(x, y) { + const R = 6378137; + const lon = (x / R) * (180 / Math.PI); + const lat = (2 * Math.atan(Math.exp(y / R)) - Math.PI / 2) * (180 / Math.PI); + return { lat, lon }; +} + +/** Rough Washington State bounding box (with the Columbia River border cams). */ +function isLikelyWashingtonCoordinate(lat, lon) { + return ( + Number.isFinite(lat) && + Number.isFinite(lon) && + lat >= 45.3 && + lat <= 49.3 && + lon >= -125 && + lon <= -116.5 + ); +} + +/** + * Fetch WSDOT highway cameras (Washington State). Keyless: one Esri JSON + * layer on WSDOT's official open-data host (the same layer the WSDOT Travel + * Center map consumes; a keyed Traveler Information API serves the identical + * population for higher-volume use). Only cameras with a positive integer id, + * finite in-state coordinates, and a frame on WSDOT's own image host are + * kept — the catalog's ~70 partner cameras (Oregon DOT, City of Seattle, + * lodges, airports) are skipped rather than proxied under someone else's + * terms. `CompassDirection` letters give a high-confidence heading; `B` + * (both directions, the majority) and `O`/null fall back to the id hash. + * The feed carries Windows-1252 bytes (en-dashes in titles) with no charset + * declaration, so the body is decoded explicitly before parsing. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadWsdotSourcesFromOpenData() { + try { + const resp = await fetch(WSDOT_CAMERAS_URL, { + headers: { Accept: 'application/json' }, + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + if (!resp.ok) { + console.warn('[CCTV] WSDOT camera download failed:', resp.status); + return []; + } + const payload = JSON.parse( + new TextDecoder('windows-1252').decode(await resp.arrayBuffer()), + ); + const rows = Array.isArray(payload?.features) ? payload.features : []; + + const cameras = []; + for (const row of rows) { + const attrs = row?.attributes; + const rawId = attrs?.CameraID; + if (!Number.isSafeInteger(rawId) || rawId <= 0) continue; + const imageUrl = String(attrs.ImageURL || '').trim(); + if (!imageUrl.startsWith(WSDOT_IMAGE_ORIGIN)) continue; + const x = row?.geometry?.x; + const y = row?.geometry?.y; + if (!Number.isFinite(x) || !Number.isFinite(y)) continue; + const { lat, lon } = webMercatorToWgs84(x, y); + if (!isLikelyWashingtonCoordinate(lat, lon)) continue; + + const cameraId = `wsdot-${rawId}`; + const heading = + WSDOT_COMPASS_HEADINGS[ + String(attrs.CompassDirection || '') + .trim() + .toUpperCase() + ]; + const hasHeading = Number.isFinite(heading); + cameras.push({ + id: cameraId, + name: String(attrs.CameraTitle || '').trim() || `WSDOT camera ${rawId}`, + city: 'Washington', + cityId: 'washington-state', + provider: 'WSDOT', + lat, + lon, + headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId), + headingConfidence: hasHeading ? 'high' : 'low', + // Same two pose personalities as the other packs: raw priors that the + // client's ground snap and manual calibration refine. + pitchDeg: hasHeading ? -24 : -18, + fovDeg: hasHeading ? 56 : 44, + rangeM: hasHeading ? 210 : 145, + mountHeightM: hasHeading ? 10 : 8, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'wsdot-open-data', + license: 'WSDOT Travel Center camera data (public low-volume feed)', + }); + } + + const maxRaw = Number( + process.env.CCTV_WSDOT_MAX_SOURCES || DEFAULT_WSDOT_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(1200, Math.floor(maxRaw))) + : DEFAULT_WSDOT_MAX_SOURCES; + const prioritized = prioritizeSources(cameras, maxCount, WSDOT_ANCHORS); + console.log( + `[CCTV] Loaded WSDOT camera sources: ${cameras.length} published (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn( + '[CCTV] WSDOT camera download error:', + error?.message || error, + ); + return []; + } +} diff --git a/src/data/cctvWsdotSource.test.mjs b/src/data/cctvWsdotSource.test.mjs new file mode 100644 index 000000000..0459f3c72 --- /dev/null +++ b/src/data/cctvWsdotSource.test.mjs @@ -0,0 +1,167 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + DEFAULT_WSDOT_MAX_SOURCES, + WSDOT_CAMERAS_URL, +} from '../../server/providers/cctv/constants.js'; +import { + loadWsdotSourcesFromOpenData, + webMercatorToWgs84, +} from '../../server/providers/cctv/sources.js'; + +/** Set (or, for `undefined`, delete) environment variables for one test. */ +function withEnv(t, env) { + for (const [name, value] of Object.entries(env)) { + const previous = process.env[name]; + t.after(() => { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + }); + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +} + +/** Silence the loaders' progress and failure logging. */ +function quiet(t) { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +} + +/** Web Mercator metres for a WGS84 coordinate (inverse of the loader's math). */ +function mercator(lat, lon) { + const R = 6378137; + return { + x: (lon * Math.PI * R) / 180, + y: R * Math.log(Math.tan(Math.PI / 4 + (lat * Math.PI) / 360)), + }; +} + +/** A `Cameras.json` point feature carrying the fields the loader reads. */ +function wsdotFeature(id, lat, lon, overrides = {}) { + const { attributes = {}, geometry } = overrides; + return { + attributes: { + CameraID: id, + CameraTitle: `Camera ${id}`, + CompassDirection: 'N', + ImageURL: `https://images.wsdot.wa.gov/nw/${id}.jpg`, + ...attributes, + }, + geometry: geometry === undefined ? mercator(lat, lon) : geometry, + }; +} + +/** Serialize a payload for the mocked fetch as raw Windows-1252 bytes. */ +function cp1252Response(payload) { + const text = JSON.stringify(payload); + const bytes = new Uint8Array(text.length); + for (let i = 0; i < text.length; i += 1) { + const code = text.charCodeAt(i); + // U+2013 EN DASH is 0x96 in Windows-1252 — the byte the live feed ships. + bytes[i] = code === 0x2013 ? 0x96 : code; + } + return new Response(bytes, { + headers: { 'Content-Type': 'application/json' }, + }); +} + +test('webMercatorToWgs84 recovers WGS84 degrees from layer geometry', () => { + // Real feature from the live layer: I-5 at the Interstate Bridge. + const { lat, lon } = webMercatorToWgs84(-13656007.102, 5719738.545); + assert.ok(Math.abs(lat - 45.62) < 0.05, `lat ${lat}`); + assert.ok(Math.abs(lon - -122.675) < 0.05, `lon ${lon}`); +}); + +test('WSDOT loader keeps official-host cameras and maps compass directions', async (t) => { + quiet(t); + withEnv(t, { CCTV_WSDOT_MAX_SOURCES: undefined }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + return cp1252Response({ + features: [ + wsdotFeature(1001, 47.6097, -122.3331, { + attributes: { + // The 0x2013 below is serialized as the feed's 0x96 byte. + CameraTitle: 'I-5 – Seattle, looking south', + CompassDirection: 'S', + }, + }), + wsdotFeature(9059, 47.66, -117.43, { + attributes: { CompassDirection: 'B' }, + }), + // Partner-hosted frame (Oregon DOT): skipped, never proxied. + wsdotFeature(1002, 45.62, -122.675, { + attributes: { + ImageURL: 'https://www.tripcheck.com/RoadCams/cams/bridge.jpg', + }, + }), + // Bad rows: non-integer id, missing geometry, out-of-state point. + wsdotFeature('1003', 47.6, -122.3), + wsdotFeature(1004, 47.6, -122.3, { geometry: null }), + wsdotFeature(1005, 34.05, -118.24), + ], + }); + }); + + const cameras = await loadWsdotSourcesFromOpenData(); + + assert.deepEqual(requested, [WSDOT_CAMERAS_URL]); + // Exactly the two valid official-host cameras survive; the partner-hosted + // frame and the malformed rows are dropped. + assert.deepEqual(cameras.map((camera) => camera.id).sort(), [ + 'wsdot-1001', + 'wsdot-9059', + ]); + + const seattle = cameras.find((camera) => camera.id === 'wsdot-1001'); + assert.equal(seattle.name, 'I-5 – Seattle, looking south'); + assert.equal(seattle.provider, 'WSDOT'); + assert.equal(seattle.url, 'https://images.wsdot.wa.gov/nw/1001.jpg'); + assert.equal(seattle.snapshotUrl, seattle.url); + assert.equal(seattle.feedType, 'image'); + assert.equal(seattle.headingDeg, 180); + assert.equal(seattle.headingConfidence, 'high'); + assert.equal(seattle.fovDeg, 56); + assert.ok(Math.abs(seattle.lat - 47.6097) < 0.001); + assert.ok(Math.abs(seattle.lon - -122.3331) < 0.001); + + // "Both directions" carries no single facing: id-hash fallback at low + // confidence, with the low-confidence pose personality. + const spokane = cameras.find((camera) => camera.id === 'wsdot-9059'); + assert.equal(spokane.headingConfidence, 'low'); + assert.equal(spokane.fovDeg, 44); + assert.ok(spokane.headingDeg >= 0 && spokane.headingDeg < 360); +}); + +test('WSDOT loader honors the max-sources cap', async (t) => { + quiet(t); + withEnv(t, { CCTV_WSDOT_MAX_SOURCES: '8' }); + const features = []; + for (let i = 0; i < 24; i += 1) { + features.push(wsdotFeature(2000 + i, 47.0 + i * 0.05, -122.9 + i * 0.05)); + } + t.mock.method(globalThis, 'fetch', async () => cp1252Response({ features })); + + const cameras = await loadWsdotSourcesFromOpenData(); + assert.equal(cameras.length, 8); +}); + +test('WSDOT loader defaults its cap and survives upstream failure', async (t) => { + quiet(t); + withEnv(t, { CCTV_WSDOT_MAX_SOURCES: undefined }); + assert.equal(DEFAULT_WSDOT_MAX_SOURCES, 250); + + t.mock.method( + globalThis, + 'fetch', + async () => new Response('', { status: 503 }), + ); + assert.deepEqual(await loadWsdotSourcesFromOpenData(), []); + + t.mock.method(globalThis, 'fetch', async () => { + throw new Error('network down'); + }); + assert.deepEqual(await loadWsdotSourcesFromOpenData(), []); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index ab9213c26..ea7adeb78 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -293,6 +293,16 @@ export const DATA_CREDITS = [ 'Open Government Licence – British Columbia. ' + 'Some cameras are supplied by partners (TransLink, the City of Vancouver, the City of Surrey, Parks Canada and others); each names its provider in the CCTV panel.', }, + { + key: 'wsdot-cctv', + html: + 'CCTV cameras & frames (Washington State): ' + + 'Washington State Department of Transportation ' + + '(Travel Center), ' + + 'used per the ' + + 'WSDOT travel information disclaimer ' + + '(low-volume public feed; catalog cached, frames proxied on demand)', + }, { key: 'tallinn-cctv', html: From 5b00dd0980480c5815c6fea346f0d892f8834d16 Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Thu, 17 Sep 2026 10:45:43 +0200 Subject: [PATCH 08/95] feat(cctv): King County road camera source pack (Washington State) Adds King County DOT Road Services' cameras as a keyless source pack, from the county's public ArcGIS FeatureServer layer ("King County and WSDOT Traffic Cameras"). 59 live cameras against the current feed. - The `outFields` list is explicit on purpose: the layer also carries asset-administration columns (including staff names) that must never be fetched. - Only live, active cameras with frames on the county's own info.kingcounty.gov image handler are registered; the feed writes http:// and the handler serves HTTPS, so URLs are upgraded and pinned before registration. WSDOT-owned rows are skipped so a camera never appears twice when a statewide WSDOT pack is present; the few city-owned cameras the county hosts (Redmond, Sammamish, Newcastle, Snoqualmie) carry their owner as the per-camera credit, DriveBC-style. - The layer publishes no facing: headings use the shared id-hash fallback at low confidence, correctable with the calibration gizmo. - Uniquely among the packs, the county publishes each camera's hardware Manufacturer/Model (Cohu 3950/3965, Bosch AUTODOME IP 5000i, Axis Q6135-LE, Siqura PD910). The model string is whitespace-collapsed (the feed carries stray newlines) and carried on the source as `model`, with an additive passthrough in normalizeSourceItem (same pattern as poseSource) so identified-hardware consumers can use it. All 59 live cameras carry a model value. - Attribution per the KCGIS Center terms, registered in the attribution popover and documented in DATA_SOURCES.md. - Defaults: the whole catalog loads (125 cap); CCTV_KINGCOUNTY_MAX_SOURCES changes the cap, CCTV_KINGCOUNTY_ENABLED=0 turns the pack off. - Tests: filtering (owner/live/status/host/geometry/bbox), HTTPS upgrade, model whitespace-collapse and passthrough, partner-city credit, env cap, upstream-failure fallback, and normalizeSourceItem model semantics. --- DATA_SOURCES.md | 2 + server/providers/cctv/catalog.js | 6 + server/providers/cctv/constants.js | 19 +++ server/providers/cctv/normalize.js | 8 ++ server/providers/cctv/sources.js | 126 ++++++++++++++++++ src/data/cctvKingCountySource.test.mjs | 171 +++++++++++++++++++++++++ src/data/dataCredits.js | 9 ++ 7 files changed, 341 insertions(+) create mode 100644 src/data/cctvKingCountySource.test.mjs diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index b1439230e..c43ae7728 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -55,6 +55,7 @@ How to read this: | **Fintraffic / Digitraffic (weathercams)** | CCTV camera catalog + frames, Finland | [CC BY 4.0](https://www.digitraffic.fi/en/terms-of-service/) — attribution REQUIRED | "Fintraffic / digitraffic.fi, license CC BY 4.0" | | **DriveBC** (Government of British Columbia) | CCTV camera catalog + frames, British Columbia highways | [Open Government Licence – British Columbia](https://www2.gov.bc.ca/gov/content/data/open-data/open-government-licence-bc) — attribution REQUIRED | "DriveBC. Contains information licensed under the Open Government Licence – British Columbia" | | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | +| **King County Road Services** (`services.arcgis.com`, King County GIS) | CCTV camera catalog + frames, unincorporated King County WA road network | [KCGIS Center terms](https://kingcounty.gov/en/dept/kcit/data-information-services/gis-center/about/terms-conditions-copyrights) — attribution required; public ArcGIS layer | "King County Road Services" (partner-city cameras name their owner) | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | @@ -89,6 +90,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. +- **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped so a camera never appears twice when a statewide WSDOT pack is present; the few city-owned cameras the county hosts (Redmond, Sammamish, Newcastle, Snoqualmie) carry their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); the model string is carried on the source as `model` for identified-hardware consumers. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index ae0ab9afa..ec5cf05b5 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -18,6 +18,7 @@ import { loadNswSourcesFromOpenData, loadCalgarySourcesFromOpenData, loadDelDOTSourcesFromOpenData, + loadKingCountySourcesFromOpenData, } from './sources.js'; /** Env kill switch: unset or anything but "0" means enabled. */ @@ -92,6 +93,11 @@ const LIVE_PACKS = [ enabled: () => envEnabled('CCTV_DELDOT_ENABLED'), load: loadDelDOTSourcesFromOpenData, }, + { + name: 'kingcounty', + enabled: () => envEnabled('CCTV_KINGCOUNTY_ENABLED'), + load: loadKingCountySourcesFromOpenData, + }, ]; /** * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index da0f2fcda..a25dc887f 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -244,6 +244,25 @@ export const DELDOT_ANCHORS = [ { lat: 39.1582, lon: -75.5244 }, // Dover (Kent) { lat: 38.6906, lon: -75.3877 }, // Georgetown (Sussex) ]; + +/** King County, WA road cameras: keyless public ArcGIS FeatureServer layer + * ("King County and WSDOT Traffic Cameras", King County DOT Road Services). + * GeoJSON with WGS84 point geometry. `outFields` is an explicit list on + * purpose: the layer also carries asset-administration columns (including + * staff names in `ModifiedBy`) that must never be fetched. Uniquely among the + * packs, the county publishes each camera's hardware `Manufacturer`/`Model` + * (Cohu 3950/3965, Bosch AUTODOME, Axis Q6135-LE, …), carried through as the + * source's `model`. */ +export const KING_COUNTY_CAMERAS_URL = + 'https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,Description,ImageURL,CamRegion,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson'; +/** Frames come only from the county's own image handler host. */ +export const KING_COUNTY_IMAGE_ORIGIN = 'https://info.kingcounty.gov/'; +/** The whole catalog is ~125 rows (~59 live county cameras); no thinning by + * default. */ +export const DEFAULT_KING_COUNTY_MAX_SOURCES = 125; +/** Prioritization anchor: Renton, roughly central to the county road network. */ +export const KING_COUNTY_ANCHORS = [{ lat: 47.4829, lon: -122.2171 }]; + /** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; /** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index b564ef440..17903335d 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -516,5 +516,13 @@ export function normalizeSourceItem(item) { // badge can distinguish them from raw automated priors (e.g. Austin Open // Data, which never sets this field). Passed through as-is to the client. poseSource: item.poseSource === 'curated' ? 'curated' : undefined, + // Optional hardware-model passthrough (additive, like poseSource): packs + // whose catalog publishes the camera's hardware model — King County's + // layer carries Manufacturer/Model per camera — pass it through so + // identified-hardware consumers can use it. Absent everywhere else. + model: + typeof item.model === 'string' && item.model.trim() + ? item.model.trim() + : undefined, }; } diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index ad8a883da..e92a8a265 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -59,6 +59,10 @@ import { DELDOT_CCTV_URL, DEFAULT_DELDOT_MAX_SOURCES, DELDOT_ANCHORS, + KING_COUNTY_CAMERAS_URL, + KING_COUNTY_IMAGE_ORIGIN, + DEFAULT_KING_COUNTY_MAX_SOURCES, + KING_COUNTY_ANCHORS, } from './constants.js'; import { toFiniteNumber, @@ -1701,3 +1705,125 @@ export async function loadDelDOTSourcesFromOpenData() { return []; } } + +/** Rough King County, WA bounding box. */ +function isLikelyKingCountyCoordinate(lat, lon) { + return ( + Number.isFinite(lat) && + Number.isFinite(lon) && + lat >= 47.0 && + lat <= 48.0 && + lon >= -123.1 && + lon <= -121.0 + ); +} + +/** + * Fetch King County road cameras (Washington State). Keyless: one public + * ArcGIS FeatureServer GeoJSON query (King County DOT Road Services), with an + * explicit `outFields` list so the layer's asset-administration columns are + * never fetched. Only cameras that are live and active (`Live`, + * `CurrentStatus`) with a positive integer id, finite in-county coordinates, + * and a frame on the county's own image handler are kept. WSDOT-owned rows in + * the layer are skipped — the statewide WSDOT pack is their home — while the + * handful of city-owned cameras the county hosts (Redmond, Sammamish, + * Newcastle, Snoqualmie) keep their owner as the per-camera credit, + * DriveBC-style. The layer publishes no facing, so every heading is the id + * hash at low confidence. + * + * Uniquely among the packs, the county publishes each camera's hardware + * `Manufacturer`/`Model`; the model string (whitespace-collapsed — the feed + * carries stray newlines) rides along as the source's `model` for + * identified-hardware consumers. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadKingCountySourcesFromOpenData() { + try { + const resp = await fetch(KING_COUNTY_CAMERAS_URL, { + headers: { Accept: 'application/json' }, + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + if (!resp.ok) { + console.warn('[CCTV] King County camera download failed:', resp.status); + return []; + } + const payload = await resp.json(); + const rows = Array.isArray(payload?.features) ? payload.features : []; + + const cameras = []; + for (const row of rows) { + const attrs = row?.properties; + const rawId = attrs?.AssetID; + if (!Number.isSafeInteger(rawId) || rawId <= 0) continue; + if (attrs.Live !== 1) continue; + if (String(attrs.CurrentStatus || '').trim() !== 'Active') continue; + const owner = String(attrs.Owner || '').trim(); + // WSDOT-owned rows belong to the statewide WSDOT pack; skipping them + // here keeps one camera from appearing twice on the globe. + if (owner === 'WSDOT') continue; + // The handler is served over HTTPS; the feed still writes http://. + const imageUrl = String(attrs.ImageURL || '') + .trim() + .replace(/^http:\/\/info\.kingcounty\.gov\//, KING_COUNTY_IMAGE_ORIGIN); + if (!imageUrl.startsWith(KING_COUNTY_IMAGE_ORIGIN)) continue; + const [lon, lat] = Array.isArray(row?.geometry?.coordinates) + ? row.geometry.coordinates + : []; + if (!isLikelyKingCountyCoordinate(lat, lon)) continue; + + const cameraId = `kingcounty-${rawId}`; + const model = String(attrs.Model || '') + .replace(/\s+/g, ' ') + .trim(); + const camera = { + id: cameraId, + name: + String(attrs.Location || '').trim() || `King County camera ${rawId}`, + city: 'King County', + cityId: 'king-county', + provider: 'King County Road Services', + lat, + lon, + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + // The low-confidence pose personality shared by the other packs. + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'kingcounty-open-data', + license: 'King County GIS open data (KCGIS Center terms of use)', + credit: owner && owner !== 'King County' ? owner : '', + }; + if (model) camera.model = model; + cameras.push(camera); + } + + const maxRaw = Number( + process.env.CCTV_KINGCOUNTY_MAX_SOURCES || + DEFAULT_KING_COUNTY_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(1200, Math.floor(maxRaw))) + : DEFAULT_KING_COUNTY_MAX_SOURCES; + const prioritized = prioritizeSources( + cameras, + maxCount, + KING_COUNTY_ANCHORS, + ); + console.log( + `[CCTV] Loaded King County camera sources: ${cameras.length} published (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn( + '[CCTV] King County camera download error:', + error?.message || error, + ); + return []; + } +} diff --git a/src/data/cctvKingCountySource.test.mjs b/src/data/cctvKingCountySource.test.mjs new file mode 100644 index 000000000..544b4f9ac --- /dev/null +++ b/src/data/cctvKingCountySource.test.mjs @@ -0,0 +1,171 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + DEFAULT_KING_COUNTY_MAX_SOURCES, + KING_COUNTY_CAMERAS_URL, +} from '../../server/providers/cctv/constants.js'; +import { loadKingCountySourcesFromOpenData } from '../../server/providers/cctv/sources.js'; +import { normalizeSourceItem } from '../../server/providers/cctv/normalize.js'; + +/** Set (or, for `undefined`, delete) environment variables for one test. */ +function withEnv(t, env) { + for (const [name, value] of Object.entries(env)) { + const previous = process.env[name]; + t.after(() => { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + }); + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +} + +/** Silence the loaders' progress and failure logging. */ +function quiet(t) { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +} + +/** A FeatureServer GeoJSON feature carrying the fields the loader reads. */ +function kingCountyFeature(id, lat, lon, overrides = {}) { + const { properties = {}, geometry } = overrides; + return { + type: 'Feature', + geometry: + geometry === undefined + ? { type: 'Point', coordinates: [lon, lat] } + : geometry, + properties: { + AssetID: id, + Location: `Road ${id} at Test Ave`, + ImageURL: `http://info.kingcounty.gov/transportation/kcdot/Roads/TrafficCameras/ImageHandler/Handler.ashx?id=cam${id}.jpg`, + Live: 1, + Model: '3950', + Manufacturer: 'Cohu', + Owner: 'King County', + CurrentStatus: 'Active', + ...properties, + }, + }; +} + +test('King County loader keeps live county cameras and upgrades frame URLs to HTTPS', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: undefined }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + return Response.json({ + type: 'FeatureCollection', + features: [ + kingCountyFeature(56, 47.70059, -122.02443, { + properties: { + Location: 'NE Novelty Hill Road at Trilogy Parkway NE', + // The live layer carries stray whitespace in model strings. + Model: 'Q6135-LE\n', + Manufacturer: 'Axis', + }, + }), + // City-owned camera the county hosts: kept, owner becomes the credit. + kingCountyFeature(90, 47.674, -122.119, { + properties: { Owner: 'City of Redmond', Model: null }, + }), + // WSDOT-owned row: the statewide WSDOT pack is its home. + kingCountyFeature(70, 47.6, -122.3, { properties: { Owner: 'WSDOT' } }), + // Off-host frame, not live, retired, bad id, bad geometry, out of county. + kingCountyFeature(71, 47.6, -122.3, { + properties: { ImageURL: 'https://example.com/cam71.jpg' }, + }), + kingCountyFeature(72, 47.6, -122.3, { properties: { Live: 0 } }), + kingCountyFeature(73, 47.6, -122.3, { + properties: { CurrentStatus: 'Retired' }, + }), + kingCountyFeature('74', 47.6, -122.3), + kingCountyFeature(75, 47.6, -122.3, { geometry: null }), + kingCountyFeature(76, 34.05, -118.24), + ], + }); + }); + + const cameras = await loadKingCountySourcesFromOpenData(); + + assert.deepEqual(requested, [KING_COUNTY_CAMERAS_URL]); + assert.deepEqual(cameras.map((camera) => camera.id).sort(), [ + 'kingcounty-56', + 'kingcounty-90', + ]); + + const novelty = cameras.find((camera) => camera.id === 'kingcounty-56'); + assert.equal(novelty.name, 'NE Novelty Hill Road at Trilogy Parkway NE'); + assert.equal(novelty.provider, 'King County Road Services'); + assert.equal( + novelty.url, + 'https://info.kingcounty.gov/transportation/kcdot/Roads/TrafficCameras/ImageHandler/Handler.ashx?id=cam56.jpg', + ); + assert.equal(novelty.snapshotUrl, novelty.url); + assert.equal(novelty.feedType, 'image'); + // Published hardware model rides along, whitespace-collapsed. + assert.equal(novelty.model, 'Q6135-LE'); + assert.equal(novelty.credit, ''); + // No facing in the layer: id-hash fallback at low confidence. + assert.equal(novelty.headingConfidence, 'low'); + assert.equal(novelty.fovDeg, 44); + assert.ok(novelty.headingDeg >= 0 && novelty.headingDeg < 360); + + const redmond = cameras.find((camera) => camera.id === 'kingcounty-90'); + assert.equal(redmond.credit, 'City of Redmond'); + assert.equal(redmond.model, undefined); +}); + +test('normalizeSourceItem passes a hardware model through and omits it otherwise', () => { + const withModel = normalizeSourceItem({ + id: 'kingcounty-56', + lat: 47.7, + lon: -122.0, + model: ' Q6135-LE ', + }); + assert.equal(withModel.model, 'Q6135-LE'); + + const withoutModel = normalizeSourceItem({ + id: 'austin-1', + lat: 30.3, + lon: -97.7, + }); + assert.equal(withoutModel.model, undefined); + assert.equal(normalizeSourceItem({ id: 'x', model: ' ' }).model, undefined); +}); + +test('King County loader honors the max-sources cap', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: '8' }); + const features = []; + for (let i = 0; i < 24; i += 1) { + features.push( + kingCountyFeature(100 + i, 47.2 + i * 0.02, -122.4 + i * 0.02), + ); + } + t.mock.method(globalThis, 'fetch', async () => + Response.json({ type: 'FeatureCollection', features }), + ); + + const cameras = await loadKingCountySourcesFromOpenData(); + assert.equal(cameras.length, 8); +}); + +test('King County loader defaults its cap and survives upstream failure', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: undefined }); + assert.equal(DEFAULT_KING_COUNTY_MAX_SOURCES, 125); + + t.mock.method( + globalThis, + 'fetch', + async () => new Response('', { status: 503 }), + ); + assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); + + t.mock.method(globalThis, 'fetch', async () => { + throw new Error('network down'); + }); + assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index ab9213c26..6c3a2507e 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -201,6 +201,15 @@ export const DATA_CREDITS = [ 'Traffic cameras (Calgary): contains information licensed under the ' + 'Open Government Licence – City of Calgary', }, + { + key: 'kingcounty-cctv', + html: + 'CCTV cameras & frames (King County, WA): King County Road Services — ' + + 'kingcounty.gov, ' + + 'per the ' + + 'KCGIS Center terms. ' + + 'A few cameras are supplied by partner cities (Redmond, Sammamish, Newcastle, Snoqualmie); each names its owner in the CCTV panel.', + }, { key: 'gbfs', html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)', From ccdf8f0dc287129bf0e45158807cec26bf478bff Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Mon, 14 Sep 2026 13:45:20 +0200 Subject: [PATCH 09/95] feat(cctv): enrich identified cameras with datasheet FOV + specs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a CCTV source pack (or a seed) names its hardware `model`, resolve it against a camera-model spec table: the coverage frustum then uses the manufacturer's horizontal FOV instead of the pose estimate, and the HUD meta line gains the model's resolution, night-vision, and PTZ specs. Cameras with no identified model are unchanged — unknown never implies anything, matching the layer's existing honesty about RAW PRIOR poses. Spec records are vendored verbatim from the CCTV Camera Database (https://github.com/ch-bas/cctv-camera-database, CC0-1.0), an open dataset of 12,000+ camera models transcribed from manufacturer datasheets. This lands a small starter table plus the join + parsing helpers; source packs that publish their hardware can grow it. Records describe camera products (optics, resolution, night vision) only — no endpoints or access data. - src/layers/cctv/modelSpecs.js: spec table, case-insensitive/alias lookup, wide-end FOV parser, HUD summary formatter - src/layers/cctv/modelSpecs.test.mjs: lookup/alias/unknown, FOV parsing across the real fov string shapes, summary rendering - src/layers/cctv/catalog.js: apply enrichment in both catalog builders - src/layers/cctv/presentation.js: label a datasheet FOV, append the spec summary, and pass fovSource + spec through the public camera state --- src/layers/cctv/catalog.js | 27 ++++++ src/layers/cctv/modelSpecs.js | 129 ++++++++++++++++++++++++++++ src/layers/cctv/modelSpecs.test.mjs | 63 ++++++++++++++ src/layers/cctv/presentation.js | 11 ++- 4 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 src/layers/cctv/modelSpecs.js create mode 100644 src/layers/cctv/modelSpecs.test.mjs diff --git a/src/layers/cctv/catalog.js b/src/layers/cctv/catalog.js index 1a6ebe7c6..be6dc7e90 100644 --- a/src/layers/cctv/catalog.js +++ b/src/layers/cctv/catalog.js @@ -1,4 +1,25 @@ import { CAMERA_SEEDS, SOURCE_ENDPOINT } from './policy.js'; +import { lookupModelSpec, horizontalFovDeg } from './modelSpecs.js'; + +/** + * Applies datasheet enrichment to a camera in place: when it names a known + * hardware `model`, the manufacturer's horizontal FOV replaces the pose + * estimate and the spec record is attached for the HUD. Unknown models — the + * common case — change nothing. + * @param {Object} camera - Camera being built (mutated). + * @param {string} [modelName] - Hardware model from a source or seed. + * @param {(v:number,min:number,max:number)=>number} clamp - Pose clamp helper. + */ +function applyModelEnrichment(camera, modelName, clamp) { + const spec = modelName ? lookupModelSpec(modelName) : null; + if (!spec) return; + const fov = horizontalFovDeg(spec); + if (fov) { + camera.fovDeg = clamp(fov, 20, 125); + camera.fovSource = 'datasheet'; + } + camera.spec = spec; +} export function createCatalog({ state: layerState, services, parts, source }) { const { CITY_POIS } = services.locations; @@ -45,6 +66,7 @@ export function createCatalog({ state: layerState, services, parts, source }) { parts.model.clamp(seed.elevationM ?? 22, 8, 80), pitchDeg: parts.model.clamp(seed.pitchDeg ?? -17, -40, -4), }; + applyModelEnrichment(camera, seed.model, parts.model.clamp); parts.model.ensureCameraPose(camera); catalog.push(camera); } @@ -205,6 +227,11 @@ export function createCatalog({ state: layerState, services, parts, source }) { : null, poseSource, }; + applyModelEnrichment( + camera, + source.model || seed?.model, + parts.model.clamp, + ); parts.model.ensureCameraPose(camera); catalog.push(camera); } diff --git a/src/layers/cctv/modelSpecs.js b/src/layers/cctv/modelSpecs.js new file mode 100644 index 000000000..e9bb48856 --- /dev/null +++ b/src/layers/cctv/modelSpecs.js @@ -0,0 +1,129 @@ +/** + * @module cctvModelSpecs + * + * Hardware-model spec enrichment for the CCTV layer. + * + * When a catalog camera names its hardware model (source packs may set + * `model`; hand-authored seeds may too), the model resolves to a datasheet + * record: the camera's frustum then uses the manufacturer's horizontal FOV + * instead of an estimate, and the HUD meta line gains the model's real specs. + * Cameras without an identified model are untouched — unknown never implies + * anything. + * + * Spec records are vendored verbatim from the CCTV Camera Database + * (https://github.com/ch-bas/cctv-camera-database, CC0-1.0) — an open dataset + * of 12,000+ camera models transcribed from manufacturer datasheets (84% with + * FOV). This module ships a small starter table; city packs that publish + * their hardware models can grow it, or vendor the full extract. + * + * Scope guardrail: records describe camera PRODUCTS (optics, resolution, + * night vision) — never endpoints, stream paths, or access/config data. + */ + +/** Verbatim records from the dataset's model-enrichment extract (v2.19.0). */ +const RECORDS = [ + { + id: 'axis-q6135-le', + brand: 'Axis', + model: 'Q6135-LE', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/axis-q6135-le/', + aliases: ['01959-004'], + fov_deg: '58.3-2.4 horizontal', + resolution: { mp: 2, w: 1920, h: 1080, label: '1080p' }, + ptz: true, + autotracking: true, + varifocal: true, + night_vision: { type: 'ir', range_m: 250 }, + environment: ['outdoor'], + ndaa_compliant: true, + }, + { + id: 'hanwha-xnp-6400rw', + brand: 'Hanwha', + model: 'XNP-6400RW', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/hanwha-xnp-6400rw/', + aliases: ['Wisenet PTZ Plus 2MP 40x IR PTZ'], + fov_deg: '65.66-1.88 horizontal / 39.4-1.09 vertical', + resolution: { mp: 2, w: 1920, h: 1080, label: '1080p' }, + ptz: true, + autotracking: true, + varifocal: true, + night_vision: { type: 'ir', range_m: 200 }, + environment: ['outdoor'], + ndaa_compliant: true, + }, + { + id: 'hikvision-ds-2cd2087g3-li2uy', + brand: 'Hikvision', + model: 'DS-2CD2087G3-LI2UY', + type: 'bullet', + url: 'https://www.cctv-database.com/camera/hikvision-ds-2cd2087g3-li2uy/', + fov_deg: '108.8 horizontal (2.8mm) / 93.3 horizontal (4mm)', + resolution: { mp: 8, w: 3840, h: 2160, label: '8MP (4K)' }, + night_vision: { type: 'color', range_m: 40 }, + environment: ['outdoor'], + ndaa_compliant: false, + }, + { + id: 'axis-f4105-lre', + brand: 'Axis', + model: 'F4105-LRE', + type: 'dome', + url: 'https://www.cctv-database.com/camera/axis-f4105-lre/', + fov_deg: '110 horizontal, 60 vertical', + resolution: { mp: 2, w: 1920, h: 1080, label: '1080p HD' }, + night_vision: { type: 'ir', range_m: 10 }, + environment: ['outdoor'], + ndaa_compliant: true, + }, +]; + +const BY_MODEL = new Map(); +for (const record of RECORDS) { + BY_MODEL.set(record.model.toLowerCase(), record); + for (const alias of record.aliases || []) + BY_MODEL.set(alias.toLowerCase(), record); +} + +/** + * Looks up a spec record by model string (case-insensitive; aliases match too). + * @param {string} model - Hardware model as named by a source pack or seed. + * @returns {Object|null} The verbatim spec record, or null when unknown. + */ +export function lookupModelSpec(model) { + if (!model) return null; + return BY_MODEL.get(String(model).trim().toLowerCase()) || null; +} + +/** + * Datasheet horizontal FOV in degrees: the FIRST number in the `fov_deg` + * string, which for varifocal/PTZ ranges ('58.3-2.4 horizontal') is the wide + * end — the honest default for a coverage cone. + * @param {Object} spec - A record from this module. + * @returns {number|null} Wide-end horizontal FOV, or null when unstated. + */ +export function horizontalFovDeg(spec) { + const match = String(spec?.fov_deg || '').match(/\d+(?:\.\d+)?/); + const value = match ? Number(match[0]) : NaN; + return Number.isFinite(value) && value > 0 ? value : null; +} + +/** + * One-line spec summary for HUD meta text. + * @param {Object} spec - A record from this module. + * @returns {string} e.g. 'Axis Q6135-LE · 1080p · IR 250m · PTZ' ('' for null). + */ +export function specSummary(spec) { + if (!spec) return ''; + const parts = [`${spec.brand} ${spec.model}`]; + if (spec.resolution?.label) parts.push(spec.resolution.label); + if (spec.night_vision) { + parts.push( + `${spec.night_vision.type.toUpperCase()}${spec.night_vision.range_m ? ` ${spec.night_vision.range_m}m` : ''}`, + ); + } + if (spec.ptz) parts.push('PTZ'); + return parts.join(' · '); +} diff --git a/src/layers/cctv/modelSpecs.test.mjs b/src/layers/cctv/modelSpecs.test.mjs new file mode 100644 index 000000000..4d6f80370 --- /dev/null +++ b/src/layers/cctv/modelSpecs.test.mjs @@ -0,0 +1,63 @@ +// src/layers/cctv/modelSpecs.test.mjs +// Hardware-model spec enrichment: lookup semantics (case-insensitive, alias +// matching, unknown → null — never a guess), datasheet-FOV parsing across the +// real fov_deg string shapes, and the HUD spec-summary line. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + lookupModelSpec, + horizontalFovDeg, + specSummary, +} from './modelSpecs.js'; + +test('lookupModelSpec matches case-insensitively and trims', () => { + const spec = lookupModelSpec(' q6135-le '); + assert.ok(spec); + assert.equal(spec.brand, 'Axis'); + assert.equal(lookupModelSpec('Q6135-LE'), spec); +}); + +test('lookupModelSpec matches aliases', () => { + const byAlias = lookupModelSpec('01959-004'); + assert.ok(byAlias); + assert.equal(byAlias.model, 'Q6135-LE'); +}); + +test('lookupModelSpec returns null for unknown or empty input — never guesses', () => { + assert.equal(lookupModelSpec('NOT-A-REAL-MODEL'), null); + assert.equal(lookupModelSpec(''), null); + assert.equal(lookupModelSpec(null), null); + assert.equal(lookupModelSpec(undefined), null); +}); + +test('horizontalFovDeg takes the wide end of a varifocal/PTZ range', () => { + // '58.3-2.4 horizontal' → 58.3 (wide), not 2.4 (tele). + assert.equal(horizontalFovDeg(lookupModelSpec('Q6135-LE')), 58.3); +}); + +test('horizontalFovDeg reads multi-variant and plain fov strings', () => { + // '108.8 horizontal (2.8mm) / 93.3 horizontal (4mm)' → first variant. + assert.equal(horizontalFovDeg(lookupModelSpec('DS-2CD2087G3-LI2UY')), 108.8); + // '110 horizontal, 60 vertical' → horizontal figure. + assert.equal(horizontalFovDeg(lookupModelSpec('F4105-LRE')), 110); +}); + +test('horizontalFovDeg returns null when no numeric FOV is stated', () => { + assert.equal(horizontalFovDeg({ fov_deg: '' }), null); + assert.equal(horizontalFovDeg({ fov_deg: 'wide dynamic range' }), null); + assert.equal(horizontalFovDeg({}), null); + assert.equal(horizontalFovDeg(null), null); +}); + +test('specSummary renders the HUD line and degrades by omission', () => { + assert.equal( + specSummary(lookupModelSpec('Q6135-LE')), + 'Axis Q6135-LE · 1080p · IR 250m · PTZ', + ); + // No night vision / no PTZ → those segments are simply absent. + assert.equal( + specSummary({ brand: 'Axis', model: 'X', resolution: { label: '4K' } }), + 'Axis X · 4K', + ); + assert.equal(specSummary(null), ''); +}); diff --git a/src/layers/cctv/presentation.js b/src/layers/cctv/presentation.js index f8f83fcd1..9fa4a7cef 100644 --- a/src/layers/cctv/presentation.js +++ b/src/layers/cctv/presentation.js @@ -1,6 +1,7 @@ import { CCTV_AMBIENT_CARD_MAX } from '../../data/cctvLod.js'; import { ACTIVE_FRAME_REFRESH_MS, IDLE_FRAME_REFRESH_MS } from './policy.js'; import { headingHudToken, isHeadingEstimated } from './headingConfidence.js'; +import { specSummary } from './modelSpecs.js'; export function createPresentation({ state: layerState, @@ -39,7 +40,8 @@ export function createPresentation({ // A synthetic bearing (headingConfidence 'low', no human calibration) // is tagged so a hashed guess never reads as a surveyed facing (#639). headingHudToken(active.camera), - `FOV ${Math.round(active.camera.fovDeg)}°`, + // Identified hardware labels its FOV as datasheet-sourced (catalog.js). + `FOV ${Math.round(active.camera.fovDeg)}°${active.camera.fovSource === 'datasheet' ? ' (DATASHEET)' : ''}`, `COVERAGE ${area.toFixed(2)}km²`, overlapCount > 0 ? `OVERLAP ${overlapCount} cams` : 'ISOLATED VIEW', `PROJ ${layerState._showProjection ? 'MONITOR' : 'OFF'}`, @@ -49,6 +51,10 @@ export function createPresentation({ ? `SRC ${String(health.sourceKind).toUpperCase()}` : `SRC ${String(active.camera.feedType || 'image').toUpperCase()}`, `${viewBand.toUpperCase()} CONTEXT`, + // Identified hardware: append the model's datasheet specs + attribution. + active.camera.spec + ? `${specSummary(active.camera.spec).toUpperCase()} · SPECS: CCTV-DATABASE.COM` + : null, ] .filter(Boolean) .join(' · '); @@ -86,6 +92,9 @@ export function createPresentation({ headingEstimated: isHeadingEstimated(camera), pitchDeg: camera.pitchDeg, fovDeg: camera.fovDeg, + // Identified-hardware enrichment passthrough (catalog.js / modelSpecs.js). + fovSource: camera.fovSource || 'estimated', + spec: camera.spec || null, rangeM: camera.rangeM, elevationM: camera.absoluteHeightM, mountHeightM: camera.mountHeightM, From 97118f478100ca4cebdba60b8eaf7f56f4e4de4e Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Thu, 17 Sep 2026 11:36:50 +0200 Subject: [PATCH 10/95] feat(cctv): vendor the model specs the integrated catalogs publish Grow the enrichment table from the 4-model starter to the hardware the currently integrated public catalogs actually publish, so identified cameras render datasheet FOV cones and spec cards instead of estimates: - King County (WA) road cameras: Cohu 3950 / 3960 (3965) / 3920 iDome (3925), Bosch AUTODOME IP 5000i. - Sarasota County (FL) + Sioux Falls (SD) open data: Bosch AUTODOME IP starlight 7000i / starlight 7000 HD / dynamic 7000 HD, the VG4 AUTODOME (500i), Bosch AUTODOME 800 (VG5-836), and Axis Q6155-E, Q6054-E, Q6000-E Mk II, and P3707-PE. All records are vendored verbatim from the CCTV Camera Database (CC0), each transcribed from a manufacturer datasheet. Analog Cohu positioners carry no fov_deg (their datasheets publish none), so they enrich the HUD specs without a datasheet cone; every other model carries a real FOV. Adds a test asserting the vendored table covers each integrated catalog's published hardware strings. --- src/layers/cctv/modelSpecs.js | 221 +++++++++++++++++++++++++++- src/layers/cctv/modelSpecs.test.mjs | 13 ++ 2 files changed, 232 insertions(+), 2 deletions(-) diff --git a/src/layers/cctv/modelSpecs.js b/src/layers/cctv/modelSpecs.js index e9bb48856..463e750ad 100644 --- a/src/layers/cctv/modelSpecs.js +++ b/src/layers/cctv/modelSpecs.js @@ -13,8 +13,11 @@ * Spec records are vendored verbatim from the CCTV Camera Database * (https://github.com/ch-bas/cctv-camera-database, CC0-1.0) — an open dataset * of 12,000+ camera models transcribed from manufacturer datasheets (84% with - * FOV). This module ships a small starter table; city packs that publish - * their hardware models can grow it, or vendor the full extract. + * FOV). This module vendors the records that the currently integrated public + * catalogs publish hardware for — King County WA (Cohu, Bosch AUTODOME, Axis + * Q6135-LE), plus the Bosch AUTODOME and Axis PTZ/multisensor models that + * appear in Sarasota County FL and Sioux Falls SD open data; packs that + * publish other hardware models can grow it, or vendor the full extract. * * Scope guardrail: records describe camera PRODUCTS (optics, resolution, * night vision) — never endpoints, stream paths, or access/config data. @@ -78,6 +81,220 @@ const RECORDS = [ environment: ['outdoor'], ndaa_compliant: true, }, + { + id: 'axis-p3707-pe', + brand: 'Axis', + model: 'P3707-PE', + type: 'panoramic', + url: 'https://www.cctv-database.com/camera/axis-p3707-pe/', + aliases: ['AXIS P3707-PE', 'P3707-PE'], + fov_deg: '108-54 horizontal per sensor (4x 1080p)', + resolution: { + mp: 8, + w: 1920, + h: 1080, + label: '4x 2MP (8MP total; 4x 1080p)', + }, + lens_count: 4, + varifocal: true, + environment: ['outdoor'], + }, + { + id: 'axis-q6000-e-mk-ii', + brand: 'Axis', + model: 'Q6000-E Mk II', + type: 'panoramic', + url: 'https://www.cctv-database.com/camera/axis-q6000-e-mk-ii/', + aliases: [ + 'AXIS Q6000-E Mk II', + 'Q6000-E Mk II 50 Hz', + 'Q6000-E Mk II 60 Hz', + ], + fov_deg: 'up to 360 combined; 113-152 horizontal per sensor', + resolution: { + mp: 8, + w: 1920, + h: 1080, + label: '4x 2MP (8MP total; 4x 1080p)', + }, + lens_count: 4, + environment: ['outdoor'], + }, + { + id: 'axis-q6054-e', + brand: 'Axis', + model: 'Q6054-E', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/axis-q6054-e/', + aliases: ['AXIS Q6054-E', 'Q6054-E 50 Hz', 'Q6054-E 60 Hz'], + fov_deg: '62.9-2.2 horizontal', + resolution: { mp: 1, w: 1280, h: 720, label: '720p HD' }, + ptz: true, + autotracking: true, + varifocal: true, + environment: ['outdoor'], + }, + { + id: 'axis-q6155-e', + brand: 'Axis', + model: 'Q6155-E', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/axis-q6155-e/', + aliases: ['AXIS Q6155-E', 'Q6155-E 50 Hz', 'Q6155-E 60 Hz'], + fov_deg: '66.7-2.36 horizontal', + resolution: { mp: 2, w: 1920, h: 1080, label: '1080p HD' }, + ptz: true, + autotracking: true, + varifocal: true, + environment: ['outdoor'], + }, + { + id: 'bosch-autodome-500i', + brand: 'Bosch', + model: 'AutoDome 500i Series (VG4)', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/bosch-autodome-500i/', + aliases: [ + 'VG4 AUTODOME', + 'VG4 AUTODOME H.264', + 'AutoDome 500i', + 'AutoDome 500i Series', + 'VG4-500i', + 'VG4-500', + ], + fov_deg: '57.8-1.7 horizontal', + resolution: { mp: 0.4, label: '540 TVL (NTSC/PAL)' }, + ptz: true, + autotracking: true, + varifocal: true, + environment: ['outdoor'], + }, + { + id: 'bosch-autodome-ip-dynamic-7000-hd', + brand: 'Bosch', + model: 'AUTODOME IP dynamic 7000 HD', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/bosch-autodome-ip-dynamic-7000-hd/', + aliases: ['AUTODOME IP dynamic 7000 HD', 'AUTODOME 7000 dynamic HD'], + fov_deg: '65-2.3 horizontal', + resolution: { mp: 2, w: 1920, h: 1080, label: '1080p HD (2MP)' }, + ptz: true, + autotracking: true, + varifocal: true, + environment: ['outdoor'], + }, + { + id: 'bosch-autodome-ip-starlight-7000-hd', + brand: 'Bosch', + model: 'AUTODOME IP starlight 7000 HD', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/bosch-autodome-ip-starlight-7000-hd/', + aliases: [ + 'AUTODOME IP starlight 7000 HD', + 'Starlight 7000HD', + 'VG5-7130-CPT4', + 'VG5-7130', + ], + fov_deg: '59-2.1 horizontal', + resolution: { mp: 1, w: 1280, h: 720, label: '720p HD (1.3MP sensor)' }, + ptz: true, + autotracking: true, + varifocal: true, + environment: ['outdoor'], + }, + { + id: 'bosch-ndp-5502-30', + brand: 'Bosch', + model: 'AUTODOME IP 5000i (NDP-5502-Z30)', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/bosch-ndp-5502-30/', + aliases: [ + 'AUTODOME IP 5000i', + 'NDP-5502-Z30', + 'NDP-5502-Z30C', + 'NDP-5502-30', + ], + fov_deg: '60.9-2.4 horizontal', + resolution: { mp: 2, w: 1920, h: 1080, label: '1080p HD (2MP)' }, + ptz: true, + varifocal: true, + environment: ['outdoor'], + }, + { + id: 'bosch-ndp-7512-z30', + brand: 'Bosch', + model: 'AUTODOME IP starlight 7000i (NDP-7512-Z30)', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/bosch-ndp-7512-z30/', + aliases: [ + 'AUTODOME IP starlight 7000i', + 'NDP-7512-Z30', + 'NDP-7512-Z30C', + 'NDP-7512-Z30K', + ], + fov_deg: '64.7-2.3 horizontal', + resolution: { mp: 2, w: 1920, h: 1080, label: '1080p HD (2MP)' }, + ptz: true, + autotracking: true, + varifocal: true, + environment: ['outdoor'], + }, + { + id: 'bosch-vg5-836-ecev', + brand: 'Bosch', + model: 'AUTODOME 800 Series HD PTZ (VG5-836-ECEV)', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/bosch-vg5-836-ecev/', + aliases: ['VG5-836-ECEV', 'VG5-836', 'AUTODOME 800 Series'], + resolution: { mp: 2, label: '1080p' }, + ptz: true, + autotracking: true, + environment: ['outdoor'], + }, + { + id: 'cohu-3920-idome', + brand: 'Cohu', + model: '3920 iDome', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/cohu-3920-idome/', + aliases: [ + '3920', + '3920 Series', + '3923', + '3925', + '3925 Dome', + 'iDome 3925', + 'Cohu iDome 3920 Series', + ], + resolution: { mp: 0.4, label: 'NTSC / 470 TVL' }, + ptz: true, + varifocal: true, + environment: ['outdoor'], + }, + { + id: 'cohu-3950-iview', + brand: 'Cohu', + model: '3950 iVIEW', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/cohu-3950-iview/', + aliases: ['3950', '3950 Series', 'Cohu 3950 Series iVIEW'], + resolution: { mp: 0.4, label: 'NTSC / 470 TVL' }, + ptz: true, + varifocal: true, + environment: ['outdoor'], + }, + { + id: 'cohu-3960', + brand: 'Cohu', + model: '3960', + type: 'ptz', + url: 'https://www.cctv-database.com/camera/cohu-3960/', + aliases: ['3960 Series', '3964', '3965', 'Cohu 3960 iView'], + resolution: { mp: 0.4, label: 'NTSC/PAL / 470 TVL' }, + ptz: true, + varifocal: true, + environment: ['outdoor'], + }, ]; const BY_MODEL = new Map(); diff --git a/src/layers/cctv/modelSpecs.test.mjs b/src/layers/cctv/modelSpecs.test.mjs index 4d6f80370..b31f2d714 100644 --- a/src/layers/cctv/modelSpecs.test.mjs +++ b/src/layers/cctv/modelSpecs.test.mjs @@ -23,6 +23,19 @@ test('lookupModelSpec matches aliases', () => { assert.equal(byAlias.model, 'Q6135-LE'); }); +test('vendored table covers the integrated catalogs published hardware', () => { + // Models the live packs publish: King County (Cohu, AUTODOME IP 5000i), + // Sarasota (Bosch AUTODOME 7000/VG4), Sioux Falls (Axis). + const cohu = lookupModelSpec('3950'); // King County publishes "3950" + assert.equal(cohu?.brand, 'Cohu'); + const autodome = lookupModelSpec('AUTODOME IP 5000i'); + assert.ok(autodome && horizontalFovDeg(autodome) > 0); + const vg4 = lookupModelSpec('VG4 AUTODOME H.264'); // Sarasota's string + assert.ok(vg4); + const axis = lookupModelSpec('Q6155-E'); // Sioux Falls + assert.equal(horizontalFovDeg(axis), 66.7); +}); + test('lookupModelSpec returns null for unknown or empty input — never guesses', () => { assert.equal(lookupModelSpec('NOT-A-REAL-MODEL'), null); assert.equal(lookupModelSpec(''), null); From 0533f78be433f28fe9085d856caf000630691670 Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Thu, 17 Sep 2026 11:56:15 +0200 Subject: [PATCH 11/95] feat(cctv): resolve verbose registry model strings via token fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some catalogs publish a full product title rather than the bare model, e.g. "AXIS Q6155-E PTZ Dome Network Camera" — which the exact model/alias match misses. Add a fallback: after the exact match, scan for a record whose DISTINCTIVE token (a >=4-char model/alias word containing a digit or hyphen, so "dome"/"network"/"ptz"/"hd" never qualify) appears as a whole word in the query. The most specific (longest) matched token wins; if two records tie on the longest matched token the result is ambiguous and returns null, so the lookup never guesses. Exact matches are unchanged and still take precedence. --- scripts/package-boundaries.json | 3 ++ src/layers/cctv/modelSpecs.js | 69 +++++++++++++++++++++++++++-- src/layers/cctv/modelSpecs.test.mjs | 37 ++++++++++++++++ 3 files changed, 105 insertions(+), 4 deletions(-) diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 189192f10..a987623a4 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1079,6 +1079,7 @@ "src/layers/cctv/ingestion.js", "src/layers/cctv/lifecycle.js", "src/layers/cctv/model.js", + "src/layers/cctv/modelSpecs.js", "src/layers/cctv/navigation.js", "src/layers/cctv/policy.js", "src/layers/cctv/presentation.js", @@ -1496,6 +1497,7 @@ "src/layers/cctv/ingestion.js", "src/layers/cctv/lifecycle.js", "src/layers/cctv/model.js", + "src/layers/cctv/modelSpecs.js", "src/layers/cctv/navigation.js", "src/layers/cctv/policy.js", "src/layers/cctv/presentation.js", @@ -2136,6 +2138,7 @@ "src/layers/cctv/ingestion.js", "src/layers/cctv/lifecycle.js", "src/layers/cctv/model.js", + "src/layers/cctv/modelSpecs.js", "src/layers/cctv/navigation.js", "src/layers/cctv/policy.js", "src/layers/cctv/presentation.js", diff --git a/src/layers/cctv/modelSpecs.js b/src/layers/cctv/modelSpecs.js index 463e750ad..1252ce013 100644 --- a/src/layers/cctv/modelSpecs.js +++ b/src/layers/cctv/modelSpecs.js @@ -297,21 +297,82 @@ const RECORDS = [ }, ]; +/** Lowercase + collapse whitespace, the shared normal form for matching. */ +function normalize(value) { + return String(value || '') + .trim() + .toLowerCase() + .replace(/\s+/g, ' '); +} + const BY_MODEL = new Map(); for (const record of RECORDS) { - BY_MODEL.set(record.model.toLowerCase(), record); + BY_MODEL.set(normalize(record.model), record); for (const alias of record.aliases || []) - BY_MODEL.set(alias.toLowerCase(), record); + BY_MODEL.set(normalize(alias), record); } /** - * Looks up a spec record by model string (case-insensitive; aliases match too). + * DISTINCTIVE tokens of a record — the whitespace words of its model and + * aliases that identify hardware rather than describe it. A token qualifies + * only when it is >= 4 chars AND contains a digit or hyphen, so generic words + * ("dome", "network", "camera", "series", "ptz") are never distinctive and a + * verbose registry string cannot match on them alone. Surrounding punctuation + * is trimmed so "(NDP-7512-Z30)" indexes as "ndp-7512-z30". + * @param {Object} record + * @returns {string[]} Unique distinctive tokens, lowercased. + */ +function distinctiveTokens(record) { + const out = new Set(); + for (const name of [record.model, ...(record.aliases || [])]) { + for (const raw of normalize(name).split(' ')) { + const token = raw.replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, ''); + if (token.length >= 4 && /[0-9-]/.test(token)) out.add(token); + } + } + return [...out]; +} + +/** + * Token index for the substring fallback: one precompiled whole-word matcher + * per distinctive token. Word boundaries are any non-alphanumeric char (or the + * string ends), so hyphenated/dotted tokens like "q6155-e" or "h.264" match as + * whole words while "700" never matches inside "7000". + */ +const TOKEN_INDEX = RECORDS.flatMap((record) => + distinctiveTokens(record).map((token) => ({ + token, + record, + re: new RegExp( + `(?:^|[^a-z0-9])${token.replace(/[.*+?^${}()|[\]\\-]/g, '\\$&')}(?:[^a-z0-9]|$)`, + ), + })), +); + +/** + * Looks up a spec record for a hardware-model string. + * + * Exact match first (case-insensitive, whitespace-collapsed) on the model and + * every alias. When that misses, a token fallback lets a VERBOSE registry + * string resolve — e.g. "AXIS Q6155-E PTZ Dome Network Camera" finds the + * Q6155-E via its distinctive "q6155-e" token. The most specific (longest) + * matched token wins; if two different records tie on the longest matched + * token the result is ambiguous and returns null (never guesses). * @param {string} model - Hardware model as named by a source pack or seed. * @returns {Object|null} The verbatim spec record, or null when unknown. */ export function lookupModelSpec(model) { if (!model) return null; - return BY_MODEL.get(String(model).trim().toLowerCase()) || null; + const query = normalize(model); + if (!query) return null; + const exact = BY_MODEL.get(query); + if (exact) return exact; + const matches = TOKEN_INDEX.filter((entry) => entry.re.test(query)); + if (!matches.length) return null; + const maxLen = Math.max(...matches.map((m) => m.token.length)); + const top = matches.filter((m) => m.token.length === maxLen); + const records = new Set(top.map((m) => m.record)); + return records.size === 1 ? top[0].record : null; } /** diff --git a/src/layers/cctv/modelSpecs.test.mjs b/src/layers/cctv/modelSpecs.test.mjs index b31f2d714..c504e14ad 100644 --- a/src/layers/cctv/modelSpecs.test.mjs +++ b/src/layers/cctv/modelSpecs.test.mjs @@ -36,6 +36,43 @@ test('vendored table covers the integrated catalogs published hardware', () => { assert.equal(horizontalFovDeg(axis), 66.7); }); +test('verbose registry strings resolve via distinctive-token fallback', () => { + // Sioux Falls publishes the full product title, not the bare model. + assert.equal( + lookupModelSpec('AXIS Q6155-E PTZ Dome Network Camera')?.model, + 'Q6155-E', + ); + assert.equal(lookupModelSpec('AXIS Q6000-E Mk II')?.model, 'Q6000-E Mk II'); + assert.equal( + lookupModelSpec('AUTODOME IP starlight 7000i camera')?.id, + 'bosch-ndp-7512-z30', + ); + // Punctuation around the model token doesn't defeat the match. + assert.equal( + lookupModelSpec('camera (P3707-PE), outdoor')?.model, + 'P3707-PE', + ); +}); + +test('token fallback never matches on generic words or wrong models', () => { + // Only generic descriptors — no distinctive hardware token — stays null. + assert.equal(lookupModelSpec('PTZ Dome Camera'), null); + assert.equal(lookupModelSpec('HD Network Camera'), null); + assert.equal(lookupModelSpec('Outdoor Dome Camera'), null); + // A real-looking but unknown model resolves to nothing, not a near neighbour. + assert.equal(lookupModelSpec('AXIS FooBar-9 Network Camera'), null); + // A generation token alone must not collide with a similar one: "7000" + // belongs to two records (dynamic / starlight 7000 HD), so a bare + // "AUTODOME 7000" is ambiguous → null rather than an arbitrary pick. + assert.equal(lookupModelSpec('AUTODOME 7000'), null); +}); + +test('exact match still wins over the token fallback', () => { + // "3965" is an alias of the 3960; exact alias match, not a token scan. + assert.equal(lookupModelSpec('3965')?.id, 'cohu-3960'); + assert.equal(lookupModelSpec('AUTODOME IP 5000i')?.id, 'bosch-ndp-5502-30'); +}); + test('lookupModelSpec returns null for unknown or empty input — never guesses', () => { assert.equal(lookupModelSpec('NOT-A-REAL-MODEL'), null); assert.equal(lookupModelSpec(''), null); From 5f8a275487338d3fc08d8e3b91dbc8d856148676 Mon Sep 17 00:00:00 2001 From: Michael Johnston Date: Tue, 15 Sep 2026 22:08:53 -0400 Subject: [PATCH 12/95] fix(providers): bound the remaining upstream body reads Route the six proxied fetches that still buffered upstream bodies without a limit through the shared capped readers in src/sources/httpBody.js: FIRMS CSV, CelesTrak TLE groups, adsb.lol /v2/mil (both the OK and relayed-status paths), TomTom flow tiles, OpenSky states/all and adsbdb enrichment lookups. Each cap is a bound sized well above the measured payload, not a budget. readResponseBytesCapped gains the same optional abort signal as the text reader, with the same cancel-on-abort plumbing and reader release, so the TomTom tile path can share the route deadline instead of reading arrayBuffer() unbounded. Where a route already had a deadline, the same signal now covers the body read as well as the headers. Two polls had no deadline at all: the adsb.lol military poll gains a 12-second one and the OpenSky states/all exchange, including its Basic-auth retry, a 20-second one; both miss into their existing STALE paths. FIRMS map-key status, the OpenSky token exchange and CCTV media are left as they are. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 + docs/CURRENT-STATE.md | 20 ++++ scripts/package-boundaries.json | 14 ++- server/providers/aircraft/adsb-lol.js | 20 +++- server/providers/aircraft/enrichment.js | 16 ++- server/providers/aircraft/opensky.js | 21 +++- server/providers/firms.js | 14 ++- server/providers/space/celestrak.js | 13 ++- server/providers/traffic.js | 15 ++- src/sources/httpBody.js | 44 +++++--- src/sources/httpBody.test.mjs | 133 ++++++++++++++++++++++++ 11 files changed, 278 insertions(+), 34 deletions(-) create mode 100644 src/sources/httpBody.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 29dbedfaa..f5433b87f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +- Bound the remaining upstream body reads (FIRMS, CelesTrak, adsb.lol, TomTom, OpenSky, adsbdb) with streaming size caps and shared deadlines; the adsb.lol military and OpenSky `states/all` polls gain deadlines of their own. + - CCTV cameras whose bearing is a guess now say so. Packs mark bearings derived from a hash of the camera id as `headingConfidence: 'low'`, but nothing read the flag, so roughly 70% of a default catalog rendered like surveyed facings. The HUD diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fdb42fc0c..7853d590e 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -827,6 +827,26 @@ feed address is read from the path, and a query-string form is refused with a 40 streaming. The 12-second deadline includes reading the body, and rejected or stalled downloads are cancelled. Development and preview use the same handler. +## Upstream body bounds + +Every proxied fetch outside CCTV media now reads its body through the shared +streaming readers in `src/sources/httpBody.js`, so a declared or observed body +over the cap is refused with `RESPONSE_TOO_LARGE` and cancelled rather than +buffered. `readResponseBytesCapped` accepts the same optional abort signal as +the text reader and cancels a body still streaming when it fires; TomTom flow +tiles use it alongside GTFS-Realtime. Caps are bounds, not budgets, sized well +above measured payloads: FIRMS 64 MiB per satellite source, CelesTrak 16 MiB +per group, OpenSky `states/all` 8 MiB, adsb.lol `/v2/mil` 8 MiB, TomTom tiles +8 MiB and adsbdb lookups 1 MiB. Where a route already had a deadline (FIRMS +60 s, CelesTrak 20 s, TomTom 15 s, adsbdb 8 s) that deadline now covers the +body read as well as the headers. Two polls that had no deadline gain one: the +adsb.lol military poll waits at most 12 seconds, and the OpenSky `states/all` +exchange, including its Basic-auth retry, at most 20. A rejected or timed-out +body is an upstream miss and follows each route's existing error and +cached-data paths. FIRMS map-key status, the OpenSky token exchange and CCTV +media keep their current handling. + + ## Remaining local service modules Overpass query validation, geometry simplification, disk caching and upstream diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 189192f10..e47e5bb16 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -113,7 +113,12 @@ "traffic-provider": { "runtime": "node", "exports": ["./server/providers/traffic"], - "modules": ["server/providers/traffic.js", "src/data/tomtomTiles.js"], + "modules": [ + "server/providers/traffic.js", + "server/providers/common/http.js", + "src/sources/httpBody.js", + "src/data/tomtomTiles.js" + ], "external": [] }, "traffic-source": { @@ -129,7 +134,12 @@ "firms-provider": { "runtime": "node", "exports": ["./server/providers/firms"], - "modules": ["server/providers/firms.js", "src/data/firmsCsv.js"], + "modules": [ + "server/providers/firms.js", + "server/providers/common/http.js", + "src/sources/httpBody.js", + "src/data/firmsCsv.js" + ], "external": [] }, "gbfs-provider": { diff --git a/server/providers/aircraft/adsb-lol.js b/server/providers/aircraft/adsb-lol.js index fc5ab0bd3..606b598cd 100644 --- a/server/providers/aircraft/adsb-lol.js +++ b/server/providers/aircraft/adsb-lol.js @@ -1,3 +1,5 @@ +import { readResponseTextCapped } from '../common/http.js'; + /** * Vite plugin: adsb.lol military aircraft proxy with 12 s response cache. * @@ -25,6 +27,10 @@ export function adsbLolProxy() { let _cooldownStatus = 0; /** Response cache TTL (ms). */ const CACHE_MS = 12000; + /** Upstream deadline (ms); the fetch previously had none and could hang a poll. */ + const UPSTREAM_TIMEOUT_MS = 12000; + /** /v2/mil measures ~67 KB; a relayed error page is bounded by the same cap. */ + const ADSBLOL_MIL_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; /** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */ const RATE_LIMIT_COOLDOWN_MS = 30000; /** Cooldown after a 5xx (ms). */ @@ -94,11 +100,17 @@ export function adsbLolProxy() { ); return; } + const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS); const upstream = await fetch('https://api.adsb.lol/v2/mil', { headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' }, + signal, }); if (upstream.ok) { - const body = await upstream.text(); + const body = await readResponseTextCapped( + upstream, + ADSBLOL_MIL_MAX_RESPONSE_BYTES, + signal, + ); _cache = body; _cacheAt = Date.now(); _cooldownUntil = 0; @@ -125,7 +137,11 @@ export function adsbLolProxy() { return; } } - const body = await upstream.text(); + const body = await readResponseTextCapped( + upstream, + ADSBLOL_MIL_MAX_RESPONSE_BYTES, + signal, + ); serve( res, upstream.status, diff --git a/server/providers/aircraft/enrichment.js b/server/providers/aircraft/enrichment.js index 400050151..9b7c1b1bb 100644 --- a/server/providers/aircraft/enrichment.js +++ b/server/providers/aircraft/enrichment.js @@ -1,5 +1,9 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; +import { readResponseJsonCapped } from '../common/http.js'; + +/** Route and aircraft lookups are a few hundred bytes; 1 MiB is a hard bound. */ +const ADSBDB_MAX_RESPONSE_BYTES = 1024 * 1024; /** * adsbdb.com enrichment proxy: callsign → route (airline + origin/destination * airports) and hex → aircraft type/registration. Free community API — cached @@ -80,12 +84,16 @@ export function adsbdbProxy() { kind === 'route' ? `https://api.adsbdb.com/v0/callsign/${encodeURIComponent(key)}` : `https://api.adsbdb.com/v0/aircraft/${encodeURIComponent(key)}`; - const res = await fetch(url, { signal: AbortSignal.timeout(8000) }); + const signal = AbortSignal.timeout(8000); + const res = await fetch(url, { signal }); if (res.ok) { + const payload = await readResponseJsonCapped( + res, + ADSBDB_MAX_RESPONSE_BYTES, + signal, + ); const data = - kind === 'route' - ? parseRoute(await res.json()) - : parseAircraft(await res.json()); + kind === 'route' ? parseRoute(payload) : parseAircraft(payload); store[key] = { at: Date.now(), data }; // data may be null — negative cache dirty = true; return data; diff --git a/server/providers/aircraft/opensky.js b/server/providers/aircraft/opensky.js index 612d01e11..f1ded6401 100644 --- a/server/providers/aircraft/opensky.js +++ b/server/providers/aircraft/opensky.js @@ -2,6 +2,7 @@ import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback import { coalesceProxyRequest, readResponseJsonCapped, + readResponseTextCapped, } from '../common/http.js'; import { requiredFiniteQueryNumber } from '../common/query.js'; // --------------------------------------------------------------------------- @@ -72,6 +73,15 @@ const ADSBLOL_POINT_CACHE_MS = 12000; const ADSBLOL_POINT_CACHE_MAX = 80; const ADSBLOL_POINT_RADIUS_NM = 250; const ADSBLOL_POINT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +/** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */ +const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +/** + * Deadline for the whole states/all exchange: headers, the Basic-auth retry + * and the body. The fetch previously had none, so a hung upstream connection + * stalled the worldwide poll indefinitely; a miss here serves STALE like any + * other upstream failure. + */ +const OPENSKY_STATES_TIMEOUT_MS = 20000; // A 200 response can still contain an old OpenSky snapshot. Past this point // the viewport-scoped adsb.lol source is more honest and keeps local motion // current instead of coasting a stale worldwide frame indefinitely. @@ -470,9 +480,10 @@ export function openSkyProxy() { } } + const signal = AbortSignal.timeout(OPENSKY_STATES_TIMEOUT_MS); let upstream = await fetch( 'https://opensky-network.org/api/states/all?extended=1', - { headers }, + { headers, signal }, ); // Auto-mode fallback: if OAuth was rejected, retry with Basic credentials if ( @@ -487,13 +498,17 @@ export function openSkyProxy() { }; upstream = await fetch( 'https://opensky-network.org/api/states/all?extended=1', - { headers: retryHeaders }, + { headers: retryHeaders, signal }, ); usedMode = 'basic'; reason = 'oauth_rejected_fallback_basic'; } - let body = await upstream.text(); + let body = await readResponseTextCapped( + upstream, + OPENSKY_STATES_MAX_RESPONSE_BYTES, + signal, + ); const sourceEpochMs = upstream.ok ? openSkySourceEpochMs(body) : null; if ( upstream.ok && diff --git a/server/providers/firms.js b/server/providers/firms.js index 636968348..c73ca10ce 100644 --- a/server/providers/firms.js +++ b/server/providers/firms.js @@ -2,6 +2,7 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js'; +import { readResponseTextCapped } from './common/http.js'; /** * NASA FIRMS live active-fire proxy with a memory + disk cache. @@ -35,6 +36,12 @@ export function firmsProxy() { 'VIIRS_SNPP_NRT', 'MODIS_NRT', ]; + /** + * Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about + * 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a + * bound, not a budget. + */ + const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024; const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); const CACHE_PATH = path.join(CACHE_DIR, 'firms.json'); @@ -83,9 +90,12 @@ export function firmsProxy() { */ async function fetchSource(key, source) { const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`; - const res = await fetch(url, { signal: AbortSignal.timeout(60_000) }); + const signal = AbortSignal.timeout(60_000); + const res = await fetch(url, { signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); - const records = parseFirmsCsv(await res.text()); + const records = parseFirmsCsv( + await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal), + ); if (records === null) throw new Error('non-CSV upstream response'); return records; } diff --git a/server/providers/space/celestrak.js b/server/providers/space/celestrak.js index a76cc590d..8baf59ba7 100644 --- a/server/providers/space/celestrak.js +++ b/server/providers/space/celestrak.js @@ -1,6 +1,7 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js'; +import { readResponseTextCapped } from '../common/http.js'; /** * Vite plugin: CelesTrak TLE proxy. @@ -48,10 +49,14 @@ export function celestrakProxy() { } } + /** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */ + const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024; + async function fetchUpstream(group) { const url = celestrakTleUrl(group); + const signal = AbortSignal.timeout(20000); const res = await fetch(url.toString(), { - signal: AbortSignal.timeout(20000), + signal, // CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a // descriptive User-Agent with a contact point. headers: { @@ -60,7 +65,11 @@ export function celestrakProxy() { }, }); if (!res.ok) throw new Error(`HTTP ${res.status}`); - const body = await res.text(); + const body = await readResponseTextCapped( + res, + CELESTRAK_MAX_RESPONSE_BYTES, + signal, + ); // An upstream error page parses to zero TLEs — treat as failure, keep cache. if (!/^1 /m.test(body)) throw new Error('no TLE lines in response'); return { at: Date.now(), body }; diff --git a/server/providers/traffic.js b/server/providers/traffic.js index 2a3c2e60a..d53db6530 100644 --- a/server/providers/traffic.js +++ b/server/providers/traffic.js @@ -1,6 +1,7 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; +import { readResponseBytesCapped } from './common/http.js'; import { isValidTileCoord as isValidTomTomTile, utcDayKey as tomtomUtcDayKey, @@ -48,6 +49,8 @@ export function tomtomProxy() { const DEFAULT_DAILY_BUDGET = 6000; const MEM_MAX_ENTRIES = 256; const UPSTREAM_TIMEOUT_MS = 15000; + /** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */ + const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; /** @type {Map} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */ const mem = new Map(); @@ -143,11 +146,15 @@ export function tomtomProxy() { 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' + `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`; recordUpstreamFetch(); // attempts count — upstream bills the request either way - const res = await fetch(url, { - signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), - }); + const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS); + const res = await fetch(url, { signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); - const buf = Buffer.from(await res.arrayBuffer()); + const bytes = await readResponseBytesCapped( + res, + TOMTOM_TILE_MAX_RESPONSE_BYTES, + signal, + ); + const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); if (buf.length === 0) throw new Error('empty tile body'); return buf; } diff --git a/src/sources/httpBody.js b/src/sources/httpBody.js index 6f176ca52..481dd5b7f 100644 --- a/src/sources/httpBody.js +++ b/src/sources/httpBody.js @@ -55,38 +55,52 @@ export async function readResponseJsonCapped(response, maxBytes, signal) { /** * Read a fetch() Response body as bytes with the same hard cap as - * readResponseTextCapped — for protobuf upstreams (GTFS-Realtime). + * readResponseTextCapped — for protobuf and tile upstreams (GTFS-Realtime, + * TomTom flow). Same early Content-Length rejection, running cap while + * streaming and cancellation on abort, so callers can treat the two alike. * Throws { code:'RESPONSE_TOO_LARGE' }. */ -export async function readResponseBytesCapped(response, maxBytes) { +export async function readResponseBytesCapped(response, maxBytes, signal) { const tooLarge = () => { const err = new Error('Upstream response too large'); err.code = 'RESPONSE_TOO_LARGE'; return err; }; const declared = Number(response.headers.get('content-length')); - if (Number.isFinite(declared) && declared > maxBytes) throw tooLarge(); + if (Number.isFinite(declared) && declared > maxBytes) { + void response.body?.cancel().catch(() => {}); + throw tooLarge(); + } + signal?.throwIfAborted(); const reader = response.body?.getReader?.(); if (!reader) { const bytes = new Uint8Array(await response.arrayBuffer()); + signal?.throwIfAborted(); if (bytes.byteLength > maxBytes) throw tooLarge(); return bytes; } + const cancel = () => { + void reader.cancel().catch(() => {}); + }; + signal?.addEventListener('abort', cancel, { once: true }); const chunks = []; let total = 0; - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - total += value.byteLength; - if (total > maxBytes) { - try { - await reader.cancel(); - } catch { - /* no-op */ - } - throw tooLarge(); + try { + for (;;) { + signal?.throwIfAborted(); + const { done, value } = await reader.read(); + signal?.throwIfAborted(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) throw tooLarge(); + chunks.push(value); } - chunks.push(value); + } catch (error) { + cancel(); + throw error; + } finally { + signal?.removeEventListener('abort', cancel); + reader.releaseLock(); } const out = new Uint8Array(total); let offset = 0; diff --git a/src/sources/httpBody.test.mjs b/src/sources/httpBody.test.mjs new file mode 100644 index 000000000..aa8e77d1a --- /dev/null +++ b/src/sources/httpBody.test.mjs @@ -0,0 +1,133 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readResponseBytesCapped } from './httpBody.js'; + +/** + * A Response that streams `chunk` forever until the reader cancels it. The + * zero high-water mark keeps the stream from pulling eagerly at construction, + * so `pulls()` counts only what a reader actually asked for. + */ +function endlessResponse(chunk, onCancel, headers = {}) { + let pulls = 0; + const stream = new ReadableStream( + { + pull(controller) { + pulls += 1; + controller.enqueue(chunk); + }, + cancel: onCancel, + }, + { highWaterMark: 0 }, + ); + return { response: new Response(stream, { headers }), pulls: () => pulls }; +} + +function chunkedResponse(chunks, headers = {}) { + const stream = new ReadableStream( + { + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + controller.close(); + }, + }, + { highWaterMark: 0 }, + ); + return new Response(stream, { headers }); +} + +const bytes = (n, fill) => new Uint8Array(n).fill(fill); + +test('returns the exact bytes of a body that fits, untouched by text decoding', async () => { + // 0xFF and 0xFE are invalid UTF-8 lead bytes; a text path would mangle them. + const payload = Uint8Array.from([0xff, 0xfe, 0x00, 0x7f, 0x80, 0x01]); + const buf = await readResponseBytesCapped(chunkedResponse([payload]), 1024); + assert.ok(buf instanceof Uint8Array); + assert.deepEqual([...buf], [...payload]); +}); + +test('joins chunks in order and reports the total length', async () => { + const buf = await readResponseBytesCapped( + chunkedResponse([bytes(3, 1), bytes(2, 2), bytes(1, 3)]), + 1024, + ); + assert.deepEqual([...buf], [1, 1, 1, 2, 2, 3]); +}); + +test('refuses on a declared Content-Length without pulling the body', async () => { + let cancelled = false; + const { response, pulls } = endlessResponse( + bytes(512, 0), + () => { + cancelled = true; + }, + { 'content-length': '4096' }, + ); + await assert.rejects(readResponseBytesCapped(response, 1024), { + code: 'RESPONSE_TOO_LARGE', + }); + assert.equal(pulls(), 0); + assert.ok(cancelled); +}); + +test('refuses a chunked body the moment it passes the cap and stops pulling', async () => { + let cancelled = false; + const { response, pulls } = endlessResponse(bytes(512, 0), () => { + cancelled = true; + }); + await assert.rejects(readResponseBytesCapped(response, 1024), { + code: 'RESPONSE_TOO_LARGE', + }); + // Three 512-byte chunks is the first total over 1024; an uncapped read of + // this never-ending stream would not terminate at all. + assert.ok(pulls() <= 3, `pulled ${pulls()} chunks`); + assert.ok(cancelled); +}); + +test('accepts a body exactly at the cap', async () => { + const buf = await readResponseBytesCapped( + chunkedResponse([bytes(512, 7), bytes(512, 7)]), + 1024, + ); + assert.equal(buf.byteLength, 1024); +}); + +test('cancels a body still streaming when the signal aborts, and never returns a partial one', async () => { + const controller = new AbortController(); + let cancelled = false; + let pulls = 0; + const stream = new ReadableStream( + { + pull(ctrl) { + pulls += 1; + ctrl.enqueue(bytes(64, 0)); + if (pulls === 2) controller.abort(); + }, + cancel() { + cancelled = true; + }, + }, + { highWaterMark: 0 }, + ); + await assert.rejects( + readResponseBytesCapped(new Response(stream), 1 << 20, controller.signal), + ); + assert.ok(cancelled); + assert.ok(pulls <= 3, `pulled ${pulls} chunks after abort`); +}); + +test('refuses immediately when the signal is already aborted', async () => { + const controller = new AbortController(); + controller.abort(); + const { response, pulls } = endlessResponse(bytes(8, 0), () => {}); + await assert.rejects( + readResponseBytesCapped(response, 1024, controller.signal), + ); + assert.equal(pulls(), 0); +}); + +test('falls back to arrayBuffer() for a body with no stream reader', async () => { + const response = new Response(bytes(16, 9)); + const buf = await readResponseBytesCapped(response, 1024); + assert.equal(buf.byteLength, 16); + assert.equal(buf[0], 9); +}); From d585d25b19c0f41746268e07ea840f11365bd0e3 Mon Sep 17 00:00:00 2001 From: Michael Johnston Date: Tue, 15 Sep 2026 22:08:54 -0400 Subject: [PATCH 13/95] chore(format): adopt the httpBody reader test Co-Authored-By: Claude Opus 5 --- scripts/format-scope.json | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index eee546dd5..d661d7a6b 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -699,6 +699,7 @@ "src/search/nominatim.js", "src/search/nominatim.test.mjs", "src/sources/httpBody.js", + "src/sources/httpBody.test.mjs", "src/sources/overpass.js", "src/sources/nominatim.js", "server/providers/transit.js", From 320a1d6bfd65b9cd96cad39f90e7aa48c93d2a67 Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:01:17 +0200 Subject: [PATCH 14/95] fix(cctv): harden the Catalonia catalog fetch and frame-link allowlist Address review feedback on #595: - Reject frame links carrying userinfo (user:pass@) in normalizeCataloniaImageUrl, so catalog data cannot inject credentials into the server-side frame request on an allowlisted host. - Fetch cameres.xml over https (verified: same 200 body, no redirect), refuse redirects with the unread body released, and read the body through readResponseTextCapped with a 1 MiB CATALONIA_MAX_CATALOG_BYTES cap instead of an unbounded resp.text(). - Add regression tests for userinfo links on every allowed host, the refused redirect, and declared and streamed oversize bodies. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- DATA_SOURCES.md | 2 +- server/providers/cctv/constants.js | 8 +- server/providers/cctv/sources.js | 38 +++++++- src/data/cctvCataloniaSource.test.mjs | 125 ++++++++++++++++++++++++-- 5 files changed, 163 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45d48a26e..1ad4538e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -295,7 +295,9 @@ Analyst records for loaded satellites, datacenters and dams, with explicit bound hotlinked Barcelona and Terrassa municipal cameras, each partner named as a credit beside the shared provider. Ambient-card refresh paced to 3 minutes, matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` - sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. + sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. The catalog + is fetched over https with redirects refused and a 1 MiB body cap, and frame + links carrying userinfo are rejected even on an allowlisted host. - Distinguish PARTIAL vessel snapshots from STALE data in the layer panel, with accepted-record counts and unchanged retention, freshness and outage safeguards. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 824cb95be..5cc0be53d 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -90,7 +90,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `gencat.cat/transit/opendata/cameres.xml` WFS/GML feed. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin, and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. +- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo is rejected even on an allowed host), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index a64b06cd0..391c35230 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -238,7 +238,8 @@ export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024; /** * Catalonia traffic cameras (Servei Català de Trànsit): one keyless WFS/GML - * XML list (verified 2026-09-15, hourly catalog refresh). The feed carries a + * XML list (verified 2026-09-15, hourly catalog refresh), fetched over https + * (verified 2026-09-27: same 200 body, no redirect). The feed carries a * `font` field naming the actual publisher per camera — SCT's own highway * cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras are * kept, each confirmed to return a real frame. Frame URLs are checked @@ -246,7 +247,10 @@ export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024; * publisher serves from its own host. */ export const CATALONIA_CAMERAS_URL = - 'http://www.gencat.cat/transit/opendata/cameres.xml'; + 'https://www.gencat.cat/transit/opendata/cameres.xml'; +/** Byte ceiling for the Catalonia catalog read (~90 KB observed 2026-09-27), + * so a runaway body cannot be buffered without limit. */ +export const CATALONIA_MAX_CATALOG_BYTES = 1024 * 1024; /** Allowed image hosts, one per kept `font` publisher in the feed. */ export const CATALONIA_IMAGE_HOSTS = Object.freeze( new Set([ diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index f32b928a2..ee83f284e 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -56,6 +56,7 @@ import { CALGARY_DOWNTOWN, CALGARY_MAX_CATALOG_BYTES, CATALONIA_CAMERAS_URL, + CATALONIA_MAX_CATALOG_BYTES, CATALONIA_IMAGE_HOSTS, CATALONIA_FONT_CREDIT, CATALONIA_FONT_ELEVATION_M, @@ -89,7 +90,10 @@ import { prioritizeSources, } from './normalize.js'; import { directionToHeading } from '../../../src/data/directionText.js'; -import { readResponseJsonCapped } from '../common/http.js'; +import { + readResponseJsonCapped, + readResponseTextCapped, +} from '../common/http.js'; /** * Fetch and parse Austin traffic camera records from the city Open Data portal. * @@ -1649,7 +1653,10 @@ export function parseCataloniaXml(xml) { /** * Validate and normalize one Catalonia feed `link` against the publisher * host allowlist (CATALONIA_IMAGE_HOSTS), upgrading to https. Rejects - * anything else, including a host that merely resembles an allowed one. + * anything else, including a host that merely resembles an allowed one, and + * any URL carrying userinfo (`https://user:pass@www.bcn.cat/...`): the + * hostname check alone would pass it, letting catalog data inject + * credentials into the server-side frame request. * * SCT's own `RenderService` frame link is a special case, for two stacked * reasons (both verified 2026-09-15): @@ -1679,6 +1686,7 @@ export function normalizeCataloniaImageUrl(rawUrl) { return ''; } if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return ''; + if (parsed.username || parsed.password) return ''; if (!CATALONIA_IMAGE_HOSTS.has(parsed.hostname)) return ''; if (parsed.hostname === 'mct.gencat.cat') { @@ -1725,19 +1733,41 @@ export function cataloniaCameraName(carretera, municipi) { * shared `provider`. The feed carries no heading, so every camera takes the * id-hash fallback, low-confidence pose (same as TfL and Fintraffic). * + * The list decides ids, coordinates, labels and which allowlisted frame path + * is proxied, so it is fetched over https, redirects are refused (the list + * host cannot be steered or downgraded) and the body read is capped at + * CATALONIA_MAX_CATALOG_BYTES. + * * @returns {Promise>} Normalized camera source objects. */ export async function loadCataloniaSourcesFromOpenData() { try { const resp = await fetch(CATALONIA_CAMERAS_URL, { headers: { Accept: 'application/xml,text/xml,*/*' }, + redirect: 'manual', signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), }); + // A response this loader will not read still owns its transport until the + // body is released, so every rejection path cancels before returning. + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn( + '[CCTV] Catalonia catalog redirected; redirects are not followed', + ); + return discard(); + } if (!resp.ok) { console.warn('[CCTV] Catalonia camera download failed:', resp.status); - return []; + return discard(); } - const xml = await resp.text(); + const xml = await readResponseTextCapped(resp, CATALONIA_MAX_CATALOG_BYTES); const rows = parseCataloniaXml(xml); if (!rows.length) return []; diff --git a/src/data/cctvCataloniaSource.test.mjs b/src/data/cctvCataloniaSource.test.mjs index d9b9f3456..a2b1ed673 100644 --- a/src/data/cctvCataloniaSource.test.mjs +++ b/src/data/cctvCataloniaSource.test.mjs @@ -4,7 +4,10 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; -import { CATALONIA_CAMERAS_URL } from '../../server/providers/cctv/constants.js'; +import { + CATALONIA_CAMERAS_URL, + CATALONIA_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/constants.js'; import { parseCataloniaXml, normalizeCataloniaImageUrl, @@ -31,6 +34,24 @@ function quiet(t) { t.mock.method(console, 'warn', () => {}); } +/** + * A response whose body is a live stream, plus a flag that flips when the + * stream is cancelled. A rejection path that returns without cancelling holds + * the transport open, so the flag is what the refusal tests actually assert. + */ +const streamingResponse = (init = {}) => { + const state = { cancelled: false }; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('` block matching the real feed's shape. */ function cataloniaFeature({ fid, @@ -143,6 +164,19 @@ test('normalizeCataloniaImageUrl accepts the other registered hosts, upgraded to assert.equal(normalizeCataloniaImageUrl('not a url'), ''); }); +test('normalizeCataloniaImageUrl rejects credential-bearing URLs on every allowed host', () => { + // The hostname matches the allowlist, so only an explicit userinfo check + // stops catalog data injecting credentials into the frame request. + for (const link of [ + 'https://user:pass@www.bcn.cat/transit/imatges/a.gif', + 'http://user@www.bcn.cat/transit/imatges/a.gif', + 'https://:pass@emap.terrassa.cat/it_terrassa/cam02.jpeg', + 'http://user:pass@mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + ]) { + assert.equal(normalizeCataloniaImageUrl(link), '', link); + } +}); + test('cataloniaCameraName joins road and municipality, skipping a redundant municipality', () => { assert.equal( cataloniaCameraName('C-58', 'Nus Trinitat'), @@ -164,8 +198,8 @@ test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publis quiet(t); withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); const requested = []; - t.mock.method(globalThis, 'fetch', async (url) => { - requested.push(String(url)); + t.mock.method(globalThis, 'fetch', async (url, init) => { + requested.push([String(url), init.redirect]); const xml = cataloniaFeed([ cataloniaFeature({ fid: 'cameres.fid-1', @@ -220,7 +254,11 @@ test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publis const cameras = await loadCataloniaSourcesFromOpenData(); - assert.deepEqual(requested, [CATALONIA_CAMERAS_URL]); + assert.deepEqual(requested, [[CATALONIA_CAMERAS_URL, 'manual']]); + assert.ok( + CATALONIA_CAMERAS_URL.startsWith('https://'), + 'the catalog is fetched over https', + ); assert.deepEqual( cameras.map((camera) => camera.id).sort(), ['cat-1', 'cat-2', 'cat-3'], @@ -264,6 +302,81 @@ test('Catalonia loader fails soft on HTTP errors, empty payloads and network err } }); +test('the Catalonia catalog fetch refuses redirects and oversized bodies', async (t) => { + t.mock.method(console, 'warn', () => {}); + // A redirect is never followed: the list host cannot be steered or + // downgraded. Its body is a live stream, so the test also proves the + // refusal releases the transport. + const seen = []; + const redirected = streamingResponse({ + status: 302, + headers: { location: 'http://www.gencat.cat/transit/opendata/cameres.xml' }, + }); + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init.redirect]); + return redirected.response; + }); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + assert.deepEqual(seen, [[CATALONIA_CAMERAS_URL, 'manual']]); + assert.equal( + redirected.state.cancelled, + true, + 'the redirect body is cancelled', + ); + + // An HTTP error body is released too. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + const failed = streamingResponse({ status: 503 }); + t.mock.method(globalThis, 'fetch', async () => failed.response); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + assert.equal(failed.state.cancelled, true, 'the error body is cancelled'); + + const validFeed = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-1', + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]); + + // A body that declares a size over the cap is refused rather than buffered, + // even though its actual content would parse. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + return new Response(validFeed, { + headers: { + 'Content-Type': 'application/xml', + 'content-length': String(CATALONIA_MAX_CATALOG_BYTES + 1), + }, + }); + }); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + + // So is a body with no declared size that only turns out too long while + // streaming — a valid feed padded past the cap. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + const oversized = + validFeed + ' '.repeat(CATALONIA_MAX_CATALOG_BYTES + 1024); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(oversized)); + controller.close(); + }, + }), + { headers: { 'Content-Type': 'application/xml' } }, + ); + }); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); +}); + test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips the request', async (t) => { quiet(t); const sourceRoot = fs.mkdtempSync( @@ -282,8 +395,8 @@ test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips t CCTV_CATALONIA_MAX_SOURCES: undefined, }); const requested = []; - t.mock.method(globalThis, 'fetch', async (url) => { - requested.push(String(url)); + t.mock.method(globalThis, 'fetch', async (url, init) => { + requested.push([String(url), init.redirect]); if (String(url) !== CATALONIA_CAMERAS_URL) { return new Response('unavailable', { status: 503 }); } From 481d0c327a36c1c8f2d6c60f44f5de98c79881ea Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Sun, 27 Sep 2026 12:09:27 +0000 Subject: [PATCH 15/95] fix(cctv): map WSDOT diagonal and BW compass codes; document WSDOT env WSDOT's layer metadata defines BW (Bothways) plus the diagonals NE/SE/SW/NW alongside N/E/S/W. Diagonal cameras were being demoted to the low-confidence id-hash fallback; map them to 45/135/225/315 degrees and treat BW (not B) as the no-single-facing code. Add CCTV_WSDOT_ENABLED and CCTV_WSDOT_MAX_SOURCES to .env.example with the other pack controls. --- .env.example | 2 ++ DATA_SOURCES.md | 2 +- server/providers/cctv/sources.js | 15 +++++++--- src/data/cctvWsdotSource.test.mjs | 46 +++++++++++++++++++++++++++++-- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index e16e85a27..45ae52099 100644 --- a/.env.example +++ b/.env.example @@ -180,6 +180,8 @@ AISSTREAM_API_KEY= # CCTV_CALGARY_ENABLED=1 # set 0 to disable the Open Calgary (Alberta) cameras # CCTV_CALGARY_MAX_SOURCES=220 # cap cameras loaded from the Open Calgary catalog # CCTV_CALGARY_ROWS_URL= # override the Open Calgary catalog URL +# CCTV_WSDOT_ENABLED=1 # set 0 to disable the WSDOT (Washington State) highway cameras +# CCTV_WSDOT_MAX_SOURCES=250 # cap cameras loaded from the WSDOT catalog, nearest Seattle/Spokane first # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 73eaf8a1f..760ac8429 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -90,7 +90,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **WSDOT highway cameras.** Keyless. The camera list is the Esri JSON layer `data.wsdot.wa.gov/travelcenter/Cameras.json` — the same layer WSDOT's own Travel Center map consumes (~1,700 point features, EPSG:3857 geometry converted to WGS84 on load; the body carries Windows-1252 bytes and is decoded as such). Only frames on WSDOT's own `images.wsdot.wa.gov` host are registered — the catalog also lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) whose own terms apply, and those are skipped rather than proxied. WSDOT's layer metadata marks the feed for "low volume" use, which the 15-minute catalog cache and per-request frame fetches respect; frames are never stored or redistributed. `CompassDirection` letters (N/E/S/W) give a high-confidence heading; the majority `B` ("both directions") rows carry no single facing and use the shared id-hash fallback at low confidence. By default the 250 cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` changes the cap and `CCTV_WSDOT_ENABLED=0` turns the pack off. A keyed WSDOT Traveler Information API serves the same population for higher-volume use and is deliberately not required here. +- **WSDOT highway cameras.** Keyless. The camera list is the Esri JSON layer `data.wsdot.wa.gov/travelcenter/Cameras.json` — the same layer WSDOT's own Travel Center map consumes (~1,700 point features, EPSG:3857 geometry converted to WGS84 on load; the body carries Windows-1252 bytes and is decoded as such). Only frames on WSDOT's own `images.wsdot.wa.gov` host are registered — the catalog also lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) whose own terms apply, and those are skipped rather than proxied. WSDOT's layer metadata marks the feed for "low volume" use, which the 15-minute catalog cache and per-request frame fetches respect; frames are never stored or redistributed. `CompassDirection` codes with a single facing (N/E/S/W and the diagonals NE/SE/SW/NW, per WSDOT's layer metadata) give a high-confidence heading; `BW` ("Bothways") rows carry no single facing and use the shared id-hash fallback at low confidence. By default the 250 cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` changes the cap and `CCTV_WSDOT_ENABLED=0` turns the pack off. A keyed WSDOT Traveler Information API serves the same population for higher-volume use and is deliberately not required here. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 1124b187f..b7fc197b0 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -1706,13 +1706,19 @@ export async function loadDelDOTSourcesFromOpenData() { } } -/** WSDOT compass letters as headings in degrees. `B` (both directions), `O` - * (other) and null carry no single facing and fall through to the id hash. */ +/** WSDOT `CompassDirection` codes as headings in degrees, per the layer's own + * metadata (N/E/S/W, NE/NW/SE/SW, BW = Bothways). `BW` and anything else + * (`O`, null, unknown codes) carry no single facing and fall through to the + * id hash. */ const WSDOT_COMPASS_HEADINGS = Object.freeze({ N: 0, + NE: 45, E: 90, + SE: 135, S: 180, + SW: 225, W: 270, + NW: 315, }); /** @@ -1749,8 +1755,9 @@ function isLikelyWashingtonCoordinate(lat, lon) { * finite in-state coordinates, and a frame on WSDOT's own image host are * kept — the catalog's ~70 partner cameras (Oregon DOT, City of Seattle, * lodges, airports) are skipped rather than proxied under someone else's - * terms. `CompassDirection` letters give a high-confidence heading; `B` - * (both directions, the majority) and `O`/null fall back to the id hash. + * terms. A single-facing `CompassDirection` (cardinal or diagonal) gives a + * high-confidence heading; `BW` (Bothways) and `O`/null fall back to the id + * hash. * The feed carries Windows-1252 bytes (en-dashes in titles) with no charset * declaration, so the body is decoded explicitly before parsing. * diff --git a/src/data/cctvWsdotSource.test.mjs b/src/data/cctvWsdotSource.test.mjs index 0459f3c72..438df7e58 100644 --- a/src/data/cctvWsdotSource.test.mjs +++ b/src/data/cctvWsdotSource.test.mjs @@ -89,7 +89,7 @@ test('WSDOT loader keeps official-host cameras and maps compass directions', asy }, }), wsdotFeature(9059, 47.66, -117.43, { - attributes: { CompassDirection: 'B' }, + attributes: { CompassDirection: 'BW' }, }), // Partner-hosted frame (Oregon DOT): skipped, never proxied. wsdotFeature(1002, 45.62, -122.675, { @@ -127,7 +127,7 @@ test('WSDOT loader keeps official-host cameras and maps compass directions', asy assert.ok(Math.abs(seattle.lat - 47.6097) < 0.001); assert.ok(Math.abs(seattle.lon - -122.3331) < 0.001); - // "Both directions" carries no single facing: id-hash fallback at low + // `BW` (Bothways) carries no single facing: id-hash fallback at low // confidence, with the low-confidence pose personality. const spokane = cameras.find((camera) => camera.id === 'wsdot-9059'); assert.equal(spokane.headingConfidence, 'low'); @@ -135,6 +135,48 @@ test('WSDOT loader keeps official-host cameras and maps compass directions', asy assert.ok(spokane.headingDeg >= 0 && spokane.headingDeg < 360); }); +test('WSDOT loader maps every documented CompassDirection code', async (t) => { + quiet(t); + withEnv(t, { CCTV_WSDOT_MAX_SOURCES: undefined }); + const codes = { + N: 0, + NE: 45, + E: 90, + SE: 135, + S: 180, + SW: 225, + W: 270, + NW: 315, + // Lower-case and padded codes are the same code. + ' nw ': 315, + }; + const noFacing = ['BW', 'O', null]; + const entries = [...Object.keys(codes), ...noFacing]; + t.mock.method(globalThis, 'fetch', async () => + cp1252Response({ + features: entries.map((code, i) => + wsdotFeature(3000 + i, 47.0 + i * 0.05, -122.9 + i * 0.05, { + attributes: { CompassDirection: code }, + }), + ), + }), + ); + + const cameras = await loadWsdotSourcesFromOpenData(); + const byId = new Map(cameras.map((camera) => [camera.id, camera])); + entries.forEach((code, i) => { + const camera = byId.get(`wsdot-${3000 + i}`); + if (code in codes) { + assert.equal(camera.headingDeg, codes[code], `heading for ${code}`); + assert.equal(camera.headingConfidence, 'high', `confidence for ${code}`); + assert.equal(camera.fovDeg, 56, `pose personality for ${code}`); + } else { + assert.equal(camera.headingConfidence, 'low', `confidence for ${code}`); + assert.equal(camera.fovDeg, 44, `pose personality for ${code}`); + } + }); +}); + test('WSDOT loader honors the max-sources cap', async (t) => { quiet(t); withEnv(t, { CCTV_WSDOT_MAX_SOURCES: '8' }); From c3f197823e2b13a1538b4a1c4cd0dada29650752 Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Sun, 27 Sep 2026 12:09:27 +0000 Subject: [PATCH 16/95] fix(cctv): serve King County model/manufacturer through /api/cctv/sources The loader and normalizer carried the hardware model, but the /sources response projection and the browser catalog both whitelist fields and dropped it, so no client consumer ever saw it. Project model and manufacturer (manufacturer disambiguates bare model strings like "3950") at both boundaries, and add a test that drives the real middleware and browser catalog end to end. --- DATA_SOURCES.md | 2 +- server/providers/cctv.js | 2 + server/providers/cctv/normalize.js | 6 ++ server/providers/cctv/sources.js | 4 + src/data/cctvKingCountySource.test.mjs | 106 +++++++++++++++++++++++++ src/layers/cctv/catalog.js | 2 + 6 files changed, 121 insertions(+), 1 deletion(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index c43ae7728..8e806208d 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -90,7 +90,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped so a camera never appears twice when a statewide WSDOT pack is present; the few city-owned cameras the county hosts (Redmond, Sammamish, Newcastle, Snoqualmie) carry their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); the model string is carried on the source as `model` for identified-hardware consumers. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. +- **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped so a camera never appears twice when a statewide WSDOT pack is present; the few city-owned cameras the county hosts (Redmond, Sammamish, Newcastle, Snoqualmie) carry their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); both are carried on the source (and served by `/api/cctv/sources`) as `model` and `manufacturer` for identified-hardware consumers. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/server/providers/cctv.js b/server/providers/cctv.js index a71e2af4d..5fb15be03 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -162,6 +162,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { sourceKind: source.sourceKind || (source.url ? 'configured' : 'fallback'), poseSource: source.poseSource, + model: source.model || '', + manufacturer: source.manufacturer || '', license: source.license, credit: source.credit || '', code: source.code || '', diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index 17903335d..c4020da34 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -524,5 +524,11 @@ export function normalizeSourceItem(item) { typeof item.model === 'string' && item.model.trim() ? item.model.trim() : undefined, + // Bare model strings collide across vendors (e.g. "3950"), so the + // publishing pack's manufacturer rides along for disambiguation. + manufacturer: + typeof item.manufacturer === 'string' && item.manufacturer.trim() + ? item.manufacturer.trim() + : undefined, }; } diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index e92a8a265..3996f37f1 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -1776,6 +1776,9 @@ export async function loadKingCountySourcesFromOpenData() { const model = String(attrs.Model || '') .replace(/\s+/g, ' ') .trim(); + const manufacturer = String(attrs.Manufacturer || '') + .replace(/\s+/g, ' ') + .trim(); const camera = { id: cameraId, name: @@ -1800,6 +1803,7 @@ export async function loadKingCountySourcesFromOpenData() { credit: owner && owner !== 'King County' ? owner : '', }; if (model) camera.model = model; + if (model && manufacturer) camera.manufacturer = manufacturer; cameras.push(camera); } diff --git a/src/data/cctvKingCountySource.test.mjs b/src/data/cctvKingCountySource.test.mjs index 544b4f9ac..a0f765446 100644 --- a/src/data/cctvKingCountySource.test.mjs +++ b/src/data/cctvKingCountySource.test.mjs @@ -1,11 +1,16 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; import { DEFAULT_KING_COUNTY_MAX_SOURCES, KING_COUNTY_CAMERAS_URL, } from '../../server/providers/cctv/constants.js'; import { loadKingCountySourcesFromOpenData } from '../../server/providers/cctv/sources.js'; import { normalizeSourceItem } from '../../server/providers/cctv/normalize.js'; +import { cctvProxy } from '../../server/providers/cctv.js'; +import { createCatalog } from '../layers/cctv/catalog.js'; /** Set (or, for `undefined`, delete) environment variables for one test. */ function withEnv(t, env) { @@ -106,6 +111,7 @@ test('King County loader keeps live county cameras and upgrades frame URLs to HT assert.equal(novelty.feedType, 'image'); // Published hardware model rides along, whitespace-collapsed. assert.equal(novelty.model, 'Q6135-LE'); + assert.equal(novelty.manufacturer, 'Axis'); assert.equal(novelty.credit, ''); // No facing in the layer: id-hash fallback at low confidence. assert.equal(novelty.headingConfidence, 'low'); @@ -115,6 +121,8 @@ test('King County loader keeps live county cameras and upgrades frame URLs to HT const redmond = cameras.find((camera) => camera.id === 'kingcounty-90'); assert.equal(redmond.credit, 'City of Redmond'); assert.equal(redmond.model, undefined); + // A manufacturer without a model identifies nothing, so it is dropped too. + assert.equal(redmond.manufacturer, undefined); }); test('normalizeSourceItem passes a hardware model through and omits it otherwise', () => { @@ -123,8 +131,10 @@ test('normalizeSourceItem passes a hardware model through and omits it otherwise lat: 47.7, lon: -122.0, model: ' Q6135-LE ', + manufacturer: ' Axis ', }); assert.equal(withModel.model, 'Q6135-LE'); + assert.equal(withModel.manufacturer, 'Axis'); const withoutModel = normalizeSourceItem({ id: 'austin-1', @@ -132,9 +142,105 @@ test('normalizeSourceItem passes a hardware model through and omits it otherwise lon: -97.7, }); assert.equal(withoutModel.model, undefined); + assert.equal(withoutModel.manufacturer, undefined); assert.equal(normalizeSourceItem({ id: 'x', model: ' ' }).model, undefined); }); +/** Drive the real `/api/cctv/sources` middleware and return its parsed body. */ +async function requestSources(plugin) { + let handler; + plugin.configureServer({ + middlewares: { + use(_route, fn) { + handler = fn; + }, + }, + }); + const res = { + writeHead(status, headers) { + Object.assign(this, { status, headers }); + }, + end(body) { + this.body = body; + }, + }; + await handler({ url: '/sources', method: 'GET' }, res); + assert.equal(res.status, 200); + return JSON.parse(res.body); +} + +/** The browser catalog with the pose math stubbed out: only field + * passthrough is under test here. */ +function browserCatalog() { + const model = { + safeNumber: (value, fallback = NaN) => + Number.isFinite(Number(value)) ? Number(value) : fallback, + normalizeHeading: (deg) => ((deg % 360) + 360) % 360, + clamp: (value, lo, hi) => Math.min(hi, Math.max(lo, value)), + headingFromId: () => 0, + normalizeFeedType: (type) => type, + ensureCameraPose: () => {}, + }; + return createCatalog({ + state: {}, + services: { locations: { CITY_POIS: {} } }, + parts: { model }, + source: {}, + }); +} + +test('King County model and manufacturer survive the /api/cctv/sources boundary into the browser catalog', async (t) => { + quiet(t); + const root = mkdtempSync(path.join(tmpdir(), 'gev-kingcounty-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + // No file/env catalog, so the live packs load; every pack but King County + // sees an upstream failure and contributes nothing. + withEnv(t, { + CCTV_SOURCES_FILE: undefined, + CCTV_SOURCES_JSON: undefined, + CCTV_FORCE_AUSTIN: undefined, + CCTV_PREFER_AUSTIN: undefined, + CCTV_KINGCOUNTY_ENABLED: undefined, + CCTV_KINGCOUNTY_MAX_SOURCES: undefined, + CCTV_MAX_SOURCES: undefined, + }); + t.mock.method(globalThis, 'fetch', async (url) => { + if (String(url) !== KING_COUNTY_CAMERAS_URL) + return new Response('', { status: 503 }); + return Response.json({ + type: 'FeatureCollection', + features: [ + kingCountyFeature(56, 47.70059, -122.02443, { + properties: { Model: 'Q6135-LE\n', Manufacturer: 'Axis' }, + }), + kingCountyFeature(90, 47.674, -122.119, { + properties: { Model: null, Manufacturer: 'Cohu' }, + }), + ], + }); + }); + + const body = await requestSources(cctvProxy({ sourceRoot: root })); + const served = new Map(body.sources.map((source) => [source.id, source])); + assert.deepEqual([...served.keys()].sort(), [ + 'kingcounty-56', + 'kingcounty-90', + ]); + assert.equal(served.get('kingcounty-56').model, 'Q6135-LE'); + assert.equal(served.get('kingcounty-56').manufacturer, 'Axis'); + // Absent hardware is an explicit empty string on the wire, like + // headingConfidence, so consumers never have to probe for the key. + assert.equal(served.get('kingcounty-90').model, ''); + assert.equal(served.get('kingcounty-90').manufacturer, ''); + + const cameras = browserCatalog().buildCatalogFromSources(body.sources); + const byId = new Map(cameras.map((camera) => [camera.id, camera])); + assert.equal(byId.get('kingcounty-56').model, 'Q6135-LE'); + assert.equal(byId.get('kingcounty-56').manufacturer, 'Axis'); + assert.equal(byId.get('kingcounty-90').model, ''); + assert.equal(byId.get('kingcounty-90').manufacturer, ''); +}); + test('King County loader honors the max-sources cap', async (t) => { quiet(t); withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: '8' }); diff --git a/src/layers/cctv/catalog.js b/src/layers/cctv/catalog.js index 1a6ebe7c6..76d530b11 100644 --- a/src/layers/cctv/catalog.js +++ b/src/layers/cctv/catalog.js @@ -204,6 +204,8 @@ export function createCatalog({ state: layerState, services, parts, source }) { ? source.groundHeights : null, poseSource, + model: String(source.model || ''), + manufacturer: String(source.manufacturer || ''), }; parts.model.ensureCameraPose(camera); catalog.push(camera); From ffa1891b01b6d39def2d2854560d6e927d4c9cfb Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Sun, 27 Sep 2026 12:09:27 +0000 Subject: [PATCH 17/95] fix(cctv): keep estimated FOV for PTZ/varifocal hardware; carry datasheet range as capability A PTZ/varifocal datasheet FOV is a capability range, not the lens's current zoom, so replacing the frustum's estimate with its wide end presented unknown optical state as known. Parse the full horizontal range (ignoring vertical, "combined" panoramic and lens-note figures), carry it as specFovMinDeg/specFovMaxDeg, and substitute the datasheet FOV only for fixed single-lens hardware. The HUD labels the range as capability beside the estimate. This also stops the Q6000-E Mk II's "360 combined" figure from being read as a 360-degree lens. --- src/layers/cctv/catalog.js | 25 ++++-- src/layers/cctv/modelSpecs.js | 64 +++++++++++--- src/layers/cctv/modelSpecs.test.mjs | 124 ++++++++++++++++++++++++---- src/layers/cctv/presentation.js | 7 +- 4 files changed, 186 insertions(+), 34 deletions(-) diff --git a/src/layers/cctv/catalog.js b/src/layers/cctv/catalog.js index be6dc7e90..499e715a0 100644 --- a/src/layers/cctv/catalog.js +++ b/src/layers/cctv/catalog.js @@ -1,11 +1,17 @@ import { CAMERA_SEEDS, SOURCE_ENDPOINT } from './policy.js'; -import { lookupModelSpec, horizontalFovDeg } from './modelSpecs.js'; +import { + lookupModelSpec, + horizontalFovRangeDeg, + fixedHorizontalFovDeg, +} from './modelSpecs.js'; /** * Applies datasheet enrichment to a camera in place: when it names a known - * hardware `model`, the manufacturer's horizontal FOV replaces the pose - * estimate and the spec record is attached for the HUD. Unknown models — the - * common case — change nothing. + * hardware `model`, the spec record is attached for the HUD and the + * datasheet's horizontal FOV range rides along as `specFovMinDeg`/ + * `specFovMaxDeg`. Only fixed-lens hardware replaces the pose estimate: a + * PTZ/varifocal range says what the lens CAN do, not where it is zoomed now. + * Unknown models — the common case — change nothing. * @param {Object} camera - Camera being built (mutated). * @param {string} [modelName] - Hardware model from a source or seed. * @param {(v:number,min:number,max:number)=>number} clamp - Pose clamp helper. @@ -13,9 +19,14 @@ import { lookupModelSpec, horizontalFovDeg } from './modelSpecs.js'; function applyModelEnrichment(camera, modelName, clamp) { const spec = modelName ? lookupModelSpec(modelName) : null; if (!spec) return; - const fov = horizontalFovDeg(spec); - if (fov) { - camera.fovDeg = clamp(fov, 20, 125); + const range = horizontalFovRangeDeg(spec); + if (range) { + camera.specFovMinDeg = range.minDeg; + camera.specFovMaxDeg = range.maxDeg; + } + const fixedFov = fixedHorizontalFovDeg(spec); + if (fixedFov) { + camera.fovDeg = clamp(fixedFov, 20, 125); camera.fovSource = 'datasheet'; } camera.spec = spec; diff --git a/src/layers/cctv/modelSpecs.js b/src/layers/cctv/modelSpecs.js index 1252ce013..a9d12e23d 100644 --- a/src/layers/cctv/modelSpecs.js +++ b/src/layers/cctv/modelSpecs.js @@ -5,8 +5,10 @@ * * When a catalog camera names its hardware model (source packs may set * `model`; hand-authored seeds may too), the model resolves to a datasheet - * record: the camera's frustum then uses the manufacturer's horizontal FOV - * instead of an estimate, and the HUD meta line gains the model's real specs. + * record and the HUD meta line gains the model's real specs. A fixed-lens + * model's horizontal FOV replaces the frustum's estimate; a PTZ/varifocal or + * multi-sensor range is carried as capability metadata beside the estimate, + * because the lens's current zoom is unknown. * Cameras without an identified model are untouched — unknown never implies * anything. * @@ -376,16 +378,58 @@ export function lookupModelSpec(model) { } /** - * Datasheet horizontal FOV in degrees: the FIRST number in the `fov_deg` - * string, which for varifocal/PTZ ranges ('58.3-2.4 horizontal') is the wide - * end — the honest default for a coverage cone. + * Datasheet horizontal FOV as the range the hardware can cover. Only + * segments stating a horizontal figure count — vertical and "combined" + * panoramic figures never do — and parenthesized lens/sensor notes are + * ignored. A zoom ('58.3-2.4 horizontal') and a lens-option list + * ('108.8 horizontal (2.8mm) / 93.3 horizontal (4mm)') both yield a range. * @param {Object} spec - A record from this module. - * @returns {number|null} Wide-end horizontal FOV, or null when unstated. + * @returns {{minDeg:number, maxDeg:number}|null} Null when unstated. */ -export function horizontalFovDeg(spec) { - const match = String(spec?.fov_deg || '').match(/\d+(?:\.\d+)?/); - const value = match ? Number(match[0]) : NaN; - return Number.isFinite(value) && value > 0 ? value : null; +export function horizontalFovRangeDeg(spec) { + const values = []; + for (const segment of String(spec?.fov_deg || '').split(/[/;,]/)) { + if (!/horizontal/i.test(segment)) continue; + const figures = segment.replace(/\([^)]*\)/g, '').split(/horizontal/i)[0]; + for (const match of figures.matchAll(/\d+(?:\.\d+)?/g)) { + const value = Number(match[0]); + if (Number.isFinite(value) && value > 0) values.push(value); + } + } + if (!values.length) return null; + return { minDeg: Math.min(...values), maxDeg: Math.max(...values) }; +} + +/** + * The datasheet horizontal FOV as the camera's CURRENT optical state, which + * holds only for fixed single-lens hardware stating one figure. PTZ, + * varifocal and multi-sensor models publish a capability range whose in-use + * value is unknown — known hardware capability is not known pose state — so + * they return null and keep their estimated FOV. + * @param {Object} spec - A record from this module. + * @returns {number|null} Fixed horizontal FOV in degrees, or null. + */ +export function fixedHorizontalFovDeg(spec) { + if (!spec || spec.ptz || spec.varifocal || (spec.lens_count || 1) > 1) + return null; + const range = horizontalFovRangeDeg(spec); + return range && range.minDeg === range.maxDeg ? range.minDeg : null; +} + +/** + * HUD token for a datasheet FOV that describes capability rather than the + * camera's current state, e.g. 'SPEC FOV 2.4–58° (CAPABILITY)'. + * @param {Object} camera - Catalog camera (see applyModelEnrichment). + * @returns {string|null} Null when the datasheet FOV already IS the camera's + * FOV (fixed lens) or no datasheet range is known. + */ +export function specFovCapabilityToken(camera) { + if (!camera || camera.fovSource === 'datasheet') return null; + const { specFovMinDeg: min, specFovMaxDeg: max } = camera; + if (!Number.isFinite(min) || !Number.isFinite(max)) return null; + const fmt = (deg) => (deg < 10 ? deg.toFixed(1) : String(Math.round(deg))); + const span = min === max ? fmt(max) : `${fmt(min)}–${fmt(max)}`; + return `SPEC FOV ${span}° (CAPABILITY)`; } /** diff --git a/src/layers/cctv/modelSpecs.test.mjs b/src/layers/cctv/modelSpecs.test.mjs index c504e14ad..06ffe3a64 100644 --- a/src/layers/cctv/modelSpecs.test.mjs +++ b/src/layers/cctv/modelSpecs.test.mjs @@ -6,9 +6,12 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { lookupModelSpec, - horizontalFovDeg, + horizontalFovRangeDeg, + fixedHorizontalFovDeg, + specFovCapabilityToken, specSummary, } from './modelSpecs.js'; +import { createCatalog } from './catalog.js'; test('lookupModelSpec matches case-insensitively and trims', () => { const spec = lookupModelSpec(' q6135-le '); @@ -29,11 +32,11 @@ test('vendored table covers the integrated catalogs published hardware', () => { const cohu = lookupModelSpec('3950'); // King County publishes "3950" assert.equal(cohu?.brand, 'Cohu'); const autodome = lookupModelSpec('AUTODOME IP 5000i'); - assert.ok(autodome && horizontalFovDeg(autodome) > 0); + assert.ok(autodome && horizontalFovRangeDeg(autodome).maxDeg > 0); const vg4 = lookupModelSpec('VG4 AUTODOME H.264'); // Sarasota's string assert.ok(vg4); const axis = lookupModelSpec('Q6155-E'); // Sioux Falls - assert.equal(horizontalFovDeg(axis), 66.7); + assert.deepEqual(horizontalFovRangeDeg(axis), { minDeg: 2.36, maxDeg: 66.7 }); }); test('verbose registry strings resolve via distinctive-token fallback', () => { @@ -80,23 +83,112 @@ test('lookupModelSpec returns null for unknown or empty input — never guesses' assert.equal(lookupModelSpec(undefined), null); }); -test('horizontalFovDeg takes the wide end of a varifocal/PTZ range', () => { - // '58.3-2.4 horizontal' → 58.3 (wide), not 2.4 (tele). - assert.equal(horizontalFovDeg(lookupModelSpec('Q6135-LE')), 58.3); +test('horizontalFovRangeDeg reads the full range across real fov_deg shapes', () => { + const range = (model) => horizontalFovRangeDeg(lookupModelSpec(model)); + // PTZ zoom: both ends, not just the wide one. + assert.deepEqual(range('Q6135-LE'), { minDeg: 2.4, maxDeg: 58.3 }); + // Vertical figures never count. + assert.deepEqual(range('XNP-6400RW'), { minDeg: 1.88, maxDeg: 65.66 }); + // Lens options: parenthesized focal lengths are not FOV figures. + assert.deepEqual(range('DS-2CD2087G3-LI2UY'), { + minDeg: 93.3, + maxDeg: 108.8, + }); + assert.deepEqual(range('F4105-LRE'), { minDeg: 110, maxDeg: 110 }); + // The "360 combined" panoramic figure is not a horizontal lens FOV. + assert.deepEqual(range('Q6000-E Mk II'), { minDeg: 113, maxDeg: 152 }); + assert.deepEqual(range('P3707-PE'), { minDeg: 54, maxDeg: 108 }); }); -test('horizontalFovDeg reads multi-variant and plain fov strings', () => { - // '108.8 horizontal (2.8mm) / 93.3 horizontal (4mm)' → first variant. - assert.equal(horizontalFovDeg(lookupModelSpec('DS-2CD2087G3-LI2UY')), 108.8); - // '110 horizontal, 60 vertical' → horizontal figure. - assert.equal(horizontalFovDeg(lookupModelSpec('F4105-LRE')), 110); +test('horizontalFovRangeDeg returns null when no numeric FOV is stated', () => { + assert.equal(horizontalFovRangeDeg({ fov_deg: '' }), null); + assert.equal(horizontalFovRangeDeg({ fov_deg: 'wide dynamic range' }), null); + assert.equal(horizontalFovRangeDeg({ fov_deg: '60 vertical' }), null); + assert.equal(horizontalFovRangeDeg({}), null); + assert.equal(horizontalFovRangeDeg(null), null); }); -test('horizontalFovDeg returns null when no numeric FOV is stated', () => { - assert.equal(horizontalFovDeg({ fov_deg: '' }), null); - assert.equal(horizontalFovDeg({ fov_deg: 'wide dynamic range' }), null); - assert.equal(horizontalFovDeg({}), null); - assert.equal(horizontalFovDeg(null), null); +test('fixedHorizontalFovDeg is the current FOV only for fixed single-lens hardware', () => { + // Fixed dome, one horizontal figure: the datasheet IS the optical state. + assert.equal(fixedHorizontalFovDeg(lookupModelSpec('F4105-LRE')), 110); + // Known hardware capability != known pose state: PTZ, multi-sensor, and a + // model sold in several lens options all stay estimated. + for (const model of [ + 'Q6135-LE', + 'XNP-6400RW', + 'AUTODOME IP 5000i', + 'P3707-PE', + 'Q6000-E Mk II', + 'DS-2CD2087G3-LI2UY', + ]) { + assert.equal(fixedHorizontalFovDeg(lookupModelSpec(model)), null, model); + } + assert.equal(fixedHorizontalFovDeg(null), null); +}); + +test('specFovCapabilityToken labels a datasheet range as capability, never as current FOV', () => { + assert.equal( + specFovCapabilityToken({ specFovMinDeg: 2.4, specFovMaxDeg: 58.3 }), + 'SPEC FOV 2.4–58° (CAPABILITY)', + ); + assert.equal( + specFovCapabilityToken({ specFovMinDeg: 113, specFovMaxDeg: 113 }), + 'SPEC FOV 113° (CAPABILITY)', + ); + // Already the camera's FOV (fixed lens) or nothing known: no token. + assert.equal( + specFovCapabilityToken({ + fovSource: 'datasheet', + specFovMinDeg: 110, + specFovMaxDeg: 110, + }), + null, + ); + assert.equal(specFovCapabilityToken({}), null); + assert.equal(specFovCapabilityToken(null), null); +}); + +/** Browser catalog with the pose math stubbed: only enrichment is under test. */ +function browserCatalog() { + const model = { + safeNumber: (value, fallback = NaN) => + Number.isFinite(Number(value)) ? Number(value) : fallback, + normalizeHeading: (deg) => ((deg % 360) + 360) % 360, + clamp: (value, lo, hi) => Math.min(hi, Math.max(lo, value)), + headingFromId: () => 0, + normalizeFeedType: (type) => type, + ensureCameraPose: () => {}, + }; + return createCatalog({ + state: {}, + services: { locations: { CITY_POIS: {} } }, + parts: { model }, + source: {}, + }); +} + +test('catalog keeps the estimated FOV for PTZ hardware and substitutes only fixed-lens datasheets', () => { + const [ptz, fixed, unknown] = browserCatalog().buildCatalogFromSources([ + { id: 'ptz', lat: 47.7, lon: -122, fovDeg: 44, model: 'Q6135-LE' }, + { id: 'fixed', lat: 47.7, lon: -122, fovDeg: 44, model: 'F4105-LRE' }, + { id: 'unknown', lat: 47.7, lon: -122, fovDeg: 44, model: 'NOPE-1' }, + ]); + + // PTZ: the frustum keeps its estimate; the datasheet range is metadata. + assert.equal(ptz.fovDeg, 44); + assert.equal(ptz.fovSource, undefined); + assert.equal(ptz.specFovMinDeg, 2.4); + assert.equal(ptz.specFovMaxDeg, 58.3); + assert.equal(ptz.spec.model, 'Q6135-LE'); + + // Fixed lens: the datasheet figure is the optical state. + assert.equal(fixed.fovDeg, 110); + assert.equal(fixed.fovSource, 'datasheet'); + + // Unknown hardware changes nothing. + assert.equal(unknown.fovDeg, 44); + assert.equal(unknown.spec, undefined); + assert.equal(unknown.specFovMaxDeg, undefined); }); test('specSummary renders the HUD line and degrades by omission', () => { diff --git a/src/layers/cctv/presentation.js b/src/layers/cctv/presentation.js index 9fa4a7cef..efa18e462 100644 --- a/src/layers/cctv/presentation.js +++ b/src/layers/cctv/presentation.js @@ -1,7 +1,7 @@ import { CCTV_AMBIENT_CARD_MAX } from '../../data/cctvLod.js'; import { ACTIVE_FRAME_REFRESH_MS, IDLE_FRAME_REFRESH_MS } from './policy.js'; import { headingHudToken, isHeadingEstimated } from './headingConfidence.js'; -import { specSummary } from './modelSpecs.js'; +import { specSummary, specFovCapabilityToken } from './modelSpecs.js'; export function createPresentation({ state: layerState, @@ -42,6 +42,9 @@ export function createPresentation({ headingHudToken(active.camera), // Identified hardware labels its FOV as datasheet-sourced (catalog.js). `FOV ${Math.round(active.camera.fovDeg)}°${active.camera.fovSource === 'datasheet' ? ' (DATASHEET)' : ''}`, + // A PTZ/varifocal datasheet range is what the lens can do, not its + // current zoom, so it is shown beside the estimate, never as it. + specFovCapabilityToken(active.camera), `COVERAGE ${area.toFixed(2)}km²`, overlapCount > 0 ? `OVERLAP ${overlapCount} cams` : 'ISOLATED VIEW', `PROJ ${layerState._showProjection ? 'MONITOR' : 'OFF'}`, @@ -94,6 +97,8 @@ export function createPresentation({ fovDeg: camera.fovDeg, // Identified-hardware enrichment passthrough (catalog.js / modelSpecs.js). fovSource: camera.fovSource || 'estimated', + specFovMinDeg: camera.specFovMinDeg ?? null, + specFovMaxDeg: camera.specFovMaxDeg ?? null, spec: camera.spec || null, rangeM: camera.rangeM, elevationM: camera.absoluteHeightM, From 5b2e160e54c8259847c34a4915cfd501f6c1bc45 Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Sun, 27 Sep 2026 12:15:13 +0000 Subject: [PATCH 18/95] docs(cctv): add the WSDOT pack to CHANGELOG, CURRENT-STATE and README CONTRIBUTING requires runtime-behavior changes to update docs/CURRENT-STATE.md and CHANGELOG.md in the same PR. The README and CURRENT-STATE table diffs are column realignment from the widened source cell. --- CHANGELOG.md | 10 ++++++++++ README.md | 42 +++++++++++++++++++++--------------------- docs/CURRENT-STATE.md | 42 +++++++++++++++++++++--------------------- 3 files changed, 52 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29dbedfaa..912e64ff4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +- Add WSDOT highway cameras (Washington State) as a keyless CCTV source pack: + WSDOT's official Travel Center camera layer, with frames registered only on + WSDOT's own image host (partner-hosted cameras such as Oregon DOT and City of + Seattle are skipped). Cameras with a single `CompassDirection`, cardinal or + diagonal, get a high-confidence heading; `BW` (Bothways) cameras use the + shared id-hash fallback at low confidence. By default the 250 cameras nearest + Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` sets the cap and + `CCTV_WSDOT_ENABLED=0` turns the pack off (bassem chagra, #644). + - CCTV cameras whose bearing is a guess now say so. Packs mark bearings derived from a hash of the camera id as `headingConfidence: 'low'`, but nothing read the flag, so roughly 70% of a default catalog rendered like surveyed facings. The HUD @@ -235,6 +244,7 @@ Add feed provenance to analyst/view answers and HUD context while retaining existing response fields and runner ownership (Matt Van Horn, #347). Analyst records for loaded satellites, datacenters and dams, with explicit bounded count/rank coverage (Matt Van Horn, #351). + - New Fire Perimeters layer (Events group): live NIFC WFIGS interagency wildfire incident perimeters as ground-clamped polygons with a containment-colored fire line, refreshed every 5 minutes from the public diff --git a/README.md b/README.md index 88690d52f..39daf9e2d 100644 --- a/README.md +++ b/README.md @@ -279,27 +279,27 @@ _Ask for radio near anywhere and the globe starts broadcasting — every station Nineteen layers and map sources. **Seventeen have a keyless path.** Some offer additional capabilities with a provider key. (🟢 no key · 🟡 free key · 🔴 metered.) -| Layer | What you get | Source | Auth | -| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------- | -| 🗺️ **Map Stack** | Esri satellite imagery, Google Photorealistic 3D, OSM, plus additional ion-hosted stacks | Esri / Google / Ion / OSM | 🟢 Esri satellite + OSM · 🟡 ion-hosted Google 3D + world terrain · 🔴 direct Google + place search | -| ✈️ **Live Flights** | 11,000+ live aircraft + route history | OpenSky + adsb.lol | 🟢 (🟡 optional for more polling credits) | -| 🎖️ **Military Flights** | ADS-B military traffic in amber | adsb.lol | 🟢 | -| 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | -| 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | -| 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | -| 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~3,600 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary. Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | -| 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | -| 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | -| 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | -| 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | -| 🧭 **Directions** | Click A and B on the globe for a street-following drive, walk or cycle route draped on the terrain with turn-by-turn steps — then FLY the camera along it. No key, no geocoder, no mic | OSRM on FOSSGIS servers (OpenStreetMap) | 🟢 | -| 🔥 **Active Fires** | Live NASA FIRMS detections, trailing 24h | NASA FIRMS | 🟡 | -| 🚀 **Space Missions** | Rolling 30-day launches with payload, stage, and recovery detail | Launch Library 2 | 🟢 (🟡 optional token raises the allowance) | -| 🎖️ **Mapped Installations** | Viewport-bounded military-site context from community mapping — incomplete by nature, and labeled that way | OpenStreetMap | 🟢 | -| 🌬️ **Wind** | Animated 10 m forecast wind with an optional speed, temperature or pressure color field and a reading at map center | NOAA GFS / ECMWF IFS | 🟢 | -| 🌦️ **Observed Weather** | Rain radar (contiguous US), Satellite clouds (GOES regional or global infrared, Clouds only or Full image) and Lightning density (Americas and Pacific) on one history timeline | NOAA nowCOAST | 🟢 | -| 🌀 **Cyclones** | Advisory positions, forecast tracks and uncertainty cones for the Atlantic and eastern/central North Pacific | NOAA NHC / CPHC | 🟢 | +| Layer | What you get | Source | Auth | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------- | +| 🗺️ **Map Stack** | Esri satellite imagery, Google Photorealistic 3D, OSM, plus additional ion-hosted stacks | Esri / Google / Ion / OSM | 🟢 Esri satellite + OSM · 🟡 ion-hosted Google 3D + world terrain · 🔴 direct Google + place search | +| ✈️ **Live Flights** | 11,000+ live aircraft + route history | OpenSky + adsb.lol | 🟢 (🟡 optional for more polling credits) | +| 🎖️ **Military Flights** | ADS-B military traffic in amber | adsb.lol | 🟢 | +| 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | +| 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | +| 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | +| 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | +| 📹 **CCTV Mesh** | ~3,850 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Washington (WSDOT). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | +| 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | +| 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | +| 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | +| 🧭 **Directions** | Click A and B on the globe for a street-following drive, walk or cycle route draped on the terrain with turn-by-turn steps — then FLY the camera along it. No key, no geocoder, no mic | OSRM on FOSSGIS servers (OpenStreetMap) | 🟢 | +| 🔥 **Active Fires** | Live NASA FIRMS detections, trailing 24h | NASA FIRMS | 🟡 | +| 🚀 **Space Missions** | Rolling 30-day launches with payload, stage, and recovery detail | Launch Library 2 | 🟢 (🟡 optional token raises the allowance) | +| 🎖️ **Mapped Installations** | Viewport-bounded military-site context from community mapping — incomplete by nature, and labeled that way | OpenStreetMap | 🟢 | +| 🌬️ **Wind** | Animated 10 m forecast wind with an optional speed, temperature or pressure color field and a reading at map center | NOAA GFS / ECMWF IFS | 🟢 | +| 🌦️ **Observed Weather** | Rain radar (contiguous US), Satellite clouds (GOES regional or global infrared, Clouds only or Full image) and Lightning density (Americas and Pacific) on one history timeline | NOAA nowCOAST | 🟢 | +| 🌀 **Cyclones** | Advisory positions, forecast tracks and uncertainty cones for the Atlantic and eastern/central North Pacific | NOAA NHC / CPHC | 🟢 | **Weather, keyless.** **Wind** animates 10 m forecast flow from NOAA GFS or ECMWF IFS, with an optional color field and a reading at map center. **Rain radar**, **Satellite clouds** and **Lightning density** are observations on one history timeline in the **WEATHER** panel (step back, **Play**, **Latest**); **Cyclone advisories** draw NHC and CPHC positions, forecast tracks and cones. On Google 3D Tiles the observed layers float above the city as translucent shells, sharpened around your view. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fdb42fc0c..edbd44565 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2747,27 +2747,27 @@ nearby place—always use station selection. Unqualified “turn on/start the ra requests use Play; a qualified Play-shaped tool call is normalized to Select so its criteria cannot be silently ignored. -| Layer | Source | File | Proxy | Update Interval | -| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------- | -| Live Flights ✈️ | OpenSky Network; bounded adsb.lol regional fallback | `src/data/flights.js` | `/api/opensky` (OAuth + fallback) | 30s | -| Military Flights 🎖️ | adsb.lol /v2/mil | `src/data/militaryFlights.js` | `/api/adsblol/mil` | 15s | -| Live AIS Vessels 🚢 | AISStream websocket | `src/data/aisLiveVessels.js` | `/api/ais-live` | 60s (+800ms visibility pass) | -| Mapped Installations ⌖ | OpenStreetMap mapped context; on-demand Google Maps Places supplement | `src/data/militaryInstallations.js` | `/api/military-installations`, `/api/google/text-search` | viewport-driven + user search; while unavailable, auto-retry 30 s → 240 s backoff | -| Earthquakes | USGS | `src/data/earthquakes.js` | — | 60s | -| Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | -| Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | -| Traffic | OSM Overpass (+ optional TomTom live flow) | `src/data/traffic.js` | `/api/overpass` + `/api/tomtom` | viewport-driven | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | -| Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | -| Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | -| Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | -| Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | -| Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | -| Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static | -| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | -| Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | -| Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | +| Layer | Source | File | Proxy | Update Interval | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------- | --------------------------------------------------------- | --------------------------------------------------------------------------------- | +| Live Flights ✈️ | OpenSky Network; bounded adsb.lol regional fallback | `src/data/flights.js` | `/api/opensky` (OAuth + fallback) | 30s | +| Military Flights 🎖️ | adsb.lol /v2/mil | `src/data/militaryFlights.js` | `/api/adsblol/mil` | 15s | +| Live AIS Vessels 🚢 | AISStream websocket | `src/data/aisLiveVessels.js` | `/api/ais-live` | 60s (+800ms visibility pass) | +| Mapped Installations ⌖ | OpenStreetMap mapped context; on-demand Google Maps Places supplement | `src/data/militaryInstallations.js` | `/api/military-installations`, `/api/google/text-search` | viewport-driven + user search; while unavailable, auto-retry 30 s → 240 s backoff | +| Earthquakes | USGS | `src/data/earthquakes.js` | — | 60s | +| Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | +| Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | +| Traffic | OSM Overpass (+ optional TomTom live flow) | `src/data/traffic.js` | `/api/overpass` + `/api/tomtom` | viewport-driven | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary + WSDOT (WA) Open Data + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | +| Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | +| Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | +| Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | +| Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | +| Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static | +| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | +| Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | +| Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | Fire Perimeters uses capped, timed server reads with stale-on-error caching and a per-client limit. Unchanged snapshots retain geometry; link checks cancel on disable or selection change, and the row legend shows reported containment. From c4f18169ebeac81272badaa9c833a4d68be98fd2 Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Sun, 27 Sep 2026 12:20:45 +0000 Subject: [PATCH 19/95] fix(cctv): raise the default catalog cap to 4,500 and enforce it over every pack The per-pack defaults already summed to 4,054 before WSDOT (4,304 with it), past the documented 4,000 cap, so a default install was trimming. The existing sum check in the DriveBC test listed packs by hand and missed Calgary and DelDOT. Replace it with a check in cctvCatalogCap.test.mjs that sums every exported DEFAULT_*_MAX_SOURCES, and drop the Calgary test that pinned the cap at exactly 4,000. --- .env.example | 2 +- CHANGELOG.md | 6 ++++- docs/CURRENT-STATE.md | 2 +- server/providers/cctv/constants.js | 7 +++-- src/data/cctvCalgary.test.mjs | 5 ---- src/data/cctvCatalogCap.test.mjs | 28 +++++++++++++++++--- src/data/cctvDriveBcSource.test.mjs | 40 +---------------------------- 7 files changed, 37 insertions(+), 53 deletions(-) diff --git a/.env.example b/.env.example index 45ae52099..5f153ca40 100644 --- a/.env.example +++ b/.env.example @@ -182,6 +182,6 @@ AISSTREAM_API_KEY= # CCTV_CALGARY_ROWS_URL= # override the Open Calgary catalog URL # CCTV_WSDOT_ENABLED=1 # set 0 to disable the WSDOT (Washington State) highway cameras # CCTV_WSDOT_MAX_SOURCES=250 # cap cameras loaded from the WSDOT catalog, nearest Seattle/Spokane first -# CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs +# CCTV_MAX_SOURCES=4500 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/CHANGELOG.md b/CHANGELOG.md index 912e64ff4..1b559c1e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,11 @@ diagonal, get a high-confidence heading; `BW` (Bothways) cameras use the shared id-hash fallback at low confidence. By default the 250 cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` sets the cap and - `CCTV_WSDOT_ENABLED=0` turns the pack off (bassem chagra, #644). + `CCTV_WSDOT_ENABLED=0` turns the pack off. The default catalog cap + (`CCTV_MAX_SOURCES`) rises from 4,000 to 4,500: the per-pack defaults already + summed to 4,054 before WSDOT, so a default install was being trimmed. A test + now sums every pack's default and fails if they exceed the cap + (bassem chagra, #644). - CCTV cameras whose bearing is a guess now say so. Packs mark bearings derived from a hash of the camera id as `headingConfidence: 'low'`, but nothing read the diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index edbd44565..892c85976 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3422,7 +3422,7 @@ silently demoting every later lookup for the session. kill switch `CCTV_FINTRAFFIC_ENABLED=0`) are the fourth pack: one keyless GeoJSON station list covering the whole country, where one _preset_ (a station's fixed view) is one camera — 806 GATHERING stations carry 2,256 in-collection presets, prioritized to 300 against seven anchors - on the main road spine. `CCTV_MAX_SOURCES` is a 4,000 catalog-wide ceiling shared round-robin + on the main road spine. `CCTV_MAX_SOURCES` is a 4,500 catalog-wide ceiling (4,000 until the WSDOT pack) shared round-robin across packs, so a lower global cap thins every region instead of starving the last pack. ~1,100 cameras total, all RAW PRIOR poses, stills-first. - **CCTV v3 UX — viewshed + calibration gizmo** (built 2026-07-05 and field diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 3ab8ce9cb..91e684be0 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -9,9 +9,12 @@ export const DEFAULT_AUSTIN_MAX_SOURCES = 250; * itself (nearest-to-anchor first); this bound only matters when the packs * together exceed it, and it is then filled round-robin across packs (see * cap.js) so no region is silently dropped. Sized above the sum of the - * default per-pack caps so a default install never trims. + * default per-pack caps so a default install never trims (enforced by + * cctvCatalogCap.test.mjs). The per-pack defaults reached 4,304 with WSDOT, + * so this is 4,500: room for one more small pack while staying under + * CCTV_MAX_SOURCES_CEILING, which also sizes the health map. */ -export const DEFAULT_CCTV_MAX_SOURCES = 4000; +export const DEFAULT_CCTV_MAX_SOURCES = 4500; /** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */ export const CCTV_MAX_SOURCES_CEILING = 5000; /** Reference point for Austin camera prioritization (Congress & 6th). */ diff --git a/src/data/cctvCalgary.test.mjs b/src/data/cctvCalgary.test.mjs index 1295c536b..4e7702059 100644 --- a/src/data/cctvCalgary.test.mjs +++ b/src/data/cctvCalgary.test.mjs @@ -12,7 +12,6 @@ import { CALGARY_IMAGE_ORIGIN, CALGARY_MAX_CATALOG_BYTES, DEFAULT_CALGARY_ROWS_URL, - DEFAULT_CCTV_MAX_SOURCES, } from '../../server/providers/cctv/constants.js'; import { CAMERA_CODE_MAX_CHARS } from '../../server/providers/cctv/normalize.js'; import { allocateSourceCap } from '../../server/providers/cctv/cap.js'; @@ -383,7 +382,3 @@ test('CCTV_CALGARY_ENABLED=0 keeps the lane from being loaded at all', async (t) Object.assign(process.env, saved); } }); - -test('the shipped catalog ceiling is not raised to make room for this pack', () => { - assert.equal(DEFAULT_CCTV_MAX_SOURCES, 4000); -}); diff --git a/src/data/cctvCatalogCap.test.mjs b/src/data/cctvCatalogCap.test.mjs index 6f1ff6c59..b0a603b4a 100644 --- a/src/data/cctvCatalogCap.test.mjs +++ b/src/data/cctvCatalogCap.test.mjs @@ -4,10 +4,9 @@ import { allocateSourceCap, resolveCatalogCap, } from '../../server/providers/cctv/cap.js'; -import { - CCTV_MAX_SOURCES_CEILING, - DEFAULT_CCTV_MAX_SOURCES, -} from '../../server/providers/cctv/constants.js'; +import * as constants from '../../server/providers/cctv/constants.js'; + +const { CCTV_MAX_SOURCES_CEILING, DEFAULT_CCTV_MAX_SOURCES } = constants; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; const pack = (name, count, prefix = name) => ({ @@ -135,3 +134,24 @@ test('the catalog applies the fair cap to configured packs end to end', async () Object.assign(process.env, saved); } }); + +test('default per-pack camera caps fit inside the default catalog cap', () => { + // Every exported DEFAULT__MAX_SOURCES counts, so a new pack cannot + // slip past this check by not being listed (Warendorf ships 1 curated + // camera and has no cap constant). + const packCaps = Object.entries(constants).filter( + ([name]) => + /^DEFAULT_[A-Z_]+_MAX_SOURCES$/.test(name) && + name !== 'DEFAULT_CCTV_MAX_SOURCES', + ); + assert.ok(packCaps.length >= 13, 'every pack cap constant is found'); + const total = packCaps.reduce((sum, [, cap]) => sum + cap, 0) + 1; + assert.ok( + total <= DEFAULT_CCTV_MAX_SOURCES, + `default packs (${total}) would be thinned by the catalog cap (${DEFAULT_CCTV_MAX_SOURCES})`, + ); + assert.ok( + DEFAULT_CCTV_MAX_SOURCES <= CCTV_MAX_SOURCES_CEILING, + 'the default cap must sit inside the CCTV_MAX_SOURCES ceiling', + ); +}); diff --git a/src/data/cctvDriveBcSource.test.mjs b/src/data/cctvDriveBcSource.test.mjs index 617bb1350..080b1edf0 100644 --- a/src/data/cctvDriveBcSource.test.mjs +++ b/src/data/cctvDriveBcSource.test.mjs @@ -4,21 +4,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; -import { - DEFAULT_AUSTIN_MAX_SOURCES, - DEFAULT_CALTRANS_MAX_SOURCES, - DEFAULT_CCTV_MAX_SOURCES, - CCTV_MAX_SOURCES_CEILING, - DEFAULT_DRIVEBC_MAX_SOURCES, - DEFAULT_TFL_MAX_SOURCES, - DEFAULT_ONTARIO_MAX_SOURCES, - DEFAULT_FINTRAFFIC_MAX_SOURCES, - DEFAULT_TXDOT_MAX_SOURCES, - DEFAULT_TALLINN_MAX_SOURCES, - DEFAULT_TARKTEE_MAX_SOURCES, - DEFAULT_NSW_MAX_SOURCES, - DRIVEBC_WEBCAMS_URL, -} from '../../server/providers/cctv/constants.js'; +import { DRIVEBC_WEBCAMS_URL } from '../../server/providers/cctv/constants.js'; import { driveBcImageCredit, loadDriveBcSourcesFromOpenData, @@ -221,30 +207,6 @@ test('CCTV catalog merges DriveBC cameras and CCTV_DRIVEBC_ENABLED=0 skips the r ); }); -test('default per-pack camera caps fit inside the default catalog cap', () => { - // Every default pack at its own cap (Warendorf ships 1 curated camera). - const packs = - DEFAULT_AUSTIN_MAX_SOURCES + - DEFAULT_CALTRANS_MAX_SOURCES + - DEFAULT_TFL_MAX_SOURCES + - DEFAULT_ONTARIO_MAX_SOURCES + - DEFAULT_FINTRAFFIC_MAX_SOURCES + - DEFAULT_DRIVEBC_MAX_SOURCES + - DEFAULT_TXDOT_MAX_SOURCES + - DEFAULT_TALLINN_MAX_SOURCES + - DEFAULT_TARKTEE_MAX_SOURCES + - DEFAULT_NSW_MAX_SOURCES + - 1; - assert.ok( - packs <= DEFAULT_CCTV_MAX_SOURCES, - `default packs (${packs}) would be thinned by the catalog cap (${DEFAULT_CCTV_MAX_SOURCES})`, - ); - assert.ok( - DEFAULT_CCTV_MAX_SOURCES <= CCTV_MAX_SOURCES_CEILING, - 'the default cap must sit inside the CCTV_MAX_SOURCES ceiling', - ); -}); - test('DriveBC credit keeps partner attribution and drops operational notes', () => { assert.equal( driveBcImageCredit( From 5dfe3f0525f1c26541829d9da699236df65634df Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Sun, 27 Sep 2026 12:23:23 +0000 Subject: [PATCH 20/95] docs(cctv): add the King County pack to CHANGELOG, CURRENT-STATE, README and .env.example CONTRIBUTING requires runtime-behavior changes to update docs/CURRENT-STATE.md and CHANGELOG.md in the same PR, and new env controls belong in .env.example with the other CCTV pack controls. The README and CURRENT-STATE table diffs are column realignment from the widened source cell. --- .env.example | 2 ++ CHANGELOG.md | 12 ++++++++++++ README.md | 42 +++++++++++++++++++++--------------------- docs/CURRENT-STATE.md | 42 +++++++++++++++++++++--------------------- 4 files changed, 56 insertions(+), 42 deletions(-) diff --git a/.env.example b/.env.example index e16e85a27..f1ff1898f 100644 --- a/.env.example +++ b/.env.example @@ -180,6 +180,8 @@ AISSTREAM_API_KEY= # CCTV_CALGARY_ENABLED=1 # set 0 to disable the Open Calgary (Alberta) cameras # CCTV_CALGARY_MAX_SOURCES=220 # cap cameras loaded from the Open Calgary catalog # CCTV_CALGARY_ROWS_URL= # override the Open Calgary catalog URL +# CCTV_KINGCOUNTY_ENABLED=1 # set 0 to disable the King County (Washington) road cameras +# CCTV_KINGCOUNTY_MAX_SOURCES=125 # cap cameras loaded from the King County catalog # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/CHANGELOG.md b/CHANGELOG.md index 29dbedfaa..810391db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +- Add King County road cameras (Washington State) as a keyless CCTV source + pack: one public King County DOT ArcGIS layer, fetched with an explicit field + list so its staff-name columns are never requested, and frames pinned to the + county's own image host and upgraded to HTTPS. WSDOT-owned rows are skipped, + and city-owned cameras the county hosts credit their owner. The layer + publishes no facing, so headings use the shared id-hash fallback at low + confidence. It does publish each camera's hardware, which is served on + `/api/cctv/sources` as `model` and `manufacturer`. `CCTV_KINGCOUNTY_MAX_SOURCES` + sets the cap and `CCTV_KINGCOUNTY_ENABLED=0` turns the pack off + (bassem chagra, #645). + - CCTV cameras whose bearing is a guess now say so. Packs mark bearings derived from a hash of the camera id as `headingConfidence: 'low'`, but nothing read the flag, so roughly 70% of a default catalog rendered like surveyed facings. The HUD @@ -235,6 +246,7 @@ Add feed provenance to analyst/view answers and HUD context while retaining existing response fields and runner ownership (Matt Van Horn, #347). Analyst records for loaded satellites, datacenters and dams, with explicit bounded count/rank coverage (Matt Van Horn, #351). + - New Fire Perimeters layer (Events group): live NIFC WFIGS interagency wildfire incident perimeters as ground-clamped polygons with a containment-colored fire line, refreshed every 5 minutes from the public diff --git a/README.md b/README.md index 88690d52f..50a86253d 100644 --- a/README.md +++ b/README.md @@ -279,27 +279,27 @@ _Ask for radio near anywhere and the globe starts broadcasting — every station Nineteen layers and map sources. **Seventeen have a keyless path.** Some offer additional capabilities with a provider key. (🟢 no key · 🟡 free key · 🔴 metered.) -| Layer | What you get | Source | Auth | -| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------- | -| 🗺️ **Map Stack** | Esri satellite imagery, Google Photorealistic 3D, OSM, plus additional ion-hosted stacks | Esri / Google / Ion / OSM | 🟢 Esri satellite + OSM · 🟡 ion-hosted Google 3D + world terrain · 🔴 direct Google + place search | -| ✈️ **Live Flights** | 11,000+ live aircraft + route history | OpenSky + adsb.lol | 🟢 (🟡 optional for more polling credits) | -| 🎖️ **Military Flights** | ADS-B military traffic in amber | adsb.lol | 🟢 | -| 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | -| 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | -| 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | -| 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~3,600 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary. Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | -| 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | -| 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | -| 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | -| 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | -| 🧭 **Directions** | Click A and B on the globe for a street-following drive, walk or cycle route draped on the terrain with turn-by-turn steps — then FLY the camera along it. No key, no geocoder, no mic | OSRM on FOSSGIS servers (OpenStreetMap) | 🟢 | -| 🔥 **Active Fires** | Live NASA FIRMS detections, trailing 24h | NASA FIRMS | 🟡 | -| 🚀 **Space Missions** | Rolling 30-day launches with payload, stage, and recovery detail | Launch Library 2 | 🟢 (🟡 optional token raises the allowance) | -| 🎖️ **Mapped Installations** | Viewport-bounded military-site context from community mapping — incomplete by nature, and labeled that way | OpenStreetMap | 🟢 | -| 🌬️ **Wind** | Animated 10 m forecast wind with an optional speed, temperature or pressure color field and a reading at map center | NOAA GFS / ECMWF IFS | 🟢 | -| 🌦️ **Observed Weather** | Rain radar (contiguous US), Satellite clouds (GOES regional or global infrared, Clouds only or Full image) and Lightning density (Americas and Pacific) on one history timeline | NOAA nowCOAST | 🟢 | -| 🌀 **Cyclones** | Advisory positions, forecast tracks and uncertainty cones for the Atlantic and eastern/central North Pacific | NOAA NHC / CPHC | 🟢 | +| Layer | What you get | Source | Auth | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------- | +| 🗺️ **Map Stack** | Esri satellite imagery, Google Photorealistic 3D, OSM, plus additional ion-hosted stacks | Esri / Google / Ion / OSM | 🟢 Esri satellite + OSM · 🟡 ion-hosted Google 3D + world terrain · 🔴 direct Google + place search | +| ✈️ **Live Flights** | 11,000+ live aircraft + route history | OpenSky + adsb.lol | 🟢 (🟡 optional for more polling credits) | +| 🎖️ **Military Flights** | ADS-B military traffic in amber | adsb.lol | 🟢 | +| 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | +| 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | +| 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | +| 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | +| 📹 **CCTV Mesh** | ~3,600 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · King County (WA). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | +| 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | +| 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | +| 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | +| 🧭 **Directions** | Click A and B on the globe for a street-following drive, walk or cycle route draped on the terrain with turn-by-turn steps — then FLY the camera along it. No key, no geocoder, no mic | OSRM on FOSSGIS servers (OpenStreetMap) | 🟢 | +| 🔥 **Active Fires** | Live NASA FIRMS detections, trailing 24h | NASA FIRMS | 🟡 | +| 🚀 **Space Missions** | Rolling 30-day launches with payload, stage, and recovery detail | Launch Library 2 | 🟢 (🟡 optional token raises the allowance) | +| 🎖️ **Mapped Installations** | Viewport-bounded military-site context from community mapping — incomplete by nature, and labeled that way | OpenStreetMap | 🟢 | +| 🌬️ **Wind** | Animated 10 m forecast wind with an optional speed, temperature or pressure color field and a reading at map center | NOAA GFS / ECMWF IFS | 🟢 | +| 🌦️ **Observed Weather** | Rain radar (contiguous US), Satellite clouds (GOES regional or global infrared, Clouds only or Full image) and Lightning density (Americas and Pacific) on one history timeline | NOAA nowCOAST | 🟢 | +| 🌀 **Cyclones** | Advisory positions, forecast tracks and uncertainty cones for the Atlantic and eastern/central North Pacific | NOAA NHC / CPHC | 🟢 | **Weather, keyless.** **Wind** animates 10 m forecast flow from NOAA GFS or ECMWF IFS, with an optional color field and a reading at map center. **Rain radar**, **Satellite clouds** and **Lightning density** are observations on one history timeline in the **WEATHER** panel (step back, **Play**, **Latest**); **Cyclone advisories** draw NHC and CPHC positions, forecast tracks and cones. On Google 3D Tiles the observed layers float above the city as translucent shells, sharpened around your view. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fdb42fc0c..2946a26e8 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2747,27 +2747,27 @@ nearby place—always use station selection. Unqualified “turn on/start the ra requests use Play; a qualified Play-shaped tool call is normalized to Select so its criteria cannot be silently ignored. -| Layer | Source | File | Proxy | Update Interval | -| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | -------------------------------------------------------- | --------------------------------------------------------------------------------- | -| Live Flights ✈️ | OpenSky Network; bounded adsb.lol regional fallback | `src/data/flights.js` | `/api/opensky` (OAuth + fallback) | 30s | -| Military Flights 🎖️ | adsb.lol /v2/mil | `src/data/militaryFlights.js` | `/api/adsblol/mil` | 15s | -| Live AIS Vessels 🚢 | AISStream websocket | `src/data/aisLiveVessels.js` | `/api/ais-live` | 60s (+800ms visibility pass) | -| Mapped Installations ⌖ | OpenStreetMap mapped context; on-demand Google Maps Places supplement | `src/data/militaryInstallations.js` | `/api/military-installations`, `/api/google/text-search` | viewport-driven + user search; while unavailable, auto-retry 30 s → 240 s backoff | -| Earthquakes | USGS | `src/data/earthquakes.js` | — | 60s | -| Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | -| Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | -| Traffic | OSM Overpass (+ optional TomTom live flow) | `src/data/traffic.js` | `/api/overpass` + `/api/tomtom` | viewport-driven | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | -| Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | -| Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | -| Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | -| Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | -| Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | -| Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static | -| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | -| Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | -| Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | +| Layer | Source | File | Proxy | Update Interval | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------- | --------------------------------------------------------- | --------------------------------------------------------------------------------- | +| Live Flights ✈️ | OpenSky Network; bounded adsb.lol regional fallback | `src/data/flights.js` | `/api/opensky` (OAuth + fallback) | 30s | +| Military Flights 🎖️ | adsb.lol /v2/mil | `src/data/militaryFlights.js` | `/api/adsblol/mil` | 15s | +| Live AIS Vessels 🚢 | AISStream websocket | `src/data/aisLiveVessels.js` | `/api/ais-live` | 60s (+800ms visibility pass) | +| Mapped Installations ⌖ | OpenStreetMap mapped context; on-demand Google Maps Places supplement | `src/data/militaryInstallations.js` | `/api/military-installations`, `/api/google/text-search` | viewport-driven + user search; while unavailable, auto-retry 30 s → 240 s backoff | +| Earthquakes | USGS | `src/data/earthquakes.js` | — | 60s | +| Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | +| Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | +| Traffic | OSM Overpass (+ optional TomTom live flow) | `src/data/traffic.js` | `/api/overpass` + `/api/tomtom` | viewport-driven | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary + King County (WA) Open Data + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | +| Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | +| Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | +| Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | +| Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | +| Submarine Cables ◠ | TeleGeography public map (bundled) | `src/data/telegeographySubmarineCables.js` | — | static | +| FIRMS Active Fires ▲ | NASA FIRMS live (VIIRS ×3 NRT + MODIS NRT, trailing 24h) | `src/data/firmsHeatmap.js` | `/api/firms` (`FIRMS_MAP_KEY`) | 10 min (proxy TTL 30 min) | +| Wind 🌬 | NOAA GFS 10 m wind (keyless, 0.25°→1° grid; animated particles) | `src/data/wind.js` | `/api/wind` | 1 h (forecast cycle) | +| Fire Perimeters 🔥 | NIFC WFIGS current interagency perimeters (keyless, paged past the 2000-record cap); InciWeb catalog + incident page origin and update time checks for verified incident-page links | `src/layers/perimeters/` via `src/app/layers/perimeters.js` | `/api/fire-perimeters` + `/api/fire-perimeters/inciweb/*` | 5 min (server caches: catalog 1 h; publication 30 min) | Fire Perimeters uses capped, timed server reads with stale-on-error caching and a per-client limit. Unchanged snapshots retain geometry; link checks cancel on disable or selection change, and the row legend shows reported containment. From cce159c93d6c7023e65589c1024bf83de45a1499 Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Sun, 27 Sep 2026 12:24:13 +0000 Subject: [PATCH 21/95] docs(cctv): document identified-hardware specs and datasheet FOV in CHANGELOG and CURRENT-STATE CONTRIBUTING requires runtime-behavior changes to update docs/CURRENT-STATE.md and CHANGELOG.md in the same PR. --- CHANGELOG.md | 9 +++++++++ docs/CURRENT-STATE.md | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29dbedfaa..21a8cb743 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +- CCTV cameras with identified hardware show the model's datasheet specs. When a + source names a camera model found in the vendored CCTV Camera Database records + (CC0), the HUD meta line adds its resolution, night vision and PTZ capability. + A fixed single-lens model's horizontal FOV replaces the estimate + (`FOV n° (DATASHEET)`); a PTZ or varifocal range is what the lens can do, not + its current zoom, so those cameras keep the estimate and show + `SPEC FOV min–max° (CAPABILITY)` beside it (bassem chagra, #452). + - CCTV cameras whose bearing is a guess now say so. Packs mark bearings derived from a hash of the camera id as `headingConfidence: 'low'`, but nothing read the flag, so roughly 70% of a default catalog rendered like surveyed facings. The HUD @@ -235,6 +243,7 @@ Add feed provenance to analyst/view answers and HUD context while retaining existing response fields and runner ownership (Matt Van Horn, #347). Analyst records for loaded satellites, datacenters and dams, with explicit bounded count/rank coverage (Matt Van Horn, #351). + - New Fire Perimeters layer (Events group): live NIFC WFIGS interagency wildfire incident perimeters as ground-clamped polygons with a containment-colored fire line, refreshed every 5 minutes from the public diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fdb42fc0c..5e1ccd447 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3390,6 +3390,13 @@ silently demoting every later lookup for the session. (`calSource: 'manual'`) or curated pose (`poseSource: 'curated'`) is never presented as estimated, matching the CAL badge. Public camera state carries `headingConfidence` and `headingEstimated` (`src/layers/cctv/headingConfidence.js`, #639). +- **CCTV identified hardware:** a camera whose source or seed names a hardware `model` that + resolves in `src/layers/cctv/modelSpecs.js` (vendored CC0 records from the CCTV Camera + Database) gets the model's specs appended to the HUD meta line. Only fixed single-lens + hardware replaces the estimated FOV, shown as `FOV n° (DATASHEET)`. PTZ, varifocal and + multi-sensor datasheets publish a capability range, not the lens's current zoom, so those + cameras keep their estimate and show `SPEC FOV min–max° (CAPABILITY)` beside it. Public + camera state carries `fovSource`, `specFovMinDeg`, `specFovMaxDeg` and `spec` (#452). - **FIRMS**: no ground clamping (zero 3D-tiles height sampling), ≤18 screen-decluttered ambient labels, click-to-inspect detail card, 2.5k/3k sprite budgets viewport-clipped by FRP. - **CCTV v2 foundation:** a pitched frustum wireframe (4 corner rays + far-cap rectangle) with a monitor plane at the frustum's From 7c7563d91b63be493312e3b89e088510274bbe97 Mon Sep 17 00:00:00 2001 From: daikaginza Date: Mon, 21 Sep 2026 10:48:00 +0700 Subject: [PATCH 22/95] fix(gbfs): rate-limit /api/gbfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy already restricted WHERE it would go — https, an allowlisted host, a station_information/station_status path, no redirects, a 12-second deadline and a 5 MB body cap — but nothing restricted HOW OFTEN, so the dev server would relay an unbounded stream of requests at a third-party bikeshare operator on any caller's behalf, one outbound fetch and one timeout timer each. Admit 120 requests/minute per client address, global backstop 3x, refused with 429 and `Retry-After` before the target is even parsed. The ceiling comes from the layer's own demand: `src/layers/bikeshare/registry.js` catalogues 15 systems, station information is cached per city after its first fetch, and station status refreshes once per city every STATUS_POLL_MS (60s) with in-flight requests deduplicated — about 15 requests a minute with everything in range, so 120 is eight times a maximal session. The method, host, path, redirect, deadline and size guards are unchanged. Refs #264 --- CHANGELOG.md | 9 ++ docs/CURRENT-STATE.md | 9 ++ scripts/format-scope.json | 3 +- scripts/package-boundaries.json | 2 + server/providers/gbfs.js | 34 +++++- src/data/gbfsProxyRateLimit.test.mjs | 152 +++++++++++++++++++++++++++ 6 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 src/data/gbfsProxyRateLimit.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 29dbedfaa..f1c6ffb53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +- Bound how often the GBFS proxy will relay. `/api/gbfs` already restricted + where it could go — https, an allowlisted host, a station feed path, no + redirects, a 12-second deadline and a 5 MB body cap — but nothing restricted + how often, so the dev server would make unlimited requests at a third-party + bikeshare operator on any caller's behalf. The route now admits 120 + requests/minute per client address, refusing the rest with 429 and + `Retry-After` before any outbound fetch. The bikeshare layer needs about 15 a + minute with every catalogued system in range, so ordinary use is unchanged. + - CCTV cameras whose bearing is a guess now say so. Packs mark bearings derived from a hash of the camera id as `headingConfidence: 'low'`, but nothing read the flag, so roughly 70% of a default catalog rendered like surveyed facings. The HUD diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fdb42fc0c..f55945224 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -827,6 +827,15 @@ feed address is read from the path, and a query-string form is refused with a 40 streaming. The 12-second deadline includes reading the body, and rejected or stalled downloads are cancelled. Development and preview use the same handler. +The route also bounds request frequency: 120 requests/minute per client +address, with a global backstop of three times that. A refused request answers +429 with `Retry-After` and `Cache-Control: no-store` and performs no outbound +fetch. The bikeshare layer's own demand is about 15 requests a minute with +every catalogued system in range, so an ordinary session never meets the limit; +the method, host and path guards run unchanged. + +Updated: September 21, 2026 + ## Remaining local service modules Overpass query validation, geometry simplification, disk caching and upstream diff --git a/scripts/format-scope.json b/scripts/format-scope.json index eee546dd5..e7b7a92d0 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -839,5 +839,6 @@ "src/ui/imageryBoxTool.test.mjs", "src/ui/recentImagery.test.mjs", "src/layers/recentImagery/testDoubles.mjs", - "scripts/qa-recent-imagery.mjs" + "scripts/qa-recent-imagery.mjs", + "src/data/gbfsProxyRateLimit.test.mjs" ] diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 189192f10..1f1db490d 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -138,6 +138,8 @@ "modules": [ "server/providers/gbfs.js", "server/providers/common/http.js", + "server/providers/common/rate-limit.js", + "src/sources/rateLimit.js", "src/data/gbfsSource.js", "src/sources/httpBody.js" ], diff --git a/server/providers/gbfs.js b/server/providers/gbfs.js index 9741bb4e6..001e94a7d 100644 --- a/server/providers/gbfs.js +++ b/server/providers/gbfs.js @@ -1,4 +1,5 @@ import { readResponseTextCapped } from './common/http.js'; +import { makeRateLimiter, clientKey } from './common/rate-limit.js'; import { isAllowedGbfsHost, isAllowedGbfsPath, @@ -13,6 +14,19 @@ const GBFS_PROXY_TIMEOUT_MS = 12000; export const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB +/** + * Requests/minute/IP admitted by the proxy. + * + * The bikeshare layer's own demand is small and known: `src/layers/bikeshare/ + * registry.js` catalogues 15 systems, station information is cached per city + * after its first fetch, and station status refreshes once per city every + * `STATUS_POLL_MS` (60 s) with in-flight requests deduplicated. Even with every + * catalogued system in camera range at once that is about 15 requests a minute, + * so 120 is eight times a maximal real session while still capping a caller + * using this proxy as an open relay against a third-party GBFS host. + */ +export const GBFS_RATE_PER_MIN = 120; + function gbfsRedirectHost(location, requestUrl) { if (!location) return ''; try { @@ -113,7 +127,12 @@ export async function fetchGbfsUpstream( * * @returns {import('vite').Plugin} */ -export function gbfsProxy() { +export function gbfsProxy({ ratePerMinute = GBFS_RATE_PER_MIN } = {}) { + const allow = makeRateLimiter({ + windowMs: 60_000, + max: ratePerMinute, + globalMax: ratePerMinute * 3, + }); const installMiddleware = (server) => { server.middlewares.use('/api/gbfs', async (req, res) => { try { @@ -126,6 +145,19 @@ export function gbfsProxy() { return; } + // Frequency, checked before the target is even parsed: the allowlist + // already says WHERE this proxy may go, nothing said HOW OFTEN. A + // refused request performs no outbound fetch and starts no timer. + if (!allow(clientKey(req))) { + res.writeHead(429, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + 'Retry-After': '10', + }); + res.end(JSON.stringify({ error: 'Rate limit exceeded' })); + return; + } + const url = new URL(req.url || '/', 'http://localhost'); const encodedTarget = url.pathname.replace(/^\/+/, ''); if (!encodedTarget) { diff --git a/src/data/gbfsProxyRateLimit.test.mjs b/src/data/gbfsProxyRateLimit.test.mjs new file mode 100644 index 000000000..72ee38152 --- /dev/null +++ b/src/data/gbfsProxyRateLimit.test.mjs @@ -0,0 +1,152 @@ +// GBFS PROXY — the frequency the allowlist never bounded. +// +// `/api/gbfs` already says WHERE it may go: an https allowlisted host, a +// station_information/station_status path, a 12 s timeout, a 5 MB body cap and +// no redirects. Nothing said HOW OFTEN, so the dev server would relay an +// unbounded stream of requests at a third-party bikeshare operator on any +// caller's behalf — one outbound fetch and one timeout timer each. +// +// These cases drive the real mounted middleware. `fetch` is stubbed per test +// and restored afterwards, so a case that reaches the network fails loudly, +// and the stub's call count is what proves a refused request costs nothing. +// +// Run with: npm test +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { gbfsProxy, GBFS_RATE_PER_MIN } from '../../server/providers/gbfs.js'; + +const STATUS_URL = 'https://gbfs.lyft.com/gbfs/2.3/bay/en/station_status.json'; +const STATION_BODY = '{"data":{"stations":[]}}'; + +/** Mount the proxy the way the dev server does; returns its route handler. */ +function mount(options) { + const routes = new Map(); + gbfsProxy(options).configureServer({ + middlewares: { + use(route, handler) { + routes.set(route, handler); + }, + }, + }); + return routes.get('/api/gbfs'); +} + +function request(handler, { target = STATUS_URL, remoteAddress, method } = {}) { + return new Promise((resolve, reject) => { + const req = { + method: method || 'GET', + // connect strips the mount prefix before the handler sees the URL. + url: '/' + encodeURIComponent(target), + headers: {}, + socket: remoteAddress ? { remoteAddress } : undefined, + }; + let status = 200; + let headers = {}; + const res = { + writeHead(code, sent) { + status = code; + headers = sent || {}; + return res; + }, + end(body = '') { + let parsed = null; + try { + parsed = body ? JSON.parse(String(body)) : null; + } catch { + parsed = String(body); + } + resolve({ status, headers, body: parsed }); + }, + }; + Promise.resolve(handler(req, res)).catch(reject); + }); +} + +/** Answer every allowlisted upstream, counting the calls that got through. */ +function stubUpstream(t) { + const asked = []; + const original = globalThis.fetch; + globalThis.fetch = async (url) => { + asked.push(String(url)); + return new Response(STATION_BODY, { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }; + t.after(() => { + globalThis.fetch = original; + }); + return asked; +} + +test('a maximal real session is never throttled', async (t) => { + const asked = stubUpstream(t); + const handler = mount(); + + // The bikeshare layer catalogues 15 systems and refreshes each one's status + // once every 60 s, so a minute of the busiest possible session is ~15 + // requests. Run eight times that and expect every one of them served. + let served = 0; + for (let i = 0; i < 120; i += 1) { + const res = await request(handler); + if (res.status === 200) served += 1; + } + assert.equal(served, 120, 'the shipped ceiling leaves a real session alone'); + assert.equal(asked.length, 120); + assert.equal(GBFS_RATE_PER_MIN, 120, 'the ceiling this case relies on'); +}); + +test('an unbounded relay is refused, and refusal costs no outbound fetch', async (t) => { + const asked = stubUpstream(t); + const handler = mount({ ratePerMinute: 5 }); + + const responses = []; + for (let i = 0; i < 12; i += 1) responses.push(await request(handler)); + + const served = responses.filter((r) => r.status === 200); + const refused = responses.filter((r) => r.status === 429); + assert.equal(served.length, 5, 'exactly the admitted budget is served'); + assert.equal(refused.length, 7); + assert.equal(refused[0].body.error, 'Rate limit exceeded'); + assert.equal(refused[0].headers['Retry-After'], '10'); + assert.equal(refused[0].headers['Cache-Control'], 'no-store'); + assert.equal( + asked.length, + 5, + 'a refused request never reaches the GBFS operator', + ); +}); + +test('a different client keeps its own budget', async (t) => { + stubUpstream(t); + const handler = mount({ ratePerMinute: 3 }); + + for (let i = 0; i < 6; i += 1) + await request(handler, { remoteAddress: '10.0.0.1' }); + const noisy = await request(handler, { remoteAddress: '10.0.0.1' }); + assert.equal(noisy.status, 429, 'the noisy client spent its own budget'); + + const quiet = await request(handler, { remoteAddress: '10.0.0.2' }); + assert.equal(quiet.status, 200, 'a quiet client is not collateral'); +}); + +test('the limiter does not displace the existing guards', async (t) => { + const asked = stubUpstream(t); + const handler = mount({ ratePerMinute: 10 }); + + const wrongMethod = await request(handler, { method: 'POST' }); + assert.equal(wrongMethod.status, 405); + + const offAllowlist = await request(handler, { + target: 'https://example.com/gbfs/station_status.json', + }); + assert.equal(offAllowlist.status, 403); + assert.equal(offAllowlist.body.error, 'GBFS host not allowed'); + + const wrongPath = await request(handler, { + target: 'https://gbfs.lyft.com/gbfs/2.3/bay/en/system_regions.json', + }); + assert.equal(wrongPath.status, 400); + + assert.equal(asked.length, 0, 'none of those reached upstream'); +}); From 9c15f093a1705b52807d44ab02a57f40249ee977 Mon Sep 17 00:00:00 2001 From: Bilawal Sidhu Date: Wed, 23 Sep 2026 21:54:43 -0500 Subject: [PATCH 23/95] fix(cyclones): accept advisories NHC publishes before their nominal time NHC stamps an advisory with its nominal time and writes it about half an hour early, so the 5-minute future limit rejected the whole feed each advisory cycle and the layer showed no storms. Allow 90 minutes ahead. --- CHANGELOG.md | 5 +++++ server/providers/cyclones.js | 5 ++++- src/data/cyclonesProxy.test.mjs | 9 +++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58103d04d..431d282f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +- Cyclone advisories no longer go dark while NHC publishes an advisory ahead + of its nominal time: advisory and position times up to 90 minutes ahead of + the server clock are accepted (previously 5 minutes), so the whole feed is + not rejected for about half an hour each advisory cycle. + - Public Overpass instances are no longer used by default. Street Traffic roads come from TomTom flow tiles, OpenFreeMap vector tiles, or both, chosen on the layer row (TomTom / OSM / Hybrid) or with `?trafficRoads=`. With a diff --git a/server/providers/cyclones.js b/server/providers/cyclones.js index fede1aade..d580357ce 100644 --- a/server/providers/cyclones.js +++ b/server/providers/cyclones.js @@ -41,6 +41,9 @@ function number(value, min, max) { const n = Number(value); return Number.isFinite(n) && n >= min && n <= max ? n : null; } +// NHC stamps an advisory with its nominal time (e.g. 03:00Z) and publishes it +// up to about half an hour before; allow that, but not a clock far ahead. +const EARLY_ADVISORY_MS = 90 * 60_000; function iso(value, now) { if ( typeof value !== 'string' || @@ -50,7 +53,7 @@ function iso(value, now) { const n = Date.parse(value); if ( !Number.isFinite(n) || - n > now + 5 * 60_000 || + n > now + EARLY_ADVISORY_MS || now - n > 12 * HOUR || new Date(n).toISOString().replace('.000Z', 'Z') !== value.replace('.000Z', 'Z') diff --git a/src/data/cyclonesProxy.test.mjs b/src/data/cyclonesProxy.test.mjs index 04729d33a..31433b326 100644 --- a/src/data/cyclonesProxy.test.mjs +++ b/src/data/cyclonesProxy.test.mjs @@ -162,6 +162,15 @@ test('malformed, duplicate, oversized, future and expired status is rejected rat assert.deepEqual(parseCycloneStatus({ activeStorms: [] }, NOW), []); }); +test('advisories published before their nominal time are accepted, far-future ones are not', () => { + // NHC wrote the 03:00Z advisory at 02:34Z. + const early = Date.UTC(2026, 8, 16, 2, 34); + assert.equal(parseCycloneStatus(status(), early)[0].issuedAt, TIME); + assert.throws(() => + parseCycloneStatus(status(), Date.UTC(2026, 8, 16, 1, 25)), + ); +}); + test('coherent geometry is retained with holes/dateline while older advisory geometry is omitted', () => { const storms = parseCycloneStatus(status(), NOW); const parts = collections(); From 03929f1a009e1a5a590dd790bcb2e38c4725adae Mon Sep 17 00:00:00 2001 From: bassem chagra Date: Thu, 1 Oct 2026 12:34:07 +0200 Subject: [PATCH 24/95] docs(cctv): refresh the CCTV Camera Database size in the model-specs module The dataset has grown from 12,000+ to 28,000+ camera models (v2.26.0, 75% with a field of view). The 17 vendored records are unchanged in v2.26.0. --- src/layers/cctv/modelSpecs.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layers/cctv/modelSpecs.js b/src/layers/cctv/modelSpecs.js index a9d12e23d..cee3e5044 100644 --- a/src/layers/cctv/modelSpecs.js +++ b/src/layers/cctv/modelSpecs.js @@ -14,8 +14,8 @@ * * Spec records are vendored verbatim from the CCTV Camera Database * (https://github.com/ch-bas/cctv-camera-database, CC0-1.0) — an open dataset - * of 12,000+ camera models transcribed from manufacturer datasheets (84% with - * FOV). This module vendors the records that the currently integrated public + * of 28,000+ camera models transcribed from manufacturer datasheets (75% with + * FOV, as of v2.26.0). This module vendors the records that the currently integrated public * catalogs publish hardware for — King County WA (Cohu, Bosch AUTODOME, Axis * Q6135-LE), plus the Bosch AUTODOME and Axis PTZ/multisensor models that * appear in Sarasota County FL and Sioux Falls SD open data; packs that From 8887ca47ebea3ca593a5c8d1956652f1739c8b8a Mon Sep 17 00:00:00 2001 From: rileygramlich Date: Wed, 23 Sep 2026 22:40:03 -0600 Subject: [PATCH 25/95] feat(air-quality): add the Air quality layer to the Weather group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reworks #568 to the shape asked for in review. Generalized from "AQHI" to a provider-backed Air quality layer, with ECCC's AQHI as the first registered provider. Readings are carried as { provider, scale, value, band } rather than a bare number, because air-quality indices are not interchangeable: AQHI runs 1-10+ on health risk, the US AQI runs 0-500 on concentration breakpoints, and Europe's EAQI is a five-step category. Naming the index a value belongs to is what lets another network be added without inventing a shared numeric meaning. Coverage is stated as "Canada only" on the provider, the layer row and the card. Adds the WEATHER panel readout (readout: true) carrying the band legend and the worst readings currently in view, with getRowControls/setRowControlsListener following the wind and cyclone cards. An unresolvable view rectangle degrades to the worst readings overall rather than to a blank card, and a rectangle crossing the antimeridian is handled. update() now takes the earthquakes layer's abort-and-guard shape: it aborts any in-flight request, passes the signal through to the source, and refuses to touch the scene when the request was superseded, aborted, or the layer was disabled while it was outstanding. disable() and destroy() abort too. Tests cover the late-response, superseded-refresh and disabled-update paths. Rebased on main: the source is registered in src/sources/reference.js, and the allocation profile was re-measured against the current worker — median 3,900 B/frame, max 4,114, 85.7 B/candidate, inside the shared 154 ceiling, budgeted at 5,200 for 26% headroom. The earlier figures no longer held. Freshness is now judged on the observation time rather than on when the last response arrived. lastUpdate only says the feed answered; a station that keeps returning a reading from four hours ago would otherwise be narrated as live by the HUD and the voice provenance envelope. getStats() reports observedAt and stale, so layerFeedState() calls the layer stale past two missed hourly publications, and degraded when a refresh fails over readings that are still good. The share-link token is 0, which is what layer-token:next allocates. Taking it moves the ledger fixtures that had hard-coded 0 as the free digit and 3 as the one after; those shift to 3 and 4. The pinned equality between the reservation ledger and LEGACY_LAYER_STATE_TOKENS also had to become "the closed legacy set plus air-quality", since that set is documented as never growing. Happy to re-pin this differently if you would rather the fixtures derive the provisional digits from nextLayerStateToken() so the next layer does not move them again. The layer and its source are registered as ./layers/air-quality and ./layers/air-quality/source, each with its own boundary group, following the perimeters pair. check:boundaries requires the package exports and the group exports to be the same set, so a layer that ships a source needs both. --- CHANGELOG.md | 10 + DATA_SOURCES.md | 9 + README.md | 1 + docs/CURRENT-STATE.md | 1 + package.json | 2 + scripts/format-scope.json | 6 + scripts/package-boundaries.json | 33 +- scripts/qa-air-quality.mjs | 171 ++++++++ src/app/constructCatalog.js | 3 + src/app/constructCatalog.test.mjs | 2 +- src/app/layers/airQuality.js | 6 + src/data/dataCredits.js | 7 + src/data/layerState.js | 5 + src/data/layerState.test.mjs | 12 +- src/data/layerStateTokenLedger.test.mjs | 63 +-- src/data/layerStateTokenReservations.json | 1 + src/layers/airQuality/index.js | 308 +++++++++++++++ src/layers/airQuality/model.js | 374 ++++++++++++++++++ src/layers/airQuality/model.test.mjs | 250 ++++++++++++ src/layers/airQuality/ownership.test.mjs | 326 +++++++++++++++ src/layers/airQuality/source.js | 78 ++++ src/overlays/worldOverlayAllocation.test.mjs | 19 + .../worldOverlayAllocation.worker.mjs | 78 +++- src/sources/reference.js | 2 + src/sources/reference.test.mjs | 3 + src/ui/layerPanel.js | 1 + 26 files changed, 1730 insertions(+), 41 deletions(-) create mode 100644 scripts/qa-air-quality.mjs create mode 100644 src/app/layers/airQuality.js create mode 100644 src/layers/airQuality/index.js create mode 100644 src/layers/airQuality/model.js create mode 100644 src/layers/airQuality/model.test.mjs create mode 100644 src/layers/airQuality/ownership.test.mjs create mode 100644 src/layers/airQuality/source.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 58103d04d..10a525546 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -600,6 +600,16 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md ## [Unreleased] +- Add a keyless Air quality layer to the Weather group, with Environment and + Climate Change Canada's AQHI as the first registered provider. Readings carry + `{ provider, scale, value, band }` so another network can be added without + inventing a shared numeric meaning across indices that do not share one; the + row and readout both state that coverage is Canada only. The WEATHER card + shows the band legend and the highest readings currently in view. AQHI is + reported as an integer from 1 with an open-ended "10+", so readings round to + that published form rather than the decimal precision the API returns. + Readings older than six hours are dropped, and the layer reports `stale` + through `getStats()` so feed provenance cannot describe an aged feed as live. - Add ECMWF IFS model selection to Wind (#464, thanks @beneduzi), with model-scoped forecast-step caches, cancellation of replaced requests, and separate issue/valid timestamps. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index ab059cebb..4e498cb6b 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -31,6 +31,7 @@ How to read this: | **The Space Devs — Launch Library 2 v2.3** | Recent launch, payload, stage, and recovery metadata for Space Missions (30d) | [The Space Devs terms of use](https://github.com/TheSpaceDevs/Tutorials/blob/main/faqs/faq_TSD.md#terms-of-use): data may be used and shared in any form; avoid forwarding it without added value; attribution is encouraged (not mandatory). [Official API limits](https://ll.thespacedevs.com/docs/): 15 unauthenticated calls/hour; optional token | "Launch Library 2 — The Space Devs" (courtesy attribution) | | **Esri World Imagery** (ArcGIS Online tile service) | The keyless satellite basemap — the default landing when no Google/ion credential is configured, and the "Esri Satellite" map stack | [Esri Master Agreement](https://www.esri.com/en-us/legal/terms/full-master-agreement): the public World Imagery service is usable in public-facing apps with attribution; no key is required for this classic endpoint, but Esri governs and can change access — an app at scale should review current ArcGIS Location Platform terms | "Powered by Esri — Source: Esri, Maxar, Earthstar Geographics, and the GIS User Community" (provider carries the service's own credit line) | | **USGS** | Earthquakes | U.S. public domain | "Data courtesy of the U.S. Geological Survey" | +| **Environment and Climate Change Canada** (MSC GeoMet OGC API) | Air quality (AQHI) — Canadian station observations | [ECCC Data Servers End-use Licence](https://eccc-msc.github.io/open-data/licence/readme_en/) (open, attribution required, no endorsement implied) | "Contains information licensed under the Environment and Climate Change Canada Data Servers End-use Licence" | | **NASA GIBS** (WMTS tiles + Worldview Snapshots) and **NASA CMR** (granule catalog) | Recent Imagery layer: HLS S30/L30 (Sentinel-2, Landsat 8/9) and VIIRS true-colour tiles draped on the map, per-day thumbnails and PNG exports; the HLS granule catalog (dates, cloud, footprints) for a selected box | U.S. public domain (NASA data); keyless, browser-direct (CORS `*`); no bulk use — one 30-day catalog query per box, tiles bounded to the box, six tile requests in flight | "We acknowledge the use of imagery provided by services from NASA's Global Imagery Browse Services (GIBS), part of NASA's Earth Science Data and Information System (ESDIS)." — [gibs.earthdata.nasa.gov](https://gibs.earthdata.nasa.gov), [HLS](https://lpdaac.usgs.gov/products/hlss30v002/) | | **NIFC WFIGS** (Wildland Fire Interagency Geospatial Services, ArcGIS feature service) | Fire Perimeters layer: current interagency wildfire incident perimeters, containment, and incident facts | U.S. public domain (interagency wildland-fire data published through the [NIFC Open Data portal](https://data-nifc.opendata.arcgis.com/)); fetched keyless through `/api/fire-perimeters` (5-minute server cache) with `exceededTransferLimit` paging | "Wildfire perimeters: National Interagency Fire Center (WFIGS)" | | **InciWeb** (inciweb.wildfire.gov) | Per-incident "InciWeb ↗" card links to official incident pages (narratives, evacuation/closure notices) | U.S. government public incident information. Links are matched best-effort by incident name and state, with incident page origin and update times checked on selection. Same-origin `/api/fire-perimeters/inciweb` routes cache the catalog for 1 hour and publication metadata for 30 minutes; linked pages remain InciWeb content | "Incident information: InciWeb (inciweb.wildfire.gov)" | @@ -72,6 +73,14 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei ### Notes on the live sources +- **Air quality (ECCC AQHI).** Keyless. Read from the MSC GeoMet OGC API (`api.weather.gc.ca`): the `aqhi-stations` catalog plus `aqhi-observations-realtime`. **Coverage is Canada only** — ECCC is the single registered provider, and the layer row and readout card both say so. Roughly 134 stations across ECCC's five administrative zones (Atlantic, Quebec, Ontario, Prairies & North, Pacific & Yukon). AQHI publishes hourly, so readings poll every 20 minutes and the station catalog, which changes a few times a year, is refetched only every 12 hours. Observations are requested with `latest=true`; without it the collection returns the full history, roughly 2,000 rows nationally. Readings older than six hours are dropped rather than displayed as current. + + Readings are stored as `{ provider, scale, value, band }` rather than a bare number, because air-quality indices are not interchangeable: AQHI runs 1-10+ on health risk, the US AQI runs 0-500 on concentration breakpoints, and Europe's EAQI is a five-step category. Naming the index a value belongs to is what lets another network be added later without inventing a shared numeric meaning that does not exist. + + AQHI is reported by ECCC as an integer from 1 to 10 with an open-ended "10+" above that. The API returns the underlying decimal (1.08, 2.32); the layer rounds to the published form rather than displaying a precision the index does not claim. + + **Quebec is thinly represented**, and that is upstream rather than a filter here: the province runs its own Info-Smog program, so only a handful of Quebec stations appear in ECCC's national feed. The reporting station count also varies hour to hour as stations report or fall silent. + - **Google Maps Platform.** You supply your own API key and are bound by [Google's ToS](https://cloud.google.com/maps-platform/terms). Google Maps Content (tiles, geocodes, places) **may not be cached, stored, rehosted, or committed** — this app only ever uses it live, which is the compliant pattern. The "Google" attribution is displayed on the globe and must stay visible. Restrict your key (see [SECURITY.md](SECURITY.md)). - **OpenSky Network.** Its license is **non-commercial**, and operational use of the REST API in a live product can require a prior written agreement with OpenSky — even for non-profit/government use. If you deploy this commercially, contact OpenSky for your own terms. The flights layer is a toggle and runs anonymously by default. - **adsb.lol flight fallback.** When OpenSky is unavailable and no last-good OpenSky response exists, the server requests a cached, capped 250 nm adsb.lol point snapshot around the current camera subpoint. This is regional observed context, not worldwide completeness; provenance is exposed in the Flights stats/context row. Military ICAOs remain reconciled through the existing dedicated military registry rather than duplicated. diff --git a/README.md b/README.md index f6a3ae886..9f69fdbd6 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,7 @@ Nineteen layers and map sources. **Seventeen have a keyless path.** Some offer a | 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | | 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | | 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | +| ◍ **Air quality** | Live AQHI on the published 1–10+ health-risk scale, with a Weather-panel card showing the band legend and the highest readings in view. **Canada only** — ECCC is the first registered provider | ECCC MSC GeoMet | 🟢 | | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | | 📹 **CCTV Mesh** | ~3,600 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary. Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 22ce1eb3b..cdaa687b8 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2848,6 +2848,7 @@ its criteria cannot be silently ignored. | Live AIS Vessels 🚢 | AISStream websocket | `src/data/aisLiveVessels.js` | `/api/ais-live` | 60s (+800ms visibility pass) | | Mapped Installations ⌖ | OpenFreeMap military areas; optional configured Overpass names and Google Places search | `src/data/militaryInstallations.js` | OpenFreeMap tiles, `/api/military-installations`, `/api/google/text-search` | viewport-driven; transient errors back off, missing capability switches to tiles | | Earthquakes | USGS | `src/data/earthquakes.js` | — | 60s | +| Air quality | ECCC MSC GeoMet OGC API (AQHI, Canada only) | `src/layers/airQuality/` | — | 20 min readings, 12 h station catalog | | Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | | Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | | Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | diff --git a/package.json b/package.json index e48881c6d..eca8aa8b7 100644 --- a/package.json +++ b/package.json @@ -168,6 +168,7 @@ "./layers/bikeshare": "./src/layers/bikeshare/index.js", "./layers/directions": "./src/layers/directions/index.js", "./layers/cctv": "./src/layers/cctv/index.js", + "./layers/air-quality": "./src/layers/airQuality/index.js", "./layers/alpr": "./src/layers/alpr/index.js", "./layers/radio": "./src/layers/radio/index.js", "./layers/submarine-cables": "./src/layers/submarineCables/index.js", @@ -228,6 +229,7 @@ "./layers/alpr/source": "./src/layers/alpr/source.js", "./layers/bikeshare/source": "./src/layers/bikeshare/source.js", "./layers/cctv/source": "./src/layers/cctv/source.js", + "./layers/air-quality/source": "./src/layers/airQuality/source.js", "./layers/earthquakes/source": "./src/layers/earthquakes/source.js", "./layers/perimeters/source": "./src/layers/perimeters/source.js", "./layers/firms/source": "./src/layers/firms/source.js", diff --git a/scripts/format-scope.json b/scripts/format-scope.json index cd836a5cb..2e27bf753 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -346,6 +346,12 @@ "src/layers/vessels/tracking.js", "src/layers/vessels/ownership.test.mjs", "src/data/earthquakes.js", + "src/app/layers/airQuality.js", + "src/layers/airQuality/index.js", + "src/layers/airQuality/model.js", + "src/layers/airQuality/model.test.mjs", + "src/layers/airQuality/ownership.test.mjs", + "src/layers/airQuality/source.js", "src/layers/earthquakes/index.js", "src/layers/earthquakes/model.js", "src/layers/earthquakes/source.js", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 276b9e0db..67f304f3f 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -2027,7 +2027,11 @@ "src/sources/requestDeadline.js", "src/data/local_data/osm_military_names/names.json", "src/layers/installations/namedMarkers.js", - "src/data/infrastructureOverlayEntry.js" + "src/data/infrastructureOverlayEntry.js", + "src/app/layers/airQuality.js", + "src/layers/airQuality/index.js", + "src/layers/airQuality/model.js", + "src/layers/airQuality/source.js" ], "external": [ "hls.js", @@ -2511,7 +2515,11 @@ "src/sources/requestDeadline.js", "src/data/local_data/osm_military_names/names.json", "src/layers/installations/namedMarkers.js", - "src/data/infrastructureOverlayEntry.js" + "src/data/infrastructureOverlayEntry.js", + "src/app/layers/airQuality.js", + "src/layers/airQuality/index.js", + "src/layers/airQuality/model.js", + "src/layers/airQuality/source.js" ], "external": [ "hls.js", @@ -2853,7 +2861,9 @@ "src/sources/httpBody.js", "src/layers/perimeters/source.js", "src/layers/perimeters/records.js", - "src/layers/submarineCables/bundledSource.js" + "src/layers/submarineCables/bundledSource.js", + "src/layers/airQuality/model.js", + "src/layers/airQuality/source.js" ], "external": [] }, @@ -3072,5 +3082,22 @@ "src/layers/perimeters/source.js" ], "external": [] + }, + "air-quality": { + "exports": ["./layers/air-quality"], + "modules": [ + "src/layers/airQuality/index.js", + "src/layers/airQuality/model.js", + "src/layers/airQuality/source.js" + ], + "external": ["cesium"] + }, + "air-quality-source": { + "exports": ["./layers/air-quality/source"], + "modules": [ + "src/layers/airQuality/model.js", + "src/layers/airQuality/source.js" + ], + "external": [] } } diff --git a/scripts/qa-air-quality.mjs b/scripts/qa-air-quality.mjs new file mode 100644 index 000000000..68e6bd505 --- /dev/null +++ b/scripts/qa-air-quality.mjs @@ -0,0 +1,171 @@ +/** + * qa-air-quality.mjs — headless proof for the Air quality layer. + * + * Checks what unit tests cannot: that the live ECCC feed answers, that readings + * reach the app in the provider-shaped form other networks will share, and that + * the Weather readout descriptor carries a legend and an in-view list. + * + * AQHI is reported by ECCC as an integer from 1 with "10+" above ten. The API + * returns the underlying decimal, so a reading arriving as 1.08 — or a station + * with no reading surfacing as a confident 1 — is a real defect this catches. + * + * Run: node scripts/qa-air-quality.mjs --url http://localhost:4173 + */ + +import puppeteer from 'puppeteer'; + +const argv = process.argv.slice(2); +const getOpt = (name, dflt) => { + const i = argv.indexOf(name); + return i >= 0 && argv[i + 1] ? argv[i + 1] : dflt; +}; +const APP_URL = getOpt('--url', 'http://localhost:4173'); + +const results = []; +function record(name, ok, detail) { + results.push({ name, ok, detail }); + const tag = ok ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m'; + console.log(` [${tag}] ${name}${detail ? ` — ${detail}` : ''}`); +} +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +async function main() { + let exitCode = 0; + const consoleErrors = []; + const browser = await puppeteer.launch({ + headless: 'new', + args: [ + '--use-gl=swiftshader', + '--enable-unsafe-swiftshader', + '--no-sandbox', + '--disable-dev-shm-usage', + ], + defaultViewport: { width: 1400, height: 900 }, + }); + + try { + const page = await browser.newPage(); + page.on('pageerror', (e) => consoleErrors.push(String(e).slice(0, 160))); + page.on('console', (m) => { + if (m.type() === 'error') consoleErrors.push(m.text().slice(0, 160)); + }); + + await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 90000 }); + await page.waitForFunction( + () => window.__godsEyeView?.viewer && window.__godsEyeView?.dataManager, + { timeout: 90000 }, + ); + await sleep(1500); + + const stats = await page.evaluate(async () => { + const dm = window.__godsEyeView.dataManager; + await dm.setEnabled('air-quality', true); + const mod = dm.layers.get('air-quality')?.module; + if (!mod) return { missing: true }; + let s = null; + for (let i = 0; i < 45; i++) { + await new Promise((r) => setTimeout(r, 1000)); + s = mod.getStats(); + if (s.count > 0 || s.error) break; + } + return s; + }); + + const loaded = !stats?.missing && !stats?.error && stats?.count > 0; + record('stations report a current index', loaded, + `count=${stats?.count} error=${stats?.error}`); + if (!loaded) exitCode = 1; + + const records = await page.evaluate( + () => + window.__godsEyeView.dataManager + .layers.get('air-quality') + ?.module.getAnalystRecords(500) || [], + ); + + // The provider-neutral contract other networks will share. + const shaped = records.every( + (r) => r.provider && r.scale && Number.isFinite(r.value) && r.band, + ); + record('every reading carries provider, scale, value and band', shaped, + records.length + ? `sample={provider:${records[0].provider}, scale:${records[0].scale}, value:${records[0].value}, band:${records[0].band}}` + : 'no records'); + if (!shaped) exitCode = 1; + + const published = records.every((r) => Number.isInteger(r.value) && r.value >= 1); + record('every AQHI value is in the published integer form, floored at 1', published, + `values=${[...new Set(records.map((r) => r.value))].sort((a, b) => a - b).join(',')}`); + if (!published) exitCode = 1; + + const zones = [...new Set(records.map((r) => r.zone).filter(Boolean))].sort(); + record('coverage spans multiple ECCC administrative zones', zones.length >= 3, + `zones=${zones.join(',') || 'none'} stations=${records.length}`); + if (zones.length < 3) exitCode = 1; + + // The card's whole point is "highest readings in view", so put the camera + // over Canada before asserting on it — the default view is over Austin, + // where an empty in-view list is correct and proves nothing. + await page.evaluate(() => { + const gev = window.__godsEyeView; + const ellipsoid = gev.viewer.scene.globe.ellipsoid; + const d2r = Math.PI / 180; + try { + gev.viewer.camera.cancelFlight(); + } catch { + /* no flight in progress */ + } + gev.viewer.camera.setView({ + destination: ellipsoid.cartographicToCartesian({ + longitude: -100 * d2r, + latitude: 55 * d2r, + height: 4_000_000, + }), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + gev.viewer.scene.requestRender?.(); + }); + await sleep(2500); + + const controls = await page.evaluate( + () => + window.__godsEyeView.dataManager + .layers.get('air-quality') + ?.module.getRowControls?.() || null, + ); + const readoutOk = + controls?.readout === true && + Array.isArray(controls?.legend?.colors) && + controls.legend.colors.length === controls.legend.labels.length && + Array.isArray(controls?.list?.items) && + controls.list.items.length > 0 && + controls.list.items.every((item) => item.id && item.label && item.value) && + /Canada only/.test(controls?.summary?.coverage || ''); + record('the Weather readout lists the worst readings in view, with a legend', readoutOk, + controls + ? `bands=${controls.legend?.labels?.join('/')} inView=${controls.list?.items?.length} coverage="${controls.summary?.coverage}"` + : 'no descriptor'); + if (!readoutOk) exitCode = 1; + + record('no console errors', consoleErrors.length === 0, + consoleErrors.length ? consoleErrors.slice(0, 2).join(' | ') : 'clean'); + if (consoleErrors.length) exitCode = 1; + } catch (e) { + console.error('\x1b[31mHarness error:\x1b[0m', e); + exitCode = 3; + } finally { + await browser.close(); + } + + const pass = results.filter((r) => r.ok === true).length; + const fail = results.filter((r) => r.ok === false).length; + console.log('\n' + '─'.repeat(60)); + console.log(` RESULT: ${pass} passed, ${fail} failed`); + console.log('─'.repeat(60) + '\n'); + process.exit(exitCode || (fail > 0 ? 1 : 0)); +} + +main().catch((e) => { + console.error(e); + process.exit(3); +}); diff --git a/src/app/constructCatalog.js b/src/app/constructCatalog.js index 7ed03d0d2..a9728f317 100644 --- a/src/app/constructCatalog.js +++ b/src/app/constructCatalog.js @@ -23,6 +23,7 @@ import { createApplicationLocalAdsb } from './layers/localAdsb.js'; import { createApplicationAwareness } from './layers/militaryAwareness.js'; import { createApplicationFirms } from './layers/firms.js'; import { createApplicationEarthquakes } from './layers/earthquakes.js'; +import { createApplicationAirQuality } from './layers/airQuality.js'; import { createApplicationFirePerimeters } from './layers/perimeters.js'; import { createApplicationCables } from './layers/submarineCables.js'; import { createInfrastructureLayers } from '../data/infrastructure.js'; @@ -53,6 +54,7 @@ const SOURCE_METHODS = Object.freeze({ weather: ['getSnapshot'], cyclones: ['getSnapshot'], earthquakes: ['getSnapshot'], + 'air-quality': ['getSnapshot'], 'fire-perimeters': ['getSnapshot'], cables: ['fetch'], }); @@ -144,6 +146,7 @@ export function createApplicationCatalog({ ...(resolveAsset ? { resolveAsset } : {}), }), createApplicationEarthquakes({ source: sources.earthquakes }), + createApplicationAirQuality({ source: sources['air-quality'] }), createApplicationFirePerimeters({ source: sources['fire-perimeters'], }), diff --git a/src/app/constructCatalog.test.mjs b/src/app/constructCatalog.test.mjs index 6043b1262..3a802edbe 100644 --- a/src/app/constructCatalog.test.mjs +++ b/src/app/constructCatalog.test.mjs @@ -39,7 +39,7 @@ test('catalogs construct distinct layers and classification from their supplied signal: b.signal, surface: fixtureSurface(b.signal), }); - assert.equal(first.layers.length, 29); + assert.equal(first.layers.length, 30); assert.ok(first.get('local-adsb'), 'Local ADS-B is registered'); assert.deepEqual( first.metadata.find(({ id }) => id === 'local-adsb'), diff --git a/src/app/layers/airQuality.js b/src/app/layers/airQuality.js new file mode 100644 index 000000000..1f25c06d9 --- /dev/null +++ b/src/app/layers/airQuality.js @@ -0,0 +1,6 @@ +import { createAirQualityLayer } from '../../layers/airQuality/index.js'; +import { overlayHost } from './overlayHost.js'; +/** Wire air-quality readings to the application overlay host. */ +export function createApplicationAirQuality(options) { + return createAirQualityLayer({ overlayHost, ...options }); +} diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index f9fc23b70..f47c454a2 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -162,6 +162,13 @@ export const DATA_CREDITS = [ }, OSM_CREDIT, OPENMAPTILES_CREDIT, + { + key: 'eccc-aqhi', + html: + 'Air quality (AQHI): Environment and Climate Change Canada — MSC GeoMet ' + + '(eccc-msc.github.io/open-data) ' + + '· ECCC Data Servers End-use Licence', + }, { key: 'overture-military-names', html: 'Military area names: Overture Maps Foundation (ODbL)', diff --git a/src/data/layerState.js b/src/data/layerState.js index eccc555f6..ea43cfd15 100644 --- a/src/data/layerState.js +++ b/src/data/layerState.js @@ -553,6 +553,11 @@ export const LAYER_STATE_TOKEN_RESERVATIONS = * owns stable URL ordering. */ export const LAYER_STATE_REGISTRY = Object.freeze([ + Object.freeze({ + id: 'air-quality', + token: '0', + disposition: 'enabled-only', + }), Object.freeze({ id: 'ais-live-vessels', token: 'a', diff --git a/src/data/layerState.test.mjs b/src/data/layerState.test.mjs index 1b504a439..3bebbc7b3 100644 --- a/src/data/layerState.test.mjs +++ b/src/data/layerState.test.mjs @@ -199,8 +199,8 @@ function encode(state) { test('production registry is exact, canonical, and rejects incomplete contracts', async () => { assert.equal(validateLayerStateRegistry(), true); - assert.equal(REGISTERED_LAYER_IDS.length, 28); - assert.equal(new Set(REGISTERED_LAYER_IDS).size, 28); + assert.equal(REGISTERED_LAYER_IDS.length, 29); + assert.equal(new Set(REGISTERED_LAYER_IDS).size, 29); assert.ok(REGISTERED_LAYER_IDS.includes('transit')); assert.deepEqual(REGISTERED_LAYER_IDS, [...REGISTERED_LAYER_IDS].sort()); assert.deepEqual(LEGACY_LAYER_STATE_TOKENS, { @@ -239,7 +239,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts' for (const [id, token] of Object.entries(LEGACY_LAYER_STATE_TOKENS)) { assert.equal(LAYER_STATE_TOKEN_RESERVATIONS[id], token); } - assert.equal(nextLayerStateToken(), '0'); + assert.equal(nextLayerStateToken(), '3'); assert.equal( nextLayerStateToken({ ...LAYER_STATE_TOKEN_RESERVATIONS, alpha: '0', bravo: '3' }), '4', @@ -247,7 +247,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts' const digitsExhausted = { ...LAYER_STATE_TOKEN_RESERVATIONS, ...Object.fromEntries( - [...'03456789'].map((digit) => [`prior-${digit}`, digit]), + [...'3456789'].map((digit) => [`prior-${digit}`, digit]), ), }; assert.equal(nextLayerStateToken(digitsExhausted), '00'); @@ -298,7 +298,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts' assert.equal( validateLayerStateAllocations( LAYER_STATE_TOKEN_RESERVATIONS, - { ...LAYER_STATE_TOKEN_RESERVATIONS, future: '0', next: '3' }, + { ...LAYER_STATE_TOKEN_RESERVATIONS, future: '3', next: '4' }, ), true, ); @@ -307,7 +307,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts' ...LAYER_STATE_TOKEN_RESERVATIONS, future: '00', }), - /next free token 0/, + /next free token 3/, ); const beforeLastDigit = { ...digitsExhausted }; delete beforeLastDigit['prior-9']; diff --git a/src/data/layerStateTokenLedger.test.mjs b/src/data/layerStateTokenLedger.test.mjs index 7b0a9263c..28223762e 100644 --- a/src/data/layerStateTokenLedger.test.mjs +++ b/src/data/layerStateTokenLedger.test.mjs @@ -122,13 +122,20 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed { ...parseLayerStateTokenReservations(reservationRows) }, { ...LAYER_STATE_TOKEN_RESERVATIONS }, ); - assert.equal(Object.keys(LAYER_STATE_TOKEN_RESERVATIONS).length, 28); + assert.equal(Object.keys(LAYER_STATE_TOKEN_RESERVATIONS).length, 29); + // The legacy assignments are a closed compatibility set that never grows, so + // the ledger is exactly that set plus every token reserved since. assert.deepEqual( { ...LAYER_STATE_TOKEN_RESERVATIONS }, - { ...LEGACY_LAYER_STATE_TOKENS }, + { ...LEGACY_LAYER_STATE_TOKENS, 'air-quality': '0' }, ); assert.throws( - () => parseLayerStateTokenReservations(reservationRows.slice(1)), + () => + parseLayerStateTokenReservations( + // Dropping any legacy row must fail. This used to slice off row 0, + // which is no longer a legacy reservation. + reservationRows.filter(([id]) => id !== 'ais-live-vessels'), + ), /Missing or changed legacy/, ); assert.throws( @@ -145,9 +152,9 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed /Duplicate layer-state token reservation/, ); assert.equal( - parseLayerStateTokenReservations([...reservationRows, ['future', '0']]) + parseLayerStateTokenReservations([...reservationRows, ['future', '3']]) .future, - '0', + '3', ); assert.throws( () => @@ -165,14 +172,14 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed test('published retired digits and competing PRs advance the merge-time allocation', () => { withPublishedBase( - { rows: [...reservationRows, ['retired-layer', '0']] }, + { rows: [...reservationRows, ['retired-layer', '3']] }, (cwd) => { const published = readPublishedLayerStateReservations('HEAD', cwd); - assert.equal(nextLayerStateToken(published), '3'); + assert.equal(nextLayerStateToken(published), '4'); assert.equal( validateLayerStateAllocations(published, { ...published, - newcomer: '3', + newcomer: '4', }), true, ); @@ -182,7 +189,7 @@ test('published retired digits and competing PRs advance the merge-time allocati ...published, competing: '0', }), - /next free token 3/, + /next free token 4/, ); assert.throws( () => @@ -196,8 +203,8 @@ test('published retired digits and competing PRs advance the merge-time allocati ); }); -test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => { - withPublishedBase({ rows: [...reservationRows, ['pr-a', '0']] }, (cwd) => { +test('PR B manually replaces provisional 3 with 4 after PR A publishes 3', () => { + withPublishedBase({ rows: [...reservationRows, ['pr-a', '3']] }, (cwd) => { mkdirSync(path.join(cwd, 'scripts')); const fixtureChecker = path.join( cwd, @@ -218,8 +225,8 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => writeCodecFixture( cwd, - [...reservationRows, ['pr-b', '0']], - [{ id: 'pr-b', token: '0' }], + [...reservationRows, ['pr-b', '3']], + [{ id: 'pr-b', token: '3' }], ); const beforeRebase = runCheck(); assert.equal(beforeRebase.status, 1); @@ -227,10 +234,10 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => writeCodecFixture( cwd, - [...reservationRows, ['pr-a', '0'], ['pr-b', '0']], + [...reservationRows, ['pr-a', '3'], ['pr-b', '3']], [ - { id: 'pr-a', token: '0' }, - { id: 'pr-b', token: '0' }, + { id: 'pr-a', token: '3' }, + { id: 'pr-b', token: '3' }, ], ); const staleRebase = runCheck(); @@ -238,16 +245,16 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => assert.match(staleRebase.stderr, /Duplicate layer-state token reservation/); const published = readPublishedLayerStateReservations('HEAD', cwd); - assert.equal(nextLayerStateToken(published), '3'); - const validBaselineRows = [...reservationRows, ['pr-a', '0']]; - writeCodecFixture(cwd, validBaselineRows, [{ id: 'pr-a', token: '0' }]); + assert.equal(nextLayerStateToken(published), '4'); + const validBaselineRows = [...reservationRows, ['pr-a', '3']]; + writeCodecFixture(cwd, validBaselineRows, [{ id: 'pr-a', token: '3' }]); const helper = spawnSync( process.execPath, [realpathSync(fixtureNext), 'pr-b'], { cwd, encoding: 'utf8' }, ); assert.equal(helper.status, 0, helper.stderr); - assert.match(helper.stdout, /pr-b: 3/); + assert.match(helper.stdout, /pr-b: 4/); assert.deepEqual( JSON.parse(readFileSync(path.join(cwd, LEDGER_PATH), 'utf8')), validBaselineRows, @@ -255,20 +262,20 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => ); writeCodecFixture( cwd, - [...reservationRows, ['pr-a', '0'], ['pr-b', '3']], + [...reservationRows, ['pr-a', '3'], ['pr-b', '4']], [ - { id: 'pr-a', token: '0' }, - { id: 'pr-b', token: '3' }, + { id: 'pr-a', token: '3' }, + { id: 'pr-b', token: '4' }, ], ); const corrected = runCheck(); assert.equal(corrected.status, 0, corrected.stderr); - assert.match(corrected.stdout, /29 published, 1 new/); + assert.match(corrected.stdout, /30 published, 1 new/); }); }); test('allocation batches cross the last digit and base-36 pair boundaries in order', () => { - const digitRows = [...'03456789'].map((digit) => [`retired-${digit}`, digit]); + const digitRows = [...'3456789'].map((digit) => [`retired-${digit}`, digit]); const allDigits = parseLayerStateTokenReservations([ ...reservationRows, ...digitRows, @@ -320,7 +327,7 @@ test('allocation batches cross the last digit and base-36 pair boundaries in ord test('an isolated valid two-character fixture round-trips an l field beyond the old 64-character cap', async () => { const cwd = mkdtempSync(path.join(tmpdir(), 'gev-layer-token-width-')); try { - const priorDigits = [...'03456789'].map((digit) => [ + const priorDigits = [...'3456789'].map((digit) => [ `qa-prior-digit-${digit}`, digit, ]); @@ -357,7 +364,7 @@ test('an isolated valid two-character fixture round-trips an l field beyond the const params = new URLSearchParams([['v', '2']]); codec.encodeLayerStateParams(params, state); assert.ok(params.get('l').length > 64, 'fixture must cross the old cap'); - assert.equal(params.get('l').length, 67); + assert.equal(params.get('l').length, 69); const restored = codec.decodeLayerStateParams(params); assert.deepEqual(restored?.enabledLayerIds, expectedLayerIds); assert.deepEqual(restored?.options, state.options); @@ -375,7 +382,7 @@ test('checker reads a complete future base ledger and rejects retired-token reus }, (cwd) => { const published = readPublishedLayerStateReservations('HEAD', cwd); - assert.equal(Object.keys(published).length, 29); + assert.equal(Object.keys(published).length, 30); assert.equal(published['retired-layer'], '00'); assert.throws( () => diff --git a/src/data/layerStateTokenReservations.json b/src/data/layerStateTokenReservations.json index 600358c7c..7329ad59f 100644 --- a/src/data/layerStateTokenReservations.json +++ b/src/data/layerStateTokenReservations.json @@ -1,4 +1,5 @@ [ + ["air-quality", "0"], ["ais-live-vessels", "a"], ["alpr-cameras", "p"], ["bhote-koshi-2026", "h"], diff --git a/src/layers/airQuality/index.js b/src/layers/airQuality/index.js new file mode 100644 index 000000000..43a76f857 --- /dev/null +++ b/src/layers/airQuality/index.js @@ -0,0 +1,308 @@ +import * as Cesium from 'cesium'; +import { + AIR_QUALITY_BANDS, + AIR_QUALITY_OVERLAY_SOURCE_ID, + AIR_QUALITY_OVERLAY_COHORT_LIMIT, + AIR_QUALITY_OVERLAY_COLLISION_CAPACITY, + AIR_QUALITY_PROVIDERS, + createAirQualityOverlayEntry, + mapAnalystRecord, + newestObservation, + readingsAreStale, + readingsInView, + selectAirQualityOverlayCohort, +} from './model.js'; +export * from './model.js'; +export { createEcccAirQualitySource } from './source.js'; + +/** Station markers are a point reading, not an extent. */ +const STATION_MARKER_RADIUS_M = 6_000; +/** Readings listed in the readout card. */ +const READOUT_LIMIT = 5; + +/** Own one air-quality display, its refresh lifecycle, and its readout card. */ +export function createAirQualityLayer({ source, overlayHost } = {}) { + if (typeof source?.getSnapshot !== 'function') + throw new TypeError('Air quality requires a snapshot source'); + if (!overlayHost) throw new TypeError('Air quality requires an overlay host'); + let _viewer = null; + let _request = null; + let _dataSource = null; + let _readings = []; + let _count = 0; + let _lastUpdate = null; + let _lastError = null; + let _enabled = false; + let rowControlsListener = null; + + const notify = () => rowControlsListener?.(); + + /** The camera's lon/lat rectangle in degrees, or null when unresolvable. */ + const viewRectangle = () => { + const rectangle = _viewer?.camera?.computeViewRectangle?.(); + if (!rectangle) return null; + const toDeg = Cesium.Math.toDegrees; + return { + west: toDeg(rectangle.west), + south: toDeg(rectangle.south), + east: toDeg(rectangle.east), + north: toDeg(rectangle.north), + }; + }; + + const layer = { + id: 'air-quality', + name: 'Air quality', + icon: '◍', + source: 'ECCC MSC GeoMet', + updateInterval: 1_200_000, + + init(viewer) { + if (_viewer) throw new Error('Air quality layer is already initialized'); + _viewer = viewer; + _dataSource = new Cesium.CustomDataSource('air-quality'); + _dataSource.show = false; + viewer.dataSources.add(_dataSource); + _readings = []; + _count = 0; + _lastUpdate = null; + _lastError = null; + _enabled = false; + overlayHost.setVisible(AIR_QUALITY_OVERLAY_SOURCE_ID, false); + }, + + enable() { + _enabled = true; + if (_dataSource) _dataSource.show = true; + overlayHost.setVisible(AIR_QUALITY_OVERLAY_SOURCE_ID, true); + notify(); + }, + + disable() { + // Abort in flight: a response arriving after disable must not redraw. + _request?.abort(); + _request = null; + _enabled = false; + if (_dataSource) _dataSource.show = false; + overlayHost.clearSource(AIR_QUALITY_OVERLAY_SOURCE_ID); + overlayHost.setVisible(AIR_QUALITY_OVERLAY_SOURCE_ID, false); + notify(); + }, + + async update() { + if (!_enabled || !_dataSource) return false; + _request?.abort(); + const request = new AbortController(); + _request = request; + try { + const readings = await source.getSnapshot({ signal: request.signal }); + // A superseded or aborted request must not touch the scene, and neither + // must one that lands after the layer was switched off. + if (request.signal.aborted || _request !== request || !_enabled) + return false; + _readings = readings; + this._render(); + _count = _readings.length; + _lastUpdate = Date.now(); + _lastError = null; + notify(); + return true; + } catch (error) { + if (request.signal.aborted || _request !== request) return false; + _lastError = error?.message || 'Air quality request failed'; + console.warn('[Data:AirQuality] Refresh failed:', _lastError); + notify(); + return false; + } finally { + if (_request === request) _request = null; + } + }, + + /** Rebuild entities and overlay labels from the current readings. */ + _render() { + if (!_dataSource) return; + _dataSource.entities.removeAll(); + const overlayEntries = []; + for (const reading of _readings) { + const position = Cesium.Cartesian3.fromDegrees( + reading.lon, + reading.lat, + ); + const color = Cesium.Color.fromCssColorString(reading.color); + _dataSource.entities.add( + new Cesium.Entity({ + id: `air-quality:${reading.id}`, + position, + ellipse: { + // Static axes, as in the earthquake layer: a per-frame axis + // re-tessellates the clamped ground primitive every frame. + semiMajorAxis: STATION_MARKER_RADIUS_M, + semiMinorAxis: STATION_MARKER_RADIUS_M, + material: new Cesium.ColorMaterialProperty(color.withAlpha(0.35)), + outline: true, + outlineColor: color.withAlpha(0.9), + outlineWidth: 2, + heightReference: Cesium.HeightReference.CLAMP_TO_GROUND, + }, + properties: { + provider: reading.provider, + scale: reading.scale, + value: reading.value, + band: reading.band, + name: reading.name, + zone: reading.zone, + risk: reading.risk, + observedMs: reading.observedMs, + note: reading.note, + lat: reading.lat, + lon: reading.lon, + }, + }), + ); + overlayEntries.push( + createAirQualityOverlayEntry({ + id: reading.id, + position, + title: `${reading.name} ${reading.label}`, + accent: reading.color, + value: reading.value, + }), + ); + } + if (_enabled) { + overlayHost.setEntries( + AIR_QUALITY_OVERLAY_SOURCE_ID, + selectAirQualityOverlayCohort(overlayEntries), + { + cohortLimit: AIR_QUALITY_OVERLAY_COHORT_LIMIT, + collisionCapacity: AIR_QUALITY_OVERLAY_COLLISION_CAPACITY, + moving: false, + }, + ); + } + }, + + destroy(viewer = _viewer) { + _request?.abort(); + _request = null; + _enabled = false; + overlayHost.clearSource(AIR_QUALITY_OVERLAY_SOURCE_ID); + overlayHost.setVisible(AIR_QUALITY_OVERLAY_SOURCE_ID, false); + if (_dataSource) { + viewer?.dataSources?.remove(_dataSource, true); + _dataSource = null; + } + _viewer = null; + _readings = []; + _count = 0; + _lastUpdate = null; + _lastError = null; + rowControlsListener = null; + }, + + /** + * Readout descriptor for the Weather panel: the band legend, and the worst + * readings currently in view. + * + * `readout: true` renders the toggle and source line only; the card owns + * the rest. Coverage is stated plainly because this layer is not global. + */ + getRowControls() { + const provider = AIR_QUALITY_PROVIDERS.eccc; + const inView = readingsInView(_readings, viewRectangle(), READOUT_LIMIT); + const newest = _readings.reduce( + (max, reading) => Math.max(max, reading.observedMs || 0), + 0, + ); + return { + readout: true, + summary: { + label: provider.label, + coverage: provider.coverage, + validTime: newest ? new Date(newest).toISOString() : undefined, + detail: `${provider.network} · ${provider.scale}`, + status: _lastError + ? _lastError + : _count === 0 + ? 'No readings available' + : `${_count} stations reporting`, + units: provider.scale, + }, + legend: { + colors: AIR_QUALITY_BANDS.map((band) => band.color), + labels: AIR_QUALITY_BANDS.map((band) => band.label), + units: provider.scale, + categorical: true, + }, + list: { + ariaLabel: 'Highest air-quality readings in view', + items: inView.map((reading) => ({ + id: reading.id, + label: reading.name, + value: `${reading.scale} ${reading.label}`, + accent: reading.color, + detail: reading.risk, + })), + }, + }; + }, + + setRowControlsListener(listener) { + rowControlsListener = typeof listener === 'function' ? listener : null; + }, + + /** + * The current reading nearest a point — what "the air quality here" means. + * @param {number} lat + * @param {number} lon + * @returns {object|null} Reading with distanceKm, or null when none loaded. + */ + nearestReading(lat, lon) { + if (!Number.isFinite(lat) || !Number.isFinite(lon) || !_readings.length) + return null; + const cosLat = Math.cos((lat * Math.PI) / 180); + let best = null; + let bestDistance = Infinity; + for (const reading of _readings) { + const dLat = (reading.lat - lat) * 111.32; + const dLon = (reading.lon - lon) * 111.32 * cosLat; + const distance = Math.hypot(dLat, dLon); + if (distance < bestDistance) { + bestDistance = distance; + best = reading; + } + } + return best ? { ...best, distanceKm: bestDistance } : null; + }, + + /** + * Snapshot readings as JSON-safe records for the analyst query engine. + * @param {number} [maxCount=500] + * @returns {Array} + */ + getAnalystRecords(maxCount = 500) { + if (!_enabled || !_readings.length) return []; + const limit = Number.isFinite(maxCount) + ? Math.max(1, Math.floor(maxCount)) + : 500; + return _readings + .slice(0, limit) + .map((reading, index) => mapAnalystRecord(reading, index)); + }, + + getStats() { + // `lastUpdate` is when a response was received; `observedAt` is when the + // network actually measured. Freshness is judged on the latter, so a feed + // that keeps answering with hours-old observations reports STALE rather + // than being narrated as live by the HUD and voice provenance envelope. + return { + count: _count, + lastUpdate: _lastUpdate, + error: _lastError, + observedAt: newestObservation(_readings), + stale: readingsAreStale(_readings), + }; + }, + }; + return layer; +} diff --git a/src/layers/airQuality/model.js b/src/layers/airQuality/model.js new file mode 100644 index 000000000..93f13404f --- /dev/null +++ b/src/layers/airQuality/model.js @@ -0,0 +1,374 @@ +/** + * @module layers/airQuality/model + * + * Air-quality readings, normalized across networks. Pure — no Cesium, no + * network. + * + * Air-quality indices are not interchangeable numbers. Canada's AQHI runs 1 to + * 10+ on health risk; the US AQI runs 0 to 500 on concentration breakpoints; + * Europe's EAQI is a five-step category. A reading is therefore carried as + * `{ provider, scale, value, band }` — who measured it, which index it is, the + * value on that index's own terms, and the band it falls in — so a second + * network can be added without retrofitting a shared numeric meaning that does + * not exist. + * + * Only ECCC's AQHI is implemented here; `AIR_QUALITY_PROVIDERS` is the seam a + * later provider registers against. + */ + +/** Band identifiers shared across providers, in ascending severity. */ +export const AIR_QUALITY_BANDS = Object.freeze([ + Object.freeze({ id: 'low', label: 'Low', color: '#4cc9f0' }), + Object.freeze({ id: 'moderate', label: 'Moderate', color: '#ffd43b' }), + Object.freeze({ id: 'high', label: 'High', color: '#ff6b35' }), + Object.freeze({ id: 'very-high', label: 'Very High', color: '#c1121f' }), +]); + +const BAND_BY_ID = new Map(AIR_QUALITY_BANDS.map((band) => [band.id, band])); + +/** Colour used when a station reports no usable reading. */ +export const AIR_QUALITY_UNKNOWN_COLOR = '#6b7785'; + +export const AIR_QUALITY_OVERLAY_SOURCE_ID = 'air-quality'; +export const AIR_QUALITY_OVERLAY_COHORT_LIMIT = 48; +export const AIR_QUALITY_OVERLAY_COLLISION_CAPACITY = 32; + +/** + * A reading this old is stale enough that displaying it would misinform. + * AQHI publishes hourly, so six hours is five missed publications. + */ +export const OBSERVATION_MAX_AGE_MS = 6 * 60 * 60 * 1000; + +/** + * How old the newest observation may be before the layer reports itself stale. + * + * This is deliberately separate from the age at which a reading is dropped. + * `lastUpdate` records when we last *received* a response, which stays current + * even when the network publishes nothing new; freshness has to be judged on + * the source's own observation time instead. AQHI publishes hourly, so a newest + * reading beyond two hours means at least one publication was missed and the + * feed should not be described as live. + */ +export const STALE_AFTER_MS = 2 * 60 * 60 * 1000; + +/** + * The newest observation time across a set of readings, or null when empty. + * @param {Array} readings + * @returns {number|null} Epoch ms. + */ +export function newestObservation(readings) { + if (!Array.isArray(readings) || readings.length === 0) return null; + let newest = null; + for (const reading of readings) { + const at = Number(reading?.observedMs); + if (Number.isFinite(at) && (newest === null || at > newest)) newest = at; + } + return newest; +} + +/** + * Is a set of readings stale, judged on source observation time? + * @param {Array} readings + * @param {number} [now=Date.now()] + * @returns {boolean} + */ +export function readingsAreStale(readings, now = Date.now()) { + const newest = newestObservation(readings); + if (newest === null) return false; // nothing to be stale about + return now - newest > STALE_AFTER_MS; +} + +/** + * Registered provider descriptors. `coverage` is stated plainly because the + * layer is not global and the row must say so. + */ +export const AIR_QUALITY_PROVIDERS = Object.freeze({ + eccc: Object.freeze({ + id: 'eccc', + scale: 'AQHI', + label: 'Air quality', + network: 'Environment and Climate Change Canada', + coverage: 'Canada only · ECCC AQHI network', + /** ECCC's published band thresholds, evaluated in order. */ + bands: Object.freeze([ + Object.freeze({ id: 'low', max: 3 }), + Object.freeze({ id: 'moderate', max: 6 }), + Object.freeze({ id: 'high', max: 10 }), + Object.freeze({ id: 'very-high', max: Infinity }), + ]), + }), +}); + +/** Band descriptor for an id, or null. */ +export function airQualityBandById(bandId) { + return BAND_BY_ID.get(String(bandId ?? '')) || null; +} + +/** Accent colour for a band id, falling back to neutral grey. */ +export function airQualityColor(bandId) { + return airQualityBandById(bandId)?.color || AIR_QUALITY_UNKNOWN_COLOR; +} + +/** + * The published AQHI value for a raw ECCC reading: an integer from 1 up. + * + * ECCC reports whole numbers with a floor of 1 — there is no "AQHI 0" and no + * "AQHI 1.08". A raw value below 1 is a legitimate low reading, not an error, + * and is published as 1. + * + * @param {number|string|null|undefined} raw - Raw `aqhi` value from the API. + * @returns {number|null} Published integer AQHI, or null when unusable. + */ +export function aqhiDisplayValue(raw) { + // Guard the empty forms BEFORE coercing: Number(null) and Number('') are both + // 0, which would otherwise floor to a confident "AQHI 1" for a station that + // reported nothing at all. + if (raw === null || raw === undefined || raw === '') return null; + if (typeof raw !== 'number' && typeof raw !== 'string') return null; + if (typeof raw === 'string' && raw.trim() === '') return null; + const value = Number(raw); + if (!Number.isFinite(value) || value < 0) return null; + return Math.max(1, Math.round(value)); +} + +/** + * The band id for a value on a provider's scale. + * @param {string} providerId + * @param {number|null} value - Already in the provider's published form. + * @returns {string|null} + */ +export function bandForValue(providerId, value) { + const provider = AIR_QUALITY_PROVIDERS[String(providerId ?? '')]; + if (!provider || !Number.isFinite(value)) return null; + return provider.bands.find((band) => value <= band.max)?.id ?? null; +} + +/** + * Display text for a value on a provider's scale. + * AQHI collapses everything above ten into the open-ended "10+". + * @param {string} providerId + * @param {number|null} value + * @returns {string} + */ +export function valueLabel(providerId, value) { + if (!Number.isFinite(value)) return '--'; + if (providerId === 'eccc') return value > 10 ? '10+' : String(value); + return String(value); +} + +/** Plain-language risk phrase for a band id. */ +export function bandRiskText(bandId) { + const band = airQualityBandById(bandId); + return band ? `${band.label} health risk` : null; +} + +/** + * Validate a station FeatureCollection into an id-keyed catalog. + * + * Atomic: a structurally broken response returns null so the caller keeps its + * last good catalog. A station missing coordinates is skipped, since one + * unusable station is not evidence the catalog is broken. + * + * @param {object} geojson - GeoJSON FeatureCollection from `aqhi-stations`. + * @returns {Map|null} Stations by location_id, or null. + */ +export function normalizeAqhiStations(geojson) { + if (!Array.isArray(geojson?.features)) return null; + const stations = new Map(); + for (const feature of geojson.features) { + const properties = feature?.properties; + if ( + !properties || + typeof properties !== 'object' || + Array.isArray(properties) + ) + return null; + const coordinates = feature?.geometry?.coordinates; + if (!Array.isArray(coordinates) || coordinates.length < 2) continue; + const [lon, lat] = coordinates.map(Number); + if (!Number.isFinite(lon) || !Number.isFinite(lat)) continue; + if (Math.abs(lat) > 90 || Math.abs(lon) > 180) return null; + const id = String(properties.location_id ?? feature.id ?? '').trim(); + if (!id) continue; + stations.set(id, { + id, + name: String(properties.location_name_en || id).trim(), + // ECCC's administrative zone: atl, que, ont, pnr, pyr. + zone: String(properties['eccc_administrative-zone'] || '').trim(), + lat, + lon, + }); + } + return stations; +} + +/** + * Join latest ECCC observations onto known stations into provider-shaped rows. + * + * The catalog is the only thing that supplies a reading its coordinates, so an + * observation without a matching station cannot be placed and is dropped. + * Readings older than OBSERVATION_MAX_AGE_MS are dropped too — a six-hour-old + * air-quality number presented as current is worse than no number. + * + * @param {object} geojson - FeatureCollection from `aqhi-observations-realtime`. + * @param {Map} stations - Known stations by location_id. + * @param {number} [now=Date.now()] - Clock seam for tests. + * @returns {Array|null} Normalized readings, or null when malformed. + */ +export function normalizeAqhiObservations(geojson, stations, now = Date.now()) { + if (!Array.isArray(geojson?.features)) return null; + if (!(stations instanceof Map)) return null; + const provider = AIR_QUALITY_PROVIDERS.eccc; + const newest = new Map(); + for (const feature of geojson.features) { + const properties = feature?.properties; + if ( + !properties || + typeof properties !== 'object' || + Array.isArray(properties) + ) + return null; + const stationId = String(properties.location_id ?? '').trim(); + const station = stations.get(stationId); + if (!station) continue; + + const value = aqhiDisplayValue(properties.aqhi); + if (value === null) continue; + const observedMs = Date.parse(properties.observation_datetime); + if (!Number.isFinite(observedMs)) continue; + if (now - observedMs > OBSERVATION_MAX_AGE_MS) continue; + + const previous = newest.get(stationId); + if (previous && previous.observedMs >= observedMs) continue; + const band = bandForValue(provider.id, value); + newest.set(stationId, { + id: `${provider.id}:${stationId}`, + provider: provider.id, + scale: provider.scale, + value, + band, + stationId, + name: station.name, + zone: station.zone, + lat: station.lat, + lon: station.lon, + observedMs, + color: airQualityColor(band), + label: valueLabel(provider.id, value), + risk: bandRiskText(band), + note: String(properties.special_notes_en || '').trim() || null, + }); + } + return [...newest.values()].sort( + (a, b) => b.value - a.value || a.stationId.localeCompare(b.stationId), + ); +} + +/** + * Build the source-owned presentation for one station label. + * @param {object} input + * @returns {object} + */ +export function createAirQualityOverlayEntry({ + id, + position, + title, + accent, + value, +}) { + return { + id: String(id), + position, + variant: 'label', + title, + accent, + // Worse air wins the label budget — the reading people need to see first. + priority: Math.round(Number(value) || 0) * 1000, + collisionGroup: 'ambient-label', + paintLane: 'ambient-label', + interactive: false, + edgeFade: 'keyhole', + horizonCull: true, + terrainOcclusion: false, + gapPx: 14, + verticalOnly: true, + placement: 'above', + }; +} + +/** Keep the worst readings, with stable identity as the tie-break. */ +export function selectAirQualityOverlayCohort( + entries, + limit = AIR_QUALITY_OVERLAY_COHORT_LIMIT, +) { + const cap = Math.max( + 0, + Math.min(AIR_QUALITY_OVERLAY_COHORT_LIMIT, Math.floor(Number(limit) || 0)), + ); + if (!Array.isArray(entries) || cap === 0) return []; + return entries + .slice() + .sort( + (a, b) => + b.priority - a.priority || String(a.id).localeCompare(String(b.id)), + ) + .slice(0, cap); +} + +/** + * The worst readings inside a lon/lat rectangle, for the readout card. + * + * A null rectangle means the camera could not resolve one (a fully oblique or + * space-level view); everything is then in scope rather than nothing, so the + * card degrades to the worst readings overall instead of going blank. + * + * @param {Array} readings + * @param {?{west:number, south:number, east:number, north:number}} rect - Degrees. + * @param {number} [limit=5] + * @returns {Array} + */ +export function readingsInView(readings, rect, limit = 5) { + if (!Array.isArray(readings)) return []; + const cap = Math.max(0, Math.floor(Number(limit) || 0)); + if (cap === 0) return []; + const inside = !rect + ? readings + : readings.filter((reading) => { + if (reading.lat < rect.south || reading.lat > rect.north) return false; + // A rectangle crossing the antimeridian has west > east. + return rect.west <= rect.east + ? reading.lon >= rect.west && reading.lon <= rect.east + : reading.lon >= rect.west || reading.lon <= rect.east; + }); + return inside + .slice() + .sort((a, b) => b.value - a.value || a.stationId.localeCompare(b.stationId)) + .slice(0, cap); +} + +/** + * Map one reading to a JSON-safe analyst record. Pure — no Cesium types. + * @param {object|null|undefined} raw + * @param {number} [index=0] + * @returns {object} + */ +export function mapAnalystRecord(raw, index = 0) { + const num = (v) => (Number.isFinite(v) ? v : null); + const text = (v) => { + const t = String(v ?? '').trim(); + return t || null; + }; + return { + id: text(raw?.id) || `AIR-${String(index).padStart(4, '0')}`, + provider: text(raw?.provider), + scale: text(raw?.scale), + value: num(raw?.value), + band: text(raw?.band), + name: text(raw?.name), + zone: text(raw?.zone), + risk: text(raw?.risk), + lat: num(raw?.lat), + lon: num(raw?.lon), + observedMs: num(raw?.observedMs), + }; +} diff --git a/src/layers/airQuality/model.test.mjs b/src/layers/airQuality/model.test.mjs new file mode 100644 index 000000000..56130c30e --- /dev/null +++ b/src/layers/airQuality/model.test.mjs @@ -0,0 +1,250 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + AIR_QUALITY_BANDS, + AIR_QUALITY_PROVIDERS, + AIR_QUALITY_UNKNOWN_COLOR, + OBSERVATION_MAX_AGE_MS, + airQualityColor, + aqhiDisplayValue, + bandForValue, + bandRiskText, + mapAnalystRecord, + normalizeAqhiObservations, + normalizeAqhiStations, + readingsInView, + selectAirQualityOverlayCohort, + valueLabel, +} from './model.js'; + +const NOW = Date.parse('2026-09-24T18:30:00Z'); +const RECENT = '2026-09-24T18:00:00Z'; +const collection = (features) => ({ type: 'FeatureCollection', features }); +const station = (id, name, zone, lon, lat) => ({ + type: 'Feature', + geometry: { type: 'Point', coordinates: [lon, lat] }, + properties: { + location_id: id, + location_name_en: name, + 'eccc_administrative-zone': zone, + }, +}); +const observation = (id, aqhi, when = RECENT) => ({ + type: 'Feature', + properties: { location_id: id, aqhi, observation_datetime: when }, +}); + +const STATIONS = collection([ + station('IAKID', 'Calgary', 'pnr', -114.0575, 51.0458), + station('OAAAA', 'Toronto', 'ont', -79.3832, 43.6532), + station('AAAAA', 'Halifax', 'atl', -63.5752, 44.6488), +]); + +// ── provider-shaped records ──────────────────────────────────────────────── + +test('readings carry the provider-neutral {provider, scale, value, band} shape', () => { + // Indices are not interchangeable numbers: AQHI is 1-10+ on health risk, + // US AQI is 0-500 on concentration. The shape names which index a value is on + // so a second network can be added without inventing a shared meaning. + const stations = normalizeAqhiStations(STATIONS); + const [row] = normalizeAqhiObservations( + collection([observation('IAKID', 2.32)]), + stations, + NOW, + ); + assert.equal(row.provider, 'eccc'); + assert.equal(row.scale, 'AQHI'); + assert.equal(row.value, 2); + assert.equal(row.band, 'low'); +}); + +test('bands map from a value on a named provider scale', () => { + assert.equal(bandForValue('eccc', 1), 'low'); + assert.equal(bandForValue('eccc', 3), 'low'); + assert.equal(bandForValue('eccc', 4), 'moderate'); + assert.equal(bandForValue('eccc', 7), 'high'); + assert.equal(bandForValue('eccc', 11), 'very-high'); + // An unregistered provider cannot be banded rather than being guessed at. + assert.equal(bandForValue('unknown-network', 5), null); + assert.equal(bandForValue('eccc', null), null); +}); + +test('the provider states its coverage plainly', () => { + assert.match(AIR_QUALITY_PROVIDERS.eccc.coverage, /Canada only/); + assert.equal(AIR_QUALITY_PROVIDERS.eccc.scale, 'AQHI'); +}); + +test("values above ten collapse to the scale's open-ended label", () => { + assert.equal(valueLabel('eccc', 10), '10'); + assert.equal(valueLabel('eccc', 11), '10+'); + assert.equal(valueLabel('eccc', null), '--'); +}); + +test('every band has a distinct colour, and an unknown band is neutral', () => { + const colors = AIR_QUALITY_BANDS.map((b) => b.color); + assert.equal(new Set(colors).size, colors.length); + assert.equal(airQualityColor('nonsense'), AIR_QUALITY_UNKNOWN_COLOR); + assert.equal(bandRiskText('high'), 'High health risk'); + assert.equal(bandRiskText(null), null); +}); + +// ── published form ───────────────────────────────────────────────────────── + +test('AQHI rounds to the published integer form and floors at 1', () => { + assert.equal(aqhiDisplayValue(1.08), 1); + assert.equal(aqhiDisplayValue(3.5), 4); + assert.equal(aqhiDisplayValue(0), 1); + assert.equal(aqhiDisplayValue('3'), 3); +}); + +test('an absent reading never becomes a confident AQHI 1', () => { + // Number(null) and Number('') are both 0, which would floor to 1. + for (const empty of [null, undefined, '', ' ', NaN, true, {}, [], -1]) { + assert.equal(aqhiDisplayValue(empty), null, `${JSON.stringify(empty)}`); + } +}); + +// ── snapshot validation ──────────────────────────────────────────────────── + +test('stations normalize into an id-keyed catalog carrying their zone', () => { + const stations = normalizeAqhiStations(STATIONS); + assert.equal(stations.size, 3); + assert.equal(stations.get('OAAAA').zone, 'ont'); +}); + +test('malformed payloads are rejected atomically', () => { + assert.equal(normalizeAqhiStations(null), null); + assert.equal(normalizeAqhiStations(collection([{ properties: null }])), null); + const stations = normalizeAqhiStations(STATIONS); + assert.equal(normalizeAqhiObservations(null, stations, NOW), null); + assert.equal( + normalizeAqhiObservations(collection([]), 'not a map', NOW), + null, + ); +}); + +test('an observation with no matching station is dropped', () => { + const stations = normalizeAqhiStations(STATIONS); + const rows = normalizeAqhiObservations( + collection([observation('IAKID', 2), observation('ZZZZZ', 9)]), + stations, + NOW, + ); + assert.equal(rows.length, 1); +}); + +test('a stale reading is dropped rather than shown as current', () => { + const stations = normalizeAqhiStations(STATIONS); + const old = new Date(NOW - OBSERVATION_MAX_AGE_MS - 60_000).toISOString(); + const rows = normalizeAqhiObservations( + collection([observation('IAKID', 3, old), observation('OAAAA', 4)]), + stations, + NOW, + ); + assert.deepEqual( + rows.map((r) => r.stationId), + ['OAAAA'], + ); +}); + +test('the newest reading per station wins and worst air sorts first', () => { + const stations = normalizeAqhiStations(STATIONS); + const rows = normalizeAqhiObservations( + collection([ + observation('IAKID', 3, '2026-09-24T16:00:00Z'), + observation('IAKID', 7, '2026-09-24T18:00:00Z'), + observation('OAAAA', 5), + ]), + stations, + NOW, + ); + assert.deepEqual( + rows.map((r) => r.value), + [7, 5], + ); +}); + +// ── readout selection ────────────────────────────────────────────────────── + +const reading = (id, value, lat, lon) => ({ + id, + stationId: id, + value, + lat, + lon, + name: id, +}); + +test('the readout lists the worst readings inside the view rectangle', () => { + const rows = [ + reading('west', 9, 51.0, -114.0), + reading('east', 4, 43.6, -79.3), + reading('mild', 2, 51.1, -114.1), + ]; + const inView = readingsInView( + rows, + { west: -115, south: 50, east: -113, north: 52 }, + 5, + ); + assert.deepEqual( + inView.map((r) => r.id), + ['west', 'mild'], + ); +}); + +test('an unresolvable view rectangle degrades to the worst overall, not to blank', () => { + // A space-level or fully oblique camera cannot produce a rectangle; the card + // should still say something useful. + const rows = [reading('a', 3, 0, 0), reading('b', 8, 10, 10)]; + assert.deepEqual( + readingsInView(rows, null, 5).map((r) => r.id), + ['b', 'a'], + ); + assert.deepEqual(readingsInView(rows, null, 0), []); + assert.deepEqual(readingsInView(null, null, 5), []); +}); + +test('a view rectangle crossing the antimeridian still selects correctly', () => { + // west > east is how a dateline-crossing rectangle is expressed. + const rows = [reading('near', 5, 0, 179), reading('far', 9, 0, 100)]; + const inView = readingsInView( + rows, + { west: 170, south: -10, east: -170, north: 10 }, + 5, + ); + assert.deepEqual( + inView.map((r) => r.id), + ['near'], + ); +}); + +test('the overlay cohort is capped and deterministic', () => { + const entries = Array.from({ length: 120 }, (_, i) => ({ + id: `s${i}`, + priority: i % 11, + })); + const cohort = selectAirQualityOverlayCohort(entries); + assert.ok(cohort.length <= 48); + assert.deepEqual( + cohort.map((e) => e.id), + selectAirQualityOverlayCohort(entries).map((e) => e.id), + ); +}); + +test('analyst records are JSON-safe and name the provider and scale', () => { + const stations = normalizeAqhiStations(STATIONS); + const [row] = normalizeAqhiObservations( + collection([observation('IAKID', 2)]), + stations, + NOW, + ); + const record = mapAnalystRecord(row, 0); + assert.deepEqual(JSON.parse(JSON.stringify(record)), record); + assert.equal(record.provider, 'eccc'); + assert.equal(record.scale, 'AQHI'); + const empty = mapAnalystRecord(undefined, 7); + assert.equal(empty.id, 'AIR-0007'); + for (const [key, value] of Object.entries(empty)) { + assert.notEqual(value, undefined, `${key} must not be undefined`); + } +}); diff --git a/src/layers/airQuality/ownership.test.mjs b/src/layers/airQuality/ownership.test.mjs new file mode 100644 index 000000000..5eeaae0e1 --- /dev/null +++ b/src/layers/airQuality/ownership.test.mjs @@ -0,0 +1,326 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createAirQualityLayer } from './index.js'; +import { createEcccAirQualitySource } from './source.js'; +import { layerFeedState } from '../../data/feedState.js'; +import { STALE_AFTER_MS } from './model.js'; + +function harness(source) { + const sources = []; + const events = []; + const viewer = { + dataSources: { + add(value) { + sources.push(value); + }, + remove(value) { + sources.splice(sources.indexOf(value), 1); + }, + }, + }; + const layer = createAirQualityLayer({ + source, + overlayHost: { + setEntries(...args) { + events.push(args); + }, + setVisible() {}, + clearSource() {}, + }, + }); + layer.init(viewer); + layer.enable(viewer); + return { layer, viewer, sources, events }; +} + +const reading = (id, value, overrides = {}) => ({ + id: `eccc:${id}`, + provider: 'eccc', + scale: 'AQHI', + value, + band: 'low', + stationId: id, + name: id, + zone: 'ont', + lat: 43.65, + lon: -79.38, + observedMs: 1_700_000_000_000, + color: '#4cc9f0', + label: String(value), + risk: 'Low health risk', + note: null, + ...overrides, +}); + +const staticSource = (rows) => ({ getSnapshot: async () => rows }); + +test('a layer requires both a snapshot source and an overlay host', () => { + assert.throws(() => createAirQualityLayer(), /snapshot source/); + assert.throws( + () => createAirQualityLayer({ source: staticSource([]) }), + /overlay host/, + ); +}); + +test('initialization is single-use and owns exactly one data source', () => { + const { layer, viewer, sources } = harness(staticSource([])); + assert.equal(sources.length, 1); + assert.throws(() => layer.init(viewer), /already initialized/); + layer.destroy(viewer); + assert.equal(sources.length, 0); +}); + +test('a refresh renders readings and publishes an overlay cohort', async () => { + const { layer, events } = harness( + staticSource([reading('A', 2), reading('B', 7)]), + ); + assert.equal(await layer.update(), true); + assert.equal(layer.getStats().count, 2); + assert.equal(layer.getStats().error, null); + assert.equal(events.at(-1)[0], 'air-quality'); +}); + +// ── the abort + enabled guard ────────────────────────────────────────────── + +test('a response landing after disable cannot redraw the scene', async () => { + // The failure this prevents: disable() clears the overlay, then an in-flight + // response resolves and repopulates it behind the user's back. + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const { layer, events } = harness({ + getSnapshot: async ({ signal } = {}) => { + await gate; + signal?.throwIfAborted(); + return [reading('A', 5)]; + }, + }); + const pending = layer.update(); + layer.disable(); + const publishedBefore = events.length; + release(); + assert.equal(await pending, false, 'a late response must not be applied'); + assert.equal( + events.length, + publishedBefore, + 'no overlay write after disable', + ); + assert.equal(layer.getStats().count, 0); +}); + +test('disable aborts the request in flight', async () => { + let seenSignal = null; + const { layer } = harness({ + getSnapshot: async ({ signal } = {}) => { + seenSignal = signal; + await new Promise((resolve) => setTimeout(resolve, 40)); + signal?.throwIfAborted(); + return []; + }, + }); + const pending = layer.update(); + layer.disable(); + await pending; + assert.ok(seenSignal, 'a signal was passed through to the source'); + assert.equal(seenSignal.aborted, true); +}); + +test('a superseded refresh does not clobber the newer one', async () => { + let call = 0; + const { layer } = harness({ + getSnapshot: async ({ signal } = {}) => { + call += 1; + const mine = call; + await new Promise((resolve) => setTimeout(resolve, mine === 1 ? 40 : 1)); + signal?.throwIfAborted(); + return [reading(`call-${mine}`, mine)]; + }, + }); + const first = layer.update(); + const second = layer.update(); + const [firstResult, secondResult] = await Promise.all([first, second]); + assert.equal(firstResult, false, 'the superseded call reports no update'); + assert.equal(secondResult, true); + assert.equal(layer.getAnalystRecords()[0].name, 'call-2'); +}); + +test('update is a no-op while the layer is disabled', async () => { + const { layer } = harness(staticSource([reading('A', 2)])); + layer.disable(); + assert.equal(await layer.update(), false); + assert.equal(layer.getStats().count, 0); +}); + +// ── readout descriptor ───────────────────────────────────────────────────── + +test('the readout descriptor carries the band legend and coverage', async () => { + const { layer } = harness(staticSource([reading('A', 2), reading('B', 9)])); + await layer.update(); + const controls = layer.getRowControls(); + assert.equal(controls.readout, true); + assert.match(controls.summary.coverage, /Canada only/); + assert.equal(controls.legend.categorical, true); + assert.deepEqual(controls.legend.labels, [ + 'Low', + 'Moderate', + 'High', + 'Very High', + ]); + assert.equal(controls.legend.colors.length, controls.legend.labels.length); + // Worst first, and every item keyed. + assert.deepEqual( + controls.list.items.map((i) => i.label), + ['B', 'A'], + ); + assert.ok(controls.list.items.every((i) => i.id)); + assert.match(controls.summary.status, /2 stations reporting/); +}); + +test('the readout reports an error rather than a silent empty card', async () => { + const { layer } = harness({ + getSnapshot: async () => { + throw new Error('ECCC HTTP 503'); + }, + }); + await layer.update(); + assert.match(layer.getRowControls().summary.status, /503/); +}); + +test('row-control listeners are notified on refresh and teardown-safe', async () => { + const { layer, viewer } = harness(staticSource([reading('A', 2)])); + let calls = 0; + layer.setRowControlsListener(() => { + calls += 1; + }); + await layer.update(); + assert.ok(calls >= 1, 'a refresh notifies the card'); + layer.destroy(viewer); + const after = calls; + layer.setRowControlsListener(() => { + calls += 1; + }); + assert.equal(calls, after); +}); + +// ── source contract ──────────────────────────────────────────────────────── + +test('the source rejects a malformed station catalog and surfaces HTTP errors', async () => { + await assert.rejects( + () => + createEcccAirQualitySource({ + fetchImpl: async () => ({ + ok: true, + json: async () => ({ features: 'no' }), + }), + }).getSnapshot(), + /Malformed AQHI station/, + ); + await assert.rejects( + () => + createEcccAirQualitySource({ + fetchImpl: async () => ({ + ok: false, + status: 503, + json: async () => ({}), + }), + }).getSnapshot(), + /ECCC HTTP 503/, + ); +}); + +test('the station catalog is cached and observations ask for latest values only', async () => { + const urls = []; + const source = createEcccAirQualitySource({ + fetchImpl: async (url) => { + urls.push(url); + return { + ok: true, + json: async () => + url.includes('aqhi-stations') + ? { + features: [ + { + geometry: { type: 'Point', coordinates: [-79.38, 43.65] }, + properties: { location_id: 'A', location_name_en: 'A' }, + }, + ], + } + : { features: [] }, + }; + }, + now: () => 1_700_000_000_000, + }); + await source.getSnapshot(); + await source.getSnapshot(); + assert.equal(urls.filter((u) => u.includes('aqhi-stations')).length, 1); + // Without latest=true the collection returns the full history (~2,000 rows). + assert.match( + urls.find((u) => u.includes('observations')), + /latest=true/, + ); +}); + +// ── feed-state honesty ───────────────────────────────────────────────────── + +test('freshness is judged on observation time, not on when we last fetched', async () => { + // lastUpdate stays current every time the network answers, even when it + // answers with nothing new. Judging freshness on it would let the HUD and + // voice describe hours-old air quality as live. + const aged = Date.now() - STALE_AFTER_MS - 60_000; + const { layer } = harness( + staticSource([reading('A', 2, { observedMs: aged })]), + ); + await layer.update(); + const stats = layer.getStats(); + assert.ok( + stats.lastUpdate >= Date.now() - 5_000, + 'we did just receive a response', + ); + assert.equal(stats.observedAt, aged, 'but the observation itself is old'); + assert.equal(stats.stale, true); + assert.equal( + layerFeedState(stats), + 'stale', + 'provenance must not read this as nominal', + ); +}); + +test('a current reading reports nominal', async () => { + const { layer } = harness( + staticSource([reading('A', 2, { observedMs: Date.now() - 10 * 60_000 })]), + ); + await layer.update(); + const stats = layer.getStats(); + assert.equal(stats.stale, false); + assert.equal(layerFeedState(stats), 'nominal'); +}); + +test('an empty feed is not called stale', async () => { + // Nothing to be stale about; the chip should not invent a fault. + const { layer } = harness(staticSource([])); + await layer.update(); + assert.equal(layer.getStats().stale, false); + assert.equal(layer.getStats().observedAt, null); +}); + +test('a failed refresh over good readings reports degraded, not nominal', async () => { + let fail = false; + const { layer } = harness({ + getSnapshot: async () => { + if (fail) throw new Error('ECCC HTTP 503'); + return [reading('A', 2, { observedMs: Date.now() - 10 * 60_000 })]; + }, + }); + await layer.update(); + assert.equal(layerFeedState(layer.getStats()), 'nominal'); + fail = true; + await layer.update(); + const stats = layer.getStats(); + assert.ok(stats.count > 0, 'the previous readings are still displayed'); + assert.equal( + layerFeedState(stats), + 'degraded', + 'showing old data after a failure is degraded', + ); +}); diff --git a/src/layers/airQuality/source.js b/src/layers/airQuality/source.js new file mode 100644 index 000000000..b5f98894c --- /dev/null +++ b/src/layers/airQuality/source.js @@ -0,0 +1,78 @@ +import { normalizeAqhiStations, normalizeAqhiObservations } from './model.js'; + +const COLLECTIONS_ROOT = 'https://api.weather.gc.ca/collections'; +const STATIONS_URL = `${COLLECTIONS_ROOT}/aqhi-stations/items`; +const OBSERVATIONS_URL = `${COLLECTIONS_ROOT}/aqhi-observations-realtime/items`; + +/** Comfortably above Canada's ~134 stations without inviting a huge page. */ +const STATION_LIMIT = 500; +const OBSERVATION_LIMIT = 500; +/** Station geometry is near-static — a few changes a year. Refetch twice a day. */ +const STATION_TTL_MS = 12 * 60 * 60 * 1000; + +/** + * Request and validate a complete ECCC AQHI snapshot before it can replace + * displayed readings. The first registered air-quality provider. + * + * Two collections are involved. `aqhi-stations` is the catalog — it supplies + * every reading its coordinates and is cached for twelve hours because it + * barely changes. `aqhi-observations-realtime` is the hourly reading feed, + * requested with `latest=true` so the response is current values rather than + * the full observation history, which is roughly 2,000 rows nationally. + * + * The station catalog is only replaced when a refresh actually returns + * stations, so a transient empty response cannot blank the layer. + * + * @param {object} [options] + * @param {Function} [options.fetchImpl] - Injected fetch, for tests. + * @param {() => number} [options.now] - Clock seam, for tests. + * @returns {{getSnapshot: Function}} + */ +export function createEcccAirQualitySource({ + fetchImpl = (...args) => globalThis.fetch(...args), + now = () => Date.now(), +} = {}) { + /** @type {Map} */ + let stations = new Map(); + let stationsFetchedAt = 0; + + const getJson = async (url, signal) => { + signal?.throwIfAborted(); + const response = await fetchImpl(url, { + headers: { Accept: 'application/json' }, + signal, + }); + if (!response.ok) throw new Error(`ECCC HTTP ${response.status}`); + const payload = await response.json(); + signal?.throwIfAborted(); + return payload; + }; + + return { + async getSnapshot({ signal } = {}) { + const clock = now(); + if (stations.size === 0 || clock - stationsFetchedAt > STATION_TTL_MS) { + const payload = await getJson( + `${STATIONS_URL}?f=json&limit=${STATION_LIMIT}`, + signal, + ); + const next = normalizeAqhiStations(payload); + if (!next) throw new Error('Malformed AQHI station response'); + // An empty refresh must not wipe a good catalog. + if (next.size > 0) { + stations = next; + stationsFetchedAt = clock; + } + } + if (stations.size === 0) throw new Error('No AQHI stations available'); + + const payload = await getJson( + `${OBSERVATIONS_URL}?f=json&latest=true&limit=${OBSERVATION_LIMIT}`, + signal, + ); + const readings = normalizeAqhiObservations(payload, stations, clock); + if (!readings) throw new Error('Malformed AQHI observation response'); + return readings; + }, + }; +} diff --git a/src/overlays/worldOverlayAllocation.test.mjs b/src/overlays/worldOverlayAllocation.test.mjs index bbaa4f168..e29785d48 100644 --- a/src/overlays/worldOverlayAllocation.test.mjs +++ b/src/overlays/worldOverlayAllocation.test.mjs @@ -11,6 +11,7 @@ import { EARTHQUAKE_OVERLAY_COHORT_LIMIT } from '../data/earthquakes.js'; import { ROCKET_MISSION_AMBIENT_OVERLAY_COHORT_LIMIT } from '../data/rocketLaunches.js'; import { RADIO_OVERLAY_COHORT_LIMIT } from '../data/radio.js'; import { CABLE_REFERENCE_LABEL_WINNER_CAP } from '../data/telegeographySubmarineCables.js'; +import { AIR_QUALITY_OVERLAY_COHORT_LIMIT } from '../layers/airQuality/model.js'; import { isCalibratedAllocationRuntime } from '../../scripts/run-unit-tests.mjs'; /** @@ -34,6 +35,7 @@ import { isCalibratedAllocationRuntime } from '../../scripts/run-unit-tests.mjs' * ---------------------+---------+------------+---------+----------------+-------------+------------- * generic below cap | 60 | 60 | 60 | 3182 | 53.0 | 4,100 * generic above cap | 250 | 250 | 96 | 10022 | 40.1 | 13,000 + * air quality | 48 | 48 | 32 | 3900 | 81.3 | 5,200 * local infrastructure | 320 | 320 | 192 | 39478 | 123.4 | 49,000 * Phase 3 + FIRMS | 338 | 338 | 210 | 41220 | 122.0 | 53,600 * Phase 3 + vessels | 451 | 451 | 311 | 67252 | 149.1 | 87,500 @@ -259,6 +261,23 @@ const WORKLOADS = [ maxBytesPerFrame: 19_000, saturated: true, }, + { + // The air-quality station cohort on its own. Measured standalone rather + // than stacked on the Phase-5 host (the submarine-cable precedent): these + // stations are sparse and, with the single registered provider, + // continental, so what matters is its own cohort cost. + // + // Re-measured against the current worker: median 3,900 B/frame, max 4,114, + // and 85.7 B/candidate at the ceiling — inside the shared 154 B/candidate + // bound, so no per-source exception is taken. The 5,200 frame budget + // carries 26.4% headroom over the measured maximum. + name: 'with the air-quality station cohort live', + profile: 'air-quality', + entries: AIR_QUALITY_OVERLAY_COHORT_LIMIT, + candidates: AIR_QUALITY_OVERLAY_COHORT_LIMIT, + maxBytesPerFrame: 5_200, + saturated: true, + }, { name: 'with the Dense detection lane active over 5,000 observations', profile: 'phase6-detection', diff --git a/src/overlays/worldOverlayAllocation.worker.mjs b/src/overlays/worldOverlayAllocation.worker.mjs index ea463f080..0314ec41d 100644 --- a/src/overlays/worldOverlayAllocation.worker.mjs +++ b/src/overlays/worldOverlayAllocation.worker.mjs @@ -26,6 +26,12 @@ import { TRACKED_OVERLAY_SOURCE_OPTIONS, } from '../data/trackedReadout.js'; import { CCTV_AMBIENT_CARD_MAX } from '../data/cctvLod.js'; +import { + AIR_QUALITY_OVERLAY_COHORT_LIMIT, + AIR_QUALITY_OVERLAY_COLLISION_CAPACITY, + AIR_QUALITY_OVERLAY_SOURCE_ID, + createAirQualityOverlayEntry, +} from '../layers/airQuality/model.js'; import { CCTV_OVERLAY_SOURCE_ID, createCctvThumbnailOverlayEntry, @@ -1101,6 +1107,68 @@ function buildSubmarineCablesWorkload(count) { ); } +/** + * The air-quality station cohort at full size. + * + * Measured standalone (the submarine-cable precedent) rather than stacked on + * the Phase-5 host: air-quality stations are sparse and, with the single + * currently-registered provider, continental — so the question a budget has to + * answer is what its own cohort costs, not what it adds to a worldwide worst + * case it never shares a viewport with. The entry shape is the same static + * `ambient-label` the earthquake layer publishes, so the per-candidate ceiling + * is the meaningful gate. + */ +function buildAirQualityWorkload(count) { + if (count !== AIR_QUALITY_OVERLAY_COHORT_LIMIT) { + throw new Error( + `air-quality requires ${AIR_QUALITY_OVERLAY_COHORT_LIMIT} entries`, + ); + } + const workload = { + entries: [], + positions: [], + drifts: [], + registrations: [], + }; + const columns = Math.max(1, Math.ceil(Math.sqrt((count * 16) / 9))); + const rows = Math.max(1, Math.ceil(count / columns)); + const spanX = columns > 1 ? 1.68 / (columns - 1) : 0; + const spanY = rows > 1 ? 1.56 / (rows - 1) : 0; + const stations = []; + for (let index = 0; index < count; index++) { + const column = index % columns; + const row = Math.floor(index / columns); + const baseX = -0.84 + column * spanX; + const baseY = -0.78 + row * spanY; + const position = new Cesium.Cartesian3(baseX, baseY, 0); + workload.positions.push(position); + workload.drifts.push({ + baseX, + baseY, + phase: index * 0.27, + rate: 0.32 + (index % 6) * 0.04, + }); + const entry = createAirQualityOverlayEntry({ + id: `air-${index}`, + position, + title: `Station ${index} ${1 + (index % 10)}`, + accent: '#4cc9f0', + value: 1 + (index % 10), + }); + entry.horizonCull = false; + stations.push(entry); + } + workload.registrations.push({ + sourceId: AIR_QUALITY_OVERLAY_SOURCE_ID, + entries: stations, + options: { + cohortLimit: AIR_QUALITY_OVERLAY_COHORT_LIMIT, + collisionCapacity: AIR_QUALITY_OVERLAY_COLLISION_CAPACITY, + }, + }); + return workload; +} + function buildPhase6DetectionWorkload(count) { const random = makeRandom(0xd37ec710); const positions = []; @@ -1206,9 +1274,13 @@ function main() { ? buildPhase5RocketMissionWorkload(ENTRY_COUNT) : PROFILE === 'all-live-radio' ? buildAllLiveRadioWorkload(ENTRY_COUNT) - : PROFILE === 'submarine-cables' - ? buildSubmarineCablesWorkload(ENTRY_COUNT) - : buildWorkload(ENTRY_COUNT); + : PROFILE === 'air-quality' + ? buildAirQualityWorkload(ENTRY_COUNT) + : PROFILE === 'submarine-cables' + ? buildSubmarineCablesWorkload( + ENTRY_COUNT, + ) + : buildWorkload(ENTRY_COUNT); const { entries, positions, drifts } = workload; const solveIntervalMs = Number(process.env.GEV_ALLOC_SOLVE_MS) || 125; const detectionActive = !!workload.detectionLayer; diff --git a/src/sources/reference.js b/src/sources/reference.js index 4bdf7e9ac..34c776164 100644 --- a/src/sources/reference.js +++ b/src/sources/reference.js @@ -1,4 +1,5 @@ import { createUsgsEarthquakeSource } from '../layers/earthquakes/source.js'; +import { createEcccAirQualitySource } from '../layers/airQuality/source.js'; import { createWfigsPerimeterSource } from '../layers/perimeters/source.js'; import { createBundledCableSource } from '../layers/submarineCables/bundledSource.js'; @@ -6,6 +7,7 @@ import { createBundledCableSource } from '../layers/submarineCables/bundledSourc export function createReferenceSources() { return { earthquakes: createUsgsEarthquakeSource(), + 'air-quality': createEcccAirQualitySource(), 'fire-perimeters': createWfigsPerimeterSource(), cables: createBundledCableSource(), }; diff --git a/src/sources/reference.test.mjs b/src/sources/reference.test.mjs index 902a72c83..285ee6d66 100644 --- a/src/sources/reference.test.mjs +++ b/src/sources/reference.test.mjs @@ -14,13 +14,16 @@ test('reference factories retain compatibility without starting acquisition or s const second = createReferenceSources(); assert.deepEqual(Object.keys(first), [ 'earthquakes', + 'air-quality', 'fire-perimeters', 'cables', ]); assert.notEqual(first.earthquakes, second.earthquakes); + assert.notEqual(first['air-quality'], second['air-quality']); assert.notEqual(first['fire-perimeters'], second['fire-perimeters']); assert.notEqual(first.cables, second.cables); assert.equal(typeof first.earthquakes.getSnapshot, 'function'); + assert.equal(typeof first['air-quality'].getSnapshot, 'function'); assert.equal(typeof first['fire-perimeters'].getSnapshot, 'function'); assert.equal(typeof first.cables.fetch, 'function'); assert.equal(requests, 0); diff --git a/src/ui/layerPanel.js b/src/ui/layerPanel.js index 57fd595f0..81ced6cf8 100644 --- a/src/ui/layerPanel.js +++ b/src/ui/layerPanel.js @@ -51,6 +51,7 @@ const PANEL_GROUPS = [ { label: 'Weather', ids: [ + 'air-quality', 'wind', 'weather-radar', 'weather-satellite', From 7485b16ded0158b804f1ac07da7e25b71eae0d5d Mon Sep 17 00:00:00 2001 From: daikaginza Date: Mon, 5 Oct 2026 17:12:41 +0700 Subject: [PATCH 26/95] fix(overpass): do not serve cached answers from a different upstream (#932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both disk caches fed by OVERPASS_UPSTREAMS key only on the query — the Overpass proxy on the sanitized request body, the installations proxy on the bbox — so an entry written against one instance is a hit for the same query against another. After an operator repoints OVERPASS_UPSTREAMS, the previous upstream's answers keep being served for 7 days, 30 for boundary-class queries, and indefinitely through the serve-stale path. An install that predates #742 has a cache written from the public instances that were the default then, and configuring your own instance does not displace it. Cached answers now carry a tag identifying the configured upstream set, and a disk read refuses an entry that does not match it, on the stale path too. With nothing configured any entry is still accepted: there is no instance to contradict, and serving last-good data at any age is what the not-configured path exists for. The tag is a hash of the resolved endpoint list, never the URLs; it is written to disk and compared on read, and is not sent to clients or surfaced in errors. One line is logged once per process when a foreign entry is ignored, naming the variable and what will happen, as local-receivers.js already does for a rejected feed. Cost worth naming: on upgrade every existing entry is untagged, so the first outage after upgrading serves nothing where it previously served stale data, until each query is fetched once from the configured instance. --- server/providers/military-installations.js | 13 +++- .../providers/military-installations/cache.js | 4 ++ server/providers/overpass/cache.js | 6 ++ server/providers/overpass/constants.js | 63 +++++++++++++++++-- server/providers/overpass/transport.js | 4 ++ src/overpassProxy.test.mjs | 52 ++++++++++++++- 6 files changed, 135 insertions(+), 7 deletions(-) diff --git a/server/providers/military-installations.js b/server/providers/military-installations.js index 08ad87d9f..7d252b8ed 100644 --- a/server/providers/military-installations.js +++ b/server/providers/military-installations.js @@ -9,7 +9,10 @@ import { fetchOverpassPayload, overpassNotConfigured, } from './overpass/transport.js'; -import { resolveOverpassUpstreams } from './overpass/constants.js'; +import { + resolveOverpassUpstreams, + resolveOverpassUpstreamTag, +} from './overpass/constants.js'; import { _militaryInstallationCache, trimMilitaryInstallationCache, @@ -74,7 +77,13 @@ function militaryInstallationsProxy() { retrievedAt: new Date().toISOString(), status: 'ready', }; - const entry = { payload, cachedAt: Date.now() }; + // Tagged with the configured upstream set so the disk layer does not serve + // this answer back after the operator repoints (see overpass/cache.js). + const entry = { + payload, + cachedAt: Date.now(), + upstreams: resolveOverpassUpstreamTag(), + }; _militaryInstallationCache.set(key, entry); trimMilitaryInstallationCache(); writeMilitaryInstallationDisk(key, entry); diff --git a/server/providers/military-installations/cache.js b/server/providers/military-installations/cache.js index c77627e29..4d77981e6 100644 --- a/server/providers/military-installations/cache.js +++ b/server/providers/military-installations/cache.js @@ -5,6 +5,7 @@ import { MILITARY_INSTALLATION_DISK_DIR, MILITARY_INSTALLATION_MAX_CACHE, } from './constants.js'; +import { cachedByConfiguredUpstream } from '../overpass/constants.js'; import path from 'node:path'; import { createHash } from 'node:crypto'; import { promises as fsp } from 'node:fs'; @@ -114,6 +115,9 @@ async function readMilitaryInstallationDisk( const entry = JSON.parse( await fsp.readFile(militaryInstallationDiskPath(cacheKey, dir), 'utf8'), ); + // Same reason as the Overpass disk cache: the key is the bbox alone, so an + // entry written against a different OVERPASS_UPSTREAMS is otherwise a hit. + if (!cachedByConfiguredUpstream(entry)) return null; if (!militaryInstallationDiskFresh(entry, maxAgeMs)) return null; return migrateMilitaryInstallationEntry(entry); } catch { diff --git a/server/providers/overpass/cache.js b/server/providers/overpass/cache.js index b4e53ab21..442f599a6 100644 --- a/server/providers/overpass/cache.js +++ b/server/providers/overpass/cache.js @@ -5,6 +5,7 @@ import { OVERPASS_DISK_DIR, OVERPASS_CACHE_MS, OVERPASS_CACHE_MAX_ENTRIES, + cachedByConfiguredUpstream, } from './constants.js'; import path from 'node:path'; import { createHash } from 'node:crypto'; @@ -47,6 +48,11 @@ async function readOverpassDisk(cacheKey, maxAgeMs) { // Older versions persisted 4xx refusals with normal data TTLs. Ignore // them on both fresh and stale reads so an upgrade can recover immediately. if (!overpassPayloadIsData(payload)) return null; + // The key is the query alone, so a file written by a different upstream — + // including the public instances that were the default before #742 — is a + // hit for the same query. Refuse it: the operator's configured instance + // must be what answers, on the stale path too. + if (!cachedByConfiguredUpstream(payload)) return null; if (Date.now() - payload.cachedAt > maxAgeMs) return null; return payload; } catch { diff --git a/server/providers/overpass/constants.js b/server/providers/overpass/constants.js index 40f244d11..2a0d4a3ef 100644 --- a/server/providers/overpass/constants.js +++ b/server/providers/overpass/constants.js @@ -1,4 +1,5 @@ import path from 'node:path'; +import { createHash } from 'node:crypto'; // --------------------------------------------------------------------------- // Overpass API proxy constants and cache state @@ -27,16 +28,68 @@ function parseOverpassUpstreams(raw) { return endpoints.slice(0, 8); } -let upstreamMemo = { raw: null, endpoints: [] }; +let upstreamMemo = { raw: null, endpoints: [], tag: 'none' }; + +/** + * Short, stable identifier for the configured upstream set. Cached responses + * carry it so an answer produced by a different instance is not served after + * the operator repoints `OVERPASS_UPSTREAMS` — the disk cache outlives the + * configuration change by 7 days, a month for boundary queries, and forever on + * the serve-stale path. It is a hash, never the URLs, and it is only ever + * compared on read: it is not sent to clients and does not appear in errors. + */ +function upstreamTag(endpoints) { + if (!endpoints.length) return 'none'; + return createHash('sha1') + .update(endpoints.join('\n')) + .digest('hex') + .slice(0, 12); +} /** Resolve after environment loading. Public Overpass instances are not used by default. */ function resolveOverpassUpstreams() { - const raw = process.env.OVERPASS_UPSTREAMS || ''; - if (upstreamMemo.raw !== raw) - upstreamMemo = { raw, endpoints: parseOverpassUpstreams(raw) }; + readUpstreamMemo(); return [...upstreamMemo.endpoints]; } +/** The configured-upstream tag stored alongside, and checked against, cache entries. */ +function resolveOverpassUpstreamTag() { + return readUpstreamMemo().tag; +} + +let reportedForeignCache = false; + +/** + * Was this cached entry produced by the upstream set configured now? + * + * With nothing configured there is no instance to contradict, and serving + * last-good data at any age is the point of the cache (the proxy's + * not-configured path depends on it), so any entry is accepted. Once an + * operator does configure an instance, only its own answers are served: an + * entry with no tag, or one from a different instance, is refused rather than + * matched on the query alone. Shared by both caches fed by these upstreams. + */ +function cachedByConfiguredUpstream(entry) { + const tag = resolveOverpassUpstreamTag(); + if (tag === 'none' || entry?.upstreams === tag) return true; + if (!reportedForeignCache) { + reportedForeignCache = true; + console.warn( + '[Overpass] ignoring cached responses produced by a different OVERPASS_UPSTREAMS; they will be refetched from the configured instance.', + ); + } + return false; +} + +function readUpstreamMemo() { + const raw = process.env.OVERPASS_UPSTREAMS || ''; + if (upstreamMemo.raw !== raw) { + const endpoints = parseOverpassUpstreams(raw); + upstreamMemo = { raw, endpoints, tag: upstreamTag(endpoints) }; + } + return upstreamMemo; +} + /** * TTL for FRESH cached Overpass responses (ms). Road geometry is static for * months — the original 45 s TTL forced a public-mirror round-trip on nearly @@ -153,6 +206,8 @@ export { OVERPASS_MAX_RESPONSE_BYTES, parseOverpassUpstreams, resolveOverpassUpstreams, + resolveOverpassUpstreamTag, + cachedByConfiguredUpstream, OVERPASS_USER_AGENT, OVERPASS_TIMEOUT_MS, }; diff --git a/server/providers/overpass/transport.js b/server/providers/overpass/transport.js index 86b9eae15..38c31bb08 100644 --- a/server/providers/overpass/transport.js +++ b/server/providers/overpass/transport.js @@ -1,6 +1,7 @@ import { OVERPASS_MAX_RESPONSE_BYTES, resolveOverpassUpstreams, + resolveOverpassUpstreamTag, OVERPASS_USER_AGENT, OVERPASS_TIMEOUT_MS, } from './constants.js'; @@ -187,6 +188,9 @@ async function fetchOverpassPayload( contentType: 'application/json', // Never retain a secret-bearing endpoint in cache or response metadata. endpoint: 'configured', + // Which configured instance produced this, as a hash rather than a URL, + // so a cached answer is not reused after the operator repoints. + upstreams: resolveOverpassUpstreamTag(), rateLimited: false, }; } catch { diff --git a/src/overpassProxy.test.mjs b/src/overpassProxy.test.mjs index 71b62aa2d..ed4539fda 100644 --- a/src/overpassProxy.test.mjs +++ b/src/overpassProxy.test.mjs @@ -12,6 +12,7 @@ import { createHash, randomUUID } from 'node:crypto'; import path from 'node:path'; import { Readable } from 'node:stream'; import createViteConfig, { fetchOverpassPayload, overpassPayloadIsData, readOverpassDisk } from '../vite.config.js'; +import { resolveOverpassUpstreamTag } from '../server/providers/overpass/constants.js'; const DATA = { status: 200, body: '{"elements":[]}' }; @@ -155,7 +156,13 @@ test('coalesced outage callers both receive last-good data, never a cached refus const directory = path.join(process.cwd(), '.gev-cache', 'overpass'); const file = path.join(directory, `${createHash('sha1').update(body).digest('hex')}.json`); await mkdir(directory, { recursive: true }); - const stale = { ...DATA, cachedAt: Date.now() - 40 * 86400000 }; + // Last-good data was fetched from the instance configured above, so it + // carries that instance's tag the way the transport now writes it. + const stale = { + ...DATA, + cachedAt: Date.now() - 40 * 86400000, + upstreams: resolveOverpassUpstreamTag(), + }; await writeFile(file, JSON.stringify(stale)); const entered = Promise.withResolvers(); const release = Promise.withResolvers(); @@ -188,3 +195,46 @@ test('coalesced outage callers both receive last-good data, never a cached refus } } }); + +test('a disk entry is not served after OVERPASS_UPSTREAMS is repointed', async (t) => { + const prior = process.env.OVERPASS_UPSTREAMS; + t.after(() => { + if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS; + else process.env.OVERPASS_UPSTREAMS = prior; + }); + const key = `overpass-upstream-switch-${randomUUID()}`; + const directory = path.join(process.cwd(), '.gev-cache', 'overpass'); + const file = path.join( + directory, + `${createHash('sha1').update(key).digest('hex')}.json`, + ); + await mkdir(directory, { recursive: true }); + try { + // Written while one instance was configured... + process.env.OVERPASS_UPSTREAMS = 'https://first.example/api/interpreter'; + const first = resolveOverpassUpstreamTag(); + await writeFile( + file, + JSON.stringify({ ...DATA, cachedAt: Date.now(), upstreams: first }), + ); + assert.ok(await readOverpassDisk(key, 60000), 'its own entry is a hit'); + + // ...is not an answer once the operator points at a different one, on the + // stale path either: the key is the query alone, so it would otherwise hit. + process.env.OVERPASS_UPSTREAMS = 'https://second.example/api/interpreter'; + assert.notEqual(resolveOverpassUpstreamTag(), first, 'tag follows the env'); + assert.equal(await readOverpassDisk(key, 60000), null, 'fresh read refused'); + assert.equal( + await readOverpassDisk(key, Infinity), + null, + 'stale read refused', + ); + + // With nothing configured there is no instance to contradict, and serving + // last-good data is what the not-configured path exists for. + delete process.env.OVERPASS_UPSTREAMS; + assert.ok(await readOverpassDisk(key, Infinity), 'unconfigured still serves'); + } finally { + await unlink(file); + } +}); From 42543c1fa1f547db940dd46b0f5c781cff11be9a Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:13:53 +0200 Subject: [PATCH 27/95] feat(bikeshare): add Oslo, Bergen and Trondheim Bysykkel Add Norway's three Urban Sharing city-bike systems to the Bike Share layer and get_bike_share. The feeds are keyless GBFS on gbfs.urbansharing.com, which joins the proxy allowlist. The proxy now sends the Client-Identifier header the operators ask clients to send. Attributed under NLOD 2.0 in DATA_SOURCES.md and the data credits. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++ DATA_SOURCES.md | 1 + docs/CURRENT-STATE.md | 5 ++- server/providers/gbfs.js | 6 ++++ src/data/dataCredits.js | 7 ++++ src/data/gbfsProxy.test.mjs | 16 +++++++++ src/data/gbfsSource.js | 1 + src/layers/bikeshare/proxyRoute.test.mjs | 44 ++++++++++++++++++++++++ src/layers/bikeshare/registry.js | 44 ++++++++++++++++++++++++ src/tools/queries/mobility.test.mjs | 25 ++++++++++++++ 10 files changed, 154 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b433f8c06..a2fc2d1e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +- Add Norway's city bikes to the Bike Share layer: Oslo, Bergen and + Trondheim Bysykkel (Urban Sharing, ~440 stations). The feeds are keyless + GBFS on `gbfs.urbansharing.com`, which joins the proxy allowlist; the proxy + now sends the `Client-Identifier: gods-eye-view` header the operators ask + for. Attributed under NLOD 2.0. `get_bike_share` covers the three cities too. + - Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera list is one keyless GeoJSON request to the agency's OGC API view of its DATEX 3.1 CCTV table, so no DATEX account is needed. Every working camera diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 734e35a69..07782fd5e 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -56,6 +56,7 @@ How to read this: | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | +| **Oslo / Bergen / Trondheim Bysykkel** (Urban Sharing, `gbfs.urbansharing.com`) | Bikeshare availability, Oslo, Bergen and Trondheim city bikes. Keyless GBFS; the proxy sends the `Client-Identifier` header the operators ask for | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED ([Oslo](https://oslobysykkel.no/en/open-data/realtime), [Bergen](https://bergenbysykkel.no/en/apne-data/sanntid), [Trondheim](https://trondheimbysykkel.no/en/open-data/realtime)) | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel" | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | | **Metro Transit** (Metropolitan Council) | Transit layer live vehicles, Minneapolis–St Paul | Metro Transit / Metropolitan Council open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to Metro Transit / Metropolitan Council (courtesy text) | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d4fe022a8..8a25ccb8c 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -961,6 +961,9 @@ The station reader addresses the proxy as `/api/gbfs/`; the feed address is read from the path, and a query-string form is refused with a 400. GBFS refuses upstream redirects and enforces its 5 MiB response cap while streaming. The 12-second deadline includes reading the body, and rejected or stalled downloads are cancelled. Development and preview use the same handler. +Every upstream request carries `Client-Identifier: gods-eye-view`, which Urban +Sharing asks of clients of the Oslo, Bergen and Trondheim Bysykkel feeds +(`gbfs.urbansharing.com`); other operators ignore it. ## Remaining local service modules @@ -2922,7 +2925,7 @@ its criteria cannot be silently ignored. | CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Urban Sharing NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | | Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | diff --git a/server/providers/gbfs.js b/server/providers/gbfs.js index 9741bb4e6..f55299992 100644 --- a/server/providers/gbfs.js +++ b/server/providers/gbfs.js @@ -11,6 +11,9 @@ import { /** Upstream fetch timeout for GBFS requests (ms). */ const GBFS_PROXY_TIMEOUT_MS = 12000; +/** Value of the Client-Identifier header sent with every upstream request. */ +export const GBFS_CLIENT_IDENTIFIER = 'gods-eye-view'; + export const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB function gbfsRedirectHost(location, requestUrl) { @@ -60,6 +63,9 @@ export async function fetchGbfsUpstream( headers: { Accept: 'application/json', 'User-Agent': 'gods-eye-view-gbfs-proxy/1.0', + // Urban Sharing (the Norwegian city-bike feeds) asks every client to + // name itself in this header; other operators ignore it. + 'Client-Identifier': GBFS_CLIENT_IDENTIFIER, }, redirect: 'manual', signal: controller.signal, diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 2967f0a2b..87358dafc 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -251,6 +251,13 @@ export const DATA_CREDITS = [ key: 'gbfs', html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)', }, + { + key: 'urbansharing-gbfs', + html: + 'City bikes (Norway): contains data under the ' + + 'Norwegian licence for Open Government data (NLOD) ' + + 'made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel', + }, { key: 'osrm-routing', // The service asks for its attribution to carry a "fix the map" link, so diff --git a/src/data/gbfsProxy.test.mjs b/src/data/gbfsProxy.test.mjs index 191cb57ba..af107b38e 100644 --- a/src/data/gbfsProxy.test.mjs +++ b/src/data/gbfsProxy.test.mjs @@ -1,6 +1,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { + GBFS_CLIENT_IDENTIFIER, GBFS_MAX_BODY_BYTES, fetchGbfsUpstream, } from '../../server/providers/gbfs.js'; @@ -56,6 +57,21 @@ test('GBFS upstream fetch passes a direct 200 through unchanged', async () => { }); }); +test('GBFS upstream fetch names the app in the Client-Identifier header', async () => { + let observedHeaders = null; + await fetchGbfsUpstream( + 'https://gbfs.urbansharing.com/oslobysykkel.no/station_status.json', + { + fetchImpl: async (_url, options) => { + observedHeaders = options.headers; + return new Response(STATION_BODY, { status: 200 }); + }, + }, + ); + assert.equal(GBFS_CLIENT_IDENTIFIER, 'gods-eye-view'); + assert.equal(observedHeaders['Client-Identifier'], GBFS_CLIENT_IDENTIFIER); +}); + test('GBFS upstream fetch forwards a non-redirect error status for the middleware to relay', async () => { const result = await fetchGbfsUpstream(STATION_URL, { fetchImpl: async () => new Response(null, { status: 404 }), diff --git a/src/data/gbfsSource.js b/src/data/gbfsSource.js index a15a806d9..f06fa24f2 100644 --- a/src/data/gbfsSource.js +++ b/src/data/gbfsSource.js @@ -8,6 +8,7 @@ const GBFS_ALLOWED_HOSTS = new Set([ 'austin.publicbikesystem.net', 'hon.publicbikesystem.net', 'chat.publicbikesystem.net', + 'gbfs.urbansharing.com', ]); /** diff --git a/src/layers/bikeshare/proxyRoute.test.mjs b/src/layers/bikeshare/proxyRoute.test.mjs index 88b98f31b..9158a6bdf 100644 --- a/src/layers/bikeshare/proxyRoute.test.mjs +++ b/src/layers/bikeshare/proxyRoute.test.mjs @@ -17,6 +17,7 @@ import { createModel } from './model.js'; import { CITY_BY_ID } from './registry.js'; const AUSTIN = CITY_BY_ID.get('austin-capmetro'); +const OSLO = CITY_BY_ID.get('oslo-bysykkel'); const STATION_INFORMATION = { data: { @@ -204,6 +205,49 @@ test('the proxy relays its cache policy and upstream host through the mounted ro ); }); +test('the Norwegian city-bike systems are served through the mounted route', async () => { + for (const [id, systemId] of [ + ['oslo-bysykkel', 'oslobysykkel.no'], + ['bergen-bysykkel', 'bergenbysykkel.no'], + ['trondheim-bysykkel', 'trondheimbysykkel.no'], + ]) { + const system = CITY_BY_ID.get(id); + assert.ok(system, `${id} is registered`); + assert.deepEqual(system.hosts, ['gbfs.urbansharing.com']); + assert.equal( + system.stationStatusUrl, + `https://gbfs.urbansharing.com/${systemId}/station_status.json`, + ); + } + + await withUpstream( + { + [OSLO.stationInformationUrl]: STATION_INFORMATION, + [OSLO.stationStatusUrl]: STATION_STATUS, + }, + async (requested) => { + const source = createBikeshareSource({ fetchImpl: mountGbfsProxy() }); + const info = await source.getStations(OSLO.stationInformationUrl); + const status = await source.getStations(OSLO.stationStatusUrl); + assert.deepEqual(requested, [ + OSLO.stationInformationUrl, + OSLO.stationStatusUrl, + ]); + assert.equal(info.data.stations.length, 1); + assert.equal(status.data.stations.length, 1); + + const response = await mountGbfsProxy()( + '/api/gbfs/' + encodeURIComponent(OSLO.stationStatusUrl), + ); + assert.equal(response.status, 200); + assert.equal( + response.headers.get('x-gbfs-upstream'), + 'gbfs.urbansharing.com', + ); + }, + ); +}); + test('the query-string shape the client used to send is refused by the mounted route', async () => { await withUpstream( { [AUSTIN.stationStatusUrl]: STATION_STATUS }, diff --git a/src/layers/bikeshare/registry.js b/src/layers/bikeshare/registry.js index ff752f664..37b7d2c4e 100644 --- a/src/layers/bikeshare/registry.js +++ b/src/layers/bikeshare/registry.js @@ -23,6 +23,26 @@ function bcycleEntry({ ...buildBcycleUrls(systemId), }; } +function urbanSharingEntry({ + id, + city, + centerLat, + centerLon, + systemId, + provider, + loadRadiusKm = 100, +}) { + return { + id, + city, + centerLat, + centerLon, + loadRadiusKm, + provider, + stationInformationUrl: `https://gbfs.urbansharing.com/${systemId}/station_information.json`, + stationStatusUrl: `https://gbfs.urbansharing.com/${systemId}/station_status.json`, + }; +} const RAW_GBFS_CITY_REGISTRY = [ { id: 'nyc-citibike', @@ -318,6 +338,30 @@ const RAW_GBFS_CITY_REGISTRY = [ centerLon: -119.6982, systemId: 'bcycle_santabarbara', }), + urbanSharingEntry({ + id: 'oslo-bysykkel', + city: 'Oslo, Norway', + centerLat: 59.9139, + centerLon: 10.7522, + systemId: 'oslobysykkel.no', + provider: 'Oslo Bysykkel', + }), + urbanSharingEntry({ + id: 'bergen-bysykkel', + city: 'Bergen, Norway', + centerLat: 60.3913, + centerLon: 5.3221, + systemId: 'bergenbysykkel.no', + provider: 'Bergen Bysykkel', + }), + urbanSharingEntry({ + id: 'trondheim-bysykkel', + city: 'Trondheim, Norway', + centerLat: 63.4305, + centerLon: 10.3951, + systemId: 'trondheimbysykkel.no', + provider: 'Trondheim Bysykkel', + }), ]; function normalizeRegistryEntry(entry) { const id = String(entry?.id || '') diff --git a/src/tools/queries/mobility.test.mjs b/src/tools/queries/mobility.test.mjs index af8142305..1626aa3dd 100644 --- a/src/tools/queries/mobility.test.mjs +++ b/src/tools/queries/mobility.test.mjs @@ -429,6 +429,31 @@ test('bike share names the systems it did not search', async () => { ); }); +test('bike share over Oslo searches Oslo Bysykkel only', async () => { + const requested = []; + const result = await composeCatalog({ + tools: coreTools, + services: { + bikeshare: { + systems: GBFS_CITY_REGISTRY, + getStations: async (url) => { + requested.push(url); + return { data: { stations: [] } }; + }, + }, + }, + }).call('get_bike_share', { + area: { lat: 59.9139, lon: 10.7522, radius_km: 5 }, + }); + assert.deepEqual(result.data.systems, ['Oslo, Norway']); + assert.deepEqual(result.data.systems_not_searched, []); + assert.ok( + requested.every((url) => + url.startsWith('https://gbfs.urbansharing.com/oslobysykkel.no/'), + ), + ); +}); + test('traffic counts only roads inside a radius area and reports missing tiles', async () => { const flow = (segments, partial = false) => ({ getStatus: async () => ({ hasKey: true }), From fa61955cf15cab33e55707aba55efc598f35d6e9 Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:02:37 +0200 Subject: [PATCH 28/95] =?UTF-8?q?feat(bikeshare):=20add=20Kolumbus,=20Fart?= =?UTF-8?q?e=20and=20Lillestr=C3=B8m=20city=20bikes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover every Norwegian station-based city-bike system with an open GBFS feed: Kolumbus Bysykkel (Stavanger) and Farte Bysykkel (Skien/Porsgrunn) through Entur Mobility, and Bysykkel Lillestrøm through JCDecaux Cyclocity. api.entur.io and api.cyclocity.fr join the GBFS allowlist, each kept to its GBFS path prefix since both hosts front other APIs. Entur serves its feeds without a .json suffix, so the endpoint checks accept either form, and requests to Entur carry the ET-Client-Name header it requires. Attributed under NLOD (Entur) and Licence Ouverte (JCDecaux). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 ++++-- DATA_SOURCES.md | 2 + docs/CURRENT-STATE.md | 10 +++- server/providers/gbfs.js | 4 +- src/data/dataCredits.js | 13 ++++++ src/data/gbfsProxy.test.mjs | 20 ++++++++ src/data/gbfsSource.js | 59 ++++++++++++++++++++++-- src/layers/bikeshare/proxyRoute.test.mjs | 52 +++++++++++++++++++++ src/layers/bikeshare/registry.js | 58 +++++++++++++++++++++-- src/tools/queries/mobility.test.mjs | 28 +++++++++-- 10 files changed, 241 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2fc2d1e2..184e55230 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,15 @@ ## [Unreleased] - Add Norway's city bikes to the Bike Share layer: Oslo, Bergen and - Trondheim Bysykkel (Urban Sharing, ~440 stations). The feeds are keyless - GBFS on `gbfs.urbansharing.com`, which joins the proxy allowlist; the proxy - now sends the `Client-Identifier: gods-eye-view` header the operators ask - for. Attributed under NLOD 2.0. `get_bike_share` covers the three cities too. + Trondheim Bysykkel (Urban Sharing), Kolumbus Bysykkel in Stavanger and + Farte Bysykkel in Skien/Porsgrunn (Entur Mobility), and Bysykkel + Lillestrøm (JCDecaux), about 760 stations in all. Every feed is keyless + GBFS. `gbfs.urbansharing.com`, `api.entur.io` and `api.cyclocity.fr` join + the proxy allowlist; the last two are kept to their GBFS paths. The proxy + now sends `Client-Identifier: gods-eye-view` on every request and + `ET-Client-Name: gods-eye-view-bikeshare` to Entur, as the operators ask, + and accepts Entur's extensionless feed paths. Attributed under NLOD and + Licence Ouverte. `get_bike_share` covers the six systems too. - Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera list is one keyless GeoJSON request to the agency's OGC API view of its diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 07782fd5e..f7e283117 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -57,6 +57,8 @@ How to read this: | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **Oslo / Bergen / Trondheim Bysykkel** (Urban Sharing, `gbfs.urbansharing.com`) | Bikeshare availability, Oslo, Bergen and Trondheim city bikes. Keyless GBFS; the proxy sends the `Client-Identifier` header the operators ask for | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED ([Oslo](https://oslobysykkel.no/en/open-data/realtime), [Bergen](https://bergenbysykkel.no/en/apne-data/sanntid), [Trondheim](https://trondheimbysykkel.no/en/open-data/realtime)) | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel" | +| **Kolumbus / Farte Bysykkel** (Entur Mobility, `api.entur.io/mobility/v2/gbfs`) | Bikeshare availability, Kolumbus Bysykkel (Stavanger/Nord-Jæren) and Farte Bysykkel (Skien/Porsgrunn). Keyless GBFS; the proxy sends the `ET-Client-Name` header Entur requires and is limited to Entur's GBFS paths | [NLOD](https://data.norge.no/nlod/en/2.0) per the [Entur Mobility API docs](https://developer.entur.org/pages-mobility-docs-mobility-v2) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Entur (Kolumbus Bysykkel, Farte Bysykkel)" | +| **JCDecaux Cyclocity** (`api.cyclocity.fr`) | Bikeshare availability, Bysykkel Lillestrøm. Keyless GBFS; the proxy is limited to JCDecaux contract paths | [Licence Ouverte / Etalab](https://developer.jcdecaux.com/files/Open-Licence-fr.pdf) (the feed's `license_url`) — attribution REQUIRED | "Bysykkel Lillestrøm: JCDecaux, Licence Ouverte" | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | | **Metro Transit** (Metropolitan Council) | Transit layer live vehicles, Minneapolis–St Paul | Metro Transit / Metropolitan Council open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to Metro Transit / Metropolitan Council (courtesy text) | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 8a25ccb8c..55056d2ea 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -963,7 +963,13 @@ streaming. The 12-second deadline includes reading the body, and rejected or stalled downloads are cancelled. Development and preview use the same handler. Every upstream request carries `Client-Identifier: gods-eye-view`, which Urban Sharing asks of clients of the Oslo, Bergen and Trondheim Bysykkel feeds -(`gbfs.urbansharing.com`); other operators ignore it. +(`gbfs.urbansharing.com`); other operators ignore it. Requests to Entur +(`api.entur.io`, Kolumbus and Farte Bysykkel) also carry +`ET-Client-Name: gods-eye-view-bikeshare`. Entur serves its feeds without a +`.json` suffix, so the endpoint check accepts `station_information` and +`station_status` with or without it. Hosts that front more than GBFS are kept to +their GBFS tree: `api.entur.io` to `/mobility/v2/gbfs/` and `api.cyclocity.fr` +(Bysykkel Lillestrøm) to `/contracts/`. ## Remaining local service modules @@ -2925,7 +2931,7 @@ its criteria cannot be silently ignored. | CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle + Urban Sharing NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Urban Sharing, Entur, JCDecaux NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | | Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | diff --git a/server/providers/gbfs.js b/server/providers/gbfs.js index f55299992..990642c8b 100644 --- a/server/providers/gbfs.js +++ b/server/providers/gbfs.js @@ -3,6 +3,7 @@ import { isAllowedGbfsHost, isAllowedGbfsPath, gbfsCacheControl, + gbfsUpstreamHeaders, } from '../../src/data/gbfsSource.js'; // --------------------------------------------------------------------------- @@ -66,6 +67,7 @@ export async function fetchGbfsUpstream( // Urban Sharing (the Norwegian city-bike feeds) asks every client to // name itself in this header; other operators ignore it. 'Client-Identifier': GBFS_CLIENT_IDENTIFIER, + ...gbfsUpstreamHeaders(new URL(url).hostname), }, redirect: 'manual', signal: controller.signal, @@ -187,7 +189,7 @@ export function gbfsProxy() { return; } - if (!isAllowedGbfsPath(upstreamUrl.pathname)) { + if (!isAllowedGbfsPath(upstreamUrl.pathname, upstreamUrl.hostname)) { res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 87358dafc..195d2487e 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -258,6 +258,19 @@ export const DATA_CREDITS = [ 'Norwegian licence for Open Government data (NLOD) ' + 'made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel', }, + { + key: 'entur-mobility-gbfs', + html: + 'City bikes (Stavanger, Skien/Porsgrunn): contains data under the ' + + 'Norwegian licence for Open Government data (NLOD) ' + + 'made available by Entur (Kolumbus Bysykkel, Farte Bysykkel)', + }, + { + key: 'jcdecaux-gbfs', + html: + 'City bikes (Lillestrøm): Bysykkel Lillestrøm, JCDecaux — ' + + 'Licence Ouverte', + }, { key: 'osrm-routing', // The service asks for its attribution to carry a "fix the map" link, so diff --git a/src/data/gbfsProxy.test.mjs b/src/data/gbfsProxy.test.mjs index af107b38e..c95256137 100644 --- a/src/data/gbfsProxy.test.mjs +++ b/src/data/gbfsProxy.test.mjs @@ -72,6 +72,26 @@ test('GBFS upstream fetch names the app in the Client-Identifier header', async assert.equal(observedHeaders['Client-Identifier'], GBFS_CLIENT_IDENTIFIER); }); +test('GBFS upstream fetch adds ET-Client-Name for Entur and only for Entur', async () => { + const headersFor = async (url) => { + let observed = null; + await fetchGbfsUpstream(url, { + fetchImpl: async (_url, options) => { + observed = options.headers; + return new Response(STATION_BODY, { status: 200 }); + }, + }); + return observed; + }; + const entur = await headersFor( + 'https://api.entur.io/mobility/v2/gbfs/v2/kolumbusbysykkel/station_status', + ); + assert.equal(entur['ET-Client-Name'], 'gods-eye-view-bikeshare'); + assert.equal(entur['Client-Identifier'], GBFS_CLIENT_IDENTIFIER); + const lyft = await headersFor(STATION_URL); + assert.equal(lyft['ET-Client-Name'], undefined); +}); + test('GBFS upstream fetch forwards a non-redirect error status for the middleware to relay', async () => { const result = await fetchGbfsUpstream(STATION_URL, { fetchImpl: async () => new Response(null, { status: 404 }), diff --git a/src/data/gbfsSource.js b/src/data/gbfsSource.js index f06fa24f2..8d4980948 100644 --- a/src/data/gbfsSource.js +++ b/src/data/gbfsSource.js @@ -9,6 +9,30 @@ const GBFS_ALLOWED_HOSTS = new Set([ 'hon.publicbikesystem.net', 'chat.publicbikesystem.net', 'gbfs.urbansharing.com', + 'api.entur.io', + 'api.cyclocity.fr', +]); + +/** + * Hosts that serve more than GBFS keep the proxy to their GBFS tree: Entur's + * api.entur.io fronts every Entur API, and api.cyclocity.fr every JCDecaux + * contract endpoint. + */ +const GBFS_HOST_PATH_PREFIXES = new Map([ + ['api.entur.io', '/mobility/v2/gbfs/'], + ['api.cyclocity.fr', '/contracts/'], +]); + +/** + * Identifying headers an operator asks every GBFS client to send. Entur rate + * limits or blocks consumers that do not send ET-Client-Name, in the form + * "company-application". + */ +const GBFS_HOST_HEADERS = new Map([ + [ + 'api.entur.io', + Object.freeze({ 'ET-Client-Name': 'gods-eye-view-bikeshare' }), + ], ]); /** @@ -29,13 +53,40 @@ export function isAllowedGbfsHost(hostname) { } /** - * Only allow station_information.json and station_status.json endpoints. + * Only allow station_information and station_status endpoints. The `.json` + * suffix is optional because Entur serves its feeds without one. When a + * hostname is given and that host serves more than GBFS, the path must also + * sit under the host's GBFS prefix. * * @param {string} pathname + * @param {string} [hostname] * @returns {boolean} */ -export function isAllowedGbfsPath(pathname) { - return /\/station_(information|status)\.json$/i.test(String(pathname || '')); +export function isAllowedGbfsPath(pathname, hostname = '') { + const path = String(pathname || ''); + if (!/\/station_(information|status)(\.json)?$/i.test(path)) return false; + const prefix = GBFS_HOST_PATH_PREFIXES.get( + String(hostname || '') + .trim() + .toLowerCase(), + ); + return !prefix || path.startsWith(prefix); +} + +/** + * Extra request headers an upstream GBFS host asks clients to send. + * + * @param {string} hostname + * @returns {Record} + */ +export function gbfsUpstreamHeaders(hostname) { + return ( + GBFS_HOST_HEADERS.get( + String(hostname || '') + .trim() + .toLowerCase(), + ) || {} + ); } /** @@ -48,7 +99,7 @@ export function isAllowedGbfsPath(pathname) { * @returns {string} Cache-Control header value. */ export function gbfsCacheControl(pathname) { - if (/\/station_information\.json$/i.test(String(pathname || ''))) { + if (/\/station_information(\.json)?$/i.test(String(pathname || ''))) { return 'public, max-age=300'; } return 'no-store'; diff --git a/src/layers/bikeshare/proxyRoute.test.mjs b/src/layers/bikeshare/proxyRoute.test.mjs index 9158a6bdf..9bb04ced0 100644 --- a/src/layers/bikeshare/proxyRoute.test.mjs +++ b/src/layers/bikeshare/proxyRoute.test.mjs @@ -248,6 +248,58 @@ test('the Norwegian city-bike systems are served through the mounted route', asy ); }); +test('every Norwegian city-bike system is registered and reachable through the mounted route', async () => { + const expected = { + 'stavanger-kolumbus': + 'https://api.entur.io/mobility/v2/gbfs/v2/kolumbusbysykkel/station_status', + 'skien-farte': + 'https://api.entur.io/mobility/v2/gbfs/v2/fartebysykkel/station_status', + 'lillestrom-bysykkel': + 'https://api.cyclocity.fr/contracts/lillestrom/gbfs/station_status.json', + }; + const upstream = {}; + for (const [id, statusUrl] of Object.entries(expected)) { + const system = CITY_BY_ID.get(id); + assert.ok(system, `${id} is registered`); + assert.equal(system.stationStatusUrl, statusUrl); + upstream[system.stationInformationUrl] = STATION_INFORMATION; + upstream[system.stationStatusUrl] = STATION_STATUS; + } + + await withUpstream(upstream, async (requested) => { + const source = createBikeshareSource({ fetchImpl: mountGbfsProxy() }); + for (const id of Object.keys(expected)) { + const system = CITY_BY_ID.get(id); + const info = await source.getStations(system.stationInformationUrl); + const status = await source.getStations(system.stationStatusUrl); + assert.equal(info.data.stations.length, 1, `${id} information`); + assert.equal(status.data.stations.length, 1, `${id} status`); + } + assert.equal(requested.length, 6); + + // Entur's extensionless station_information keeps its 5-minute cache. + const kolumbus = CITY_BY_ID.get('stavanger-kolumbus'); + const response = await mountGbfsProxy()( + '/api/gbfs/' + encodeURIComponent(kolumbus.stationInformationUrl), + ); + assert.equal(response.headers.get('cache-control'), 'public, max-age=300'); + }); +}); + +test('the mounted route keeps multi-API hosts to their GBFS tree', async () => { + await withUpstream({}, async (requested) => { + const mounted = mountGbfsProxy(); + for (const target of [ + 'https://api.entur.io/journey-planner/v3/station_status', + 'https://api.cyclocity.fr/other/station_status.json', + ]) { + const response = await mounted('/api/gbfs/' + encodeURIComponent(target)); + assert.equal(response.status, 400, target); + } + assert.deepEqual(requested, []); + }); +}); + test('the query-string shape the client used to send is refused by the mounted route', async () => { await withUpstream( { [AUSTIN.stationStatusUrl]: STATION_STATUS }, diff --git a/src/layers/bikeshare/registry.js b/src/layers/bikeshare/registry.js index 37b7d2c4e..89508d42e 100644 --- a/src/layers/bikeshare/registry.js +++ b/src/layers/bikeshare/registry.js @@ -43,6 +43,27 @@ function urbanSharingEntry({ stationStatusUrl: `https://gbfs.urbansharing.com/${systemId}/station_status.json`, }; } +function enturEntry({ + id, + city, + centerLat, + centerLon, + systemId, + provider, + loadRadiusKm = 100, +}) { + const base = `https://api.entur.io/mobility/v2/gbfs/v2/${systemId}`; + return { + id, + city, + centerLat, + centerLon, + loadRadiusKm, + provider, + stationInformationUrl: `${base}/station_information`, + stationStatusUrl: `${base}/station_status`, + }; +} const RAW_GBFS_CITY_REGISTRY = [ { id: 'nyc-citibike', @@ -362,6 +383,35 @@ const RAW_GBFS_CITY_REGISTRY = [ systemId: 'trondheimbysykkel.no', provider: 'Trondheim Bysykkel', }), + enturEntry({ + id: 'stavanger-kolumbus', + city: 'Stavanger, Norway', + centerLat: 58.9, + centerLon: 5.72, + systemId: 'kolumbusbysykkel', + provider: 'Kolumbus Bysykkel', + }), + enturEntry({ + id: 'skien-farte', + city: 'Skien/Porsgrunn, Norway', + centerLat: 59.16, + centerLon: 9.64, + systemId: 'fartebysykkel', + provider: 'Farte Bysykkel', + loadRadiusKm: 40, + }), + { + id: 'lillestrom-bysykkel', + city: 'Lillestrøm, Norway', + centerLat: 59.9556, + centerLon: 11.0492, + loadRadiusKm: 40, + stationInformationUrl: + 'https://api.cyclocity.fr/contracts/lillestrom/gbfs/station_information.json', + stationStatusUrl: + 'https://api.cyclocity.fr/contracts/lillestrom/gbfs/station_status.json', + provider: 'Bysykkel Lillestrøm', + }, ]; function normalizeRegistryEntry(entry) { const id = String(entry?.id || '') @@ -390,11 +440,13 @@ function normalizeRegistryEntry(entry) { ) { throw new Error(`GBFS entry "${id}" must use https URLs`); } - // Validate that URLs end with expected GBFS endpoint filenames - if (!/\/station_information\.json$/i.test(stationInformationUrl.pathname)) { + // Validate that URLs end with expected GBFS endpoint names (Entur omits .json) + if ( + !/\/station_information(\.json)?$/i.test(stationInformationUrl.pathname) + ) { throw new Error(`GBFS entry "${id}" station_information URL is invalid`); } - if (!/\/station_status\.json$/i.test(stationStatusUrl.pathname)) { + if (!/\/station_status(\.json)?$/i.test(stationStatusUrl.pathname)) { throw new Error(`GBFS entry "${id}" station_status URL is invalid`); } diff --git a/src/tools/queries/mobility.test.mjs b/src/tools/queries/mobility.test.mjs index 1626aa3dd..07482946b 100644 --- a/src/tools/queries/mobility.test.mjs +++ b/src/tools/queries/mobility.test.mjs @@ -429,7 +429,7 @@ test('bike share names the systems it did not search', async () => { ); }); -test('bike share over Oslo searches Oslo Bysykkel only', async () => { +test('bike share over Oslo searches only the nearby Norwegian systems', async () => { const requested = []; const result = await composeCatalog({ tools: coreTools, @@ -445,15 +445,35 @@ test('bike share over Oslo searches Oslo Bysykkel only', async () => { }).call('get_bike_share', { area: { lat: 59.9139, lon: 10.7522, radius_km: 5 }, }); - assert.deepEqual(result.data.systems, ['Oslo, Norway']); + assert.deepEqual(result.data.systems.sort(), [ + 'Lillestrøm, Norway', + 'Oslo, Norway', + ]); assert.deepEqual(result.data.systems_not_searched, []); assert.ok( - requested.every((url) => - url.startsWith('https://gbfs.urbansharing.com/oslobysykkel.no/'), + requested.every( + (url) => + url.startsWith('https://gbfs.urbansharing.com/oslobysykkel.no/') || + url.startsWith('https://api.cyclocity.fr/contracts/lillestrom/'), ), ); }); +test('bike share over Stavanger searches Kolumbus Bysykkel', async () => { + const result = await composeCatalog({ + tools: coreTools, + services: { + bikeshare: { + systems: GBFS_CITY_REGISTRY, + getStations: async () => ({ data: { stations: [] } }), + }, + }, + }).call('get_bike_share', { + area: { lat: 58.97, lon: 5.7331, radius_km: 5 }, + }); + assert.deepEqual(result.data.systems, ['Stavanger, Norway']); +}); + test('traffic counts only roads inside a radius area and reports missing tiles', async () => { const flow = (segments, partial = false) => ({ getStatus: async () => ({ hasKey: true }), From 2e50836b080c00ef20894aa940bfa63c75f23c54 Mon Sep 17 00:00:00 2001 From: SM-26 <39513660+SM-26@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:43:36 +0200 Subject: [PATCH 29/95] feat: add a Docker install path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vite's dev server is the whole app here — it serves the client and runs every API proxy as middleware — so one container running `npm run dev` covers it, and HOST/PORT are already env-driven. Three details are deliberate: - The port publishes to 127.0.0.1. Docker's usual "4173:4173" binds every host interface, which would silently make the README's LAN warning the default for a server that brokers configured provider keys. - The repo is bind-mounted, so keys saved from Provider Settings, .gev-cache and .gev-logs persist on the host exactly as they do under `npm run dev`. node_modules stays in the image so a host install cannot shadow the Linux-built native deps. - The container runs as the host user, since everything it writes lands in that bind mount. Vite's dep optimizer still needs to write inside the image's node_modules, hence the two loosened directories. Puppeteer's Chromium download is skipped: it is only needed by `npm run test:track`, which the README points at the host. Co-Authored-By: Claude Opus 5 --- .dockerignore | 6 ++++++ CHANGELOG.md | 6 ++++++ Dockerfile | 28 ++++++++++++++++++++++++++++ README.md | 24 ++++++++++++++++++++++++ compose.yaml | 21 +++++++++++++++++++++ 5 files changed, 85 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 compose.yaml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..493b503c9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +node_modules +.git +.env +.gev-cache +.gev-logs +dist diff --git a/CHANGELOG.md b/CHANGELOG.md index 68ab1c52d..d5d88e9dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,12 @@ [TheSmokeDev](https://github.com/TheSmokeDev) for the credential handling from #653. +- **Docker install path.** `docker compose up` builds the pinned Node image and + serves the app on `http://localhost:4173` with no local Node install. The repo + is bind-mounted and the container runs as the host user, so `.env`, + `.gev-cache`, and `.gev-logs` persist on the host exactly as they do under + `npm run dev`. + - Stdio servers from one install share a panel key so a page read from one process can make requests through another. Concurrent malformed-key repairs now select one winner; unavailable storage or a busy repair retains the diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..b4706ffdb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +# syntax=docker/dockerfile:1 +# +# The dev server is the whole app: Vite serves the client AND runs the API +# proxies (ADS-B, AIS, Overpass, OpenAI Realtime, ...) as middleware, so there +# is nothing else to containerise. See compose.yaml for the usual entry point. +FROM node:24-slim + +# Puppeteer is only used by the QA scripts; skip its ~150MB Chromium download. +ENV PUPPETEER_SKIP_DOWNLOAD=1 +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN npm ci +# compose runs the container as the host user so bind-mounted files stay +# yours, but node_modules is baked in as root. Vite only needs to write its +# dep-optimizer cache, so open up those two directories rather than chowning +# a few hundred MB to a UID we cannot know at build time. +RUN mkdir -p node_modules/.vite && chmod 777 node_modules node_modules/.vite + +COPY . . + +# Bind to all interfaces, otherwise the published port hits nothing. Vite also +# relaxes allowedHosts when HOST is 0.0.0.0, which is what lets the browser +# reach it as "localhost". +ENV HOST=0.0.0.0 +ENV PORT=4173 +EXPOSE 4173 +CMD ["npm", "run", "dev"] diff --git a/README.md b/README.md index 5d97dfb2f..3a21bd189 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,30 @@ See [docs/PERFORMANCE.md](docs/PERFORMANCE.md). **macOS shortcut:** `./scripts/dev-fresh.sh` clears the Vite cache and pulls any configured keys straight from the Keychain. It starts keyless too. +### Path 3 — Docker + +No Node install needed; the container publishes the same dev server. + +```bash +git clone https://github.com/bilawalsidhu/gods-eye-view.git +cd gods-eye-view +docker compose up +``` + +Open **`http://localhost:4173`** as above. The repo is bind-mounted, so `.env`, +`.gev-cache` and `.gev-logs` live on the host and source edits still hot-reload. +The container binds `0.0.0.0` *inside* the container, but the port is published +to `127.0.0.1` only, so the default matches the native one: nobody else can +reach your server. Widening that mapping puts you under +[Sharing an instance](#-sharing-an-instance) — keys and all. + +`node_modules` deliberately stays inside the image, so it survives in an +anonymous volume across restarts: after a `git pull` that changes dependencies, +recreate it with `docker compose down -v && docker compose up --build`. The +image also skips Puppeteer's Chromium download, which keeps it small but leaves +`npm run test:track` to the host — `npm run build` and `npm test` run fine in +the container. + ### Then power it up — in the app, not in a file Keys are upgrades, not prerequisites. When you want one, click the **POWER UP** diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 000000000..7826bf376 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,21 @@ +services: + gods-eye-view: + build: . + # Everything the app writes (.env with your keys, .gev-cache, .gev-logs) + # lands in the bind-mounted repo, so run as you, not root. UID/GID are + # unset on Docker Desktop, where the 1000 fallback is ignored anyway. + user: "${UID:-1000}:${GID:-1000}" + ports: + # Publish to loopback only. Docker's default ("4173:4173") would bind + # every host interface, which quietly turns the LAN warning in the README + # into the default: this server brokers whatever provider keys you have + # configured to anyone who can reach it. Widen this deliberately, not by + # accident. + - "127.0.0.1:4173:4173" + volumes: + # Bind the source so edits hot-reload and so keys pasted into the setup + # UI land in the .env on your host instead of vanishing with the + # container. node_modules stays in the image: sharp and friends are + # compiled for Linux and must not be shadowed by a host install. + - .:/app + - /app/node_modules From 1c6c291774e0efd479ad9b54d71e6813f87d73cf Mon Sep 17 00:00:00 2001 From: SM-26 <39513660+SM-26@users.noreply.github.com> Date: Fri, 25 Sep 2026 11:31:07 +0200 Subject: [PATCH 30/95] fix(docker): let Provider Settings admit the host through the published port Docker delivers the host's own browser to the container from the container network's gateway, never from 127.0.0.1, so the loopback-only admission gate answered every POWER UP request with 403 and no key could be saved. The README and CHANGELOG claimed otherwise; both are corrected here. - keySetupCore: GEV_KEY_SETUP_TRUSTED_PEERS, an exact-address list (no wildcards, prefixes or hostnames) admitted alongside loopback. Unset trusts nothing, and the sharing, Host, Origin, Content-Type and proxy-header checks all still apply to a listed address. Twelve assertions pin each refusal. - scripts/docker-start.mjs: the container entry point, shaped after scripts/pinokio-start.mjs. It reads the default gateway from /proc/net/route, trusts exactly that address, and starts Vite. No gateway (--network none) trusts nothing. The pure helpers are unit-tested and a contract test pins the Dockerfile CMD and the loopback-only publish. - Dockerfile: install git. `npm test` builds fixtures with `git init`, so the suite failed 71 tests inside the image and the README's claim that it runs in the container was wrong. - SECURITY.md and .env.example document the setting and why it is safe only behind a 127.0.0.1 publish; README gains the Linux/NAS id note and .dockerignore drops macOS AppleDouble sidecars, both reported on the PR. Verified in Docker on a clean build: saving a key from the panel writes the bind-mounted .env (0600 in the container), it survives `compose down`/`up`, and REMOVE comments it back out. A foreign Origin, a foreign Host, a proxy header and a sibling container on the same network are each refused. `npm run doctor`, `npm test` (5067 passed, 1 skipped, 0 failed) and `npm run build` are green inside the container. Reported-by: @jklaze Co-Authored-By: Bogdan Petrut Co-Authored-By: Claude Opus 5 --- .dockerignore | 3 + .env.example | 7 ++ CHANGELOG.md | 10 ++- Dockerfile | 9 ++- README.md | 35 ++++++++-- SECURITY.md | 11 +++ scripts/docker-start.mjs | 132 ++++++++++++++++++++++++++++++++++++ src/dockerLauncher.test.mjs | 81 ++++++++++++++++++++++ src/keySetupCore.mjs | 32 ++++++++- src/keySetupCore.test.mjs | 17 +++++ 10 files changed, 328 insertions(+), 9 deletions(-) create mode 100644 scripts/docker-start.mjs create mode 100644 src/dockerLauncher.test.mjs diff --git a/.dockerignore b/.dockerignore index 493b503c9..3f5bc8a9d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,6 @@ node_modules .gev-cache .gev-logs dist +# AppleDouble sidecars from a macOS `tar` copy; `npm test` would otherwise +# collect `._*.test.mjs` as test files. +._* diff --git a/.env.example b/.env.example index c89680b30..825910d3d 100644 --- a/.env.example +++ b/.env.example @@ -95,6 +95,13 @@ OPENAI_HUD_SUMMARY_MODEL=gpt-5-nano # spend protection set a provider-side budget: OpenAI platform -> Settings -> # Limits (usage limits). # GEV_RATELIMIT_OPENAI_PER_MIN=30 # the default; uncomment only to change it +# Optional: exact socket addresses (comma-separated, no wildcards or prefixes) +# that Provider Settings treats as this machine in addition to loopback. Leave +# unset. The Docker launcher sets it to the container's gateway at start, +# because that is where Docker delivers your own browser from; it is safe only +# while the port is published on 127.0.0.1. Read SECURITY.md before setting it +# by hand. +# GEV_KEY_SETUP_TRUSTED_PEERS= # Optional: Mapillary imagery for the Street Level layer. Free client token from # https://www.mapillary.com/dashboard/developers (register an app, copy its diff --git a/CHANGELOG.md b/CHANGELOG.md index d5d88e9dc..658749625 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,7 +111,15 @@ serves the app on `http://localhost:4173` with no local Node install. The repo is bind-mounted and the container runs as the host user, so `.env`, `.gev-cache`, and `.gev-logs` persist on the host exactly as they do under - `npm run dev`. + `npm run dev` — Provider Settings included. The launcher + (`scripts/docker-start.mjs`) trusts the container's gateway address for that + panel, since that is where Docker delivers the host's own browser from; the + opt-in behind it, `GEV_KEY_SETUP_TRUSTED_PEERS`, takes exact addresses only + and is unset (so loopback-only, unchanged) under every other launcher. + Thanks to [jklaze](https://github.com/jklaze) for finding and diagnosing the + container-to-host 403, and to [14traits](https://github.com/14traits), + [PoBruno](https://github.com/PoBruno) and [milu70hub](https://github.com/milu70hub) + for testing on Apple silicon, Docker Engine on Linux, and NAS hosts. - Stdio servers from one install share a panel key so a page read from one process can make requests through another. Concurrent malformed-key repairs diff --git a/Dockerfile b/Dockerfile index b4706ffdb..de89347f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,10 @@ FROM node:24-slim ENV PUPPETEER_SKIP_DOWNLOAD=1 WORKDIR /app +# `npm test` builds fixtures with `git init`, so the suite cannot run in a +# container without it (71 failures on node:24-slim, all ENOENT on git). +RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* + COPY package.json package-lock.json ./ RUN npm ci # compose runs the container as the host user so bind-mounted files stay @@ -25,4 +29,7 @@ COPY . . ENV HOST=0.0.0.0 ENV PORT=4173 EXPOSE 4173 -CMD ["npm", "run", "dev"] +# The launcher is `npm run dev` plus one step: it trusts the container's +# gateway address for Provider Settings, because that is where Docker delivers +# the host's own browser from (never 127.0.0.1). See scripts/docker-start.mjs. +CMD ["node", "scripts/docker-start.mjs"] diff --git a/README.md b/README.md index 3a21bd189..5ab3ae693 100644 --- a/README.md +++ b/README.md @@ -156,11 +156,34 @@ docker compose up ``` Open **`http://localhost:4173`** as above. The repo is bind-mounted, so `.env`, -`.gev-cache` and `.gev-logs` live on the host and source edits still hot-reload. -The container binds `0.0.0.0` *inside* the container, but the port is published -to `127.0.0.1` only, so the default matches the native one: nobody else can -reach your server. Widening that mapping puts you under -[Sharing an instance](#-sharing-an-instance) — keys and all. +`.gev-cache` and `.gev-logs` live on the host, edits still hot-reload, and keys +you save from Provider Settings survive `docker compose down`. The port is +published to `127.0.0.1` only, so the default matches the native one. + +Publishing it wider is the [Sharing an instance](#-sharing-an-instance) opt-in, +and it hands out **Provider Settings** too: the launcher trusts the container +gateway as this machine, and a wider publish delivers everyone else from that +same address. For an instance you mean to share, put the keys in `.env` by hand +instead. [SECURITY.md](SECURITY.md) has the detail. + +
+File ownership on Linux and NAS hosts + +On Linux, export your ids first so the files the container writes stay yours: +`export UID GID` before `docker compose up`. On a NAS or anywhere the share has +its own owner, set them explicitly instead (`UID=1026 GID=100 docker compose +up`) — Docker Desktop on macOS and Windows maps ownership for you. + +
+ +
+IPv6-only networks, rebuilding, and the test suites + +The launcher reads the IPv4 default gateway, so an IPv6-only Docker network +finds none: it logs `No default gateway; Provider Settings stays loopback-only` +and the panel refuses every save. Put your keys in `.env` by hand there, or name +the gateway yourself with `GEV_KEY_SETUP_TRUSTED_PEERS` after reading +[SECURITY.md](SECURITY.md). `node_modules` deliberately stays inside the image, so it survives in an anonymous volume across restarts: after a `git pull` that changes dependencies, @@ -169,6 +192,8 @@ image also skips Puppeteer's Chromium download, which keeps it small but leaves `npm run test:track` to the host — `npm run build` and `npm test` run fine in the container. +
+ ### Then power it up — in the app, not in a file Keys are upgrades, not prerequisites. When you want one, click the **POWER UP** diff --git a/SECURITY.md b/SECURITY.md index 53437a83c..23dfee4e6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -84,6 +84,17 @@ The dev server is a **key broker**: every server-side key above is spendable by - **Local-only by default.** `./scripts/dev-fresh.sh` (and the Vite config itself) bind to `localhost`, so only your machine can reach the server — and only local names are accepted (`allowedHosts` stays restricted, which also blunts DNS-rebinding tricks). Binding to all interfaces keeps that restriction: IP addresses and `localhost` work, and LAN hostnames must be listed in `GEV_ALLOWED_HOSTS` (for example `GEV_ALLOWED_HOSTS=globe.lan`); suffix and wildcard entries are ignored. - **LAN exposure is an explicit opt-in**: `HOST=0.0.0.0 ./scripts/dev-fresh.sh`. The launcher prints a prominent warning plus your LAN URL. Understand what opting in means: **every device on that network can drive the proxies and spend your OpenAI / Google / OpenSky / AISStream / TomTom / FIRMS quota** for as long as the server runs. Do this only on networks you trust. - **App-level throttles (on by default):** `GEV_RATELIMIT_OPENAI_PER_MIN` (default `30`) and `GEV_RATELIMIT_GOOGLE_PER_MIN` (default `120`) — the values the Pinokio build already ships — cap the cost-bearing endpoints per client IP per minute (over-limit requests receive a sanitized `429`). Both defaults sit well above what the app itself generates; set either to exactly `0` to run unthrottled. They are **per-IP, process-local, in-memory guards** — they reset on restart, a caller with several addresses gets a bucket per address, and they are **not billing caps**. +- **Docker trusts one address, and only because the port is on loopback.** + Docker delivers the host's own browser to the container from the container + network's gateway, never from `127.0.0.1`, so a loopback-only Provider + Settings refuses the machine running it. `scripts/docker-start.mjs` therefore + sets `GEV_KEY_SETUP_TRUSTED_PEERS` to exactly that gateway address. The list + takes exact addresses only (no wildcards, prefixes or hostnames), is unset + under every other launcher, and relaxes none of the sharing, Host, Origin, + Content-Type or proxy-header checks. It is safe precisely because + `compose.yaml` publishes on `127.0.0.1`: **widen that mapping and every LAN + client arrives from the same gateway address, which hands them Provider + Settings too.** Treat publishing beyond loopback as the LAN opt-in above. - **Provider-side budgets are the real backstop.** For hard spend protection, configure limits where the money is: OpenAI platform usage limits, Google Cloud budget alerts + per-API quotas, and equivalent controls for any other keyed provider. - **Pinokio LAN and Cloudflare sharing are refused.** The current supported Pinokio release re-reads sharing state when an app registers its Open URL and diff --git a/scripts/docker-start.mjs b/scripts/docker-start.mjs new file mode 100644 index 000000000..25ddf76ba --- /dev/null +++ b/scripts/docker-start.mjs @@ -0,0 +1,132 @@ +#!/usr/bin/env node +/** + * Docker launcher — the container's entry point (see Dockerfile). + * + * `compose.yaml` publishes the port on the host's loopback, but Docker still + * delivers the host's own browser to the container FROM the container + * network's gateway address, never from 127.0.0.1. Provider Settings admits + * loopback only, so without this the POWER UP panel refuses the very machine + * running the container. This launcher reads the gateway out of the kernel + * routing table and declares exactly that one address trusted, then starts + * Vite the way scripts/pinokio-start.mjs does. + */ +import { readFileSync, realpathSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { isDirectInvocation } from './pinokio-install.mjs'; +import { loadViteFromCanonicalRoot } from './pinokio-start.mjs'; + +const MODULE_PATH = fileURLToPath(import.meta.url); +const ROOT = realpathSync(path.resolve(path.dirname(MODULE_PATH), '..')); +const ROUTE_TABLE = '/proc/net/route'; +/** Host names the native launcher accepts; the container matches it. */ +const LOCAL_ALLOWED_HOSTS = Object.freeze(['localhost', '127.0.0.1', '.local']); + +/** + * Extract the IPv4 default gateway from Linux `/proc/net/route` text. The + * addresses there are little-endian hex and only the default route (a + * destination of 00000000 on an interface flagged up) names a gateway. + * Returns null when there is none — the `--network none` case, which must + * never fabricate an address. + * @param {string} text Raw routing-table contents. + * @returns {string|null} Dotted-quad gateway, or null. + */ +export function defaultGatewayFromRouteTable(text) { + const lines = String(text || '').split(/\r?\n/).slice(1); + for (const line of lines) { + const fields = line.trim().split(/\s+/); + if (fields.length < 4) continue; + const [, destination, gateway, flags] = fields; + if (destination !== '00000000' || !/^[0-9a-f]{8}$/i.test(gateway)) continue; + if ((Number.parseInt(flags, 16) & 0x1) === 0) continue; // RTF_UP + const octets = [6, 4, 2, 0].map((offset) => + Number.parseInt(gateway.slice(offset, offset + 2), 16), + ); + if (octets.every((octet) => octet === 0)) continue; + return octets.join('.'); + } + return null; +} + +/** + * Decide this launch's trusted-peer list. An operator value always wins, so a + * compose override stays authoritative; otherwise the gateway alone is + * trusted. No gateway trusts nothing and Provider Settings stays loopback-only, + * exactly as under `npm run dev`. + * @param {{env?: NodeJS.ProcessEnv, routeTable?: string}} input + * @returns {string} Value for GEV_KEY_SETUP_TRUSTED_PEERS ('' trusts nothing). + */ +export function resolveTrustedPeers({ env = process.env, routeTable = '' } = {}) { + const explicit = String(env.GEV_KEY_SETUP_TRUSTED_PEERS ?? '').trim(); + if (explicit) return explicit; + return defaultGatewayFromRouteTable(routeTable) ?? ''; +} + +/** + * Host headers this container answers to. A wildcard bind is unavoidable + * inside a container, but build/vite.js reads that as "allow every Host" + * (allowedHosts: true), dropping the DNS-rebinding protection `npm run dev` + * keeps. The published port only ever delivers the host's own browser, so pin + * the same names the native launcher uses, and still honour an operator's + * GEV_ALLOWED_HOSTS so a LAN hostname stays addable here too. See issue #21. + * @param {NodeJS.ProcessEnv} [env] + * @returns {string[]} + */ +export function resolveAllowedHosts(env = process.env) { + const configured = String(env.GEV_ALLOWED_HOSTS ?? '') + .split(',') + .map((host) => host.trim()) + .filter(Boolean); + return [...new Set([...LOCAL_ALLOWED_HOSTS, ...configured])]; +} + +function readRouteTable() { + try { + return readFileSync(ROUTE_TABLE, 'utf8'); + } catch { + return ''; + } +} + +async function start() { + const trustedPeers = resolveTrustedPeers({ routeTable: readRouteTable() }); + if (trustedPeers) { + // Set before Vite is imported: the dev server snapshots process.env during + // configuration, and the admission gate reads it per request. + process.env.GEV_KEY_SETUP_TRUSTED_PEERS = trustedPeers; + console.log(`[Docker] Provider Settings trusts the container gateway (${trustedPeers}).`); + } else { + console.log('[Docker] No default gateway; Provider Settings stays loopback-only.'); + } + + const port = Number.parseInt(process.env.PORT, 10) || 4173; + const { createServer } = await loadViteFromCanonicalRoot(ROOT); + const server = await createServer({ + root: ROOT, + server: { + // The container's own namespace. compose publishes this to the host's + // 127.0.0.1 only; widening that mapping is the LAN opt-in. + host: '0.0.0.0', + port, + strictPort: true, + allowedHosts: resolveAllowedHosts(), + }, + }); + await server.listen(); + server.printUrls(); + console.log(`[Docker] Ready at http://localhost:${port}/`); + + for (const signal of ['SIGINT', 'SIGTERM']) { + process.once(signal, async () => { + await server.close(); + process.exit(0); + }); + } +} + +if (isDirectInvocation(process.argv[1], MODULE_PATH)) { + start().catch((error) => { + console.error(`[Docker] Start refused: ${error.message}`); + process.exitCode = 1; + }); +} diff --git a/src/dockerLauncher.test.mjs b/src/dockerLauncher.test.mjs new file mode 100644 index 000000000..85ac525a4 --- /dev/null +++ b/src/dockerLauncher.test.mjs @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; +import { + defaultGatewayFromRouteTable, + resolveAllowedHosts, + resolveTrustedPeers, +} from '../scripts/docker-start.mjs'; + +// The shape of /proc/net/route inside a bridge-networked container: +// little-endian hex addresses, the default route first, an interface route +// after. +const BRIDGE_ROUTES = [ + 'Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT', + 'eth0\t00000000\t0100A8C0\t0003\t0\t0\t0\t00000000\t0\t0\t0', + 'eth0\t0000A8C0\t00000000\t0001\t0\t0\t0\t00F0FFFF\t0\t0\t0', + '', +].join('\n'); + +test('the container gateway is decoded from the kernel routing table', () => { + assert.equal(defaultGatewayFromRouteTable(BRIDGE_ROUTES), '192.168.0.1', 'little-endian hex, default route only'); + assert.equal( + defaultGatewayFromRouteTable(BRIDGE_ROUTES.replace('0100A8C0', '011FAC')), + null, + 'a malformed gateway field is not an address', + ); + assert.equal( + defaultGatewayFromRouteTable(BRIDGE_ROUTES.replace('\t0003\t', '\t0002\t')), + null, + 'a default route that is not up is ignored', + ); + assert.equal( + defaultGatewayFromRouteTable(BRIDGE_ROUTES.replace('0100A8C0', '00000000')), + null, + 'a zero gateway is not an address', + ); + assert.equal( + defaultGatewayFromRouteTable( + BRIDGE_ROUTES.split('\n').filter((line) => !line.includes('00000000\t0100A8C0')).join('\n'), + ), + null, + '--network none has no default route and must not invent one', + ); + assert.equal(defaultGatewayFromRouteTable(''), null); + assert.equal(defaultGatewayFromRouteTable(undefined), null); +}); + +test('an operator list wins, and no gateway trusts nothing', () => { + assert.equal(resolveTrustedPeers({ env: {}, routeTable: BRIDGE_ROUTES }), '192.168.0.1'); + assert.equal(resolveTrustedPeers({ env: { GEV_KEY_SETUP_TRUSTED_PEERS: ' 10.9.8.7 ' }, routeTable: BRIDGE_ROUTES }), '10.9.8.7'); + assert.equal( + resolveTrustedPeers({ env: { GEV_KEY_SETUP_TRUSTED_PEERS: ' ' }, routeTable: BRIDGE_ROUTES }), + '192.168.0.1', + 'a blank override falls through to the gateway', + ); + assert.equal(resolveTrustedPeers({ env: {}, routeTable: '' }), '', 'loopback-only when there is no gateway'); +}); + +test('the container answers the same host names as the native launcher', () => { + assert.deepEqual(resolveAllowedHosts({}), ['localhost', '127.0.0.1', '.local'], 'a wildcard bind never means "any Host header"'); + assert.deepEqual( + resolveAllowedHosts({ GEV_ALLOWED_HOSTS: ' globe.lan , nas.internal ' }), + ['localhost', '127.0.0.1', '.local', 'globe.lan', 'nas.internal'], + 'an operator can still add a LAN hostname', + ); + assert.deepEqual( + resolveAllowedHosts({ GEV_ALLOWED_HOSTS: ' , localhost , ' }), + ['localhost', '127.0.0.1', '.local'], + 'blank and duplicate entries change nothing', + ); +}); + +test('the Docker install path keeps its loopback-only, launcher-owned contract', () => { + const dockerfile = readFileSync(new URL('../Dockerfile', import.meta.url), 'utf8'); + assert.match(dockerfile, /^CMD \["node", "scripts\/docker-start\.mjs"\]$/m, 'the launcher is the container entry point'); + + const compose = readFileSync(new URL('../compose.yaml', import.meta.url), 'utf8'); + assert.match(compose, /^\s+- "127\.0\.0\.1:4173:4173"$/m, 'the port is published on the host loopback only'); + assert.doesNotMatch(compose, /GEV_KEY_SETUP_TRUSTED_PEERS/, 'trust is derived at launch, never hard-coded'); + assert.doesNotMatch(compose, /network_mode/, 'the container keeps its own network namespace'); +}); diff --git a/src/keySetupCore.mjs b/src/keySetupCore.mjs index 6c3cecb2e..73044f003 100644 --- a/src/keySetupCore.mjs +++ b/src/keySetupCore.mjs @@ -126,6 +126,29 @@ const LOCAL_HOSTNAMES = new Set(['localhost', '127.0.0.1', '::1', '[::1]']); /** Socket addresses that count as this machine. */ const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']); +/** + * Whether a socket address is one the operator has explicitly declared to be + * this machine through GEV_KEY_SETUP_TRUSTED_PEERS: a comma-separated list of + * EXACT addresses, never wildcards, prefixes or hostnames. Unset trusts + * nothing, which is every launcher but one. + * + * The shipped use is the Docker launcher (scripts/docker-start.mjs). A port + * published on the host's loopback still reaches the container FROM the + * container network's gateway, never from 127.0.0.1, so the loopback check + * alone refuses the very machine running the server. Admitting that one + * address relaxes nothing else: the sharing, Host, Origin, Content-Type and + * proxy-header checks all still apply to it. + */ +function isTrustedPeerAddress(remoteAddress, env) { + const normalize = (value) => + String(value || '').trim().toLowerCase().replace(/^::ffff:/, ''); + const address = normalize(remoteAddress); + if (!address) return false; + return String(env?.GEV_KEY_SETUP_TRUSTED_PEERS || '') + .split(',') + .some((entry) => entry.trim() !== '' && normalize(entry) === address); +} + /** Parse an exact local request authority from a Host header. */ function localAuthority(hostHeader, protocol) { const raw = String(hostHeader || '') @@ -224,7 +247,9 @@ export function isSharingEnabled(env = {}) { * outright — a credential-writing endpoint has no business existing on a * shared instance, and tunnel traffic reaches the server FROM loopback, so * the socket check below cannot carry that boundary alone; - * - loopback socket: refuses LAN peers when the server is bound wide; + * - loopback socket: refuses LAN peers when the server is bound wide. The + * only exception is an exact address listed in + * GEV_KEY_SETUP_TRUSTED_PEERS, and every check below still applies to it; * - local Host header: tunnel and DNS-rebinding traffic carries a foreign * Host even when the socket says loopback; * - exact same Origin on POST: a hostile web page can make a browser POST to @@ -274,7 +299,10 @@ export function admitKeySetupRequest({ error: 'Provider Settings is disabled while sharing is enabled', }; } - if (!LOOPBACK_ADDRESSES.has(String(remoteAddress || ''))) { + if ( + !LOOPBACK_ADDRESSES.has(String(remoteAddress || '')) && + !isTrustedPeerAddress(remoteAddress, env) + ) { return { ok: false, status: 403, diff --git a/src/keySetupCore.test.mjs b/src/keySetupCore.test.mjs index 05f406009..0b66e59b8 100644 --- a/src/keySetupCore.test.mjs +++ b/src/keySetupCore.test.mjs @@ -258,6 +258,23 @@ test('the admission gate refuses every non-local shape, one assertion per refusa assert.equal(admitKeySetupRequest({ ...local, env: { PINOKIO_SHARE_LOCAL: '1' } }).ok, false, 'LAN sharing disables the surface'); // A LAN peer reaching a wide-bound server. assert.equal(admitKeySetupRequest({ ...local, remoteAddress: '192.168.1.20' }).ok, false, 'non-loopback socket refused'); + // A container publishes its port on the host's loopback, yet the host's own + // browser still arrives from the container network's gateway. Only an exact + // address the operator listed admits it, and nothing else is relaxed. + const trusted = { GEV_KEY_SETUP_TRUSTED_PEERS: ' 10.0.0.9 , 172.29.73.1 ' }; + const gateway = { ...local, remoteAddress: '172.29.73.1' }; + assert.equal(admitKeySetupRequest(gateway).ok, false, 'container gateway refused without the opt-in'); + assert.equal(admitKeySetupRequest({ ...gateway, env: trusted }).ok, true, 'listed peer admitted'); + assert.equal(admitKeySetupRequest({ ...gateway, remoteAddress: '::ffff:172.29.73.1', env: trusted }).ok, true, 'IPv4-mapped form of a listed peer admitted'); + assert.equal(admitKeySetupRequest({ ...local, remoteAddress: '192.168.1.20', env: trusted }).ok, false, 'unlisted peer still refused'); + assert.equal(admitKeySetupRequest({ ...gateway, env: { GEV_KEY_SETUP_TRUSTED_PEERS: '*' } }).ok, false, 'wildcards are not addresses'); + assert.equal(admitKeySetupRequest({ ...gateway, env: { GEV_KEY_SETUP_TRUSTED_PEERS: '172.29.73.0/24' } }).ok, false, 'prefixes are not addresses'); + assert.equal(admitKeySetupRequest({ ...gateway, env: { GEV_KEY_SETUP_TRUSTED_PEERS: '' } }).ok, false, 'an empty list trusts nothing'); + assert.equal(admitKeySetupRequest({ ...gateway, env: { GEV_KEY_SETUP_TRUSTED_PEERS: ' , , ' } }).ok, false, 'blank entries trust nothing'); + assert.equal(admitKeySetupRequest({ ...gateway, env: { ...trusted, PINOKIO_SHARE_LOCAL: '1' } }).ok, false, 'sharing still disables the surface for a listed peer'); + assert.equal(admitKeySetupRequest({ ...gateway, env: trusted, hostHeader: 'abc.trycloudflare.com' }).ok, false, 'a listed peer still needs a local Host'); + assert.equal(admitKeySetupRequest({ ...gateway, env: trusted, origin: 'https://evil.example' }).ok, false, 'a listed peer still needs the exact Origin'); + assert.equal(admitKeySetupRequest({ ...gateway, env: trusted, proxyHeaders: { 'x-forwarded-for': '203.0.113.5' } }).ok, false, 'a listed peer still refuses proxied requests'); // Tunnel and DNS-rebinding traffic carries a foreign Host over a loopback socket. assert.equal(admitKeySetupRequest({ ...local, hostHeader: 'abc.trycloudflare.com' }).ok, false, 'foreign Host refused'); assert.equal(admitKeySetupRequest({ ...local, hostHeader: 'workstation.local:4173' }).ok, false, 'non-localhost hostnames refused'); From 79a6a0db5cd86f130f73e2c4d29e32bcd47cf343 Mon Sep 17 00:00:00 2001 From: SM-26 <39513660+SM-26@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:39:32 +0200 Subject: [PATCH 31/95] fix(docker): defer to main's Host allowlist and name an unwritable checkout Two findings from PoBruno's Docker Engine on Linux run (#208), and one from re-running the suite in the container after rebasing: - The launcher pinned its own allowedHosts, including '.local'. Vite reads a leading dot as a suffix match, so `Host: anything.local` reached the page and the data proxies. Main has since moved the policy into build/allowedHosts.js, which drops suffix entries on purpose and applies before the app's routes, so the launcher's override is deleted rather than patched. As a side effect GEV_ALLOWED_HOSTS now also works from `.env`, as it does under `npm run dev`. - A checkout the container cannot write - typically a clone made as root on a NAS or Proxmox host - made Save Keys answer 500 and never created .gev-cache, with nothing logged. The launcher now warns at start, naming the owner, the ids it runs as, and the two fixes. - On Docker Desktop the bind-mounted repo always shows an owner other than the container user, so git refuses it as "dubious ownership" and the tests that shell out to git fail (src/data/bundledJson.test.mjs, new on main). The image now marks /app, and only /app, as a safe directory. Verified in a container on this base: `anything.local` 403 on the page and on /api routes, localhost 200, GEV_ALLOWED_HOSTS honoured from the environment and from .env, the warning on a root-owned /app run as 1000:1000, and bundledJson.test.mjs 6/6. Reported-by: PoBruno Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 5 ++++ scripts/docker-start.mjs | 51 ++++++++++++++++++++++++------------- src/dockerLauncher.test.mjs | 22 +++++++--------- src/keySetupCore.mjs | 5 +++- 4 files changed, 51 insertions(+), 32 deletions(-) diff --git a/Dockerfile b/Dockerfile index de89347f9..b1b0876d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,11 @@ WORKDIR /app # `npm test` builds fixtures with `git init`, so the suite cannot run in a # container without it (71 failures on node:24-slim, all ENOENT on git). RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* +# The repo is bind-mounted, so inside the container it can show an owner other +# than the user the container runs as - always, on Docker Desktop. git then +# refuses it as "dubious ownership", which fails the tests that shell out to +# git. It is the user's own checkout, so trust that one path. +RUN git config --system --add safe.directory /app COPY package.json package-lock.json ./ RUN npm ci diff --git a/scripts/docker-start.mjs b/scripts/docker-start.mjs index 25ddf76ba..47519af97 100644 --- a/scripts/docker-start.mjs +++ b/scripts/docker-start.mjs @@ -10,7 +10,7 @@ * routing table and declares exactly that one address trusted, then starts * Vite the way scripts/pinokio-start.mjs does. */ -import { readFileSync, realpathSync } from 'node:fs'; +import { accessSync, constants, readFileSync, realpathSync, statSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { isDirectInvocation } from './pinokio-install.mjs'; @@ -19,8 +19,6 @@ import { loadViteFromCanonicalRoot } from './pinokio-start.mjs'; const MODULE_PATH = fileURLToPath(import.meta.url); const ROOT = realpathSync(path.resolve(path.dirname(MODULE_PATH), '..')); const ROUTE_TABLE = '/proc/net/route'; -/** Host names the native launcher accepts; the container matches it. */ -const LOCAL_ALLOWED_HOSTS = Object.freeze(['localhost', '127.0.0.1', '.local']); /** * Extract the IPv4 default gateway from Linux `/proc/net/route` text. The @@ -63,21 +61,21 @@ export function resolveTrustedPeers({ env = process.env, routeTable = '' } = {}) } /** - * Host headers this container answers to. A wildcard bind is unavoidable - * inside a container, but build/vite.js reads that as "allow every Host" - * (allowedHosts: true), dropping the DNS-rebinding protection `npm run dev` - * keeps. The published port only ever delivers the host's own browser, so pin - * the same names the native launcher uses, and still honour an operator's - * GEV_ALLOWED_HOSTS so a LAN hostname stays addable here too. See issue #21. - * @param {NodeJS.ProcessEnv} [env] - * @returns {string[]} + * A checkout the container cannot write fails quietly: Provider Settings + * answers 500 and `.gev-cache` is never created, with nothing in the log. The + * usual cause is a clone made as root on a NAS or Proxmox host while the + * container runs as 1000:1000. Say so at start, naming both sets of ids. + * @param {{root: string, writable: boolean, owner: {uid: number, gid: number}, runAs: {uid: number, gid: number}}} input + * @returns {string|null} A warning, or null when the checkout is writable. */ -export function resolveAllowedHosts(env = process.env) { - const configured = String(env.GEV_ALLOWED_HOSTS ?? '') - .split(',') - .map((host) => host.trim()) - .filter(Boolean); - return [...new Set([...LOCAL_ALLOWED_HOSTS, ...configured])]; +export function unwritableRootWarning({ root, writable, owner, runAs }) { + if (writable) return null; + return ( + `[Docker] ${root} is owned by ${owner.uid}:${owner.gid} but the container runs as ` + + `${runAs.uid}:${runAs.gid}, so saving keys and caching will fail. Either give ` + + `the checkout to that user (chown -R ${runAs.uid}:${runAs.gid} on the host) or ` + + `run as its owner (env UID=${owner.uid} GID=${owner.gid} docker compose up)` + ); } function readRouteTable() { @@ -89,6 +87,24 @@ function readRouteTable() { } async function start() { + let writable = true; + try { + accessSync(ROOT, constants.W_OK); + } catch { + writable = false; + } + if (!writable) { + const { uid, gid } = statSync(ROOT); + console.warn( + unwritableRootWarning({ + root: ROOT, + writable, + owner: { uid, gid }, + runAs: { uid: process.getuid?.() ?? -1, gid: process.getgid?.() ?? -1 }, + }), + ); + } + const trustedPeers = resolveTrustedPeers({ routeTable: readRouteTable() }); if (trustedPeers) { // Set before Vite is imported: the dev server snapshots process.env during @@ -109,7 +125,6 @@ async function start() { host: '0.0.0.0', port, strictPort: true, - allowedHosts: resolveAllowedHosts(), }, }); await server.listen(); diff --git a/src/dockerLauncher.test.mjs b/src/dockerLauncher.test.mjs index 85ac525a4..ae9987b01 100644 --- a/src/dockerLauncher.test.mjs +++ b/src/dockerLauncher.test.mjs @@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs'; import test from 'node:test'; import { defaultGatewayFromRouteTable, - resolveAllowedHosts, + unwritableRootWarning, resolveTrustedPeers, } from '../scripts/docker-start.mjs'; @@ -56,18 +56,14 @@ test('an operator list wins, and no gateway trusts nothing', () => { assert.equal(resolveTrustedPeers({ env: {}, routeTable: '' }), '', 'loopback-only when there is no gateway'); }); -test('the container answers the same host names as the native launcher', () => { - assert.deepEqual(resolveAllowedHosts({}), ['localhost', '127.0.0.1', '.local'], 'a wildcard bind never means "any Host header"'); - assert.deepEqual( - resolveAllowedHosts({ GEV_ALLOWED_HOSTS: ' globe.lan , nas.internal ' }), - ['localhost', '127.0.0.1', '.local', 'globe.lan', 'nas.internal'], - 'an operator can still add a LAN hostname', - ); - assert.deepEqual( - resolveAllowedHosts({ GEV_ALLOWED_HOSTS: ' , localhost , ' }), - ['localhost', '127.0.0.1', '.local'], - 'blank and duplicate entries change nothing', - ); +test('an unwritable checkout is named at start instead of failing quietly', () => { + const ids = { owner: { uid: 0, gid: 0 }, runAs: { uid: 1000, gid: 1000 } }; + assert.equal(unwritableRootWarning({ root: '/app', writable: true, ...ids }), null, 'a writable checkout says nothing'); + const warning = unwritableRootWarning({ root: '/app', writable: false, ...ids }); + assert.match(warning, /owned by 0:0/, 'names the owner'); + assert.match(warning, /runs as 1000:1000/, 'names the ids the container runs as'); + assert.match(warning, /chown -R 1000:1000/, 'offers handing the checkout to the container user first'); + assert.match(warning, /env UID=0 GID=0 docker compose up/, 'or the env form, which bash does not ignore'); }); test('the Docker install path keeps its loopback-only, launcher-owned contract', () => { diff --git a/src/keySetupCore.mjs b/src/keySetupCore.mjs index 73044f003..7ecc58eb8 100644 --- a/src/keySetupCore.mjs +++ b/src/keySetupCore.mjs @@ -141,7 +141,10 @@ const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']); */ function isTrustedPeerAddress(remoteAddress, env) { const normalize = (value) => - String(value || '').trim().toLowerCase().replace(/^::ffff:/, ''); + String(value || '') + .trim() + .toLowerCase() + .replace(/^::ffff:/, ''); const address = normalize(remoteAddress); if (!address) return false; return String(env?.GEV_KEY_SETUP_TRUSTED_PEERS || '') From 3ade710ce0b783b7e1dd36433f45eaf9c7ac40e4 Mon Sep 17 00:00:00 2001 From: SM-26 <39513660+SM-26@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:53:08 +0200 Subject: [PATCH 32/95] docs(docker): say where data lives, and fix the UID/GID commands The Path 3 section assumed readers already knew how a bind-mounted container persists state (14traits, #208). It now lists what the app saves and where, with the NAS and reverse-proxy detail in the collapsed block that was already there, so the open part stays about the same length. It also fixes two commands that did not do what they said in bash, confirmed in bash and dash by PoBruno: `UID=1026 GID=100 docker compose up` prints "UID: readonly variable" and still runs as the caller's UID, and `export UID GID` exports an empty GID because bash does not define one. Both are now `env UID=... GID=...`, which `docker compose config` resolves to the requested ids. Suggested-by: 14traits Co-Authored-By: Claude Opus 5.5 --- README.md | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5ab3ae693..7ac0b8e3e 100644 --- a/README.md +++ b/README.md @@ -155,24 +155,37 @@ cd gods-eye-view docker compose up ``` -Open **`http://localhost:4173`** as above. The repo is bind-mounted, so `.env`, -`.gev-cache` and `.gev-logs` live on the host, edits still hot-reload, and keys -you save from Provider Settings survive `docker compose down`. The port is -published to `127.0.0.1` only, so the default matches the native one. +Open **`http://localhost:4173`** as above. Everything the app saves stays in the folder you cloned, so `docker compose down` never loses anything: -Publishing it wider is the [Sharing an instance](#-sharing-an-instance) opt-in, -and it hands out **Provider Settings** too: the launcher trusts the container -gateway as this machine, and a wider publish delivers everyone else from that -same address. For an instance you mean to share, put the keys in `.env` by hand -instead. [SECURITY.md](SECURITY.md) has the detail. +- `.env` - your API keys, whether saved from Provider Settings or typed in by hand +- `.gev-cache/` - downloaded map and satellite data, so later starts are faster +- `.gev-logs/` - logs + +Back up that folder and you've backed up everything; delete `.env` to forget your keys. Source edits still hot-reload. The port is published to `127.0.0.1` only, so the default matches the native one. + +Publishing it wider is the [Sharing an instance](#-sharing-an-instance) opt-in, and it hands out **Provider Settings** too: the launcher trusts the container gateway as this machine, and a wider publish delivers everyone else from that same address. For an instance you mean to share, put the keys in `.env` by hand instead. [SECURITY.md](SECURITY.md) has the detail.
-File ownership on Linux and NAS hosts +Running on Linux, a NAS, or behind a reverse proxy -On Linux, export your ids first so the files the container writes stay yours: -`export UID GID` before `docker compose up`. On a NAS or anywhere the share has -its own owner, set them explicitly instead (`UID=1026 GID=100 docker compose -up`) — Docker Desktop on macOS and Windows maps ownership for you. +Run it as the user who should own the files it writes. `id` shows the numbers; for yourself on Linux: + +```bash +env UID=$(id -u) GID=$(id -g) docker compose up +``` + +On a NAS, clone onto the share you want the data on and use the share owner's ids: + +```bash +cd /volume1/docker +git clone https://github.com/bilawalsidhu/gods-eye-view.git +cd gods-eye-view +env UID=1026 GID=100 docker compose up -d +``` + +Keep the `env`: bash treats `UID` as read-only, so a plain `UID=1026` in front of a command is silently ignored. If the container can't write the checkout, it says so at start. Docker Desktop on macOS and Windows maps ownership for you. + +Behind a reverse proxy, put your keys in `.env` or pass them as environment variables - Provider Settings refuses proxied requests on purpose - and add the hostname you serve it on, for example `GEV_ALLOWED_HOSTS=globe.example.lan` (see [Sharing an instance](#-sharing-an-instance)).
From 7e9b8d949fb3f6bf88d3f93954150289ca562c24 Mon Sep 17 00:00:00 2001 From: mateo monsalve Date: Sun, 11 Oct 2026 16:18:27 -0300 Subject: [PATCH 33/95] =?UTF-8?q?feat(bikeshare):=20add=20Bogot=C3=A1=20(T?= =?UTF-8?q?embici)=20and=20Santiago=20(Bike=20Ita=C3=BA)=20GBFS=20feeds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both systems publish keyless GBFS 2.3 feeds on publicbikesystem.net, the same platform as the existing Austin, Honolulu and Chattanooga entries (listed in MobilityData's systems.csv). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +++ DATA_SOURCES.md | 2 +- src/data/dataCredits.js | 2 +- src/layers/bikeshare/registry.js | 24 ++++++++++++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68ab1c52d..d83deed3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +- Add Bogotá (Tembici) and Santiago (Bike Itaú) to the GBFS bikeshare registry, + the layer's first stations in Colombia and Chile. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 0b8eac358..2226f527b 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -56,7 +56,7 @@ How to read this: | **DriveBC** (Government of British Columbia) | CCTV camera catalog + frames, British Columbia highways | [Open Government Licence – British Columbia](https://www2.gov.bc.ca/gov/content/data/open-data/open-government-licence-bc) — attribution REQUIRED | "DriveBC. Contains information licensed under the Open Government Licence – British Columbia" | | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | -| **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | +| **GBFS (Lyft / BCycle / PBSC)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle, Tembici / Bike Itaú) + its `license_url` | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | | **Metro Transit** (Metropolitan Council) | Transit layer live vehicles, Minneapolis–St Paul | Metro Transit / Metropolitan Council open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to Metro Transit / Metropolitan Council (courtesy text) | diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 2967f0a2b..48ad026fb 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -249,7 +249,7 @@ export const DATA_CREDITS = [ }, { key: 'gbfs', - html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)', + html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle, Tembici / Bike Itaú)', }, { key: 'osrm-routing', diff --git a/src/layers/bikeshare/registry.js b/src/layers/bikeshare/registry.js index ff752f664..51c10f17b 100644 --- a/src/layers/bikeshare/registry.js +++ b/src/layers/bikeshare/registry.js @@ -167,6 +167,30 @@ const RAW_GBFS_CITY_REGISTRY = [ 'https://chat.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', provider: 'Bike Chattanooga', }, + { + id: 'bogota-tembici', + city: 'Bogotá, Colombia', + centerLat: 4.6533, + centerLon: -74.0836, + loadRadiusKm: 90, + stationInformationUrl: + 'https://bogota.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://bogota.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Tembici Bogotá', + }, + { + id: 'santiago-bike-itau', + city: 'Santiago, Chile', + centerLat: -33.4372, + centerLon: -70.6506, + loadRadiusKm: 90, + stationInformationUrl: + 'https://santiago.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://santiago.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Bike Itaú Santiago', + }, bcycleEntry({ id: 'boulder-bcycle', city: 'Boulder, CO', From 97492e28fe259802fea1eba885f80031a5aa8f06 Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:04:55 +0200 Subject: [PATCH 34/95] docs: list the Catalonia pack under Unreleased and in the README The changelog entry sat below the 0.2.0/0.2.1 release headings after those were cut; move it to [Unreleased] beside the Norway pack. Name Catalonia in the README's CCTV Mesh row the way the other packs are listed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 18 ++++++++--------- README.md | 56 ++++++++++++++++++++++++++-------------------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7862af2c8..55ebdb1bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -142,6 +142,15 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. +- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~135 + cameras from a WFS/GML feed that aggregates SCT's own highway cameras with + hotlinked Barcelona and Terrassa municipal cameras, each partner named as a + credit beside the shared provider. Ambient-card refresh paced to 3 minutes, + matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` + sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. The catalog + is fetched over https with redirects refused and a 1 MiB body cap, and frame + links carrying userinfo are rejected even on an allowlisted host. + ## [0.2.1] — 2026-10-02 — God's Eye View through Your Agent (with security fixes and other features) - Ship each bundled data pack once. The region, marine, admin-boundary, @@ -648,15 +657,6 @@ Analyst records for loaded satellites, datacenters and dams, with explicit bound "~50k/day" free-tier figure in the proxy, `.env.example` and `DATA_SOURCES.md`. Still an application-side ceiling, not a billing cap. -- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~135 - cameras from a WFS/GML feed that aggregates SCT's own highway cameras with - hotlinked Barcelona and Terrassa municipal cameras, each partner named as a - credit beside the shared provider. Ambient-card refresh paced to 3 minutes, - matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` - sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. The catalog - is fetched over https with redirects refused and a 1 MiB body cap, and frame - links carrying userinfo are rejected even on an allowlisted host. - - Distinguish PARTIAL vessel snapshots from STALE data in the layer panel, with accepted-record counts and unchanged retention, freshness and outage safeguards. diff --git a/README.md b/README.md index 5d97dfb2f..bb191d9e7 100644 --- a/README.md +++ b/README.md @@ -291,28 +291,28 @@ _Ask for radio near anywhere and the globe starts broadcasting — every station Twenty layers and map sources. **Seventeen have a keyless path.** Some offer additional capabilities with a provider key. (🟢 no key · 🟡 free key · 🔴 metered.) -| Layer | What you get | Source | Auth | -| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------- | -| 🗺️ **Map Stack** | Esri satellite imagery, Google Photorealistic 3D, OSM, plus additional ion-hosted stacks | Esri / Google / Ion / OSM | 🟢 Esri satellite + OSM · 🟡 ion-hosted Google 3D + world terrain · 🔴 direct Google + place search | -| ✈️ **Live Flights** | 11,000+ live aircraft + route history | OpenSky + adsb.lol | 🟢 (🟡 optional for more polling credits) | -| 🎖️ **Military Flights** | ADS-B military traffic in amber | adsb.lol | 🟢 | -| 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | -| 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | -| 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | -| 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | -| 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | -| 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary first; Panoramax, KartaView and Google Street View planned | Mapillary | 🟡 Mapillary client token | -| 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | -| 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | -| 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | -| 🧭 **Directions** | Click A and B on the globe for a street-following drive, walk or cycle route draped on the terrain with turn-by-turn steps — then FLY the camera along it. No key, no geocoder, no mic | OSRM on FOSSGIS servers (OpenStreetMap) | 🟢 | -| 🔥 **Active Fires** | Live NASA FIRMS detections, trailing 24h | NASA FIRMS | 🟡 | -| 🚀 **Space Missions** | Rolling 30-day launches with payload, stage, and recovery detail | Launch Library 2 | 🟢 (🟡 optional token raises the allowance) | -| 🎖️ **Mapped Installations** | Viewport-bounded military-site context from community mapping — incomplete by nature, and labeled that way | OpenStreetMap | 🟢 | -| 🌬️ **Wind** | Animated 10 m forecast wind with an optional speed, temperature or pressure color field and a reading at map center | NOAA GFS / ECMWF IFS | 🟢 | -| 🌦️ **Observed Weather** | Rain radar (contiguous US), Satellite clouds (GOES regional or global infrared, Clouds only or Full image) and Lightning density (Americas and Pacific) on one history timeline | NOAA nowCOAST | 🟢 | -| 🌀 **Cyclones** | Advisory positions, forecast tracks and uncertainty cones for the Atlantic and eastern/central North Pacific | NOAA NHC / CPHC | 🟢 | +| Layer | What you get | Source | Auth | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------- | +| 🗺️ **Map Stack** | Esri satellite imagery, Google Photorealistic 3D, OSM, plus additional ion-hosted stacks | Esri / Google / Ion / OSM | 🟢 Esri satellite + OSM · 🟡 ion-hosted Google 3D + world terrain · 🔴 direct Google + place search | +| ✈️ **Live Flights** | 11,000+ live aircraft + route history | OpenSky + adsb.lol | 🟢 (🟡 optional for more polling credits) | +| 🎖️ **Military Flights** | ADS-B military traffic in amber | adsb.lol | 🟢 | +| 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | +| 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | +| 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | +| 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | +| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Catalonia (Servei Català de Trànsit). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | +| 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary first; Panoramax, KartaView and Google Street View planned | Mapillary | 🟡 Mapillary client token | +| 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | +| 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | +| 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | +| 🧭 **Directions** | Click A and B on the globe for a street-following drive, walk or cycle route draped on the terrain with turn-by-turn steps — then FLY the camera along it. No key, no geocoder, no mic | OSRM on FOSSGIS servers (OpenStreetMap) | 🟢 | +| 🔥 **Active Fires** | Live NASA FIRMS detections, trailing 24h | NASA FIRMS | 🟡 | +| 🚀 **Space Missions** | Rolling 30-day launches with payload, stage, and recovery detail | Launch Library 2 | 🟢 (🟡 optional token raises the allowance) | +| 🎖️ **Mapped Installations** | Viewport-bounded military-site context from community mapping — incomplete by nature, and labeled that way | OpenStreetMap | 🟢 | +| 🌬️ **Wind** | Animated 10 m forecast wind with an optional speed, temperature or pressure color field and a reading at map center | NOAA GFS / ECMWF IFS | 🟢 | +| 🌦️ **Observed Weather** | Rain radar (contiguous US), Satellite clouds (GOES regional or global infrared, Clouds only or Full image) and Lightning density (Americas and Pacific) on one history timeline | NOAA nowCOAST | 🟢 | +| 🌀 **Cyclones** | Advisory positions, forecast tracks and uncertainty cones for the Atlantic and eastern/central North Pacific | NOAA NHC / CPHC | 🟢 | **Weather, keyless.** **Wind** animates 10 m forecast flow from NOAA GFS or ECMWF IFS, with an optional color field and a reading at map center. **Rain radar**, **Satellite clouds** and **Lightning density** are observations on one history timeline in the **WEATHER** panel (step back, **Play**, **Latest**); **Cyclone advisories** draw NHC and CPHC positions, forecast tracks and cones. On Google 3D Tiles the observed layers float above the city as translucent shells, sharpened around your view. @@ -435,7 +435,7 @@ Seven keys. Five have a free tier, and the two 🔴 ones are metered: | --- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 🟡 | **Cesium ion** | 🗺️ Google Photorealistic 3D, world terrain, and additional ion-hosted imagery stacks. The free Community plan is for eligible individual, personal/non-commercial use and has quotas | [cesium.com/ion](https://cesium.com/ion) — use a public `assets:read` token and check current [pricing/eligibility](https://cesium.com/platform/cesium-ion/pricing/) | | 🔴 | **Google Maps** | Direct Google Photorealistic 3D + Google place search ([Map Tiles API](https://developers.google.com/maps/documentation/tile)) | [Google Cloud Console](https://console.cloud.google.com/) — URL-restrict it | -| 🔴 | **OpenAI** | 🎙️ Voice + AI HUD summary through an API key. Provider Settings also offers **USE CHATGPT OAUTH** for cloud voice without removing the API-key option | [platform.openai.com](https://platform.openai.com) — API-key usage is metered, see costs below | +| 🔴 | **OpenAI** | 🎙️ Voice + AI HUD summary through an API key. Provider Settings also offers **USE CHATGPT OAUTH** for cloud voice without removing the API-key option | [platform.openai.com](https://platform.openai.com) — API-key usage is metered, see costs below | | 🟡 | **AISStream** | 🚢 Live global ships | [aisstream.io](https://aisstream.io) — free signup | | 🟡 | **NASA FIRMS** | 🔥 Live active fires | [firms.modaps.eosdis.nasa.gov](https://firms.modaps.eosdis.nasa.gov/api/map_key/) — free | | 🟡 | **TomTom** | 🚦 Live flow speeds and congestion colors for the simulated traffic layer | [my.tomtom.com](https://my.tomtom.com/keys) — free tier available | @@ -509,11 +509,11 @@ RTMP-only sources are not supported by this integration. Honest numbers, roughly, as of mid-2026 — always check the provider pricing pages: -| | Cost reality | -| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **🟢 Most layers** | **$0, no signup.** OpenSky anon, USGS, CelesTrak, adsb.lol, city CCTV, Radio Browser, GBFS, Launch Library 2, transit feeds, OSRM routing, NOAA and ECMWF weather, OpenStreetMap ALPR mapping, bundled datasets. | -| **🟡 The free-key tier** | **$0 with a signup.** AISStream, FIRMS, TomTom, OpenSky, plus Cesium ion for eligible personal/non-commercial use. Provider quotas and eligibility still apply. | -| **🗺️ Google 3D tiles** | **Free through an eligible Cesium ion Community account within its quota; metered through a direct Google key.** Use the direct route for Google place search or commercial deployment, verify current provider terms, and set budget alerts where billing is enabled. | +| | Cost reality | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **🟢 Most layers** | **$0, no signup.** OpenSky anon, USGS, CelesTrak, adsb.lol, city CCTV, Radio Browser, GBFS, Launch Library 2, transit feeds, OSRM routing, NOAA and ECMWF weather, OpenStreetMap ALPR mapping, bundled datasets. | +| **🟡 The free-key tier** | **$0 with a signup.** AISStream, FIRMS, TomTom, OpenSky, plus Cesium ion for eligible personal/non-commercial use. Provider quotas and eligibility still apply. | +| **🗺️ Google 3D tiles** | **Free through an eligible Cesium ion Community account within its quota; metered through a direct Google key.** Use the direct route for Google place search or commercial deployment, verify current provider terms, and set budget alerts where billing is enabled. | | **🔴 OpenAI voice** | API-key sessions keep the live spend readout, STD/MINI model toggle, $2 warning, and **$5 hard cap**. ChatGPT OAuth is an explicit choice in Provider Settings. Its usage is tracked without an API-dollar estimate or spend cap. The voice context window stays deliberately short. | Google's direct 3D route is surprisingly generous: the first 1,000 Photorealistic From 492ecb484918bb5000a973e566ab71148e70e5be Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:07:54 +0200 Subject: [PATCH 35/95] docs(cctv): state that offline Catalonia cameras show the publisher's still The Catalonia feed has no camera status field, so an offline camera keeps its catalog entry and SCT or Barcelona answer its frame with their own "image unavailable" still (HTTP 200). Say so in DATA_SOURCES.md and stop the loader comment from implying every camera returns a live frame. Co-Authored-By: Claude Opus 5.5 --- DATA_SOURCES.md | 2 +- server/providers/cctv/sources.js | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index cdadba27e..1ec77f705 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -91,7 +91,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo is rejected even on an allowed host), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. +- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo is rejected even on an allowed host), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. The feed carries no camera status field, so a camera its publisher has taken offline still registers and shows the publisher's own "image unavailable" still (SCT and Barcelona both serve one with HTTP 200), the same as other packs without a status field. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 53017a2fe..d9a1c3ac3 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -1733,9 +1733,12 @@ export function cataloniaCameraName(carretera, municipi) { * Fetch Catalonia traffic cameras (Servei Català de Trànsit), keyless: one * WFS/GML XML list. The feed carries a `font` field naming the real * publisher per camera; only SCT's own highway cameras and the hotlinked - * Barcelona (IMI) and Terrassa municipal cameras are kept, each confirmed to - * return a real frame — frame URLs are checked against a host allowlist - * (CATALONIA_IMAGE_HOSTS), so a row from any other publisher is dropped. + * Barcelona (IMI) and Terrassa municipal cameras are kept — the publishers + * whose frame links were confirmed to serve images. Frame URLs are checked + * against a host allowlist (CATALONIA_IMAGE_HOSTS), so a row from any other + * publisher is dropped. The feed carries no camera status, so a camera its + * publisher has taken offline still registers and serves that publisher's + * own "image unavailable" still. * Each non-SCT camera carries its publisher as a `credit` alongside the * shared `provider`. The feed carries no heading, so every camera takes the * id-hash fallback, low-confidence pose (same as TfL and Fintraffic). From a4d6181fb99764d2f5493c8238224bac4f7b125e Mon Sep 17 00:00:00 2001 From: Jeff Blakely <7987380+jeffblakely@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:26:05 -0400 Subject: [PATCH 36/95] feat(bikeshare): add Detroit, Pittsburgh, Aspen and Kona & Hilo Register four live US systems on Lyft's PBSC platform: MoGo (Detroit, 81 stations), POGOH (Pittsburgh, 60), WE-cycle (Aspen, 101) and HIBike (Kona and Hilo, 23). The proxy already allows *.publicbikesystem.net, so no server change is needed. HIBike's centre sits between Kailua-Kona and Hilo so a camera over either town loads the whole system. The test checks each system is registered, passes the proxy's host and endpoint checks, and has an activation radius covering its farthest station as measured against the live feed. --- CHANGELOG.md | 4 ++ src/layers/bikeshare/registry.js | 50 ++++++++++++++++++++++ src/layers/bikeshare/usPbscCities.test.mjs | 31 ++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 src/layers/bikeshare/usPbscCities.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index b24ac95ed..5fd137e89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +- Add Detroit (MoGo), Pittsburgh (POGOH), Aspen (WE-cycle) and Kona & Hilo + (HIBike) to the GBFS bikeshare registry, the layer's first stations in + Michigan and Pennsylvania. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/src/layers/bikeshare/registry.js b/src/layers/bikeshare/registry.js index ff752f664..38d417b04 100644 --- a/src/layers/bikeshare/registry.js +++ b/src/layers/bikeshare/registry.js @@ -48,6 +48,56 @@ const RAW_GBFS_CITY_REGISTRY = [ 'https://gbfs.lyft.com/gbfs/2.3/chi/en/station_status.json', provider: 'Divvy', }, + { + id: 'detroit-mogo', + city: 'Detroit, MI', + centerLat: 42.3314, + centerLon: -83.0458, + loadRadiusKm: 90, + stationInformationUrl: + 'https://detroit.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://detroit.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'MoGo', + }, + { + id: 'pittsburgh-pogoh', + city: 'Pittsburgh, PA', + centerLat: 40.4406, + centerLon: -79.9959, + loadRadiusKm: 90, + stationInformationUrl: + 'https://pittsburgh.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://pittsburgh.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'POGOH', + }, + { + id: 'aspen-we-cycle', + city: 'Aspen, CO', + centerLat: 39.1911, + centerLon: -106.8175, + loadRadiusKm: 90, + stationInformationUrl: + 'https://aspen.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://aspen.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'WE-cycle', + }, + // HIBike serves both Kailua-Kona and Hilo, about 95 km apart; the centre sits + // between them so a camera over either town loads the whole system. + { + id: 'hawaii-island-hibike', + city: 'Kona & Hilo, HI', + centerLat: 19.68, + centerLon: -155.54, + loadRadiusKm: 90, + stationInformationUrl: + 'https://kona.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://kona.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'HIBike', + }, { id: 'dc-capital-bikeshare', city: 'Washington, DC', diff --git a/src/layers/bikeshare/usPbscCities.test.mjs b/src/layers/bikeshare/usPbscCities.test.mjs new file mode 100644 index 000000000..4ff15c370 --- /dev/null +++ b/src/layers/bikeshare/usPbscCities.test.mjs @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { CITY_BY_ID } from './registry.js'; +import { isAllowedGbfsHost, isAllowedGbfsPath } from '../../data/gbfsSource.js'; +import { CITY_RANGE_BASE_KM } from './policy.js'; + +// Each city's PBSC host, and its farthest station from the registered centre +// as measured against the live feed (km), so a camera over any part of the +// system activates it. +const CITIES = { + 'detroit-mogo': { host: 'detroit.publicbikesystem.net', provider: 'MoGo', farthestKm: 22.4 }, + 'pittsburgh-pogoh': { host: 'pittsburgh.publicbikesystem.net', provider: 'POGOH', farthestKm: 9.1 }, + 'aspen-we-cycle': { host: 'aspen.publicbikesystem.net', provider: 'WE-cycle', farthestKm: 42.6 }, + 'hawaii-island-hibike': { host: 'kona.publicbikesystem.net', provider: 'HIBike', farthestKm: 53.8 }, +}; + +test('the added US PBSC systems are registered and readable through the proxy', () => { + for (const [id, expected] of Object.entries(CITIES)) { + const city = CITY_BY_ID.get(id); + assert.ok(city, `${id} is registered`); + assert.equal(city.provider, expected.provider); + assert.deepEqual(city.hosts, [expected.host]); + for (const feed of [city.stationInformationUrl, city.stationStatusUrl]) { + const url = new URL(feed); + assert.ok(isAllowedGbfsHost(url.hostname), `${id} host ${url.hostname}`); + assert.ok(isAllowedGbfsPath(url.pathname), `${id} path ${url.pathname}`); + } + const radius = Math.max(CITY_RANGE_BASE_KM, city.loadRadiusKm); + assert.ok(radius > expected.farthestKm, `${id} radius ${radius} covers its farthest station`); + } +}); From fc48eda113576cfcf9a4cc606c8a4554e4c6681e Mon Sep 17 00:00:00 2001 From: Aditya <205600203+Rohilalala@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:31:11 +0530 Subject: [PATCH 37/95] fix(radio): reject null-body and out-of-range mirror statuses instead of crashing fetchPinnedRadioResponse built `new Response(stream, { status })` inside the raw https response callback, which runs outside the surrounding promise. A Radio Browser mirror answering 204, 205 or 304 (null-body statuses), or any status outside 200-599, made the Response constructor throw. Nothing caught it, so the whole dev/preview server process exited and every layer went down, not only radio. The catalog's mirror failover never saw the error. Mirror the guard local-receivers.js already uses: null-body statuses get a null body, statuses outside 200-599 are rejected, and anything thrown in the callback rejects the promise so the existing error handling and mirror failover run. --- CHANGELOG.md | 5 +++ docs/CURRENT-STATE.md | 9 ++-- server/providers/radio/transport.js | 41 ++++++++++++------ src/data/radioProxy.test.mjs | 64 +++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b24ac95ed..66b87d770 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +- A Radio Browser mirror answering 204, 205 or 304, or a status outside + 200–599, no longer crashes the local dev/preview server. Null-body statuses + get an empty body; anything else that cannot become a response fails that + mirror so the next one is tried. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 26d56e66b..9de0ebf31 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1135,9 +1135,12 @@ CCTV catalog acquisition, source normalization and frame/media delivery now live in separate modules. Each CCTV factory owns its catalog and health state; its `sourceRoot` option resolves relative source files against the application root. Radio Browser station normalization, restricted outbound transport and directory -caching are separate modules. Node-only package entries expose both provider -factories. Routes, payloads, fallback behavior and existing dev/preview hook -registration are preserved; browser rendering is unchanged. +caching are separate modules. The pinned transport gives a mirror's 204, 205 or +304 reply an empty body and fails any reply outside 200–599 without throwing +outside its promise, so an unusual mirror status cannot crash the server. +Node-only package entries expose both provider factories. Routes, payloads, +fallback behavior and existing dev/preview hook registration are preserved; +browser rendering is unchanged. ## Terrain, traffic, fire and bike-share provider modules diff --git a/server/providers/radio/transport.js b/server/providers/radio/transport.js index 7bbe33bcd..c9922bb2f 100644 --- a/server/providers/radio/transport.js +++ b/server/providers/radio/transport.js @@ -1,6 +1,9 @@ import https from 'node:https'; import { Readable } from 'node:stream'; import { isNonGlobalIpv4 } from './stations.js'; + +const NULL_BODY_STATUSES = new Set([204, 205, 304]); + export function radioMirrorOrigin(value) { const hostname = String(value ?? '') .toLowerCase() @@ -124,19 +127,33 @@ export function fetchPinnedRadioResponse(url, options, addresses) { }, }, (response) => { - const headers = new Headers(); - for (const [name, value] of Object.entries(response.headers)) { - if (Array.isArray(value)) - value.forEach((item) => headers.append(name, item)); - else if (value !== undefined) headers.set(name, String(value)); + // Nothing thrown here may escape: this callback runs outside the + // promise, so an exception would be uncaught and crash the server. + try { + const status = response.statusCode; + // A `Response` only represents statuses up to 599; Node never hands + // this callback a 1xx. + if (status > 599) + throw new Error( + `Radio Browser returned unsupported status ${status}`, + ); + const headers = new Headers(); + for (const [name, value] of Object.entries(response.headers)) { + if (Array.isArray(value)) + value.forEach((item) => headers.append(name, item)); + else if (value !== undefined) headers.set(name, String(value)); + } + const nullBody = NULL_BODY_STATUSES.has(status); + const upstream = new Response( + nullBody ? null : Readable.toWeb(response), + { status, statusText: response.statusMessage || '', headers }, + ); + if (nullBody) response.resume(); + resolve(upstream); + } catch (error) { + response.destroy(); + reject(error); } - resolve( - new Response(Readable.toWeb(response), { - status: response.statusCode || 500, - statusText: response.statusMessage || '', - headers, - }), - ); }, ); request.on('error', reject); diff --git a/src/data/radioProxy.test.mjs b/src/data/radioProxy.test.mjs index 2f090b5f9..60463205b 100644 --- a/src/data/radioProxy.test.mjs +++ b/src/data/radioProxy.test.mjs @@ -1,5 +1,9 @@ import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import https from 'node:https'; +import { Readable } from 'node:stream'; import test from 'node:test'; +import { fetchPinnedRadioResponse } from '../../server/providers/radio/transport.js'; import { createRadioProxyMiddleware, isPublicRadioAddress, @@ -469,3 +473,63 @@ test('proxy rejects any hostname resolution containing a forbidden address befor } assert.equal(fetchCount, 0); }); + +// Answer every pinned request with `status`, invoking the response callback +// asynchronously as `https.request` does, outside the caller's promise. +// The returned promise rejects on an uncaught exception, so a regression fails +// the race at once instead of leaving the fetch pending. +function mockMirrorStatus(t, status, statusMessage = '') { + let crash; + const crashed = new Promise((_resolve, reject) => (crash = reject)); + const onUncaught = (error) => + crash(new Error(`uncaught exception: ${error.message}`)); + process.on('uncaughtException', onUncaught); + t.after(() => process.off('uncaughtException', onUncaught)); + t.mock.method(https, 'request', (_url, _options, callback) => { + const request = new EventEmitter(); + request.end = () => + setImmediate(() => + callback( + Object.assign(Readable.from([]), { + statusCode: status, + statusMessage, + headers: { 'content-type': 'application/json' }, + }), + ), + ); + return request; + }); + return crashed; +} + +function fetchMirror() { + return fetchPinnedRadioResponse( + new URL('https://de1.api.radio-browser.info/json/servers'), + { headers: {} }, + [{ address: '93.184.216.34', family: 4 }], + ); +} + +for (const status of [204, 205, 304]) { + test(`a mirror answering null-body status ${status} resolves with a null body instead of crashing`, async (t) => { + const crashed = mockMirrorStatus(t, status); + const response = await Promise.race([fetchMirror(), crashed]); + assert.equal(response.status, status); + assert.equal(response.body, null); + }); +} + +for (const status of [600, 999]) { + test(`a mirror answering out-of-range status ${status} rejects instead of crashing`, async (t) => { + const crashed = mockMirrorStatus(t, status); + await assert.rejects( + Promise.race([fetchMirror(), crashed]), + new RegExp(`unsupported status ${status}`), + ); + }); +} + +test('a mirror answering an invalid reason phrase rejects instead of crashing', async (t) => { + const crashed = mockMirrorStatus(t, 200, 'OK\x7f'); + await assert.rejects(Promise.race([fetchMirror(), crashed]), TypeError); +}); From b90bcf1829c4a44f16ceb9ee166a3785cb0345b4 Mon Sep 17 00:00:00 2001 From: Aditya <205600203+Rohilalala@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:00:04 +0530 Subject: [PATCH 38/95] fix(flights): don't treat OpenSky's position-less rows as a partial snapshot OpenSky's state-vector spec allows null longitude/latitude, and every live /states/all response carries such rows (about 50 of 7,800). The normaliser rejects them, so admitRecords reported complete: false on effectively every poll. While a snapshot is incomplete, FlightRecords.absence() retains any aircraft received in the last five minutes, which meant the stale fade, MISSING_POLL_LIMIT and the landed fast-cull never ran: vanished and landed aircraft stayed on the globe at full opacity for up to five minutes. admitRecords now takes an optional unpositionedId(row). A rejected row it names is a contact without a position: its id is returned in retainIds and the row does not count against completeness. OpenSky opts in for rows with an ICAO24 address and strictly null longitude and latitude; the id comes from the same helper the normaliser uses. Rows with no usable id, out-of-range or half-null coordinates, truncated rows and positioned duplicates still make the snapshot incomplete, and a nonempty feed with no positioned rows is still malformed. absence() retains exactly the retainIds aircraft under the existing five-minute bound, except likely-landed ones, which keep the landed limit. Everything else ages out normally. Military readsb admission does not opt in and is unchanged. --- CHANGELOG.md | 5 +++ docs/CURRENT-STATE.md | 8 ++++- src/layers/flights/records.js | 8 +++-- src/layers/flights/records.test.mjs | 24 ++++++++++++++ src/layers/flights/snapshotRenderer.js | 1 + src/sources/live/aircraft.js | 17 ++++++++-- src/sources/live/contract.js | 20 ++++++++++-- src/sources/live/contract.test.mjs | 44 ++++++++++++++++++++++++++ 8 files changed, 118 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b24ac95ed..c608afa5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +- Flights age out vanished aircraft again. OpenSky's routine position-less rows + no longer mark every snapshot partial, which had kept vanished and landed + aircraft at full opacity for up to five minutes; those rows retain only their + own aircraft, and a likely-landed one still leaves on the landed limit. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 26d56e66b..15d5c50f7 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -825,7 +825,13 @@ addition to the locally accumulated trail, never a promise of complete coverage. Snapshot coverage, completeness and freshness are separate fields. A partially admitted aircraft snapshot retains absent contacts for up to five minutes before -the usual missed-poll eviction. Unknown snapshot times remain unknown in stats. An invalid nonempty +the usual missed-poll eviction. An OpenSky row with an ICAO24 address but null +longitude and latitude is a contact without a position, not a malformed row: it +leaves the snapshot complete and is listed in `retainIds`, so civil flights +retain that aircraft under the same five-minute bound, unless it is likely +landed, while other absent aircraft fade and leave on the missed-poll limits. A +nonempty snapshot with no positioned rows is still malformed. Military readsb +admission is unchanged. Unknown snapshot times remain unknown in stats. An invalid nonempty snapshot retains the previous display. Empty vessel refreshes retain the existing first-connect grace and warm-data behavior. Known source failures have bounded messages; arbitrary HTTP response bodies are not surfaced as diagnostics. diff --git a/src/layers/flights/records.js b/src/layers/flights/records.js index 43809f516..fdb65c62c 100644 --- a/src/layers/flights/records.js +++ b/src/layers/flights/records.js @@ -258,10 +258,12 @@ export class FlightRecords { return { icao24, prevMeta, meta, groundFlipped, fixEpochMs }; } - absence(id, { complete, likelyLanded }) { - // Partial admissions do not prove absence, but retention remains bounded. + absence(id, { complete, retainIds, likelyLanded }) { + // Partial admissions and position-less rows do not prove absence, but + // retention remains bounded; a likely-landed position-less contact + // still takes the landed limit. if ( - !complete && + (!complete || (retainIds?.has(id) && !likelyLanded)) && Date.now() - (this.data.get(id)?.observedReceiptMs ?? 0) < 300000 ) return 'retain'; diff --git a/src/layers/flights/records.test.mjs b/src/layers/flights/records.test.mjs index d001c7811..783b6d06a 100644 --- a/src/layers/flights/records.test.mjs +++ b/src/layers/flights/records.test.mjs @@ -106,3 +106,27 @@ test('ground transitions preserve last render height while a surface cell is col assert.equal(next.meta.renderAltitudeM, 1040); assert.equal(next.meta.wasAirborne, true); }); + +test('complete snapshots age out vanished aircraft and retain position-less ones unless landed', () => { + const store = records(); + for (const id of ['abc123', 'def456', 'ghi789']) + store.receive(observation({ id, reference: id }), view()); + // abc123 and ghi789 are still listed without a position; def456 vanished. + const absent = (id, likelyLanded = false) => + store.absence(id, { + complete: true, + retainIds: new Set(['abc123', 'ghi789']), + likelyLanded, + }); + assert.deepEqual( + [absent('def456'), absent('def456'), absent('def456')], + ['stale', 'stale', 'remove'], + ); + assert.equal(absent('abc123'), 'retain'); + assert.equal(store.missingPolls.has('abc123'), false); + // A landed contact that lost its position takes the landed fast-cull. + assert.equal(absent('ghi789', true), 'remove'); + // Position-less retention stays bounded like partial admission. + store.data.get('abc123').observedReceiptMs = Date.now() - 300001; + assert.equal(absent('abc123'), 'stale'); +}); diff --git a/src/layers/flights/snapshotRenderer.js b/src/layers/flights/snapshotRenderer.js index f6e74b125..ec6c2933b 100644 --- a/src/layers/flights/snapshotRenderer.js +++ b/src/layers/flights/snapshotRenderer.js @@ -243,6 +243,7 @@ export function createFlightSnapshotRenderer({ if (currentIcaos.has(icao24)) continue; const absence = records.absence(icao24, { complete: snapshot.complete, + retainIds: snapshot.retainIds, likelyLanded: queries._likelyLanded(icao24), }); if (absence === 'retain') continue; diff --git a/src/sources/live/aircraft.js b/src/sources/live/aircraft.js index 9534a5c3c..1ba3af603 100644 --- a/src/sources/live/aircraft.js +++ b/src/sources/live/aircraft.js @@ -6,10 +6,12 @@ import { finite, } from './contract.js'; +const openSkyId = (row) => + Array.isArray(row) ? cleanText(row[0]).toLowerCase() : ''; + /** Normalize the OpenSky state-vector route, including its regional fallback. */ export function normalizeOpenSkyAircraft(row) { - if (!Array.isArray(row)) return null; - const id = cleanText(row[0]).toLowerCase(); + const id = openSkyId(row); if (!id || !coordinates(row[6], row[5])) return null; return { id, @@ -71,6 +73,16 @@ export function normalizeReadsbAircraft(row, snapshotTimeMs) { }; } +/** + * OpenSky allows null longitude/latitude for a contact that is still + * broadcasting. Strict nulls also reject rows truncated before the position. + */ +function openSkyUnpositionedId(row) { + return Array.isArray(row) && row[5] === null && row[6] === null + ? openSkyId(row) + : ''; +} + export function openSkySnapshot( payload, { @@ -84,6 +96,7 @@ export function openSkySnapshot( payload?.states, normalizeOpenSkyAircraft, 'OpenSky', + openSkyUnpositionedId, ); const observedAtMs = epoch(payload?.time, 1000); const ageMs = observedAtMs == null ? null : Math.max(0, now - observedAtMs); diff --git a/src/sources/live/contract.js b/src/sources/live/contract.js index 2415ff43c..d99b55f0d 100644 --- a/src/sources/live/contract.js +++ b/src/sources/live/contract.js @@ -53,17 +53,30 @@ export function coordinates(latitude, longitude) { ); } -/** Admit a snapshot atomically; an all-invalid nonempty feed is unavailable. */ -export function admitRecords(rows, normalize, label) { +/** + * Admit a snapshot atomically; an all-invalid nonempty feed is unavailable. + * `unpositionedId(row)` names a rejected row that is a valid contact without + * a position: it is returned in `retainIds` and does not make the snapshot + * incomplete. + */ +export function admitRecords(rows, normalize, label, unpositionedId) { if (!Array.isArray(rows)) throw new LiveSourceError('malformed', `Malformed ${label} response`); const records = []; const ids = new Set(); + const retainIds = new Set(); + let unpositioned = 0; for (const row of rows) { const record = normalize(row); if (record && !ids.has(record.id)) { records.push(record); ids.add(record.id); + } else if (!record) { + const id = unpositionedId?.(row); + if (id) { + retainIds.add(id); + unpositioned += 1; + } } } if (rows.length && !records.length) { @@ -71,7 +84,8 @@ export function admitRecords(rows, normalize, label) { } return { records, - complete: records.length === rows.length, + retainIds, + complete: records.length + unpositioned === rows.length, rejectedCount: rows.length - records.length, }; } diff --git a/src/sources/live/contract.test.mjs b/src/sources/live/contract.test.mjs index 391fe1e72..143a7fade 100644 --- a/src/sources/live/contract.test.mjs +++ b/src/sources/live/contract.test.mjs @@ -67,6 +67,50 @@ test('mixed-invalid and duplicate admissions are explicitly incomplete; all-inva assert.equal(openSkySnapshot({ states: [] }).complete, true); }); +test('position-less OpenSky rows are seen, not missing; malformed rows stay incomplete', () => { + // OpenSky's spec allows null longitude/latitude; every live /states/all + // carries such rows. + const unpositioned = ['DEF456', ...aircraft.slice(1)]; + unpositioned[5] = null; + unpositioned[6] = null; + const snapshot = openSkySnapshot({ states: [aircraft, unpositioned] }); + assert.equal(snapshot.records.length, 1); + assert.equal(snapshot.complete, true); + assert.deepEqual([...snapshot.retainIds], ['def456']); + assert.equal(snapshot.rejectedCount, 1); + const malformed = [...aircraft]; + malformed[5] = 181; + const garbled = openSkySnapshot({ + states: [aircraft, unpositioned, malformed], + }); + assert.equal(garbled.complete, false); + assert.deepEqual([...garbled.retainIds], ['def456']); + // No usable id, only half a position, a truncated row, or no row at all. + for (const row of [ + [' ', ...unpositioned.slice(1)], + Object.assign([...unpositioned], { 5: -97 }), + ['DEF456'], + null, + ]) + assert.equal(openSkySnapshot({ states: [aircraft, row] }).complete, false); + // Repeated position-less rows stay complete. + const repeated = openSkySnapshot({ + states: [aircraft, unpositioned, unpositioned], + }); + assert.equal(repeated.complete, true); + // A feed with no positions at all still cannot prove absence. + assert.throws(() => openSkySnapshot({ states: [unpositioned] }), { + code: 'malformed', + }); + // Military readsb admission keeps its existing semantics. + const military = readsbSnapshot( + { ac: [{ hex: 'abc123', lat: 30, lon: -97 }, { hex: 'def456' }] }, + { observedAtMs: null }, + ); + assert.equal(military.complete, false); + assert.equal(military.retainIds.size, 0); +}); + test('readsb units and position/contact ages are normalized once against the source epoch', () => { const row = normalizeReadsbAircraft( { From afa9e763f211d224c014f506899707cc254f2da6 Mon Sep 17 00:00:00 2001 From: Jeff Blakely <7987380+jeffblakely@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:06:41 -0400 Subject: [PATCH 39/95] test(bikeshare): check PBSC activation from real station coordinates Assert through computeInRangeCities that a camera over each system's outermost live-feed station activates it, including both Kona and Hilo for HIBike, and that HIBike stays off over Honolulu. Drop Pennsylvania from the changelog: Philadelphia's Indego is already registered. --- CHANGELOG.md | 2 +- src/layers/bikeshare/usPbscCities.test.mjs | 66 ++++++++++++++++++---- 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fd137e89..eedaf06d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ - Add Detroit (MoGo), Pittsburgh (POGOH), Aspen (WE-cycle) and Kona & Hilo (HIBike) to the GBFS bikeshare registry, the layer's first stations in - Michigan and Pennsylvania. + Michigan. - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/src/layers/bikeshare/usPbscCities.test.mjs b/src/layers/bikeshare/usPbscCities.test.mjs index 4ff15c370..aba2d20ec 100644 --- a/src/layers/bikeshare/usPbscCities.test.mjs +++ b/src/layers/bikeshare/usPbscCities.test.mjs @@ -1,19 +1,48 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { CITY_BY_ID } from './registry.js'; +import { createModel } from './model.js'; +import { createViewport } from './viewport.js'; import { isAllowedGbfsHost, isAllowedGbfsPath } from '../../data/gbfsSource.js'; -import { CITY_RANGE_BASE_KM } from './policy.js'; -// Each city's PBSC host, and its farthest station from the registered centre -// as measured against the live feed (km), so a camera over any part of the -// system activates it. +// Each city's PBSC host, and the stations farthest from its registered centre +// in the live feed (checked 2026-10-13). HIBike's centre sits between its two +// towns, so it lists the outermost station in each. const CITIES = { - 'detroit-mogo': { host: 'detroit.publicbikesystem.net', provider: 'MoGo', farthestKm: 22.4 }, - 'pittsburgh-pogoh': { host: 'pittsburgh.publicbikesystem.net', provider: 'POGOH', farthestKm: 9.1 }, - 'aspen-we-cycle': { host: 'aspen.publicbikesystem.net', provider: 'WE-cycle', farthestKm: 42.6 }, - 'hawaii-island-hibike': { host: 'kona.publicbikesystem.net', provider: 'HIBike', farthestKm: 53.8 }, + 'detroit-mogo': { + host: 'detroit.publicbikesystem.net', + provider: 'MoGo', + stations: [{ name: 'Robina Ave & W Twelve Mile Rd', lat: 42.50288, lon: -83.18921 }], + }, + 'pittsburgh-pogoh': { + host: 'pittsburgh.publicbikesystem.net', + provider: 'POGOH', + stations: [{ name: 'Wilkinsburg Park & Ride', lat: 40.446951, lon: -79.888593 }], + }, + 'aspen-we-cycle': { + host: 'aspen.publicbikesystem.net', + provider: 'WE-cycle', + stations: [{ name: 'Cowen Dr | Hwy 133', lat: 39.413115, lon: -107.220634 }], + }, + 'hawaii-island-hibike': { + host: 'kona.publicbikesystem.net', + provider: 'HIBike', + stations: [ + { name: 'Kona Community Center', lat: 19.642932, lon: -156.004962 }, + { name: 'Kealoha Beach Park, Hilo', lat: 19.733317, lon: -155.029202 }, + ], + }, }; +// The production range check, with no viewer or fetches behind it. +const parts = { model: createModel({ state: {}, services: {}, parts: {}, source: null }) }; +const { computeInRangeCities } = createViewport({ + state: {}, + services: { render: { governorRequestRender() {} } }, + parts, + source: null, +}); + test('the added US PBSC systems are registered and readable through the proxy', () => { for (const [id, expected] of Object.entries(CITIES)) { const city = CITY_BY_ID.get(id); @@ -25,7 +54,24 @@ test('the added US PBSC systems are registered and readable through the proxy', assert.ok(isAllowedGbfsHost(url.hostname), `${id} host ${url.hostname}`); assert.ok(isAllowedGbfsPath(url.pathname), `${id} path ${url.pathname}`); } - const radius = Math.max(CITY_RANGE_BASE_KM, city.loadRadiusKm); - assert.ok(radius > expected.farthestKm, `${id} radius ${radius} covers its farthest station`); } }); + +test('a camera over each outermost station activates its system', () => { + for (const [id, expected] of Object.entries(CITIES)) { + for (const station of expected.stations) { + assert.ok( + computeInRangeCities({ lat: station.lat, lon: station.lon }).has(id), + `${station.name} activates ${id}`, + ); + } + } +}); + +test('HIBike stays inactive away from Hawaii Island', () => { + // Honolulu is about 270 km from the HIBike centre. + assert.equal( + computeInRangeCities({ lat: 21.3069, lon: -157.8583 }).has('hawaii-island-hibike'), + false, + ); +}); From bdd64660e7d9d0de31ce58dc8450a09e7a632214 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 18:17:09 -0400 Subject: [PATCH 40/95] feat: ship a web app manifest and home-screen icons Adding the app to a phone home screen produced a blank tile: index.html linked only an SVG favicon, and Safari ignores SVG for apple-touch-icon, substituting a screenshot of the page. There was no manifest at all, so Android had nothing to install against either. Generate the rasters from public/logo.svg with sharp, which the repo already carries. The output is committed so a plain `vite build` needs no image toolchain; re-run scripts/generate-icons.mjs after changing the logo. The logo's viewBox is wider than it is tall, so each icon centres the mark on a square #0a0a0f canvas rather than stretching it, inset to 78% normally and 54% for the maskable variant to survive Android's circular crop. The SVG favicon stays as the browser tab icon. --- CHANGELOG.md | 6 +++ docs/CURRENT-STATE.md | 16 +++++++ index.html | 10 +++++ public/icons/icon-180.png | Bin 0 -> 12055 bytes public/icons/icon-192.png | Bin 0 -> 12993 bytes public/icons/icon-32.png | Bin 0 -> 1160 bytes public/icons/icon-512-maskable.png | Bin 0 -> 28226 bytes public/icons/icon-512.png | Bin 0 -> 40995 bytes public/manifest.webmanifest | 21 +++++++++ scripts/format-scope.json | 4 +- scripts/generate-icons.mjs | 70 +++++++++++++++++++++++++++++ src/tooling/webAppIcons.test.mjs | 70 +++++++++++++++++++++++++++++ 12 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 public/icons/icon-180.png create mode 100644 public/icons/icon-192.png create mode 100644 public/icons/icon-32.png create mode 100644 public/icons/icon-512-maskable.png create mode 100644 public/icons/icon-512.png create mode 100644 public/manifest.webmanifest create mode 100644 scripts/generate-icons.mjs create mode 100644 src/tooling/webAppIcons.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 032c6fe2a..eb61ad1db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +- Ship a web app manifest and raster home-screen icons, so adding the app to a + phone home screen or installing it on desktop uses the God's Eye View mark + instead of a blank tile. The icons are generated from `public/logo.svg` by + `node scripts/generate-icons.mjs` and committed, so a plain `vite build` needs + no image toolchain. The existing SVG favicon still serves the browser tab. + - GBFS v3 station names use English translations when available, then the first usable translation or short name. Empty or malformed names no longer appear as `[object Object]` or interrupt station loading; legacy string names remain diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 0d3ef31e0..f8e6bff3a 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1131,6 +1131,22 @@ services, local voice and standalone key setup. Importing them starts no network acquisition. Browser layer lifecycle, rendering and voice execution stay in their existing modules. +## Web app install icons + +`public/manifest.webmanifest` declares the installable app: standalone display, +`#0a0a0f` background and theme, and 192/512 icons plus a 512 maskable variant. +`index.html` links the manifest, a 180px `apple-touch-icon` and a 32px PNG +favicon; the existing SVG favicon still serves the browser tab. Safari ignores an +SVG apple-touch-icon and substitutes a page screenshot, so the raster is what +makes an installed icon appear at all. + +`node scripts/generate-icons.mjs` renders every raster from `public/logo.svg` and +writes `public/icons/`. Output is committed, so building needs no image +toolchain; re-run the script after changing the logo. Ordinary icons inset the +mark to 78% of the canvas and the maskable variant to 54%, keeping it inside the +circle Android may crop to. `src/tooling/webAppIcons.test.mjs` checks the links, +the manifest fields and that each committed PNG is square at its declared size. + ## Local build preview After `npm run build`, `npm run preview` serves the built app with the same data diff --git a/index.html b/index.html index 2e630c4a8..31db11b06 100644 --- a/index.html +++ b/index.html @@ -5,6 +5,16 @@ God's Eye View + + + + + + + + diff --git a/public/icons/icon-180.png b/public/icons/icon-180.png new file mode 100644 index 0000000000000000000000000000000000000000..5655a9e705643b9a36e8d27666014bc940e3d94f GIT binary patch literal 12055 zcmcJVRZtyF)8`NF8rsDPN z%t5j;ND)?ST@sYFPFzy!8!5!v@E~ZsWRKRR##QatBtA~iy(lEjHzaiCE(*>{Pw7qnR)+3aUh{nldt6vBXg4g- zZr|dHh*@h@6{UU5mD+2H*evtwZlJ*&ZC4s5eznP*3HuwyQqTdT*C8fw=?8m=D38w> zhUsl2$7T?!Hhh;e-=!|x7&nQqN`Z-3A|dS<$xI}T&{h^xK`i8=2WKw`k~Cm#LNTg=C?5wmf{N|oJ943+ zUzC&L)IR7o3)US*70UJlP-!B0GL5^sTlaHujUP{-z#vn20)XzSa|e) z+N>?)pob1DguGj^W&YBxdKkeNLx@Jo71E3F$py9*sh$3A*vf;)NP)Z^W3t^KPhS+D zqBX}A`zBoGjNMb+JWoU6P z-$|f~(^1NMYtKLgTgH(GjCH(_co=h;2E$^CTtG816o#wGFkzM#tvB5kaAD=>(LfH1 zoB(?*BLDOdaC<>Mt3Uy!L zFMj0`0MG$0y3gzEe|9YKcX8 z^B^(PGM)M=j-nshiL!pYvy z{B7L$Z8Qv_#cf!d_+l`)ezp6fef=2?VXN8w-NG+Ut5}ZC?MzF01D7?QJd?rtrpfZ? zAkJf^=+duXEZ0AO>@&fl`z@8h|1JGO@QqKSOc*mjX>|*TX8@!HJ!f&2Q-?ses$Er z-pd=VQS(yu@r3)qt0Eq?Hpy*LoS^>Rxh)yOCZH+O{n6hqZ;-W9Ttc%l@7LeW z@5Qm5BC+53A%*c3{+gV}FV3;m2)L0BR)s%!;4paweNO$RxGe=3Y^)=7>F8)}6+^i6 zQeycZ^IFQGRE(WvQULm!jOhqC9UXn5Gu$NTY_$1Me7DJU-v@cR2?vDFn?Lg-wYET; zQ4A$gk}{+UikOA9Mog_Y#+tTolkhJ4negjAi6c{p;Dz23PAjGCw9&sHsM!fpqkf$|u872eYulGx*>ddjbwz_i}c~W=&sM@RNVn z52Pl6Wm4`>!E-Q1+f@|l$UP}LlU*EP!?=iuSQYE1>D$HM-L09k{^95?M>A5NzYhj~ zjjBzFg`|CPSn}eBChehyi9TLol1Kd5t%i)%E}hRp_fB2OgO1`;h_OJ*jUhjAl%+pP z^Q%1u=Hw+#bX*!|-iy?k2}$eR&5+&n{BbMaKWJH2L(rFxq$t9Y%8K{8qH1GJ+YrI@J#++80_A#Tx`gPKb z%wx4Rck1)S(q>$A!&p~tPGs{G#)38=bpN{PM&J~0JrCIhXn1>-nlwCI59#y8zxeJY zv&2tUomI!Sxc8Pz;}m<|Uj^_@sPLO}8r!zZmet^(4CHkE z%EMSRj*jyvIB2{{Gwu5lr z@a&&VuPcl`y@!q5zx&5*Ej^zhoizR=-RM;aABK9ktDk3JnYnc|mKZY#$Y@+4<}Jly zDiL!&n3~Z~!#cRAePI;Mwe#l_%qdLFn9dabH~{G*r{fHjWA&o_dR}~AvYHKYxme%Z z??&DF8GphCngA#)yY)N?Xi5CLAveKsNI{kO1Q!L3qyCvDFVXt1W|Xdum&@*=tI(1Y(=|kV!sHI06C$g1XEnAKft&7iJKiZ5=8rAyEV*BnsqmH0G*zqg36L^S>LCjkUud_;CIQh_7P`tGfbZw1|9Hc&5j}r{NNMszBWuBG zKDv&9$ERWfN#rw#z}IDhM0V>#eonFL=hg~v&=R~t}AWpS4jZ|$PiM+l0g)X`QD5a6#+_e-I%5W5*H2w zSp2s(XmZ4YV4PrizG5jJlXue2g5iLePl-G%_7z%XUx$>^}UO?aMuFTS07tcFHZeno7eg5bjuu5Ows?1zHB@?=K0coT&&hO%(D>{D;Pupn=AkZ zYIhTEy&|H_zd3PMh6=kiLUBT(Q7NdPK$6F7z@#4QN?u_I-}PFXRy`i`ER8W1=n%Q$ zSKGk4GaBN*2NsdMj-4tZ#6NUtDnb8dPc!u(Kg&id#YCIZ+5)?vZtBB{c=+RTlEDQp z`bu89LwWD>Y|T;Qqzq263$O3*@ZTFY3@|=J)^};h=8?YL_-iv1#R-ZJ*#eQ7hP^B{@ySBjpmtV0Nn1lx6U$9GTUpw|`}+Dy>gv>%Z}!JW_g#LMvlu(REc&-#8lG)yd2O99}AyIZ>M$)TE z9^?VAltHHq^|4PYS|GZ4KmA_;2YiU&MPxGDSo2wENOvPfSxeu2sls|>_k=MEFK775^!kt`(6C=MrXv=3N9qE^2=wg zNO}F1Y;Lh# zkj(T}4ejPvU(KRU{gTf1)usJqwLB&eC+3!}O3f>Vyv z)CqW2gx%(tu5)~=Y$t%}d4Mqszy?WSBQ-TA<~eMMC9{46Gmx4Z0zbuGFQKi=(TIz{ zveU8OtTxCrHLG;l@Q{TJV*eOHy;bZ6gd;Ur7sUBd9jOPQsw_A0!)GW7fHI{EBM`sn z^UmKqM|I7JBjHx5W@^_mul|f6dn3Nib8A7AWSiy|dKoaRZnQSX#E2G`RtH+*`k80XX2}rgVZyyXceL^H zYX~)!jRWkQ&3N2N!$va^;02vyl#>@P!JTJaA$hu+O!L#c8&@5@{iLUO zzCckR&Nhf9@$b&zU-)4BStzlec@la>D(0oBM#-Dz>J4Psm($00#mbDJe_EYc&E?$aIQnY9^ncUZA_07tdw zQnkJu?u3D{iVI|x9Q~&8g;5b3E-<>LQdw=k^YNd@!NB|(J-0`j6?{XXbW;HgV>Hb1 zF>ImF6P~?x*NiF^uD7Z2>}@n`jMzT9)xM(Ep}?ArB=fmLx%jJZ=&BNrM8Ks&$SGI+ z=Vm_(ij$Lo0)A%`Ln-NJx_2p7o#C?Z$?P>j@t9vDK+xl89w0LHsvT8*yKP1JR z5=R9W#EJS4TCJ~GRf4*w+Yqhq@fo6TjlXFud?{GV?n9-i1fp-_W_#|z-A5@v zIPM_3KP(X9pVRdU+Pw|M&YV4pGV;x~S$bgV2q6-Enq(?ZMph+(3yCL8<0#K|d-nph zNjF46_&|g|tFlvT8QtXvx0Am+sJK0~28J9}W6b`N zH}oBEsnv28AH3gij1^-esu)4?FCLS~Rb{yx)@X0y39ocNC2%=6Nj=zoCjhx+fUS$b zb{h{zy*swm>MQSP))$fe_7V@I?w06Yq{QtwUSmkhjp&ZDUqVv*+#>qUw^`J0e2CkT z*%*Q$$532T=!vULu=(Y(e+OKYw4?ngnTtr_Gm09U+0+z9*OrQ>#~Q^!wUZ2Fs{U*E z-Jsg9;n%w|!V!y4R(F$u(T8Mxu2(F>mZ!SjN*FQ{vc>WsZsO_N?Wo(WyyiKvYtGBZ zfn{|VbL_O%?5{+|6dSZi(%ysx+^`-wl5W>Q23AD0I(_J%18%rFPDQ_JzIG24fj|B`PZQfSw>*iasC78I}2+Nu>Hxg(gZ z=5EC{=B`S0#1oFQkO)mY-ES6@Z+Np&c_4MRt$6OWON{^P9P<@YVt`_uP2SM4pdDh< zJ7x_pqIxR7u>zERGs7KzSv<&@-2$<%gLDLL0meXPxXY+yShV1Tk5G%O>J%S>wItsB zHlf7xjSHh#B4aJ3`#Z&x&rtd{jLE2zQNJb-?i%_g>gO?ML3p1*!)6`ugH%8+kA^#8 ztlg<@B+Po&pp+1Z>29JceVT90fz#A4QXhV*aw-nPu0`j2TqD_>gSRpGIiMN)Ga5WU zPLHpHmdt6HDO*wdnmrj4#!b66sU59o;Q-YGlqm+$rFkz?42?rH)G}oFYSAOXy{@lm zaFJhi{Hl);G<}F~|IGHf&N(+Xk^5be50}^5Zb*+A)lUIt158lgEwvT0iV8S1t(e>C zfJZA!R$JDyN&43jtDm{AshLYnND7K5*6ip|uE+9K05z@phTt=-kjVgf!CV{2uJ+FI(jxwI_e9P?bkH zqOq)~en9-ZdiZ?D4pA3=ieO-SLAF#!_)cr7km7KYUo7}gu;eCQZa`H@uZM`l`WN7iX9lo>UJ9{^7RxTEk~;dAcHF+Je(iv&F>1benmQ3|T1lgKH;6 zdvRB~QvdY8MCax>=@q-&Rx`GV{g1=t2OJ$U^CUhB+hSvmJByy3b3h( z_SeU1_21IF8v!_Jhxh$;+C&Rugq9Nb|mY z))_d2PI+oSPZ6vewI0f}uSassA7np~*8~a^@`dVGcPMfg`Ax?AH#uyqihy#BQ+J*o z4PR~hH>8CC@5d26hXHpj$A=G}n%_#Vcc&Rk08jvz~DyM}6{wO4~IsHY7&p5X?rF2gM?EqNk zgZNhkLbjp$$qW*!OfKR&%t66QZij7s=9T=lW1tpRk(0T#8*R&^VQ%o zM6uT+Y&yJQG8UUnQK9V?8E7KS(WAjh-5J1_4eakyhVQ_V-`i+1JAH44DHb8>7tr&C%gk=R$6T?dy_wgN`WjQ${=DYvSHbne=n1 z?qm*+5Q{Rn-|Q5-sHKA%`gx45sS>tgAwA-Wf;N+r_0_YQR@EmCIh+eVuMWF3sJoF? zv>`=1`Vb_B!Wn%=eu+3P8j;XyE-rN*&gCmEw+;5puiw00p?J(6V4CrE-JxQqphDWw z>j=st()C35B*8pS-O76=3=6(dF-Bk(8&RgMjC5P;n zNC<6uPiJ1Z^oc2FG0o1Ydy>UNt4T^q8l!sOd9pfa*ATrh5Oe^Oj=SR?)a>jkoJmrR z{90cb_=0$dOP@%DpjG+zZ zO53KOz`+fu8I@|0caYO!x-o0!iFXVLie*;zo{+07j%&9=iXF4`XecLkgZ6v6-4QJQ z`)*LOfaLrE#3KPJ*}DQcJWVUqD`-$0EP|CFQE;etgE$>4s|Ts=rHRaIY3CzHz6NVr z!eVViQ`?c5`YU{m8lC8a)%W~VE10tPhA9cSeu#92x(-ViG-e;{d?Q_O1a@}6WMLg& zEyW{svU?h$xsbydapt=3DJbPS3!$}JZt?{*rx+=LJ; zJ|$b>L@o5<>~jTHzXYr8iGizK#1fKUEMfi`hFF)oo$VyA9{*w}-|mnYg^sJp2MUcL zSu727JiiT+G#H!%DjkLD+!5ryw<+V|)b1N~uk(Qv4`%p|w~MC$G2fq>x)V=>kOqV! zBO{}I@6J}#*F36)fjLtoU`h2N=L3#upGNr8WxYhSf#b?0fOa{VLU^Bc&-a?E>ZxKo zw)zkrjsa!O)T?M*+EERDZ6ksh{E%ruZCf+UKT&4rAxP$UwA#Q0%tbuz$i*}Qc4(CZz@hFnfB5;d8Z}1PLX)^ z4Fl(9D6DR5#@AF!uMwHKy5lk5i_DdUQXp2q<_&M5roQDoRP5jP7U5;{iVO`%PVY76 z_Nstk%Umrx5$gaWQrZijzhPTl%O5L6c`+cB@H zcRO(qbj)+7f~bopUwZCaoz4l?kU~dCcnA45Y5u>aMde#M}F;L z3^pw;fqSIH#NNt8#;%QE-~dkGB3$g5 zSs(5fxdh(Qn{q1|A>iE3>ije8`uYd%m%8XTo2ss=rFiIWrtay*(~T3I*{z^v!}d+U z-TC@?S8Q(Rfu=|k$r7al@Y0tYS$BACv8PGo1hQypYfkc|7xLxaqtW-tN1&s79Q|sk zvnhz%VV$nYsM}9yL2}_)>Cji|ucB02^Tq1v0#6&Ck%(=I3XR6ZNlVG!Yr{xEe;u)P zw2HT4Z}!SlS7ZRDgcUFKg=+#fER6NMLG7@7?y`R<+_W{%Y70)WmsZB?Z1y9xR%oNu zBthB0GMwRtZlk@dTr^?I{Fj?-jcT_v{P;W6wvD6YF)@Xuf zbCWyHczGXde^-{^NnGM2@YNV0Fe~hI-%eTCdGb@0K8~!>m zv?C9Xry{i4lLHU(r7gpGKVjFPn-(=`2azma>!pZG;#?6}j||d-q`XZj^lZ0BX8B88 z;%s%Bi2^oUtAM;Qj~=IyL6|QsQ|ZziZmpe3cAUX%gGxgWL7&iQHqh+r6GVZ?x@Ub? z{y@#S15dW*t>dCg?fey$SA80K!HQ;%v(|R-!gs!hotCJN@+&SM&8N;R$o@VBBNYEt1^__q z{Feow;F?-^SQOVaicUJ(>h1kJCRp@Z)eUT%=W1z1)eAH~?P{LK8Mw_>(IrxaOh3S5q$$l!)Vby*19}8fK z?8yU|*HJTntYl>2j~^~5Zmx4x zusUql*tFqTl=p;Lgsg2S3(CcCJ2M7#cmH7{*3ikj=4(p!`x=0f2~1vU%FD}JZCZ!5 z<7!96tw-9y0!JKJH%|7f6q=nL!U)i;K>{YcH1CYm?g z`+gzlCb~XsIquOLC{1atQp{OJ-k^B+n8U^JN>=Caq_#_?W0K!z{EFBlh(Crcd!0z@ zia4a;=4&R*39gYDr;EP|G>iD-jPSFlj-Oiha*DE;zc86~K{vFP^>E+)s^T|Ek%^eb z#b(^9aHmt#nEU7Z_vbtAb0edzz>D-X+P6)X55i}AEE5hSeYIQ{{^}z^{!70d^(DuH z3u+V+9V<&9XPTHMM$9a^SzqUON_w)`(Uu9&jTsM>UbAVq^j65Rx!@5_oR-CfU8Oap zVRnd`hn~Q|YYf;Dppm9owFP4be?|(tdW2X@t+fqibE3^RHvalLwYY%qmO~dHl_-th zZA?;-yaCP_+q&3E@qC(`hz|HO7c}x=GZeWI8K}!`Bt{|KhOG5W?*4?c-~0Zp)A?!Wd?Sba?Z?m*|L$Yjq>4;3X1cjku2+ZvHYlqzg&c>Zv@_qFHV68G;h&l)56Mlm`g#^a(v`8f+Kq@S z$AKLt{5E~IJa`e{*E1W$HCU&E#88r{YuNzI_YitU@+0yGB1?6pKnGG0zwX#J=UxwC`pH}HG*2`D#Gs@)Z{2J#bIQrpma6Pt(*8Hru z1u(q6@xXL7>sZD_A|5&U-qVJj&RE~;dSJs58eZc*kM~`d@eMZE;JohE)sCGYpsz`D z0Irrqo+t1g*g8>$$m{V%n`iENFmyi#d4{-ooG?qgUg>h{pnl8leCip%>$orchLS4% z-2@w~*7D27?`VCY!Vxai7>!uEa#A_^BF)bG`CXtTSJqBd z`LdZ*Jw4q!q=VH24+oNt(0K!q_coHv{-BNz;eg0OLR?o`)onnj5xN23c50~xY`YDd+F9@>Hp_&gs{r18RUL24sf4@>|U zWWHP9(lrr@lye*~u?X!{?S(qDziM!F(D7G#7_cVx)q-c>g{Vq@ZDv)sRtT0}7*{<6 z;D8Qn9G-0JzKW`Ow}6!*xe9%PmMW{ucD+}gU7y=YJp%{=e;^TJs58U9#L)P2IUar_ z!X7zfC37C+U5)8Msi_f48U7viv2L_hJEO^QUE3{77v6GpA_jLd@HfwU9TTG2^fjhH z1%QnT+k&5?0S$P%{ecEU8Xow!J%#f#&h~3~_QJ9ZXrXsp`wi|c{3wTc{1z(J`&ynP z!H0t1&J6=~28TWz!gK#X6bCrCUX{Sfa@dh1Wvojz(DDfKOzW@r5XALGB-&|NK-;tT zq^x`&hN+UxI1;1aBs~b_Km^l9%qDKI?&!%_-Xr}<1a?`;Sb%-IJm*TZPdd%Z;f$#+ z*ViSpA+!%9zu}t|3a7!#1XwuFIwf9qY>zm<`;~vz**(9$(J78k~`p8;$}nh z-*9CV!= z?fK?TZ$)gwK`(G+GI__gYd6)8;A*HTHn30<+1!+dlpS{U(v zsdbiy)ICCOF{Rmyr0}+k=eNHm7_>Iz8=(o&V(mt0X$`g>>Kk;W^bovJtGzA*-fsqz zqLMn7)vcQ+0ftN8>j{>L-6lBbAD|J^#R5d zYv#KNq>>FK`Fskcbk?uPv<@_9H)Sv5&-7nnR51HDWT-h=;=2v^m z{B8tKF^u5f*!&h$841J>lRaDsXn+sibOOCHR z-7=a$u?pGPTdqh3Q||?-9k-6FRF<}IN3;I49*4QF>szI7RWVHWS@QM*)ZQqm&AuSs zw3stC>5`oSLp~J=uX&^&hA}`-j`ZtVMmo)as!&*BPJ+tAV+Z&oj}p7syZ+Fg++S$1 z?(nN6`x-;~si4s4o3hmsd!NV?Lxl*aup1jsn0@73sb|5*ubE%Stoz64^OWhw7NPMn=t-8J(V|G!+;yE{j%*yPFvCNGt(uqt zJ=NyX5x315Vy&r_wslf!LC@D*(O>T6U;@tGB$o2F z7epX`wN(;fH+%*ypRW>fUh9bJ@TFCy_X4QmOgwFEgG`c)Mp)*l=6MjECZk>sksbB; zunyfo-5$EyuaM{i5wPcU$_5>wokz)ZY}&D?n(-@H?}TI`v;m0v&2Z>u2kJJf1(CV|Zg`i5cCJr0=|u(T7wxHGG4hg(aPT}K zZ|T__sS>rmyV~`AXTkacLJ(-c&^Fmw>*ESf7*cfY5#ILD-oG}mOZ?9L?F_0C&&4y> zyHAso(7*mszRIkk@c9JZYO*@$X+z%%yZC3yQ;^cn~ti@Nw762du zNQ()pd97dSd*)y)JqD%$qf*?1OX&QK*S1%CCo$7XBGIkkLWtxmYo%Zk3s=c8m6TH0 zanWO;FxaPuFxhR;OGvF2SBoU%3(4|!Ubi!OuV=Eoucxd3T)2F55xVVkXL_?9Z~9I= z`zh`&s+vkgraY}XLT$bfOja#yT2;TgAb4< zzrRNPHe_F@73wnr&jz1oaHwqm8b1fSiCTI47u5t zp$iaqous4VtW&i784hy;draNybY5W}tJj2Hk86uB`dO)4j-6zt=*k>U%>{GK8o|=y z^XW96LT;Os6Rl0sL_|s){SzSTWnr-;MDC{7pDqrSNC~-S8C^QDF z=m2wtIcWuqVbQ~YZiUU#e(QV0WSG`}Q7k_LTEN=jHzQ_-MX764*m2Ok9d`4<0G44gW%`4wxwI#-JrYsdA(k1&i!1^rXXyh@PsZ z`++#|OzaHxSdfld)}X?eNlBxDw;|9G2Li!vAvI-UNq8xhi2zdPKkq<{HWw zRN7!qZ9a#fEX2aX9k2YweUW^u=4(J*pr#=azE$k8wlu$$J;J6iCt;^^CDXkIimH z7H?R1_0)Zij8Sp<0$fM*HOPH+Ox7Tv=4sybStIt2E$1WV>nji%;B#iLrBHGqc4TRO zd~JUmUTmib8mFd?Q|GoxLsp=$O4DyBovq%onG8zkOic7r=M=uDYzTgY*%0_;%xHDS zpt86Q;6aSS=dK}7Oiv-5L2!3xy;sPSVId&@2->!=gD8rcEuVu{+F{`xD;!%w)=oB? zsd&GQS9VXOB?ecjZrt0<;xJEL{C>kbP2%*##|)1r^9*=I)NemrQ4_GHY58M&{`Icg ziWO|{$VwZ&9;7*#9+Cmu0{4o5Lb%&u1Hm;>qB_o0Y4mMCb$$1nDq1xvz<_TTW&v>w z7!WF@8ln2}61C0d_m|Q7bLBg(ycrMl&M3d>nE-vwFICcoq=YFue9mh(S`2!@#klfK zuhf>fSXcB6_y(1aI6dMyD(`sb^jl+G`^1kCfZ0)+5Nu<73_-Izjqk0NB{j*D)J)D| zSnK>(C5!5I`0^TO_03Dzf(L^Q&p*vIRZs?D%p@Q?UR3c$JIj=0C#+ZbpBLU7zb>tp z25QTzd) zUw|p6|DJYVijOV}fs#F$Koanv+A9?moC}BflIQ{Y&zF|FZhrg@!xy}n}UV-Q}+ zr|ZLxux_S z)@B_oEyuA22fpf|cj=5DsRhV-Qc;oTCv1(S(-rS2IBsh&GwPlkbuT}7cQ9psgim`< zUbU(XJUS}})IkGq>Ro};s|%hNO#-}M(cel8cmosh0)`MA|?wm@avJ zzjf7hkK#sM$FhyD&mQ@X+C%l$gw)Tlz>f`;5mkY=B#nTV3il}!4^@S?1EAnK0N4Mf zL%tuY=eb}NOk|6E2&e7$5UeJsBXptiojU-lvWFiFd&%zQ$vxbOv>e(2?pT--?;oN) zK(L`toSGS<&2OGRMbAwlm+6rlYtuyC4Mm7I1tbdo*!2w7a8@b9etGFxamY)^gWi&( zY2sd@pssO7XbLc~Tx*#1uoKR3^)%lmf+)!Q;g>*aY%3Zp)%fu~m|wpmo5{rsjV+t+;*U%fvy+-T}9T*k8ZS!9RFP zbXgEhs6r}*Y`HV0*LpV+CsI^8&h4`c+$8jng8y4mfR2;eNDb~0V4T2s0b+>#2U4mw6To&%}YQMb{#IF~Z;b&m`@nv~cp#Fb<4J0T${pB{f zkM(tV-HJU#yz>zs_0<`AwM&xoG2;`VX7(UBc-w{cF2{uKd5gKzMUxe@xQh*|ni(fC z8_E$xn7cpQ;4*vC>_Tf~{(y5$eL`<&X6BcKeQ>Ds_4gl@=U{Ft9G%Pc)@of`aR&Fq zSv&@3|KOP+7DP~S*adh+SD{|CXZ61o&vlOwcDsCi7kGS-pl4J;`-$^S<;Pm6#hrs+ z4XJ1`ZvVUiEmtKX07-EG_m0wFuk$X{xxnVoRQdMf)q))z2RH}E=(qoLpFSrqWRn%u z7@U|n53XJP!1@e>;SK%Wmc9h4g?9Q?#(zxR@qMp^1-qYC!7Kde!7E`sf1Ovliu??aSVR=qV+^UOEiqRp8pQzsx#(;-XrzIkh;bf{?$F3JQ{KJq5zSDW z4?J{5K+^7k*(S!rbi1jnSt6}l{&TIS=ofMIx_*!TO4@6=U{7UF(Sa(LmWE37 z@5ju60rW52wDfV#yo#BP`$YU3emgsbcs~vCUPzW;C2D#nP|zXxX;V2+^?vb>9>Ryn z3v1nOZO3l@2$JN{(PfO6fdI@DuU2!A)WaR$TO#JzoTm>z;rq~i5y~IFbn_PRh}OBg zh5=5%DL3qfsdD#&uEO?4_smkGXR2zZAeu)R=5sLF9SlBny5;F&wq` z!!3h8?)r^X7Y~gMnZvt!9PnaX z>3?cod6--9bkA7vl;>*)cTMxWCE3E*E7sz>s_&g`8n&ee@J-b(dA}}OuZdT=^gV(L zIQc#3F&mmzB@fVU=8OVCEgk^|$B<&e~2+rIRm01#c1=)Pxuq}ZT z9Tyn(MOS~nu+G3?tQJt{;YhA)L$+@|+wr?nTn^krW=HsRySb7;%@SRKH(IU_JDu#LR)IEsZZr3@24_kS8R{**qZ}u6FVgrX zsvgSvqns|^yGWs z%c(h-jW}iJ);$Xr7?q*s(*$je0)z|SbpRbmP(vv++9*=<$%>lG=@~W<7|KlR#kvn& zEI!QK&H5*O0L;k>!ncURujl;&s&0bQLk0YO+<@$GEj?!QjaZaB`go=+><1e4-gHk& z-&Z@=Rj?!#F?S{^D(Z#H43~^ndzg%@Y{gS;m)FuakW2EPD`WPtAoAoOB`FU$iWB*UtvK=8fZMN607saz8eCP;)vi~g3 zhGi_Er%&mw^z+}Jxv@of2mos~WScmgPbku@j1|K(s#1(0L-*W$ z?qC9PwuS`O4-?b$qgSZ22Jxf&5gJ10=c9bB(pnVTU-!@e8KHI-A}eq;VjHL=Rom1_ z`RCE0>rlMWax6V-Nd0cOJ-;!Fuz{S0rLyhL`PR4eOP`JOyZkmy?7n-(x#}!jI5{4{ z7jBijnhma05RRZ76hZDxm-yxNw-MyHosC6CF zM-})c{1$@%`bMU9;XvWC7YZ3OLn(0eS9~Po9bTmx{e8xQCOU^EFvlLmwVAK#6Mk4c zHhB2_oVkFjV2W(4_=}J4NnSb$@om@(v8XF%H{4EK^=FxVV+`5Z?x07_cB|ZB)(I zeqhn3LMg{#$F6A9Kh0sP!8SH!pnm3t;6vv(ryV-I>Mp$KGrz@vv{|IRNP%&TE)m7%Cm^A`Rp1ErFh(L&_tnfmZGD1PxtC zVq(_upY0NrKeXv`jVB+cnI9zKT%UEE8%0Ltq4Q-jTUIYYxLd0zV zF11YdbUlpVG#2Uyv@kzzN;91s>`YU{6)I#EGsE|AK`j}xyH~IR=3%E1;xLn2YgKp zU!4(-fLm>K3HAhLo&aiYp<&`a$L^(xvxXZ*oGe<^VyaD|R;MDCDbP0x-a$vnPG3|0 zpHk3Qsq;_h)W`Fc*{(a$(%Dwa;Gdt!Zov*4=zKrdiudEogdBPc$FAV`wUoE-=VDt= zHPPFR;-0g>Wv>Xx3HJJWMP)i|e*rKNCj&XR@X?8ustJzJ{{T8{V(L653@qx}hOpVS zAnG*&^D0leGPlC^WUNBnN{VNHY(@p0vIwU^yet>8BYKhkDb$;(MaGrmqy3J9APLf^ zfVW}I_t&_W4WlIZ8pMPQ@PzCCd6F7&)B?ZyzJTg@kd%W8^s*TIQ<+s?&& zQ*&N=o?Y0&Sm7uA{BKk>l~^k=(~E_Ls~$p^=-2+Du+iXcVfWh`E?~Q6Q#qe77Pk_t zxk=+ckbZW4$$ji`>%V3T*D2WG9@n!78}zQ|R-+OH;3|TNRn?gM@&3^+xTCzgj(76S z&=cbzJom$mau7QS=WJK{wIS>K3`{p~+Ze$6wBhGV`_t zgQHPcl=@~OJ1=nOzY$=S;7=E%$4;x-{Z}vrhPfOH5v*&)K0<7qN{fw!4tbSsG=j6^ z^vcqK3n|_tqmI5r{ZIKtqIjH2SykSw*4s}Dig?BKdR$$)dDE0VS&wVMH5>p?fDD_d zF{fj;?0t|aqEwOJ-s#J6gTe?Qm)A3{_Je+(>gECdr{T7*>;V$WLeXz3ZSL8ci?rM! z_o%)lTPmkR_g~F{$FI*7vlgo@wmGpZ6w5N?#L4YM?$s(Qjfomqz@2B>eixdNs(x$$ zmwy;&#x&w=;tWuY)cQ|?xMC5Z2|NZ2yCCxiHDw%*;!40>$77PP=+z`ZMWy0UVP>=Y zJum#gZll%S4Z`cmdDwe@Mk|S-jZI|Xg^V-_IS-{-Y;w#$)jg(hlR3rxp`63< zKvvhSP?V-8vhU%%ey2DKd$`kVjTVc|93l~mi(#N)C|&f_ZV0oYiD*Q|%IVchC_ZBW zP6swMI5X6q&+K@~EWjI>Rd+w7`WDXUwVEj|sxMAw#HTe!W0Enpr=K7;y&=FAab1oefqa+tJpus6XcD~R2!fd$;&g$`SxW5H$*`Okv2Tl z?!0Omm>iRylDiE8u0C0FPwll%$Ynk!;sAhbP*h(lXDL-UX-y~d2igM#9>)Y1Ib}%W z)A-D5-_*OR<*?|*O^h`l`})xV@G@h3vE%L*`MIL^+Gh8|;NBIYh-{td-0LbF^qM;e z;5h(ZS!qqMZebuCsU9bfYt{W1DG9&5qw#ST&|ax!!O~dbonv*?3E^Z!ix3n`&Bjh? z`8nZSiXCy8uLg8aq+jVtY4ban<{zgGEXis)w<1nRKAB`swV%o1RvY{T7@KeNRjxH` zJm$z^q>IWW-TE+I@I(d%-uqTo^h6&qXxp%MzItxSvm-p$YBwyrUz>KQ5I@(fbxyC< z*$V_>bcH)Nflq5Yx5gSx=4)^2aZb25^_6yZJ_f+_p<{uFDx8=BDFSRYxNgA+#Z+4H z_^G=UtahwDcOPcK0M^>d(kIvu!Zn-YyT~-Ce+?DCqj6?}Vg5Z>onJ|FK>RVK!EoMs z!vA>0q`JWVOXxLsI&w!-x2%VNb>i|D3fGL34UM_)qjMO*632^rHH*RuAmR*0aW0y_ zN62D{w#vVTKWvg?_ryHqZFQzD19CZ#)F|UF`@W8Y7f=m{sXH?P>RB=-mOI^T9mAOz z3b`W=f-5fP;!j}uUJblrc|0zxwvfQ@Ev#mot@|P=(9j_=+*bJGj#p})xnLI8=+7gz zG|4G%cx?eZ;5<|RZM*O^aM`3A5vYP_K&IYg9-AW6Sughp4ndj)sF&@GtHuCPi8*kn zM=mqaERT>4&oI-N-QU_R307X(Un#S!=>I-4(G%w2E!-7A{>lBg#*H#B8~|+d0Ag|q z3LWTr<3VVi*MsEP^IXnhEIOao0cW3O`SYCX*#kJ0W7w1oX;`Hdm<_7?DR29>mw?g6 zxMUZ|j!YhX&e>0h`>1T^QH9w2L%44w-G@PiaP)u+K091^U4CD*35G#myPCC4kvEx0*wWb9?** zY(WRltW|X$S=WYP%k1|_e}5+4ug-(p-Br@=n+zsWIen<1Htr5K*){RXU{KxF`#$1s z8wl+xwJa3~^DmOZ5BapTc)l+fqUK&MlVzAIP^FR4lg6~JCu@_N^1nVfF~x1v82p1d z^I>HRwnYf|&-)R^mvd%#xXyxpgVT=7?fFZUui=mQzn5#1qugLPb%wUSzOg^dT*p)|k?YOiJj#ezC)laS;u0W1xzpCw?c%3EpjWo@&t{XEOhwbt zxR#mVv!)lj*TY_H66gy=+{cy;Po-pv24d-$WpH5V&t4+V^An=}RM{$9=)c8Y{&nO{ zD_LymXm|ml2p`XlEjhbP>2Y)e!6xh*av@nd6Ul7M1$!Cq|SRHPs0NJvcTUFRxZWzxP$sM z^GGQve}gA~GK&sf1Ok_7|DO0RO}R@4$GP_rnGFax-jDAcYWYEm$HELT&H-J9Y_LkQ zHgB|v+j;cN{&@%`99Xuq2dSh^68ZFiV2JG$^7dQ?6hELrvZD z2(VIK)#_x@1eHT&?6k>S2`l;EzjfmDu`)_3`W6;ddo-cIvhp-v#;movHlL4dnZXF& zL4`7CyhL(*z9W+#@CJur+=7GE zM=Xgbk+957xKc=Q^mxuNbq>ho=i2p2aaPSh>7l!JIk-M$b4rz(r}YX%HsIbzb=*pq zlKLISA!9Kn^#%P~czFDJmH=cE%6IHiH-@c|fcjPGPb*(L9Z%)6a zcAkpwGo5+KEPfQN=-kzY`2P5g&x+x36Vv$UF$+@hItK8xV}Sx6ei0^5IZ2n{;XoWD z_i&5@q6-}|8y^^sbRxO(RqXn`>gDx!n<*cu4wOs#)g6CyG559yb6NHR7Nd3_j(=94 zDXFZ-RCnSCfxr^$XhRJ*X*FKfu+~_Fn(EqNMnH2FD3H-R&nED9Y8{{fdMF-XZD;f|pqXN$ zp&aHJi9L|zbe6f__x8KhT-AoZdH_^Wm`Gf-@H)Oq?qH31gL1gg*q&#QR2BO3$&m;T zE3UXt?b@&Kh_|^0_0eCvY%<45__)>m2GFd!8;p*#G*cvw4<@GJxJK1VMAPo<-L4dK zHsp9}p+>7PqG7uSViZ=#4xM|w^ll5mAUt63XGbkB00AI}>`*Z~S<8ZZo_ft`kv04u z?!hZc7kG_n++IJHeKpNcS2&c)b(5@t0FOPUW!7_n%^PcW7ALS)b{`YBjS%(f2Y{i( zJi|O|#uCPzz5-k*FqjSuGX9wXC|-rlZ^*rEccIlyhL9A)U|=eW_#3@x&mK3*E=9|{ zD}6{c;20P3By)5iShiOcB1L~I7l}`nD@S8ii^cLq!WjoO_jzV>lt}3HW2|uB1m9k} zbKcatSMI+L1m%CkPrW zq{cDB6sCJdOq4w1P5I}5ub`QvR4e6iKNJp`FII#~PvyvE%;;9ASFrtN7=vqq)Ci}o zWy@v>T{cpv%juX5zU5JH^uefV3yL-RJm0X!hnQck(LL54I;+7BdR%)aT<8qFUf?N} z)ARg>Fb09ZcJ@;V>k&4ifxf@X=YD5qx3vot5m0Pu+lQqmK&c1sX{0f9k3M~ny89fu z(3a$(90f_c)u(WInjLr%br=P$rtP`P5UM~gCXMJK81I_SiR}!T3tUp$^Q!EW0r~y5 zv7Pu^Hqb}_g19Wwc7!Zp^henmLcu#pep?g)b-|a_#zNSoO#As! zFl(aexqPQkB6t_GsS|HVV;4`*l&gjszcqha3$eD|4z%i=b}ne!lHSd75^zVPeSRhn zuGGOVY#!%*pP~~o%s#c*r9=s8jBH)}N+?^_R12Zy&L(!9Qdv=!@iw-a7uJe^f=}Ua zfReEX*gR)MGFRw4mSfl8U}7?_nh~jjZ{_@`vR}k+qyP}5BV>sFJ~Vee-C0JO1am&i z_JvfSN-DmStE)=o7--1BmETV(Pq5EP#Pww;DJcn0-R|+-Na1B=W&OC-)=trIWiDv| z0$W`L<;5^%lGYl*qHQHQ45cUnzb(RXk0@%*n?1)rf=*;+^aiEKa6U{tSo17xp}ynQ zSW!`_lqv@w7RQsMWPfYKwt$J@rr9Sk5ksK-R3(8`XY|)<=U%OiHC`)OPO$&cK*f%C zx74eyt4~xGG^0vU7p*Mpcy*g1&8;d;%Aje9IOg|ZxXlLZ+<$vY4v(HiFLU95zRYEX zQK3}(l{aOj)DrmY79ZZd-yXoMQ4vVwQBmuByq#DDg7fctp*D@U=a}Chj!zAjYV*)E zi!(!#oKWT<&hKv8l5~<7K9XWv&1qzdfj}r7!?RJ_#o$-%kn9baVy#2FN zx4z2jy!*=-GFZe-bk)l0W1oLM({_2XFju(BNij;9f~9s`)vm>UX9s6?sO01PV`JaWWTJC*VN7v#2H!pVslP^0_%3Sapi(X?De?(k2wx6kN z2FA%Y>UTpJ#L2BCjoTi0DeJ?+V)8g|77ywpZk`O`4*xX96H$eQ57WQp8t71{@r?QU z)k2w59FY*H!}LW8Y>U-Z@*f3=eX@Aw(Ox4@j?K`g9ZRreG|EGW-7Xz_a}7!TO!bW| zKL!N@U4BEJJ+CHnlypB7&6XeXCc@07Uc+612Vpd8^gE*zGgKqvd$nlBvh9dM@g|x^ zw2Wbo7p(H7v1hLp>F+Z(6_nEr%!$x3ls90b1rRt35vm=+$oG)3EJG`;JW!QQMac;g zFiJq1FGC#b3qflqdP$Oh^KnHPX*sO$kh4ci@V^(zGi%nPGz;7adpT|T0v8@L9%;49 zJTE@25s1uc8EMx3z>xS;Mj{?a2(_iASmHdR{)V88vgSQmW*wCmAGje^wM3Zh8f8g) zEB$yJ0L0|x=X&KQIa0WN{5%T5aJ5t^2FDdh&Xwerd&GBu72W^GlJ#`NknK`yW`17J z=@)-VB+eXM6MsKvbGiapzdgqD_KD`}RV4F_jwMf~9IsRLN=V?~_n%i;Nugxnb|G*g z=!(o6!Oir_lXHEPjLq+*4(wqD7(;#t!A}!a=V&^w0Zio>uVuHUHv`37=ml!;P)*Sl z+bSFOB-=HR^0kR20Go1ey1x1iPxL>(w$cpvTvxsZZ2^AMO`cGyz?JWPAUay<`MK8! zn>zD+?HSF|T_Di=lt!|uDAftuQt4M3(?LHi4c2JC)(vBOo>{1$zsi@7IUva#m%m)1 z{vEc2)wIF!jDM>1Fozw^;Wl%RGLemzx=fl8WTUD=wFHQ$?9c{!GEM05$Ow710lldpJX63N_mTYIFCpju?z_p89zxHkaKQL zv|R1oE{@HFaJ||PO}M92jPX{(wIrUD>Y$QN`YQ3b9i-4SmTg_(dA?+i}^m`lpBJ4kg5LPqA-Q|*6yhu%|Kr4^1 zmc9I64H4^e453?VdIwS0Y3K&8+y-YYcgm#1Om=aO&~14X!Zwp6( zf*Sx%z=cL&70xy8@ZZ=Ba7aa(kzsk0j$D_Mt?Wy%QjClMaZxT-)tfHCHI4m-9AciC z5-p5r0P>WA#tV=0G((_wXi^7ijN8ejjKmn+;&YEY1}mdm%}r&@FRfR^i!T$yiTIfK zhv)h5m4C3)w+qn7v9^C`V^1w~6gb9w$KkYj!Bp9!GJ-tO+M!V>x9%P0+&|1ZPnI}0 z_l9HO5ZM#_u6)`_t&a1ngn^93nl;cK!g30JXh& z!{BR~z;CPCD*{(&BYEb%OY9VCk<5;JSSG#MYS*KBm?4Z4C5Rbv6D9&IMegqP8UNRo z@`QauC#RTh|DU8Ryp4sX3Hs}kbsKTkuT#+`yZWeI9N`zoRZAOAE-*kWF^x!ab7Kho z%;WS`8pxUekMSw6+P<#?F{O)K9xK=sJMGMyy2|yz}5%L%czHpY=uUh}+R-2sNxO zRDnlOY?zb)Wc5MevK1yCh!~1F6SUV>G8N^89G5)Hr?}G_l#vzO z3%kthsxIW>U%AOctckNQZY<6yaAE6!vkId^?i0R)=m^Ic8w;`(*flcr=}Nf=QkNa- zshAjT7|F?N5k!b6qyX|YG_VI-F(;xf9<5tpB|0EXo;+V!L_B#eS1E!}t~V`jX4rs} z6RbeClrDi)TN5}wM z#wK-TmS$oL9!&SW47MO6@n#*nXBCtLu#hB6rYRCTQK?%ZWy^#$l7?g_cnT|RruzM| z5#fFi3_Jv!XByaj6q2AI9XWB`nUM!{84}M)^=d?*uU|^%Z;l{Sq5O8h>Ud0~91dm- zXcM>pOiP-(2t1^g1x=C>YLz5m$Yh3o=A5H^5a|u$7QT@i`v zs%7L3pt$U@>xI+e&N22pN&0Qe7Q!jPHCVK~FGt)0YFchhtB=ka7IVudnRy>Bh!0yd z6(>k1YU1jL)?Wa|h#B4my3)zalmlWLgzHRSbr6VnOu7tOoA!v)_@f|kmJ~NaM;$lv(c&CLU=md(LVjaAua0mwq<4sx^<7Ya=fxcDa z5Bp9qe=e&{;dg@$zbeth9vP@>k=?^EL!>r27JW$1f&dc&=9Yx0XtYts0tOida>W=8b;`U)nrg0OY$v)yTB2C`lL6Q_DSs) z25mzHiXKrUXV%UqXO~~FIknwzVq<2k%e?69%DG@8?BH4LH!O^@bfgA&{?{_VNp`es za@EHEGEk}_d9afKE3%1{YIkSnZM69OV*y+JsUdfyXb>g8gO>GiA>+VAFm{B+br4%c zp%A~6ysGcO$90hKRGPuq!d8Q$O>f6)nh1g1KGcV-K;>B|P&3q$J4Ox!;`x{c@qb&z z^}Cj4#yZ6OfIjAoScY0#BnGBb)!&<6i;)n9PG%m50TxuOkl(Q&VyY2lB_}`1U&*=u scTEIzK4HlJ{gwZB`iwN7$`@g}Tk5l-_%Gi7G${aSaRsqj5u=d*0e}Voj{pDw literal 0 HcmV?d00001 diff --git a/public/icons/icon-32.png b/public/icons/icon-32.png new file mode 100644 index 0000000000000000000000000000000000000000..a19279f0690e87da20ab7989e3ef4dd401336f59 GIT binary patch literal 1160 zcmV;31b6$1P)Z2dVs{n=L9$xrv!m9B-Y z-B?-Im9{Hu!xRyW*)*Uauw)ve2E5T2U8a*@OuUe2;)Rh!;xLAVEJLOvf}jD^p@a%_ zV@@95@7u1pn5&sFB)#}@dcLpceV_Mv-{(1pWn?Jve=e2=kOq(jaNhw~Oe?pj6vb;j zP**fVTj>+m;&n|Zkjjyv$oX#oWak${Q{4(cV9grPiXeha+f&=hD?+BrE<2E{P zhg8#w+?A#U0-(r-+SUx?`Ve&GZpZ{cyD5n1=O?i1@;LS~C&dOFux z-7r@BA=S1ab4lL62avV&VHh^VAzA96d&CWGjUUc8`p}-7#uHP(k|Yqh11KkePP*Tm z{0qMK-iPGa0@><DRlrGxoKk@Mn@1j^Loh6C@Nn*3=bvDrG!nBcfp3v zATl%zcJefuC#JEC_A4d<&zt?k#8yaUH7N5uCfb%=ph^vZpqeU>DUda^5Bb(AJbm*H z0t8@R;x_z12HL#-}g8eiO|1a0UzC0yBcAP#VSK%i@>xJBK-qkw`m}c527U$0z2{b z31GM-I@Lt+pIt26N>wo)_aFg}h^o#0E@5VhtYxV?VWu)4dRK>V zK|Kv((dq_RyPw7O8%gXU=gXUzal!(+a%YUuzWqS27(RJTJ?Y;C*%h7Bj@f+Pe#I2b zKuqWzfV++u@{=0(5GEB?=)Tf9(jL#|)(Qlg5i zfj!U>5H8OmO2)4|u{E$;cnJUBy?S5J4-#d88Y{~;2&&a2QT{E!@0Izg^e0&wKpMdP a2jFkM^v(e@-H2fT0000H*De~YP$<@7g(8Jg+@ZL%SaEkP?ouRpC|ZgaC|2AxxI-uuFD}7@ySpYg zyx*Vujd8voXPkS_9%Jw9tc)y~Ypyw;`K-AjRF!3MvB|Lk006GsXDM|602TR)3cz}X z{J8L%x01ZG+>Z7LD{6Plr_teH!uk6Qye^l=#o{ zKYa%P2>$aW@V|=w_y0okUw{1ntNQ;oi2v_<|KF%Uq-3B-Szw*OW}^Pk`Uo7zhJSJD4kdudF`L+Y}7*=U*Sj9&7^4&?kAD41ey7p@=N z`RIuAOnV~7LZF_5s`R#1J5@9(%Q0S8aX7c8*ft&Aj%eoF9zR~6HO|urR`OLjLdYAV za!HEWx(;H~tlMDUb046a@_sarQEl&DA}&a z3wcYbrWrdh^`VhEw0g{DwoP=p_Zt1FRvOMdJ_nf`%86p7Wx$Vp@ks9XM#>nZOM{X3 z#(gBf1*xt*(L~_taH5bWuJrZqTq(+TKWNr4U~K~3WYo<*2$l2cHm@iSWbnP&SJ*PXGWg|+i0G(mT?%I za}vAhc9^J?Gtb&Xub!oOK~BX6o7*AM1QAF ztj*E9jWwf05j%|ff!!i0Wfw=3r*1OIYB)7V9tr9(1=iLy$-T?AS#=!jVvO`RT&3CT zuKw4Behgoy!@?V?zYd zu%pZEd4qxe-ybALs$Dlf4Pr+Qtd6?|r89?(&xYCIsWVq`M* zBTZs$PnXv|(q=TeRNc(7EMj8|k&blIp(XI1tW*z?+v+bcl;u>I>rq&e=*xd8EA{aO zwSiPz+w$7xwf-x#-^@FzLP2zBjxKDd$Wck~r5x}^t9s@L);iZ|W+x9tmayh>rHVQj za2D8bM4WHD&STdmx%p{%t*Jl{DysJ_vH4Rm_$i}z`azv>N9qUduU7Pp8+C;h7E9Fz zPwIkd{Gd9DnF<&Yl<`S3XCs?-3u5#(d5SoMbq7|5*2q%T?O;KSZ{0{<(UVASP0uVd zl)NCp&~tc^*p>T^xqGW$MxT`4>}U6G5O<9JP3h~nqaTHb8ZN{C=`2C(<9oIew8pVb zj<%()T!_qoqwITPjP@Pr!bCN0ojYq4UFxZ$jug_rZ8*+3<-1VTG}f^Lo;Z=Og7hv7 zA2A{%Ntr<$pT~2hK43RhR9Rd~qTry)30g$y=GGC-Gd+`1rN2xmmZBNd7Tvk@==XI> zSW3@BRdQiNLy9V}XHo2!#_~&|n8##j3^4B}Lo5e{?PRV=Y27uG@Dkt5{XRV~YXk@A z0DJx8Pt3(+xYXa-niD>oiBGqzv|gm_GSX&`EFU`I--B~|fa*d;%!#zJ={3%1UmO9a4+Y*=6ZOajTe84SlA|)|$7cRj2Ejw;|HG})vUwS_7 zR$eERKhliQLrK4HRUh-o2lI13vir_l8hhU3GiqI~{jwf9N-EpcxKq^mwArDL;bsNP zJ}~WJztQ~A8}A;W_)^h~v@5qj*q`Xtky_V{YmcM@2aVzmue_^NVvuw#UeJ_lGGhXs z>meEvu_$Zkp9gJLgsS{l$QK>1TE-d{@J_SpHu#AY_>$h*28MwE>;!V0%m>+*^;D2_@X>5A0e7Z4Jmz{!Hm{n z_b>hHHWFsr;~(ZXJ=OHD+BUDww_%Eu!qM=B)I~sF`;jKsQp2u^_K2p4f`cO9Wzv?~ieViP_IW8rG zq_F#W59QcqEXQ+9FQvxMQ_{?jhoq>|iT&|mejkLmg&U(df6YHzWO=f$kmpKXRQ`8= zhm<}Y-7^}L?)2=A^!B{pT*OK2aIX!xOc9$D_zN3>Qvwg4X+i?bvG>W8D9^B)O-~I6 zDcq=%lr!X(w%D@8B9q5fwCszI&xm=whUurW5W+6IUpFD`+rvd8+bRVQgn5h!AdU|V zS^ACp5=!YGw$1rfV;rITMB@pA6%kfF!5D_RHb9CmmKgw`)91ZE4m4^Hoy`p3&X-Sc zn0%hwMgGpBBg1zdG_AVbONSG_iJmP!xakXZv(Tcj)dNLc%`Wr6 zK8;cOA)7O$&AQc1sLlN-MEw@TBmjRjX}R4$-bo{knao`Pjy03A<*vo&F;!Qkbam4a zsS(r(tWyGC%F0EB1p3L&u`cF1x~ol*zDScs|Dhu$5$~{tPW>MvYx3Ml<5jD%_jd}> z#U3?|7~ic!sz{cgE|=BqPrQ1HK}I*0n8x7dlvf@_xq@!CQ2vIWcfR8KSZ}%P+GblR!-sHusqABsnL=FGB@>41~ofeb|-!?jWHhw zyw>^6Hb>m#8j{;vasZ0`3bkIdK%k>Q;(OWnfWIT&9q+ zcLeL#I7^tax#1SO%9C6EOHty<@=J5t>bc;kJxP~(Rw>}}y+uhRYrd!V9>zCDqZyG+ zHtq5q!-Y2zok1#ajXh;Lvh`CXZzd1tI(b1A(KXeR7k+ni*W*1>m5d0YVETYCyeGwU zqW|}%wu?@O4Z%V9U83u;evdEL${Rzo?kk()%^8IO>FO{HA!RQZ#JQiuZ!;Tj&b!;A z^8)1bm~1lH*)6-)a-p>>m*1boU5pXpeEE_LryAsmx7@awydS$HQz<@=k{e zSUg?FH_7^>Bnrk1{K9kUT`U@(YZe(}9Z4faA(3H6HELhC&#`Ib{q?vchV|LF$ewmc z=y;{c7S>JsiCe3&jji=w&8>dRRnbb>6?k(HafBJGI`RkU-6QCT0#0m8?;pzCdxf*& z_W#Ci+-)NmQxU^Ez`paYFvBP(b)!<-B90W*g2Ww;~~C5NT=r! zbBf<L5Fjvj?)bl>BfH(rF~tV?(fp^eB_pgmSBRuj>=I~$Yq`N>1*Fu%5WnB2djAn;_6TI~W*JseF%Eml%xT)g9UId%jkH(EBc z@W9x=_+v1^PzGweQJ=57|L4B&C|yr=bjEQ&B?r^*<6TPCCJTnpfeo3(+NWi`LU8`( zbm)5avKG=&cO}?7IYiaiyWW;M`Qe-$To0OQo5Rrb>;5o*28OY5{Lnc;o!{*I@?!;~ ze#Q+vX2|}J=T2~!=zGE#B(z6d7lr2CS@XY^S2E(~sM9NVC~}fh>+``IefZ;w>|%AS z=r`x7XR$0Lj+*70Hi7?ujC!t+zWR96bxsE7dK7g6{Vg?umgM2IURwGm?ZF~?{qHA# zAJ2wKr}wOG_=_*hq@dtA6s9~5Z99VU4!BzvZSi!79Pg;b>4&o3QSS@D^ft!Sb%{%(Z3f8(M$&v|LF+{&hqKKxO}(s_wz75BtNAOZvs8WOxMxA?J5b%*a|es zv?VYTqzd{qN$OfhKU1VB$%NCK{zH<{th<}*$cthS;vt#7BMlqNaiU@_)eYvI^ok)T` z+S~{$h~I>ew_Ox&_Q3E^XhPoj$FpzO1&_OGgYGc+{7yYoR!=tZr&g;ac}GeKZnm9y zeR|}vt#$qg{n`?%J5Awc)Vh}wF>4Z%3e3+ro1^ZOEdQDY$V5R-C?K+VshBE3wDh4T zRH?UF$Y`a29~2KA&v6f-CWc2BtARUsFFF%`kpvK8^|VPkWgFat@^m~V-POL0)7ugE zgatrIK&|M3aU02w1-?kPjE`n;9>Tb&3g*Fd+%s=u_H#j&F#HWB?a-r zcW%h?*M9LjjCuj*m&1*!xEwf5nn#T*V!lug1fL-*ogl5LRWHxl?76%E+~Rp)E$lw3 zFn!~?VCJnE;tLpJvVD2Y7#)9 z?stpgBQ@RhoveP<4xVIq1KdSzNBPl>J zp~ys4pJaF*b@XA6!M<1oE3}7mJo3=V3=j%Z#s+w+q`n`=zBLpQ^BDuLXLPU0{=jMv zC^UQ{;Jo0eK#6+kRg{spj~AErvdA%5bm^U*O^t7Up=2OiG?y$g+6|nH`5@>|+)Z?e zR{fp&JC~kl>E^dL>67WgICYL;&)YO=+s;QV-Li~Y{n_E&F=DcpCILfts5e)AyoieH z7qRqwDB1@BWm*8v`sQrs*NyMUDS>M^9IeCO9t4(O!yXcTQU^>~<7|d*HmVObP_j-Z z@A#u8&Y|Gp5pWa#E~|qY63Rk8EikO7A`M~Z?l;YueKy0M#XdO&6D8db1ykn53R|~` zD$U>Yk)UBks?Tn);N{2}0Zikon~y^2k+%7r{$=NsD92hOe-dZR9eNS zdVN45?-Ty(3l6Hic(>8?favukxLu9MVZaj=z&=RG0 zU)b2@o|T0bZETj+@?=l&GPaTu@6x_pBMy9xiZpHyj=QQt6XvJMgdJ-gI*!e=9Xujb zZ8gGmXLt7R3siGsgT!~yzpwYeF3&*^#s|4xMGkh=_T5J?ITRX;6e*N^EmbRWbf3jI zYm-8~_tNo4X@Q8m^BLnVwxAp_h~#UU{6{!Y&E2YTj}*mkqzj|bCyE#^a;UEs6{!*s zQpzD2?>fl{Sb)dYmmbAphiS<)PssNRr(ed1xkK8L!>#?ctpewpOBhk>^=YU6g@8Wi zZ{T29vlBb5$o-OviZ_aCvSUOK_Uz8hl6RF_c12{+ERMVYptNjt_C;_^y$Ny4sr8oQM#`=nLfXfWH?+1bw6uMjSlfXBBl{{}Hb0 zuqt670>|05#l-2!@ckwo4spRYsWpDxYA@8ewQa)m0|)vJNZF6dV1#N_C=@Yq?cBX~ zTn8YiQOpJCAZ~C%U4~3K`+k`X%5k(?9aZ+Pu{w9sWHtP`^p=R#T(RFndG@c`<5Y!` z6DlfEI_iFGj8c+OBEf$mZ)M86ex)lSjAR6F6zH6M?t&oYuJ&xlP5Jr;X9a&694%j% zY>bCHTGp7bU0s3s9e+&>S~r|HAR*&0kbzf51pcgN5Io)s`HVDD7f z|B=+>btav7oS8)M=T5@}_UD_eHwOxtM0z{tzK$cep02){!Wg|Zw+U>a0gJu*<7YZ; zH+Tg*Q}$}ik$`rAq^?dR33KuSg#fU^F~6NPF_$;Z;oIdi6bW*GHzz$~mamY=&H;(V zv|zS63`@9rNL8-BVpaWg`b zrS4UTr)qpV@0!Uo3kN=&bG+^-3J?*diNkwNCsb8mQ)fs{iUCjlAscn3hd)*>S^8dH zt6}fskAWBeqL{UyUvl4vvK6s;U|P6qOgtlB{Redr{fCDn?p8F5KGuFngs2*rCISCM zi=Agv5l@*z$;Cb={bt@)WD`~NmM!qSPrz&^hVqpVf&Jkx($3EI3nLLhK=F&A)zPRE zNm->xL@q1dBp3I%tE>K>`cu2FN|`d1H6t5#(`km*zdsHP3}_k~AN{#GT_4!|8ejGi z?Q4N$oc#{gS2hFbN?%~Yep%LIEN@C3?~iOdM({A+$L>$GqeEl)6En_(|)C_`ViK{PP*?H#~A#_ zSHX9!%25rsT87H<)%El8WNne8+vEHCWwaAD5mfUlZx~|#%&MFf$zbVQHF(rF+}#nyPi_1=GwZ9 z4`tYgBy7z?KNnzgUn&?6TG)_2TyNauE5Ttv8duYsuaL3)RvOqK?ZCR1QRAXq&Mui;WdX~;WiO_2BuXn4cO^gBkOpdSMY`%_N)es0gEmn=Oy{(C zTucLH4KwF8*){VYq8>sElh>pM|=trvO!qEvIqF~Sga}bWpYLK-!?yaMaBHTb-eO#_`+U- zJ2GE_7ZoGqz z0d4zlJySaNmvCJ72l`tmHmm#kUtWm*)c;q0k#ry3C9F;9wq{AbUXs!l5U_;LnT z!(m%a5+Uuih4kVsKkDuaJjV~*iDEA+w=pz7Crv3Vq|wh*XlZxR)!|DnQ}zJKzt~Xa z2Z*ZgI+pB-^CkTBJU$nTga9pZ|r_STy*Dl$xR@APICCePxYg0UK$sJXax)p4YS{Z<*WP65sS?PQ&2|ixW!K zw~7EH{(8y<=szb=mOf3?oax#2KyvfV?AWG;?O9`%4h6E(mWdIvhQ4nSq-9TkW~B8j zz=CuM-`;)KrfO`Mv7dc0N`rLS%If3P+f$Z5GdrN-rrnvpm(#-S_L>w#Mp6aba~i5j z&&h_l))d0+frgVG3|Jmm4nJXtbX+GIH047dS4EF{Jgy}*N0fnK;=r5FE;xwxH=jVe zNu4CeI>wV!3b_Pe$Sw(c(U^6!*LM4qz~B4axe(yYo~Fil7iE|lfN7%EC!^;&ol;dH zd)c)3^EM%k8OG?_r*2{2Evc_Q7zB~;r*oN=YRR_# z(Gd?U=99D>j);$)G#dPd+A>RcZ2K~f*O0kv1Vbe>t6 z&Y^(q1I@R8muQw{b>j_J(vl~vNHQn81@qy>)E2!?)i>N>)8l%rR?&;Kat67Y>iFn}X3 z8qLH;PqFRcM{A7fw(@y#kDMtkl%3a+V-AcZedtR53QW*HA!aTF$jE6rw2nWia=xO(N%<_VZ;rF5l;tFHb}xnIYva2?_Q9XKX{*Q3}~TBc*2R{w+tQCkN#K9hj@EDX76yR zxI+d_e7mbCvc(OU76Oc|e(Et`$_jmbhIi&|v}2%8UvDv6o%^+o)gpnJCwU-c#N07m zjFGJ>%_k6}k&)_+*9R0VzzU!xHws;J9^tEsG&4Q$yjA4&= z)|qmZ@#g*XljuOvau3wDO%kP9*oYp0TnrIoM@ll;8%?o9`J#nartC5NP`oG#FJP5V zAN*3lPFtj|Z~J%PmfO5wqGG< zV&rAf`Ao8ijcjENzBYh}W4UB*+-tV}3QD5Bn=!AB^!@|UB>45Jx3KSfPE8Z;eicop zb6oApDyJxDC?|8IQ+X!hU+be>7H-lWJDSiw*f|VWZC4u2!!iVZeIW<(@a4`1y@vhtNM_F<&`u?#X61-8W{lA!9cZ|zsO=c>fV{H#< zXlM-7+hyDycU%%S=jK+h!Fvied{#XUk3sh=+ZFe=zK@n4x6Xx`0Vxl`tUj6E4f5y@ zub=M>Iel9r-}x)JUoE*ZKdM3&&^QTh9WUaLSXd00uX@b(?(GRI7VKNv&e6zna}mstWmM{lN8ccN_0+bV?UrS{dRL!_6ykLm9;BFq)S`vg~)#M z_meeaKq)lXh&lx>;^%XxeaN4fli)JpILcO(0F8ll0Nv~2R97IE^Ilq?7p{K zZWC={u!GEV*3%RK^**YE>i&hR;yj~ri44HO@z7uJ(7G;wF zxIP+%FrjfNG1Dvdo^xyo0KoT@b7j?~kTMdIedNr(k;8oFHbg+A6hhMof#!(&pnUm@ zB#TNYTj5I%6YyW_5=SOBjV9BMG}Djj^dP9Rew`=Vbg^2n;lLY=m52I`5f;!fBgaVd z?M(aKHaln51n`9E+PeT~av6|80;u=m59rZ>wP47GTK};JvxLTe!OZeJXp7GBF@C*l zaY1>V>6SD48s)Ubo}WhH!lK)TcJG1qcpkHd_lGJi6=t#G*im%$XqYMj^dLl47 zKOr#YXYzfZUOT2wEDk2GQB|}U5CErG3_}`mW+o{Hpz0&hmvBslNYWMok zKG#3r>ls}>9d9u1baS6I;{P&i+pL}n|BznCti}`7Bn!UrBf7EUM5i+{QwPQLBoo

jis$0avX_KC%q z_$BhBrC-JkIh zMlujLf8}P)SnOYBT6ESDkA{I;Tr#_%G9oXEAs;e&mN+~YWhaR9Tq)kvl8+O(Bb?-;|t@OOVzhQ$0!#k^0u@h9PY zyGa0D@@wNE`k|wew1_|ccVPhl!tpf1E?S>Fo|c8L90P&09ztxOKaTR~_@mrNVL!-- zQ;T|Z8XAxy$QmT456WlFB}_uusP>cawK85S7{UQrN8e;t5iL;19G2;%8)j^;92Yyn zpZ!7VfTI0BSziyleMv!l!%el>Fu+&;$00dV0AJHQXfr&^D`eGqxi-0L0Z5Z9*`+{8 zEt^?D5w-QwMa|601ZE3hQm!bMTx}5V{_*g9zka-YOq;6D?jK<#(Ne#C^>zJz=3uH{ z=uGD9C;1mh6O#S$>**e*911DaJ3X2z4&2O+fT=^ph)I&c#54}i^5Ku}M)z8}4wc6z zAT#D35vMkaL=o2^G!?-HEXzgrNZ|`a`3z=u_(fMy;X>mMKRz8)sWE=2^Y4xYHlXAi z+1HJf?OVuw{Myf-izV1aBg4mz|9CjaI3vO zU$78{N=VmB?J=9|r-c`D0jXC~pCOgvvI`RF1Qj{C81eOttE^R<)?t_Rqy$G3oWz(s z-cQIH>HY56{U&tz$uArc9+l0G8JiM@u=l$u;K3O~?(i?VifM-;tdb=`Z}|FAmO$UcTH* zrcqvP0pxi~ROC#*NDvBC9DebwO%J-@V^V-Sfo+~wJ&&|&07=?br7;nS8U`b$`!#T# z@^Fi?@T1DL(1g}|1~cE%2#G4{mic>=utr@jgrHjwxn&N&KGxuG2U!+KFcNBPQ=OIx zJ7Y!&8G;hpa}d5jgI9ow^wEU-URagZzNqQ=8WcAA_B62iW^YLH8&Tx?gmR2O<{JNs zTJXuXzM$Q6p`f%EV3Ct>=nE@ zN2qD>w))^Npz@yzC?|}~$j_Q+K)QG^4%!-D3dhd;@XEATr5NQ$F1a?LEZpB^hppRi zD+Usfe-m#@(G|*@E^me5eX<5s;Rov#kqo5Fj~_rXmAiNss`ZEN)ap}t>r#X> zWWf;d@K7fF@>E#(%>NGKTre6oI@70Pv<+XfEbFf~V{3k|R8jgbSp+gmh^bes6O2h- z@p!`%J8F{4{WkpeP3=~TJ&Lh+U;J^GBEZQ)B}^td&8s~?-}>^*M~S(@>p&w1{`xhe zg?o97h@L8)OE^cvBUL2Zn_^ci)}|e8h3n@?z+3cIj;>D9Tm5eMtHmi$xX2bK=^|-R zZK+;3KjmP#5RQ`izFFHJj)-a|9nar3;Wh;Pz!%bdM8LN<%wiEcC6l6~_eb}!xc8HlrDYZx<6Lniq}A;*>6+OtdRE=X`9rL0c%dbwZRRv2$WAvzC{UpMAYwVRO~lQ zB)z|aR4dmb9`%;8s@;;ez2{zKD?@o`6|AlXU1& zr6J@ni`wW+S&`?L2|h2nKH8qdmuOL9jxL*J=&ow8v`u=eD#Xzlc5QIpkoKt^^_4S< zu)1T0rdsZ#le;cf&$}cf2JaQSZ{%`xtHw-js-ik*>#{ zS6%nq$|`2_j5Iw=X(jegm2CYT`V)mvU&q?Z7&ypUkS8(q4&L^|B zau^<#7(9(=r6B$S*bB!TEzMuYsGW7JKlt??W&+LxGcLEnuQ)z9`>8fl99Jw~4)V5P zQfvM=e);m{fzd<3_mRltZ}ilGvC47w9F3bxj{xL;OOv{y3vs*s{ozKM&?K8fzLy&I zGnI!86Y-7+0AtDLZoh{lVPY=p4!Yd({(L;s*)$I^9F*z2lA)PHp7QvCH;wulth$W> z8s+CMzKk{C)iaJ08|jN*_LrOlYgCRV z$JH{U9rL#C%rcalIfCMkUl3+)huYV1)S@hI$=<;_tOfzBxPe zi-RHg@L))mLR$oYAYSvG9%i|jfLMuMA08Q5I7U+k!$jO9w>_>u(J(iv9G zZj-Gi*kL>{#D_;oo}f*^f{9S%iB?9L!Wv>QKb+Dw-6cV139C|}`^A=DQ@7?74}3C< z=s9DDv=NG4e&1HB=Xa0LcB5R6GACt_mOpHp8XX|^ee!n`ytb|($FTljW6jxOZSs9l ztj@x8h;kk;#H3#X!CY2k4YS$x&*bu@!qqtL(%KN7Mj;r9^aXcp+-=ym6nFWnhM_z* z-{2r~^Kb)BlY<10z>Q~V$`~kPTiMkF6qme36anzq`)#Vl?Eq)s@^+_YuXU6#O2ADj zt&r3m>#u3N{xSA$o52Hv5>rVB3utC$A$p3cbM)$EK5af^Px&AY+m$ zdX}9RnL&+Pc`79jf1znTprK!EQ&nHOS4a@OZ;$lRBEkEb1c24|gn_nwCR(}lgQFQY z<4br=E+vQC;DL&;eewH=b3PYG8AYdW*S>{c+zx57xL8CCPy~~~sr`Ly6*7Hx{p0-3 zRyqy$s4gv^mP8W-Jzix7BX=3x3;a)257E9C?F+Wwut@v0xE8-W(}7g(WX^+P<#+Qd zsC9L;s>u52e)!`A-f6B6mhGzNg?D(JQv1!rL!DsX+1bUPFb{v|?H&RbvhoLm}@ zuBw6GUW*i8+*62NqzkR0(EMsp|J%ES@%BIrn6U9%Yx%xwttT@0`1<XduGUa zycY~PuUp@QlzBWvmu4k-N)ifwOub6MoNwMWJQb(=0i%kI@d_E znW~tl8@|+!9N7LrFD4F6O#wXR(#dxqzQ_xoMYSuYNZIz#4`yiqMK`WsI&AQ+v&|tU zLBZ6Ci3wEmv)cejW#v6SQi?dsKC$Y}qN7)%e**uu$F^$Fg7i;*fo|*1tRt}Bi8nRw zStop;;^BHIBThz{;^d0CnyESOT4dDkA zr3|Q0P=n{s_l+OcQn8S1@GJ5&0rhHF#iF;Ni+Md8+=s%H>tP(yp_sIwpW_ssw2~J% z_Pv&NUVTUwk4)@@a=p55?!X#2jfljkRtBICkDE{5Xa1H5dnsqIVQ}mPUsyP9ffXro zlovuLEqfiZpb0{s=;e;JO!ETwPXugPGnLCF=#|Vs8vJGzQH(SdKwQrGMNA zQ31c-@01-R_^MQBqr?sGh7A%R_QKo3bPG0B9$UG;DJ__192)@Y991f=s4JI49Y>9s zVb`Ch5ul5RN7R*DQp6qVN>6g>GQJA^)I+-0)etQGpR-HQ5$V5E$KQPJ@m9*A9KtJ$ z%?oULIHW?n_?@~T;K@bVVe_vi4k1W%h1TrWXp4Dx;*w!U40xsWHh}%t*&?x#K|M=a z>vY+`k0eZ}9&*<|I^s6c{QLAVU1?pnhU_6JfP)IL&;9^$Rr;ILJ%3H7_Udxf@1f-W z7;;lOZ>%$lvwh!|Ipx`@E@H3iC|)c7cWacNe`!E7RiBuxsebdo#7nh%Q%S&y$@-{j648^j8BU zsl9f8*euOLoS0w3GHzPtjm;ZVJ2U{*rL;m!*?dHRQaR8qj&q}?s;&_#O{MC-W(x99 zOJ?+w<)sl}H*q`PXE@BTL(P_809dHTLlOX^P6K&?*GK$*E0J%9pH+Tp%rJ1y!{txb zfrD<{Mtxa5*U+23bcd~Y-KCg-?I#<>$pY{U;FqmDS>4+v<6*%n6JbWzlVSUndAJMC z+Q!!njte4UvFXlRpjNmKGT{=m3Djd{JdVc=ItVa=meS$w(RS|E`WdU(WnrH}`4v%$ zF#wHssgx{|9vnp48rTvUL9e>v5aCD-Qo{v=Pfz1>7WKyIpT6T~Mb#vh*}X4M@UdYi z4(-{xB>@cUtM8!s;!@x6yAu06-c!*gN1-*m+8uYEl(}`Iqbe&qaPYucUTk&?&O#gG zO*E7>se5QyQ=ETGOdYuHJdY5>;^tEA#U9G?{#}(qZR9y;NtM+%tz&ZNDWj-NuhYl) zb-HXjAQm&T;gRoB#MF>T-xID0DYTa7YtI3w7F3^YrRdCAtLPZ*pEn2x97iZsPqAqd zGv=ZpZQF0{ewUYrO-uCC%mD7b@`(^tQBUbTaumEQyDcnt$|8F-)-y+R&1BgZFjsKR z5TDOZxZo!h3gI-dfCN$^Uj&Knul zj;@~#aTL{#iY=T!-tKztB$yy#pQV$bY(~F1JokBN4JrN5dGEh9TiaaLbD+DNYc&0( zYdU5U#q0f(SNr~yT%2<~7*0mIhCEH9odqCk);uTpiNZpjKudL^kxFWWWW87>zfZA~ zlm{YX35o`ameJ>q87`1CUFex+WGwmBsNSHHR?msb*+vnrS(e2Ab?!v+TQukfaUMFAXR@0ITeqn8~<2(O3 z?1hH=?Ojko>8eT3!;4}l4bdp_Y?n(5;gVa8Q~BNo;7W8>)-zA3&VfhLBmr-V}*3L2Z~;bO#2;*wv^`<8THgdjTJ7L zyB7b@Hgr>f#p*FSS&kWB!9Ph#;^F{sHE6vtt{^e7%42JdFrJ4uFBODu+cLQe4R?8* zP1^c%i8)E(T4C!&#(EVz0_Fe4pHe5B_nt9|txStz&py(m*?W_zGEhBxH#yC0jbd>% z_TXf(&Gl>f_%+V2w4*z2Ji$2BxVAP{Q>Y^y^ge5{hZbrL4N1iTxJvPI&bS%^P888~tQyY94H_J583Dbq)W^30y-HP&^^An6#{m~@Bm&Y? zT=YvE)VMsP!#|T+*>KUAwQ+F=w|+_V9SLVXz8$X{y!S8o2e65-ev8D9MrzkQIA(b2 zZ13Os7x_|RzL`@p*ePOHkFto*g*PJ8!dTm(JPayDa1}8}!<)?zzzSz=4NKOCA<=-V zB>v(D17k0s?L91=-`NQaAjto&{9}pPI1(#J^ganidZ5|cY2S!zZdDL#6T}`8gA?dY zs@2iqc{u+XJMgV!m{&)#0(;|QJum-j*|x)T4yJNxmMp0H_|Mi-&qIw$-l}=FX`{T} zbBuQnmGI^$HBw=QmRm8ktRV*}uBWhFfhgPCww$xxwCnp?xmDZPB)v@(O zOW^bQy=56yU0tiXhWQF7GtYC-wdh46Po%LtdGpuvBn({=-AzC4_>)WirAST=A3{RH zD;z+MHrutpQV{KR7UIG8wOGEM7x2K810@_I&|U94+1)vxl&mZTb|C!m-nSz*_zG$* zMa4@JL@`@l67bl&3@&Ji=5^!c?(-$L?b*7R!CUyU@0e#Ux1sRpW*fXE=r&qy!*1NB zVg4}5*hZK6jR0^%pbUKZY!oVuVueKSX6K`qLK8aA9&G;oXvr+1tRKukNy1+#NiTmM z_;V!DmKI>uc0t~({9uFV;`gO$wwrkeO%@*m_+NZufF)6T|Desw{Y?mX?}Y{U#y~9e z9?*o=HJzQa#h0I-Plg@nwA5m20(Mt>c%WV|(Fhyx;Itp|EHM7N;PDQp$ZCAchve~+ z(&WRZbDj#t_u3y~M`ngMQq*= zq~3<%dTOK$E0TvwmpFb#tCiC&Jo52cpDBs8)quypmQWUca`41enKwn`<$NaIbInH{ z(6u*K2`(hm&q6hF;`LpeFLl|P_t$JR{_e~XD5F$-nsGhe07CSgtit9almlOh$1BV~ z^eW|xBp6v&Pf@!#KgbaSe*bil0$l9b-;n!U?0jJaY}8{+%r80 zH>ZTeL>#J$mgXhH476-|8cO$IN)-7@2E4Y!_9etsz4HpYW(nGzb-{SkC>KXX z7c~r`P9l-n$y3yyQ-oQzH=YMBIOUxJkLq*)pO1`}CdDnhESXJr#&W_?v(5Wr<{Nr} z;`NXB#r_YtEfY3voTM1apHy_+%>j-t zKKGRTIyQ7K{0mq0o}|ifDXGtx=gk6SFQe8jP0e@-K&E}#Q7K#Kv zpedC3g^kjBUATzW>jA)l`NWP`1d}b0W>CW`HvXgocdfjPx$*_`*4pNppZbgsKaxH4 zg-t+AT$j&_S`kiv%seg2r78HGE(8!BB8 zAWi;SXUPhlC4#xSNZM3##&M@+Lr7jD4)Q>#8`z&5c?5Tgx6->Vj zzBsydl7N2AYKFI^Hv_;U=inBQvd=Xp{$$-htujE9OOF4s_pRNv;QkCfG~A>G;k5lY zRcEU)-{IfdcMC1oleE@ub1s25E+9-+G>2B``d&Z~VmKss2Y@&X%}mU5J?^*X&NCbIubRT=o}&<$2pQV% z4q>RKX-j0Rsvyr@qtnXPO{@GM-?Z<@=LLbQP8o4}D3&YQ0E z28tLuJSZv64}vBy+-7pVN@En7XP-QgCm*4$1~k*5ejE@bN+EYn5yWh@d71I{gLI4^ ze!WRRB)j-&^eQk*)TIiRSJz`G3o>Hxqy}F$eLlAGDj)Ndi_vrKI~Oe@Vdi$ujtTvw zhqP(RZu;--js95_N4(mSP|w~N9#?UYpT@5)ub!v;*;J@kQg>B_p2-z1L+QKhLMYIf zOEdbMkkgJ8_l-!(VDqLbXc@g0Rcc@Wc%&$9du`+)?OKM|0KaH;wqQK#OAJCw{lqh03nRL~LK|EIdQ3TrFs+I9mJmqIDD zP_&c^4#kQ~DFuqV26u{UfFK20oMMHd#ob+k7PsQ=?ivV$?8W;Zd^89Id+I}4rFM1~BSA=4!|J@Z4Sdjou49;<(hIUfH%6j zbjn7&&!DV~m7VaI<3?s>a zlMx(U^oh^{vu&ZF9p7*M*e%^cu3;NS?W;lIZR&EgIgmJ>^h0I?u?YiIISN@9+!ZMO z^Ij96GDw;}{YA~aq{ZHQ%usP!I5o>0XuCS1(J8_vo`Q%7M7k5Q-@pBNk+jh1kpM`c zh>6#XN{Olx%N503yfk%v`Uj?tE(K;4&+_A&?izGmgT=P3Fh7nF+gSNc9TwCia*HLa zT7hWrxYu(u3%nmoqko09I6W;R-W>|na+{n~#&8%3^oQ|rA_|@(viP3<6u?URV(E57 z-x>UY>#|J6qveEBKU1zrZPe~BOZp*Hw%7NQdT(poF-F!Mw3MkBTJ<&)6%wHx;Q;bEfFZ^Qk$Qpkw!WOHU&&#tgl1cF8tuE*ceUA^PXK42 zfN494*SF=8yG?4qa;nIlk_|5w>>V_Gw+6Am8v9H?r-*Wi!dP;JJqbu~-hwbcVqq$v zK_e!oseiFDBv?4iAnA9(gLVhkZ*<`la$oJ|iDbQ(sQf`#^~E&Y7~ecyV^-+5#9~$Y z`(39In-3LQ?{jtVu!bTNxy);ZHws_X>rl47;gJlgCng?Ydl+smn&2bZ^cG$+!{wQO z;haU>Jw#(uLds zv^VUch#!jnPRQ$~OwWk@6z4)|kebI8_Sw?WvZ`y^KZSHb5a-R zSlJ7dffrCw@v9nsISom_fwyJ{t_I-O1bTB-^-Y%gY%c%6SV>^jt=?|A{o0tO%br@; zKrA5S*2_H^%V(RNW5X!9S@`Cv@l#-B7nza|HLu-oOON6$&JT^o9XI0Y$i!j~C1;@q z&C|h$uc7h81=;OcRa(_e_qhN=Dd;ppC(mi-jXtZ$s|B4K;xGHf1q`3hvyN{DW7gw> zmMk6X{X08{h(t}Gf|QE&v4yBmT$fEnDhKvpc9`tp&>l#mq(}sfZZO#fDJ)Z%t*e;6 zE&uuRXqu`D-SWNI_PM2jnQbL0x?dP&h=XbO&TY%qP~E$Qj+1n(p}DsVc|)&NHL6i$ z>X9;fO+p)3Cy@oH8Mq+ zV<+J&nQ8uV*D(cDY`*`jX?rCP#ACyX*{0hMsLu_Rh7`JOAkaC7r{|ZR^kO>tOp@&i z1!j03il3GOTz%8)^1z85rho_F4ZmujtJE#TDI+7$l3h>)rG$0 z&@l9$krJgNEx#HLYJinS2GCnV|8dr{v2&HCBFrlElidb-ZK~91)6UF_eYN z-fUa{Sr0V;%WuF?_9O7YU$cj)-UK#Tb$uuKIxTwBpnLR6)A3yczq^5*M%!L`e1Ztr z2(^S{NmV^VRn1y{{xDrqr*b2muGR{_Mk)%_-qH?cB@jxZd@osRt{uz%cd^S-)D=%K`LX5WMX=lF&i4;uIEx<>HA`gdOMR{P z?KN0`PiBn3nS&|#wfTA&$J1&Ga(sT(`s%g#l~8fJu;V;)6{hY(&%t`xl-(ZsjqjaT zPkefW)3y=Q$JIaN19WO_Wk}ASsdQz~%Pu!B{_;E>N%=7Da}=7M>gj(2sfUY)?o4!_ zeVDJcEBb|eTN0J-Tbk51b5+}9i}H>i`6Zt?=!Rm@EUR&`+ZlphbX5IADoz&a^h$Y{ zPa8mdxamP)LdP~1ZLhj5{wMUERk&gxKWqBH)PV~&|Mn@2r-k0AoV}t-q>~X+hsU7)Te`BU-(M zGz-q6ofg?EAS}Gc-9?p5KPA+VYw7=5%m<4qYj#ntGv1{S!G8!O({)TPH^UnDr6^>C z%ox&SfP_$^C6v@v1xf{gw~`` zpLy2Hl>sWktaYBX0>8ZzDBTr|qO&HAN=nZC%x8>zH8S`Fr^4HzuQC*a5YG@D70QK8cHOVTjJUd>+)ZLeuTtolTZzW$6?%hgS zI}R2HqX9LL;b{yPnk>(J85)t-b@pv9>A>;ZiC_H?5uH5Su?<_vC)>R_I;wFrb0Kc% zkSbjtb_>_>}vNXX|SyhmJr>cNrvO%I-^$oLjQUuXx`BDR&Lz?OsTn$kzdCG=`pJjmY$gxVEO_Cg&Y! zd^hS8`L$_Xb;r5+>ay1_!{Y_~Iw(HT=Z0s}_l!bUL=w|YbI{WvYS*0?bw&O3Ustk>LP%xI+@k16AqSN)4~l#; zy3i*#EB^(zRc4F9vjyuLgXquxwL8Z&)`Q*O{*Na{mgFzZHdpu?t-LmVpE7PmjEQt+ z13q_s^GwKl@=$mdd4`+*RsbgV{$-u~vpABTw7e?QRotoK;E>H<+LKp4XI95--^yB~ zOL4a~BUWnWaeAK_E5&Jwl~Tj&<^3>l zKP0PXlSDP&j$AAt-?&ibL`Zn<@{i$%J`FUMU?xL3+4Ds8cU1kyl@tqDlDSJ1&bH%3 z#*y$xN)b40`c2YSf)mfH%g2)ut6IHHoOUpmAAf)AGv)rSmMKrevz(8avxUXimVby! z!s(r@TPFX#@~DN`P-+;l4K6qxd&c@<>0PP+B1AhB?-48LWma#EEm_0Tb=&?vp4t1f z7f@u~X|~jhJQNRP??YpFDsph|jn|h{^Bc(Y?L}B;`q*XC2dAVdKoO30smj7^ZF;75 zPtL`rB|JPA9_L#nbd&{*`850?%_m+rhPVk?%{KbbFnIfjfNm{Id>ESK^ZUmt#g(Qe z+vE<_?ETz^ z?+r$S+tM*j`-?Q1yG|w_>A)y;KC73>X zW1z$LY|)jE2QbC&-X;*YW`|8sv#R)6@U|5K7`dTK#-mumC#R=R0{E8qAu<$J>=xqWY(oGy3dOJx9wzY%EzpuSkzd=Uiq zao}Yi3|m5uo(Pl8Y#_mbdBj)THw)ib^-P~9B^ce@PPtcQJI(b%Tzxf5hhdU0IDa`Y zP8K{<@f>8CjAjO)Yl7)@CV!n9Gt_t}#+NlTfTBoSU&#EWih8c_(aAQ(Q!xKb<32^5 z!R~FEV1ku%**{KVd&3c(q#{=8Ox&D)?oNQXr0q+)%}toKwa>=4n8}Q3ZVywWXKyB7 z81zchv2d})O{{}1EE#?1!%pWCn!dIGyX2AD@=G7AhlHP>rsO>qymlGw5e)=eq$S=E zZo-h_4H#J3g$PZzy@h{yQE#<(jeiX7=aCfdmByZ2qaRj_1afvHcc+I~$CrBbW-XQ} zEPnDxdoZa_OUxBqWFnp%#YZwznaY4Z=D3a#T*qLSm_MOsKj;(Ay9OXOe`>5F_W8!k z;Eoznlx4$BPf0&f&gW zm;#;szl`ipEMhtqG0hoL22hgR-lr-6!Kb3T17;ovJ8?VYU=pQbmGUdp)eM@NZHc$^ z+iGN{G4>@O|GQ_?(xF~n`jv(Eq3yyR$bVBnJ<9a@=7&8jCuzIU0XCIlR!}1om?X^2 zX8o^{@h^+-B*xRHsuhV|AaR0k)|fH#3VgFXa-4?w5ez>7^?eC{vXMp%ZRas({<^ck zOGKu0Gr0xZRx`BfL?ZzT#IPhKKBmbOJXjT>-BG5yQE>Fi4E>O854L~$$`j&waubf8?WXYu3(*b6Lc4kIqlEG!e;*Gt+jPpXSCr0XeXvGX z@|v3-$;yEb>qPJ@9gCcS^6bVrX>iZs-J?7--<#tCc2NDLy`PNkdXRmqoCUV(T~8Qv zqBF{i$EtB{r)uIg@7Ni9PDnua{fGq1gU}m$0~{l4lvcgtn$EK znMDKgaoOC<@Mz&shotSX2s01bZYR05yXHb3wb{zxmQA>||FH59ZHSoEqY0ajmv7{= zC=Bk=5zm7kz+~M&-+^8CWKAs#c+q$RS&$MTB+&ea(!&cK_N$zIe z*PPv(WY=V=O!c2%AQX4`7*;-z%UZNG128X`p8H?lXOhAl_fApV1a?6dSPv0n?}`-# z!~Nn$n`e*m_uA(QoDwIj4h_nrpa%H09-ozy3VUU(adcjh-(1oR<63=^-)7Hdr1f6& z*bReu z(dMX;(;Se320Z_=+j(*SF`i&tUb`jkZ2)YJJOW+J`}xw=|4cy?t^#zG~zlt11=7|`&x#W5n23r z(XLC3;>V-k^z>b8xjOYHo9z1DaWwNt73bcXsdzr>Bq1zBKl_z3%odtT`}9$ zzSb>Ra|ses9$khz5GfLcYgNaw-K;T}yChl^Wll8$fG((B}x{6HVY z9@Xeor%ZXfb?h56(jRac;FP%Gc<3s1BNaY&6ublt1Mrocld&)5TFDj}vi{#&&I%P< zPY|9$LmH+%^2Ik^- zN(yVfzEQlQ!>KwH$;KAKVzbG@Plut$bvTwCU;W3;|J}KZoHSbER?W`Uk+C0%bY4@b z@KrUCagOogxQjJmiVZR-{tDj3rx7;%V5S@2@!;FhoHG`>4te7un25{B!L{ zk-m>>_Cl0mjk0#U0jvol`l~l-e}n9Jd_3;>pl83Vq}y)TTRvG}@-wt4pecn2Qj;fp z+{itBu9q4@g6D%5Sa*o~DC0G~3hZI$eZVvXQ^!Ni3&j#yaMOO7|_@ z%8DayX~Y0SN8zBk30=*keUMGYpn>~=_z=wrf?8*FO)##{!5=3Hq3@-H>LI$;>wS8= z@(ip@E`kgBec9?h3+=st$HGN?oiIVi2j?DedjU3WH&p9u`qnG9&2!FD_`hYa(Q(sS zD|Wu5O6YPJbp8&FCd)PUR>9fjR53bJM8IA22S*L-^KTGpU>r(4rfu6b^Dy_3m=i)xo>quHE31dhdmGVDbUkf2$6?&*iTwy7L`q4a1G0ZdP-DZQ0*{K!0 zY+U%9;IxM72}uOx1rQ{|8hNF{Y~ek0^23ao0RQ0)ThD46&f^K8#o0N`3mWSRSBKcI z!dBP*oeAWE-F$x3SnogLG%K4idQV}4M@?hr&`x6drgV@`mLSgVF1v)9C!Hh-dI7`ER#h>#}xBuoAIoF(6vp=bD*ya zx+bji=>F~)TI#N&HxY1Z3ui)~sh3I*=mk6Iit(Vuzw_^UD%-T7GS?O9suAS(ZS?(& zaM@}VJM$IH4ye61I-s+6}>u>7Q;%2#I-p;0&%7e9bsZn~6g8Jr~=E zO2<(OFcNq=icUkJHHYy*s2iRjw&OlB@G|#%uW`t4e#Nv`@ST%@p|UP-X|l5asPMvZ zZC4KffR7{x6Aj9C(4;BzDHzhQP`XEwvu0BxWsG~I53Ch_)ZBQ-pJ(srrbTVT5|`z0 zQ+ouS;Ev{DWJ_@nr-NL%Gc-&I7>9MmibU@R+3D$4uNv54=9F%UQoR11XOVcIQiS4$ zW3xFv;!oXD&Q;xl_;#gO{Z$?mad}_sXb+8gHx4K^_WiP?PBGX{p8i;Sv)VQc7Em>1 z$_e>FHjluf1Te!uDzoQ`b9HFKI%82IlugNh_*D!c@_wZpN}1x6pV0R8BW z1QWIRycxkM^K7#&g!&f$wrO!*%uusWbo8!>VT$pGT$Nq?{9E6y$($bJu(~dnX@e&U%2wK=$ zW3$k*?2d?bR>xlEhkI-;ny1pyUe)aJ<(??$7i>q&v(ymi7_n$h^9kLtq@J!O=HB1> zRlX_C%dcjuMLc>w+tAO!Ah=bRKF%Na((5wUfX|lD z@|j&VDHw5t&HYXCG}W_W#Ov%W;|WyojHnf}3IFxcg4s1BHeFC5Q_prLxg<2}m)d>H zA4x@3`%?B_M2U_9v`2es!z0`{rW8#13a zVfMY#FOOt^BoX5oe;rkH9-`ff=gi^A9ctIIC^*r z>z{T{*B;g?Vt!qtLe;C=yk&deQ?~14UXY;`cobc!{#eC$`uw2e#3F3(Ng(=!F*%K$ z6JnL-(E~$r!cq)C$^c!A9!Z@75X=z2dv@mq9eE`pfnEFi6OVAm7Oc$LX4<|WzVC9` z8#3GDrlp~EdQ#BB_HhFC`CA;<7ost9pFYi-KZ)-QatppE)~mPD-3#%#14!W(9`ZRA zqEek(hmOZU%L$}Frc-ff6C-ZsMloM$9;t2gJIoO1Rz41F!uMu0Qq-sraMLV!O(fC%O zj+HVa5^n;RFw4y>wZQO7FWF^VVeag_>1tL`;w$^-f(tyfU)&lb%R}4 z>u9#CkeOpkP!IfFu#(&6?M?FMb(46O<;{0=vlDNOMuR3YaH)|08G+Ey%l8D5-A4|m zhj9<^WPEAsWr`GtZ|^!4=DEG&1VOIGA>t_|>Bu5ZVE#cznB_m#y0O>8Rf&SBY%A!U z5A0kyw*_&ZZG+F+EeIzM(NCO570zRtbGKR_fBZHmX#d(AEedXuxR+V{%RK#Utmfsk zl4Xy}|H0=9KslLL-J~Sr0$8gMejYUQ)*)a&s(~FUu8|Df?UGdAvKnn-!1NI?XKJq6 zpJAlN<;;88;=w4tQSi-%i0aSKakKQM@$mUPwE4@TNi6*-qapLsi)G)a(v{^Vhuacv zj~l!_UOOx^IBgsKnF}7ijW$`l!&(bRNfktO^^c$s153)V3~j{%xihWRKJ%}pp%V!6 zmIF*ru>1_@pn~q=$I<0=R9HgpBiw$5694kSV2q2zX_OQDhp+PEh(yBtJfj}X*|XB{ zw>9**^7Fuv(5qK*9XfScQ$mS!T+AUa&(Geq)&u=HxwmOQhPUA{2@2FdRo(^&UrT;i z9QE6O{85X)X7mNgUk2x@T(HTtFxi=5E&y?FpLU3g$Y{p@$}J?lyCrmi0!)a0 zFjC{OWWaUai2#`y5`;iV1Hy5>J4TOv)-rR{uKB$-!^$>F(h0RlmbOp@=yV)+3m?^T z2$P>xS17(bzCQo*Pr^w)J6s$01sM1Pw=Glj?V6X%7mrC~Y#e*$B`r9sDB(xq;P$D$ zpawI|cFUZ&fNiarOCuS#@KNcx3S$(D0ilDhE|ZHtJ|1j6zW9--Vj04{@Zw=)bFURonQo5R}MRV>t@y_eZ(wO-=@t++Spn1mPX&iY6YMQ@~Uou9qV;HZ^ zrm{s(h*fUnY}@e(w>Cjm=w&zkrR`?>>y@=aU|;rgJUBi&qI`+idzY5d>krKon}rOE zw(5Vb#_@;)H1%KjO!s^@Y`}O+?H(3^@+pzt5<_YftOasY@`nI}znRZC059`7YXIw& zgdDrx+&}VYEaoRNEh@Og%30BtuX5v5A}C20e?6bAcL*XU(=_;VTJ~ZY-uZ;*a5p8` zxZC2CMagm`i8pCE>1GQ_lwKHYQOY9}6ES2^hNUoEvPUi+Nj4lilZlmBY_Crf1=&ii zc)hyn|HgiR?OWacH<%T~o?`rWdbu-cZv2uoP1*g7pb<~l?b-9k3gCC(tknuHxXyw~ zQYo*k?P(JMI!MFbCOmYg1f19;?6AM#W|c{)Cq=L!(V4>ZD{R?GZa!Y&kC|`#AfFkD z(fpY6q@)iiUpkXkXFE@Why8hEtM&Zok28J~5gE?=)u zHvb_u`E_m)C&~<~q64ke_A4$4My9+AyV4NYjTrHv4Z{|VWVIgQtufCK7NOWy%8aRl z={cz;|E(kRWi|PJzOjJp$g$p5ElX7FD7lT&)HU%kkel{*dZPHa)-4gTwmD)4=|>Gj zTT-VL&PArMVK{mzwt%tD!-tkF_gq;gW6zTw9Sk(%mdx# zuA=W~?^g1xcRD7W{tX0ub1fr2Sa7c1OY$zE6hbPy(uv%#@pxS;^s&YaK)y2*dRoDb zAic0@*B#TzXFWDMch4#^DwpiO5tIo1)fEUv4*LUk^k*11;hx(+>I!4XG;UBx&YI=^ z>c+e5s*EzFJvM`%;!iI9%86DK*?^SXB16`HZIpaJTWF2&6a%$Etx*1Y^yy9LTYc39 zw>K5f9S+Vet#^h-20A*@Ozh*Nm*;+K2``hAo^lZjJZ^p7w4C}bikAFz!MpVR=P*=> z#hIQbIn3l@wlYurz2MQNO+Pb5(>l+V&Cqm8S<6wtUzkX3VwPuJ&+Wp_m@qNOL9-ry z&*fEXw`QC4Y1(qQt^Z9wxNt0~Y(V%tB~g@L7<}37^)0PWv#6BcbF1ny>R4Fg(^{!l z%2)6Hk+LxoFthvUK-nVSr2^0O8DtZ(1*eISezh;uQGa)P{ehgt7g+1akI8KO_s~Pv!H=;b?@PYO8yVlq6{qsFZPLyQQM=d>r`rQ%H?KI3VsJ%Qn?p7D9 zK0E!GXZbwW1JQ;HmO6F11(Pw3xv|L2jh41vcNxFWFa-UEpbeel0FebaGetnONa z(|Z9T92F?R&JS?ts=*N8{ns_QjErxB~w$1o)G zzfb*d$NtOZ|8ojhBL44p|DVhMXH5UwvHwqZA_fsX`fp$5zuXwTP5l2Bi2wHK|0Qn! kDTR-v5{3c)59}m9;{X5v literal 0 HcmV?d00001 diff --git a/public/icons/icon-512.png b/public/icons/icon-512.png new file mode 100644 index 0000000000000000000000000000000000000000..644a5c8eae4afb4859ea929be2c517a19e834b39 GIT binary patch literal 40995 zcmdSAWmKDA@GcrcaCdi?0;NDH?p7RH+$ru-oZv-@LvaXh#kIH;X^XqNJ4J%$roaC^ zAJ5lw&;5|~X1&Q;duGqfv*($a9i^@+kA+T-4gdhK6cuDN0RSNUD-eK+41ZjE&pf~% zXf6tR?f?Md!M_helb@zN06+^+l#$f-UOF-I^4I<}N9~J7M^^+uV+2k4of9QshwvSu z(#xtJf@Cs-f})Wq3c}X>gDC;2P&|XuXbiltC@NdBX`6(0)@b5J#CNto_*Z(K~ z01EnVWn==tf1;5p1O7)e9)$m-4(70oGCfuW=iMjjaFi z+5Zdt<$vwi|8K)-lg|5IYPv`L0wXFS=rh{{FBgr}l#sNI6OD}yFDZTd*WiO(0@^Ov zwGtyXd+5$42797_6;jbxL(S9mEtHY(PtUD^(?#=<2YkRKV(cIRJyBZ0azg!|J3(M? zUOsyw1Qq{~yN3T4Z*+Q=YluMR>Ntv>z`ht{N$nQbgrHtw&uhnofPQIp9&6#0HD5v@ zE@3kaLT(|{A8+QQ2gCemKix@1o(dz;3&npMM#UqbN005yytMvf`X4^1WTCugu|-eP zSRIm_O7KbGKNMrlZdn!xUQIoy-9gKV^9UA59{keS-wfGN0=L+>)7;``RDe*fi zt*3E`IVeIn^}KPF_Rhlk7PBCxNBYA#{^C1dO_1~o(-enE#k%niL-!@f1s>2|Uw`|( z{)e(z;$iv3UPodqtex&W0)-Cz;LFOsj6864EAhpk>c?DPmY38K|DPge`6F(dDZZfY z7|efO**}UvqXsd1*GYGI#5%1_ki>VsY&*(^SV+@zl9x} zGG+HX70bHJp#NEr{{qeCRn7$g@>oO%8HEu|2Qf_ukdt5nElYKkP=c*>Xy-cQ7ESgW zvX>4$eks`&`lGZtv`)d?%G}C~&5DpZXe-OBH5C^mXXQqN@E_<;pxB8OVtXEaEZyVN z0ToXb%jdg{2@DTt)e#=}sivr2;6;U}sU*Br!PXQ~+6g*CTcv1|_oy-a$lFtnKJ^y0 z|M8Rn_xHN%f7aUs6@Lv)7T&U1!enKw^wIAo%z8n$b`0YrT+Am3Fq8yiUJrmDC7cPh z*AzO;Q?xaQ2VSDL3~pzh8YEA{lo?6!bPN$+%+Q{mt^YaROPpk$)x@Et0=)qLk@iqe zN(|dmqhte5fH)@oh0Gs>7179FhiHnd(J8^^(nwk~sk|Kg_a}cZ_!TVKAdd9nB>zs@ z!B4^kUnKKVZjvI#(gy!1qxd`66=nvY|K1r~ppB(TT|?T6MwZf?x2O97wTy}^vIrXK zI}1=M)$uqdK(}^EnmQ&4X?x%Z%oYDsrJXfS=EY)Z*g!b-WrfqoT9eyBzPL>9 zI=&;TpasbUp<6o(IDa~OF*ERTg(~5lT@;}*+}T7; z9?yi3r(BuyfStlZyQ>08@>*De4{7{+0#V;}Wz#E^kUNV!cZ|Bpl}M;~Cp&zU?U1o# z2^gulE&rL~M1LOY7~J*{P>jd(Cf3iRfax`*(yDXM(8ef?xq+b zzH6rN!;W3X+T#Y|t%{<^_XVginN0;2Uj=7RkLNrAzG;`d-x*5MM6hDo62?sZ%{IV1 zlIEtK&BobHV?g)j-|;|=LVnUXxc7D{>Miy4yi%;=WX7rpu}i_5cBOMgET7^QYS)0H zfJw9R!kl`Va?+WDBJyc_`o?n}=vlPAY~5i+D?hgtemVQ6MlncVA;9h0Zr%7x`>HR+g=j}lm7_2!j1I^Z%SJaZe)nRKyEexN~$83 zouhh4TFRL}upg3$GQ^_D$|hJ2>cc=$U>3Diz(wD4@WM*$ofcG~QI}|r##GM;$)t$A z##Fbk6#C8og*IbrKjmL4=Ta$HCh@x=5*VzL#htS@#X28=azsvu zQRVCL7q|v)d)E{OnHif1ewvriCJS9Ui`vgp9s;)XKBk(EWMn?p<>E@z*$ zDJhXuvRbyRGyW|yfv9BNB6YZO?OKVMbNfb~Gs$XFl$g{dR#9~D{cZbT=n~#=&kK^$ zE=m}S4`;aZJGvfI1+rMs$X@#7BTI;Ebz~8$%`Y3efW0zGg-=w^A4eZ)iJ9^P3>vrh zv;Hlhj{fAKKQcI-d!|LqzMQxg$oP5AkC#7t4lFdbPhlU;@XJhSSF1nAtKr*|4}*ed zJUO;|iHKcm#v2qg^Y18sgbTi_?UfZzr575f^?5shZp>Wi{tJ8bDjp*>ET^`&u*=G# z*Ie=F*QSVcY$K_hA~qY>UVvw0VnwQ3PR2rP-hDcwz>0hdIz+zTihYV7(XjF8iZnDc zttOU76bXpMf*tSO|6$@&z$E@#bvNs2VR2O!E={ki9b`3f1ighX6TK3M^t0`XDUm;; z?%W0PPbauO2)#DEbL(U((C#qxNeuJ@kEffXiSc5hNw zkR3l*IL{DnVqsAT^ulfs8Wul9mB_DKK;2$GS+vl1}XiyKF-=Yal*wW;;#;|1Bv z$zfL3*z7X!-&n)(4u$}hl$RK;JEkoPwC@wO{&N5{D;!LDAd_m zg~p+2W)W>^)g!TwE%(^vXByHp@38fWHrQJI{2rxDQR9{em?R!g3`L#`q_#dhLv}JJ zC!7a4Nn8G6r+4~rtoRf#pY1W z4rB7Wm0ZmHV)iukiO2M}_5qKFpyka0lA`RuT}F{a)(FARTPSvI;`y38+D3ZZNq8r^ zIB9}_k3ml&EQ-KHHef4qN_LKUYn(1=i%6&V3wQYteVAH=FoM$n8$(S58w!4Smjb(5 zSAQ{ap?P_rNrC+p{{HyH(F*%PVmkQ{%j2-eIea>cvBm2SQPCAtUDC90cbLM#@F3pes$@fXvR6BZLsBmS4lKO(tDHDy=R zzm&%#vO};U`5Mmag3f0NGh5=rE}rN+#6!(yq;dKypyEFzp1@~V*H#zt;$~Dy2_NL+ z`dX1N=t>Q6B2|^bH#X)bilS-^&=6E~8i%41=&39+EUz4F3r&$WWwLe*sRFPVgHnTv z7EnGA`>2`017U%>>(%mc#g&#$z~IF_ZtBuxl1lmvuzUs5G}ic= z=#{LpoUI_@Jl(}Shjdaup)??gKY?&Wo+(#a{gHM)A0wX@nAZ-`PB>Zy@dArFbUb33 zXPw9Hghki1*&T>wwhCCTWXKwu55$HaG2j8y-*p(MT%7zj!zzbBO+1%+zkMPjeyVn< zX~8pzK417;4^cVoZA|oF&n%J9x<3Y|&++Q}Hyiew>oAs2s7}kxRBiDc$OUz2w6u_7 zgJFqJ-BEH4rfh*~TGbuByw5=~erne#y!9G;nLBkp-T&!|{L0*oR_6<&lUSV1fL_s> z9EBr#+@Y-}REzz`s}h^l*%cQ4v+xLMU0alc+Jld+&sR0vbr75Ef zQ*lr?7$Yqiq5h!wBzDgWKh(~{%Ii-9!j8Ly7I!f@DDyY)^4VHJ<_nbiUo3|&o%J3( zeJs(F=FV)8##gL6Y9vaXB!j0db2IxaX2f0&-w9Wa?omR(zQmTwd1&v&o_3r`u8Ds2 z=&bkrty+dOs+rg zr+pL3>(q-HSWo}Hai~Uvm9&RlW|7`T#dm<-)zksxqh634GJ{r4EfO9?PEu4(t6YIt z_hUL7@HgwUz;Ghdrdj#X=Z8*__8Y}Mon1y*x9@lgxQ3412HoW}eD4(^T3rb?zd^-z zHk|l4E>>||jvufR;E8Ob4c-Yay@a`xUh>(zh~5wpRwEJ`^-?w7Gb zNq7FSgETV&GuTjWSO}FLt^1>s!a^HCLGwviZB=^@?CjC(<&mbmYw$03cW{vKSwyX4 zI~@?lg`HUc31w%~9!t;%-Kyh-cqRDe(Q{_^JnH!26sKH=wnkpfCY&Lhiw?N3JX!4# z$=F+|fce@zbqdOCkN~jOgAiE8&T(NB#dXE5B9p-duZFs+k&Ea|7po~jEsUmqkMibk zyUY@39d7k|AJH7vGmE1u-8_f(aC4Bj>8znaU!Jn$L$*D+b3z-h5V~gfs5aWqQD*`! z`Ca^9L*fcFh}Q_^}1wEgR|7Mg<24dfnwvKp~fd}!}M@R zOq-pvs~)G(fouz(enN!Lc-mL!af#?#42lt_?gv4?(c+R_0O~rk^<->l`!^?oOTIQ)6pT+&W^12452f6HG8}^v%?j*f|*Lu z87al;e$2^VxBWKh7~OaqEe}EoKlY>6biLe{DDT( zEBoQFtOV4$vZn46jPbZj(d(gZ$W8z5#@#mf8$MuY92PR%w42L;Vpt^7YCgoJFftly zHTq<~HW9s^AM$Sb!>5WW=2kbo%@_Q48joY`nHRk9boYhXF3|F`1ylRn z@=tQ0MsnnH&lhi2JeJW>v1aAiS#*pxk;& z;}hOt_#rg`oW+Uxu%lh94?FmTn_(H?$j99BmT>eVwJc|PCmC?5xUHnO<^$@T*AZ~l z>nyxUuzoKZAomQ+c^za7TZw541m7K6=TBbkW1@h)@BqLYkTO^+FT~95p4ld-d!76b zifsFB@86m(&tC!lpd73kN`7+v<_fbyTB}Z0-HHs!VhirCRSJMlh!b)892Ea56HyCS z)I)ID66U2RwV4o}=ROEOO|rioV=kX)#SSf=`Cbqq{#i!rs({1lo4odu>tFt&1h3@K9ey=or)P_@c_{dH`?XiQs#5hCp|d(H zm?gK&b?G5^9flSVmlIa|cXZ-nv0>wG2xe9*iuktkq2zM~#k%lA;*I9&>Squ@k}1l8 zl4Qo6MlRb2n|y<>HymuT-vp%voxeXXjPK1PUm~x0+!H4CohC$eqS7yP0N5fEmRY6^ zkM|v|k>p=|7{V*Vs5NL}#9j2^JNHut+%fZHM+b)6Q4AJHYkGAW3AFyKE+h&ScXZke zM>KRhs!2-+YQ&_?wzlU1!CypCXmQofo5Y@**)N{&)D(VZw&u=57GEA}I_-t-+Oq(- z9l74f%z1v&^EV2Xdo8fxsGmb7(tlKzi@QP|H4uBQ1LXhgw8v3@@%H%Wur%X;y39gL z2*(`AiscuJFg!LYM|ogmnRR@-R_cVcX?Lia1jQO+20S9FX??Vt@XDSJk6(SbO4hrZ z8vZ5~oot92H8SJfh!bn;-v$j)Wi8Q4))FuN*-kA8eD&6RdQC_CSBQ~2;r^sFSJ}x% zcYs=KNH+^AAb++^MKjanIJ4~b{7V~Bbj{5uH*S#lxEw;k2Yy=9;|pJ8m5weshENrM z1BG420WpG<(NPMSBAxwUOp|B@Bu1$jrU8$A1N%4IFZx|Ad#dW`n|fPuZ&P2me4>WQDmhhgsb^f>E$xgr|hguC+x51Z&flt zv>I{B=ML}jadj3x!Dbxe6_@j4?#|HEr;5yI!SWH zCI+;~pS(yn7iD;XZhbq?iEu{3E2pmgZ3gYT4Lh$N$4^4TPnDw_c#it1sxLlTG)U{@ zc6nOA*W2xJ4x&A+8+t!cFFiI(Yh?S@bk`VhfdA?l#e%gFoy*@yEJXoud(mbUT_Uo5 z$&btsA#3v`kITd+1~|RycDN_}=8SiR9u14QuH@x^bzCeDDEoQ2nKq#!9~FwKV?asAr^I6u zt4Pt@!%hg~efp7~w6*i8MeKxb zeBMhK;G_XkC)J-vgCD>A`7B~0(q@`52(<|xZCG|P2Q}*ur0xDkCI^v!6GA_NpM-*X zp7&2W9)(9S&l%0#jqVFt#?M`YfxoT1``GpT6)wZ32tOH&qJW*tsL}vt9A{yEfe*7T z7myi$-rOyUuBV92pwzf6@ix@~=Ej#(IeGemyzrkiHl^T_=v?qG!)rfz zX}oqv<=2a%6Jt7@ve_}E#F?YjQBi6+*&>lYz_FGU7JR14e%n;2J|2ZLIZk}){L-`H zqAcdWIcF+2N4j630&avwUijvY*XKXKeLB#x@qJRfI=>NR$1F&g01>=k;&i?cGwu^f zBs1@IfZ()BBe#i$cfJ_}Dd3iHBi0_aOsJMKAUUx6SHKoQ&FkSQm#bd*9yaQ(acJbg zYC44mFPmy+>p6Wye07DqHVSdtcS?vWl`X7qxPB}ajNWq;P{==VylpnhJeOwo?j;>I zqJ)}cQVVbzZaW(b>=8a z$QD*<4px~LHq9i#J&XH*V}M_4Icb2KGoHjLq;d&|7dObvov|NOBJ zlcj|V&(dM{)A2d&)265TV`0t6e81P(mv#@n??%DNsIxObZcm^@&SUy0cV!brP5 zwO&$Y{q)m3+&JqQH+g>OmXVgZw6vn#iaic3$(HoI3BEA`(k_}&gq*2D&&iJ2(aNWO z$=<81T9nySzx*$t7(ow+DAM_H0M6-kaUXR5iK^<7URKi~<6pNr$IosF5cOA&Zj68? zSI&g5fgYa_3|+>#8`;l1cfwxd2g!W5fFYD9NDyGgS+;YGn2#JXLSDV z9od6vPLM~t<_D-o~u(H6@*3jf?stBuYv*z=BZbJ0mI=QJA#uI|jrAJ1~?dXl|_$IJu zDGU?;S<t<3aG+pP1wpv|~6))Y0)(4TaB{0At_^8a;im z;OXoIRDVsEP|hor&1CBex}qKd1(^{oK_K#9n6{?h@L&I2av71*L5o%SMzpM`zdiB7QgKo5cV46;jB3!9GrH48uhv|qFp+I#s$Hm);DcFs!&0P!l& zCl%WUg(Z_$&q>mT%=(TB9lFn{0M!}Ue?U+bgQAiQ-B9w?aZ|VP)m85_X%WI2^IOU) zkOi;xM^hcHBpy%t;&QE#2>f0SUtN@0gA>P#i9v-p$t_oV)V`j#<`y zhqhl{pwLJv`=$7uYdd?;O}`k=~K+bHD%7?=v;5!&p=jbI(@JKr(QbNi)VpF z?>U^&PL`S+tS#u6UpUKJ@l}(Gw(~3$C|}4>Axr<5xj(N+>SvMyD5&M1Uk>q~k8ZH2 zO6Neu+{c9`TG?=;hM(far=N}VTsLSzUz+)ghcn(<6jsTE*sRoeY>FxH+C1jC?P5h& z1TVf7`57Awk7XySoBl+>;!N>aGFc0)obqXjG<({*7;6y(1qP$W7kK+sc|EPIYFTPp z_&l7##P2;cLY1ZGo5ybtvmqZODc3Wj%KHQnBw)+wt-?hM0RnB~;G^_3aYeZo=Vs&2 zT!fY?#0I`1A{Pt+s-ri!z;vH;J5*FYr!XQ zinhl=(IKFjzzW#of~ddMU&?o+Bj`@>c_yodZO`J38oU>Xo3q+Knp9~1YX-OynfZ_* zEIKoU^`keWctU=t{BK{TcyaJ3`9#^=kIS&-{RPEQe)U%?!Tguq3%^L4sgnt8K(1fbb2EptY_< z&pSV|oNAsk+Q{v&jO}xLv3}bH%YD32aN|N(Yn+My6}HyY#xI-K)iVSMug_F3;2nt0 ztbhFf0!NJ)S=Acdxsva_NwNwZlzz7!POn5VAL7s6_4-enSYaby$$MmVCr~;Pj)f+R z)EI=I>A&n9@H~@+Q{Q?TgJI4p0CMqr&XIs~=}|wT_k9x8)M$iXQJeeE!*lm!%0({M zP;he-Y+BhSNJhWQjPpi$4(u2n&TXi5N-8~~&fbo8SfR6fvnK}aY{oE7_4^6J%V&xS z2&$W-!UN>@!z=bnr!)o=YPW!CFw1AE?WMTwv8`)sflQcp1|MlTRk#Xxp%Z+(Z*nO7 z!%QWYaMP~+mqTga%jd+n!Fp6>VL#rx~BkzkKd zD`zPrr{#7U{bm8N7vkvg?=IQcc}QL=NE65GWRCB$zO-ZnI%i_5`>wWDmRyL9!Lbv$ zs6O>=K#t%YL$a6bmAXL5(NFY{$0Aj;w4xUc++uhGIDg(-qMc9+T4HR*bW#cL>QsNw<+?q*0}(AQ0j@EV#>ei|9%3SHU#_WDAv zpH=(}Wa-5Rv^)>L@_tCVslL&I*5o81zp4#2si_t|1m0~bPOk>{d!3uLVw_se&Pk01 zOP-NGGrS6Y!yU5S#lMlxLq^F&)iWtTNqKV=+`k$0aC6-MbbpoK(P4=9CyiTv(SZ99 zO67SR!tni-V6;-t>oZ@M?}4b4YNF?Ef2ZWWnDker@V|5YZBg{g3#O4?nfR)_CxE*P zlkRkh;)Ux=;bf{yiqkdkypN;lc2&}R1P{i9WRBBiNTV|%^{eq)@(CVXF{9t?2QhTxQpS?MLu>(J5&)R^98lv{{9Tl@km4BP z=XGVu>Ao>rbw1}d^U|tfd2`x_sC?~4t zE^Y%ar4#klQ^q3BYg^Dd|9v*Uw-|AP6YWdtaZpfx*k>kb!q7U7-f zYiMzm$U%kW@xGAHBnYjfYik<^(>RpN(9A#Brcv}NUp~3-2Ye|?3La7b9O3*l(}Hll zRbx_g206EwDI9Ry=;FZX;6<%_}Ix_L%Tm zOrXIT07nzls*6nxUg$4Uw}#`rOD!R#COHjLRZ?x{Ghy1Rr0-`B>n!=Gsc3J}R1r{$ zG+4FDVKd68x-EJQw%>uF)L?Y-uIo1W5ZR4yIhlo#MUUrx2Vp;;-k20u5tu0Q$;`8% z&ZfA6=rL7H0Fu6%cY@VfsBv23WYek(ZSUPd=VRxAAQ~@wASx)v_XyWBDTT^J2!n1D zFQR)H-Z5b1+szp7NZhT}WEW#)fA4GJyiVL#*OPr&X;^SC$sTIn)bm(O&9Zg{_aFAb zXQ=`rYY*GNwbX4%r7Xl8xiYT2!UTa4`GNZ3BF&tuUahbiyhYFNzp7!TWnaplf=Op? z90oBa!^gAi=A6MNc#O^~6+cR_{jh_!2j;K4&lX#HUV~o?)^-`b8lJ1hiW5=pt!cxd z6nuYfGbyFxQXRt|APRyc%CSklErV_BA#DgkVWzMLwyElTIvB9U*G z9e(!`pSkIz&=Z)y&<;|Quy;6F#fSIp;10HFx%C5`tzm_N#!ZpsB|KJWhuHh_vB=^|b-*`s6U|7`kJh?x zHPe!j{)hI0w~Sl&9yx4c3tAzGTi0x8BJ?eDQ0x(ZP~^E|s1{DJ_~!YpM!l{(*9g>l z)BVmvUlY`Pd2KPMZ8pV7>JAm<**ienKhN(zK1s2&L>ilrJclP{9ADl^IqKUP~HT3!Ww0Okf#lA_{-T=+@X^==vY({Av&K7E%)Y%df z?6JxqXCAuW&3!z6Dd2&H=|JY-a6&Rr>sDGt5*cth^Jh&*5X^DF|(wh5U!wmBwFW8>wXt2DH}?fnO|EMY)KuP5C6SQWXPylFwI&<_G6TTkVH=0WGd^lJ!d|*pngaPA zi6Ww;lRk}q)2D7K!?A*uBH$fhYcv{<`qu(X(5y^^7mDWIRV&~`F~H77B(T7@C*bxS ztByTuQTJlUVrMY;*_RdkgtqTDyYo8D<@!TW4TBLXEC0FgU?9_@WCGI>=p z>z6!pqwh+L_Za=8KV;M8yvga)Z_M@W(NNE~W5Io=uV0eD3EdKS;Wd57tzWB_ZQk8B z^2!Zjxk8t;eS?Lr!^efE!c>HlTPXOV#o1c&{TEKU-Cg-P4XBAmA7vGXs-06UO7pk> z_FzTO-Cgn$LfUVTAN0rT(fCsL6Dw}m?%R3e^Y`2w)}*Y{!OSXY%hvw0VNw6O#GBH= zwIqN)c1HnknfgYuEQbr_{fdS_`DdtPs6X5u%3N-lUA1UcXg$aVH3A*eM6}hdK5Oa4`a5xig2)nC2cgu!^WFYLNg0|HU zCk1>{)zuyFH^mPKBTGSL)++9P+0V6SmZbWvkITv+ao%LDHv4$Kim@yH{$xBq7Dqt&PuwtKUC z*8|%4i}KDgiaMw-BdbWOQ8+~W$4k`O8^)>zprS2THi-6$wwREHANyBPn8xAA%z}vo zMp=XB+iUR2uRk1j$$Npa54lq(Su4q~d>bX+hE9`Mu< z*vWx%RgU{De9eex0Q9z^pdvNGvwUaZ$Qd0aYJ^!O&lw zTy~GN7qn3?YoD?~)P7FoQAs0c`|yX_3G+hfV@Lj?=)i4map(a5@R^0t$Hz2P7*%zU zY2!$zgDQ;hBsZ{x{o-sXw)Dl>LmBxgyy-8)+57wM`{om{%CqgGxA@CGJD8Esw+mJRflEkUX1DLIPQG zCvw{dnHy3U9G#FFzk*RK!S_@j&A-5YH`$x*{!Ufh5M2E|-Zd2%Ki(CT8ed)cAWUNV zs=APOVrr(>1jf>^*0d8i^}O(-(k%x^{hLf7&Sso5xb~!>rl*i(Rosl&bdR%N58yUV z@OiJ$zlQAIQB+O@p=znnWt_U-qT5Oo@h=iqyk-faY>1#S4?^q6(aR{&@k_M_3_m)Q zwKeX(MTM}si431eBbtls)GvGq$P^l0vreo=xTgxsp#{a7TxY+?i(dh~3z0jabu1z= zh4Io;nezPeBz})Z8&qr99U9#0?Y3GM)S@3B3`XbIy+9prcyqwk$bo?y0XOOwUE9?A zupzL02yfqP3Lp&_(-sgHuiv{gx4=ZOq-yuA1ZFVpJL6`Cb{ z1&H|k&V@Hr8a|7z{Mg&9*pt8)YqP0pSobA45(ke>dCuK$U>L<#;B-Ut&5Qu}=WNCv z6D~?(=&{~5VQO=q;{9T+;x!)$&B%h7F;|+zKVjx^Gg4&;;uV*A1^0q_nr*fS!c+t) zel}H8Z?956rMAHkmrL1}lF7?OOK!8+XNWH}CWt4rpQS^yI;i~k-fCBLMkKxdDoX*r zKqk7%Qxi|yTgE^~&z8LJaeb}{fa#Ym#_Itu(xoos+P5AMg5BOgw&Nx5kJHh-H<{R0 zok)NicX#%r3J*AsINB;4{<^CiL8@!P%byLpK6Z{)3?L$te?(Cx6*R~-VxZ&FKG$+Y z&i3!l7Bp;^x_SoI;n2D_mSNV8#)>KpXo3?bxFuwZzQXmmRj+^0hFgSxYEn+u@^>NQ z2*LER?`A|{kqT}7lgfZlLHRUT3jOX(6qBR--nhLEJ!ku<>!s^cR$yO=@9Fzh78rgY=nW9hgXX;d)l+HjTf@< zj+w*MjavH@u?PDJgw&18bnmQlh#0B-)n*joRP)_8I2v3_ zrQV}p6l#FDgO$6?(1G7*hP$0rXS+-Jp5M$6@6RW#(#pLv#qje2t)?7QiQ$sdG0$g$ zUufhff_^|3p^bK$3f0+mXy|sS_$w28iV>iV(1)!sSvOl2IIc}9(8fNN{|StF@9v0! zVgsdS+`-7YnySe?o$F*L%fSiXvJrd*;j11o1)9F+bEM`x*_nej+(L#FAoR_b9wO;|M$ig}UpXM_GVzq|&p(UO;<+q#&+G*+9Xm_*mzvZ& zG&tYwG}LbN$NY2;e3s{blSff|SgD#Lw9zx>hcXxS!Q^TuBeVF~wXNrJl=}g-=aE(B z`V3`^6Df{tsR;#AFm5gzE=B(d$ex1O;#irY+;zBetiGSx!f9f^nzkNv$=y|OznT-c z{=Bh0y)w}-smk+7n*XKj>sZCOczN{iqa)VaLR+O7E~Gq1A2M3!-~CSO4Q@#G%rcoL zZ00-&wVejcA4oH753re>#LKaM6aw*RSjwg6~%X{R_KPgb)`@K@C$b8O>fiwzE+~ zIdjB9)<^ZWyV|VHi#3j2~i?)I-Hp>>K$$ygWVQgdQ&VJ_zh`Yh>9b z-B7Ll2C98gS|XbG#za@uS$1E%?)n_p^#MGB?qk8=q_CF>OM8hUyO0?K$Kak+x1okpw}E4O>BoLo84J5-kF;+jItU`+7Nn(oTTqv?b^oZ6 z6*oKu>z9)ZThd)c;%%zp6g!=L_sFd5ciIsICmkOb?{mepevIAI_+w(JisQtcb*|7n zHT?#VBox0-1l;8><*E~ej3kb|pExEq>0PQq-)3roCfnhACQ$VM9q`_ZaJkw1kC;7Y1I;zZg+_y8WP@KnOX}Y=z9n zTs&&!XO7YRWa{9lB`qX;$h+-PX*Kc_W ztBVY|=&@`w{^*5B#g?x!Zl}mR2R`cJfo!RU1P1a`YQEXbAL@C1$gFrbv>DolE>GQE zga*K?K3uqFGvK%}r4-{_h559sBHzS|ImO(}$#o*(mrXrzL9>*92MzuP`~u33`^OH7 zXkrl_1<6>Jb?IPW)?cPS6Fk5Af$eLKLlDcW^huG*7a2Z^lZ(WA^!->TM%F__{cUpb z1gN|_4B-vYAL4>(S-0^hb6Yb9%ameRxtKNcb6yI%%D+~+=g7;DSu)XVc-6{=fzHT>G8@4~rHF`STdW|+{ac!QJuw{aA!zobD<+C{C>ZO0{20l19gHvlYN`7%Z zw<=&Cw^)`l+P0suPg^F-Aw7<+=N0oCP&8OBhTgGQt7-iPhRZgs!+t=ga3;EF@u-_C zF`+Ozm<>M$@f$};N>|(mtNwalls<^tLXY_uY{J0|E4oty7IFoba#OF~roGUVb3Ch+ zp6#_`>vsJZ(uz$xvFETlRFv5-XlibE>evtM#y{(TPB8-*Q5${Q|7MOuO&hUrYQCM; zY7dgeW}g*V$;R?Zczlej;|1V*Dh2UPCZDQOK9+v7!kZ`VoLfc8TV|VJM!1 zYgW&jmuX+>_;7hl z6PhIRI#6snT*$2GyB?pDSOAU!<+y-sAgg$sBoEs!wzeW2&&eDK6?ob3uq&CF0qs`c zdL$vcW->@QLYV%Cle5W<+|LR4nAR!xTVnR?&R>O(dP@+!EzPj#JTy3v4B=KnOjF^F zBx;-&&26=A+rNo)mT;08sK<{iTxcECe(2x)Z;SV=Y*Bvc z3@y|2FUbGlR(SrS=7XU@TSiwk`@N_}?<*x%9M)YI>wCX4{gizKpa@Q&xbIZ?S28$-Gwy*5(ONZyg4dlqRnkeqV_ zyUhS+W2L_BUS=<3?&go0H4HVk4a`>sjB#Vg$}xpG+KGud8*yI2eLtl3m=UBAZPhX z^9;zdhHCKr;%OB(b$^0-fDL(?b<;HUnw25lZxm<}A@m2ghCcwmw> zn#7-vnNWp90B#1I79?$~?=Qnp&+&$98r%JkBhUIoUE4)z6jgY^o!bU~t16z81))x; z1cAA#gP@}WQ|K$YlE40*gd)&(J?Z|D?*`CUz#Cp*Likm*vqjeWGl?qGjq0Z=!97>*^bMAh)Cf0qT(cO(1;hAc|v1 zhIz8f9}C!XoP=wg%X$mLH~h`m!`I2U&ddS{oX#8UMjImYgT;JXYM=f~vDqXGZC?_t z(e_@)Eyi(##mCWAvtKVIGq|J?TDp{J=>Tu16E#S-yn=L90DRMD0Fkj@q$0k%P~C^@ zkr%%+Nz@|AkyD<9J>rBk3y~L>-7fb??Te%B?CI1{@9gZPIJ)NkuUPQu&{ z=U*G}>j7~OH=-0zud8`!*T zUpO**rmtTQ3#Iy8!+I?#@xF&=0N-1|6p_@q5$^ulmsV02mzHZi&(7yRjZ9C$xWH$! z7X^Er5aJJ@psx|Em9ATCE{55$KHG>al5JL;g1NqHj(Ldu%$lJEOJ2qEi3C%aT^4XU z*F=z*&j#Xn72=t~Rc+KbTuUwSx7Gmj7@_DkIqjmJd95(#^y@ET^6bRbR;B<|Z&Ef~ zKISRDWHkxaAy#I72a1C1A`1HFUclDzDO}+mwk;{ z@9x(LFN9o`+@;fwU^dm}YzTsRnzw&%BX;cyZHD5cgphOWe4eIZsx%cCIh>J-uYl0W@&aQR3NFwz$$H$?6m0l(jkwAjv|*=9Uyrj|aS z<$_vugoyg(y4QUt8Kn%(d^a#H2A*0OTgFjZ5)6nd z95%$?-&}=-f$Y%${HH(I5hY6f-AMV`kO{rq{#fT*5GKq3pV^=236RYUGRcg1 zV)4>U;Fpck>kSlo{)H~yfbYR?>vPZDmCzRV zeaq&-k=7aSuyr;MjBK>N+qV1@4cE*^JeTjXD`Bp*&F~|^Y!jROt7br2>+IiJdVZ+OWs#?=;u?|M3`1khI zm*AC+(BS%6p+tAJDb3r0aWn;~SZB5JW!QGJ4*T>*#ht3OD*MI=QI4@1_I|lyuVNOq; z6qf&dmD=k}S4Th&io?UUOY z1`5C@32g!8U{IOK0QJ4_5W)V_0d%(HW0U`*M+OkHl{IGb?-w~>#&AQ8rmzm*7~}v3 z;HGcxRRdfIV0qXh@`$_6632eh($|k~W)*t8FI#DfDjvV2aa&FDE=<&t#LX=_7>ZY4 z3jCvHCPA701KXnh#r1+mU8dEf^j2&P%nfolTY(O2ZpFsObvMuh->}(j&Xrg{P4K=b zPMyfZ&0wh%$O)<{#)MUfWhX{O`m8u}>6PxSwN~;uTD{e6@WK$?;R#>1Y`~@Ik$d!I<60%zaR6SEe>~G&yVg1phI@!Gay~D z$_(Q~Lw8}!AX#B*ax|4#?YBy)Htoj$Nh-3M-bouA`}Vxo6MfqJ z+SSr^0fSd zKxNFsOS9$J<`dHcKB0dDYchoXb}CEg*7gBvyfNmLYWjLYJg;p3xvX{RwY2|Uwzdq# zW`b1@4LM%YE7cl?19=ATK<9_8AN|R$SBm{acUF!*9zEV^B?p~`5;BPDTy~z|S=g>w zvOf^90A6#(#b~-0?u=2|j%Vj*<@(-XQD;8Gr0KX4S~*kSd_uBIyyXt0?^$V!1CXKn z`Su8+58yAD6=pcO7y~%;9OTVa%dEQmcQhAxLb%$im(ij>m3f zaH{q%DV1AW>IQ0+04@5FcLughpL_VP6;lTpiAi z@UcPbG~FzKdOlDLADqZ z#Vc#jA;J3QA~X3#^Qe7k$Cyc5vVMe`)%w$acpS$hgODy{>sR@A3SzaQ{^Pk$T>ooL zt5ZTNAk>S(a!f~8-;&tk&TsB=Ah>LrOZrx3pl+bKj1eHVcR-6s6Fulm6}{0;{C=^h z6T~bmCntA`3$M2c+}tL>22+O7;qG8H|TAcNtbhgZ^0X}9RI5v2fq9)

bx8zif3J20`T6&j2qMb*^Tgq7PQxLD%N1b6=(~7SFXbj>uiA!E#JvFdo0S ztM!h@=$>{71lkT1t}rP#e!|-o7~;wJ!_Ec_8~vq2EmGV9PjzgoJME)@IPdCQtLX}0 zg=UcPFO@XhV2|h<0m@r8c z)b$h0skbUeEx>mAi5 z14A5)zJPCHY72m-$dD{0#^)16HLY}cQG?^HVR-` z-Xlc{VPg+lKe|cj_Y9MF1Og%{$&_S7uqH@xM#n~A^hcqdkLw7CT69_+^PtbA&BqNf z8pZu9=9lE{h-`Y0qLPv$4T$!V3SVGy5g7~;!J5%hHqVTowH>)6o~GjJh2+)3%`&0+ zcII^mT7H{d`~75Vj+lL>X5-oRuM@Akbr=W`n3 zNts)sKgf;BvB=)SSuiKGXY7sr@J&tf%Vw!Fj z_3V#xotKiK=e!>A&VUkv!4Zjb`h~7M-?3k7qi7G38AZ_hCu@E#Zokr`j}yUQp(^^^ zbAgNkw?>;Gf=$eK%$3o$m@q}Ow8aL7NI-)eGx^}R z*e6;xlXvN)Qo)S&98;&AaHeO2dKgH$mJfqAS3#mmi z&wJp<)OUAUA`fbk%|XU!2rpSID2tmKh91>QYUQ0~-`S}Te$oww

7L;^)-nI@IeMbNH!s_w#LA@T7m(+o%?$JmW%xb>g;aHG&V64o zP2;@%{WpU+=(ayI6_gCH9>Bx%1g1mQz8-C84E9oF7@?M(=SYE-G^!&l$sSPl>tD?A zgAZ4z^q0~BZAe0Vv<|{u_=f@E6DHy=9;F?_zGFx9+1R791ODl$wQ&3`)bAm}w-O!K ze@V9g1p|QP2UzENR--C=zR=W+X?)Kf`MEn>{NX06=;7BJD~Bj#$D5MSC0o&#EOfo^i$^zozGZZ zwC6y{pGT+nB<8g=7H?as5cIDFyC?W&f3s*R0!Vhv%aZ7%6))ZSN3)!Mn5x&^Fy;4F zt}q`}-fH8pZ7O2Ga2TqsP~&}`bZ7{+MO3A6^g$%El99WvS2{2%{C)PYU>s}VLz?H( zOV*-oV{M8JBG``Azki0U31_5i0y!}WU(NS^vT$*LB}+1-6(s#jESNxDD3KFHx&x2- z*f)U*r3J`Wj*!b?Aac&7W{q-J1-`NC#1`*tU zKbpiPzX6cV9;K%x;V#kVKb~NR_gNkXWv-BoTCTU}h#$CqNXz~|T*|t$d66?rUXk&4 zQ2BO1@C_(PzVC%!1*^vh226R57Y!ecek$uIhB1b)p;Xv%G`Wnu%gKuG`PuMVSzo4JK)T1FsU{t%fbz6O0JGKCfOGoywLyx~pvymmdp=<1Ku zMZ0@GBfUc0ktaYy<0c29Qd=!NQpnUnX`cOarm*g8+f`$5SLRPlNj@zmm^Mxc@}Y8I zb8WnH8>-%mC`?#Sl0jC1LO+q43u1cunCQQ011NqMqa!A*$^9ZrsfNFUXDlD0FNVYUO|C0JJx{qlL4rBy#2Dmy6a#vH{|kc<%g8c{N2)r zIs3VUFCFNGR8!}-ubXd!B14VZ=}1o!wH}R7i%t8b56gHJ(*l8>&N$D{wO-mHwg9peFM$~t@~)`bifNR!VF0I0%%9EvM8i0x%)Xg^@8 z8U>mg_9J#lr&99r!H0Vei09kfY<0fEgxY=TH@aU(1={cFRDCYW4%>~xnf2uJVTt`f z?o2;`x-<4QU0O0J01r0UrCI|~Klts4oP1six_^*S7yVQH1_Cfq#GYok@!Qj1vobfW zGcp@)bXf^Fmj4#Cs{WVd3-vH%`hg*H{ zQM+JSe-dj4ylxXurI6a1!@Vl&S@!dbi}xKNr8G6)o?SB~&@cIU0e92fuac73jo6?* z_gBnN_{h~epI!C%Bnsm11##>YKi_mkz%r=Ok;*{lAP#<%{+yG7bWGR;PSKIBV*yHX z)r!>j+68QrJ86HCa~Ef~E}jd`v&CA65pS9j`yk!U>vt|aDU!}x#U&;eC-xuRx;o)9 z0_Th@EUlRUfkRwRS>ZF28W5nbIBy)352_#ka{q5p)%6~CyW8J}-3&oY{)S7%Zz$p`Ll3xj`BfZ+@8bh;Q*~wS<}r}x=1MzOfH;Esy7%n@oaUm z@pUtrv5QMvv|^tA!O>>@8!MwZUKHg+^EKQm%e6NK%b@Ur{|0*&TJ=<-?2*}^Ke(_= z2~9*X*U_iH0zxa}@?iiIZSopb-h3Dn5NF)7TPg;NY$O((+N=_kmF99iOlWD?xXH#F z!I-*tZxI2xe!dn}3pFFno-}WnoBtL^=9rnAdqn{RyDRh+iN;Y7lPlN6dEcExof0Bg zv-6(xYKv>%ZP_PIho|!prg>Vk*qJ|J1lviR3jelF{6tKSH)B)(u9P1=EAlQ>EWW3( ze@%*k_?^W93saRspmuD#?Q-uMlZ7-X25OkR_}^&zV?11wCEr|PM9sT2GJzDSxV6MH!*P9b zY;w`17v7~Uz1LB&g&%VkoLU72t94bo;(xQ7odbu~#{oD^OE0;-uKt6kK9J?1y$O)s zIG8O>+QS6s=!$E!vz36VYA3kA%-H<{(lY0B&+t}SU&qLXQ?y$nRZuMvz@ZY>zBkGy zssA`nlrMvzuQY}-0&2KoT3_y+OfyAMZR`cY&wfjprJU^$PxPPv_Aewzoct*Kd+-5? zSKRXGhjhJ7$u$X5r1;gcb)pCi#>h0x{k&xj3kjX+wG`KlcD2072`o5e%NIJ(rs|Y? zRMelRVkp4pZw#uVM_9TApqpe>836_#c zBD)NP+{J~PT~e&;HKfCxPfJFSjRui#^ZSvqdpFDWI!pj6D(dhphFsC>=KqWl0b8q& zvz?c$QKadKws*;4goMz@4`v-*!fAOE^=qXcrL#r*#N7*bylIB&r#s4oP=-ywS=hfl zsmYewOHZ zKdBf@49J4LsNT^{*dai}_|$P={B+eL9e(^J5?-+8sN|mEBLOVlqU(KS#`kAGKfm)X z6#OD0pwC>DUNIADKYHzR$AmL&?elzLVM+Yb_Rdq8`i;cBxrs!~lF|FOHu0YYMcYev zy?>bB&Xv<_y`QG>|Acv>_djV@6f)_1`9YqO)B;p0|NNP020*X9MjjW1!x+bkV}a>-Rs8=G%6f} zIvk!SQ8R4y%bA@@WWi*7o$E0*%ktzna8O2eNPfk%5K1CIJPW_S8i#ZV=)rERdb(16 z{_e7Sn4ih>xDZC^i4U45<+7ys^UAT*u5`0u!RMPYHlXn?FYMFJTx$s>eWWU?o&c8O zFTqlM8p)*w!;&LklATBtA|ftB@cGK+95F0u9TEbCZ9Vk%^zdU!%E|`aow2Ak9Jl`M z^u_-l)v07X6V;%B&i{(-i(mi5{lQW>5gEpzZa4mESI2@c69v4I@2)DV z&Uk(SFM{{$R7bfc*`o|TWqF+9bt|>&dZ@(~5fv>HI;^C>Ni8oYCy7wA;*!oZWf2$w zVn3;aUrBr<6&-6waTIWsd+U$mShBO85y)EPQA#MQWE8g@NM*?J$iRG%S)BI z|68=KBbVeSl=PpxiGm@Uc-u1Xbg=%ygu(>Sf`@5*tYi3vFk!hcn|T#OcndQKCv3H; z#&KkG`v`}6L37^j=*{T0ykaC-R0U|!@RrwBc3Y0v; zflEYse0;7lU@6`87_eR`o9Q>_8{!a`zucXBxSrk#$KU#gg(IOtSL~z_o%2pa`!`miQ+wG_BU0Y5IP*PJ6!b zK7?#MX43Kx-)ZBLOww|ow zIhXDC0+Lek>J7t6?dQAcPV&)S;2O-pI*y=u+nXQXMUzyx?VB(t=TZiY&|}W z_-DAVTB?2lLIdOX)TDmeh0fzznW!U?3b|BpW-f_80=?OtqJG%cSXjGtW#Wz80{X0Y z^}_aJ+Tf_>=wlF4PQ;WI+kzAg{9amScNNl*a`~X6buvPdfb-!|jl82_5VZ`6d3KJ3 zaaO%Q`V-C#Xg%p$T{n#`m!~VSNB~%pAdo|^N@Y>SHJUbd*+;sO(%j!o?4KNvywBs8^HuiHh`X2vtxl3gv3QlvhGn8PQ{ZXTZ_k&GgT3cNfkRrt7L48 zbF`pMv2NTU#Um$Kj>!Qvw$dW2MRn?P(cm?D@)JyTc0wZ+Z>fG5Gv;r%-b+K~208)o z%?bV&VRZ$@tgluyK=zDh%^&Kwimv4ehxy58)=(9-2x5CcArLJI57ag7wu8AcEe5DL znm%lW97@)i&&*;cloQO1m1TM@Vd;xDF==OBE=Ji`nMer+Bzv*__m-p}Hx@k$ zw_aprFL!l=5WGWiM^k%V<%yK)Ob-U#WcZIEBZ>zV8zzUuQ;mhNu%zQkE(KOAWRL z&nZg%i#=z!vxs^M1sacQS8V`C@ad_J^A+uj?bYuKJp9of3Qm~%8S}p5t6ZhiDD7X0(#QN6!9JcX;#7%YyUe@TkUP3 z%htqx1mx5FKJj74AfYO*UI3{irm51;(*?)b`u-zFp1fptVKbyNwrq8y**koPR;eVZ z;Ek-rK*9TRLdhur0|TSTF8=$4JG0-k;ON=SUH-_qU+N1LYF>@C^Kj>j64Rb-on)J| zUMremlEE8Heu95_^Oz@<6%-0cZd4H=_&vQYiKvxmpR)HK5}z!yWK2d5<@s2ocU!bR zdK!|mn;LiTZPvMrR^Oy{3wmu>tirgnWMLe%Wbb3b3F`VTXGZl{$#4a|78 zH%hn(7X9p>D`N&MDPNE7dQbV?ry3_hqrO?4Z*-R&88v)Qj&IWbd=_l`pP#XHcHmzm zaKUCC(ds5N3n;2*l1qf=&1W|VktEtCWo9@(T6QFLN;)VWg8pd^#otiJ4!%6;0oZ0J(k2u@OGG@R@x`v7wfp!gk|31qfV&}KE_(r zM=ksfsj_&q;{}Um315bp2jiM=m1u+|CUHfP(IQ+TnMVIQ^_~ zu(u$SZLzxDA6Z&s8Wn~%AORnrIt!;+;gsUGAO@48ffjgv2wVMj@>9JB#pcVOUp!-3m&v-lCOk&r z*#&ZOMc20GTwvJzLAAHAOVHnjqPxpy){~{SccNDlG9^cEI62E(=3G-b5f!qwGwKK~ z2?q?$s7Rs8`k{U11!ue>b82E?V+Lx%8|#95KV+EdF$b#ap}jZdJ82aMiQWuAy1vGQ z28X6w&u*_^TBEgih@Egxa3u1|5)I5aa>TD6bL+xe--i8-Y=@Yi8u}W~*sl9x2%m`L zHS6IaS&`PhPxbYbA`d7eIFy?>!XlHDlCjN?yCgtcl8r;rk&U9bd z{vM|Nn9j!M(hM*&F}>)tZ3;24YxOMnbHu@o==E6BBqE(?eD zh!s!*T90bi5z90!xM4T*4&MX*-XZ0}UbKN^o=#f6>`1Vr&1+s%eB5`QKj+SNPVkw) z5+|~vJQ(G&sGXF^9`w`O4H(^o`heBJ{{VO(GIv+vt<&E4{rt zEMJVSRd?@xY?E2*!j2v>{33*rQwhmRJq8d9QE2y4SNRg&i4^Q~QcgdXtRG0ge^}k) zXpO7Ek*}jkkuf!U6XYXB5Pc%1wCqU01e24u%&-I)zwo^m)Utl!W0M&(0dIIUN+t}E z{B$B2T`LhzIvPrOIXcWF*}oitYCSEoPHsA2@aeeLL?b-g8RqZ&9vI$2E9e zc@iAs*3D5Vj57XktqEVn7jWjuJHo`P#s^8tP*o0OIGy>fv`xQ!vLXuTXGrvmrjW3i z54KxvDXvH!M$u-@`tHADMo<4!xCL?MwMBA=UdR2^cK1W}=Mh2)O-^S{OUEV!PHhDXO^RM+bIGsaHqZJPUp2+Yn7 z5ueE6()!{<%c`9kjN{a0T`Vt4{i{S>L2T|U)cwj;v(e7*mw?TGhbeo1l2NWhWqog3 zce*?A*cGBYcTq>roGKl+)$$R&;b=oP_pDL5pAZ@(Gl-#u2-(rPwP=!?Z{v@3uK&GA zRZ2#md$a(X82sp3O#QJ0ILe`jRV1mYNqN;j2GaBWV2arPQ_(Uc2aB@%y2INwJAqdS zsuNTrLd_`#sR-RU+xM#N8HG3{JSXx3Btb^P2Da(p5lt^X5Y$nE6xAIy3U-p9IzsSl zURVl*&kKBaf8QO5FJ9ELEAIOM%k`<{R7d=EDmH`593YA!RJF!c4)>QK&+?6>r|2&k zcDfUMBwb$OMSk9L?5=H^>h775Fyr~3DbWKN;29YGb&0HRuyQq#v?su;~Wo6LF^vEwDK@(Fi|N9B*S4|&#c zo3T&?d4MLWdeX9qNsQPJm|fcLgZJ(I?d-fPSTbk*@sP!Vky)jan3}pm524X~BWwVH zPN@&BHK<&QwEf%1)H;C1wdRlcChxdMUmU7+^9c^ZgKPY}Ah0k9)m;>En?i{v96#1!(;aylgZJA^&)_LYmSmkLb^_wcW=A+z=tvXd15fi!1P<)y>|O6k z)aJV6>e*~^y6ia6L!^@Ptx6-{O z<(|Jo35SnXYfw)g|4~|+^U#$I?*9G~Ut<0eJJnDo_p7QH1O2@*)8(7`;ElK+k1j6( zIX5bs1^X;{?@#EChnb=ZDJic6kehy$2AjzkUNOTw10!uVhBEo^^uw1TGG={e=Q;?f z;Od498wL7nB+|b^0LC6HOCL_T*>y?NG? zDImwOBXZWJ<=M{K!v8<{;p413CkucJ?`*$}iZ1!H=0PBWwY7dv($qKGrzGQ| zY83AeG22W{K9$cYtWwMfwqreFK_o||(a!g&%^6Y+?XxK1u{>_cn*!uxYu3W9bn*&r zY$*6>&Wds_?WL8uvf`^E2C_fYQyU)byZ5-S6UW7&n*#06r(BfN2YUx}Q8+=#)m(2% zF+PK6s|;(+3M8K36`$=}-A*DZKkD<`cO;!kxHL3PjDu;-FYyZhn#0M8$KZC&%9A)lIW+U2aeN`#))G zXZk1l^e26BANu^jAWep!had7dyNY$465_DXJq!WH-K(jN&R@tnq@Szmd?AQk-clcT zq`o}3pd>t!8S+n8JDK2Dt9R#gpSHRI&r#!@?WBFBGuIP-apZc#`$R*&>tnPcZ`|T9 zzuYeuMLlS-0lw!4yppZHq4CutDgo<+X8|}@AN{TOvjqI4<)lU^<45GO_FhP@#ebE} zmR+=s>WHuPoG9(bIi51FLJ-(Zv*Tk)-=>rs(bJVIVrOl%KS)9AsSk0W^tf|u07%nR zad)ys-*v>CqUNWU0VH9?l4efh5SkvylY7G{vuOI?3tha*84)}$lW9+-Pj7x)73#*L^IYhF`I+W~QxcQ-tiP1zz$0?aA zCK=$x2>fmuNaubG=(97nQB=ez*RMb9Ir5GWc{8?bx%YHq*rJVv{_Obxy)~t9J&bVX zI9hL0vlVu6ITE7lxNSM_#PCr#aUmUvT6}I_i2E`&?HExJv*?6?R%lH{-hWI-dqvs> z2Z1!VV;x%UKEc?FcouXyZP8zut^cPxC?PDPAai;#OU7kA1+bx|qnW>!$4v(r1lJai z*lD2z1nEL&!EaM;;Tmgs86I(hb}NeI2F{s?g$oP0t^z6xnM3ilJ25h#r<%nuEv;!X zcV8Zf+W0Z=w#8_>N!*#v5>%@mF8TJE4~5xyblly!(T|wdar%^B+&6Oj9GW?_q*b4c z=~a%qgoq16(`ymjiO?&nj8AFk*G>Y%fFi;mc@}F94<^y|`p1uJw&*je=UUEPBKvzW zAvOglSZt&W^-+Ps(|8I$f(z0>;Uh)3jS%stTj)DO-Ad0LTr9)JpFM9G&upe>mP~wX zkM)~&nx_-u;-tTP`9d<+DDlJa-NP~Bme{F8kOBBRzWDrX#W%W|Byhx5R+Bm~udl6r zHR3Av{0Ef&QCH-EjLtE8||{G}$OVSojX=z;bLomA=U z#9ysqC-?4I2907yo4XoQkSSd^9Oob*1l**k7`l1PlFVpJf;ChkX)Q~rug{?SGiz=m zNA^?yt+tl?uJdtajNH#_2Ww<@zbC!*!)EKXn<;yVUR)b^_2sLby{$xh@_A?8nKJol zeWsx0rYiB! z`Q?LaR%Ib6c;*(kE2aS!?}3aXJ3%aha|d{1zul8Oc%xDx^I9V2M#0YeDy)gHbcX`I zYHt$0e{*q~Zu#+IeXBQRX7lTSRVT-;HCA+V(Y&)!)8{gLL=sN*;9;Y*K~m6v5*RRH zZf@29?-L;#8b1pZG0~J#!)y%{#R?7gbk1C18*dn2bJk?py5vF6^JfIXt{q7zcZ+In<{pzmm!HAw&{I2Me&slGwn)~Aw*QLP9 zP+yYuQuL@@1@F>w>M7Kd_-w+jAV)7%`-uMr69cBN<}tu;s}en&?tD6GFbx{nT@JZ! z!D;S37tS|SBrl%8im=P&#NR7smGtA0bF8?|?tIR53x&}jpaR7ZE3btEQ}ef0{KE`LS+rM12VF88A#33OatXuHU1IlO4ShesEM_r zrs9j));&;(s7sw21|Su-Xyo;hvK{Y2lbZ)S$9x9*l=KimY416z$xi9;&ShpA)4k&0 z;&+c|tn&>uIYVty=vbm=B-{8wq%Q~zz=0C1|Ip7FcW-Jw1B{nAK>4_v{*O&7Nk~At z0JjF1|44Xu|61)4eEzLzBK`5;R03Xn6AEm`k2jgv;R`+^aIcFw>7bPOq6qNq^FRyv z^H(hXa%jGDqs_A8`0RG*MUr`ZVzzieFm9xN-ZjU?LQDj{D6g(+qqyP#=mrG?)`D(d z^=`h5mmf1sqx33(Z9*`~DR4-bR+DFg|mw{+rc$6#(PI z#@$pU^d+MI!A{wEoew$&}49DcXHEMjLh_U)5!9MR9&8Yb)ujGJd=ehIG zS}d2%!mIJ%r*m4G+#HFQbZaEfa00+Jk!OkS#3}1 z>0wwEf8MiVzEk=(@xCzT)r(>K`m+~`LI*>W&DM1zy}^HGs1J@qXXz}ujjRa?!+im8 zmK||-zX++yb#wJ+YT#==5JTg^*`}2!b3=*^x=$Y`4RrS^rNRr&v_1dUBd8>)JnImIQq~$*W zdutIkKIVvAP`UQ5Hd18a?jc)#=c>Mi8qyiXx3hXMa&<#n&(+v+ruR4lRd5*qU@kC$ z-Gn#^9t*855q49^8Ra}ZUh@r`(S|gbe%b&29PRfTJaXc1yDN$NsWKPDlF46Buzo*F zQ#?CoXc2O82h%1tjqs*^m;E}3s$)(ET97r1uSH<6sc(zwbC93ZVvl;v_T~G9&$bVB z{!{&p^E`A%$hs$YnrFhm?crFYelpEQlg#(3m~Fd|W8tdM^HQ@>198tcw}@?5@h5s3 z;}pBq=f?82E6o+_PRyadij`Mq@zQ?wlL*PP6aWh!b?49ZuuTUQui$@AW`RkKWc;BN zY7xBHZ7@h6JDqNcRU(lj5BrnaF_#+!t3~xcAnD(gy#2>E5`ppS>1e)qC|SpM3zJLh zU@r7sRN&G5?5%|6U@psO`))L%AnV$Ija8hE+Z@!S@`y*`GS80n06)W%u%8+8t?O1o z?@}gn9n}y&(L%=eJF%-`tz9gHdS-S557nw)$Bt0Zqixu7fy_F>D83F_**+90c**Xa zTQzmxx~eW!;6_t8hve`v{d%nkK}L31e+ib3#4DMe!~fnnUPSAc{5)CdD_F7dusx~q zKFB2&eXB@`Ix$00;|Mjhq` zi$0a{%z0rTMby71Mbn%LE9T%H!ya^s?E4l!F$TE@$tiF#;`)lyEXWqpiJD3!-IZe= zfD4ZKMSSlqyrK`F*vT2oQ8Xb7mlCC)PaL1Un^CFT@6||KYkh%PuWVEkiO@#iNL0 zn$A@nt!5jj#}lV&QM-%kl`!72Z`V<(Jd*A+jZ8Q97O0{WmKxUT*Dr6qA7!vw9(61j zP}EZH6aao}S*g_)tqH1I%?X3E)@Qx1su+cRX@T}nS1&V>;-Kf~Ddb%r{MSCK&_n+E z3)-vowvPtoD4BJ$DanNNr%-ujk*_C5`(4?W58M<_J5^o33<5`XPv(}N)|ysxH%*W~ zEcMASjja|;zY>#ZZ);Z>&(Al1sT);A!Y;R}zJ&N}gXMA;0ShhMoh4MM?-B;GzdcyN z0mW-s*$GvS9JYe**z82fe+hWs$Z>^4K=TfJ2hxmG#6wy+(@hac&<`=+qejm1viwk* zVO0#Mv!xd8z&?SDl<$MqL^%iR{?EPF_c+($h&k;q7N-@yBv-wiJ~j73UI(}oo`(^| zd)9pOu6ZP6%`^`dzvB!LeVcyMIGFz8ZStYOI~byHB!}SXS`iiO7zn^ z%OP75{(F6E-PL%qN<9^Ws467>Dth8Sls!7c9|U}C)S$@96(1|bWpvR2UO+{{*z@yp zbFFDCv>n}wb8kFdPK~}SHk+9VwJYiS@Y|Wv*cPa-^R5skSuihTJH(Prc$~Ou|P-EmQa;)n<#>7Y|A^b~^jE z;cXNO(iuY zs5IShV3w;>9(T>BQXLyspplkimgz_M;_J_45>q`@@z*zhm@{(g?K|X7h~q@Do(Eoh z@42`8yD{GQ_f1;Ir%oe>Bq_d^%_BejTZD5zP*Z#pKBPS}-T231qa4!Ht=mZlz(`#x zUYY$0;o7(n(|8?-jnjfxhi8O++bU4R>b~OStHHg;Db_AjBH++PpuAQ$dNrg*cCX&H zAabO3TfhRth}|!5J*)#lg(k;5ACru=E{QcfZ}ppSc~UswGU%G&VXXLiu8Rj1PtlqeXvay5W3jbYxP_HRKv!;!M;XFma#FuVuLlX*dn;n{VV=Ne+3*W$Li)a;!&5Qs8=V!t!tJxJbKDAdPp}q3=`TydZ*g?a6*`+~!74ufbVJ zPOkCx&t9MmV#)G_!Jcw^#4JOXnNwlyXs1gz^S9xK;Bnij7(-WN`{a^_z3z zE8#@4&lPo1vzDTq& z35y6t4VZcSYjX(Q!YsU%Jw&1K>aE?>kl?6h2~7SYPI~=Ime+bHwhV$(rW2L$AL1!$d6@0K|VlT@3G$)H?7&8pGHR zxA3>7#Q;E)uOus_-Cp}1=FMffqjV%T#DN_*X_a#n`s2TqRC#(ZRydpA37Q$bsatl^ zuueKCpc492LDwtF*plX!zc-rg z-MI63SXI=4z7}Wox!N15!}?Lg=W6@Koh3})gXf4~Ty?wRk#tuV|-2ay)t=bLP$VzHa(|k6PW20;k#uYL?7G@#d zpy;@aWlB{a2^^R~JDbT;%mLQn+NG#nOrkuF1bf%%tKpuVkCHFh^&ZwT{bVpH8=6Q+ zm#%hn65i|g&kbgXg?Dr2CY=44Q}^C~RcR03+jah&AWUN5n43bUClqF3$NkwLj||g> z8vXp2}e?WKxlptR*N}cL&Uz#um#I zV4R4R?aA$#nJ~3bEvz5R@~g^7flXA(_QE?INoJt4?-XDxqU~unja6Vmn4#YEh~udK7Fp27zd=r zXutp@PD9ezG!q!QWFp5EtXEVVlc1}N)*#DUnQS~f0xjgLGUw?z8o#lb45K)QrKQ^t2b{PG|1J?j~|GQnB!Gx7XQjdUmR>$C0gqL(5N z{p*hICw<>&iPNduj)Non;vx;>XzyoKp1#SOty;_&j-Hj8eFJs;l8wnNq9vgcRH9z4 z-1@&ydUODChy?QCk+xIDSmbo~UfqtOih0ucY%cSuedY|GcE~AF8}rasy|u_PU-$Ws zCyq+);>aqCFA-M)Z_fEl@!ODfO4gq64ggL9oGg4F_=DLNz5sVR2_il9Bm8X7{XE)e zxQvvWl3(r1b$4fYYt?O9v^m`?mCr+UsKZYx*VAR!T?&lYL_amfS@^#wr@KJjca#1D z0Q^-4rZ$h;E6)y+0&**S3m^#?F_tp-Ie|tJwnZ+Ix6}#U&cJ6(RP55XqFndeXw*1) zn49{r){gtJ1d$CUn&<3Km@xTD_%-> z;Tp<~lS}?mC9GA0E!p%RUg&n#1$^*ycE}g$`b18f{WnHIqa}%GW9?FPOhyDLA&phI zv}&Hi0K`WJY*(xgsl9z3_Rg4))`{Y$t}j~kXyhxm>(3dyBkWBZ33TkE-y1vYyts{T`BQK2cvb>9cZnE zkprULBf*};-DlhKHv6)#bmMP_4$O0AJ;CQ>Y^0OY46Ofm}7vP?X5xwe5dhM z-e4ZMtdIc6VkRPsS7s9Z+q;i$FR7iZ8yLw?H9%cd1dGf$)Q6Y<_!Zp4Ir9xz=y(zAa3{uZs~_nj1Bl}q*#Z~C8HXD3G=Z0f|cPH?b=aE=c3h!p}94b@q$ z#$tKo3O!^eaV`{^gCPAs^WwrRY;at|H`&qO>?P!>}`B|BO6HuY<`}w?03}ojG$;HzA!P(WVPc=IP2ayi$0PhvtN;& ztWJNu_(Y|4tStu>{l%gwh{%hhE8!cQkd1gmls}b&kXw<$@2~TNfh%spvGiGz?~(@{ zh)#O=y*UJ;_e~Dkn-fA>gQ8xkDSHLjx{Y_y}!eZdYZpK`+| z)4Q*Ach!?^-32hx5u?nfCshs2le)i#3?tCb=8g?ri9W6xpp<(3Dp|c-M6}hypu!l zjwp5ao-a;EYu>X1F#kNa-pzE4LW`7D?}>?_8i$x`r^U~#Rz6zXzRy=%qF*PKdB#Vg zX#8=NEQA8=3dPT-3|}C|H9PGILe6u-$iY_WiE8CP#?G)Vl7&S{-STA<#-6PYL2fL zH)tl}wwj;t>BL=(!*va{Tv~aY1J-0*4)Dyivqd>T;H_|}VMyJQG^YEKyc(wr3X&P) zjJLF#$}K4hbe(%*g;YOVZEjY;egUQjaqU4l^v*;*>X9iOQ4p2Er=MYI!$us3)KYPnz2ny3bTzByg!}4l zH8mI(7R@xsWD7m(*l1;Z+lT_ZZem@1a6zxL9(+wnBhK^GLinYLk{NY{kP5(=TED~_ zaHfkdc)ss>n%rI|nm6_6!tnaLq{A|c!Hpn^6m&*7=fcmD>dYu96U<4$p(5w~?LG&i z9AI?}e5rWvL(8SBD!FtOYyg9e z8dVIXd@q2sWnvXUTDeA()kwS-C$m)NeIGv78-m|{Sp>M=mb_7LrpZSy>l2KagE)kR zoIjDvd9*O0sE8hP4(`hwmN>lqLbq+*I=`8kSn2S4R!+tozC8)?Nk1c(o*FsA5t&`@ zG~r1g(7KDdB;(0*Fty$FQ1~n*V}B%v43!VWp7rCIV^FmT5>6uGRZ<}&93Z9Zs(#R= zDQ3%qpzzX=Lu`@W7n_)n@F9^hi+MT|O5qwotJ^dWbH?Qn)Dc@&pSiN5$v&>SyqD6w zE(Y0ryJCVq9VjlKAPjq|NxWwntV9oj`d^f~&p$Yn76Lwh-FPI~)|TzKCtBw5VL_v> z(9i)T)b%@m1mkgA>J9C~M>REK%jHGq)1B!1nF_A(P22tA;=9B6c>1hSaV_AIyr=|I zJctN%8a5G$dn$^a5-U6iv?9Flq^7S^0xp5t*O4tW!VAsOXtG?dPm;f8>yo7ukqlKenO7rJ@S_cuUF3!*=Lk=e1Sutu^~qb-xO%X+0g0? zH@2+bCW9lW^2uVSOd*?$@6gr!j7*nzxwkRu>VtW|TW?sM{{-Ffp01uzy`}^TGS~2r zBM&Fmm005F>&NR59X$}DO5)V`k-k!s6zNG(9i1=Hf**6ta>eW;#(|751(jo|Wi~Rx z>;+fF?BL9DNsrmOHw!j5ZSWKqHpRQ$_N{aKl7smtpBGQ@uUv50**ndwEAFzeg;m8r z-=m%E@{}YY?Vb@m3J0>f>*CXuqceai^)ta(S)Gr-0#?=bEo;%z^p)O%z$?NutGTIY zESv^TETsK7XDG2eCkzv~Y)HV7b8_oOLz4`DyF0-i9EPCu(K=Nna$fE2SoC7Pm(ap& zUhTM&jHx$j;?@;(_E=A&!N8p^)v`{_q#!Y!Xn0;XcTL1G)s9iA5Tw_Mg$E_3uU_?4jb+`3y{E zNg{zXv&Y9i3e!{$7bo&B3&z4?++`|qt#46=^vE#@ub&TBlgP3GC*XESTd?F-C%^gW z{q}A>tV~?kt_xgiin)ZhjaiXW>+8Ohq@a#Fn&}bFIXKNqq?$cXy6mcd89>tvo*dRE zRd4Hz;!%H^ac4ph5Lp578?;#Yh-m6b5{;&;a>nmnnbuE59g6q81Qi~Sn#udS!*07v zHNNz_>-AYA(dB&45C$hn@$l&geIRUz$>lXFRk}+(Kok5&-5O1l|2(JpVxB3cth_D1 z(5M*k;1VbFSmyHFGAluEFkCKide!ve!hG%{gkke?%*GKn;hbbU%n_Asogg%2WJ1?(1ylYHh&7c4~nlHg0s?!!1*S* zB^PPBiEV3ye-^ubrLJ!u4`>xGgVZw<+!i1qH4P zGaGYh$t-}~XNHx;88~6zwALMz-pXmDVkuZiVa*>h%GF;d_yh=nH#tnm9-JJU*Omar${LHN846=T?@)6ej1mdi6W(YX|fGq2jB- zjR#_m2lShnnJS!e`J=y>+a9+Wat4xG3BNq_)^{I$25Z7;7IlnPmHePF#F%tPnL>e4 za@ag!C^2=PS|H?k0paoA=&x2Xi8V+<`*kfuYNDtioAtfSW~y@|-$bkUUM!A_VUs7% z4dmI~Dw-Axs1L zhVSd119S7VN`eEw`Hn@%xJV=ZdZ>RIgNDHRlYA7(%6fX&(LL_UM9dDfA*@34so2A{ zHAA*Z3V{H8iwwi!rLgnGBENa1vurZ;ZI#b}i_v37ari=}*0)e~AB?*YV5jcXT3wXi zzv*^jL2Jmqd`18mrKxWTMOf~5P4ICn3^Ygb^8r?$>uR=#RZugmGm)N=?979~FZa=4 zp&8yZDq-WjVZo0Wcosb7@^{-z-A0uTrobmaopPci{}aLcLPJ~8n%7LbRXUfy=Ppox z8cPSzI&0nD72*u6*K#LQ8= z6w9qk&f=Y=eXQJ+J=P1t*48bEDQ~uSc&==Udk(9^ehyrR$wba{79BOx!#S>bQlfP^+F4FqwEE*<4F-7f$oY-nl4SpI5&dl|C|=xe$CCLv}q{X zBk<=O;f=(BX##2Jgyvr5h{kC)xI8oqKMT%~rdelQize zCVO^oGb6qm(G>>eQZoS&90M1H?gNAZxJa;O5u4})354x>X%O)ya7hn+I~P^sebFlb zBYQ$q4@BH$A**00D_rP(`I@J>MG5;A#I%^86RUe3zjt&Qm45k-m$lsW^LlPP3PPB; z-PdkU2~0M$ONDJ+lcOW+0Q+%VGP9sQk>`wzxXA(uObExB>h1 z0QVw%*c6Bk6aj%jL4^EwS@DCf3DPdoVPD|vz(#+!#uu5Lb_N=k={;$!4DvH112PMY z!)$b)*jrAeN;uIltSg~TGs22I4f~p{blbjD=xXMBc(!ll1jO1UyKNPxt;xXxsSV&k z7+Rn?8D&xhh;*fO3y1i9AJR45@};mERB0E4YkR&9AMbW#5@*>=Vuo3|QY%2`{7M|^> zo?${|OR)^MWkNJBFt5BU$i%4Ox;MFWR$PV|T;+>&Jt7-gnpZosWW5n%F2Nq%A!8GI zP21K*lPqXBWAWMaxlr1#N37FwHhS9M`0}3ao@33=K72n!dYCpWS2SoaD54Ffg%c?K zVF2SnhsAYfWI)iMo44t_fZ=D+3UA?jI$p|OaYp-?0CErTHf%y3J4>ma9J;Xd4vaY)RwPA&}-O z1TA8iek=R*bdGr=I4>jsX<+($L5G}f?pEbnTHB_~IR$1#OXj9WpGY6)R%T$E_Nca( zy_@*vrcZeWhKEfyP@fCR)t%^;yW(O1SpdyH@~ZQNF%F@g`Hd#e?OF~E`tUKWCKIUA z(zw~r`YnicsUmx7oXx||-5+X^0S!OvsK}8(tRy{Hv4%>Jh%g}S=)Qz*XH86y*|CleK+__A1z~*y6O&9n&huYi}sO>6gN0;+%6}xrim_0D<-%@|&uW zMs107B;oqQIC1LZB%jRXjcd-$7{Fz>a;#0lk@by;X}@ngG9x)|dZh=q$ZkX~8@^%U zIZA&lEq7on@wjd`E)-|>+RS{ZBO?G@N2djem>vo_w7_v&jRM+mK-|?|+t&Ve!jX+% z;GvJv-QvpONP=CgiETDss{bx6k|bHgjpSw(WoGfDhjL?3G+MN$bY924gn~b!de@Yz zQEHi6U{%ijn{fTb^Ou>+yYlfEUTo3Li3yL2Tbc{+!00BXHqxg1iqWa-1G~MLYXEYD zMednThWf6-GbBEIjiV}AzQ0E#4xjzBOR!4^-FWZr=jB7!TpzA5O7MF>i-`Fg``Hy5M1Q|V_wZ=?PcYFqPG-4m%7dRX!n z=c8i5pm4Be_kvN8WM53x)9}g`*HZ0m6IAx$&jPn7c9a-AW(Kn zKG|YpoJqs0I!R%Lmj0wVO9fVQPBOd0L>8Z(k?E)RR}{tq4#b^0*XVrvO0GL#nBv~1 z49-MV1eoa3_5P_9)ty)vb$UZXnVp3NP>Miad6$+X$a*K47u;m-QnT?B@1l@+3w_ym zV2g~E5rE%e;4d>S#N$1MWBu>gS4u%f@(;AM&4m-C@`UXbMRS$&^<88)M!llU;y!dV z{tO`p)JPCxG{Kwfr5url%(XJ}H{T9bB!njTGi%c;8x-Y|bBSmVuu{(PljX$vxiS zBo>7O0ca>0KKNti(RCP&2V<3LGKt zR3{xzUgWY&FVj^PmA*<_RvBEbT-XfIAbLn~0zOqR^BR6h7Uv&lK*ThF=Ff5k0aAG% zBiCzm0Qk&{)`9ad@9oD(D>sn#8T|_Z)6Ly6LJgkxV0p#O5%2cBK8=u=q&u7wPsbFOxNal$Mr5*BNtdK9I zlKOb=>euEcpH>h5LAsq~Qu901MgfjO5A4?! z(*-ayaEh9MhB-lhfIvh5szY^5kaQu$y{%YHPv`)NwvPKmpQXW<7OaDW$zxPy6r8yjD;Js8qBwnUe&Lu z&I;Oz>Xa=&I0mPVo+m&tC<)>~OYEn$}MTiadq zRCyApFSn5L*n_uwErfs-3#uq0OMruFKW;vL{?!WMrIK}Pf&RkcV>qZdLfKLUtwF zR|GH!5XWETRbK!B0s)ZHUnKx5KtX?%|E%uS#sJX2{mMUKp(OKLF4UkVQ`)4O`C`A}D_Oyya~e*pu?U6B9) literal 0 HcmV?d00001 diff --git a/public/manifest.webmanifest b/public/manifest.webmanifest new file mode 100644 index 000000000..574b90faf --- /dev/null +++ b/public/manifest.webmanifest @@ -0,0 +1,21 @@ +{ + "name": "God's Eye View", + "short_name": "God's Eye", + "description": "Live spatial intelligence in the browser.", + "start_url": "/", + "scope": "/", + "display": "standalone", + "orientation": "any", + "background_color": "#0a0a0f", + "theme_color": "#0a0a0f", + "icons": [ + { "src": "/icons/icon-192.png", "sizes": "192x192", "type": "image/png" }, + { "src": "/icons/icon-512.png", "sizes": "512x512", "type": "image/png" }, + { + "src": "/icons/icon-512-maskable.png", + "sizes": "512x512", + "type": "image/png", + "purpose": "maskable" + } + ] +} diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 662d33a6b..d910536a2 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -981,5 +981,7 @@ "src/tools/globePanel.test.mjs", "src/costEndpointThrottle.test.mjs", "src/sources/gbfsStations.test.mjs", - "scripts/qa-gbfs-names.mjs" + "scripts/qa-gbfs-names.mjs", + "scripts/generate-icons.mjs", + "src/tooling/webAppIcons.test.mjs" ] diff --git a/scripts/generate-icons.mjs b/scripts/generate-icons.mjs new file mode 100644 index 000000000..3c16eb46e --- /dev/null +++ b/scripts/generate-icons.mjs @@ -0,0 +1,70 @@ +#!/usr/bin/env node +// Renders the home-screen and favicon rasters from `public/logo.svg`. +// +// Re-run after changing the logo: `node scripts/generate-icons.mjs`. The output +// is committed, because the install icons have to exist in a plain `vite build` +// and contributors should not need an image toolchain to ship the app. +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import sharp from 'sharp'; + +const REPO_ROOT = path.resolve( + fileURLToPath(new URL('.', import.meta.url)), + '..', +); +const LOGO = path.join(REPO_ROOT, 'public', 'logo.svg'); +const OUT_DIR = path.join(REPO_ROOT, 'public', 'icons'); + +/** `--bg-dark` from src/ui/styles/foundation.css, so the tile matches the app. */ +const BACKGROUND = '#0a0a0f'; + +/** + * Fraction of the canvas the mark is allowed to occupy. + * + * A maskable icon may be cropped to a circle inscribed in the middle 80% of the + * square, so its mark stays well inside that circle; the other icons are shown + * as drawn and only need optical breathing room. + */ +const INSET = { normal: 0.78, maskable: 0.54 }; + +const TARGETS = [ + { file: 'icon-32.png', size: 32, inset: INSET.normal }, + { file: 'icon-180.png', size: 180, inset: INSET.normal }, + { file: 'icon-192.png', size: 192, inset: INSET.normal }, + { file: 'icon-512.png', size: 512, inset: INSET.normal }, + { file: 'icon-512-maskable.png', size: 512, inset: INSET.maskable }, +]; + +/** Fit the logo inside `size * inset` and centre it on an opaque square. */ +async function render(svg, { size, inset }) { + const box = Math.round(size * inset); + // The logo's viewBox is wider than it is tall, so `contain` bounds it by + // width and leaves the vertical padding uneven — centring is done by the + // composite below rather than by the resize. + const mark = await sharp(svg, { density: 384 }) + .resize(box, box, { + fit: 'contain', + background: { r: 0, g: 0, b: 0, alpha: 0 }, + }) + .png() + .toBuffer(); + return sharp({ + create: { + width: size, + height: size, + channels: 4, + background: BACKGROUND, + }, + }) + .composite([{ input: mark, gravity: 'centre' }]) + .png({ compressionLevel: 9 }) + .toBuffer(); +} + +const svg = await readFile(LOGO); +await mkdir(OUT_DIR, { recursive: true }); +for (const target of TARGETS) { + await writeFile(path.join(OUT_DIR, target.file), await render(svg, target)); + console.log(`icons: wrote ${target.file} (${target.size}px)`); +} diff --git a/src/tooling/webAppIcons.test.mjs b/src/tooling/webAppIcons.test.mjs new file mode 100644 index 000000000..2267872fe --- /dev/null +++ b/src/tooling/webAppIcons.test.mjs @@ -0,0 +1,70 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import sharp from 'sharp'; + +const REPO_ROOT = path.resolve( + fileURLToPath(new URL('.', import.meta.url)), + '../..', +); +const PUBLIC_ROOT = path.join(REPO_ROOT, 'public'); +const INDEX_HTML = readFileSync(path.join(REPO_ROOT, 'index.html'), 'utf8'); + +/** `public/` is served at the site root, so a manifest href maps to a file by dropping the leading slash. */ +function publicFile(href) { + assert.ok(href.startsWith('/'), `${href} must be a root-relative URL`); + return path.join(PUBLIC_ROOT, href.slice(1)); +} + +const manifestHref = INDEX_HTML.match( + /]+rel="manifest"[^>]+href="([^"]+)"/, +)?.[1]; + +test('index.html links a web app manifest', () => { + assert.ok(manifestHref, 'no in index.html'); +}); + +test('iOS gets a raster home-screen icon', () => { + // Safari ignores SVG for apple-touch-icon and falls back to a screenshot of + // the page, which is the blank tile this whole file exists to prevent. The + // existing `rel="icon"` SVG covers the browser tab and nothing else. + const href = INDEX_HTML.match( + /]+rel="apple-touch-icon"[^>]+href="([^"]+)"/, + )?.[1]; + assert.ok(href, 'no in index.html'); + assert.match(href, /\.png$/, 'apple-touch-icon must be a PNG'); + assert.doesNotThrow(() => readFileSync(publicFile(href))); +}); + +test('the manifest declares a standalone installable app', () => { + const manifest = JSON.parse(readFileSync(publicFile(manifestHref), 'utf8')); + assert.equal(manifest.display, 'standalone'); + assert.equal(manifest.start_url, '/'); + for (const field of ['name', 'short_name', 'background_color', 'theme_color']) + assert.ok(manifest[field], `manifest is missing ${field}`); + // Android's install prompt needs both, and uses the 512 for the splash screen. + const sizes = new Set(manifest.icons.map((icon) => icon.sizes)); + for (const required of ['192x192', '512x512']) + assert.ok(sizes.has(required), `manifest declares no ${required} icon`); + // Without a maskable icon Android pads the square into a circle and clips + // the mark; with one it crops to the safe zone instead. + assert.ok( + manifest.icons.some((icon) => icon.purpose?.includes('maskable')), + 'manifest declares no maskable icon', + ); +}); + +test('every declared icon is a square PNG at its declared size', async () => { + const manifest = JSON.parse(readFileSync(publicFile(manifestHref), 'utf8')); + for (const icon of manifest.icons) { + const meta = await sharp(publicFile(icon.src)).metadata(); + const [width, height] = icon.sizes.split('x').map(Number); + assert.equal(meta.format, 'png', `${icon.src} is not a PNG`); + // A size the manifest overstates is worse than a missing one: the platform + // trusts the declaration and scales whatever it finds. + assert.equal(meta.width, width, `${icon.src} width`); + assert.equal(meta.height, height, `${icon.src} height`); + } +}); From 51b89d3e4a9bf5edcbea046e91ecaf28c5de67e8 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 18:56:08 -0400 Subject: [PATCH 41/95] feat(cctv): add the City of Toronto RESCU camera pack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The province's 511 pack covers Ontario's highways; this adds the municipal network underneath it — roughly 336 cameras across downtown arterials plus the Gardiner, DVP, Allen and Lake Shore, from the City's own open data. The list is keyless and carries no frame URL, so frame URLs are built from the convention the dataset documents (`CameraImages/loc####.jpg`) and a camera number validated as plain digits. The frame proxy is therefore pinned to the portal origin by construction rather than by trusting a field. The list fetch refuses redirects, bounds the body it buffers, and releases the transport on every rejection path. The dataset publishes no camera facing. `D1`-`D4` look like a compass and map trivially onto one, but they describe the *comparison* images — static reference stills captured in 2016 and served from a different path — not the live pan-tilt-zoom frame. A heading derived from them would be confidently wrong for every camera in the city, so every camera takes the shared id-hash fallback at low confidence, as headingless Austin, TfL, Fintraffic and Calgary cameras do. A test pins that down so it is not "fixed" later. Names stay verbatim: the feed's labels are upper-case, and title-casing would mangle MCCOWAN RD and ST HELEN'S AVE more often than it helped. Carries the Open Government Licence – Toronto attribution the licence requires. Disable with `CCTV_TORONTO_ENABLED=0`. --- CHANGELOG.md | 12 + DATA_SOURCES.md | 3 + README.md | 2 +- docs/CURRENT-STATE.md | 2 +- server/providers/cctv/catalog.js | 6 + server/providers/cctv/constants.js | 32 +++ server/providers/cctv/normalize.js | 11 + server/providers/cctv/sources.js | 181 +++++++++++- src/data/cctvToronto.test.mjs | 423 +++++++++++++++++++++++++++++ src/data/dataCredits.js | 6 + 10 files changed, 675 insertions(+), 3 deletions(-) create mode 100644 src/data/cctvToronto.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 032c6fe2a..bc81d10fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +- Add City of Toronto RESCU traffic cameras as a keyless CCTV source pack: the + ~336 municipal cameras the Ontario 511 pack does not carry — downtown + arterials plus the Gardiner, DVP, Allen and Lake Shore. The list is the + City's own open data feed; frames are built from the documented + `loc####.jpg` convention and a digits-only camera number, so the frame proxy + is pinned to the portal origin by construction. The dataset publishes no + camera facing — its `D1`-`D4` fields describe static 2016 comparison images, + not the live pan-tilt-zoom frame — so headings use the shared id-hash + fallback at low confidence and are corrected with the calibration gizmo. + Carries the required Open Government Licence – Toronto attribution. Disable + with `CCTV_TORONTO_ENABLED=0`. + - GBFS v3 station names use English translations when available, then the first usable translation or short name. Empty or malformed names no longer appear as `[object Object]` or interrupt station loading; legacy string names remain diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 040e53afc..47d517763 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -57,6 +57,7 @@ How to read this: | **DriveBC** (Government of British Columbia) | CCTV camera catalog + frames, British Columbia highways | [Open Government Licence – British Columbia](https://www2.gov.bc.ca/gov/content/data/open-data/open-government-licence-bc) — attribution REQUIRED | "DriveBC. Contains information licensed under the Open Government Licence – British Columbia" | | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | +| **City of Toronto Open Data** (`opendata.toronto.ca`) | CCTV camera catalog + frames, Toronto city streets | [Open Government Licence – Toronto](https://www.toronto.ca/city-government/data-research-maps/open-data/open-data-licence/) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – Toronto" | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | @@ -92,6 +93,8 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. +- **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. + - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/README.md b/README.md index 9ce8d1ff0..938c7f8a2 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,7 @@ Twenty layers and map sources. **Seventeen have a keyless path.** Some offer add | 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | | 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📹 **CCTV Mesh** | Public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Toronto. Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | | 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary and Google Street View; Panoramax and KartaView planned | Mapillary · Google | 🟡 Mapillary token · 🔴 Google Maps key | | 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 0d3ef31e0..eb674d95c 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3071,7 +3071,7 @@ its criteria cannot be silently ignored. | Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | | Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | | Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | -| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | +| CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + City of Toronto RESCU + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | | Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 9bc3b70ec..e53acba80 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -19,6 +19,7 @@ import { loadCalgarySourcesFromOpenData, loadDelDOTSourcesFromOpenData, loadVegvesenSourcesFromOpenData, + loadTorontoSourcesFromOpenData, } from './sources.js'; /** Env kill switch: unset or anything but "0" means enabled. */ @@ -98,6 +99,11 @@ const LIVE_PACKS = [ enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'), load: loadVegvesenSourcesFromOpenData, }, + { + name: 'toronto', + enabled: () => envEnabled('CCTV_TORONTO_ENABLED'), + load: loadTorontoSourcesFromOpenData, + }, ]; /** * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 6298e86a0..6f896be59 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -275,6 +275,38 @@ export const DELDOT_ANCHORS = [ { lat: 39.1582, lon: -75.5244 }, // Dover (Kent) { lat: 38.6906, lon: -75.3877 }, // Georgetown (Sussex) ]; +/** + * City of Toronto RESCU traffic cameras: the municipal network the province's + * 511 pack does not cover — downtown arterials plus the Gardiner, DVP, Allen + * and Lake Shore. One keyless list for the whole city (~336 cameras), from the + * City's open data portal, refreshed within two minutes of any change. + * + * The list is served as JSONP (`jsonTMCEarthCamerasCallback({...})`); the + * portal publishes the same data as .xml and .csv beside it. + */ +export const TORONTO_RESCU_LIST_URL = + 'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/Data/tmcearthcameras.json'; +/** + * The only origin Toronto camera frames may come from. The list carries no URL + * field: frame URLs are BUILT from this origin and a strictly-digits camera + * number (`loc####.jpg`, the convention the dataset documents), so no upstream + * field can steer the frame proxy off-host. + */ +export const TORONTO_RESCU_IMAGE_ORIGIN = + 'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/CameraImages/'; +/** The whole city fits under this; the cap exists to stay tunable, not to trim. */ +export const DEFAULT_TORONTO_MAX_SOURCES = 336; +/** Yonge / Queen: the prioritization anchor. */ +export const TORONTO_DOWNTOWN = { lat: 43.6532, lon: -79.3832 }; +/** Toronto's lakeshore core sits near 76 m; the ravines and the north of the + * city run higher. This prior only has to be close — the client's one-shot + * ground snap corrects it wherever 3D tiles are loaded. */ +export const TORONTO_GROUND_ELEVATION_M = 90; +/** Hard ceiling on the RESCU list body. The whole city is ~336 rows and under + * 60 KB; this only exists so an upstream that streams an unbounded body cannot + * be buffered without limit. */ +export const TORONTO_MAX_CATALOG_BYTES = 4 * 1024 * 1024; + /** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; /** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index e62ba33f3..be913bc76 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -553,3 +553,14 @@ export function normalizeSourceItem(item) { poseSource: item.poseSource === 'curated' ? 'curated' : undefined, }; } + +/** City of Toronto bounding box (Steeles to the lake, Etobicoke to Scarborough). */ +export function isLikelyTorontoCoordinate(lat, lon) { + return ( + isPlausibleLatLon(lat, lon) && + lat >= 43.55 && + lat <= 43.9 && + lon >= -79.7 && + lon <= -79.1 + ); +} diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 7ecd3434d..e03701de9 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -55,6 +55,12 @@ import { DEFAULT_CALGARY_MAX_SOURCES, CALGARY_DOWNTOWN, CALGARY_MAX_CATALOG_BYTES, + TORONTO_RESCU_LIST_URL, + TORONTO_RESCU_IMAGE_ORIGIN, + DEFAULT_TORONTO_MAX_SOURCES, + TORONTO_DOWNTOWN, + TORONTO_GROUND_ELEVATION_M, + TORONTO_MAX_CATALOG_BYTES, CCTV_SOURCE_FETCH_TIMEOUT_MS, DELDOT_CCTV_URL, DEFAULT_DELDOT_MAX_SOURCES, @@ -83,12 +89,16 @@ import { isLikelyNswCoordinate, isLikelyCalgaryCoordinate, isLikelyNorwayCoordinate, + isLikelyTorontoCoordinate, cameraDisplayCode, rowArrayToObject, prioritizeSources, } from './normalize.js'; import { directionToHeading } from '../../../src/data/directionText.js'; -import { readResponseJsonCapped } from '../common/http.js'; +import { + readResponseJsonCapped, + readResponseTextCapped, +} from '../common/http.js'; /** * Fetch and parse Austin traffic camera records from the city Open Data portal. * @@ -1863,3 +1873,172 @@ export async function loadVegvesenSourcesFromOpenData() { return []; } } + +/** + * The JSONP envelope the City of Toronto wraps its camera list in. Anchored at + * both ends, so only that exact callback is unwrapped — a payload that merely + * contains the name somewhere is not treated as the list. + */ +const RESCU_JSONP = /^jsonTMCEarthCamerasCallback\(([\s\S]*)\)\s*;?$/; + +/** + * Unwrap the RESCU camera list. The portal serves the list as JSONP; should it + * ever drop the wrapper, the bare JSON document still parses. Anything else — + * a foreign callback, a truncated body, an error page — yields no cameras + * rather than a thrown loader. + * + * @param {string} text - Raw response body. + * @returns {Array} The `Data` rows, or [] when the body is unusable. + */ +export function parseRescuCameraList(text) { + const body = String(text ?? '').trim(); + if (!body) return []; + const unwrapped = RESCU_JSONP.exec(body)?.[1] ?? body; + try { + const parsed = JSON.parse(unwrapped); + const rows = parsed?.Data ?? parsed?.data; + return Array.isArray(rows) ? rows : []; + } catch { + return []; + } +} + +/** + * Convert one RESCU list row into a camera source. + * + * The row carries no URL: the frame URL is built from the documented + * `loc####.jpg` convention and a camera number validated as plain digits, so + * the frame proxy is pinned to the city origin by construction and no upstream + * field — present or planted — can steer it. + * + * No compass heading exists in this dataset. `D1`-`D4` are the directions of + * the *comparison* images (static reference stills captured in 2016, served + * from a different path); the live frame is one pan-tilt-zoom camera pointing + * wherever the operator left it. Treating those letters as a facing would put + * a confident, wrong bearing on every camera in the city, so every row takes + * the id-hash fallback and the low-confidence pose, as headingless Austin, + * TfL, Fintraffic and Calgary cameras do. + * + * @param {object} row - One entry from the RESCU `Data` array. + * @returns {object|null} Normalized camera source, or null when unusable. + */ +export function torontoCameraToSource(row) { + if (!row || typeof row !== 'object') return null; + const number = String(row.Number ?? row.number ?? '').trim(); + // Plain digits only: a number is about to become a URL path segment. + if (!/^\d{1,6}$/.test(number)) return null; + + const lat = toFiniteNumber(row.Latitude ?? row.latitude); + const lon = toFiniteNumber(row.Longitude ?? row.longitude); + if (!isLikelyTorontoCoordinate(lat, lon)) return null; + + const cameraId = `toronto-${number}`; + // The feed's own label, verbatim. It is upper-case ("MCCOWAN RD", "ST + // HELEN'S AVE"); title-casing it would mangle more names than it fixed. + const name = + String(row.Name ?? row.name ?? '') + .replace(/\s+/g, ' ') + .trim() || `Toronto Camera ${number}`; + + return { + id: cameraId, + name, + city: 'Toronto', + cityId: 'toronto', + provider: 'City of Toronto', + lat, + lon, + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + groundElevationM: TORONTO_GROUND_ELEVATION_M, + feedType: 'image', + url: `${TORONTO_RESCU_IMAGE_ORIGIN}loc${number}.jpg`, + snapshotUrl: `${TORONTO_RESCU_IMAGE_ORIGIN}loc${number}.jpg`, + sourceKind: 'toronto-rescu-open-data', + license: + 'Contains information licensed under the Open Government Licence – Toronto', + // Unselected-label code: the intersection, so a camera at rest reads as a + // place rather than as its id. + code: cameraDisplayCode(name.toUpperCase()), + }; +} + +/** + * Fetch City of Toronto RESCU traffic cameras from the City's open data + * portal, keyless. One JSONP list for the whole city; frames are stills on the + * same portal origin. + * + * The list fetch refuses redirects (`redirect: 'manual'`) so the list host + * cannot be steered, and caps the body it will buffer. + * + * Attribution: Open Government Licence – Toronto, registered in + * src/data/dataCredits.js. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadTorontoSourcesFromOpenData() { + try { + const endpoint = + process.env.CCTV_TORONTO_LIST_URL || TORONTO_RESCU_LIST_URL; + const resp = await fetch(endpoint, { + headers: { Accept: 'application/json, text/javascript' }, + redirect: 'manual', + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + // A response this loader will not read still owns its transport until the + // body is released, so every rejection path cancels before returning. + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn( + '[CCTV] Toronto camera list redirected; redirects are not followed', + ); + return discard(); + } + if (!resp.ok) { + console.warn('[CCTV] Toronto camera download failed:', resp.status); + return discard(); + } + const text = await readResponseTextCapped(resp, TORONTO_MAX_CATALOG_BYTES); + const rows = parseRescuCameraList(text); + if (!rows.length) return []; + + const cameras = []; + const seen = new Set(); + for (const record of rows) { + const camera = torontoCameraToSource(record); + if (!camera || seen.has(camera.id)) continue; + seen.add(camera.id); + cameras.push(camera); + } + const maxRaw = Number( + process.env.CCTV_TORONTO_MAX_SOURCES || DEFAULT_TORONTO_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(500, Math.floor(maxRaw))) + : DEFAULT_TORONTO_MAX_SOURCES; + const prioritized = prioritizeSources(cameras, maxCount, [ + TORONTO_DOWNTOWN, + ]); + console.log( + `[CCTV] Loaded Toronto camera sources: ${cameras.length} (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn( + '[CCTV] Toronto camera download error:', + error?.message || error, + ); + return []; + } +} diff --git a/src/data/cctvToronto.test.mjs b/src/data/cctvToronto.test.mjs new file mode 100644 index 000000000..c2e3c0e1d --- /dev/null +++ b/src/data/cctvToronto.test.mjs @@ -0,0 +1,423 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + parseRescuCameraList, + torontoCameraToSource, + loadTorontoSourcesFromOpenData, +} from '../../server/providers/cctv/sources.js'; +import { + TORONTO_RESCU_LIST_URL, + TORONTO_RESCU_IMAGE_ORIGIN, + TORONTO_MAX_CATALOG_BYTES, + DEFAULT_CCTV_MAX_SOURCES, +} from '../../server/providers/cctv/constants.js'; +import { CAMERA_CODE_MAX_CHARS } from '../../server/providers/cctv/normalize.js'; +import { allocateSourceCap } from '../../server/providers/cctv/cap.js'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; + +/** + * A response whose body is a live stream, plus a flag that flips when the + * stream is cancelled. A rejection path that returns without cancelling holds + * the transport open, so the flag is what the refusal tests actually assert. + */ +const streamingResponse = (init = {}) => { + const state = { cancelled: false }; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('j')); + }, + cancel() { + state.cancelled = true; + }, + }); + return { response: new Response(body, init), state }; +}; + +/** One RESCU row, shaped like the live `tmcearthcameras.json` payload. */ +const row = (overrides = {}) => ({ + Number: '8048', + Name: 'BATHURST ST & BLOOR ST W', + Latitude: '43.665249', + Longitude: '-79.41114', + D1: 'n', + D2: 'e', + D3: 's', + D4: 'w', + Group: 'Arterial', + ...overrides, +}); + +/** The city wraps the list in a JSONP callback; this is that envelope. */ +const jsonp = (rows) => + `jsonTMCEarthCamerasCallback(${JSON.stringify({ Data: rows })});`; + +test('a RESCU row maps to a source on the constructed frame URL', () => { + const source = torontoCameraToSource(row()); + assert.equal(source.id, 'toronto-8048'); + assert.equal(source.name, 'BATHURST ST & BLOOR ST W'); + assert.equal(source.city, 'Toronto'); + assert.equal(source.cityId, 'toronto'); + assert.equal(source.provider, 'City of Toronto'); + assert.equal(source.lat, 43.665249); + assert.equal(source.lon, -79.41114); + assert.equal(source.feedType, 'image'); + assert.equal(source.sourceKind, 'toronto-rescu-open-data'); + assert.equal( + source.license, + 'Contains information licensed under the Open Government Licence – Toronto', + ); + assert.equal( + source.url, + `${TORONTO_RESCU_IMAGE_ORIGIN}loc8048.jpg`, + 'the documented loc####.jpg convention', + ); + assert.equal(source.url, source.snapshotUrl); + assert.ok(source.url.startsWith(TORONTO_RESCU_IMAGE_ORIGIN)); +}); + +test('the frame URL is built from the camera number, never read from the payload', () => { + // The list carries no URL field at all, so an added one must stay inert: the + // frame proxy is pinned to the city origin by construction. + const planted = torontoCameraToSource( + row({ + url: 'https://evil.example/x.jpg', + snapshotUrl: 'https://evil.example/x.jpg', + Url: 'https://evil.example/x.jpg', + }), + ); + assert.equal(planted.url, `${TORONTO_RESCU_IMAGE_ORIGIN}loc8048.jpg`); + assert.equal(planted.snapshotUrl, planted.url); + + // A number that is not plain digits cannot reach the URL builder: path + // traversal and absolute URLs are refused outright rather than escaped. + for (const Number_ of [ + '../../etc/passwd', + '8048/../../x', + 'https://evil.example/1', + '80 48', + '8048.jpg', + '-8048', + '', + ' ', + null, + ]) { + assert.equal( + torontoCameraToSource(row({ Number: Number_ })), + null, + `refused: ${String(Number_)}`, + ); + } +}); + +test('the comparison directions never become a camera heading', () => { + // D1-D4 are the city's *comparison image* directions (static 2016 reference + // stills served from ComparisonImages/), not a facing for the live frame: + // these cameras pan. The letters read like a compass and map trivially onto + // one (n -> 0, w -> 270), so the temptation to wire them up is real — and a + // heading derived from them would be confidently wrong for every camera in + // the city. Nobody may "fix" this. + const base = torontoCameraToSource(row()); + for (const dirs of [ + { D1: 'n', D2: 'e', D3: 's', D4: 'w' }, + { D1: '', D2: 'e', D3: '', D4: 'w' }, + { D1: '', D2: '', D3: '', D4: '' }, + { D1: 's', D2: '', D3: '', D4: '' }, + ]) { + const source = torontoCameraToSource(row(dirs)); + assert.equal(source.headingConfidence, 'low'); + assert.ok(Number.isFinite(source.headingDeg)); + // Same id, same fallback heading, whatever the directions say. + assert.equal(source.headingDeg, base.headingDeg); + } +}); + +test('the two cameras at one intersection are kept apart', () => { + // 8047 and 8048 are both "BATHURST ST & BLOOR ST W", five metres apart. They + // are distinct units, so both survive, and the id-hash fallback fans their + // gizmos apart instead of stacking them on one bearing. + const a = torontoCameraToSource( + row({ Number: '8047', Latitude: '43.665223', Longitude: '-79.411083' }), + ); + const b = torontoCameraToSource(row()); + assert.equal(a.id, 'toronto-8047'); + assert.equal(b.id, 'toronto-8048'); + assert.notEqual(a.url, b.url); + assert.notEqual(a.headingDeg, b.headingDeg); +}); + +test('rows outside Toronto and unusable geometry are dropped', () => { + // Calgary: a plausible lat/lon, but not a Toronto camera. + assert.equal( + torontoCameraToSource(row({ Latitude: '51.0453', Longitude: '-114.1413' })), + null, + ); + // Null island, and the empty strings that would become it. + assert.equal( + torontoCameraToSource(row({ Latitude: '0', Longitude: '0' })), + null, + ); + assert.equal( + torontoCameraToSource(row({ Latitude: '', Longitude: '' })), + null, + ); + assert.equal( + torontoCameraToSource(row({ Latitude: 'north', Longitude: 'west' })), + null, + ); + assert.equal(torontoCameraToSource({}), null); + assert.equal(torontoCameraToSource(null), null); +}); + +test('a nameless row still gets a label', () => { + const source = torontoCameraToSource(row({ Name: ' ' })); + assert.equal(source.name, 'Toronto Camera 8048'); +}); + +test('the JSONP envelope is unwrapped, and only that envelope', () => { + assert.deepEqual(parseRescuCameraList(jsonp([row()])), [row()]); + // Whitespace and a missing trailing semicolon are both tolerated. + assert.equal( + parseRescuCameraList(`\n ${jsonp([row()]).replace(/;$/, '')} \n`).length, + 1, + ); + // Should the city ever drop the wrapper, the bare document still parses. + assert.equal( + parseRescuCameraList(JSON.stringify({ Data: [row()] })).length, + 1, + ); + + // Anything else yields no cameras rather than a thrown loader. + for (const text of [ + 'otherCallback({"Data":[]})', + 'jsonTMCEarthCamerasCallback(', + 'jsonTMCEarthCamerasCallback({"Data":', + '{"Data":"not-an-array"}', + '{"nope":[]}', + '503', + '', + null, + ]) { + assert.deepEqual(parseRescuCameraList(text), [], `rejected: ${text}`); + } +}); + +test('the loader reads the keyless list and collapses duplicate numbers', async (t) => { + t.mock.method(console, 'log', () => {}); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + return new Response( + jsonp([ + row(), + // Same camera number twice: one camera, not two. + row({ Name: 'BATHURST ST & BLOOR ST W (duplicate)' }), + row({ + Number: '8001', + Name: 'YORK ST & BREMNER BLVD / RAPTORS WAY', + Latitude: '43.643120', + Longitude: '-79.381386', + }), + row({ Number: '../evil' }), + ]), + ); + }); + const cameras = await loadTorontoSourcesFromOpenData(); + assert.deepEqual(requested, [TORONTO_RESCU_LIST_URL]); + assert.deepEqual( + cameras.map((camera) => camera.id), + // Nearest downtown first: York & Bremner is at the core, Bathurst & Bloor + // is three kilometres out. + ['toronto-8001', 'toronto-8048'], + ); + assert.equal(cameras[1].name, 'BATHURST ST & BLOOR ST W'); +}); + +test('an upstream failure yields an empty pack and releases the response', async (t) => { + t.mock.method(console, 'warn', () => {}); + // A 503 can still arrive with a streaming body; returning without cancelling + // it would hold the connection until the socket times out. + const failed = streamingResponse({ status: 503 }); + t.mock.method(globalThis, 'fetch', async () => failed.response); + assert.deepEqual(await loadTorontoSourcesFromOpenData(), []); + assert.equal(failed.state.cancelled, true, 'the failed body is cancelled'); + + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + throw new Error('network down'); + }); + assert.deepEqual(await loadTorontoSourcesFromOpenData(), []); +}); + +test('the list fetch refuses redirects and oversized bodies', async (t) => { + t.mock.method(console, 'warn', () => {}); + // A redirect is never followed: the list host cannot be steered. Its body is + // a live stream, so the test also proves the refusal releases the transport. + const seen = []; + const redirected = streamingResponse({ + status: 302, + headers: { location: 'https://evil.example/cameras.json' }, + }); + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init.redirect]); + return redirected.response; + }); + assert.deepEqual(await loadTorontoSourcesFromOpenData(), []); + assert.deepEqual(seen, [[TORONTO_RESCU_LIST_URL, 'manual']]); + assert.equal( + redirected.state.cancelled, + true, + 'the redirect body is cancelled', + ); + + // A body over the cap is refused rather than buffered. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + return new Response(jsonp([row()]), { + headers: { + 'content-length': String(TORONTO_MAX_CATALOG_BYTES + 1), + }, + }); + }); + assert.deepEqual(await loadTorontoSourcesFromOpenData(), []); + + // So is a body that only declares its size once it is already too long. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + const oversized = 'x'.repeat(TORONTO_MAX_CATALOG_BYTES + 1024); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(oversized)); + controller.close(); + }, + }), + ); + }); + assert.deepEqual(await loadTorontoSourcesFromOpenData(), []); +}); + +test('the unselected label is the intersection, trimmed to the code width', () => { + assert.equal(torontoCameraToSource(row()).code, 'BATHURST ST & BLOOR ST W'); + const long = torontoCameraToSource( + row({ + Name: 'BAYVIEW AVE & POTTERY RD / PRIVATE ACCESS @ 620m SOUTH OF - 550 BAYVIEW AVE (EVERGREEN)', + }), + ); + assert.ok(long.code.length <= CAMERA_CODE_MAX_CHARS); + assert.ok(long.code.endsWith('…')); + assert.ok(long.code.startsWith('BAYVIEW AVE & POTTERY')); +}); + +test('a reduced catalog cap thins every pack instead of dropping Toronto', () => { + // Toronto merges last in LIVE_PACKS, so a positional slice would delete it + // outright. The round-robin allocation must give it its share. + const lane = (name, count) => ({ + name, + sources: Array.from({ length: count }, (_, i) => ({ id: `${name}-${i}` })), + }); + const { sources, packs } = allocateSourceCap( + [lane('austin', 250), lane('calgary', 215), lane('toronto', 336)], + 30, + ); + assert.equal(sources.length, 30); + assert.deepEqual( + packs.map((p) => [p.name, p.kept]), + [ + ['austin', 10], + ['calgary', 10], + ['toronto', 10], + ], + ); + // Toronto contributes its own highest-priority cameras, in its own order. + assert.deepEqual( + sources.filter((s) => s.id.startsWith('toronto-')).map((s) => s.id), + Array.from({ length: 10 }, (_, i) => `toronto-${i}`), + ); +}); + +/** + * Serve the RESCU list to the Toronto endpoint and an empty payload to every + * other pack, so one catalog refresh exercises the registration without + * reaching the network. Returns the URLs that were requested. + */ +const runCatalogWithMockedUpstreams = async (t) => { + const requested = []; + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async (url) => { + const href = String(url); + requested.push(href); + if (href === TORONTO_RESCU_LIST_URL) { + return new Response( + jsonp([ + row({ + Number: '8001', + Name: 'YORK ST & BREMNER BLVD / RAPTORS WAY', + Latitude: '43.643120', + Longitude: '-79.381386', + }), + row(), + ]), + ); + } + return Response.json([]); + }); + const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); + return { requested, sources }; +}; + +test('the pack is registered, so its cameras reach the served catalog', async (t) => { + const saved = { ...process.env }; + try { + delete process.env.CCTV_SOURCES_FILE; + delete process.env.CCTV_SOURCES_JSON; + delete process.env.CCTV_TORONTO_ENABLED; + const { requested, sources } = await runCatalogWithMockedUpstreams(t); + assert.ok( + requested.includes(TORONTO_RESCU_LIST_URL), + 'the registered lane reaches its upstream', + ); + assert.deepEqual( + sources.filter((s) => s.cityId === 'toronto').map((s) => s.id), + ['toronto-8001', 'toronto-8048'], + 'Toronto cameras reach the served catalog through the registered lane', + ); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + } +}); + +test('CCTV_TORONTO_ENABLED=0 keeps the lane from being loaded at all', async (t) => { + const saved = { ...process.env }; + try { + delete process.env.CCTV_SOURCES_FILE; + delete process.env.CCTV_SOURCES_JSON; + process.env.CCTV_TORONTO_ENABLED = '0'; + const { requested, sources } = await runCatalogWithMockedUpstreams(t); + assert.equal( + requested.includes(TORONTO_RESCU_LIST_URL), + false, + 'the disabled lane never reaches its upstream', + ); + assert.deepEqual( + sources.filter((s) => s.cityId === 'toronto'), + [], + ); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + } +}); + +test('the shipped catalog ceiling is not raised to make room for this pack', () => { + assert.equal(DEFAULT_CCTV_MAX_SOURCES, 4000); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 2967f0a2b..d6f2f3ad4 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -247,6 +247,12 @@ export const DATA_CREDITS = [ 'Norwegian licence for Open Government data (NLOD) ' + 'distributed by Statens vegvesen', }, + { + key: 'toronto-cctv', + html: + 'Traffic cameras (Toronto): contains information licensed under the ' + + 'Open Government Licence – Toronto', + }, { key: 'gbfs', html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)', From 7bdeff06a937550046c1e1b66aa976542ad1b565 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 19:22:18 -0400 Subject: [PATCH 42/95] fix(cctv): base the Toronto ground prior on the city's real relief MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pack shipped a flat 90 m. Sampling a DEM at all 336 camera sites puts the network between 76 m (Queens Quay) and 209 m (Keele & Steeles), mean 116 m: a median error of 14 m against the old value, 88 m at the 90th percentile and 119 m at worst, with the Allen Road cameras — 170-197 m — the furthest underground. No constant fits a city with that much relief; the precomputed ground heights are what place a camera properly, and until this pack has them the prior is the fallback for a sidecar miss or an edited pose. The mean is at least wrong in both directions rather than only one. This value is not part of the served pose, so it changes no pose hash and invalidates no precomputed entry. --- server/providers/cctv/constants.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 6f896be59..9c817a02b 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -298,10 +298,12 @@ export const TORONTO_RESCU_IMAGE_ORIGIN = export const DEFAULT_TORONTO_MAX_SOURCES = 336; /** Yonge / Queen: the prioritization anchor. */ export const TORONTO_DOWNTOWN = { lat: 43.6532, lon: -79.3832 }; -/** Toronto's lakeshore core sits near 76 m; the ravines and the north of the - * city run higher. This prior only has to be close — the client's one-shot - * ground snap corrects it wherever 3D tiles are loaded. */ -export const TORONTO_GROUND_ELEVATION_M = 90; +/** The city climbs from 76 m at the lake to 209 m at Steeles, so no single + * number fits: this is the mean across the 336 camera sites. Placement comes + * from the precomputed ground-height sidecar wherever a camera has an entry; + * this prior is only the fallback for a sidecar miss or an edited pose, and + * the client's ground snap corrects it wherever 3D tiles are loaded. */ +export const TORONTO_GROUND_ELEVATION_M = 116; /** Hard ceiling on the RESCU list body. The whole city is ~336 rows and under * 60 KB; this only exists so an upstream that streams an unbounded body cannot * be buffered without limit. */ From fc2c372ffcc7cbb67ecda42bd8e27a08be719ca5 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Thu, 1 Oct 2026 15:01:32 -0400 Subject: [PATCH 43/95] fix: keep install assets inside a non-root base Manifest start_url, scope and icon src resolve against the manifest URL, so root-relative values sent an app served under a path prefix to the domain root. Make them relative, and add a build test under base: '/example/' that asserts every emitted link and manifest URL stays inside the base. index.html is unchanged: Vite already rewrites its links under base. --- CHANGELOG.md | 2 + docs/CURRENT-STATE.md | 6 +++ public/manifest.webmanifest | 10 ++-- src/tooling/webAppIcons.test.mjs | 85 ++++++++++++++++++++++++++++++-- 4 files changed, 95 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb61ad1db..0c07f3c64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ instead of a blank tile. The icons are generated from `public/logo.svg` by `node scripts/generate-icons.mjs` and committed, so a plain `vite build` needs no image toolchain. The existing SVG favicon still serves the browser tab. + Manifest paths are relative to the manifest, so a build served under a path + prefix installs and launches inside that prefix. - GBFS v3 station names use English translations when available, then the first usable translation or short name. Empty or malformed names no longer appear as diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index f8e6bff3a..3d51bad2a 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1140,12 +1140,18 @@ favicon; the existing SVG favicon still serves the browser tab. Safari ignores a SVG apple-touch-icon and substitutes a page screenshot, so the raster is what makes an installed icon appear at all. +Manifest `start_url`, `scope` and icon paths are relative to the manifest, and +Vite rewrites the `index.html` links under `base`, so a build served from a path +prefix installs and launches inside that prefix. + `node scripts/generate-icons.mjs` renders every raster from `public/logo.svg` and writes `public/icons/`. Output is committed, so building needs no image toolchain; re-run the script after changing the logo. Ordinary icons inset the mark to 78% of the canvas and the maskable variant to 54%, keeping it inside the circle Android may crop to. `src/tooling/webAppIcons.test.mjs` checks the links, the manifest fields and that each committed PNG is square at its declared size. +It also builds the install links under `base: '/example/'` and asserts every +emitted link and manifest URL stays inside that base. ## Local build preview diff --git a/public/manifest.webmanifest b/public/manifest.webmanifest index 574b90faf..cf4b2b791 100644 --- a/public/manifest.webmanifest +++ b/public/manifest.webmanifest @@ -2,17 +2,17 @@ "name": "God's Eye View", "short_name": "God's Eye", "description": "Live spatial intelligence in the browser.", - "start_url": "/", - "scope": "/", + "start_url": ".", + "scope": ".", "display": "standalone", "orientation": "any", "background_color": "#0a0a0f", "theme_color": "#0a0a0f", "icons": [ - { "src": "/icons/icon-192.png", "sizes": "192x192", "type": "image/png" }, - { "src": "/icons/icon-512.png", "sizes": "512x512", "type": "image/png" }, + { "src": "icons/icon-192.png", "sizes": "192x192", "type": "image/png" }, + { "src": "icons/icon-512.png", "sizes": "512x512", "type": "image/png" }, { - "src": "/icons/icon-512-maskable.png", + "src": "icons/icon-512-maskable.png", "sizes": "512x512", "type": "image/png", "purpose": "maskable" diff --git a/src/tooling/webAppIcons.test.mjs b/src/tooling/webAppIcons.test.mjs index 2267872fe..6a6cdb350 100644 --- a/src/tooling/webAppIcons.test.mjs +++ b/src/tooling/webAppIcons.test.mjs @@ -1,9 +1,12 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync } from 'node:fs'; +import { rm, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import sharp from 'sharp'; +import { build } from 'vite'; +import { makeFixtureRoot } from './fixtureRoot.mjs'; const REPO_ROOT = path.resolve( fileURLToPath(new URL('.', import.meta.url)), @@ -12,7 +15,7 @@ const REPO_ROOT = path.resolve( const PUBLIC_ROOT = path.join(REPO_ROOT, 'public'); const INDEX_HTML = readFileSync(path.join(REPO_ROOT, 'index.html'), 'utf8'); -/** `public/` is served at the site root, so a manifest href maps to a file by dropping the leading slash. */ +/** `public/` is served at the site root, so an index.html href maps to a file by dropping the leading slash. */ function publicFile(href) { assert.ok(href.startsWith('/'), `${href} must be a root-relative URL`); return path.join(PUBLIC_ROOT, href.slice(1)); @@ -22,6 +25,18 @@ const manifestHref = INDEX_HTML.match( /]+rel="manifest"[^>]+href="([^"]+)"/, )?.[1]; +/** + * Manifest URLs resolve against the manifest, not the page. Keeping them + * relative is what lets a build served under a path prefix install inside it. + */ +function manifestFile(src) { + assert.ok( + !src.startsWith('/') && !/^[a-z][a-z\d+.-]*:/i.test(src), + `${src} must be relative to the manifest`, + ); + return path.join(path.dirname(publicFile(manifestHref)), src); +} + test('index.html links a web app manifest', () => { assert.ok(manifestHref, 'no in index.html'); }); @@ -41,7 +56,8 @@ test('iOS gets a raster home-screen icon', () => { test('the manifest declares a standalone installable app', () => { const manifest = JSON.parse(readFileSync(publicFile(manifestHref), 'utf8')); assert.equal(manifest.display, 'standalone'); - assert.equal(manifest.start_url, '/'); + assert.equal(manifest.start_url, '.'); + assert.equal(manifest.scope, '.'); for (const field of ['name', 'short_name', 'background_color', 'theme_color']) assert.ok(manifest[field], `manifest is missing ${field}`); // Android's install prompt needs both, and uses the 512 for the splash screen. @@ -59,7 +75,7 @@ test('the manifest declares a standalone installable app', () => { test('every declared icon is a square PNG at its declared size', async () => { const manifest = JSON.parse(readFileSync(publicFile(manifestHref), 'utf8')); for (const icon of manifest.icons) { - const meta = await sharp(publicFile(icon.src)).metadata(); + const meta = await sharp(manifestFile(icon.src)).metadata(); const [width, height] = icon.sizes.split('x').map(Number); assert.equal(meta.format, 'png', `${icon.src} is not a PNG`); // A size the manifest overstates is worse than a missing one: the platform @@ -68,3 +84,66 @@ test('every declared icon is a square PNG at its declared size', async () => { assert.equal(meta.height, height, `${icon.src} height`); } }); + +test('install assets stay inside a non-root Vite base', async (t) => { + // The real index.html pulls the whole Cesium app into a build, so only its + // install links are copied, verbatim, into a fixture page. + const tags = INDEX_HTML.match( + /]+rel="(?:icon|apple-touch-icon|manifest)"[^>]*>/g, + ); + assert.ok(tags?.length >= 3, 'index.html lost its install links'); + const root = await makeFixtureRoot('gev-webapp-icons-'); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile( + path.join(root, 'index.html'), + `${tags.join('')}`, + ); + const base = '/example/'; + const origin = 'https://host.test'; + const result = await build({ + root, + configFile: false, + envFile: false, + logLevel: 'silent', + base, + publicDir: PUBLIC_ROOT, + build: { write: false }, + }); + const html = result.output.find( + (item) => item.fileName === 'index.html', + ).source; + const hrefs = [...html.matchAll(/]+href="([^"]+)"/g)].map( + (match) => match[1], + ); + assert.equal(hrefs.length, tags.length); + for (const href of hrefs) { + assert.ok(href.startsWith(base), `${href} escapes ${base}`); + assert.doesNotThrow(() => + readFileSync(path.join(PUBLIC_ROOT, href.slice(base.length))), + ); + } + + // Vite copies public/ only when it writes, and copies it verbatim, so the + // deployed manifest is the committed one, served at the emitted href. + const manifestUrl = new URL( + html.match(/]+rel="manifest"[^>]+href="([^"]+)"/)[1], + origin, + ); + const manifest = JSON.parse(readFileSync(publicFile(manifestHref), 'utf8')); + const inBase = (value) => { + const url = new URL(value, manifestUrl); + assert.ok( + url.href.startsWith(origin + base), + `${value} resolves to ${url.href}, outside ${base}`, + ); + return url; + }; + assert.equal(inBase(manifest.start_url).href, origin + base); + assert.equal(inBase(manifest.scope).href, origin + base); + for (const icon of manifest.icons) { + const url = inBase(icon.src); + assert.doesNotThrow(() => + readFileSync(path.join(PUBLIC_ROOT, url.pathname.slice(base.length))), + ); + } +}); From b8ccfd1648755d31d18644cbeec9d7768ffedb0a Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 20:22:45 -0400 Subject: [PATCH 44/95] feat(bikeshare): add Bike Share Toronto MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Toronto runs the same PBSC platform as the Austin, Honolulu and Chattanooga systems already in the registry, and publishes the identical GBFS v2 endpoint pair under `toronto.publicbikesystem.net`. The proxy allowlist already admits the host through its `*.publicbikesystem.net` wildcard and the paths end in the expected `station_information.json` / `station_status.json`, so the system needs no new plumbing — only the registry entry that names it. Verified against the live feeds: 1,069 stations in both documents, all 1,069 joining on `station_id`. The layer reads `num_bikes_available`, which Toronto populates normally, so availability renders without a vendor-specific mapping. It does not populate the standard `num_bikes_available_types`, reporting per-type counts under PBSC ids (ICONIC/BOOST/EFIT/ASTRO) instead; nothing reads that field today, and this entry deliberately does not translate it rather than adding a mapping with no consumer. The center is downtown at 43.6532, -79.3832 with the group's 90 km load radius. The stations span roughly 37 km east-west, from Etobicoke to Scarborough, so a tighter radius would drop the outer ends on a downtown pan. The new registry test is the reason this was cheap to be sure about: it holds every city's feed URLs against the same host and path predicates the proxy enforces, so a city the proxy would silently refuse now fails the suite instead of rendering an empty layer. Credited to Bike Share Toronto (Toronto Parking Authority). The City's open-data package declares no licence and the feed publishes no `license_url`, so it is treated as attribution-only, like the other GBFS operators, rather than claiming a licence that was never granted. --- CHANGELOG.md | 6 +++ DATA_SOURCES.md | 1 + docs/CURRENT-STATE.md | 2 +- src/data/dataCredits.js | 4 +- src/layers/bikeshare/registry.js | 12 ++++++ src/layers/bikeshare/registry.test.mjs | 58 ++++++++++++++++++++++++++ 6 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 src/layers/bikeshare/registry.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 032c6fe2a..98e0066f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +- Add Bike Share Toronto (1,069 stations) to the bikeshare registry. The system + publishes the same PBSC GBFS v2 feeds as the Austin, Honolulu and Chattanooga + entries, so it needs no new plumbing; a registry test now holds every city's + feed URLs against the proxy host and path allowlists, so a city the proxy + would refuse fails the suite instead of rendering an empty layer. + - GBFS v3 station names use English translations when available, then the first usable translation or short name. Empty or malformed names no longer appear as `[object Object]` or interrupt station loading; legacy string names remain diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 040e53afc..3e1e0f114 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -58,6 +58,7 @@ How to read this: | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | +| **Bike Share Toronto** (`toronto.publicbikesystem.net`) | Bikeshare station availability, Toronto | Operator GBFS feed. The City's open-data package declares no licence and the feed publishes no `license_url`, so it is treated as attribution-only like the other GBFS operators. | "Bike Share Toronto (Toronto Parking Authority)" | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | | **Metro Transit** (Metropolitan Council) | Transit layer live vehicles, Minneapolis–St Paul | Metro Transit / Metropolitan Council open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to Metro Transit / Metropolitan Council (courtesy text) | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 0d3ef31e0..550d76d57 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3074,7 +3074,7 @@ its criteria cannot be silently ignored. | CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Bike Share Toronto) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | | Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 2967f0a2b..2277abf4a 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -249,7 +249,9 @@ export const DATA_CREDITS = [ }, { key: 'gbfs', - html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)', + html: + 'Bikeshare availability: GBFS operator feeds ' + + '(e.g. Austin BCycle; Bike Share Toronto, operated by the Toronto Parking Authority)', }, { key: 'osrm-routing', diff --git a/src/layers/bikeshare/registry.js b/src/layers/bikeshare/registry.js index ff752f664..e9096b4a9 100644 --- a/src/layers/bikeshare/registry.js +++ b/src/layers/bikeshare/registry.js @@ -167,6 +167,18 @@ const RAW_GBFS_CITY_REGISTRY = [ 'https://chat.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', provider: 'Bike Chattanooga', }, + { + id: 'toronto-bike-share', + city: 'Toronto, ON', + centerLat: 43.6532, + centerLon: -79.3832, + loadRadiusKm: 90, + stationInformationUrl: + 'https://toronto.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://toronto.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Bike Share Toronto', + }, bcycleEntry({ id: 'boulder-bcycle', city: 'Boulder, CO', diff --git a/src/layers/bikeshare/registry.test.mjs b/src/layers/bikeshare/registry.test.mjs new file mode 100644 index 000000000..30e708957 --- /dev/null +++ b/src/layers/bikeshare/registry.test.mjs @@ -0,0 +1,58 @@ +// BIKESHARE REGISTRY — a city entry is only useful if the proxy will serve it. +// +// A registry entry and the GBFS proxy allowlist are edited in different files, +// so a city can look correctly registered while every one of its requests is +// rejected at the proxy. These cases hold the two together: each entry is +// checked against the same host and path predicates the proxy enforces, so a +// new city that the proxy would refuse fails here rather than silently +// rendering an empty layer. +// +// Run with: npm test +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { GBFS_CITY_REGISTRY, CITY_BY_ID } from './registry.js'; +import { isAllowedGbfsHost, isAllowedGbfsPath } from '../../data/gbfsSource.js'; + +test('every registered city is reachable through the GBFS proxy allowlist', () => { + for (const city of GBFS_CITY_REGISTRY) { + for (const raw of [city.stationInformationUrl, city.stationStatusUrl]) { + const url = new URL(raw); + assert.equal( + isAllowedGbfsHost(url.hostname), + true, + `${city.id}: host ${url.hostname} is not allowlisted`, + ); + assert.equal( + isAllowedGbfsPath(url.pathname), + true, + `${city.id}: path ${url.pathname} is not allowlisted`, + ); + } + } +}); + +test('Bike Share Toronto is registered against the PBSC v2 feeds', () => { + const toronto = CITY_BY_ID.get('toronto-bike-share'); + assert.ok(toronto, 'toronto-bike-share is missing from the registry'); + assert.equal(toronto.city, 'Toronto, ON'); + assert.equal(toronto.provider, 'Bike Share Toronto'); + assert.equal( + toronto.stationInformationUrl, + 'https://toronto.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + ); + assert.equal( + toronto.stationStatusUrl, + 'https://toronto.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + ); +}); + +test('the Toronto center sits inside the system service area', () => { + const toronto = CITY_BY_ID.get('toronto-bike-share'); + // Bounding box of the 1,069 live stations, sampled from the published + // station_information feed: 43.588..43.813 N, -79.603..-79.123 W. + assert.ok(toronto.centerLat > 43.588 && toronto.centerLat < 43.813); + assert.ok(toronto.centerLon > -79.603 && toronto.centerLon < -79.123); + // The system spans roughly 37 km east-west, so the load radius has to clear + // half that from the center or downtown pans lose the outer stations. + assert.ok(toronto.loadRadiusKm >= 25); +}); From 001043f74eb8db26cd9c9f41edd798dfbb2b180d Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 20:55:04 -0400 Subject: [PATCH 45/95] fix(bikeshare): let one dead city feed fail without failing the layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The layer recorded failure in a single `_error` string. Every city feed is fetched independently, so one retired operator set that flag for the whole layer — and the only code that cleared it ran on a successful *activation*, which `runProximityCheck` skips for a city that already has rendered points. A dead feed that re-entered camera range therefore set the flag on every camera move and nothing ever cleared it: a permanent LOAD FAILED over a map drawing every other city's stations. Failure is now tracked per city in `_failedCityIds`, and `getStats()` reports it only when the failures left nothing to draw. A city that recovers removes its own entry rather than waiting for a neighbour to succeed. This was reachable only because Buffalo sits 95 km from Toronto, inside the 100 km activation radius — the one dead city in the registry with a healthy neighbour in range. The other four failed alone, where LOAD FAILED with an empty map reads as correct, which is why the latch went unnoticed. Probing all 33 registry systems found five dead. Three had moved and are repointed: Reddy Bikeshare (Buffalo) to Social Bicycles, Tugo (Tucson) and Trinity Metro (Fort Worth) to PBSC, all verified serving stations. Two are dropped because the systems wound down rather than moved — Broward B-cycle 404s at every documented endpoint, and CoGo's replacement Lyft feed publishes zero stations with status last reported in March 2025. `gbfs.cogobikeshare.com` no longer resolves, so its allowlist entry goes too. All 31 remaining systems were verified live, and every registry URL was checked against the proxy host and path allowlists. --- CHANGELOG.md | 14 ++++ src/data/gbfsSource.js | 2 +- src/layers/bikeshare/controls.js | 9 ++- .../bikeshare/failureReporting.test.mjs | 80 +++++++++++++++++++ src/layers/bikeshare/ingestion.js | 3 +- src/layers/bikeshare/lifecycle.js | 4 +- src/layers/bikeshare/registry.js | 54 ++++++------- src/layers/bikeshare/state.js | 11 ++- src/layers/bikeshare/viewport.js | 4 +- src/layers/transit/lifecycle.test.mjs | 1 + 10 files changed, 144 insertions(+), 38 deletions(-) create mode 100644 src/layers/bikeshare/failureReporting.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 98e0066f6..344e32952 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +- Stop one retired bikeshare feed from reporting the whole layer as failed. + City feeds are fetched independently, but failure was recorded in a single + flag that only a successful *activation* cleared — and an already-loaded city + never activates again. A dead feed re-entering camera range therefore latched + LOAD FAILED permanently over a map still drawing every other city's stations. + Failure is now tracked per city and reported only when it leaves the layer + with nothing to draw; a city that recovers clears its own failure. + + Repointed the three feeds whose operators had moved: Reddy Bikeshare (Buffalo) + to Social Bicycles, Tugo (Tucson) and Trinity Metro (Fort Worth) to PBSC. + Dropped CoGo (Columbus) and Broward B-cycle (Fort Lauderdale), whose systems + have wound down — CoGo's feed reports no stations and last moved a bike in + March 2025. All 31 remaining systems were verified live. + - Add Bike Share Toronto (1,069 stations) to the bikeshare registry. The system publishes the same PBSC GBFS v2 feeds as the Austin, Honolulu and Chattanooga entries, so it needs no new plumbing; a registry test now holds every city's diff --git a/src/data/gbfsSource.js b/src/data/gbfsSource.js index a15a806d9..f0a94d789 100644 --- a/src/data/gbfsSource.js +++ b/src/data/gbfsSource.js @@ -4,7 +4,7 @@ const GBFS_ALLOWED_HOSTS = new Set([ 'gbfs.bluebikes.com', 'gbfs.bcycle.com', 'gbfs.biketownpdx.com', - 'gbfs.cogobikeshare.com', + 'reddybikeshare.socialbicycles.com', 'austin.publicbikesystem.net', 'hon.publicbikesystem.net', 'chat.publicbikesystem.net', diff --git a/src/layers/bikeshare/controls.js b/src/layers/bikeshare/controls.js index 8bbe70120..9e9389c58 100644 --- a/src/layers/bikeshare/controls.js +++ b/src/layers/bikeshare/controls.js @@ -70,7 +70,14 @@ export function createControls({ state: layerState, services, parts, source }) { ? `syncing ${layerState._activeCityIds.size} city feeds...` : 'scanning nearby systems...'; } - if (layerState._error) stats.error = layerState._error; + // A failed city is only the layer's failure when it left nothing on the + // map. While any city still renders, the layer is working and says so. + if ( + layerState._failedCityIds.size > 0 && + layerState._stationRenderMap.size === 0 + ) { + stats.error = 'GBFS fetch error'; + } return stats; }, }; diff --git a/src/layers/bikeshare/failureReporting.test.mjs b/src/layers/bikeshare/failureReporting.test.mjs new file mode 100644 index 000000000..63034078b --- /dev/null +++ b/src/layers/bikeshare/failureReporting.test.mjs @@ -0,0 +1,80 @@ +// BIKESHARE FAILURE REPORTING — one dead city is not a dead layer. +// +// City feeds are fetched independently, but the layer reported failure through +// a single flag. A city whose operator had retired its feed set that flag on +// every proximity check, and the only code that cleared it ran on a successful +// *activation* — which an already-loaded city never repeats. The result was a +// permanent LOAD FAILED over a map drawing a thousand live stations. +// +// These cases pin the rule that replaced it: the layer reports failure only +// when it has nothing to show, and a city that comes back clears its own +// failure rather than waiting for a neighbour to succeed. +// +// Run with: npm test +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createState } from './state.js'; +import { createControls } from './controls.js'; + +function makeLayer() { + const services = { + overlays: { + setOverlayEntries() {}, + setOverlaySourceVisible() {}, + clearOverlaySource() {}, + }, + }; + const state = createState({ services }); + const parts = { queries: { collectDetectableStations: () => [] } }; + const { methods } = createControls({ state, services, parts, source: {} }); + return { state, methods }; +} + +/** Stand in for a rendered station without reaching into the renderer. */ +function renderStations(state, count) { + for (let i = 0; i < count; i += 1) state._stationRenderMap.set(`s${i}`, {}); + state._count = state._stationRenderMap.size; +} + +test('a failed city does not fail a layer that is drawing stations', () => { + const { state, methods } = makeLayer(); + renderStations(state, 1069); + state._failedCityIds.add('buffalo-reddy'); + + const stats = methods.getStats(); + assert.equal(stats.count, 1069); + assert.equal( + stats.error, + undefined, + 'a layer drawing 1069 stations reported LOAD FAILED', + ); +}); + +test('the layer reports failure when a failed city left it with nothing', () => { + const { state, methods } = makeLayer(); + state._failedCityIds.add('buffalo-reddy'); + + const stats = methods.getStats(); + assert.equal(stats.count, 0); + assert.equal(stats.error, 'GBFS fetch error'); +}); + +test('a layer with nothing to draw and nothing failed is not an error', () => { + const { state, methods } = makeLayer(); + + // Out of range of every city is empty, not broken. + assert.equal(methods.getStats().error, undefined); +}); + +test('a recovered city clears its own failure', () => { + const { state, methods } = makeLayer(); + state._failedCityIds.add('buffalo-reddy'); + assert.equal(methods.getStats().error, 'GBFS fetch error'); + + state._failedCityIds.delete('buffalo-reddy'); + assert.equal( + methods.getStats().error, + undefined, + 'the failure outlived the city that caused it', + ); +}); diff --git a/src/layers/bikeshare/ingestion.js b/src/layers/bikeshare/ingestion.js index 732ec9c90..cda677766 100644 --- a/src/layers/bikeshare/ingestion.js +++ b/src/layers/bikeshare/ingestion.js @@ -182,13 +182,14 @@ export function createIngestion({ ) return; parts.rendering.applyStatusToPoints(cityId, statusMap); + layerState._failedCityIds.delete(cityId); } catch (error) { if (error?.name === 'AbortError') return; console.warn( `[Data:Bikeshare] ${cityId} status update error:`, error, ); - layerState._error = 'GBFS status update failed'; + layerState._failedCityIds.add(cityId); } }), ); diff --git a/src/layers/bikeshare/lifecycle.js b/src/layers/bikeshare/lifecycle.js index 17af85f11..3e46b5972 100644 --- a/src/layers/bikeshare/lifecycle.js +++ b/src/layers/bikeshare/lifecycle.js @@ -51,7 +51,7 @@ export function createLifecycle({ layerState._lastUpdate = null; layerState._loading = false; layerState._loadingOps = 0; - layerState._error = null; + layerState._failedCityIds.clear(); layerState._limitWarned = false; layerState._overlayHost.setVisible( @@ -75,7 +75,7 @@ export function createLifecycle({ */ enable(viewer) { layerState._enabled = true; - layerState._error = null; + layerState._failedCityIds.clear(); layerState._pointCollection.show = true; layerState._overlayHost.setVisible( BIKESHARE_SELECTED_OVERLAY_SOURCE_ID, diff --git a/src/layers/bikeshare/registry.js b/src/layers/bikeshare/registry.js index e9096b4a9..5cdf971ad 100644 --- a/src/layers/bikeshare/registry.js +++ b/src/layers/bikeshare/registry.js @@ -143,18 +143,6 @@ const RAW_GBFS_CITY_REGISTRY = [ 'https://hon.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', provider: 'Biki', }, - { - id: 'columbus-cogo', - city: 'Columbus, OH', - centerLat: 39.9612, - centerLon: -82.9988, - loadRadiusKm: 90, - stationInformationUrl: - 'https://gbfs.cogobikeshare.com/gbfs/2.3/en/station_information.json', - stationStatusUrl: - 'https://gbfs.cogobikeshare.com/gbfs/2.3/en/station_status.json', - provider: 'CoGo', - }, { id: 'chattanooga-bikechatt', city: 'Chattanooga, TN', @@ -246,14 +234,6 @@ const RAW_GBFS_CITY_REGISTRY = [ systemId: 'bcycle_pacersbikeshare', provider: 'Pacers Bikeshare', }), - bcycleEntry({ - id: 'fort-lauderdale-broward', - city: 'Fort Lauderdale, FL', - centerLat: 26.1224, - centerLon: -80.1373, - systemId: 'bcycle_broward', - provider: 'Broward B-cycle', - }), bcycleEntry({ id: 'memphis-bcycle', city: 'Memphis, TN', @@ -268,22 +248,32 @@ const RAW_GBFS_CITY_REGISTRY = [ centerLon: -93.625, systemId: 'bcycle_desmoines', }), - bcycleEntry({ + { + // Tugo moved off BCycle to PBSC; the bcycle_tugo feed is gone (404). id: 'tucson-tugo', city: 'Tucson, AZ', centerLat: 32.2226, centerLon: -110.9747, - systemId: 'bcycle_tugo', + loadRadiusKm: 100, + stationInformationUrl: + 'https://tucson.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://tucson.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', provider: 'Tugo', - }), - bcycleEntry({ + }, + { + // Trinity Metro moved off BCycle to PBSC; the bcycle_fortworth feed is gone. id: 'fort-worth-trinity', city: 'Fort Worth, TX', centerLat: 32.7555, centerLon: -97.3308, - systemId: 'bcycle_fortworth', + loadRadiusKm: 100, + stationInformationUrl: + 'https://fortworth.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://fortworth.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', provider: 'Trinity Metro', - }), + }, bcycleEntry({ id: 'omaha-heartland', city: 'Omaha, NE', @@ -307,14 +297,20 @@ const RAW_GBFS_CITY_REGISTRY = [ centerLon: -82.394, systemId: 'bcycle_greenville', }), - bcycleEntry({ + { + // Reddy left BCycle for Social Bicycles; the bcycle_reddy feed is gone + // (404). This is the endpoint the MobilityData GBFS registry lists. id: 'buffalo-reddy', city: 'Buffalo, NY', centerLat: 42.8864, centerLon: -78.8784, - systemId: 'bcycle_reddy', + loadRadiusKm: 100, + stationInformationUrl: + 'https://reddybikeshare.socialbicycles.com/opendata/station_information.json', + stationStatusUrl: + 'https://reddybikeshare.socialbicycles.com/opendata/station_status.json', provider: 'Reddy Bikeshare', - }), + }, bcycleEntry({ id: 'las-vegas-rtc-bike-share', city: 'Las Vegas, NV', diff --git a/src/layers/bikeshare/state.js b/src/layers/bikeshare/state.js index 381384147..728c1c124 100644 --- a/src/layers/bikeshare/state.js +++ b/src/layers/bikeshare/state.js @@ -99,9 +99,16 @@ export function createState({ services }) { layerState._loadingOps = 0; - /** Most recent error message string, or null. */ + /** + * @type {Set} City ids whose most recent fetch failed. + * + * Per city rather than one layer-wide flag: feeds are fetched independently, + * so a retired operator must not mark a layer that other cities are still + * populating as failed. The layer reports failure only when these cost it + * everything it had to draw — see `getStats()` in controls.js. + */ - layerState._error = null; + layerState._failedCityIds = new Set(); /** Whether the MAX_TOTAL_POINTS cap warning has already been logged. */ diff --git a/src/layers/bikeshare/viewport.js b/src/layers/bikeshare/viewport.js index e4c03a690..b4e63e1dd 100644 --- a/src/layers/bikeshare/viewport.js +++ b/src/layers/bikeshare/viewport.js @@ -168,11 +168,11 @@ export function createViewport({ state: layerState, services, parts, source }) { return; parts.rendering.applyStatusToPoints(cityId, statusMap); layerState._lastUpdate = Date.now(); - layerState._error = null; + layerState._failedCityIds.delete(cityId); } catch (error) { if (error?.name === 'AbortError') return; console.warn(`[Data:Bikeshare] ${cityId} activate error:`, error); - layerState._error = 'GBFS fetch error'; + layerState._failedCityIds.add(cityId); deactivateCity(cityId); layerState._activeCityIds.delete(cityId); } finally { diff --git a/src/layers/transit/lifecycle.test.mjs b/src/layers/transit/lifecycle.test.mjs index 795bbc654..8a2d39edf 100644 --- a/src/layers/transit/lifecycle.test.mjs +++ b/src/layers/transit/lifecycle.test.mjs @@ -3586,6 +3586,7 @@ test('actual Traffic, Bikeshare and Transit lifecycles retain sensitivity in eve _pointCollection: {}, _overlayHost: { setVisible: noop }, _cityRuntime: new Map(), + _failedCityIds: new Set(), }, services: { sprites: { restoreSpriteOrder: noop }, From 00d2b73b6af3cfd6cd56323226d86b33719e1f57 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Wed, 14 Oct 2026 13:10:40 +0100 Subject: [PATCH 46/95] feat(street-level): Panoramax as a fourth, keyless imagery provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Panoramax, the open federated street-level commons, registers beside Mapillary, Google Street View and KartaView. Its meta-catalogue (api.panoramax.xyz) is keyless and CORS-open, so it runs in the browser: - coverage from its vector tiles: sequence lines at z11–14 below 30 km (longest first under the per-tile cap), and above that the grid layer's density cells as binned points, not counted as sequences; - a click on a line opens that sequence's picture nearest the click (STAC search, any direction), without moving the camera, then any sequence's if it has none within 50 m; the 360° and since filters go to the lines and to the API; - cones from the z15 tiles' pictures layer around the open picture (~110 bytes a picture, against ~7.5 kB for a STAC item), re-centred when a picture opens outside them; ‹ › follow the picture's prev/next links; - the caption names the author (not an account named by an email address) and the picture's licence (StreetPose `license`), and links to the picture in the Panoramax viewer; the globe credits Panoramax. KartaView's viewer becomes the shared photoViewer.js (flat , 360° on panorama.js, ‹ ›, keyboard, pinch, context loss), driven by a per-provider PhotoSources table; the request limiter moves to limiter.js. Panoramax shows the 2,048 px SD derivative and fetches the HD original only when the viewer is wide enough, once, shrunk while it decodes (never above 8,192 px). Orange #ff6f00 (Panoramax's own), off by default, share option 0.m.0_0.r.panoramax-1. The MCP panel allows the meta-catalogue and the biggest instances. Tests: policy, decode, source, viewer, provider, and a Panoramax fixture in the hermetic browser gate. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 + DATA_SOURCES.md | 1 + README.md | 10 +- docs/CURRENT-STATE.md | 43 +- scripts/fixtures/street-level/panoramax.mjs | 256 +++++++ scripts/format-scope.json | 18 +- scripts/package-boundaries.json | 27 +- scripts/qa-street-level.mjs | 357 ++++++++- src/app/layers/streetLevel.js | 6 +- src/app/layers/streetLevel.test.mjs | 19 +- src/layers/streetLevel/limiter.js | 49 ++ .../{providers/kartaview => }/panorama.js | 0 .../kartaview => }/panorama.test.mjs | 0 src/layers/streetLevel/photoViewer.js | 705 ++++++++++++++++++ src/layers/streetLevel/policy.js | 1 + .../streetLevel/providers/kartaview/policy.js | 3 - .../streetLevel/providers/kartaview/source.js | 53 +- .../streetLevel/providers/kartaview/viewer.js | 659 ++-------------- .../providers/mapillary/coverageFixture.mjs | 6 +- .../streetLevel/providers/panoramax/decode.js | 178 +++++ .../providers/panoramax/decode.test.mjs | 173 +++++ .../streetLevel/providers/panoramax/index.js | 461 ++++++++++++ .../streetLevel/providers/panoramax/policy.js | 220 ++++++ .../providers/panoramax/policy.test.mjs | 195 +++++ .../providers/panoramax/provider.test.mjs | 526 +++++++++++++ .../streetLevel/providers/panoramax/source.js | 184 +++++ .../providers/panoramax/source.test.mjs | 176 +++++ .../streetLevel/providers/panoramax/viewer.js | 63 ++ .../providers/panoramax/viewer.test.mjs | 251 +++++++ src/layers/streetLevel/registry.js | 1 + src/layers/streetLevel/state.js | 1 + src/layers/streetLevel/viewerHost.js | 1 + src/testSupport/panoramaxFixtures.mjs | 129 ++++ src/tooling/streetLevelQa.test.mjs | 94 +++ src/tools/globePanel.js | 14 +- src/ui/streetLevelPresentation.js | 1 + src/ui/styles/street-level.css | 52 +- 37 files changed, 4235 insertions(+), 711 deletions(-) create mode 100644 scripts/fixtures/street-level/panoramax.mjs create mode 100644 src/layers/streetLevel/limiter.js rename src/layers/streetLevel/{providers/kartaview => }/panorama.js (100%) rename src/layers/streetLevel/{providers/kartaview => }/panorama.test.mjs (100%) create mode 100644 src/layers/streetLevel/photoViewer.js create mode 100644 src/layers/streetLevel/providers/panoramax/decode.js create mode 100644 src/layers/streetLevel/providers/panoramax/decode.test.mjs create mode 100644 src/layers/streetLevel/providers/panoramax/index.js create mode 100644 src/layers/streetLevel/providers/panoramax/policy.js create mode 100644 src/layers/streetLevel/providers/panoramax/policy.test.mjs create mode 100644 src/layers/streetLevel/providers/panoramax/provider.test.mjs create mode 100644 src/layers/streetLevel/providers/panoramax/source.js create mode 100644 src/layers/streetLevel/providers/panoramax/source.test.mjs create mode 100644 src/layers/streetLevel/providers/panoramax/viewer.js create mode 100644 src/layers/streetLevel/providers/panoramax/viewer.test.mjs create mode 100644 src/testSupport/panoramaxFixtures.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index d0df3bc64..8c347fa44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,19 @@ ## [Unreleased] +- Street Level: Panoramax is the fourth provider, keyless. Light its chip + (orange; off by default, one provider at a time): from orbit dots show + where the federated commons has pictures, and below about 30 km its + sequences draw as lines from `api.panoramax.xyz` vector tiles. A click on a + line opens that sequence's picture nearest the click without moving the + camera, flat or 360°, with cones along the sequence around it and ‹ › / + ← → following the picture's own links. The 360° and since filters apply, + in the lines and in the lookup. The caption names each picture's author + and licence (CC BY-SA 4.0, Licence Ouverte 2.0…) and links to it in the + Panoramax viewer; HD originals load only for an expanded viewer, once. + Share links carry it as `0.m.0_0.r.panoramax-1`. KartaView's photo viewer + is now shared (`photoViewer.js`, `panorama.js`). + - Street Level: KartaView is the third provider, keyless. Light its chip (purple; off by default, one provider at a time) and zoom in below about 6 km: the lines are the streets KartaView has photos of (z13–14 tiles from diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 23d9cfc82..944e4e3fc 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -23,6 +23,7 @@ How to read this: | **Mapillary** (vector tiles, Graph API, MapillaryJS) | Street Level: coverage lines, image cones and the embedded viewer. Coverage tiles cached for 24 h | Imagery [CC BY-SA 4.0](https://www.mapillary.com/terms); API under the Mapillary terms with a client token | "© Mapillary contributors, CC BY-SA 4.0", shown on the globe while the provider is on | | **Google Street View** (Maps JavaScript API) | Street Level, on the Google 3D map only (Google's terms): Google's panorama viewer and nearest-panorama lookups; panorama dates and addresses kept for the session only | Google Maps Platform ToS (your own key + billing; every panorama load is billed) | "Street View imagery © Google" on the globe while the provider is on, and Google's logo and terms in the viewer | | **KartaView** (`api.kartaview.org` coverage tiles and photo API, `cdn.kartaview.org` image proxy, `storage*.openstreetcam.org` photos) | Street Level: coverage lines (z13–14 JSON tiles), nearest-photo lookups, the photo's sequence and the photo itself, fetched by the browser. Keyless; nothing cached beyond the session | Imagery [CC BY-SA 4.0](https://kartaview.org/terms) (KartaView terms §4); the public API needs no token | "© KartaView contributors, CC BY-SA 4.0", shown on the globe while the provider is on | +| **Panoramax** (`api.panoramax.xyz` meta-catalogue: vector tiles and STAC search; pictures from the federated instance holding each one, e.g. `panoramax.openstreetmap.fr`, `panoramax.ign.fr`) | Street Level: coverage lines and density points, nearest-picture lookups, a sequence's pictures as cones, and the picture itself, fetched by the browser. Keyless; tiles cached by the browser for an hour (the API's own header) | Per picture: mostly [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/), IGN's [Licence Ouverte 2.0](https://www.etalab.gouv.fr/licence-ouverte-open-licence/); the API needs no token | "© Panoramax contributors, licence per picture" on the globe while the provider is on; the caption names each picture's author and licence | | **OpenStreetMap** | Map and place data (roads, military areas and names, ALPR, place search, outlines); via vector tiles, hourly ALPR extract, bundled data, Nominatim and optional self-configured Overpass | [ODbL 1.0](https://opendatacommons.org/licenses/odbl/1-0/) | © OpenStreetMap contributors | | **Google Map Tiles API** (Photorealistic 3D Tiles) + Places/Geocoding | The 3D globe, voice scene context, and on-demand nearby installation search | Google Maps Platform ToS (proprietary, your own key + billing) | "Google" / "Google Maps" logo — **shown in-app**, required | | **OpenSky Network** | Primary worldwide live-flight snapshot | Non-commercial research/education license | Schäfer et al., _"Bringing Up OpenSky"_, IPSN 2014 + opensky-network.org | diff --git a/README.md b/README.md index b58a2eaec..8fef6c826 100644 --- a/README.md +++ b/README.md @@ -302,7 +302,7 @@ Twenty layers and map sources. **Eighteen have a keyless path.** Some offer addi | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | | 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | -| 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary, Google Street View and KartaView; Panoramax planned | Mapillary · Google · KartaView | 🟢 KartaView · 🟡 Mapillary token · 🔴 Google Maps key | +| 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary, Google Street View, KartaView and Panoramax | Mapillary · Google · KartaView · Panoramax | 🟢 KartaView, Panoramax · 🟡 Mapillary token · 🔴 Google Maps key | | 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | | 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | | 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | @@ -340,13 +340,13 @@ _The Space Missions layer replaying a Falcon 9 ascent — labeled `RECONSTRUCTED ## 📷 Street Level -> Street-level photos from any provider in one panel. This build ships **Mapillary** (a free client token), **Google Street View** (your Google Maps key) and **KartaView** (no key). +> Street-level photos from any provider in one panel. This build ships **Mapillary** (a free client token), **Google Street View** (your Google Maps key), **KartaView** and **Panoramax** (no key). -**Setup.** At [mapillary.com/dashboard/developers](https://www.mapillary.com/dashboard/developers), register an application with **Read** access and copy its **Client Token** (`MLY|…`). Paste it into **POWER UP → Mapillary**, or set `MAPILLARY_CLIENT_TOKEN` in `.env` and restart. Without it the panel reads **KEY REQUIRED**. **Google Street View** uses `GOOGLE_MAPS_API_KEY` with the **Maps JavaScript API** enabled for it in Google Cloud; its chip starts off, since every panorama is billed to the key. **KartaView** needs nothing: its public API is keyless. +**Setup.** At [mapillary.com/dashboard/developers](https://www.mapillary.com/dashboard/developers), register an application with **Read** access and copy its **Client Token** (`MLY|…`). Paste it into **POWER UP → Mapillary**, or set `MAPILLARY_CLIENT_TOKEN` in `.env` and restart. Without it the panel reads **KEY REQUIRED**. **Google Street View** uses `GOOGLE_MAPS_API_KEY` with the **Maps JavaScript API** enabled for it in Google Cloud; its chip starts off, since every panorama is billed to the key. **KartaView** and **Panoramax** need nothing: their public APIs are keyless. -**Use.** Turn on **Street Level** under DATA LAYERS → Cameras, or with the pill in the STREET LEVEL panel header. Each provider has a chip in its own colour (Mapillary green), and one is lit at a time: lighting one switches the others off. Mapillary, which is free, is lit by default. The **IMAGERY** (ALL / 360° / FLAT) and **SINCE** filters apply to Mapillary; Street View panoramas are all 360°, so with Street View lit the filters and legend are hidden. Click a coverage line for its image cones and a cone to open the photo. Light **KARTAVIEW** (purple) and zoom in below about 6 km: its lines are the streets KartaView has photos of, and a click on one opens the photo nearest the click, flat or 360° (drag to look around), with cones along its drive; **‹ ›** or ← → step through the drive (Shift+← →, ↑ ↓ and + − look around a 360° photo; pinch zooms on a phone). Photos KartaView has archived are skipped, and where all are archived the panel says so. KartaView photos are undated on its map, so the filters do not apply to it. On the **Google 3D** map (Google's terms keep Street View off other maps, where it does not appear), light **GOOGLE STREET VIEW** and click a street from below 3 km, or press **OPEN STREET VIEW** for the view centre, to open Google's panorama there; step along its arrows. Above the photo: **EXPAND**, **FIT / FILL** (Mapillary, and flat KartaView photos), and **FOLLOW** (the globe camera follows the photo's view; Google 3D map only). The caption links to the image on the provider's site, and the globe credits each provider that is on ("© Mapillary contributors, CC BY-SA 4.0", "Street View imagery © Google", "© KartaView contributors, CC BY-SA 4.0"). KartaView's own photo pages fail for signed-out visitors, so its link opens the full-size photo. +**Use.** Turn on **Street Level** under DATA LAYERS → Cameras, or with the pill in the STREET LEVEL panel header. Each provider has a chip in its own colour (Mapillary green), and one is lit at a time: lighting one switches the others off. Mapillary, which is free, is lit by default. The **IMAGERY** (ALL / 360° / FLAT) and **SINCE** filters apply to Mapillary; Street View panoramas are all 360°, so with Street View lit the filters and legend are hidden. Click a coverage line for its image cones and a cone to open the photo. Light **KARTAVIEW** (purple) and zoom in below about 6 km: its lines are the streets KartaView has photos of, and a click on one opens the photo nearest the click, flat or 360° (drag to look around), with cones along its drive; **‹ ›** or ← → step through the drive (Shift+← →, ↑ ↓ and + − look around a 360° photo; pinch zooms on a phone). Photos KartaView has archived are skipped, and where all are archived the panel says so. KartaView photos are undated on its map, so the filters do not apply to it. Light **PANORAMAX** (orange) for the open, federated commons (strongest in France and Belgium): from orbit its dots show where it has pictures, and below about 30 km its lines are sequences. A click on a line opens that sequence's picture nearest the click without moving the camera, flat or 360°, with cones along the sequence around it; **‹ ›** or ← → step through it, and both filters apply. Its caption names the picture's author and licence (mostly CC BY-SA 4.0; IGN's are Licence Ouverte 2.0). On the **Google 3D** map (Google's terms keep Street View off other maps, where it does not appear), light **GOOGLE STREET VIEW** and click a street from below 3 km, or press **OPEN STREET VIEW** for the view centre, to open Google's panorama there; step along its arrows. Above the photo: **EXPAND**, **FIT / FILL** (Mapillary, and flat KartaView and Panoramax photos), and **FOLLOW** (the globe camera follows the photo's view; Google 3D map only). The caption links to the image on the provider's site, and the globe credits each provider that is on ("© Mapillary contributors, CC BY-SA 4.0", "Street View imagery © Google", "© KartaView contributors, CC BY-SA 4.0", "© Panoramax contributors, licence per picture"). KartaView's own photo pages fail for signed-out visitors, so its link opens the full-size photo. -Drag the panel header to float the panel, and resize it from any edge. Double-click the header or collapse it to put it back in the rail. Share links carry the provider switches and filters (`0.m.0` Mapillary off, `0.m.0_0.r.google-1` Street View on, `0.m.0_0.r.kartaview-1` KartaView on (a link that leaves out `m` keeps Mapillary, the default, lit), `0.p.p` panoramas only, `0.s.365` the last year). To add a provider, implement the contract in `src/layers/streetLevel/registry.js` and register it in `src/app/layers/streetLevel.js`. +Drag the panel header to float the panel, and resize it from any edge. Double-click the header or collapse it to put it back in the rail. Share links carry the provider switches and filters (`0.m.0` Mapillary off, `0.m.0_0.r.google-1` Street View on, `0.m.0_0.r.kartaview-1` KartaView on, `0.m.0_0.r.panoramax-1` Panoramax on (a link that leaves out `m` keeps Mapillary, the default, lit), `0.p.p` panoramas only, `0.s.365` the last year). To add a provider, implement the contract in `src/layers/streetLevel/registry.js` and register it in `src/app/layers/streetLevel.js`. ## 🎖️ Field Missions diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index f595ce519..7b28ca40b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3084,7 +3084,7 @@ provider draws in one colour (`PROVIDER_COLORS` in `src/layers/streetLevel/policy.js`); 360° cones are rings and the selected sequence is GEV cyan. Share options: `m` (Mapillary), `r` (the other providers' switches by id, e.g. `google-1` for Street View), `p` (`a`/`p`/`f` panoramas), `s` (since, days). Mapillary, Google -Street View and KartaView are registered. Without +Street View, KartaView and Panoramax are registered. Without `MAPILLARY_CLIENT_TOKEN` the panel reads KEY REQUIRED and the controls are disabled. With it, coverage draws as z0–5 overview points from orbit and z11–14 sequence lines below 60 km; the proxy strips the unused `image` layer from z14 @@ -3244,14 +3244,49 @@ photo that was opening is drawn on the new canvas instead. FIT / FILL hide for panoramas (`supportsRenderModes`). The caption has no place name (KartaView's is where the drive began) and links to the full-size photo through the image proxy, since kartaview.org's photo -pages fail for signed-out visitors. +pages fail for signed-out visitors. The viewer itself is `photoViewer.js`, +shared with Panoramax: each provider hands it a `PhotoSources` table (first +and sharper flat and panorama URLs, and the caption fields). + +Panoramax (`providers/panoramax/`) is keyless and browser-side: the +meta-catalogue `api.panoramax.xyz` (CORS open) federates every instance, and +each picture is served by the instance holding it (OpenStreetMap France, +IGN, MapComplete…; IGN's redirect to its OVH object storage). Its chip +(orange, `#ff6f00`, Panoramax's own) starts off. Coverage comes from its +vector tiles (`/api/map/{z}/{x}/{y}.mvt`, an hour's HTTP cache): sequence +lines from z14 below 1.8 km above ground, z13 to 5 km, z12 to 12 km and z11 +to 30 km (a dense z11 tile over Paris is ~1.4 MB and 3,800 sequences; each +tile draws its 1,600 longest), and above 30 km the `grid` layer's 0.1° +density cells, binned to at most 96×96 points a tile (a z0 tile holds +~70,000 cells) and not counted as sequences: the panel says to zoom in. Both +filters apply: lines carry their day and camera type, and a cell stays while +it holds pictures of the kind asked for (cells are undated). A click on a +line asks the core (`actions.openNearest`) for that sequence's picture +nearest the click (`/api/search?place_position…&collections=`, any +direction: `place_fov_tolerance=180`), without framing, then any sequence's +if it has none within 50 m; the filters go to the API (`filter= +field_of_view=360`, `datetime=/..`). Once the picture is on screen +its sequence is highlighted and drawn as cones from the z15 tiles around it +(their `pictures` layer, ~110 bytes a picture, against ~7.5 kB for a STAC +item), plus a neighbour within 150 m of an edge; the cones re-centre when a +picture opens outside them. ‹ › follow the picture's own `prev`/`next` +links, so stepping costs one small search per picture. The viewer shows the +2,048 px SD derivative (equirectangular for a 360° picture) and swaps in the +HD original only when the viewer is wider than SD (flat) or needs a 4,096 px +panorama (expanded), once, shrunk while it decodes; originals wider than +8,192 px are never fetched. The caption names the author (not an account +named by an email address) and the picture's licence (CC BY-SA 4.0, Licence +Ouverte 2.0…; StreetPose `license`), and links to the picture in the +Panoramax viewer. The MCP globe panel allows the meta-catalogue and the +three biggest instances; a picture on another instance shows only in the +app. Street Level has two browser gates. `npm run qa:street-level -- --url ` runs against real Mapillary and needs `MAPILLARY_CLIENT_TOKEN`. `npm run qa:street-level:fixtures -- --url ` answers every Mapillary request from fixtures (`providers/mapillary/coverageFixture.mjs`, -`scripts/fixtures/street-level/`), so it needs no network and covers the whole -photo flow. CI runs the fixture gate and `qa:panel-resize` +`scripts/fixtures/street-level/`), and KartaView's and Panoramax's too, so it +needs no network and covers the whole photo flow. CI runs the fixture gate and `qa:panel-resize` against a production build served by `vite preview` with a dummy token. Directions is a keyless front end to the routing the voice agent already diff --git a/scripts/fixtures/street-level/panoramax.mjs b/scripts/fixtures/street-level/panoramax.mjs new file mode 100644 index 000000000..768a7ff62 --- /dev/null +++ b/scripts/fixtures/street-level/panoramax.mjs @@ -0,0 +1,256 @@ +import { tileBounds } from '../../../src/layers/streetLevel/tileMath.js'; +import { encodePanoramaxTile } from '../../../src/testSupport/panoramaxFixtures.mjs'; +import { metresApart, photoJpeg } from './mapillaryGraph.mjs'; + +/** + * Panoramax's meta-catalogue and one instance's pictures answered from + * fixtures, so the gate's Panoramax steps need no network. Two sequences + * drive east along the same Paris street (PNX_LINE), their pictures + * interleaved, so a click on one line must open that line's picture, not + * the other's that is as near. Every seventh picture of the first is 360°. + * The meta-catalogue's pictures live on panoramax.openstreetmap.fr; the + * first sequence is CC BY-SA 4.0 by "Fixture Mapper", the second Licence + * Ouverte by an account named by an email address (never shown). + */ + +export const PNX_SEQUENCES = Object.freeze([ + 'a1a1a1a1-0000-4000-8000-000000000001', + 'b2b2b2b2-0000-4000-8000-000000000002', +]); +export const PNX_LINE = Object.freeze({ + lat: 48.8874, + west: 2.3445, + east: 2.3545, +}); +export const PNX_PARK = Object.freeze({ + lon: 2.3495, + lat: 48.8874, + height: 700, + pitch: -1.4, +}); +export const PNX_CREATOR = 'Fixture Mapper'; +const PNX_SPACING_M = 10; +const METRES_PER_DEG_LON = 111_320 * Math.cos((PNX_LINE.lat * Math.PI) / 180); +const API = 'https://api.panoramax.xyz/api'; +const HOST = 'https://panoramax.openstreetmap.fr'; + +const CORS = Object.freeze({ 'Access-Control-Allow-Origin': '*' }); + +/** Calls answered since the last reset, by kind. */ +let asked = { search: 0, hd: 0 }; + +export function resetPanoramaxFixture() { + asked = { search: 0, hd: 0 }; +} + +/** How many HD originals were downloaded since the last reset. */ +export function panoramaxHdDownloads() { + return asked.hd; +} + +/** + * Both sequences' pictures, west to east. The second's sit 5 m along from + * the first's, on the same street. + */ +export function panoramaxPictures() { + const step = PNX_SPACING_M / METRES_PER_DEG_LON; + const count = Math.floor((PNX_LINE.east - PNX_LINE.west) / step); + return PNX_SEQUENCES.flatMap((sequenceId, s) => + Array.from({ length: count }, (_, i) => ({ + id: `${sequenceId.slice(0, 8)}-${String(i).padStart(4, '0')}`, + sequenceId, + index: i, + count, + lon: Number((PNX_LINE.west + (i + s / 2) * step).toFixed(7)), + lat: PNX_LINE.lat, + isPano: s === 0 && i % 7 === 3, + datetime: + s === 0 ? '2025-06-01T10:00:00+00:00' : '2023-03-01T09:00:00+00:00', + })), + ); +} + +function item(picture, pictures) { + const sibling = (delta) => + pictures.find( + (p) => + p.sequenceId === picture.sequenceId && + p.index === picture.index + delta, + ); + const link = (rel, other) => + other + ? [ + { + rel, + href: `${API}/collections/${other.sequenceId}/items/${other.id}`, + }, + ] + : []; + const first = picture.sequenceId === PNX_SEQUENCES[0]; + const files = `${HOST}/derivates/${picture.id}`; + return { + type: 'Feature', + id: picture.id, + collection: picture.sequenceId, + geometry: { type: 'Point', coordinates: [picture.lon, picture.lat] }, + providers: [ + { + name: first ? PNX_CREATOR : 'someone@fixture.example.fr', + roles: ['producer'], + }, + ], + properties: { + datetime: picture.datetime, + license: first ? 'CC-BY-SA-4.0' : 'etalab-2.0', + 'view:azimuth': 90, + 'pers:interior_orientation': picture.isPano + ? { field_of_view: 360, sensor_array_dimensions: [5760, 2880] } + : { field_of_view: 70, sensor_array_dimensions: [4032, 3024] }, + }, + assets: { + hd: { href: `${files}/hd.jpg` }, + sd: { href: `${files}/sd.jpg` }, + thumb: { href: `${files}/thumb.jpg` }, + }, + links: [...link('prev', sibling(-1)), ...link('next', sibling(1))], + }; +} + +/** The tile `z/x/y` in the meta-catalogue's shape. */ +export function panoramaxTile(z, x, y) { + const tile = { x, y, z }; + const { west, east, south, north } = tileBounds(x, y, z); + if (PNX_LINE.lat < south || PNX_LINE.lat > north) return new Uint8Array(0); + const from = Math.max(PNX_LINE.west, west); + const to = Math.min(PNX_LINE.east, east); + if (from >= to) return new Uint8Array(0); + if (z <= 5) + return encodePanoramaxTile(tile, { + grid: [{ lon: (from + to) / 2, lat: PNX_LINE.lat, pano: 30, flat: 170 }], + }); + if (z < 7) return new Uint8Array(0); + const sequences = PNX_SEQUENCES.map((id, s) => ({ + id, + type: s === 0 ? 'equirectangular' : 'flat', + date: s === 0 ? '2025-06-01' : '2023-03-01', + parts: [ + [ + [from, PNX_LINE.lat], + [to, PNX_LINE.lat], + ], + ], + })); + if (z < 15) return encodePanoramaxTile(tile, { sequences }); + const pictures = panoramaxPictures() + .filter((p) => p.lon >= west && p.lon < east) + .map((p) => ({ + id: p.id, + lon: p.lon, + lat: p.lat, + sequenceId: p.sequenceId, + heading: 90, + ts: p.datetime.replace('T', ' ').replace('+00:00', '+00'), + type: p.isPano ? 'equirectangular' : 'flat', + })); + return encodePanoramaxTile(tile, { sequences, pictures }); +} + +/** A call named by host, path shape and parameter names. */ +export function describePanoramaxCall(method, url) { + const keys = [...url.searchParams.keys()].sort(); + const path = url.pathname + .replace(/\/map\/\d+\/\d+\/\d+\.mvt$/, '/map/{z}/{x}/{y}.mvt') + .replace( + /^\/derivates\/[\w-]+\/(sd|hd|thumb)\.jpg$/, + '/derivates/{id}/$1.jpg', + ); + return `${method} ${url.hostname}${path}${keys.length ? `?${keys.join('&')}` : ''}`; +} + +const json = (body) => ({ + status: 200, + contentType: 'application/geo+json', + headers: CORS, + body: JSON.stringify(body), + known: true, +}); + +/** + * Answer one Panoramax request; `known` is false for a call no fixture + * covers, so the gate fails on a new one instead of missing it. + */ +export function answerPanoramaxRequest({ method, url }) { + const address = new URL(url); + if (method === 'OPTIONS') + return { status: 204, headers: CORS, body: '', known: true }; + const pictures = panoramaxPictures(); + if (address.hostname === new URL(HOST).hostname) { + const file = address.pathname.match( + /^\/derivates\/([\w-]+)\/(sd|hd|thumb)\.jpg$/, + ); + if (!file || !pictures.some((p) => p.id === file[1])) + return { status: 404, headers: CORS, body: '', known: false }; + if (file[2] === 'hd') asked.hd++; + return { + status: 200, + contentType: 'image/jpeg', + headers: CORS, + body: photoJpeg(), + known: true, + }; + } + if (address.hostname !== 'api.panoramax.xyz') + return { status: 404, headers: CORS, body: '', known: false }; + const tile = address.pathname.match(/^\/api\/map\/(\d+)\/(\d+)\/(\d+)\.mvt$/); + if (tile) { + const bytes = panoramaxTile( + Number(tile[1]), + Number(tile[2]), + Number(tile[3]), + ); + return bytes.length + ? { + status: 200, + contentType: 'application/vnd.mapbox-vector-tile', + headers: CORS, + body: Buffer.from(bytes), + known: true, + } + : { status: 204, headers: CORS, body: '', known: true }; + } + if (address.pathname !== '/api/search') + return { status: 404, headers: CORS, body: '', known: false }; + asked.search++; + const q = address.searchParams; + if (q.has('ids')) { + const wanted = q.get('ids').split(','); + return json({ + type: 'FeatureCollection', + features: pictures + .filter((p) => wanted.includes(p.id)) + .map((p) => item(p, pictures)), + }); + } + const [lon, lat] = (q.get('place_position') || '').split(',').map(Number); + const radius = Number((q.get('place_distance') || '0-50').split('-')[1]); + const filter = q.get('filter') || ''; + const since = q.get('datetime')?.split('/')[0]; + const near = pictures + .filter( + (p) => !q.get('collections') || p.sequenceId === q.get('collections'), + ) + .filter((p) => + filter === 'field_of_view=360' + ? p.isPano + : filter === 'field_of_view<360' + ? !p.isPano + : true, + ) + .filter((p) => !since || Date.parse(p.datetime) >= Date.parse(since)) + .map((p) => ({ p, metres: metresApart({ lon, lat }, p) })) + .filter(({ metres }) => metres <= radius) + .sort((a, b) => a.metres - b.metres) + .slice(0, Number(q.get('limit')) || 10) + .map(({ p }) => item(p, pictures)); + return json({ type: 'FeatureCollection', features: near }); +} diff --git a/scripts/format-scope.json b/scripts/format-scope.json index a77fc11f9..73959131f 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -912,15 +912,28 @@ "src/layers/streetLevel/providers/kartaview/source.js", "src/layers/streetLevel/providers/kartaview/decode.js", "src/layers/streetLevel/providers/kartaview/viewer.js", - "src/layers/streetLevel/providers/kartaview/panorama.js", + "src/layers/streetLevel/panorama.js", "src/layers/streetLevel/lineCoverage.js", "src/layers/streetLevel/sequenceCones.js", "src/layers/streetLevel/providers/kartaview/policy.test.mjs", "src/layers/streetLevel/providers/kartaview/decode.test.mjs", "src/layers/streetLevel/providers/kartaview/source.test.mjs", - "src/layers/streetLevel/providers/kartaview/panorama.test.mjs", + "src/layers/streetLevel/panorama.test.mjs", "src/layers/streetLevel/providers/kartaview/viewer.test.mjs", "src/layers/streetLevel/providers/kartaview/provider.test.mjs", + "src/layers/streetLevel/providers/panoramax/index.js", + "src/layers/streetLevel/providers/panoramax/policy.js", + "src/layers/streetLevel/providers/panoramax/source.js", + "src/layers/streetLevel/providers/panoramax/decode.js", + "src/layers/streetLevel/providers/panoramax/viewer.js", + "src/layers/streetLevel/photoViewer.js", + "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/providers/panoramax/policy.test.mjs", + "src/layers/streetLevel/providers/panoramax/decode.test.mjs", + "src/layers/streetLevel/providers/panoramax/source.test.mjs", + "src/layers/streetLevel/providers/panoramax/viewer.test.mjs", + "src/layers/streetLevel/providers/panoramax/provider.test.mjs", + "src/testSupport/panoramaxFixtures.mjs", "src/layers/streetLevel/lineCoverage.test.mjs", "src/testSupport/kartaViewFakes.mjs", "src/testSupport/googleStreetViewFakes.mjs", @@ -935,6 +948,7 @@ "scripts/qa-browserEvidence.mjs", "scripts/fixtures/street-level/mapillaryGraph.mjs", "scripts/fixtures/street-level/kartaView.mjs", + "scripts/fixtures/street-level/panoramax.mjs", "src/layers/streetLevel/filter.test.mjs", "src/layers/streetLevel/registry.test.mjs", "src/layers/streetLevel/params.test.mjs", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index ba1f0c36a..2e83eb2cc 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1234,7 +1234,14 @@ "src/layers/streetLevel/providers/kartaview/source.js", "src/layers/streetLevel/providers/kartaview/decode.js", "src/layers/streetLevel/providers/kartaview/viewer.js", - "src/layers/streetLevel/providers/kartaview/panorama.js", + "src/layers/streetLevel/providers/panoramax/index.js", + "src/layers/streetLevel/providers/panoramax/policy.js", + "src/layers/streetLevel/providers/panoramax/source.js", + "src/layers/streetLevel/providers/panoramax/decode.js", + "src/layers/streetLevel/providers/panoramax/viewer.js", + "src/layers/streetLevel/photoViewer.js", + "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/panorama.js", "src/layers/streetLevel/lineCoverage.js", "src/layers/streetLevel/sequenceCones.js", "src/layers/streetLevel/providers/mapillary/source.js", @@ -2202,7 +2209,14 @@ "src/layers/streetLevel/providers/kartaview/source.js", "src/layers/streetLevel/providers/kartaview/decode.js", "src/layers/streetLevel/providers/kartaview/viewer.js", - "src/layers/streetLevel/providers/kartaview/panorama.js", + "src/layers/streetLevel/providers/panoramax/index.js", + "src/layers/streetLevel/providers/panoramax/policy.js", + "src/layers/streetLevel/providers/panoramax/source.js", + "src/layers/streetLevel/providers/panoramax/decode.js", + "src/layers/streetLevel/providers/panoramax/viewer.js", + "src/layers/streetLevel/photoViewer.js", + "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/panorama.js", "src/layers/streetLevel/lineCoverage.js", "src/layers/streetLevel/sequenceCones.js", "src/layers/streetLevel/providers/mapillary/source.js", @@ -2741,7 +2755,14 @@ "src/layers/streetLevel/providers/kartaview/source.js", "src/layers/streetLevel/providers/kartaview/decode.js", "src/layers/streetLevel/providers/kartaview/viewer.js", - "src/layers/streetLevel/providers/kartaview/panorama.js", + "src/layers/streetLevel/providers/panoramax/index.js", + "src/layers/streetLevel/providers/panoramax/policy.js", + "src/layers/streetLevel/providers/panoramax/source.js", + "src/layers/streetLevel/providers/panoramax/decode.js", + "src/layers/streetLevel/providers/panoramax/viewer.js", + "src/layers/streetLevel/photoViewer.js", + "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/panorama.js", "src/layers/streetLevel/lineCoverage.js", "src/layers/streetLevel/sequenceCones.js", "src/layers/streetLevel/providers/mapillary/source.js", diff --git a/scripts/qa-street-level.mjs b/scripts/qa-street-level.mjs index 153d7514c..7437b2be8 100644 --- a/scripts/qa-street-level.mjs +++ b/scripts/qa-street-level.mjs @@ -3,8 +3,9 @@ * Browser QA for the Street Level layer against a running server. Run with * `npm run qa:street-level -- --url http://localhost:4173`. * - * `--fixtures` (the CI mode) answers every *.mapillary.com and KartaView - * request from fixtures, so nothing reaches Mapillary or KartaView. The server still needs a token + * `--fixtures` (the CI mode) answers every *.mapillary.com, KartaView and + * Panoramax request from fixtures, so nothing reaches those providers. The + * server still needs a token * because the bundle carries it; CI uses a dummy one. Fixture runs also stage * a rejected key and a keyless second server. * @@ -46,6 +47,17 @@ import { kartaViewPhotos, resetKartaViewFixture, } from './fixtures/street-level/kartaView.mjs'; +import { + answerPanoramaxRequest, + describePanoramaxCall, + PNX_CREATOR, + PNX_LINE, + PNX_PARK, + PNX_SEQUENCES, + panoramaxHdDownloads, + panoramaxPictures, + resetPanoramaxFixture, +} from './fixtures/street-level/panoramax.mjs'; const PANEL_ID = 'street-level-panel'; @@ -62,6 +74,7 @@ export const EXPECTED_PROVIDERS = Object.freeze([ 'mapillary', 'google', 'kartaview', + 'panoramax', ]); /** @@ -245,6 +258,8 @@ export async function serveFixtures(page, fixture) { { urlPattern: '*://api.kartaview.org/*' }, { urlPattern: '*://cdn.kartaview.org/*' }, { urlPattern: '*://*.openstreetcam.org/*' }, + { urlPattern: '*://api.panoramax.xyz/*' }, + { urlPattern: '*://panoramax.openstreetmap.fr/*' }, ], }); const fulfil = (requestId, { status, headers = {}, contentType, body }) => @@ -280,6 +295,14 @@ export async function serveFixtures(page, fixture) { answer = answerKartaViewRequest({ method, url: request.url }); fixture.calls.add(call); if (!answer.known) fixture.unknown.push(call); + } else if ( + address.hostname === 'api.panoramax.xyz' || + address.hostname === 'panoramax.openstreetmap.fr' + ) { + const call = describePanoramaxCall(method, address); + answer = answerPanoramaxRequest({ method, url: request.url }); + fixture.calls.add(call); + if (!answer.known) fixture.unknown.push(call); } else if ( address.pathname === '/api/mapillary/status' && fixture.configured !== undefined @@ -909,7 +932,7 @@ async function main() { assert.equal(street.renderModes, true, 'FIT / FILL for a flat photo'); assert.ok( await settle(page, () => { - const img = document.querySelector('.sl-kv-photo'); + const img = document.querySelector('.sl-photo-flat'); return img && !img.hidden && img.naturalWidth > 0; }), 'the photo is on screen', @@ -944,7 +967,7 @@ async function main() { ); } const before = street.imageId; - await clickPanelControl(page, '.sl-kv-step-next'); + await clickPanelControl(page, '.sl-photo-step-next'); assert.ok( await uiUntil( (u, was) => @@ -996,10 +1019,10 @@ async function main() { assert.equal(u.street.error, null); assert.equal(u.street.renderModes, false, 'no frame to fit'); assert.equal( - await page.$eval('.sl-kv-pano', (canvas) => canvas.hidden), + await page.$eval('.sl-photo-pano', (canvas) => canvas.hidden), false, ); - const box = await page.$eval('.sl-kv-pano', (canvas) => { + const box = await page.$eval('.sl-photo-pano', (canvas) => { const r = canvas.getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2 }; }); @@ -1016,7 +1039,7 @@ async function main() { // A GPU reset: the panorama comes back on a new canvas, and drags. assert.ok( await page.evaluate(() => { - const canvas = document.querySelector('.sl-kv-pano'); + const canvas = document.querySelector('.sl-photo-pano'); window.__kvLostCanvas = canvas; const lose = canvas .getContext('webgl') @@ -1030,7 +1053,7 @@ async function main() { await settle( page, () => { - const canvas = document.querySelector('.sl-kv-pano'); + const canvas = document.querySelector('.sl-photo-pano'); return ( canvas && canvas !== window.__kvLostCanvas && @@ -1077,6 +1100,323 @@ async function main() { await setStack(original); } + /** + * Panoramax: keyless orange coverage with both filters, a line click + * opening that sequence's picture nearest the click (author, licence, + * cones, ›), a 360° picture whose original is fetched once on EXPAND. + * Live runs park over Paris, where Panoramax has pictures. Runs on a + * draped map; leaves Mapillary lit. + */ + async function panoramaxSteps() { + resetPanoramaxFixture(); + const ui = () => + page.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('street-level') + .module.getUIState(), + ); + const uiUntil = (test, arg = null, options) => + settle( + page, + `(${test})(window.__godsEyeView.dataManager.layers.get('street-level').module.getUIState(), ${JSON.stringify(arg)})`, + undefined, + options, + ); + const lit = (id) => + uiUntil( + (u, want) => + u.providers.find((p) => p.id === want)?.on === true && + u.providers.filter((p) => p.on).length === 1, + id, + ); + const module = (call, ...args) => + page.evaluate( + (name, list) => + window.__godsEyeView.dataManager.layers + .get('street-level') + .module[name](...list), + call, + args, + ); + const stack = () => + page.evaluate(() => + window.__godsEyeView.mapStackController.getActiveId(), + ); + const setStack = async (id) => { + await page.evaluate( + (stackId) => + window.__godsEyeView.mapStackController.setStack(stackId), + id, + ); + await settle( + page, + (stackId) => + window.__godsEyeView.mapStackController.getActiveId() === stackId, + id, + ); + }; + const camera = () => + page.evaluate(() => { + const c = window.__godsEyeView.viewer.camera.positionCartographic; + return { + lon: (c.longitude * 180) / Math.PI, + lat: (c.latitude * 180) / Math.PI, + height: c.height, + }; + }); + const pnxChip = '#sl-provider-chips [data-chip-id="panoramax"]'; + const original = await stack(); + await module('closeViewer'); + await setStack('esri-imagery'); + for (let attempt = 0; attempt < 4; attempt++) { + await page.evaluate((at) => { + const v = window.__godsEyeView.viewer; + v.camera.cancelFlight?.(); + const C = v.camera.positionCartographic.constructor; + v.camera.setView({ + destination: v.scene.globe.ellipsoid.cartographicToCartesian( + C.fromDegrees(at.lon, at.lat, at.height), + ), + orientation: { heading: 0, pitch: at.pitch, roll: 0 }, + }); + }, PNX_PARK); + await sleep(1000); + } + try { + await step( + 'Panoramax: its chip draws keyless orange coverage, keeps both filters, credits Panoramax and lights in the share link', + async () => { + await clickPanelControl(page, pnxChip); + assert.ok(await lit('panoramax'), 'Panoramax alone is lit'); + await waitForCoverage(page); + const u = await ui(); + assert.equal(u.keyRequired, false); + assert.deepEqual(u.filterControls, { pano: true, since: true }); + assert.equal( + await page.$eval(pnxChip, (chip) => + chip.style.getPropertyValue('--chip-color'), + ), + '#ff6f00', + ); + assert.match( + await page.$eval('#sl-coverage-meta', (node) => node.textContent), + /sequences in view · click a line for the photo there/, + ); + await page.waitForFunction( + () => + document.body.innerHTML.includes('Panoramax contributors'), + { timeout: 15_000 }, + ); + assert.ok( + await settle(page, () => + (new URLSearchParams(location.hash.slice(1)).get('lo') || '') + .split('_') + .some( + (entry) => + entry.startsWith('0.') && entry.includes('panoramax-1'), + ), + ), + `the share link lights Panoramax: ${await page.evaluate(() => location.hash)}`, + ); + }, + ); + await step( + 'Panoramax: a click on a line opens that sequence’s picture nearest the click, camera still, with author and licence, cones and ›', + async ({ skip }) => { + // Points on the street: the fixture's, or live pictures there. + let points = null; + if (fixtures) + points = Array.from({ length: 9 }, (_, i) => [ + PNX_LINE.west + + ((i + 2) / 13) * (PNX_LINE.east - PNX_LINE.west), + PNX_LINE.lat, + ]); + else { + const near = await fetch( + `https://api.panoramax.xyz/api/search?place_position=${PNX_PARK.lon},${PNX_PARK.lat}&place_distance=0-150&place_fov_tolerance=180&limit=12`, + ) + .then((r) => r.json()) + .catch(() => null); + points = near?.features?.map((f) => f.geometry.coordinates); + if (!points?.length) + return skip('no Panoramax picture near the park'); + } + const handle = await page + .waitForFunction( + findPickPoint, + { timeout: 60_000, polling: 500 }, + { + prefix: 'pnx:seq:', + id: null, + points, + billboardIds: null, + }, + ) + .catch(() => null); + if (!handle && !fixtures) return skip('no Panoramax line in view'); + assert.ok(handle, 'a Panoramax line is pickable'); + const at = await handle.jsonValue(); + const parked = await camera(); + await page.mouse.click(at.x, at.y); + await page.waitForFunction( + () => { + const s = window.__godsEyeView.dataManager.layers + .get('street-level') + .module.getUIState().street; + return (s.imageId && !s.loading) || s.error; + }, + { timeout: 60_000 }, + ); + const street = (await ui()).street; + assert.equal(street.error, null, `viewer error: ${street.error}`); + assert.equal(street.providerId, 'panoramax'); + assert.equal( + street.sequenceId, + at.id, + 'the picture is from the line clicked', + ); + assert.ok(street.license, 'the caption names the licence'); + assert.match( + street.externalUrl, + /^https:\/\/api\.panoramax\.xyz\/#pic=/, + ); + await sleep(1500); + const now = await camera(); + assert.ok( + Math.abs(now.lon - parked.lon) < 1e-6 && + Math.abs(now.lat - parked.lat) < 1e-6 && + Math.abs(now.height - parked.height) < 0.5, + `the camera did not move: ${JSON.stringify({ parked, now })}`, + ); + if (fixtures) { + const caption = await page.$eval( + '#sl-image-by', + (node) => node.textContent, + ); + if (street.sequenceId === PNX_SEQUENCES[0]) + assert.equal(caption, `Image by ${PNX_CREATOR} · CC BY-SA 4.0`); + else + assert.equal( + caption, + 'Licence Ouverte 2.0', + 'an account named by an email address is not shown', + ); + const pictures = panoramaxPictures().filter( + (p) => p.sequenceId === street.sequenceId, + ); + assert.ok( + await uiUntil( + (u, want) => + u.sequence.selectedId === want && + !u.sequence.loading && + u.sequence.images > 0, + street.sequenceId, + { timeout: 30_000 }, + ), + `cones along the sequence: ${JSON.stringify((await ui()).sequence)}`, + ); + assert.ok((await ui()).sequence.images <= pictures.length); + } + const before = street.imageId; + await clickPanelControl(page, '.sl-photo-step-next'); + assert.ok( + await uiUntil( + (u, was) => + u.street.imageId && + u.street.imageId !== was && + !u.street.loading && + !u.street.error, + before, + { timeout: 30_000 }, + ), + '› opened the next picture', + ); + const after = (await ui()).street; + assert.equal(after.sequenceId, street.sequenceId); + if (fixtures) { + const all = panoramaxPictures(); + const from = all.find((p) => p.id === before); + assert.equal( + after.imageId, + all.find( + (p) => + p.sequenceId === from.sequenceId && + p.index === from.index + 1, + ).id, + '› followed the picture’s next link', + ); + } + }, + ); + if (fixtures) + await step( + 'Panoramax: a 360° picture opens on the panorama without FIT / FILL, drags, and EXPAND fetches its original once', + async () => { + const pano = panoramaxPictures().find((p) => p.isPano); + await module('openImage', 'panoramax', pano.id); + assert.ok( + await uiUntil( + (u, want) => + u.street.imageId === want && + !u.street.loading && + u.street.isPano, + pano.id, + { timeout: 60_000 }, + ), + `the panorama opened: ${JSON.stringify((await ui()).street)}`, + ); + const u = await ui(); + assert.equal(u.street.renderModes, false, 'no frame to fit'); + const box = await page.$eval('.sl-photo-pano', (canvas) => { + const r = canvas.getBoundingClientRect(); + return { x: r.x + r.width / 2, y: r.y + r.height / 2 }; + }); + const bearing = u.street.bearing; + await page.mouse.move(box.x, box.y); + await page.mouse.down(); + for (let i = 1; i <= 8; i++) + await page.mouse.move(box.x - i * 15, box.y); + await page.mouse.up(); + assert.ok( + await uiUntil((v, was) => v.street.bearing !== was, bearing), + 'dragging the panorama turned its bearing', + ); + assert.equal(panoramaxHdDownloads(), 0, 'docked: SD only'); + await clickPanelControl(page, '#sl-viewer-expand'); + assert.ok( + await settle(page, () => + Boolean(document.querySelector('.sl-viewer-wrap-expanded')), + ), + 'the viewer expanded', + ); + const deadline = Date.now() + 30_000; + while (panoramaxHdDownloads() === 0 && Date.now() < deadline) + await sleep(250); + await sleep(2000); + assert.equal( + panoramaxHdDownloads(), + 1, + 'the original, once, for the expanded viewer', + ); + await page.keyboard.press('Escape'); + }, + ); + else + await skipStep( + 'fixtures only', + 'Panoramax: a 360° picture opens on the panorama without FIT / FILL, drags, and EXPAND fetches its original once', + ); + } finally { + await module('closeViewer'); + await clickPanelControl( + page, + '#sl-provider-chips [data-chip-id="mapillary"]', + ); + assert.ok(await lit('mapillary'), 'Mapillary is lit again'); + await setStack(original); + } + } + async function keyedSteps() { // A late startup flight only shows itself by moving the camera, so park // and check it stays put rather than reading a state. @@ -2386,6 +2726,7 @@ async function main() { }, ); await kartaViewSteps(); + await panoramaxSteps(); await step( 'a reload with the layer on keeps a collapsed panel collapsed, and the share link carries its ui token', async () => { diff --git a/src/app/layers/streetLevel.js b/src/app/layers/streetLevel.js index f176d97de..b441d12d7 100644 --- a/src/app/layers/streetLevel.js +++ b/src/app/layers/streetLevel.js @@ -2,6 +2,7 @@ import { createStreetLevelLayer } from '../../layers/streetLevel/index.js'; import { createMapillaryProvider } from '../../layers/streetLevel/providers/mapillary/index.js'; import { createGoogleProvider } from '../../layers/streetLevel/providers/google/index.js'; import { createKartaViewProvider } from '../../layers/streetLevel/providers/kartaview/index.js'; +import { createPanoramaxProvider } from '../../layers/streetLevel/providers/panoramax/index.js'; import * as sprites from '../../data/spriteOrder.js'; import * as picking from '../../data/pickRegistry.js'; import * as input from '../../data/inputOwnership.js'; @@ -12,8 +13,8 @@ import * as render from '../../renderGovernor.js'; const googleApiKey = () => globalThis.window?.__GOOGLE_MAPS_API_KEY__ || null; /** - * Mapillary, Google Street View and KartaView, once the Mapillary source is - * supplied; else none. + * Mapillary, Google Street View, KartaView and Panoramax, once the Mapillary + * source is supplied; else none. */ function defaultProviders(sources) { if (sources.mapillary == null) return []; @@ -21,6 +22,7 @@ function defaultProviders(sources) { createMapillaryProvider({ source: sources.mapillary }), createGoogleProvider({ getApiKey: googleApiKey }), createKartaViewProvider(), + createPanoramaxProvider(), ]; } diff --git a/src/app/layers/streetLevel.test.mjs b/src/app/layers/streetLevel.test.mjs index c2c3ebb16..16e2cc4b5 100644 --- a/src/app/layers/streetLevel.test.mjs +++ b/src/app/layers/streetLevel.test.mjs @@ -28,7 +28,7 @@ test('explicit providers replace the default source and an empty list removes it ); }); -test('the defaults are Mapillary (given its source), Street View and KartaView, which start off and travel as provider switches', async () => { +test('the defaults are Mapillary (given its source), Street View, KartaView and Panoramax, which start off and travel as provider switches', async () => { const { createDefaultLayerState } = await import('../../data/layerState.js'); const mapillary = { getStatus() {}, @@ -37,7 +37,12 @@ test('the defaults are Mapillary (given its source), Street View and KartaView, nearestImages() {}, }; const layer = createApplicationStreetLevel({ sources: { mapillary } }); - assert.deepEqual(layer.providerIds, ['mapillary', 'google', 'kartaview']); + assert.deepEqual(layer.providerIds, [ + 'mapillary', + 'google', + 'kartaview', + 'panoramax', + ]); // A fresh page and a restored default agree: Street View is billed, so off. layer.setParams(createDefaultLayerState().options['street-level']); assert.equal(layer.getParams().google, false); @@ -45,12 +50,20 @@ test('the defaults are Mapillary (given its source), Street View and KartaView, // A link that lit Street View carries it in the shared switch field. layer.setParams({ providerSwitches: 'google-1' }); assert.equal(layer.getParams().google, true); - assert.equal(layer.getParams().providerSwitches, 'google-1*kartaview-0'); + assert.equal( + layer.getParams().providerSwitches, + 'google-1*kartaview-0*panoramax-0', + ); // KartaView is keyless and free, but one source shows at a time. layer.setParams({ providerSwitches: 'kartaview-1' }); assert.equal(layer.getParams().kartaview, true); assert.equal(layer.getParams().google, false); assert.equal(layer.getParams().mapillary, false); + // As is Panoramax: lighting it puts KartaView out. + layer.setParams({ providerSwitches: 'panoramax-1' }); + assert.equal(layer.getParams().panoramax, true); + assert.equal(layer.getParams().kartaview, false); + assert.equal(layer.getParams().mapillary, false); // Without the application's Street Level source there are none. assert.deepEqual(createApplicationStreetLevel({}).providerIds, []); }); diff --git a/src/layers/streetLevel/limiter.js b/src/layers/streetLevel/limiter.js new file mode 100644 index 000000000..c986bf356 --- /dev/null +++ b/src/layers/streetLevel/limiter.js @@ -0,0 +1,49 @@ +/** + * A gate letting at most `limit` tasks run at once, first in first out, or + * newest first with `lifo` (the last request is the one the user waits on). + */ +export function createLimiter(limit, { lifo = false } = {}) { + let running = 0; + const queue = []; + function next() { + while (running < limit && queue.length) { + const { task, resolve, reject, signal } = lifo + ? queue.pop() + : queue.shift(); + if (signal?.aborted) { + reject(signal.reason); + continue; + } + running++; + Promise.resolve() + .then(task) + .then(resolve, reject) + .finally(() => { + running--; + next(); + }); + } + } + return { + run(task, { signal } = {}) { + return new Promise((resolve, reject) => { + const item = { task, resolve, reject, signal }; + queue.push(item); + signal?.addEventListener( + 'abort', + () => { + const at = queue.indexOf(item); + if (at === -1) return; + queue.splice(at, 1); + reject(signal.reason); + }, + { once: true }, + ); + next(); + }); + }, + get pending() { + return running + queue.length; + }, + }; +} diff --git a/src/layers/streetLevel/providers/kartaview/panorama.js b/src/layers/streetLevel/panorama.js similarity index 100% rename from src/layers/streetLevel/providers/kartaview/panorama.js rename to src/layers/streetLevel/panorama.js diff --git a/src/layers/streetLevel/providers/kartaview/panorama.test.mjs b/src/layers/streetLevel/panorama.test.mjs similarity index 100% rename from src/layers/streetLevel/providers/kartaview/panorama.test.mjs rename to src/layers/streetLevel/panorama.test.mjs diff --git a/src/layers/streetLevel/photoViewer.js b/src/layers/streetLevel/photoViewer.js new file mode 100644 index 000000000..49dac9949 --- /dev/null +++ b/src/layers/streetLevel/photoViewer.js @@ -0,0 +1,705 @@ +import { + createPanoramaRenderer, + PANO_FOV_DEFAULT_DEG, + wrapDegrees, +} from './panorama.js'; + +/** A photo download (tens of kilobytes to a few megabytes). */ +export const IMAGE_TIMEOUT_MS = 15_000; + +/** Panorama texture widths: enough for a docked viewer, and for an expanded one. */ +export const PANO_MID_SIZE = Object.freeze([2048, 1024]); +export const PANO_FULL_SIZE = Object.freeze([4096, 2048]); + +/** + * Load an image within the timeout. A CORS-clean one (for WebGL) is decoded + * off the main thread into an ImageBitmap where the browser can: uploading + * a decoded bitmap costs a fraction of uploading an (90 ms → 15 ms for + * 4096×2048 on a phone-class CPU). `resize` shrinks a bitmap while decoding + * it (a panorama larger than any texture the viewer uploads). + * @param {string} url + * @param {{crossOrigin?: boolean, timeoutMs?: number, resize?: [number, number]|null, failure?: string}} [options] + */ +export function loadImage( + url, + { + crossOrigin = true, + timeoutMs = IMAGE_TIMEOUT_MS, + resize = null, + failure = 'The photo could not be loaded', + } = {}, +) { + if ( + crossOrigin && + url && + typeof globalThis.createImageBitmap === 'function' && + typeof globalThis.fetch === 'function' + ) + return loadBitmap(url, timeoutMs, resize, failure); + return new Promise((resolve, reject) => { + if (!url) return reject(new Error(failure)); + const image = new Image(); + if (crossOrigin) image.crossOrigin = 'anonymous'; + image.decoding = 'async'; + const timer = setTimeout(() => { + image.src = ''; + reject(new Error(failure)); + }, timeoutMs); + image.onload = () => { + clearTimeout(timer); + resolve(image); + }; + image.onerror = () => { + clearTimeout(timer); + reject(new Error(failure)); + }; + image.src = url; + }); +} + +async function loadBitmap(url, timeoutMs, resize, failure) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { + mode: 'cors', + signal: controller.signal, + }); + if (!response.ok) throw new Error(failure); + const blob = await response.blob(); + return await (resize + ? createImageBitmap(blob, { + resizeWidth: resize[0], + resizeHeight: resize[1], + resizeQuality: 'high', + }) + : createImageBitmap(blob)); + } catch { + throw new Error(failure); + } finally { + clearTimeout(timer); + } +} + +/** The panorama size a canvas this wide can show sharply at this field of view. */ +export function panoSizeFor(canvasPixels, fovDeg) { + const needed = (Math.max(1, canvasPixels) * 360) / Math.max(1, fovDeg); + return needed > PANO_MID_SIZE[0] ? PANO_FULL_SIZE : PANO_MID_SIZE; +} + +/** + * @typedef {object} PhotoSources Where a provider's photo comes from. + * @property {(photo: object) => boolean} hasPano A 360° photo with an equirectangular image. + * @property {(photo: object) => {url: string, width: number}|null} panoFirst + * The panorama shown first, and its texture width (0 when it is only a preview). + * @property {(photo: object, size: [number, number]) => {url: string, resize?: [number, number]|null, timeoutMs?: number}|null} panoSharp + * A sharper panorama for a canvas that needs `size`, or null. + * @property {(photo: object) => Array} flatFirst Flat versions to try in turn. + * @property {(photo: object, pixels: number) => string|null} flatSharp + * A sharper flat version for a picture this many device pixels wide, or null. + * @property {(photo: object) => {title?: string|null, creator?: string|null, license?: string|null, externalUrl: string, capturedAtPrecision?: 'day'|'month'}} describe + */ + +/** + * Viewer adapter for providers whose photos are plain image files: a flat + * photo in an , a 360° photo on a WebGL panorama (drag to look, wheel + * or pinch to zoom), and ‹ › to step along the sequence (also ← →). Poses + * follow the view direction. + * + * Photos are normalized by the provider: `{id, sequenceId, lon, lat, + * heading, isPano, capturedAt}` plus whatever its `sources` read. + * + * @param {object} options + * @param {string} options.providerId + * @param {string} options.name Provider name for labels and messages, e.g. 'KartaView'. + * @param {PhotoSources} options.sources + * @param {(photoId: string) => Promise} options.getPhoto A normalized photo. + * @param {(photo: object, delta: -1|1) => Promise} options.step The photo `delta` along the sequence, or null at its end. + * @param {(imageId: string) => void} options.onStep Open the photo ‹ › chose (through the core, so loading and errors show). + * @param {(photo: object) => void} [options.onShown] A photo is on screen. + * @param {object} [options.render] Render governor. + * @param {(url: string, options?: object) => Promise} [options.load] + * @param {(canvas: HTMLCanvasElement) => object} [options.createRenderer] + * @returns {import('./registry.js').ViewerAdapter} + */ +export function createPhotoViewer({ + providerId, + name, + sources, + getPhoto, + step, + onStep, + onShown = () => {}, + render, + load = loadImage, + createRenderer = createPanoramaRenderer, +}) { + const LOAD_FAILED = `The ${name} photo could not be loaded`; + const listeners = new Set(); + let root = null; + let picture = null; + let canvas = null; + let prevButton = null; + let nextButton = null; + let renderer = null; + /** WebGL failed once: 360° photos show their front view instead. */ + let rendererFailed = false; + let photo = null; + /** What shows: 'flat', 'pano' or null. */ + let showing = null; + let openSeq = 0; + let renderMode = 'letterbox'; + let drag = null; + let poseFrame = 0; + let removeListeners = []; + /** The document the viewer was built in (new canvases come from it). */ + let documentOf = null; + /** A context loss's look direction, for the reopen of that photo. */ + let restoreView = null; + /** Width of the panorama texture on screen (0: the preview). */ + let panoWidth = 0; + /** Width of a sharper panorama on its way (resizes come in bursts). */ + let panoComing = 0; + /** The sharper flat version on screen or on its way (one download each). */ + let flatSharpUrl = null; + /** The photo an open is fetching: ‹ › step on from it, not the old one. */ + let openingId = null; + /** ‹ › chose this photo: keep the panorama's look direction for it. */ + let keepViewFor = null; + /** ‹ › presses run one after another, each from where the last one went. */ + let steps = Promise.resolve(); + + const loadAs = (url, options) => + load(url, { failure: LOAD_FAILED, ...options }); + + function requestRender() { + render?.governorRequestRender?.(`${providerId}-viewer`); + } + + function emit() { + if (!photo || !showing) return; + const view = showing === 'pano' ? renderer.getView() : null; + const heading = photo.heading; + const described = sources.describe(photo); + const pose = { + providerId, + imageId: photo.id, + position: { lon: photo.lon, lat: photo.lat }, + bearing: + heading === null || heading === undefined + ? null + : view + ? wrapDegrees(heading + view.yaw) + : heading, + tilt: view ? view.pitch : 0, + altitude: null, + isPano: showing === 'pano', + capturedAt: photo.capturedAt, + capturedAtPrecision: described.capturedAtPrecision || 'day', + title: described.title ?? null, + creator: described.creator ?? null, + license: described.license ?? null, + sequenceId: photo.sequenceId, + externalUrl: described.externalUrl, + }; + for (const listener of [...listeners]) { + try { + listener(pose); + } catch { + /* listener errors are the core's to log */ + } + } + requestRender(); + } + + /** Re-emit on the next frame: a drag moves the view many times a frame. */ + function emitSoon() { + if (poseFrame) return; + poseFrame = requestAnimationFrame(() => { + poseFrame = 0; + emit(); + }); + } + + function syncButtons() { + const enabled = Boolean(photo?.sequenceId) && showing !== null; + if (prevButton) prevButton.disabled = !enabled; + if (nextButton) nextButton.disabled = !enabled; + if (enabled) checkEnds(photo); + } + + /** Disable ‹ or › at once where the sequence ends (it also warms the neighbours). */ + function checkEnds(target) { + for (const [delta, button] of [ + [-1, prevButton], + [1, nextButton], + ]) + step(target, delta).then( + (id) => { + if (photo === target && button && !id) disable(button); + }, + () => {}, + ); + } + + /** Disable a step button, keeping keyboard focus in the viewer. */ + function disable(button) { + if (button.ownerDocument?.activeElement === button) root?.focus?.(); + button.disabled = true; + } + + function show(kind) { + showing = kind; + if (picture) picture.hidden = kind !== 'flat'; + if (canvas) canvas.hidden = kind !== 'pano'; + if (root) root.dataset.kind = kind || ''; + syncButtons(); + } + + function applyRenderMode() { + if (picture) + picture.style.objectFit = renderMode === 'fill' ? 'cover' : 'contain'; + } + + function ensureRenderer() { + if (renderer || rendererFailed || !canvas) return renderer; + try { + renderer = createRenderer(canvas); + } catch { + rendererFailed = true; + renderer = null; + } + return renderer; + } + + /** + * A lost WebGL context (GPU reset, memory pressure) takes the panorama + * with it: drop the renderer, and show the photo again on a new one. + */ + function onContextLost(event) { + event.preventDefault?.(); + const lost = renderer; + renderer = null; + // The new renderer starts facing the photo's heading: keep the user's look. + if (showing === 'pano' && photo && lost) + restoreView = { id: photo.id, view: lost.getView() }; + try { + lost?.destroy(); + } catch { + /* already gone */ + } + // A lost canvas hands back its lost context: draw on a new element. + const fresh = makeCanvas(documentOf); + fresh.hidden = canvas.hidden; + canvas.replaceWith?.(fresh); + canvas = fresh; + // Show the photo again through the core, unless a newer one is opening. + if (showing === 'pano' && photo && openingId === null) { + const again = photo.id; + show(null); + openingId = again; + onStep(again); + } + } + + /** The panorama canvas, wired for drag, pinch, wheel and context loss. */ + function makeCanvas(doc) { + const element = doc.createElement('canvas'); + element.className = 'sl-photo-pano'; + element.hidden = true; + listen(element, 'pointerdown', onPointerDown); + listen(element, 'pointermove', onPointerMove); + listen(element, 'pointerup', onPointerUp); + listen(element, 'pointercancel', onPointerUp); + listen(element, 'wheel', onWheel, { passive: false }); + listen(element, 'webglcontextlost', onContextLost); + return element; + } + + /** + * Step along the sequence from the photo showing, or from the one on its + * way after an earlier press, and open it through the core. + */ + function stepBy(delta) { + steps = steps.then(async () => { + const fromId = openingId ?? photo?.id; + if (!fromId || !root) return; + let from; + let nextId = null; + try { + from = await getPhoto(fromId); + if (from.sequenceId) nextId = await step(from, delta); + } catch { + // Not the end of the sequence, just no answer: the press is lost. + return; + } + // Closed, or another photo was opened meanwhile. + if ((openingId ?? photo?.id) !== fromId) return; + if (!nextId) { + const button = delta < 0 ? prevButton : nextButton; + if (button && photo?.id === fromId) disable(button); + return; + } + openingId = nextId; + keepViewFor = nextId; + onStep(nextId); + }); + } + + /** Pointers on the panorama: one drags the view, two pinch the zoom. */ + const pointers = new Map(); + let pinch = null; + + function pinchSpan() { + const [a, b] = [...pointers.values()]; + return Math.max(1, Math.hypot(a.x - b.x, a.y - b.y)); + } + + function onPointerDown(event) { + if (showing !== 'pano' || !renderer) return; + if (event.pointerType === 'mouse' && event.button !== 0) return; + pointers.set(event.pointerId, { x: event.clientX, y: event.clientY }); + canvas.setPointerCapture?.(event.pointerId); + if (pointers.size === 2) { + drag = null; + pinch = { span: pinchSpan(), fov: renderer.getView().fov }; + return; + } + drag = { + id: event.pointerId, + x: event.clientX, + y: event.clientY, + view: renderer.getView(), + }; + canvas.classList.add('sl-photo-dragging'); + } + + function onPointerMove(event) { + if (pointers.has(event.pointerId)) + pointers.set(event.pointerId, { x: event.clientX, y: event.clientY }); + if (pinch && pointers.size === 2 && renderer) { + renderer.setView({ fov: pinch.fov * (pinch.span / pinchSpan()) }); + return; + } + if (!drag || event.pointerId !== drag.id || !renderer) return; + const { fov } = drag.view; + const width = Math.max(1, canvas.clientWidth); + const height = Math.max(1, canvas.clientHeight); + const vertical = fov; + const horizontal = fov * (width / height); + renderer.setView({ + yaw: drag.view.yaw - ((event.clientX - drag.x) / width) * horizontal, + pitch: drag.view.pitch + ((event.clientY - drag.y) / height) * vertical, + }); + emitSoon(); + } + + function onPointerUp(event) { + pointers.delete(event.pointerId); + if (pointers.size < 2) pinch = null; + if (!drag || event.pointerId !== drag.id) return; + drag = null; + canvas.classList.remove('sl-photo-dragging'); + } + + function onWheel(event) { + if (showing !== 'pano' || !renderer) return; + event.preventDefault(); + const { fov } = renderer.getView(); + renderer.setView({ fov: fov * Math.exp(event.deltaY * 0.001) }); + } + + function onKeyDown(event) { + if ( + event.defaultPrevented || + event.altKey || + event.ctrlKey || + event.metaKey + ) + return; + const pano = showing === 'pano' && renderer; + const turn = { ArrowLeft: -1, ArrowRight: 1 }[event.key]; + // Shift+← → look around a panorama; ← → step along the sequence. + if (turn && event.shiftKey && pano) { + event.preventDefault(); + const { yaw, fov } = renderer.getView(); + renderer.setView({ yaw: yaw + (turn * fov) / 4 }); + emitSoon(); + } else if (turn) { + event.preventDefault(); + stepBy(turn); + } else if ((event.key === 'ArrowUp' || event.key === 'ArrowDown') && pano) { + event.preventDefault(); + const { pitch, fov } = renderer.getView(); + renderer.setView({ + pitch: pitch + ((event.key === 'ArrowUp' ? 1 : -1) * fov) / 6, + }); + emitSoon(); + } else if (['+', '=', '-', '_'].includes(event.key) && pano) { + event.preventDefault(); + const { fov } = renderer.getView(); + renderer.setView({ + fov: fov * (event.key === '+' || event.key === '=' ? 0.85 : 1 / 0.85), + }); + } + } + + function listen(target, type, handler, options) { + target.addEventListener(type, handler, options); + removeListeners.push(() => + target.removeEventListener(type, handler, options), + ); + } + + function build(host) { + const doc = host.ownerDocument; + documentOf = doc; + root = doc.createElement('div'); + root.className = 'sl-photo-viewer'; + root.dataset.provider = providerId; + root.tabIndex = 0; + root.setAttribute('role', 'group'); + root.setAttribute( + 'aria-label', + `${name} photo. ← and → step along the sequence. On a 360° photo, drag or use Shift+← →, ↑ ↓ to look around, and + − to zoom`, + ); + picture = doc.createElement('img'); + picture.className = 'sl-photo-flat'; + picture.alt = `${name} street-level photo`; + picture.draggable = false; + picture.hidden = true; + canvas = makeCanvas(doc); + const nav = (label, glyph, delta) => { + const button = doc.createElement('button'); + button.type = 'button'; + button.className = `sl-photo-step sl-photo-step-${delta < 0 ? 'prev' : 'next'}`; + button.textContent = glyph; + button.title = label; + button.setAttribute('aria-label', label); + button.disabled = true; + listen(button, 'click', (event) => { + event.stopPropagation(); + stepBy(delta); + }); + return button; + }; + prevButton = nav('Previous photo in this sequence (←)', '‹', -1); + nextButton = nav('Next photo in this sequence (→)', '›', 1); + root.append(picture, canvas, prevButton, nextButton); + host.append(root); + listen(root, 'keydown', onKeyDown); + applyRenderMode(); + } + + function teardown() { + pointers.clear(); + pinch = null; + for (const remove of removeListeners) remove(); + removeListeners = []; + if (poseFrame) cancelAnimationFrame(poseFrame); + poseFrame = 0; + drag = null; + renderer?.destroy(); + renderer = null; + root?.remove(); + root = picture = canvas = prevButton = nextButton = null; + showing = null; + photo = null; + } + + /** Device pixels across the viewer (its picture or canvas). */ + function viewerPixels() { + const ratio = Math.min(globalThis.devicePixelRatio || 1, 2); + const element = showing === 'pano' ? canvas : picture; + return (element?.clientWidth || root?.clientWidth || 0) * ratio; + } + + /** A sharper version once the first is on screen, if nothing replaced it. */ + async function upgrade(target, seq, kind) { + // A 360° photo shown flat (no WebGL) keeps its front view. + if (kind === 'flat' && target.isPano) return; + if (kind === 'pano') { + const size = panoSizeNow(); + if (size[0] <= Math.max(panoWidth, panoComing)) return; + const sharp = sources.panoSharp(target, size); + if (!sharp?.url) return; + panoComing = size[0]; + try { + const image = await loadAs(sharp.url, { + crossOrigin: true, + resize: sharp.resize ?? null, + ...(sharp.timeoutMs ? { timeoutMs: sharp.timeoutMs } : {}), + }); + if (seq !== openSeq || photo !== target || showing !== 'pano') { + image.close?.(); + return; + } + if (!renderer || size[0] <= panoWidth) { + image.close?.(); + return; + } + renderer.setImage(image); + panoWidth = size[0]; + requestRender(); + } catch { + /* the first version stays */ + } finally { + if (panoComing === size[0]) panoComing = 0; + } + return; + } + const url = sources.flatSharp(target, viewerPixels()); + if (!url || url === flatSharpUrl) return; + flatSharpUrl = url; + try { + const image = await loadAs(url, { crossOrigin: false }); + if (seq !== openSeq || photo !== target || showing !== 'flat') return; + picture.src = image.src; + requestRender(); + } catch { + // The first version stays; a later resize may try again. + if (flatSharpUrl === url && photo === target) flatSharpUrl = null; + } + } + + /** The sharp panorama this viewer needs now: 2,048 docked, 4,096 expanded. */ + function panoSizeNow() { + const ratio = Math.min(globalThis.devicePixelRatio || 1, 2); + const width = (canvas?.clientWidth || 0) * ratio; + return panoSizeFor(width, renderer?.getView().fov ?? PANO_FOV_DEFAULT_DEG); + } + + async function showPano(target, seq, keepView) { + // If the panorama fails, the caller shows the front view rather than + // waiting on it twice. + const first = sources.panoFirst(target); + if (!first?.url) throw new Error(LOAD_FAILED); + const image = await loadAs(first.url, { crossOrigin: true }); + if (seq !== openSeq) { + image.close?.(); + return false; + } + // The context may have been lost while this loaded: build on the new + // canvas, or let the caller show the front view. + if (!ensureRenderer()) throw new Error(LOAD_FAILED); + const restore = restoreView?.id === target.id ? restoreView.view : null; + restoreView = null; + if (restore) renderer.setView(restore); + else if (!keepView || showing !== 'pano') + renderer.setView({ yaw: 0, pitch: 0, fov: PANO_FOV_DEFAULT_DEG }); + photo = target; + // Shown first: a hidden canvas measures 0 and would be sized 1×1. + show('pano'); + renderer.setImage(image); + panoWidth = first.width || 0; + panoComing = 0; + return true; + } + + async function showFlat(target, seq) { + const urls = sources.flatFirst(target).filter(Boolean); + let image; + let failure = null; + for (const url of urls) { + try { + image = await loadAs(url, { crossOrigin: false }); + break; + } catch (error) { + failure = error; + if (seq !== openSeq) return false; + } + } + if (!image) throw failure || new Error(LOAD_FAILED); + if (seq !== openSeq) return false; + picture.src = image.src; + flatSharpUrl = null; + photo = target; + show('flat'); + return true; + } + + async function openPhoto(id, seq, keepView) { + const target = await getPhoto(id); + if (seq !== openSeq) return; + let shown = false; + if (sources.hasPano(target) && ensureRenderer()) { + try { + shown = await showPano(target, seq, keepView); + } catch { + // The panorama failed: its front view is better than nothing. + if (seq !== openSeq) return; + shown = await showFlat(target, seq); + } + } else shown = await showFlat(target, seq); + if (!shown) return; + emit(); + onShown(target); + upgrade(target, seq, showing); + } + + const adapter = { + async mount(host) { + if (!host) throw new Error('Street Level viewer has no host element'); + if (root && root.parentNode === host) return; + teardown(); + rendererFailed = false; + build(host); + }, + + /** Resolves once the photo is on screen and its pose was emitted. */ + async open(imageId) { + if (!root) throw new Error(`${name} viewer is not mounted`); + const id = String(imageId); + const seq = ++openSeq; + const keepView = keepViewFor === id; + keepViewFor = null; + openingId = id; + try { + await openPhoto(id, seq, keepView); + } finally { + if (openingId === id && seq === openSeq) openingId = null; + } + }, + + close() { + openSeq++; + openingId = null; + keepViewFor = null; + photo = null; + flatSharpUrl = null; + show(null); + if (picture) picture.removeAttribute('src'); + renderer?.clear?.(); + }, + + unmount() { + openSeq++; + openingId = null; + keepViewFor = null; + flatSharpUrl = null; + teardown(); + }, + + resize() { + renderer?.resize(); + // Expanded: a docked viewer's panorama or flat version is now too soft. + if (showing === 'pano' && photo && panoSizeNow()[0] > panoWidth) + upgrade(photo, openSeq, 'pano'); + else if (showing === 'flat' && photo) upgrade(photo, openSeq, 'flat'); + }, + + /** FIT / FILL frame a flat photo; a panorama has no frame to fit. */ + supportsRenderModes: () => showing !== 'pano', + + setRenderMode(mode) { + renderMode = mode === 'fill' ? 'fill' : 'letterbox'; + applyRenderMode(); + }, + + onPose(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }; + return adapter; +} diff --git a/src/layers/streetLevel/policy.js b/src/layers/streetLevel/policy.js index 84f9e2fc7..0d6fdadb8 100644 --- a/src/layers/streetLevel/policy.js +++ b/src/layers/streetLevel/policy.js @@ -9,6 +9,7 @@ export const PROVIDER_COLORS = Object.freeze({ mapillary: '#05cb63', google: '#4285f4', kartaview: '#635bff', + panoramax: '#ff6f00', }); /** Colours every provider shares: the selection highlight and the marker. */ diff --git a/src/layers/streetLevel/providers/kartaview/policy.js b/src/layers/streetLevel/providers/kartaview/policy.js index 200033449..f4ca5a5d2 100644 --- a/src/layers/streetLevel/providers/kartaview/policy.js +++ b/src/layers/streetLevel/providers/kartaview/policy.js @@ -100,9 +100,6 @@ export const SEQUENCE_PAGE_EDGE = 12; /** Equirectangular panorama sizes: a quick preview, then the sharp one. */ export const PANO_PREVIEW_SIZE = Object.freeze([1024, 512]); -/** Enough for a docked viewer; an expanded one gets the full size. */ -export const PANO_MID_SIZE = Object.freeze([2048, 1024]); -export const PANO_FULL_SIZE = Object.freeze([4096, 2048]); /** A flat photo's first version, then its sharp one. */ export const FLAT_VIEW_SIZE = Object.freeze([1280, 1280]); export const FLAT_FULL_SIZE = Object.freeze([2560, 2560]); diff --git a/src/layers/streetLevel/providers/kartaview/source.js b/src/layers/streetLevel/providers/kartaview/source.js index 9e926a9b0..11637514e 100644 --- a/src/layers/streetLevel/providers/kartaview/source.js +++ b/src/layers/streetLevel/providers/kartaview/source.js @@ -15,6 +15,9 @@ import { TILE_CONCURRENCY, TILE_TIMEOUT_MS, } from './policy.js'; +import { createLimiter } from '../../limiter.js'; + +export { createLimiter } from '../../limiter.js'; /** An HTTP or API failure; `retryAfterSec` is set while KartaView rate-limits. */ export class KartaViewSourceError extends Error { @@ -54,56 +57,6 @@ function retryAfterSeconds(response) { return Number.isFinite(value) && value >= 0 ? value : null; } -/** - * A gate letting at most `limit` tasks run at once, first in first out, or - * newest first with `lifo` (the last request is the one the user waits on). - */ -export function createLimiter(limit, { lifo = false } = {}) { - let running = 0; - const queue = []; - function next() { - while (running < limit && queue.length) { - const { task, resolve, reject, signal } = lifo - ? queue.pop() - : queue.shift(); - if (signal?.aborted) { - reject(signal.reason); - continue; - } - running++; - Promise.resolve() - .then(task) - .then(resolve, reject) - .finally(() => { - running--; - next(); - }); - } - } - return { - run(task, { signal } = {}) { - return new Promise((resolve, reject) => { - const item = { task, resolve, reject, signal }; - queue.push(item); - signal?.addEventListener( - 'abort', - () => { - const at = queue.indexOf(item); - if (at === -1) return; - queue.splice(at, 1); - reject(signal.reason); - }, - { once: true }, - ); - next(); - }); - }, - get pending() { - return running + queue.length; - }, - }; -} - /** * KartaView's public API, browser-side and keyless (it allows any origin). * It speaks HTTP/1.1, so the browser opens six connections to it: tiles take diff --git a/src/layers/streetLevel/providers/kartaview/viewer.js b/src/layers/streetLevel/providers/kartaview/viewer.js index ae04b0f5a..31c052cb9 100644 --- a/src/layers/streetLevel/providers/kartaview/viewer.js +++ b/src/layers/streetLevel/providers/kartaview/viewer.js @@ -1,632 +1,81 @@ import { - createPanoramaRenderer, - PANO_FOV_DEFAULT_DEG, - wrapDegrees, -} from './panorama.js'; + createPhotoViewer, + loadImage as loadAnyImage, +} from '../../photoViewer.js'; import { FLAT_FULL_SIZE, FLAT_VIEW_SIZE, IMAGE_TIMEOUT_MS, + KARTAVIEW_NAME, KARTAVIEW_PROVIDER_ID, - PANO_FULL_SIZE, - PANO_MID_SIZE, PANO_PREVIEW_SIZE, kartaViewPhotoUrl, resizedImageUrl, } from './policy.js'; +export { panoSizeFor } from '../../photoViewer.js'; + const LOAD_FAILED = 'The KartaView photo could not be loaded'; -/** - * Load an image within the timeout. A CORS-clean one (for WebGL) is decoded - * off the main thread into an ImageBitmap where the browser can: uploading - * a decoded bitmap costs a fraction of uploading an (90 ms → 15 ms for - * 4096×2048 on a phone-class CPU). - */ -export function loadImage( - url, - { crossOrigin = true, timeoutMs = IMAGE_TIMEOUT_MS } = {}, -) { - if ( - crossOrigin && - url && - typeof globalThis.createImageBitmap === 'function' && - typeof globalThis.fetch === 'function' - ) - return loadBitmap(url, timeoutMs); - return new Promise((resolve, reject) => { - if (!url) return reject(new Error(LOAD_FAILED)); - const image = new Image(); - if (crossOrigin) image.crossOrigin = 'anonymous'; - image.decoding = 'async'; - const timer = setTimeout(() => { - image.src = ''; - reject(new Error(LOAD_FAILED)); - }, timeoutMs); - image.onload = () => { - clearTimeout(timer); - resolve(image); - }; - image.onerror = () => { - clearTimeout(timer); - reject(new Error(LOAD_FAILED)); - }; - image.src = url; +/** Load a KartaView photo (see photoViewer.js `loadImage`). */ +export function loadImage(url, options = {}) { + return loadAnyImage(url, { + timeoutMs: IMAGE_TIMEOUT_MS, + failure: LOAD_FAILED, + ...options, }); } -async function loadBitmap(url, timeoutMs) { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), timeoutMs); - try { - const response = await fetch(url, { - mode: 'cors', - signal: controller.signal, - }); - if (!response.ok) throw new Error(LOAD_FAILED); - return await createImageBitmap(await response.blob()); - } catch { - throw new Error(LOAD_FAILED); - } finally { - clearTimeout(timer); - } -} - -/** The panorama size a canvas this wide can show sharply at this field of view. */ -export function panoSizeFor(canvasPixels, fovDeg) { - const needed = (Math.max(1, canvasPixels) * 360) / Math.max(1, fovDeg); - return needed > PANO_MID_SIZE[0] ? PANO_FULL_SIZE : PANO_MID_SIZE; -} +/** + * Where KartaView photos come from. Only KartaView's image proxy can shrink + * a 13,000 px panorama, so panoramas and flat upgrades go through it. + * @type {import('../../photoViewer.js').PhotoSources} + */ +const KARTAVIEW_SOURCES = Object.freeze({ + hasPano: (photo) => Boolean(photo.isPano && photo.urls.pano), + panoFirst: (photo) => ({ + url: resizedImageUrl(photo.urls.pano, PANO_PREVIEW_SIZE), + width: 0, + }), + panoSharp: (photo, size) => ({ + url: resizedImageUrl(photo.urls.pano, size), + }), + // The proxy's 1,280 px version is a few hundred kB; KartaView's own + // "large thumbnail" can be 1.7 MB. Either serves when the other fails. + // A 360° photo's front view is only on storage. + flatFirst: (photo) => [ + photo.isPano ? null : resizedImageUrl(photo.urls.full, FLAT_VIEW_SIZE), + photo.urls.view, + ], + flatSharp: (photo) => resizedImageUrl(photo.urls.full, FLAT_FULL_SIZE), + describe: (photo) => ({ + // KartaView's only place name is the sequence's, where the drive began. + title: null, + creator: null, + externalUrl: kartaViewPhotoUrl(photo), + }), +}); /** * Viewer adapter for KartaView photos: a flat photo in an , a 360° - * photo on a WebGL panorama (drag to look, wheel to zoom), and ‹ › to step - * along the sequence (also ← →). Poses follow the view direction. + * photo on a WebGL panorama, and ‹ › along the sequence (photoViewer.js). * * @param {object} options * @param {(photoId: string) => Promise} options.getPhoto A normalized photo (policy.js normalizePhoto). - * @param {(photo: object, delta: -1|1) => Promise} options.step The photo `delta` along the sequence, or null at its end. - * @param {(imageId: string) => void} options.onStep Open the photo ‹ › chose (through the core, so loading and errors show). - * @param {(photo: object) => void} [options.onShown] A photo is on screen. - * @param {object} [options.render] Render governor. + * @param {(photo: object, delta: -1|1) => Promise} options.step + * @param {(imageId: string) => void} options.onStep + * @param {(photo: object) => void} [options.onShown] + * @param {object} [options.render] * @param {(url: string, options?: object) => Promise} [options.load] * @param {(canvas: HTMLCanvasElement) => object} [options.createRenderer] * @returns {import('../../registry.js').ViewerAdapter} */ -export function createKartaViewViewer({ - getPhoto, - step, - onStep, - onShown = () => {}, - render, - load = loadImage, - createRenderer = createPanoramaRenderer, -}) { - const listeners = new Set(); - let root = null; - let picture = null; - let canvas = null; - let prevButton = null; - let nextButton = null; - let renderer = null; - /** WebGL failed once: 360° photos show their front view instead. */ - let rendererFailed = false; - let photo = null; - /** What shows: 'flat', 'pano' or null. */ - let showing = null; - let openSeq = 0; - let renderMode = 'letterbox'; - let drag = null; - let poseFrame = 0; - let removeListeners = []; - /** The document the viewer was built in (new canvases come from it). */ - let documentOf = null; - /** A context loss's look direction, for the reopen of that photo. */ - let restoreView = null; - /** Width of the panorama texture on screen (0: the preview). */ - let panoWidth = 0; - /** The photo an open is fetching: ‹ › step on from it, not the old one. */ - let openingId = null; - /** ‹ › chose this photo: keep the panorama's look direction for it. */ - let keepViewFor = null; - /** ‹ › presses run one after another, each from where the last one went. */ - let steps = Promise.resolve(); - - function requestRender() { - render?.governorRequestRender?.('kartaview-viewer'); - } - - function emit() { - if (!photo || !showing) return; - const view = showing === 'pano' ? renderer.getView() : null; - const heading = photo.heading; - const pose = { - providerId: KARTAVIEW_PROVIDER_ID, - imageId: photo.id, - position: { lon: photo.lon, lat: photo.lat }, - bearing: - heading === null - ? null - : view - ? wrapDegrees(heading + view.yaw) - : heading, - tilt: view ? view.pitch : 0, - altitude: null, - isPano: showing === 'pano', - capturedAt: photo.capturedAt, - capturedAtPrecision: 'day', - // KartaView's only place name is the sequence's, where the drive began. - title: null, - creator: null, - sequenceId: photo.sequenceId, - externalUrl: kartaViewPhotoUrl(photo), - }; - for (const listener of [...listeners]) { - try { - listener(pose); - } catch { - /* listener errors are the core's to log */ - } - } - requestRender(); - } - - /** Re-emit on the next frame: a drag moves the view many times a frame. */ - function emitSoon() { - if (poseFrame) return; - poseFrame = requestAnimationFrame(() => { - poseFrame = 0; - emit(); - }); - } - - function syncButtons() { - const enabled = Boolean(photo?.sequenceId) && showing !== null; - if (prevButton) prevButton.disabled = !enabled; - if (nextButton) nextButton.disabled = !enabled; - if (enabled) checkEnds(photo); - } - - /** Disable ‹ or › at once where the sequence ends (it also warms the neighbours). */ - function checkEnds(target) { - for (const [delta, button] of [ - [-1, prevButton], - [1, nextButton], - ]) - step(target, delta).then( - (id) => { - if (photo === target && button && !id) disable(button); - }, - () => {}, - ); - } - - /** Disable a step button, keeping keyboard focus in the viewer. */ - function disable(button) { - if (button.ownerDocument?.activeElement === button) root?.focus?.(); - button.disabled = true; - } - - function show(kind) { - showing = kind; - if (picture) picture.hidden = kind !== 'flat'; - if (canvas) canvas.hidden = kind !== 'pano'; - if (root) root.dataset.kind = kind || ''; - syncButtons(); - } - - function applyRenderMode() { - if (picture) - picture.style.objectFit = renderMode === 'fill' ? 'cover' : 'contain'; - } - - function ensureRenderer() { - if (renderer || rendererFailed || !canvas) return renderer; - try { - renderer = createRenderer(canvas); - } catch { - rendererFailed = true; - renderer = null; - } - return renderer; - } - - /** - * A lost WebGL context (GPU reset, memory pressure) takes the panorama - * with it: drop the renderer, and show the photo again on a new one. - */ - function onContextLost(event) { - event.preventDefault?.(); - const lost = renderer; - renderer = null; - // The new renderer starts facing the photo's heading: keep the user's look. - if (showing === 'pano' && photo && lost) - restoreView = { id: photo.id, view: lost.getView() }; - try { - lost?.destroy(); - } catch { - /* already gone */ - } - // A lost canvas hands back its lost context: draw on a new element. - const fresh = makeCanvas(documentOf); - fresh.hidden = canvas.hidden; - canvas.replaceWith?.(fresh); - canvas = fresh; - // Show the photo again through the core, unless a newer one is opening. - if (showing === 'pano' && photo && openingId === null) { - const again = photo.id; - show(null); - openingId = again; - onStep(again); - } - } - - /** The panorama canvas, wired for drag, pinch, wheel and context loss. */ - function makeCanvas(doc) { - const element = doc.createElement('canvas'); - element.className = 'sl-kv-pano'; - element.hidden = true; - listen(element, 'pointerdown', onPointerDown); - listen(element, 'pointermove', onPointerMove); - listen(element, 'pointerup', onPointerUp); - listen(element, 'pointercancel', onPointerUp); - listen(element, 'wheel', onWheel, { passive: false }); - listen(element, 'webglcontextlost', onContextLost); - return element; - } - - /** - * Step along the sequence from the photo showing, or from the one on its - * way after an earlier press, and open it through the core. - */ - function stepBy(delta) { - steps = steps.then(async () => { - const fromId = openingId ?? photo?.id; - if (!fromId || !root) return; - let from; - let nextId = null; - try { - from = await getPhoto(fromId); - if (from.sequenceId) nextId = await step(from, delta); - } catch { - // Not the end of the sequence, just no answer: the press is lost. - return; - } - // Closed, or another photo was opened meanwhile. - if ((openingId ?? photo?.id) !== fromId) return; - if (!nextId) { - const button = delta < 0 ? prevButton : nextButton; - if (button && photo?.id === fromId) disable(button); - return; - } - openingId = nextId; - keepViewFor = nextId; - onStep(nextId); - }); - } - - /** Pointers on the panorama: one drags the view, two pinch the zoom. */ - const pointers = new Map(); - let pinch = null; - - function pinchSpan() { - const [a, b] = [...pointers.values()]; - return Math.max(1, Math.hypot(a.x - b.x, a.y - b.y)); - } - - function onPointerDown(event) { - if (showing !== 'pano' || !renderer) return; - if (event.pointerType === 'mouse' && event.button !== 0) return; - pointers.set(event.pointerId, { x: event.clientX, y: event.clientY }); - canvas.setPointerCapture?.(event.pointerId); - if (pointers.size === 2) { - drag = null; - pinch = { span: pinchSpan(), fov: renderer.getView().fov }; - return; - } - drag = { - id: event.pointerId, - x: event.clientX, - y: event.clientY, - view: renderer.getView(), - }; - canvas.classList.add('sl-kv-dragging'); - } - - function onPointerMove(event) { - if (pointers.has(event.pointerId)) - pointers.set(event.pointerId, { x: event.clientX, y: event.clientY }); - if (pinch && pointers.size === 2 && renderer) { - renderer.setView({ fov: pinch.fov * (pinch.span / pinchSpan()) }); - return; - } - if (!drag || event.pointerId !== drag.id || !renderer) return; - const { fov } = drag.view; - const width = Math.max(1, canvas.clientWidth); - const height = Math.max(1, canvas.clientHeight); - const vertical = fov; - const horizontal = fov * (width / height); - renderer.setView({ - yaw: drag.view.yaw - ((event.clientX - drag.x) / width) * horizontal, - pitch: drag.view.pitch + ((event.clientY - drag.y) / height) * vertical, - }); - emitSoon(); - } - - function onPointerUp(event) { - pointers.delete(event.pointerId); - if (pointers.size < 2) pinch = null; - if (!drag || event.pointerId !== drag.id) return; - drag = null; - canvas.classList.remove('sl-kv-dragging'); - } - - function onWheel(event) { - if (showing !== 'pano' || !renderer) return; - event.preventDefault(); - const { fov } = renderer.getView(); - renderer.setView({ fov: fov * Math.exp(event.deltaY * 0.001) }); - } - - function onKeyDown(event) { - if ( - event.defaultPrevented || - event.altKey || - event.ctrlKey || - event.metaKey - ) - return; - const pano = showing === 'pano' && renderer; - const turn = { ArrowLeft: -1, ArrowRight: 1 }[event.key]; - // Shift+← → look around a panorama; ← → step along the sequence. - if (turn && event.shiftKey && pano) { - event.preventDefault(); - const { yaw, fov } = renderer.getView(); - renderer.setView({ yaw: yaw + (turn * fov) / 4 }); - emitSoon(); - } else if (turn) { - event.preventDefault(); - stepBy(turn); - } else if ((event.key === 'ArrowUp' || event.key === 'ArrowDown') && pano) { - event.preventDefault(); - const { pitch, fov } = renderer.getView(); - renderer.setView({ - pitch: pitch + ((event.key === 'ArrowUp' ? 1 : -1) * fov) / 6, - }); - emitSoon(); - } else if (['+', '=', '-', '_'].includes(event.key) && pano) { - event.preventDefault(); - const { fov } = renderer.getView(); - renderer.setView({ - fov: fov * (event.key === '+' || event.key === '=' ? 0.85 : 1 / 0.85), - }); - } - } - - function listen(target, type, handler, options) { - target.addEventListener(type, handler, options); - removeListeners.push(() => - target.removeEventListener(type, handler, options), - ); - } - - function build(host) { - const doc = host.ownerDocument; - documentOf = doc; - root = doc.createElement('div'); - root.className = 'sl-kv-viewer'; - root.tabIndex = 0; - root.setAttribute('role', 'group'); - root.setAttribute( - 'aria-label', - 'KartaView photo. ← and → step along the sequence. On a 360° photo, drag or use Shift+← →, ↑ ↓ to look around, and + − to zoom', - ); - picture = doc.createElement('img'); - picture.className = 'sl-kv-photo'; - picture.alt = 'KartaView street-level photo'; - picture.draggable = false; - picture.hidden = true; - canvas = makeCanvas(doc); - const nav = (label, glyph, delta) => { - const button = doc.createElement('button'); - button.type = 'button'; - button.className = `sl-kv-step sl-kv-step-${delta < 0 ? 'prev' : 'next'}`; - button.textContent = glyph; - button.title = label; - button.setAttribute('aria-label', label); - button.disabled = true; - listen(button, 'click', (event) => { - event.stopPropagation(); - stepBy(delta); - }); - return button; - }; - prevButton = nav('Previous photo in this sequence (←)', '‹', -1); - nextButton = nav('Next photo in this sequence (→)', '›', 1); - root.append(picture, canvas, prevButton, nextButton); - host.append(root); - listen(root, 'keydown', onKeyDown); - applyRenderMode(); - } - - function teardown() { - pointers.clear(); - pinch = null; - for (const remove of removeListeners) remove(); - removeListeners = []; - if (poseFrame) cancelAnimationFrame(poseFrame); - poseFrame = 0; - drag = null; - renderer?.destroy(); - renderer = null; - root?.remove(); - root = picture = canvas = prevButton = nextButton = null; - showing = null; - photo = null; - } - - /** A sharper version once the first is on screen, if nothing replaced it. */ - async function upgrade(target, seq, kind) { - // A 360° photo shown flat (no WebGL) keeps its front view. - if (kind === 'flat' && target.isPano) return; - const size = kind === 'pano' ? panoSizeNow() : FLAT_FULL_SIZE; - if (kind === 'pano' && size[0] <= panoWidth) return; - const url = - kind === 'pano' - ? resizedImageUrl(target.urls.pano, size) - : resizedImageUrl(target.urls.full, size); - if (!url) return; - try { - const image = await load(url, { crossOrigin: kind === 'pano' }); - if (seq !== openSeq || photo !== target || showing !== kind) return; - if (kind === 'pano') { - if (!renderer || size[0] <= panoWidth) return; - renderer.setImage(image); - panoWidth = size[0]; - } else picture.src = image.src; - requestRender(); - } catch { - /* the first version stays */ - } - } - - /** The sharp panorama this viewer needs now: 2,048 docked, 4,096 expanded. */ - function panoSizeNow() { - const ratio = Math.min(globalThis.devicePixelRatio || 1, 2); - const width = (canvas?.clientWidth || 0) * ratio; - return panoSizeFor(width, renderer?.getView().fov ?? PANO_FOV_DEFAULT_DEG); - } - - async function showPano(target, seq, keepView) { - // Only the proxy can shrink a 13,000 px panorama: if it fails, the - // caller shows the front view rather than waiting on it twice. - const image = await load( - resizedImageUrl(target.urls.pano, PANO_PREVIEW_SIZE), - { crossOrigin: true }, - ); - if (seq !== openSeq) return false; - // The context may have been lost while this loaded: build on the new - // canvas, or let the caller show the front view. - if (!ensureRenderer()) throw new Error(LOAD_FAILED); - const restore = restoreView?.id === target.id ? restoreView.view : null; - restoreView = null; - if (restore) renderer.setView(restore); - else if (!keepView || showing !== 'pano') - renderer.setView({ yaw: 0, pitch: 0, fov: PANO_FOV_DEFAULT_DEG }); - photo = target; - // Shown first: a hidden canvas measures 0 and would be sized 1×1. - show('pano'); - renderer.setImage(image); - panoWidth = 0; - return true; - } - - async function showFlat(target, seq) { - // The proxy's 1,280 px version is a few hundred kB; KartaView's own - // "large thumbnail" can be 1.7 MB. Either serves when the other fails. - // A 360° photo's front view is only on storage. - const proxied = target.isPano - ? null - : resizedImageUrl(target.urls.full, FLAT_VIEW_SIZE); - const sources = [proxied, target.urls.view].filter(Boolean); - let image; - let failure = null; - for (const url of sources) { - try { - image = await load(url, { crossOrigin: false }); - break; - } catch (error) { - failure = error; - if (seq !== openSeq) return false; - } - } - if (!image) throw failure || new Error(LOAD_FAILED); - if (seq !== openSeq) return false; - picture.src = image.src; - photo = target; - show('flat'); - return true; - } - - async function openPhoto(id, seq, keepView) { - const target = await getPhoto(id); - if (seq !== openSeq) return; - let shown = false; - if (target.isPano && target.urls.pano && ensureRenderer()) { - try { - shown = await showPano(target, seq, keepView); - } catch { - // The panorama failed: its front view is better than nothing. - if (seq !== openSeq) return; - shown = await showFlat(target, seq); - } - } else shown = await showFlat(target, seq); - if (!shown) return; - emit(); - onShown(target); - upgrade(target, seq, showing); - } - - const adapter = { - async mount(host) { - if (!host) throw new Error('Street Level viewer has no host element'); - if (root && root.parentNode === host) return; - teardown(); - rendererFailed = false; - build(host); - }, - - /** Resolves once the photo is on screen and its pose was emitted. */ - async open(imageId) { - if (!root) throw new Error('KartaView viewer is not mounted'); - const id = String(imageId); - const seq = ++openSeq; - const keepView = keepViewFor === id; - keepViewFor = null; - openingId = id; - try { - await openPhoto(id, seq, keepView); - } finally { - if (openingId === id && seq === openSeq) openingId = null; - } - }, - - close() { - openSeq++; - openingId = null; - keepViewFor = null; - photo = null; - show(null); - if (picture) picture.removeAttribute('src'); - renderer?.clear?.(); - }, - - unmount() { - openSeq++; - openingId = null; - keepViewFor = null; - teardown(); - }, - - resize() { - renderer?.resize(); - // Expanded: a docked viewer's 2,048 px panorama is now too soft. - if (showing === 'pano' && photo && panoSizeNow()[0] > panoWidth) - upgrade(photo, openSeq, 'pano'); - }, - - /** FIT / FILL frame a flat photo; a panorama has no frame to fit. */ - supportsRenderModes: () => showing !== 'pano', - - setRenderMode(mode) { - renderMode = mode === 'fill' ? 'fill' : 'letterbox'; - applyRenderMode(); - }, - - onPose(listener) { - listeners.add(listener); - return () => listeners.delete(listener); - }, - }; - return adapter; +export function createKartaViewViewer({ load = loadImage, ...options }) { + return createPhotoViewer({ + providerId: KARTAVIEW_PROVIDER_ID, + name: KARTAVIEW_NAME, + sources: KARTAVIEW_SOURCES, + load, + ...options, + }); } diff --git a/src/layers/streetLevel/providers/mapillary/coverageFixture.mjs b/src/layers/streetLevel/providers/mapillary/coverageFixture.mjs index 8fb94821b..9f9f0fa2f 100644 --- a/src/layers/streetLevel/providers/mapillary/coverageFixture.mjs +++ b/src/layers/streetLevel/providers/mapillary/coverageFixture.mjs @@ -1,6 +1,6 @@ import { PbfWriter } from 'pbf'; -const zigzag = (value) => (value << 1) ^ (value >> 31); +export const zigzag = (value) => (value << 1) ^ (value >> 31); /** Tile-local [x, y] (0..extent) of a lon/lat inside tile (x, y, z). */ export function lonLatToTileLocal([lon, lat], { x, y, z }, extent = 4096) { @@ -13,7 +13,7 @@ export function lonLatToTileLocal([lon, lat], { x, y, z }, extent = 4096) { } /** One MVT layer writer with deduplicated keys and appended values. */ -function layerWriter(name, extent) { +export function layerWriter(name, extent) { const keys = new Map(); const values = []; const features = []; @@ -62,7 +62,7 @@ function layerWriter(name, extent) { } /** MVT MoveTo/LineTo commands; the cursor carries over between parts. */ -function lineGeometry(parts) { +export function lineGeometry(parts) { const out = []; let cx = 0; let cy = 0; diff --git a/src/layers/streetLevel/providers/panoramax/decode.js b/src/layers/streetLevel/providers/panoramax/decode.js new file mode 100644 index 000000000..02ff0579f --- /dev/null +++ b/src/layers/streetLevel/providers/panoramax/decode.js @@ -0,0 +1,178 @@ +import { PbfReader } from 'pbf'; +import { VectorTile } from '@mapbox/vector-tile'; +import { tileLocalToLonLat } from '../../tileMath.js'; +import { OVERVIEW_BINS_PER_AXIS, panoramaxTimeMs } from './policy.js'; + +/** Epoch ms of a sequence's day ('2025-08-11'), or 0 when undated. */ +function dayMs(value) { + return panoramaxTimeMs(typeof value === 'string' ? value : '') ?? 0; +} + +/** + * Decode a Panoramax coverage tile (`/api/map/{z}/{x}/{y}.mvt`). + * + * - `sequences` (z7–15): one line per sequence, its parts kept apart so no + * segment is drawn across a gap, with its day and camera type; longest + * first, so a cap on lines drawn drops the shortest. + * - `grid` (z0–6): picture-density cells, as overview points binned to at + * most OVERVIEW_BINS_PER_AXIS² a tile (a z0 tile holds ~70,000 cells). A + * cell has 360° and flat counts but no date. + * - `pictures` (z15) is left for `decodePictures`. + * @param {Uint8Array} bytes + * @param {{x: number, y: number, z: number}} address + * @returns {{sequences: Array<{id: string, parts: Array>, capturedAt: number, isPano: boolean}>, overview: Array<{id: string, lon: number, lat: number, capturedAt: null, isPano: boolean, hasPano: boolean, hasFlat: boolean, weight: number}>}} + */ +export function decodeCoverageTile(bytes, address) { + const result = { sequences: [], overview: [] }; + if (!bytes || !bytes.length) return result; + const tile = new VectorTile(new PbfReader(bytes)); + const { x, y, z } = address; + + const sequences = tile.layers.sequences; + if (sequences) { + const lengths = new Map(); + for (let i = 0; i < sequences.length; i++) { + const feature = sequences.feature(i); + const props = feature.properties || {}; + const parts = []; + let length = 0; + for (const line of feature.loadGeometry()) { + if (line.length < 2) continue; + for (let k = 1; k < line.length; k++) + length += Math.hypot( + line[k].x - line[k - 1].x, + line[k].y - line[k - 1].y, + ); + parts.push( + line.map((point) => + tileLocalToLonLat(point.x, point.y, sequences.extent, x, y, z), + ), + ); + } + if (!parts.length || props.id == null) continue; + const sequence = { + id: String(props.id), + parts, + capturedAt: dayMs(props.date), + isPano: props.type === 'equirectangular', + }; + lengths.set(sequence, length); + result.sequences.push(sequence); + } + result.sequences.sort((a, b) => lengths.get(b) - lengths.get(a)); + } + + const grid = tile.layers.grid; + if (grid) { + const bins = new Map(); + const size = grid.extent / OVERVIEW_BINS_PER_AXIS; + for (let i = 0; i < grid.length; i++) { + const feature = grid.feature(i); + const props = feature.properties || {}; + const point = feature.loadGeometry()?.[0]?.[0]; + if (!point) continue; + const pano = Number(props.nb_360_pictures) || 0; + const flat = Number(props.nb_flat_pictures) || 0; + const weight = Number(props.nb_pictures) || pano + flat; + if (weight <= 0) continue; + const key = `${Math.floor(point.x / size)}:${Math.floor(point.y / size)}`; + const bin = bins.get(key); + if (!bin) { + bins.set(key, { point, weight, pano, flat, top: weight }); + continue; + } + // The bin sits on its busiest cell and counts them all. + bin.weight += weight; + bin.pano += pano; + bin.flat += flat; + if (weight > bin.top) { + bin.top = weight; + bin.point = point; + } + } + for (const [key, bin] of bins) { + const [lon, lat] = tileLocalToLonLat( + bin.point.x, + bin.point.y, + grid.extent, + x, + y, + z, + ); + result.overview.push({ + id: `${z}/${x}/${y}/${key}`, + lon, + lat, + capturedAt: null, + isPano: bin.pano > bin.flat, + hasPano: bin.pano > 0, + hasFlat: bin.flat > 0, + weight: bin.weight, + }); + } + } + return result; +} + +/** A picture's sequence ids: the tile stores them as a JSON array in a string. */ +function sequenceIdsOf(props) { + const raw = props.sequences; + if (typeof raw === 'string' && raw.startsWith('[')) { + try { + const ids = JSON.parse(raw); + if (Array.isArray(ids)) return ids.map(String); + } catch { + /* fall through to the first sequence */ + } + } + return props.first_sequence != null ? [String(props.first_sequence)] : []; +} + +/** + * The pictures of a z15 tile that belong to `sequenceId`, as cones draw + * them (unsorted). + * @param {Uint8Array} bytes + * @param {{x: number, y: number, z: number}} address + * @param {string} sequenceId + * @returns {Array<{id: string, lon: number, lat: number, compassAngle: number, capturedAt: number, isPano: boolean}>} + */ +export function decodePictures(bytes, address, sequenceId) { + if (!bytes || !bytes.length) return []; + const tile = new VectorTile(new PbfReader(bytes)); + const layer = tile.layers.pictures; + if (!layer) return []; + const { x, y, z } = address; + const wanted = String(sequenceId); + const out = []; + for (let i = 0; i < layer.length; i++) { + const feature = layer.feature(i); + const props = feature.properties || {}; + // A cheap test first: most pictures in a tile are other sequences'. + if ( + typeof props.sequences === 'string' && + !props.sequences.includes(wanted) + ) + continue; + if (!sequenceIdsOf(props).includes(wanted) || props.id == null) continue; + const point = feature.loadGeometry()?.[0]?.[0]; + if (!point) continue; + const [lon, lat] = tileLocalToLonLat( + point.x, + point.y, + layer.extent, + x, + y, + z, + ); + const heading = Number(props.heading); + out.push({ + id: String(props.id), + lon, + lat, + compassAngle: Number.isFinite(heading) ? heading : 0, + capturedAt: panoramaxTimeMs(String(props.ts ?? '')) ?? 0, + isPano: props.type === 'equirectangular', + }); + } + return out; +} diff --git a/src/layers/streetLevel/providers/panoramax/decode.test.mjs b/src/layers/streetLevel/providers/panoramax/decode.test.mjs new file mode 100644 index 000000000..397380dad --- /dev/null +++ b/src/layers/streetLevel/providers/panoramax/decode.test.mjs @@ -0,0 +1,173 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { decodeCoverageTile, decodePictures } from './decode.js'; +import { OVERVIEW_BINS_PER_AXIS } from './policy.js'; +import { encodePanoramaxTile } from '../../../../testSupport/panoramaxFixtures.mjs'; +import { tileCentre } from '../../../../testSupport/streetLevelFakes.mjs'; + +const near = (a, b, eps = 1e-4) => Math.abs(a - b) < eps; + +test('sequence lines keep their parts apart, with their day and camera type', () => { + const { tile, lon, lat } = tileCentre(2.35, 48.85); + const bytes = encodePanoramaxTile(tile, { + sequences: [ + { + id: 'pano-seq', + type: 'equirectangular', + date: '2024-03-05', + parts: [ + [ + [lon - 0.001, lat], + [lon, lat], + ], + [ + [lon + 0.0005, lat], + [lon + 0.001, lat], + ], + ], + }, + { + id: 'flat-seq', + parts: [ + [ + [lon, lat - 0.001], + [lon, lat], + ], + ], + }, + ], + }); + const { sequences, overview } = decodeCoverageTile(bytes, tile); + assert.equal(overview.length, 0); + assert.equal(sequences.length, 2); + const pano = sequences.find((s) => s.id === 'pano-seq'); + const flat = sequences.find((s) => s.id === 'flat-seq'); + assert.equal(pano.isPano, true); + assert.equal(pano.capturedAt, Date.UTC(2024, 2, 5)); + assert.equal(pano.parts.length, 2, 'the gap is not bridged'); + assert.ok(near(pano.parts[1][1][0], lon + 0.001)); + assert.equal(flat.isPano, false); +}); + +test('density cells bin to a bounded number of points that keep the counts and kinds', () => { + const tile = { x: 0, y: 0, z: 0 }; + // Two cells in one bin, one far away, one empty. + const grid = [ + { lon: 2.35, lat: 48.85, pano: 0, flat: 5 }, + { lon: 2.36, lat: 48.85, pano: 30, flat: 0 }, + { lon: -73.9, lat: 40.7, pano: 1, flat: 0 }, + { lon: 10, lat: 10, pano: 0, flat: 0 }, + ]; + const { overview } = decodeCoverageTile( + encodePanoramaxTile(tile, { grid }), + tile, + ); + assert.equal(overview.length, 2, 'Paris cells share a bin; empty cells go'); + const paris = overview.find((p) => p.lon > 0); + assert.equal(paris.weight, 35); + assert.equal(paris.hasPano, true); + assert.equal(paris.hasFlat, true); + assert.equal(paris.isPano, true, 'mostly 360°'); + assert.equal(paris.capturedAt, null, 'cells are undated'); + assert.ok(near(paris.lon, 2.36, 0.2), 'sits on the busiest cell'); + const nyc = overview.find((p) => p.lon < 0); + assert.deepEqual([nyc.hasPano, nyc.hasFlat, nyc.weight], [true, false, 1]); +}); + +test('lines come longest first, so a cap on lines drawn drops the shortest', () => { + const { tile, lon, lat } = tileCentre(2.35, 48.85); + const span = (id, metres) => ({ + id, + parts: [ + [ + [lon, lat], + [lon + metres / 73_000, lat], + ], + ], + }); + const { sequences } = decodeCoverageTile( + encodePanoramaxTile(tile, { + sequences: [span('short', 20), span('long', 900), span('mid', 300)], + }), + tile, + ); + assert.deepEqual( + sequences.map((s) => s.id), + ['long', 'mid', 'short'], + ); +}); + +test('a dense grid tile never draws more than the bins allow', () => { + const tile = { x: 0, y: 0, z: 0 }; + const grid = []; + for (let i = 0; i < 300; i++) + for (let j = 0; j < 60; j++) + grid.push({ lon: -170 + i * 1.1, lat: -60 + j * 2 }); + const { overview } = decodeCoverageTile( + encodePanoramaxTile(tile, { grid }), + tile, + ); + assert.ok(overview.length <= OVERVIEW_BINS_PER_AXIS ** 2); + assert.ok(overview.length > 1000); + assert.equal( + overview.reduce((sum, p) => sum + p.weight, 0), + grid.length, + 'every picture is still counted', + ); +}); + +test('an empty tile decodes to nothing', () => { + for (const bytes of [new Uint8Array(0), null, undefined]) + assert.deepEqual(decodeCoverageTile(bytes, { x: 0, y: 0, z: 0 }), { + sequences: [], + overview: [], + }); + assert.deepEqual( + decodePictures(new Uint8Array(0), { x: 0, y: 0, z: 15 }, 's'), + [], + ); +}); + +test('a z15 tile’s pictures of one sequence become cones; other sequences’ are skipped', () => { + const { tile, lon, lat } = tileCentre(2.35, 48.85, 15); + const bytes = encodePanoramaxTile(tile, { + pictures: [ + { + id: 'a', + lon, + lat, + sequenceId: 's1', + heading: 45, + type: 'equirectangular', + ts: '2023-12-12 07:56:37+00', + }, + { id: 'b', lon: lon + 0.0001, lat, sequenceId: 's2' }, + { id: 'c', lon: lon + 0.0002, lat, sequenceId: 's1' }, + ], + }); + const images = decodePictures(bytes, tile, 's1'); + assert.deepEqual( + images.map((i) => i.id), + ['a', 'c'], + ); + assert.equal(images[0].compassAngle, 45); + assert.equal(images[0].isPano, true); + assert.equal(images[0].capturedAt, Date.UTC(2023, 11, 12, 7, 56, 37)); + assert.ok(near(images[1].lon, lon + 0.0002)); + assert.deepEqual(decodePictures(bytes, tile, 'nope'), []); + // A tile without pictures (below z15) has none to give. + const lines = encodePanoramaxTile(tile, { + sequences: [ + { + id: 's1', + parts: [ + [ + [lon, lat], + [lon + 0.001, lat], + ], + ], + }, + ], + }); + assert.deepEqual(decodePictures(lines, tile, 's1'), []); +}); diff --git a/src/layers/streetLevel/providers/panoramax/index.js b/src/layers/streetLevel/providers/panoramax/index.js new file mode 100644 index 000000000..cda9015c0 --- /dev/null +++ b/src/layers/streetLevel/providers/panoramax/index.js @@ -0,0 +1,461 @@ +import { createLineCoverage, sequenceIdFromPick } from '../../lineCoverage.js'; +import { createSequenceCones, thinImages } from '../../sequenceCones.js'; +import { passesImageryFilter } from '../../filter.js'; +import { + lonToTileX, + latToTileY, + overviewZoomForHeight, + tileBounds, +} from '../../tileMath.js'; +import { metresBetween } from '../../view.js'; +import { decodeCoverageTile, decodePictures } from './decode.js'; +import { createPanoramaxSource } from './source.js'; +import { createPanoramaxViewer } from './viewer.js'; +import { + COLORS, + COVERAGE_LINES_BELOW_M, + COVERAGE_TUNING, + COVERAGE_Z12_BELOW_M, + COVERAGE_Z13_BELOW_M, + COVERAGE_Z14_BELOW_M, + LINE_CLICK_HINT, + NEAREST_LIMIT, + NEAREST_RADIUS_M, + NO_COVERAGE_HINT, + OVERVIEW_HINT, + PANORAMAX_CREDIT_HTML, + PANORAMAX_LABEL, + PANORAMAX_NAME, + PANORAMAX_PROVIDER_ID, + PICK_PREFIX, + PICTURE_TILE_EDGE_M, + PICTURE_TILE_ZOOM, + RATE_LIMITED_MESSAGE, + normalizeItem, +} from './policy.js'; + +export { PANORAMAX_PROVIDER_ID } from './policy.js'; + +/** Pictures kept in memory by id (lookups and ‹ › fill it). */ +const PHOTO_CACHE_SIZE = 2000; + +/** + * Sequence lines at z11–14 below 30 km; above, the density grid at z5 and + * coarser. + */ +export function panoramaxZoomFor(heightM) { + if (!Number.isFinite(heightM)) return null; + if (heightM <= COVERAGE_LINES_BELOW_M) { + const zoom = + heightM <= COVERAGE_Z14_BELOW_M + ? 14 + : heightM <= COVERAGE_Z13_BELOW_M + ? 13 + : heightM <= COVERAGE_Z12_BELOW_M + ? 12 + : 11; + return { zoom, kind: 'sequence' }; + } + return { + zoom: Math.min(5, overviewZoomForHeight(heightM) ?? 5), + kind: 'overview', + }; +} + +/** + * The imagery filter for lines (dated, typed) and grid cells (undated, with + * 360° and flat counts): a cell stays while it holds pictures of the kind + * asked for. + */ +export function passesCoverage(record, filter) { + if ('hasPano' in record) { + if (filter?.pano === 'pano') return record.hasPano; + if (filter?.pano === 'flat') return record.hasFlat; + return true; + } + return passesImageryFilter(record, filter); +} + +/** Least-recently-used map: reading an entry renews it. */ +function createLru(limit) { + const map = new Map(); + return { + get(key) { + if (!map.has(key)) return undefined; + const value = map.get(key); + map.delete(key); + map.set(key, value); + return value; + }, + set(key, value) { + map.delete(key); + map.set(key, value); + while (map.size > limit) map.delete(map.keys().next().value); + }, + delete: (key) => map.delete(key), + }; +} + +/** The z15 tile holding a point. */ +function pictureTileOf({ lon, lat }) { + const z = PICTURE_TILE_ZOOM; + return { x: lonToTileX(lon, z), y: latToTileY(lat, z), z }; +} + +/** + * The z15 tiles whose pictures surround a point: its own, and the + * neighbours it sits within PICTURE_TILE_EDGE_M of (at most four). + */ +export function pictureTilesAround(point) { + const home = pictureTileOf(point); + const box = tileBounds(home.x, home.y, home.z); + const near = (lon, lat) => + metresBetween(point, { lon, lat }) <= PICTURE_TILE_EDGE_M; + const dx = near(box.west, point.lat) ? -1 : near(box.east, point.lat) ? 1 : 0; + const dy = near(point.lon, box.north) + ? -1 + : near(point.lon, box.south) + ? 1 + : 0; + const max = 2 ** home.z; + const tiles = [home]; + const add = (x, y) => { + if (y < 0 || y >= max) return; + tiles.push({ x: (x + max) % max, y, z: home.z }); + }; + if (dx) add(home.x + dx, home.y); + if (dy) add(home.x, home.y + dy); + if (dx && dy) add(home.x + dx, home.y + dy); + return tiles; +} + +const tileKey = ({ x, y, z }) => `${z}/${x}/${y}`; + +/** + * Panoramax Street Level provider: the federated, keyless, open + * street-level commons. Coverage from its vector tiles (density points + * from orbit, sequence lines near the ground); a click on a line opens that + * sequence's picture nearest the click, with cones along the sequence + * around it and ‹ › to step through it. Flat pictures show as photos, 360° + * ones on the shared WebGL panorama. + * @param {{source?: ReturnType, viewerOptions?: object}} [options] + * @returns {import('../../registry.js').StreetLevelProvider} + */ +export function createPanoramaxProvider({ + source = createPanoramaxSource(), + viewerOptions = {}, +} = {}) { + return Object.freeze({ + id: PANORAMAX_PROVIDER_ID, + name: PANORAMAX_NAME, + label: PANORAMAX_LABEL, + requiresKeyId: null, + pickPrefix: PICK_PREFIX.root, + colors: COLORS, + credit: Object.freeze({ html: PANORAMAX_CREDIT_HTML }), + defaultOn: false, + coverageLabel: Object.freeze({ + noun: 'sequences', + action: LINE_CLICK_HINT, + }), + + create(context) { + const state = { + context, + services: context.services, + viewer: null, + keyRequired: false, + keyRejected: false, + statusKnown: true, + coverage: { + zoom: null, + tiles: new Map(), + stale: new Map(), + staleTimer: null, + pending: new Map(), + lastError: null, + holdUntil: 0, + holdTimer: null, + debounceTimer: null, + removeCameraListener: null, + terrainReady: null, + hint: '', + kind: null, + }, + sequence: { + selectedId: null, + images: [], + cache: new Map(), + collection: null, + loading: false, + abort: null, + }, + }; + const photos = createLru(PHOTO_CACHE_SIZE); + /** Where each sequence's cones centre: the picture last opened in it, or a click. */ + const anchors = new Map(); + /** The z15 tiles the cones were drawn from. */ + let drawn = null; + /** A line click: its sequence's pictures win the next nearest lookup there. */ + let preferred = null; + + function remember(item) { + const photo = normalizeItem(item); + if (photo) photos.set(photo.id, photo); + return photo; + } + + /** Lookups in flight, shared: ‹ › and the viewer ask for the same picture. */ + const inflight = new Map(); + function getPhoto(photoId) { + const id = String(photoId); + const cached = photos.get(id); + if (cached) return Promise.resolve(cached); + let lookup = inflight.get(id); + if (!lookup) { + lookup = source + .getItem(id) + .then((item) => { + const photo = item && remember(item); + if (!photo) + throw new Error('Panoramax has no position for this picture'); + return photo; + }) + .finally(() => inflight.delete(id)); + inflight.set(id, lookup); + } + return lookup; + } + + /** The picture `delta` along its sequence, from its own links (null at an end). */ + async function step(photo, delta) { + if (!photo?.sequenceId) return null; + return (delta < 0 ? photo.prevId : photo.nextId) || null; + } + + const coverage = createLineCoverage({ + state, + fetchTile: async (tile, { signal }) => + decodeCoverageTile(await source.getTile(tile, { signal }), tile), + zoomFor: panoramaxZoomFor, + colors: COLORS, + pickPrefix: PICK_PREFIX.sequence, + renderKey: 'panoramax-coverage', + tuning: COVERAGE_TUNING, + messages: { rateLimited: RATE_LIMITED_MESSAGE }, + passes: passesCoverage, + }); + + const cones = createSequenceCones({ + state, + async loadImages(sequenceId, { signal }) { + const anchor = anchors.get(sequenceId); + if (!anchor) return []; + const tiles = pictureTilesAround(anchor); + const lists = await Promise.all( + tiles.map(async (tile) => + decodePictures( + await source.getTile(tile, { signal }), + tile, + sequenceId, + ), + ), + ); + signal?.throwIfAborted(); + drawn = { sequenceId, tiles: new Set(tiles.map(tileKey)) }; + const byId = new Map(lists.flat().map((image) => [image.id, image])); + return thinImages( + [...byId.values()].sort((a, b) => a.capturedAt - b.capturedAt), + ); + }, + colors: COLORS, + pickPrefix: PICK_PREFIX.image, + spriteId: 'street-level:panoramax-cones', + renderKey: 'panoramax-sequence', + onHighlight: (sequenceId, selected) => + coverage.recolorSequence(sequenceId, selected), + }); + + /** + * The cones show the z15 tiles around one picture: once a picture of + * the selected sequence opens outside them (‹ ›, a cone), re-centre. + */ + function recentreCones(photo) { + if (!photo?.sequenceId) return; + if (state.sequence.selectedId !== photo.sequenceId) return; + if (drawn?.sequenceId !== photo.sequenceId) return; + if (drawn.tiles.has(tileKey(pictureTileOf(photo)))) return; + cones.select(photo.sequenceId, { force: true }); + } + + const viewer = createPanoramaxViewer({ + getPhoto: async (photoId) => { + const photo = await getPhoto(photoId); + if (photo.sequenceId) anchors.set(photo.sequenceId, photo); + return photo; + }, + step, + onStep: (photoId) => + context.actions.openImage(photoId, { frame: false }), + onShown: recentreCones, + render: context.services?.render, + ...viewerOptions, + }); + + /** Nearest of `items` within the search radius of a point, or null. */ + function nearestOf(items, point) { + let best = null; + for (const item of items) { + const photo = remember(item); + if (!photo) continue; + const metres = metresBetween(point, photo); + if (metres > NEAREST_RADIUS_M) continue; + if (!best || metres < best.metres) best = { photo, metres }; + } + return best?.photo ?? null; + } + + return { + async status() { + return { configured: true }; + }, + + init(cesiumViewer) { + state.viewer = cesiumViewer; + cones.ensureCollections(cesiumViewer); + cones.setVisible(false); + }, + + activate(cesiumViewer) { + state.viewer = cesiumViewer; + cones.setVisible(true); + coverage.attach(cesiumViewer); + }, + + deactivate() { + coverage.detach(); + coverage.clear(); + coverage.resetErrors(); + cones.clearSelection(); + cones.setVisible(false); + drawn = null; + preferred = null; + }, + + destroy(cesiumViewer) { + coverage.detach(); + coverage.clear(); + coverage.resetErrors(); + cones.destroy(cesiumViewer); + }, + + refreshCoverage: () => coverage.refresh(), + + /** The filter changed: redraw. Each draw reads it from the context. */ + setFilter() { + coverage.rebuild(); + cones.rerender(); + }, + + setSurface() { + coverage.setSurface(); + cones.rerender(); + }, + + coverageStats() { + const lines = state.coverage.kind === 'sequence'; + // Density dots are not sequences: they are not counted as such. + const count = lines ? coverage.sequenceCount() : 0; + const loading = state.coverage.pending.size > 0; + const empty = + lines && !loading && count === 0 && !state.coverage.lastError; + const fallback = empty + ? NO_COVERAGE_HINT + : state.coverage.kind === 'overview' + ? OVERVIEW_HINT + : ''; + return { + count, + zoom: state.coverage.zoom, + kind: state.coverage.kind, + loading, + hint: state.coverage.hint || fallback, + error: state.coverage.lastError, + keyRequired: false, + keyRejected: false, + }; + }, + + /** + * A line opens its sequence's picture nearest the click (the cones + * follow once it is on screen); a cone opens its picture. + */ + handlePick(pickId, { groundPoint } = {}) { + if (pickId.startsWith(PICK_PREFIX.image)) { + context.actions.openImage(pickId.slice(PICK_PREFIX.image.length)); + return true; + } + const sequenceId = sequenceIdFromPick(pickId, PICK_PREFIX.sequence); + if (!sequenceId) return false; + const point = groundPoint?.(); + if (!point) { + // No ground under the click: show the line selected, nothing more. + cones.select(sequenceId); + return true; + } + anchors.set(sequenceId, { ...point, sequenceId }); + preferred = { sequenceId, lon: point.lon, lat: point.lat }; + context.actions.openNearest(point); + return true; + }, + + selectSequence: (sequenceId) => + cones.select(sequenceId, { force: true }), + clearSequence: () => { + cones.clearSelection(); + drawn = null; + }, + sequenceStats: () => ({ + selectedId: state.sequence.selectedId, + images: state.sequence.images.length, + loading: state.sequence.loading, + }), + + /** + * The nearest picture within the search radius that passes the + * imagery filter, or null. After a click on a line, that line's + * sequence is asked first. + */ + async nearestImage({ lat, lon }, { signal } = {}) { + const point = { lat, lon }; + const filter = context.getFilter(); + const wanted = + preferred && metresBetween(preferred, point) < 1 + ? preferred.sequenceId + : null; + preferred = null; + const ask = (collections) => + source.nearestItems( + { + lat, + lon, + radius: NEAREST_RADIUS_M, + limit: NEAREST_LIMIT, + collections, + pano: filter.pano, + sinceMs: filter.sinceMs, + }, + { signal }, + ); + if (wanted) { + const hit = nearestOf(await ask(wanted), point); + if (hit) return hit.id; + } + return nearestOf(await ask(null), point)?.id ?? null; + }, + + viewer, + }; + }, + }); +} diff --git a/src/layers/streetLevel/providers/panoramax/policy.js b/src/layers/streetLevel/providers/panoramax/policy.js new file mode 100644 index 000000000..b163be640 --- /dev/null +++ b/src/layers/streetLevel/providers/panoramax/policy.js @@ -0,0 +1,220 @@ +import { COLORS as SHARED_COLORS, PROVIDER_COLORS } from '../../policy.js'; + +export { NEAREST_RADIUS_M } from '../../policy.js'; + +/** Identity and tuning for the Panoramax street-level provider (keyless). */ +export const PANORAMAX_PROVIDER_ID = 'panoramax'; +export const PANORAMAX_NAME = 'Panoramax'; +export const PANORAMAX_LABEL = 'PANORAMAX'; + +/** + * The Panoramax meta-catalogue: one keyless, CORS-enabled STAC API over + * every federated instance (OpenStreetMap France, IGN, MapComplete…). Its + * pictures are served by the instance that holds them. + */ +export const PANORAMAX_API = 'https://api.panoramax.xyz/api'; +/** The meta-catalogue's web viewer, where the caption link goes. */ +export const PANORAMAX_VIEWER = 'https://api.panoramax.xyz/'; + +/** Stable id prefixes for picked primitives; every one starts with `pnx:`. */ +export const PICK_PREFIX = Object.freeze({ + root: 'pnx:', + sequence: 'pnx:seq:', + image: 'pnx:img:', +}); + +export const COLORS = Object.freeze({ + coverage: PROVIDER_COLORS.panoramax, + selected: SHARED_COLORS.selected, +}); + +/** + * Sequence lines come from z7–15 tiles; z11–14 are drawn (a dense z11 tile + * over Paris is ~1.4 MB and 3,800 sequences, z12 ~800 kB). Higher up, the + * z0–5 `grid` layer's density cells (0.1°, ~10 km) draw as points: below + * 30 km they would be a few dots over a whole city. + */ +export const COVERAGE_Z14_BELOW_M = 1_800; +export const COVERAGE_Z13_BELOW_M = 5_000; +export const COVERAGE_Z12_BELOW_M = 12_000; +export const COVERAGE_LINES_BELOW_M = 30_000; +export const COVERAGE_TUNING = Object.freeze({ + maxTiles: 9, + overviewMaxTiles: 6, + // Each tile draws at most 1,600 lines, its longest first: a dense z11 + // tile over Paris holds 3,800, and a lower cap left the city centre + // looking emptier than its suburbs. (KartaView draws as many.) + maxSequences: 14_400, + lineWidthPx: 2.5, + overviewPointPx: 4, + moveDebounceMs: 320, + viewRangeMinM: 2_500, + viewRangePerHeight: 10, + viewNearM: 1_000, +}); +/** + * Grid cells per overview tile, at most: a z0 tile holds ~70,000 cells + * (1.9 MB), drawn as this many binned points. + */ +export const OVERVIEW_BINS_PER_AXIS = 96; + +export const RATE_LIMITED_MESSAGE = + 'Panoramax is busy — coverage resumes on its own'; +export const NO_COVERAGE_HINT = 'no Panoramax pictures in view'; +/** From high up the dots are where pictures are, not sequences to click. */ +export const OVERVIEW_HINT = 'zoom in to click a Panoramax sequence'; + +/** Cones come from z15 tiles' `pictures` layer (~110 bytes a picture). */ +export const PICTURE_TILE_ZOOM = 15; +/** Within this of a z15 tile's edge, the neighbouring tile's pictures join. */ +export const PICTURE_TILE_EDGE_M = 150; + +export const TILE_TIMEOUT_MS = 20_000; +export const API_TIMEOUT_MS = 12_000; +/** Real tiles are under 6 MB (z6 over France); refuse anything absurd. */ +export const MAX_RESPONSE_BYTES = 16 * 1024 * 1024; +/** Requests at once to the API (tiles have their own budget). */ +export const API_CONCURRENCY = 4; +export const TILE_CONCURRENCY = 6; + +/** Pictures asked for around a point; the API returns them nearest first. */ +export const NEAREST_LIMIT = 10; +/** Any picture within the radius, whichever way it looks (2–180°). */ +export const NEAREST_FOV_TOLERANCE_DEG = 180; + +/** A full-resolution panorama is only fetched up to this width (memory). */ +export const PANO_HD_MAX_WIDTH = 8_192; +/** An HD panorama can be several megabytes. */ +export const HD_TIMEOUT_MS = 30_000; +/** Panoramax's SD derivative is 2,048 px wide, its thumbnail 500 px. */ +export const SD_WIDTH = 2_048; + +/** On-globe credit: licences are per picture, named in the caption. */ +export const PANORAMAX_CREDIT_HTML = + 'Street Level: imagery © Panoramax contributors, licence per picture'; + +export const LINE_CLICK_HINT = 'click a line for the photo there'; + +/** Licence ids Panoramax uses, as the caption names them. */ +const LICENSE_LABELS = Object.freeze({ + 'CC-BY-SA-4.0': 'CC BY-SA 4.0', + 'CC-BY-4.0': 'CC BY 4.0', + 'CC0-1.0': 'CC0', + 'etalab-2.0': 'Licence Ouverte 2.0', +}); + +/** The caption's licence name for a STAC licence id, or null. */ +export function licenseLabel(license) { + if (typeof license !== 'string' || !license.trim()) return null; + const id = license.trim(); + if (id === 'proprietary' || id === 'other') return null; + return LICENSE_LABELS[id] || id.slice(0, 40); +} + +/** Deep link to a picture in the Panoramax web viewer. */ +export function panoramaxPictureUrl(pictureId) { + const id = String(pictureId ?? '').trim(); + if (!id) return PANORAMAX_VIEWER; + return `${PANORAMAX_VIEWER}#${new URLSearchParams({ pic: id, focus: 'pic' })}`; +} + +/** Epoch ms for a capture time ('2023-12-12T07:54:47+00:00', '2023-12-12 07:56:37+00'), or null. */ +export function panoramaxTimeMs(value) { + if (typeof value !== 'string' || !value) return null; + // Tile timestamps are Postgres text: a space and an hour-only offset. + const iso = /\d[T ]\d/.test(value) + ? value + .replace(' ', 'T') + .replace(/([+-]\d{2})$/, '$1:00') + .replace(/\.(\d{3})\d+/, '.$1') + : value; + const ms = Date.parse(iso); + return Number.isFinite(ms) && ms > Date.UTC(1990, 0, 1) ? ms : null; +} + +/** An https address, or null (asset links come from federated instances). */ +function httpsUrl(value) { + try { + const url = new URL(String(value)); + return url.protocol === 'https:' ? url.href : null; + } catch { + return null; + } +} + +/** Last path segment of an item's `rel` link: the linked picture's id. */ +function linkedId(item, rel) { + const href = item?.links?.find?.((link) => link?.rel === rel)?.href; + if (typeof href !== 'string') return null; + const id = href.split(/[?#]/)[0].split('/').filter(Boolean).pop(); + return id ? decodeURIComponent(id) : null; +} + +/** + * Who took the picture. Producers are account names, and some accounts + * are named by an email address: those are not repeated in the caption. + */ +function creatorOf(item) { + const props = item.properties || {}; + const names = [ + props['geovisio:producer'], + ...(Array.isArray(item.providers) + ? item.providers + .filter((p) => p?.roles?.includes?.('producer')) + .map((p) => p.name) + : []), + ]; + const name = names.find((value) => typeof value === 'string' && value.trim()); + if (!name || /\S+@\S+\.\S+/.test(name)) return null; + return name.trim().slice(0, 80); +} + +/** A 360° picture: a full-sphere field of view or an equirectangular projection. */ +export function isEquirectangular(item) { + const props = item?.properties || {}; + const orientation = props['pers:interior_orientation']; + if (Number(orientation?.field_of_view) >= 360) return true; + return props['pers:projection'] === 'equirectangular'; +} + +/** + * A STAC item (from search or a collection) in the shape the provider and + * viewer use, or null without an id or a position. + * @returns {{id: string, sequenceId: string|null, lon: number, lat: number, heading: number|null, isPano: boolean, capturedAt: number|null, creator: string|null, license: string|null, width: number|null, prevId: string|null, nextId: string|null, urls: {thumb: string|null, sd: string|null, hd: string|null}}|null} + */ +export function normalizeItem(item) { + if (!item || item.id == null) return null; + const [lon, lat] = Array.isArray(item.geometry?.coordinates) + ? item.geometry.coordinates.map(Number) + : [NaN, NaN]; + if (!Number.isFinite(lat) || !Number.isFinite(lon)) return null; + if (Math.abs(lat) > 90 || Math.abs(lon) > 180) return null; + const props = item.properties || {}; + const assets = item.assets || {}; + const azimuth = + props['view:azimuth'] === '' || props['view:azimuth'] == null + ? NaN + : Number(props['view:azimuth']); + const width = Number( + props['pers:interior_orientation']?.sensor_array_dimensions?.[0], + ); + return { + id: String(item.id), + sequenceId: item.collection != null ? String(item.collection) : null, + lon, + lat, + heading: Number.isFinite(azimuth) ? ((azimuth % 360) + 360) % 360 : null, + isPano: isEquirectangular(item), + capturedAt: panoramaxTimeMs(props.datetime), + creator: creatorOf(item), + license: licenseLabel(props.license), + width: Number.isFinite(width) && width > 0 ? width : null, + prevId: linkedId(item, 'prev'), + nextId: linkedId(item, 'next'), + urls: { + thumb: httpsUrl(assets.thumb?.href), + sd: httpsUrl(assets.sd?.href), + hd: httpsUrl(assets.hd?.href), + }, + }; +} diff --git a/src/layers/streetLevel/providers/panoramax/policy.test.mjs b/src/layers/streetLevel/providers/panoramax/policy.test.mjs new file mode 100644 index 000000000..5d93d7956 --- /dev/null +++ b/src/layers/streetLevel/providers/panoramax/policy.test.mjs @@ -0,0 +1,195 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + COLORS, + PICK_PREFIX, + isEquirectangular, + licenseLabel, + normalizeItem, + panoramaxPictureUrl, + panoramaxTimeMs, +} from './policy.js'; +import { PROVIDER_COLORS } from '../../policy.js'; +import { stacItem } from '../../../../testSupport/panoramaxFixtures.mjs'; + +test('Panoramax draws in its own orange, and every pick id starts pnx:', () => { + assert.equal(COLORS.coverage, '#ff6f00'); + assert.equal(PROVIDER_COLORS.panoramax, '#ff6f00'); + for (const prefix of Object.values(PICK_PREFIX)) + assert.ok(prefix.startsWith('pnx:')); + // One colour per source. + assert.equal( + new Set(Object.values(PROVIDER_COLORS)).size, + Object.keys(PROVIDER_COLORS).length, + ); +}); + +test('capture times parse in the API’s ISO form, the tiles’ Postgres form, and as bare days', () => { + assert.equal( + panoramaxTimeMs('2023-12-12T07:54:47+00:00'), + Date.UTC(2023, 11, 12, 7, 54, 47), + ); + assert.equal( + panoramaxTimeMs('2023-12-12 07:56:37+00'), + Date.UTC(2023, 11, 12, 7, 56, 37), + ); + assert.equal( + panoramaxTimeMs('2025-08-13 07:45:51.98+00'), + Date.UTC(2025, 7, 13, 7, 45, 51, 980), + ); + // Microseconds are cut to what Date.parse reads. + assert.equal( + panoramaxTimeMs('2025-08-03T14:38:51.838000+00:00'), + Date.UTC(2025, 7, 3, 14, 38, 51, 838), + ); + // A sequence's day: its trailing "-11" is the day, not a UTC offset. + assert.equal(panoramaxTimeMs('2025-08-11'), Date.UTC(2025, 7, 11)); + assert.equal( + panoramaxTimeMs('2025-08-13 09:00:00+02'), + Date.UTC(2025, 7, 13, 7), + ); + for (const bad of ['', 'junk', '1970-01-01', null, 42]) + assert.equal(panoramaxTimeMs(bad), null, String(bad)); +}); + +test('a STAC item becomes a picture: position, heading, 360°, licence, links and assets', () => { + const photo = normalizeItem( + stacItem({ + id: 'a', + lon: 2.35, + lat: 48.85, + sequenceId: 's', + heading: 370, + pano: true, + license: 'etalab-2.0', + prevId: 'p', + nextId: 'n', + }), + ); + assert.deepEqual( + { + id: photo.id, + sequenceId: photo.sequenceId, + lon: photo.lon, + lat: photo.lat, + heading: photo.heading, + isPano: photo.isPano, + license: photo.license, + width: photo.width, + prevId: photo.prevId, + nextId: photo.nextId, + creator: photo.creator, + }, + { + id: 'a', + sequenceId: 's', + lon: 2.35, + lat: 48.85, + heading: 10, + isPano: true, + license: 'Licence Ouverte 2.0', + width: 5760, + prevId: 'p', + nextId: 'n', + creator: 'tester', + }, + ); + assert.equal( + photo.urls.sd, + 'https://panoramax.example.org/pictures/a/sd.jpg', + ); + assert.equal(photo.capturedAt, Date.UTC(2025, 5, 1, 10)); +}); + +test('a picture without links, heading or a usable position', () => { + const bare = normalizeItem({ + id: 7, + geometry: { coordinates: [1, 2] }, + properties: {}, + }); + assert.equal(bare.id, '7'); + assert.equal(bare.heading, null); + assert.equal(bare.prevId, null); + assert.equal(bare.nextId, null); + assert.equal(bare.sequenceId, null); + assert.equal(bare.isPano, false); + assert.equal(bare.capturedAt, null); + assert.deepEqual(bare.urls, { thumb: null, sd: null, hd: null }); + for (const item of [ + null, + { geometry: { coordinates: [1, 2] } }, + { id: 'x', geometry: { coordinates: ['a', 2] } }, + { id: 'x', geometry: { coordinates: [181, 2] } }, + { id: 'x', geometry: { coordinates: [1, 91] } }, + { id: 'x' }, + ]) + assert.equal(normalizeItem(item), null, JSON.stringify(item)); +}); + +test('asset links that are not https are dropped', () => { + const item = stacItem({ id: 'a', lon: 1, lat: 2 }); + item.assets.sd.href = 'http://insecure.example/sd.jpg'; + item.assets.hd.href = 'javascript:alert(1)'; + const photo = normalizeItem(item); + assert.equal(photo.urls.sd, null); + assert.equal(photo.urls.hd, null); + assert.ok(photo.urls.thumb.startsWith('https://')); +}); + +test('an account named by an email address is not shown as the creator', () => { + const named = (producer) => + normalizeItem(stacItem({ id: 'a', lon: 1, lat: 2, producer })).creator; + assert.equal(named('someone@city.example.fr'), null); + assert.equal(named('Robot8A'), 'Robot8A'); + assert.equal(named(' padded '), 'padded'); + assert.equal(named('x'.repeat(200)).length, 80); + // The producer role among the providers serves when the property is missing. + const item = stacItem({ id: 'a', lon: 1, lat: 2 }); + delete item.properties['geovisio:producer']; + item.providers = [ + { name: 'host', roles: ['host'] }, + { name: 'mapper', roles: ['producer'] }, + ]; + assert.equal(normalizeItem(item).creator, 'mapper'); +}); + +test('360°: a full field of view, or the equirectangular projection', () => { + assert.equal( + isEquirectangular({ + properties: { 'pers:interior_orientation': { field_of_view: 360 } }, + }), + true, + ); + assert.equal( + isEquirectangular({ properties: { 'pers:projection': 'equirectangular' } }), + true, + ); + assert.equal( + isEquirectangular({ + properties: { 'pers:interior_orientation': { field_of_view: 92 } }, + }), + false, + ); + assert.equal(isEquirectangular(null), false); +}); + +test('licences read as the caption names them', () => { + assert.equal(licenseLabel('CC-BY-SA-4.0'), 'CC BY-SA 4.0'); + assert.equal(licenseLabel('etalab-2.0'), 'Licence Ouverte 2.0'); + assert.equal(licenseLabel('CC0-1.0'), 'CC0'); + assert.equal(licenseLabel('ODbL-1.0'), 'ODbL-1.0'); + assert.equal(licenseLabel('proprietary'), null); + assert.equal(licenseLabel(''), null); + assert.equal(licenseLabel(undefined), null); + assert.equal(licenseLabel('x'.repeat(100)).length, 40); +}); + +test('the caption link opens the picture in the Panoramax viewer', () => { + assert.equal( + panoramaxPictureUrl('d5467bbb-4357'), + 'https://api.panoramax.xyz/#pic=d5467bbb-4357&focus=pic', + ); + assert.equal(panoramaxPictureUrl(''), 'https://api.panoramax.xyz/'); + assert.equal(panoramaxPictureUrl(null), 'https://api.panoramax.xyz/'); + assert.ok(panoramaxPictureUrl('a&b=c').includes('pic=a%26b%3Dc')); +}); diff --git a/src/layers/streetLevel/providers/panoramax/provider.test.mjs b/src/layers/streetLevel/providers/panoramax/provider.test.mjs new file mode 100644 index 000000000..e0d5eae61 --- /dev/null +++ b/src/layers/streetLevel/providers/panoramax/provider.test.mjs @@ -0,0 +1,526 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + createPanoramaxProvider, + panoramaxZoomFor, + passesCoverage, + pictureTilesAround, +} from './index.js'; +import { NO_COVERAGE_HINT, OVERVIEW_HINT, PICK_PREFIX } from './policy.js'; +import { validateProviders } from '../../registry.js'; +import { tileBounds } from '../../tileMath.js'; +import { fakeHost, until } from '../../../../testSupport/kartaViewFakes.mjs'; +import { + coverageViewer, + tileCentre, +} from '../../../../testSupport/streetLevelFakes.mjs'; +import { + encodePanoramaxTile, + stacItem, +} from '../../../../testSupport/panoramaxFixtures.mjs'; + +const ORIGIN = { lon: 2.3495, lat: 48.8874 }; +/** A point `metres` east of ORIGIN. */ +const east = (metres) => ({ + lon: ORIGIN.lon + metres / 73_300, + lat: ORIGIN.lat, +}); + +/** + * A sequence `seq` of `count` pictures 5 m apart heading east from ORIGIN + * (`offset` m in), each linked to its neighbours. + */ +function sequence(seq, count, { offset = 0, pano = false } = {}) { + return Array.from({ length: count }, (_, i) => { + const at = east(offset + i * 5); + return stacItem({ + id: `${seq}-${i}`, + sequenceId: seq, + ...at, + pano, + prevId: i ? `${seq}-${i - 1}` : null, + nextId: i < count - 1 ? `${seq}-${i + 1}` : null, + }); + }); +} + +/** A source over `items`, answering nearest lookups by distance, recording calls. */ +function fakeSource({ items = [], tiles = () => new Uint8Array(0) } = {}) { + const calls = { nearest: [], items: [], tiles: [] }; + const metres = (a, b) => + Math.hypot((a.lon - b.lon) * 73_300, (a.lat - b.lat) * 111_200); + return { + calls, + getTile: async (tile) => { + calls.tiles.push(`${tile.z}/${tile.x}/${tile.y}`); + return tiles(tile); + }, + getItem: async (id) => { + calls.items.push(id); + return items.find((item) => item.id === id) ?? null; + }, + nearestItems: async (query) => { + calls.nearest.push(query); + const at = { lon: query.lon, lat: query.lat }; + return items + .filter( + (item) => !query.collections || item.collection === query.collections, + ) + .map((item) => ({ + item, + d: metres(at, { + lon: item.geometry.coordinates[0], + lat: item.geometry.coordinates[1], + }), + })) + .filter(({ d }) => d <= query.radius) + .sort((a, b) => a.d - b.d) + .slice(0, query.limit) + .map(({ item }) => item); + }, + }; +} + +function fakeContext({ filter = { pano: 'all', sinceMs: null } } = {}) { + const actions = { opened: [], nearest: [], errors: [], instance: null }; + return { + actions, + context: { + services: {}, + getFilter: () => filter, + isActive: () => true, + groundCaster: null, + meshSampler: null, + getSurface: () => 'draped', + notify() {}, + actions: { + openImage: async (id, options) => { + actions.opened.push({ id, options }); + // As the core does, errors included (a test may not mount a viewer). + await actions.instance?.viewer.open(id).catch((error) => { + actions.errors.push(error.message); + }); + return true; + }, + openNearest: (point) => actions.nearest.push(point), + reportError: (message) => actions.errors.push(message), + }, + }, + }; +} + +function setup(sourceOptions, contextOptions) { + const source = fakeSource(sourceOptions); + const loads = []; + const definition = createPanoramaxProvider({ + source, + viewerOptions: { + load: async (url) => { + loads.push(url); + return { src: url }; + }, + createRenderer: () => { + throw new Error('no WebGL in Node'); + }, + }, + }); + const { context, actions } = fakeContext(contextOptions); + const instance = definition.create(context); + actions.instance = instance; + return { definition, instance, source, actions, loads }; +} + +test('Panoramax registers keyless and off by default, with both filters and its own wording', () => { + const { definition } = setup(); + assert.equal(definition.id, 'panoramax'); + assert.equal(definition.label, 'PANORAMAX'); + assert.equal(definition.requiresKeyId, null); + assert.equal(definition.defaultOn, false); + assert.equal(definition.filters, undefined, 'pano and since both apply'); + assert.notEqual(definition.highlightsSequence, false, 'lines are sequences'); + assert.match(definition.credit.html, /Panoramax<\/a> contributors/); + assert.equal( + definition.coverageLabel.action, + 'click a line for the photo there', + ); + assert.doesNotThrow(() => validateProviders([definition])); +}); + +test('lines at z11–14 below 30 km; density points from z5 down to z0 above', () => { + assert.deepEqual(panoramaxZoomFor(500), { zoom: 14, kind: 'sequence' }); + assert.deepEqual(panoramaxZoomFor(1_800), { zoom: 14, kind: 'sequence' }); + assert.deepEqual(panoramaxZoomFor(4_000), { zoom: 13, kind: 'sequence' }); + assert.deepEqual(panoramaxZoomFor(12_000), { zoom: 12, kind: 'sequence' }); + assert.deepEqual(panoramaxZoomFor(25_000), { zoom: 11, kind: 'sequence' }); + assert.deepEqual(panoramaxZoomFor(30_000), { zoom: 11, kind: 'sequence' }); + assert.deepEqual(panoramaxZoomFor(30_001), { zoom: 5, kind: 'overview' }); + assert.deepEqual(panoramaxZoomFor(100_000), { zoom: 5, kind: 'overview' }); + assert.deepEqual(panoramaxZoomFor(2_000_000), { zoom: 3, kind: 'overview' }); + assert.deepEqual(panoramaxZoomFor(20_000_000), { zoom: 0, kind: 'overview' }); + assert.equal(panoramaxZoomFor(NaN), null); +}); + +test('the imagery filter: lines by type and day, density cells by what they hold', () => { + const old = Date.UTC(2020, 0, 1); + const since = { pano: 'all', sinceMs: Date.UTC(2024, 0, 1) }; + assert.equal( + passesCoverage({ isPano: false, capturedAt: old }, since), + false, + ); + assert.equal( + passesCoverage( + { isPano: true, capturedAt: Date.UTC(2025, 0, 1) }, + { + pano: 'pano', + sinceMs: null, + }, + ), + true, + ); + const cell = { hasPano: false, hasFlat: true, capturedAt: null }; + assert.equal(passesCoverage(cell, { pano: 'pano', sinceMs: null }), false); + assert.equal(passesCoverage(cell, { pano: 'flat', sinceMs: null }), true); + assert.equal(passesCoverage(cell, since), true, 'cells are undated: kept'); + assert.equal(passesCoverage(cell, null), true); +}); + +test('cones read the z15 tile around a picture, and its neighbours near an edge', () => { + const home = tileCentre(ORIGIN.lon, ORIGIN.lat, 15); + assert.equal(pictureTilesAround(home).length, 1, 'mid-tile: one tile'); + const box = tileBounds(home.tile.x, home.tile.y, 15); + const corner = { lon: box.west + 0.0002, lat: box.north - 0.0002 }; + const tiles = pictureTilesAround(corner).map((t) => `${t.x}/${t.y}`); + const { x, y } = home.tile; + assert.deepEqual(tiles, [ + `${x}/${y}`, + `${x - 1}/${y}`, + `${x}/${y - 1}`, + `${x - 1}/${y - 1}`, + ]); + // Across the antimeridian the column wraps. + const dateLine = pictureTilesAround({ lon: -179.9999, lat: 0.5 }); + assert.equal(dateLine[1].x, 2 ** 15 - 1); +}); + +test('the nearest picture within 50 m that passes the filter; filters go to the API', async () => { + const items = [ + ...sequence('far', 1, { offset: 80 }), + ...sequence('a', 3, { offset: 10 }), + ]; + const { instance, source } = setup( + { items }, + { filter: { pano: 'pano', sinceMs: 123 } }, + ); + const id = await instance.nearestImage(east(0)); + assert.equal(id, 'a-0'); + assert.equal(source.calls.nearest.length, 1); + assert.equal(source.calls.nearest[0].pano, 'pano'); + assert.equal(source.calls.nearest[0].sinceMs, 123); + assert.equal(source.calls.nearest[0].collections, null); + assert.equal(await instance.nearestImage(east(300)), null, 'none near'); +}); + +test('a click on a line opens that sequence’s picture nearest the click, not another’s', async () => { + const items = [ + ...sequence('other', 2, { offset: 0 }), + ...sequence('clicked', 4, { offset: 8 }), + ]; + const { instance, actions, source } = setup({ items }); + assert.equal( + instance.handlePick(`${PICK_PREFIX.sequence}clicked~1`, { + groundPoint: () => east(1), + }), + true, + ); + assert.deepEqual(actions.nearest, [east(1)]); + // The core asks the provider for the nearest picture to that point. + assert.equal(await instance.nearestImage(east(1)), 'clicked-0'); + assert.equal(source.calls.nearest[0].collections, 'clicked'); + // Only once: the next lookup (OPEN, another click) is any sequence's. + assert.equal(await instance.nearestImage(east(1)), 'other-0'); +}); + +test('a line click whose sequence has nothing near falls back to any picture there', async () => { + const items = [ + ...sequence('near', 1), + ...sequence('clicked', 1, { offset: 400 }), + ]; + const { instance, source } = setup({ items }); + instance.handlePick(`${PICK_PREFIX.sequence}clicked`, { + groundPoint: () => east(0), + }); + assert.equal(await instance.nearestImage(east(0)), 'near-0'); + assert.deepEqual( + source.calls.nearest.map((q) => q.collections), + ['clicked', null], + ); +}); + +test('a cone opens its picture; foreign ids are not picked', () => { + const { instance, actions } = setup(); + assert.equal(instance.handlePick(`${PICK_PREFIX.image}abc`), true); + assert.equal(actions.opened[0].id, 'abc'); + assert.equal(instance.handlePick('kv:line:1'), false); + assert.equal(instance.handlePick('pnx:other'), false); +}); + +/** A provider and its viewer mounted, opening pictures through the fake core. */ +async function mounted(sourceOptions, contextOptions) { + const env = setup(sourceOptions, contextOptions); + const poses = []; + env.instance.viewer.onPose((pose) => poses.push(pose.imageId)); + const host = fakeHost(); + await env.instance.viewer.mount(host); + const [, , prev, next] = host.children[0].children; + return { ...env, poses, prev, next }; +} + +test('› and ‹ step along the picture’s links without moving the camera, asking only for the next picture', async () => { + const items = sequence('s', 3); + const env = await mounted({ items }); + await env.instance.viewer.open('s-0'); + assert.ok(await until(() => env.prev.disabled), 'the first picture: no ‹'); + env.next.fire('click'); + assert.ok(await until(() => env.poses.at(-1) === 's-1')); + assert.deepEqual(env.actions.opened.at(-1), { + id: 's-1', + options: { frame: false }, + }); + env.next.fire('click'); + assert.ok(await until(() => env.poses.at(-1) === 's-2')); + assert.ok(await until(() => env.next.disabled), 'the last picture: no ›'); + assert.deepEqual(env.source.calls.items, ['s-0', 's-1', 's-2']); + env.prev.fire('click'); + assert.ok(await until(() => env.poses.at(-1) === 's-1')); + assert.equal(env.source.calls.items.length, 3, 's-1 came from the cache'); +}); + +test('one lookup for a picture asked twice at once', async () => { + const items = sequence('s', 1); + const { instance, source } = setup({ items }); + let release; + const gate = new Promise((resolve) => (release = resolve)); + const getItem = source.getItem; + source.getItem = async (id) => { + await gate; + return getItem(id); + }; + const host = fakeHost(); + await instance.viewer.mount(host); + const first = instance.viewer.open('s-0'); + const second = instance.viewer.open('s-0'); + release(); + await Promise.all([first, second]); + assert.deepEqual(source.calls.items, ['s-0']); +}); + +test('a picture Panoramax does not know fails the open with a reason', async () => { + const env = await mounted({ items: [] }); + await assert.rejects(env.instance.viewer.open('ghost'), /no position/); +}); + +/** Cones on a stand-in Cesium viewer; `count()` reads how many are drawn. */ +async function withCones(t, env) { + const Cesium = await import('cesium'); + // The cone glyph draws on a canvas: any drawing call is fine. + const drawing = new Proxy({}, { get: () => () => {}, set: () => true }); + const saved = globalThis.document; + globalThis.document = { + createElement: () => ({ getContext: () => drawing }), + }; + t.after(() => (globalThis.document = saved)); + let collection = null; + env.instance.init({ + scene: { + frameState: { mode: Cesium.SceneMode.SCENE3D }, + updateHeight: () => () => {}, + getHeight: () => undefined, + primitives: { add: (c) => (collection = c), remove() {} }, + preRender: { addEventListener: () => () => {} }, + }, + camera: { positionWC: new Cesium.Cartesian3(0, 0, 1e8) }, + }); + return { count: () => collection?.length ?? 0 }; +} + +/** z15 tiles holding every item of `items` as a picture. */ +const pictureTiles = (items) => (tile) => { + if (tile.z !== 15) return new Uint8Array(0); + const box = tileBounds(tile.x, tile.y, tile.z); + const inside = items.filter( + ({ + geometry: { + coordinates: [lon, lat], + }, + }) => + lon >= box.west && lon < box.east && lat <= box.north && lat > box.south, + ); + return encodePanoramaxTile(tile, { + pictures: inside.map((item) => ({ + id: item.id, + lon: item.geometry.coordinates[0], + lat: item.geometry.coordinates[1], + sequenceId: item.collection, + ts: item.properties.datetime.replace('T', ' ').replace('+00:00', '+00'), + })), + }); +}; + +test('once a picture is on screen its sequence’s cones load from the z15 tile around it', async (t) => { + const items = [...sequence('s', 6), ...sequence('x', 3, { offset: 2 })]; + const env = await mounted({ items, tiles: pictureTiles(items) }); + const cones = await withCones(t, env); + await env.instance.viewer.open('s-2'); + await env.instance.selectSequence('s'); + assert.ok(await until(() => !env.instance.sequenceStats().loading)); + assert.equal(env.instance.sequenceStats().selectedId, 's'); + assert.equal(env.instance.sequenceStats().images, 6, 'only sequence s'); + assert.equal(cones.count(), 6); + assert.ok(env.source.calls.tiles.every((key) => key.startsWith('15/'))); +}); + +test('a sequence selected with no picture or click to centre on draws no cones', async (t) => { + const items = sequence('s', 3); + const env = await mounted({ items, tiles: pictureTiles(items) }); + await withCones(t, env); + env.instance.handlePick(`${PICK_PREFIX.sequence}s`, { + groundPoint: () => null, + }); + assert.ok(await until(() => !env.instance.sequenceStats().loading)); + assert.equal(env.instance.sequenceStats().selectedId, 's', 'highlighted'); + assert.equal(env.instance.sequenceStats().images, 0); + assert.deepEqual(env.source.calls.tiles, []); + assert.deepEqual(env.actions.nearest, []); +}); + +test('the cones re-centre when › walks into another z15 tile', async (t) => { + // 400 pictures 5 m apart span 2 km: more than two z15 tiles (~800 m). + const items = sequence('long', 400); + const env = await mounted({ items, tiles: pictureTiles(items) }); + await withCones(t, env); + await env.instance.viewer.open('long-0'); + await env.instance.selectSequence('long'); + assert.ok(await until(() => !env.instance.sequenceStats().loading)); + const firstTiles = new Set(env.source.calls.tiles); + const first = env.instance.sequenceStats().images; + assert.ok(first > 0 && first < 400, `part of the sequence: ${first}`); + // Jump to the far end: the cones follow. + await env.instance.viewer.open('long-399'); + assert.ok( + await until(() => + env.source.calls.tiles.some((key) => !firstTiles.has(key)), + ), + 'a new tile was read', + ); + assert.ok(await until(() => !env.instance.sequenceStats().loading)); + assert.ok(env.instance.sequenceStats().images > 0); +}); + +/** The provider drawing coverage on a stand-in viewer above one z14 tile. */ +function coverageSetup( + t, + { sequences = [], grid = [], filter, height = 900 } = {}, +) { + const centre = tileCentre(ORIGIN.lon, ORIGIN.lat); + let hold = null; + const env = setup( + { + tiles: (tile) => { + const bytes = encodePanoramaxTile(tile, { sequences, grid }); + return hold ? hold.then(() => bytes) : bytes; + }, + }, + { filter }, + ); + const viewer = coverageViewer({ ...centre, height }, t); + env.instance.init(viewer); + t.after(() => env.instance.destroy(viewer)); + return { + ...env, + viewer, + centre, + holdTiles() { + let release; + hold = new Promise((resolve) => (release = resolve)); + return () => { + hold = null; + release(); + }; + }, + }; +} + +const line = (id, options = {}) => ({ + id, + parts: [ + [ + [ORIGIN.lon - 0.001, ORIGIN.lat], + [ORIGIN.lon + 0.001, ORIGIN.lat], + ], + ], + ...options, +}); + +test('sequence lines draw near the ground, and the imagery filter thins them', async (t) => { + const env = coverageSetup(t, { + sequences: [ + line('new-pano', { type: 'equirectangular', date: '2025-06-01' }), + line('old-flat', { date: '2019-06-01' }), + ], + }); + env.instance.activate(env.viewer); + assert.ok(await until(() => env.instance.coverageStats().count === 2)); + assert.equal(env.instance.coverageStats().kind, 'sequence'); + assert.equal(env.instance.coverageStats().keyRequired, false); + env.instance.deactivate(); + + const filtered = coverageSetup(t, { + filter: { pano: 'pano', sinceMs: null }, + sequences: [ + line('new-pano', { type: 'equirectangular' }), + line('old-flat'), + ], + }); + filtered.instance.activate(filtered.viewer); + assert.ok(await until(() => filtered.instance.coverageStats().count === 1)); +}); + +test('coverage hints: nothing while loading, “no pictures” once an empty view has loaded', async (t) => { + const env = coverageSetup(t); + const release = env.holdTiles(); + env.instance.activate(env.viewer); + assert.ok(await until(() => env.instance.coverageStats().loading)); + assert.equal(env.instance.coverageStats().hint, ''); + release(); + assert.ok(await until(() => !env.instance.coverageStats().loading)); + assert.equal(env.instance.coverageStats().hint, NO_COVERAGE_HINT); +}); + +test('from high up the density grid draws points, not counted as sequences, and the panel says to zoom in', async (t) => { + const env = coverageSetup(t, { + height: 200_000, + grid: [{ lon: ORIGIN.lon, lat: ORIGIN.lat, pano: 2, flat: 3 }], + }); + env.instance.activate(env.viewer); + assert.ok( + await until(() => env.instance.coverageStats().kind === 'overview'), + ); + assert.ok(await until(() => !env.instance.coverageStats().loading)); + assert.equal(env.instance.coverageStats().zoom, 5); + assert.equal(env.instance.coverageStats().count, 0, 'dots are not sequences'); + assert.equal(env.instance.coverageStats().hint, OVERVIEW_HINT); +}); + +test('switching off clears the cones and forgets a pending line click', async (t) => { + const items = sequence('s', 3); + const env = await mounted({ items, tiles: pictureTiles(items) }); + await withCones(t, env); + env.instance.handlePick(`${PICK_PREFIX.sequence}s`, { + groundPoint: () => east(0), + }); + env.instance.deactivate(); + assert.equal(env.instance.sequenceStats().selectedId, null); + // The click's preference is gone: a later lookup asks any sequence. + await env.instance.nearestImage(east(0)); + assert.equal(env.source.calls.nearest.at(-1).collections, null); +}); diff --git a/src/layers/streetLevel/providers/panoramax/source.js b/src/layers/streetLevel/providers/panoramax/source.js new file mode 100644 index 000000000..3ddb6e1e9 --- /dev/null +++ b/src/layers/streetLevel/providers/panoramax/source.js @@ -0,0 +1,184 @@ +import { createLimiter } from '../../limiter.js'; +import { + API_CONCURRENCY, + API_TIMEOUT_MS, + MAX_RESPONSE_BYTES, + NEAREST_FOV_TOLERANCE_DEG, + NEAREST_LIMIT, + NEAREST_RADIUS_M, + PANORAMAX_API, + TILE_CONCURRENCY, + TILE_TIMEOUT_MS, +} from './policy.js'; + +/** An HTTP or API failure; `retryAfterSec` is set while Panoramax is busy. */ +export class PanoramaxSourceError extends Error { + constructor(message, { status = 0, retryAfterSec = null } = {}) { + super(message); + this.name = 'PanoramaxSourceError'; + this.status = status; + this.retryAfterSec = retryAfterSec; + } +} + +const NO_ANSWER = 'Panoramax did not answer in time — try again'; + +/** Seconds from a Retry-After header, or null. */ +function retryAfterSeconds(response) { + const header = response.headers?.get?.('retry-after'); + const value = header == null || header === '' ? NaN : Number(header); + return Number.isFinite(value) && value >= 0 ? value : null; +} + +/** The body as bytes, refusing more than MAX_RESPONSE_BYTES whatever the headers say. */ +async function readBounded(response) { + const tooMuch = () => + new PanoramaxSourceError('Panoramax sent too much data', { + status: response.status, + }); + const length = Number(response.headers?.get?.('content-length')); + if (length > MAX_RESPONSE_BYTES) throw tooMuch(); + const reader = response.body?.getReader?.(); + if (!reader) { + const bytes = new Uint8Array(await response.arrayBuffer()); + if (bytes.byteLength > MAX_RESPONSE_BYTES) throw tooMuch(); + return bytes; + } + const chunks = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > MAX_RESPONSE_BYTES) { + reader.cancel().catch(() => {}); + throw tooMuch(); + } + chunks.push(value); + } + const body = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return body; +} + +/** + * Every network path the Panoramax provider uses, browser-side and keyless: + * the meta-catalogue allows any origin. Vector tiles for coverage and + * cones, STAC search for single pictures and nearest ones. + * @param {{fetchImpl?: typeof fetch, api?: string}} [options] + */ +export function createPanoramaxSource({ + fetchImpl = (...args) => globalThis.fetch(...args), + api = PANORAMAX_API, +} = {}) { + const root = api.replace(/\/+$/, ''); + const apiLimiter = createLimiter(API_CONCURRENCY, { lifo: true }); + const tileLimiter = createLimiter(TILE_CONCURRENCY); + + /** GET within `timeoutMs`; the caller's abort wins over the timeout. */ + async function get(url, { signal, timeoutMs, empty = [] }) { + const controller = new AbortController(); + const onAbort = () => controller.abort(signal.reason); + if (signal?.aborted) throw signal.reason; + signal?.addEventListener('abort', onAbort, { once: true }); + const timer = setTimeout( + () => controller.abort(new PanoramaxSourceError(NO_ANSWER)), + timeoutMs, + ); + try { + const response = await fetchImpl(url, { signal: controller.signal }); + if (empty.includes(response.status)) return null; + if (response.status === 429 || response.status === 503) + throw new PanoramaxSourceError('Panoramax is busy', { + status: response.status, + retryAfterSec: retryAfterSeconds(response) ?? 5, + }); + if (!response.ok) + throw new PanoramaxSourceError( + `Panoramax answered HTTP ${response.status}`, + { status: response.status }, + ); + return await readBounded(response); + } catch (error) { + if (signal?.aborted) throw signal.reason; + if (controller.signal.aborted) throw controller.signal.reason; + if (error instanceof PanoramaxSourceError) throw error; + throw new PanoramaxSourceError(NO_ANSWER); + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + } + } + + /** A STAC search, through the API gate (newest request first). */ + async function search(params, { signal } = {}) { + const url = `${root}/search?${new URLSearchParams(params)}`; + const bytes = await apiLimiter.run( + () => get(url, { signal, timeoutMs: API_TIMEOUT_MS }), + { signal }, + ); + let payload; + try { + payload = JSON.parse(new TextDecoder().decode(bytes)); + } catch { + throw new PanoramaxSourceError('Panoramax sent an unreadable answer'); + } + return Array.isArray(payload?.features) ? payload.features : []; + } + + return { + api: root, + + /** Raw vector-tile bytes; an empty array where there is no coverage. */ + async getTile({ x, y, z }, { signal } = {}) { + const url = `${root}/map/${z}/${x}/${y}.mvt`; + const bytes = await tileLimiter.run( + () => + get(url, { signal, timeoutMs: TILE_TIMEOUT_MS, empty: [204, 404] }), + { signal }, + ); + return bytes ?? new Uint8Array(0); + }, + + /** One picture (a STAC item) by id, or null. */ + async getItem(id, { signal } = {}) { + const [item] = await search({ ids: String(id), limit: '1' }, { signal }); + return item && String(item.id) === String(id) ? item : null; + }, + + /** + * Pictures within `radius` metres of a point, nearest first, whichever + * way they look; `collections` keeps one sequence's, `pano` and + * `sinceMs` filter as the imagery filter does. + */ + async nearestItems( + { + lat, + lon, + radius = NEAREST_RADIUS_M, + limit = NEAREST_LIMIT, + collections = null, + pano = 'all', + sinceMs = null, + }, + { signal } = {}, + ) { + const params = { + place_position: `${Number(lon).toFixed(7)},${Number(lat).toFixed(7)}`, + place_distance: `0-${Math.min(1000, Math.max(1, Math.round(radius)))}`, + place_fov_tolerance: String(NEAREST_FOV_TOLERANCE_DEG), + limit: String(Math.min(100, Math.max(1, limit))), + }; + if (collections) params.collections = String(collections); + if (pano === 'pano') params.filter = 'field_of_view=360'; + else if (pano === 'flat') params.filter = 'field_of_view<360'; + if (Number.isFinite(sinceMs) && sinceMs > 0) + params.datetime = `${new Date(sinceMs).toISOString()}/..`; + return search(params, { signal }); + }, + }; +} diff --git a/src/layers/streetLevel/providers/panoramax/source.test.mjs b/src/layers/streetLevel/providers/panoramax/source.test.mjs new file mode 100644 index 000000000..7f2e87237 --- /dev/null +++ b/src/layers/streetLevel/providers/panoramax/source.test.mjs @@ -0,0 +1,176 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { PanoramaxSourceError, createPanoramaxSource } from './source.js'; +import { MAX_RESPONSE_BYTES } from './policy.js'; + +/** A fetch answering from `respond(url)`, recording every URL asked. */ +function fakeFetch(respond) { + const urls = []; + const fetchImpl = async (url, { signal } = {}) => { + urls.push(url); + const answer = await respond(url, signal); + if (answer instanceof Response) return answer; + const { status = 200, body = '', headers = {} } = answer; + return new Response(status === 204 ? null : body, { status, headers }); + }; + return { urls, fetchImpl }; +} + +const json = (value) => ({ body: JSON.stringify(value) }); + +test('a tile is the raw vector-tile bytes; 204 and 404 mean no coverage', async () => { + const { urls, fetchImpl } = fakeFetch((url) => + url.includes('/14/') + ? { body: new Uint8Array([1, 2, 3]) } + : { status: 204 }, + ); + const source = createPanoramaxSource({ fetchImpl }); + assert.deepEqual( + [...(await source.getTile({ x: 8299, y: 5634, z: 14 }))], + [1, 2, 3], + ); + assert.equal(urls[0], 'https://api.panoramax.xyz/api/map/14/8299/5634.mvt'); + assert.equal((await source.getTile({ x: 1, y: 1, z: 13 })).length, 0); + const missing = createPanoramaxSource({ + fetchImpl: fakeFetch(() => ({ status: 404 })).fetchImpl, + }); + assert.equal((await missing.getTile({ x: 1, y: 1, z: 13 })).length, 0); +}); + +test('a busy API says so and when to come back; other failures name the status', async () => { + const busy = createPanoramaxSource({ + fetchImpl: fakeFetch(() => ({ + status: 429, + headers: { 'retry-after': '12' }, + })).fetchImpl, + }); + await assert.rejects(busy.getTile({ x: 0, y: 0, z: 0 }), (error) => { + assert.ok(error instanceof PanoramaxSourceError); + assert.equal(error.status, 429); + assert.equal(error.retryAfterSec, 12); + return true; + }); + const down = createPanoramaxSource({ + fetchImpl: fakeFetch(() => ({ status: 503 })).fetchImpl, + }); + await assert.rejects(down.getTile({ x: 0, y: 0, z: 0 }), (error) => { + assert.equal(error.retryAfterSec, 5, 'a default wait without the header'); + return true; + }); + const broken = createPanoramaxSource({ + fetchImpl: fakeFetch(() => ({ status: 500 })).fetchImpl, + }); + await assert.rejects(broken.getItem('x'), /HTTP 500/); +}); + +test('an oversized answer is refused while it streams', async () => { + let pulled = 0; + const huge = new ReadableStream({ + pull(controller) { + pulled++; + controller.enqueue(new Uint8Array(4 * 1024 * 1024)); + }, + }); + const source = createPanoramaxSource({ + fetchImpl: async () => new Response(huge, { status: 200 }), + }); + await assert.rejects(source.getTile({ x: 0, y: 0, z: 0 }), /too much data/); + assert.ok(pulled * 4 * 1024 * 1024 <= MAX_RESPONSE_BYTES + 8 * 1024 * 1024); + // A declared length over the cap is refused before reading. + const declared = createPanoramaxSource({ + fetchImpl: fakeFetch(() => ({ + body: 'x', + headers: { 'content-length': String(MAX_RESPONSE_BYTES + 1) }, + })).fetchImpl, + }); + await assert.rejects(declared.getItem('a'), /too much data/); +}); + +test('a picture by id is a STAC search; another id in the answer is not taken for it', async () => { + const { urls, fetchImpl } = fakeFetch((url) => + url.includes('ids=good') + ? json({ features: [{ id: 'good' }] }) + : json({ features: [{ id: 'other' }] }), + ); + const source = createPanoramaxSource({ fetchImpl }); + assert.deepEqual(await source.getItem('good'), { id: 'good' }); + assert.equal(await source.getItem('bad'), null); + const asked = new URL(urls[0]); + assert.equal(asked.pathname, '/api/search'); + assert.equal(asked.searchParams.get('ids'), 'good'); + assert.equal(asked.searchParams.get('limit'), '1'); +}); + +test('nearest pictures: any direction within the radius, filtered as the imagery filter asks', async () => { + const { urls, fetchImpl } = fakeFetch(() => json({ features: [] })); + const source = createPanoramaxSource({ fetchImpl }); + await source.nearestItems({ lat: 48.85, lon: 2.35 }); + await source.nearestItems({ + lat: 48.85, + lon: 2.35, + collections: 'seq', + pano: 'pano', + sinceMs: Date.UTC(2025, 0, 1), + radius: 5000, + limit: 500, + }); + await source.nearestItems({ lat: 48.85, lon: 2.35, pano: 'flat' }); + const [plain, filtered, flat] = urls.map((u) => new URL(u).searchParams); + assert.equal(plain.get('place_position'), '2.3500000,48.8500000'); + assert.equal(plain.get('place_distance'), '0-50'); + assert.equal(plain.get('place_fov_tolerance'), '180'); + assert.equal(plain.get('collections'), null); + assert.equal(plain.get('filter'), null); + assert.equal(plain.get('datetime'), null); + assert.equal(filtered.get('collections'), 'seq'); + assert.equal(filtered.get('filter'), 'field_of_view=360'); + assert.equal(filtered.get('datetime'), '2025-01-01T00:00:00.000Z/..'); + assert.equal(filtered.get('place_distance'), '0-1000', 'capped'); + assert.equal(filtered.get('limit'), '100', 'capped'); + assert.equal(flat.get('filter'), 'field_of_view<360'); +}); + +test('an unreadable search answer, or one with no features, is handled', async () => { + const garbled = createPanoramaxSource({ + fetchImpl: fakeFetch(() => ({ body: '' })).fetchImpl, + }); + await assert.rejects(garbled.getItem('a'), /unreadable/); + const empty = createPanoramaxSource({ + fetchImpl: fakeFetch(() => json({ type: 'FeatureCollection' })).fetchImpl, + }); + assert.deepEqual(await empty.nearestItems({ lat: 0, lon: 0 }), []); +}); + +test('the caller’s abort wins; a silent API times out with a readable message', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const hang = (url, signal) => + new Promise((_, reject) => + signal.addEventListener('abort', () => reject(signal.reason)), + ); + const source = createPanoramaxSource({ + fetchImpl: fakeFetch(hang).fetchImpl, + }); + const controller = new AbortController(); + const asked = source.getItem('a', { signal: controller.signal }); + await Promise.resolve(); + controller.abort(new Error('overtaken')); + await assert.rejects(asked, /overtaken/); + const slow = source.getTile({ x: 0, y: 0, z: 0 }); + // Let the request start before the clock runs out. + for (let i = 0; i < 5; i++) await Promise.resolve(); + t.mock.timers.tick(60_000); + await assert.rejects(slow, /did not answer in time/); +}); + +test('a network failure reads as no answer, not as a stack trace', async () => { + const source = createPanoramaxSource({ + fetchImpl: async () => { + throw new TypeError('Failed to fetch'); + }, + }); + await assert.rejects(source.getTile({ x: 0, y: 0, z: 0 }), (error) => { + assert.ok(error instanceof PanoramaxSourceError); + assert.match(error.message, /did not answer/); + return true; + }); +}); diff --git a/src/layers/streetLevel/providers/panoramax/viewer.js b/src/layers/streetLevel/providers/panoramax/viewer.js new file mode 100644 index 000000000..df666237b --- /dev/null +++ b/src/layers/streetLevel/providers/panoramax/viewer.js @@ -0,0 +1,63 @@ +import { createPhotoViewer, loadImage } from '../../photoViewer.js'; +import { + HD_TIMEOUT_MS, + PANORAMAX_NAME, + PANORAMAX_PROVIDER_ID, + PANO_HD_MAX_WIDTH, + SD_WIDTH, + panoramaxPictureUrl, +} from './policy.js'; + +/** + * Where Panoramax pictures come from: the instance holding each one serves + * a 2,048 px SD derivative (equirectangular for a 360° picture) and the HD + * original. SD shows first; HD only once the viewer is wide enough to use + * it. A 360° original is shrunk to the texture size while it decodes. + * @type {import('../../photoViewer.js').PhotoSources} + */ +export const PANORAMAX_SOURCES = Object.freeze({ + hasPano: (photo) => Boolean(photo.isPano && (photo.urls.sd || photo.urls.hd)), + panoFirst: (photo) => + photo.urls.sd + ? { url: photo.urls.sd, width: SD_WIDTH } + : { url: photo.urls.hd, width: photo.width || SD_WIDTH }, + panoSharp: (photo, size) => { + const width = photo.width || 0; + if (!photo.urls.hd || !photo.urls.sd) return null; + // An original no wider than the SD one adds nothing; a huge one would + // take hundreds of megabytes to decode. + if (width <= SD_WIDTH || width > PANO_HD_MAX_WIDTH) return null; + const target = Math.min(size[0], width); + return { + url: photo.urls.hd, + resize: [target, target / 2], + timeoutMs: HD_TIMEOUT_MS, + }; + }, + // A 360° picture shown flat (no WebGL) is its whole equirectangular image. + flatFirst: (photo) => [photo.urls.sd, photo.urls.hd, photo.urls.thumb], + flatSharp: (photo, pixels) => + photo.urls.sd && photo.urls.hd && pixels > SD_WIDTH ? photo.urls.hd : null, + describe: (photo) => ({ + title: null, + creator: photo.creator, + license: photo.license, + externalUrl: panoramaxPictureUrl(photo.id), + }), +}); + +/** + * Viewer adapter for Panoramax pictures: flat in an , 360° on the + * shared WebGL panorama, ‹ › along the sequence (photoViewer.js). + * @param {object} options See photoViewer.js `createPhotoViewer`. + * @returns {import('../../registry.js').ViewerAdapter} + */ +export function createPanoramaxViewer({ load = loadImage, ...options }) { + return createPhotoViewer({ + providerId: PANORAMAX_PROVIDER_ID, + name: PANORAMAX_NAME, + sources: PANORAMAX_SOURCES, + load, + ...options, + }); +} diff --git a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs new file mode 100644 index 000000000..d1ec2732e --- /dev/null +++ b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs @@ -0,0 +1,251 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { PANORAMAX_SOURCES, createPanoramaxViewer } from './viewer.js'; +import { normalizeItem } from './policy.js'; +import { fakeHost, until } from '../../../../testSupport/kartaViewFakes.mjs'; +import { stacItem } from '../../../../testSupport/panoramaxFixtures.mjs'; + +const photo = (options) => + normalizeItem(stacItem({ lon: 2.35, lat: 48.85, ...options })); + +const PHOTOS = { + flat: photo({ id: 'flat', prevId: 'p0', nextId: 'flat2', heading: 30 }), + flat2: photo({ id: 'flat2', prevId: 'flat', heading: 40 }), + pano: photo({ id: 'pano', pano: true, heading: 100, width: 5760 }), + bigPano: photo({ id: 'bigPano', pano: true, width: 11_000 }), + smallPano: photo({ id: 'smallPano', pano: true, width: 2048 }), + email: photo({ id: 'email', producer: 'someone@city.example.fr' }), +}; + +function fakeRenderer() { + const view = { yaw: 0, pitch: 0, fov: 80 }; + const images = []; + return { + images, + setImage: (image) => images.push(image), + setView(next) { + Object.assign(view, next); + }, + getView: () => ({ ...view }), + resize() {}, + clear() {}, + destroy() {}, + }; +} + +function setup() { + const loads = []; + const renderers = []; + const stepped = []; + const viewer = createPanoramaxViewer({ + getPhoto: async (id) => { + if (!PHOTOS[id]) throw new Error('no such picture'); + return PHOTOS[id]; + }, + step: async (from, delta) => (delta < 0 ? from.prevId : from.nextId), + onStep: (id) => { + stepped.push(id); + viewer.open(id).catch(() => {}); + }, + load: async (url, options) => { + loads.push({ url, ...options }); + return { src: url, width: 1, height: 1 }; + }, + createRenderer: () => { + const renderer = fakeRenderer(); + renderers.push(renderer); + return renderer; + }, + }); + const poses = []; + viewer.onPose((pose) => poses.push(pose)); + return { viewer, poses, loads, renderers, stepped, host: fakeHost() }; +} + +const parts = (host) => { + const root = host.children[0]; + const [picture, canvas, prev, next] = root.children; + return { root, picture, canvas, prev, next }; +}; + +test('a flat picture shows its SD version with author, licence and a Panoramax link', async () => { + const { viewer, poses, loads, host } = setup(); + await viewer.mount(host); + const { root, picture, canvas } = parts(host); + assert.equal(root.dataset.provider, 'panoramax', 'its colour in the CSS'); + assert.match(root.attributes['aria-label'], /^Panoramax photo/); + await viewer.open('flat'); + assert.equal(picture.hidden, false); + assert.equal(canvas.hidden, true); + assert.equal( + picture.src, + 'https://panoramax.example.org/pictures/flat/sd.jpg', + ); + assert.deepEqual( + loads.map((l) => l.url), + [PHOTOS.flat.urls.sd], + ); + const pose = poses.at(-1); + assert.equal(pose.providerId, 'panoramax'); + assert.equal(pose.bearing, 30); + assert.equal(pose.isPano, false); + assert.equal(pose.creator, 'tester'); + assert.equal(pose.license, 'CC BY-SA 4.0'); + assert.equal(pose.sequenceId, 'seq-1'); + assert.equal( + pose.externalUrl, + 'https://api.panoramax.xyz/#pic=flat&focus=pic', + ); + assert.equal(viewer.supportsRenderModes(), true); +}); + +test('no creator in the caption for an account named by an email address', async () => { + const { viewer, poses, host } = setup(); + await viewer.mount(host); + await viewer.open('email'); + assert.equal(poses.at(-1).creator, null); +}); + +test('HD replaces SD only once the viewer is wide enough to use it', async (t) => { + const { viewer, loads, host } = setup(); + await viewer.mount(host); + const { picture, root } = parts(host); + t.after(() => delete globalThis.devicePixelRatio); + globalThis.devicePixelRatio = 1; + picture.clientWidth = 400; + await viewer.open('flat'); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(loads.length, 1, 'docked: SD is sharp enough'); + // EXPAND on a wide screen. + picture.clientWidth = root.clientWidth = 2600; + viewer.resize(); + assert.ok(await until(() => picture.src === PHOTOS.flat.urls.hd)); + viewer.resize(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal( + loads.filter((l) => l.url === PHOTOS.flat.urls.hd).length, + 1, + 'asked once', + ); +}); + +test('a 360° picture shows its SD equirectangular image, then HD shrunk to 4,096 when expanded', async (t) => { + const { viewer, poses, loads, renderers, host } = setup(); + await viewer.mount(host); + const { canvas, picture } = parts(host); + t.after(() => delete globalThis.devicePixelRatio); + globalThis.devicePixelRatio = 1; + canvas.clientWidth = 400; + await viewer.open('pano'); + assert.equal(canvas.hidden, false); + assert.equal(picture.hidden, true); + assert.equal(viewer.supportsRenderModes(), false, 'no frame to fit'); + assert.deepEqual(loads[0], { + url: PHOTOS.pano.urls.sd, + crossOrigin: true, + failure: 'The Panoramax photo could not be loaded', + }); + assert.equal(poses.at(-1).isPano, true); + assert.equal(poses.at(-1).bearing, 100); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(loads.length, 1, 'docked: the 2,048 px SD is enough'); + canvas.clientWidth = 1800; + // EXPAND resizes more than once while the panel settles. + viewer.resize(); + viewer.resize(); + viewer.resize(); + assert.ok(await until(() => renderers[0].images.length === 2)); + assert.equal( + loads.filter((l) => l.url === PHOTOS.pano.urls.hd).length, + 1, + 'the original is fetched once', + ); + const hd = loads.at(-1); + assert.equal(hd.url, PHOTOS.pano.urls.hd); + assert.deepEqual(hd.resize, [4096, 2048]); + assert.equal(hd.timeoutMs, 30_000); +}); + +test('HD panoramas too big to decode, or no bigger than SD, are never fetched', () => { + const size = [4096, 2048]; + assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.bigPano, size), null); + assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.smallPano, size), null); + assert.deepEqual( + PANORAMAX_SOURCES.panoSharp({ ...PHOTOS.pano, width: 3000 }, size).resize, + [3000, 1500], + 'never enlarged', + ); + // Without an SD derivative the HD one is the first and only. + const hdOnly = { ...PHOTOS.pano, urls: { ...PHOTOS.pano.urls, sd: null } }; + assert.deepEqual(PANORAMAX_SOURCES.panoFirst(hdOnly), { + url: hdOnly.urls.hd, + width: 5760, + }); + assert.equal(PANORAMAX_SOURCES.panoSharp(hdOnly, size), null); + assert.equal(PANORAMAX_SOURCES.flatSharp(PHOTOS.flat, 2048), null); + assert.equal( + PANORAMAX_SOURCES.flatSharp(PHOTOS.flat, 2049), + PHOTOS.flat.urls.hd, + ); + assert.equal( + PANORAMAX_SOURCES.hasPano({ isPano: true, urls: { sd: null, hd: null } }), + false, + ); +}); + +test('› and ‹ follow the picture’s own links; the ends disable the buttons', async () => { + const { viewer, poses, stepped, host } = setup(); + await viewer.mount(host); + const { next, prev } = parts(host); + await viewer.open('flat'); + next.fire('click'); + assert.ok(await until(() => poses.at(-1)?.imageId === 'flat2')); + assert.deepEqual(stepped, ['flat2']); + assert.ok(await until(() => next.disabled), 'flat2 is the last'); + assert.equal(prev.disabled, false); +}); + +test('without WebGL a 360° picture shows its whole equirectangular image flat', async () => { + const loads = []; + const viewer = createPanoramaxViewer({ + getPhoto: async () => PHOTOS.pano, + step: async () => null, + onStep() {}, + load: async (url) => { + loads.push(url); + return { src: url }; + }, + createRenderer: () => { + throw new Error('no WebGL'); + }, + }); + const host = fakeHost(); + await viewer.mount(host); + await viewer.open('pano'); + const { picture } = parts(host); + assert.equal(picture.hidden, false); + assert.equal(picture.src, PHOTOS.pano.urls.sd); +}); + +test('a picture whose SD fails falls back to HD, then the thumbnail', async () => { + const tried = []; + const viewer = createPanoramaxViewer({ + getPhoto: async () => PHOTOS.flat, + step: async () => null, + onStep() {}, + load: async (url) => { + tried.push(url); + if (!url.endsWith('thumb.jpg')) throw new Error('gone'); + return { src: url }; + }, + }); + const host = fakeHost(); + await viewer.mount(host); + await viewer.open('flat'); + assert.deepEqual(tried, [ + PHOTOS.flat.urls.sd, + PHOTOS.flat.urls.hd, + PHOTOS.flat.urls.thumb, + ]); + assert.equal(parts(host).picture.src, PHOTOS.flat.urls.thumb); +}); diff --git a/src/layers/streetLevel/registry.js b/src/layers/streetLevel/registry.js index 68fff8444..0e5d40143 100644 --- a/src/layers/streetLevel/registry.js +++ b/src/layers/streetLevel/registry.js @@ -82,6 +82,7 @@ export function filtersOf(provider) { * @property {'day'|'month'} [capturedAtPrecision] How exactly the provider dates images (default 'day'). * @property {string|null} [title] A place name for the caption, e.g. the address. * @property {string|null} creator + * @property {string|null} [license] The image's licence when it is per image, e.g. 'CC BY-SA 4.0'. * @property {string|null} sequenceId * @property {string} externalUrl Deep link to the image on the provider's site. */ diff --git a/src/layers/streetLevel/state.js b/src/layers/streetLevel/state.js index fd92e05ec..a7bf85e37 100644 --- a/src/layers/streetLevel/state.js +++ b/src/layers/streetLevel/state.js @@ -26,6 +26,7 @@ export function freshStreet() { renderModes: true, sequenceId: null, creator: null, + license: null, externalUrl: null, loading: false, error: null, diff --git a/src/layers/streetLevel/viewerHost.js b/src/layers/streetLevel/viewerHost.js index 3a562c21e..8992d4905 100644 --- a/src/layers/streetLevel/viewerHost.js +++ b/src/layers/streetLevel/viewerHost.js @@ -38,6 +38,7 @@ export function createViewerHost({ state, parts }) { title: pose.title || null, sequenceId: pose.sequenceId ?? null, creator: pose.creator || null, + license: pose.license || null, externalUrl: pose.externalUrl || null, // FIT / FILL, unless this image has no frame to fit (a panorama). renderModes: diff --git a/src/testSupport/panoramaxFixtures.mjs b/src/testSupport/panoramaxFixtures.mjs new file mode 100644 index 000000000..47e807604 --- /dev/null +++ b/src/testSupport/panoramaxFixtures.mjs @@ -0,0 +1,129 @@ +// Panoramax stand-ins for tests and the hermetic QA gate: vector tiles in +// the meta-catalogue's shape, and STAC items. +import { PbfWriter } from 'pbf'; +import { + layerWriter, + lineGeometry, + lonLatToTileLocal, + zigzag, +} from '../layers/streetLevel/providers/mapillary/coverageFixture.mjs'; + +/** + * Encode a Panoramax `/api/map/{z}/{x}/{y}.mvt` tile from lon/lat features. + * @param {{x: number, y: number, z: number}} tile + * @param {{sequences?: Array<{id: string, date?: string, type?: string, parts: Array>}>, grid?: Array<{lon: number, lat: number, pano?: number, flat?: number}>, pictures?: Array<{id: string, lon: number, lat: number, sequenceId: string, heading?: number, ts?: string, type?: string}>, extent?: number}} [content] + */ +export function encodePanoramaxTile( + tile, + { sequences = [], grid = [], pictures = [], extent = 4096 } = {}, +) { + const pbf = new PbfWriter(); + if (sequences.length) { + const layer = layerWriter('sequences', extent); + for (const sequence of sequences) + layer.add( + 2, + lineGeometry( + sequence.parts.map((part) => + part.map((point) => lonLatToTileLocal(point, tile, extent)), + ), + ), + { + id: sequence.id, + type: sequence.type ?? 'flat', + date: sequence.date ?? '2025-06-01', + model: 'Test camera', + }, + ); + pbf.writeMessage(3, (_, out) => layer.write(out), null); + } + const point = (lon, lat) => { + const [px, py] = lonLatToTileLocal([lon, lat], tile, extent); + return [9, zigzag(px), zigzag(py)]; + }; + if (grid.length) { + const layer = layerWriter('grid', extent); + for (const cell of grid) { + const pano = cell.pano ?? 0; + const flat = cell.flat ?? 1; + layer.add(1, point(cell.lon, cell.lat), { + nb_pictures: pano + flat, + nb_360_pictures: pano, + nb_flat_pictures: flat, + }); + } + pbf.writeMessage(3, (_, out) => layer.write(out), null); + } + if (pictures.length) { + const layer = layerWriter('pictures', extent); + for (const picture of pictures) + layer.add(1, point(picture.lon, picture.lat), { + id: picture.id, + ts: picture.ts ?? '2025-06-01 10:00:00+00', + heading: picture.heading ?? 90, + sequences: JSON.stringify([picture.sequenceId]), + first_sequence: picture.sequenceId, + type: picture.type ?? 'flat', + }); + pbf.writeMessage(3, (_, out) => layer.write(out), null); + } + return new Uint8Array(pbf.finish()); +} + +/** + * A STAC item as the meta-catalogue returns it. + * @param {{id: string, lon: number, lat: number, sequenceId?: string, heading?: number, pano?: boolean, datetime?: string, producer?: string, license?: string, width?: number, prevId?: string|null, nextId?: string|null, host?: string}} options + */ +export function stacItem({ + id, + lon, + lat, + sequenceId = 'seq-1', + heading = 90, + pano = false, + datetime = '2025-06-01T10:00:00+00:00', + producer = 'tester', + license = 'CC-BY-SA-4.0', + width = pano ? 5760 : 4032, + prevId = null, + nextId = null, + host = 'https://panoramax.example.org', +}) { + const api = 'https://api.panoramax.xyz/api'; + const links = [ + { rel: 'self', href: `${api}/collections/${sequenceId}/items/${id}` }, + ]; + if (prevId) + links.push({ + rel: 'prev', + href: `${api}/collections/${sequenceId}/items/${prevId}`, + }); + if (nextId) + links.push({ + rel: 'next', + href: `${api}/collections/${sequenceId}/items/${nextId}`, + }); + return { + type: 'Feature', + id, + collection: sequenceId, + geometry: { type: 'Point', coordinates: [lon, lat] }, + providers: [{ name: producer, roles: ['producer'] }], + properties: { + datetime, + license, + 'view:azimuth': heading, + 'geovisio:producer': producer, + 'pers:interior_orientation': { + ...(pano ? { field_of_view: 360 } : {}), + sensor_array_dimensions: [width, pano ? width / 2 : 3024], + }, + }, + assets: { + hd: { href: `${host}/pictures/${id}/hd.jpg` }, + sd: { href: `${host}/pictures/${id}/sd.jpg` }, + thumb: { href: `${host}/pictures/${id}/thumb.jpg` }, + }, + links, + }; +} diff --git a/src/tooling/streetLevelQa.test.mjs b/src/tooling/streetLevelQa.test.mjs index 11680c5d3..354ca3edf 100644 --- a/src/tooling/streetLevelQa.test.mjs +++ b/src/tooling/streetLevelQa.test.mjs @@ -743,3 +743,97 @@ test('KartaView fixtures draw the drive, answer nearest first, and refuse the st 'GET api.kartaview.org/2.0/photo/{id}', ); }); + +test('Panoramax fixtures: two sequences on one street, searches honour sequence and filters, unknown calls are flagged', async () => { + const { + answerPanoramaxRequest, + describePanoramaxCall, + PNX_LINE, + PNX_SEQUENCES, + panoramaxHdDownloads, + panoramaxPictures, + panoramaxTile, + resetPanoramaxFixture, + } = await import('../../scripts/fixtures/street-level/panoramax.mjs'); + const { decodeCoverageTile, decodePictures } = await import( + '../layers/streetLevel/providers/panoramax/decode.js' + ); + const { normalizeItem } = await import( + '../layers/streetLevel/providers/panoramax/policy.js' + ); + resetPanoramaxFixture(); + const pictures = panoramaxPictures(); + assert.deepEqual( + [...new Set(pictures.map((p) => p.sequenceId))], + [...PNX_SEQUENCES], + ); + assert.ok(pictures.some((p) => p.isPano)); + // Coverage: both lines in a z14 tile on the street; cones in z15. + const z = 14; + const x = lonToTileX((PNX_LINE.west + PNX_LINE.east) / 2, z); + const y = latToTileY(PNX_LINE.lat, z); + const lines = decodeCoverageTile(panoramaxTile(z, x, y), { x, y, z }); + assert.deepEqual( + lines.sequences.map((s) => s.id).sort(), + [...PNX_SEQUENCES].sort(), + ); + const t15 = { + x: lonToTileX(pictures[20].lon, 15), + y: latToTileY(PNX_LINE.lat, 15), + z: 15, + }; + assert.ok( + decodePictures(panoramaxTile(15, t15.x, t15.y), t15, PNX_SEQUENCES[0]) + .length > 0, + ); + // A search for one sequence near the street gives only its pictures, nearest first. + const at = pictures[30]; + const ask = (query) => + JSON.parse( + answerPanoramaxRequest({ + method: 'GET', + url: `https://api.panoramax.xyz/api/search?${query}`, + }).body, + ).features.map(normalizeItem); + const own = ask( + `place_position=${at.lon},${at.lat}&place_distance=0-50&limit=5&collections=${PNX_SEQUENCES[1]}`, + ); + assert.ok(own.length > 0); + assert.ok(own.every((p) => p.sequenceId === PNX_SEQUENCES[1])); + assert.equal(own[0].nextId && typeof own[0].nextId, 'string'); + const panos = ask( + `place_position=${at.lon},${at.lat}&place_distance=0-200&limit=50&filter=field_of_view%3D360`, + ); + assert.ok(panos.length > 0 && panos.every((p) => p.isPano)); + assert.deepEqual( + ask( + `place_position=${at.lon},${at.lat}&place_distance=0-200&limit=50&datetime=2024-01-01T00:00:00.000Z/..`, + ).every((p) => p.sequenceId === PNX_SEQUENCES[0]), + true, + 'the second sequence is older', + ); + const one = ask(`ids=${at.id}&limit=1`); + assert.equal(one[0].id, at.id); + // Its pictures, and only those, are served; HD downloads are counted. + const file = (name) => + answerPanoramaxRequest({ + method: 'GET', + url: `https://panoramax.openstreetmap.fr/derivates/${name}`, + }); + assert.equal(file(`${at.id}/sd.jpg`).status, 200); + assert.equal(file(`${at.id}/hd.jpg`).status, 200); + assert.equal(panoramaxHdDownloads(), 1); + assert.equal(file('nope/sd.jpg').known, false); + const unknown = answerPanoramaxRequest({ + method: 'GET', + url: 'https://api.panoramax.xyz/api/collections/x/items', + }); + assert.equal(unknown.known, false); + assert.equal( + describePanoramaxCall( + 'GET', + new URL('https://api.panoramax.xyz/api/map/14/1/2.mvt'), + ), + 'GET api.panoramax.xyz/api/map/{z}/{x}/{y}.mvt', + ); +}); diff --git a/src/tools/globePanel.js b/src/tools/globePanel.js index 16e32eec6..0bfa5c2df 100644 --- a/src/tools/globePanel.js +++ b/src/tools/globePanel.js @@ -15,9 +15,9 @@ export const PANEL_WORKER_PRELUDE_PATH = 'cesium/worker-prelude.js'; /** * Sites the app's browser code loads from directly: map imagery, 3D tiles, * terrain, fonts, and Street Level's Mapillary Graph API and image CDN (its - * coverage tiles come through the app's server), Google Street View and - * KartaView. Everything from the app's - * own server comes through the MCP server. + * coverage tiles come through the app's server), Google Street View, + * KartaView and Panoramax. Everything from the app's own server comes + * through the MCP server. */ const PROVIDER_ORIGINS = Object.freeze([ 'https://tile.googleapis.com', @@ -49,6 +49,14 @@ const PROVIDER_ORIGINS = Object.freeze([ 'https://api.kartaview.org', 'https://cdn.kartaview.org', 'https://*.openstreetcam.org', + // Panoramax's meta-catalogue, and the federated instances holding most of + // its pictures (a picture on another instance shows only in the app). + 'https://api.panoramax.xyz', + 'https://panoramax.openstreetmap.fr', + 'https://panoramax.ign.fr', + // IGN's pictures redirect to its OVH object storage. + 'https://panoramax-storage-public-fast.s3.gra.perf.cloud.ovh.net', + 'https://panoramax.mapcomplete.org', ]); export const MCP_APP_MIME_TYPE = 'text/html;profile=mcp-app'; diff --git a/src/ui/streetLevelPresentation.js b/src/ui/streetLevelPresentation.js index 9ecfd077e..f0b844532 100644 --- a/src/ui/streetLevelPresentation.js +++ b/src/ui/streetLevelPresentation.js @@ -133,6 +133,7 @@ function presentViewer(state) { const left = []; if (street.title) left.push(street.title); if (street.creator) left.push(`Image by ${street.creator}`); + if (street.license) left.push(street.license); return { open: street.open === true, loading: street.loading === true && !street.imageId, diff --git a/src/ui/styles/street-level.css b/src/ui/styles/street-level.css index 0e7258978..ea9efee1c 100644 --- a/src/ui/styles/street-level.css +++ b/src/ui/styles/street-level.css @@ -7,6 +7,9 @@ /* KartaView's purple for lines; a lighter one where it is text (WCAG AA). */ --sl-kartaview: #635bff; --sl-kartaview-text: #8f8aff; + /* Panoramax's orange; the darker one sits behind white text. */ + --sl-panoramax: #ff6f00; + --sl-panoramax-fill: #b84e00; --sl-text: 0.62rem; --sl-text-small: 0.6rem; } @@ -621,8 +624,11 @@ button.sl-status:focus-visible { font-size: 1.3rem; } -/* ── KartaView photo viewer ──────────────────────────────────────────────── */ -.sl-kv-viewer { +/* ── Photo viewer (KartaView, Panoramax) ─────────────────────────────────── */ +.sl-photo-viewer { + /* The lit source's colour: a focus ring, and behind the white ‹ › glyphs. */ + --sl-photo-ring: var(--sl-kartaview-text); + --sl-photo-fill: color-mix(in srgb, var(--sl-kartaview) 85%, transparent); position: absolute; inset: 0; background: #000; @@ -630,23 +636,29 @@ button.sl-status:focus-visible { overflow: hidden; } -.sl-kv-viewer:focus-visible { +.sl-photo-viewer:focus-visible { outline: none; } /* The photo and canvas fill the viewer and would cover a ring drawn on it: draw the ring on a layer above them (clicks pass through). */ -.sl-kv-viewer:focus-visible::after { +.sl-photo-viewer:focus-visible::after { content: ''; position: absolute; inset: 0; z-index: 3; - border: 2px solid var(--sl-kartaview-text); + border: 2px solid var(--sl-photo-ring); pointer-events: none; } -.sl-kv-photo, -.sl-kv-pano { +/* White on #ff6f00 is 2.9:1; the darker orange behind ‹ › is 4.6:1. */ +.sl-photo-viewer[data-provider='panoramax'] { + --sl-photo-ring: var(--sl-panoramax); + --sl-photo-fill: var(--sl-panoramax-fill); +} + +.sl-photo-flat, +.sl-photo-pano { position: absolute; inset: 0; width: 100%; @@ -654,25 +666,25 @@ button.sl-status:focus-visible { user-select: none; } -.sl-kv-photo { +.sl-photo-flat { object-fit: contain; } -.sl-kv-pano { +.sl-photo-pano { cursor: grab; touch-action: none; } -.sl-kv-pano.sl-kv-dragging { +.sl-photo-pano.sl-photo-dragging { cursor: grabbing; } -.sl-kv-photo[hidden], -.sl-kv-pano[hidden] { +.sl-photo-flat[hidden], +.sl-photo-pano[hidden] { display: none; } -.sl-kv-step { +.sl-photo-step { position: absolute; top: 50%; z-index: 2; @@ -688,27 +700,27 @@ button.sl-status:focus-visible { cursor: pointer; } -.sl-kv-step-prev { +.sl-photo-step-prev { left: 6px; } -.sl-kv-step-next { +.sl-photo-step-next { right: 6px; } -.sl-kv-step:hover:not(:disabled), -.sl-kv-step:focus-visible { - background: color-mix(in srgb, var(--sl-kartaview) 85%, transparent); +.sl-photo-step:hover:not(:disabled), +.sl-photo-step:focus-visible { + background: var(--sl-photo-fill); outline: none; } -.sl-kv-step:disabled { +.sl-photo-step:disabled { opacity: 0.3; cursor: default; } /* Nothing to step through until a photo shows. */ -.sl-kv-viewer:not([data-kind='flat']):not([data-kind='pano']) .sl-kv-step { +.sl-photo-viewer:not([data-kind='flat']):not([data-kind='pano']) .sl-photo-step { display: none; } From bce3044e0c25fb5aaea6146984f768d24275caec Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Wed, 14 Oct 2026 13:34:25 +0100 Subject: [PATCH 47/95] =?UTF-8?q?fix(street-level):=20Panoramax=20function?= =?UTF-8?q?al=20QA=20=E2=80=94=20nearest=20lookup,=20line=20cap=20order,?= =?UTF-8?q?=20hostile=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From an adversarial functional QA round against the live API: - The nearest lookup missed pictures looking away from the click (place_position leaves them out at any tolerance): it now runs a bounding-box search beside it and ranks both by distance. A line clicked from high up searches its sequence within height / 100 (50–400 m). - The per-tile line cap kept the newest sequences, not the longest (lineCoverage re-sorted them): providers can now keep their decoded order (`keepOrder`). - A long producer name froze the tab (quadratic email regex): linear check. A malformed link escape no longer fails the lookup; null or string coordinates no longer land at 0,0. - A line with no camera type or day stays drawn under the filters. - A sky click on a line changes nothing (as KartaView), instead of selecting a sequence with no cones. - An unreadable tile reads as such; a failed neighbour z15 tile only thins the cones; unknown or refused ids say the picture is gone. - Shared viewer: the sharp-panorama guard is per open (a late download of the previous photo no longer lets a second one start); an HD-only 360° picture is shrunk or skipped like any original; panorama.js read the bitmap size after closing it, so 2:1 power-of-two panoramas clamped instead of wrapping. - Tests for each, and for the gaps mutation testing found (preferred point, cone order, no reload within drawn tiles, newest-first gate, stale bitmap release, flat upgrade retry). Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 14 +- scripts/qa-street-level.mjs | 2 +- src/layers/streetLevel/lineCoverage.js | 12 +- src/layers/streetLevel/panorama.js | 4 +- src/layers/streetLevel/panorama.test.mjs | 30 +++ src/layers/streetLevel/photoViewer.js | 23 ++- src/layers/streetLevel/photoViewer.test.mjs | 134 +++++++++++++ .../streetLevel/providers/panoramax/decode.js | 8 +- .../providers/panoramax/decode.test.mjs | 18 ++ .../streetLevel/providers/panoramax/index.js | 123 +++++++++--- .../streetLevel/providers/panoramax/policy.js | 42 +++- .../providers/panoramax/policy.test.mjs | 32 +++ .../providers/panoramax/provider.test.mjs | 186 +++++++++++++++++- .../streetLevel/providers/panoramax/source.js | 55 ++++-- .../providers/panoramax/source.test.mjs | 57 +++++- .../streetLevel/providers/panoramax/viewer.js | 36 +++- .../providers/panoramax/viewer.test.mjs | 11 +- src/tooling/streetLevelQa.test.mjs | 10 +- 18 files changed, 699 insertions(+), 98 deletions(-) create mode 100644 src/layers/streetLevel/photoViewer.test.mjs diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 7b28ca40b..7cf173b2c 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3262,10 +3262,16 @@ density cells, binned to at most 96×96 points a tile (a z0 tile holds filters apply: lines carry their day and camera type, and a cell stays while it holds pictures of the kind asked for (cells are undated). A click on a line asks the core (`actions.openNearest`) for that sequence's picture -nearest the click (`/api/search?place_position…&collections=`, any -direction: `place_fov_tolerance=180`), without framing, then any sequence's -if it has none within 50 m; the filters go to the API (`filter= -field_of_view=360`, `datetime=/..`). Once the picture is on screen +nearest the click, within the camera height / 100 (50–400 m: from high up +a click lands well off a line), without framing, then any sequence's within +50 m. Each lookup is two searches side by side, ranked by distance in the +browser: `place_position` (nearest first, but even at +`place_fov_tolerance=180` it leaves out pictures looking away from the +point) and a bounding box (those too, but in no order); the filters go to +both (`filter=field_of_view=360`, `datetime=/..`). A line is dated +by its sequence's first day, so a sequence still growing can hide under +SINCE; a line with no camera type or day stays drawn. Lines keep the +decoded longest-first order under the per-tile cap (`keepOrder`). Once the picture is on screen its sequence is highlighted and drawn as cones from the z15 tiles around it (their `pictures` layer, ~110 bytes a picture, against ~7.5 kB for a STAC item), plus a neighbour within 150 m of an edge; the cones re-centre when a diff --git a/scripts/qa-street-level.mjs b/scripts/qa-street-level.mjs index 7437b2be8..52251ba2c 100644 --- a/scripts/qa-street-level.mjs +++ b/scripts/qa-street-level.mjs @@ -2809,7 +2809,7 @@ async function main() { ); if (fixtures) await step( - 'every Mapillary and KartaView request was answered from a fixture', + 'every Mapillary, KartaView and Panoramax request was answered from a fixture', () => { console.log( ` (answered: ${[...fixture.calls].sort().join(' | ')})`, diff --git a/src/layers/streetLevel/lineCoverage.js b/src/layers/streetLevel/lineCoverage.js index 85edd97c1..9c363a07e 100644 --- a/src/layers/streetLevel/lineCoverage.js +++ b/src/layers/streetLevel/lineCoverage.js @@ -72,6 +72,7 @@ export function meshBoxInRange(box, centre) { * @property {number} viewRangeMinM Street-zoom coverage reaches at least this far… * @property {number} viewRangePerHeight …or camera height times this. * @property {number} viewNearM Ground always covered around a tilted camera. + * @property {boolean} [keepOrder] Cap a tile's lines in the order decoded (default: newest first). */ /** @@ -559,10 +560,13 @@ export function createLineCoverage({ north: bounds.north + pad, }; if (kind === 'sequence') { - // Newest first; the per-tile cap applies to what passes the filter. - entry.sequenceList = decoded.sequences.sort( - (a, b) => (b.capturedAt || 0) - (a.capturedAt || 0), - ); + // Newest first, unless the provider ordered them itself; the + // per-tile cap applies to what passes the filter. + entry.sequenceList = tuning.keepOrder + ? decoded.sequences + : decoded.sequences.sort( + (a, b) => (b.capturedAt || 0) - (a.capturedAt || 0), + ); entry.sequences = new Map(); entry.total = decoded.sequences.length; } else { diff --git a/src/layers/streetLevel/panorama.js b/src/layers/streetLevel/panorama.js index 50f348a9b..72f5718a0 100644 --- a/src/layers/streetLevel/panorama.js +++ b/src/layers/streetLevel/panorama.js @@ -188,13 +188,15 @@ export function createPanoramaRenderer(canvas) { gl.UNSIGNED_BYTE, image, ); + // Read before the close: a closed bitmap measures 0×0. + const { width, height } = image; // The pixels are on the GPU now; a bitmap's copy can go. image.close?.(); // Linear only: a mipmapped lookup draws a seam where longitude wraps. gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); const repeat = - isPowerOfTwo(image.width) && isPowerOfTwo(image.height) + isPowerOfTwo(width) && isPowerOfTwo(height) ? gl.REPEAT : gl.CLAMP_TO_EDGE; gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, repeat); diff --git a/src/layers/streetLevel/panorama.test.mjs b/src/layers/streetLevel/panorama.test.mjs index 5bb6380d1..6a0c7c8b8 100644 --- a/src/layers/streetLevel/panorama.test.mjs +++ b/src/layers/streetLevel/panorama.test.mjs @@ -167,3 +167,33 @@ test('an image uploads as RGBA and its bitmap is released; closing hands the tex ); assert.equal(renderer.hasImage(), false); }); + +test('a power-of-two bitmap wraps in longitude even though it is closed once uploaded', (t) => { + const { gl, renderer } = stubPanorama(t); + const wraps = []; + gl.texParameteri = (target, name, value) => wraps.push([name, value]); + gl.TEXTURE_WRAP_S = 'WRAP_S'; + gl.REPEAT = 'REPEAT'; + gl.CLAMP_TO_EDGE = 'CLAMP'; + const bitmap = { + width: 2048, + height: 1024, + // A closed ImageBitmap measures 0×0. + close() { + this.width = 0; + this.height = 0; + }, + }; + renderer.setImage(bitmap); + assert.deepEqual( + wraps.find(([name]) => name === 'WRAP_S'), + ['WRAP_S', 'REPEAT'], + ); + wraps.length = 0; + renderer.setImage({ width: 2000, height: 1000 }); + assert.deepEqual( + wraps.find(([name]) => name === 'WRAP_S'), + ['WRAP_S', 'CLAMP'], + 'other sizes clamp (WebGL 1 repeats powers of two only)', + ); +}); diff --git a/src/layers/streetLevel/photoViewer.js b/src/layers/streetLevel/photoViewer.js index 49dac9949..705519446 100644 --- a/src/layers/streetLevel/photoViewer.js +++ b/src/layers/streetLevel/photoViewer.js @@ -90,8 +90,8 @@ export function panoSizeFor(canvasPixels, fovDeg) { /** * @typedef {object} PhotoSources Where a provider's photo comes from. * @property {(photo: object) => boolean} hasPano A 360° photo with an equirectangular image. - * @property {(photo: object) => {url: string, width: number}|null} panoFirst - * The panorama shown first, and its texture width (0 when it is only a preview). + * @property {(photo: object) => {url: string, width: number, resize?: [number, number]|null}|null} panoFirst + * The panorama shown first, its texture width (0 when it is only a preview), and a size to shrink it to. * @property {(photo: object, size: [number, number]) => {url: string, resize?: [number, number]|null, timeoutMs?: number}|null} panoSharp * A sharper panorama for a canvas that needs `size`, or null. * @property {(photo: object) => Array} flatFirst Flat versions to try in turn. @@ -158,8 +158,8 @@ export function createPhotoViewer({ let restoreView = null; /** Width of the panorama texture on screen (0: the preview). */ let panoWidth = 0; - /** Width of a sharper panorama on its way (resizes come in bursts). */ - let panoComing = 0; + /** The sharper panorama on its way, for which open (resizes come in bursts). */ + let panoComing = null; /** The sharper flat version on screen or on its way (one download each). */ let flatSharpUrl = null; /** The photo an open is fetching: ‹ › step on from it, not the old one. */ @@ -520,10 +520,12 @@ export function createPhotoViewer({ if (kind === 'flat' && target.isPano) return; if (kind === 'pano') { const size = panoSizeNow(); - if (size[0] <= Math.max(panoWidth, panoComing)) return; + if (size[0] <= panoWidth) return; + if (panoComing?.seq === seq && size[0] <= panoComing.width) return; const sharp = sources.panoSharp(target, size); if (!sharp?.url) return; - panoComing = size[0]; + const coming = { seq, width: size[0] }; + panoComing = coming; try { const image = await loadAs(sharp.url, { crossOrigin: true, @@ -544,7 +546,7 @@ export function createPhotoViewer({ } catch { /* the first version stays */ } finally { - if (panoComing === size[0]) panoComing = 0; + if (panoComing === coming) panoComing = null; } return; } @@ -574,7 +576,10 @@ export function createPhotoViewer({ // waiting on it twice. const first = sources.panoFirst(target); if (!first?.url) throw new Error(LOAD_FAILED); - const image = await loadAs(first.url, { crossOrigin: true }); + const image = await loadAs(first.url, { + crossOrigin: true, + resize: first.resize ?? null, + }); if (seq !== openSeq) { image.close?.(); return false; @@ -592,7 +597,7 @@ export function createPhotoViewer({ show('pano'); renderer.setImage(image); panoWidth = first.width || 0; - panoComing = 0; + panoComing = null; return true; } diff --git a/src/layers/streetLevel/photoViewer.test.mjs b/src/layers/streetLevel/photoViewer.test.mjs new file mode 100644 index 000000000..5276ea375 --- /dev/null +++ b/src/layers/streetLevel/photoViewer.test.mjs @@ -0,0 +1,134 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createPhotoViewer } from './photoViewer.js'; +import { fakeHost, until } from '../../testSupport/kartaViewFakes.mjs'; + +const PHOTOS = { + a: { id: 'a', sequenceId: 's', lon: 1, lat: 2, heading: 0, isPano: true }, + b: { id: 'b', sequenceId: 's', lon: 1, lat: 2, heading: 0, isPano: true }, + f: { id: 'f', sequenceId: 's', lon: 1, lat: 2, heading: 0, isPano: false }, +}; + +/** Sources whose panoramas and sharper flat versions are named after the photo. */ +const SOURCES = { + hasPano: (photo) => photo.isPano, + panoFirst: (photo) => ({ url: `${photo.id}-first`, width: 2048 }), + panoSharp: (photo) => ({ url: `${photo.id}-sharp` }), + flatFirst: (photo) => [`${photo.id}-first`], + flatSharp: (photo, pixels) => (pixels > 1000 ? `${photo.id}-sharp` : null), + describe: () => ({ externalUrl: 'https://example.org' }), +}; + +/** A viewer whose loads wait until `release(url)` (or fail with `fail(url)`). */ +function setup() { + const pending = new Map(); + const loads = []; + const images = []; + const viewer = createPhotoViewer({ + providerId: 'test', + name: 'Test', + sources: SOURCES, + getPhoto: async (id) => PHOTOS[id], + step: async () => null, + onStep() {}, + load: (url) => + new Promise((resolve, reject) => { + loads.push(url); + pending.set(url, { resolve, reject }); + }), + createRenderer: () => ({ + setImage: (image) => images.push(image.src), + setView() {}, + getView: () => ({ yaw: 0, pitch: 0, fov: 80 }), + resize() {}, + clear() {}, + destroy() {}, + }), + }); + const release = (url) => { + const closed = { value: false }; + pending.get(url).resolve({ src: url, close: () => (closed.value = true) }); + pending.delete(url); + return closed; + }; + const fail = (url) => { + pending.get(url).reject(new Error('gone')); + pending.delete(url); + }; + return { viewer, loads, images, release, fail, host: fakeHost() }; +} + +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)); +const parts = (host) => { + const [picture, canvas] = host.children[0].children; + return { picture, canvas }; +}; + +test('a panorama that arrives after another photo opened is released, not shown', async () => { + const env = setup(); + await env.viewer.mount(env.host); + const first = env.viewer.open('a'); + await tick(); + const second = env.viewer.open('b'); + await tick(); + const stale = env.release('a-first'); + await first; + assert.equal(stale.value, true, 'its bitmap is closed'); + env.release('b-first'); + await second; + assert.deepEqual(env.images, ['b-first']); +}); + +test('one sharp panorama download per open, even across a quick switch and a burst of resizes', async (t) => { + const env = setup(); + t.after(() => delete globalThis.devicePixelRatio); + globalThis.devicePixelRatio = 1; + await env.viewer.mount(env.host); + const { canvas } = parts(env.host); + canvas.clientWidth = 1800; + const opening = env.viewer.open('a'); + await tick(); + env.release('a-first'); + await opening; + await tick(); + assert.ok(env.loads.includes('a-sharp'), 'expanded: the sharp one is asked'); + // Photo b opens while a's sharp version is still on its way. + const next = env.viewer.open('b'); + await tick(); + env.release('b-first'); + await next; + await tick(); + assert.ok(env.loads.includes('b-sharp')); + // a's download finishes late; resizes must not ask for b's again. + env.release('a-sharp'); + await tick(); + env.viewer.resize(); + env.viewer.resize(); + await tick(); + assert.equal(env.loads.filter((url) => url === 'b-sharp').length, 1); + env.release('b-sharp'); + await tick(); + assert.deepEqual(env.images, ['a-first', 'b-first', 'b-sharp']); +}); + +test('a sharper flat version that failed is tried again on the next resize', async (t) => { + const env = setup(); + t.after(() => delete globalThis.devicePixelRatio); + globalThis.devicePixelRatio = 1; + await env.viewer.mount(env.host); + const { picture } = parts(env.host); + picture.clientWidth = 1200; + const opening = env.viewer.open('f'); + await tick(); + env.release('f-first'); + await opening; + await tick(); + env.fail('f-sharp'); + await tick(); + assert.equal(picture.src, 'f-first'); + env.viewer.resize(); + await tick(); + assert.equal(env.loads.filter((url) => url === 'f-sharp').length, 2); + env.release('f-sharp'); + assert.ok(await until(() => picture.src === 'f-sharp')); +}); diff --git a/src/layers/streetLevel/providers/panoramax/decode.js b/src/layers/streetLevel/providers/panoramax/decode.js index 02ff0579f..19b470846 100644 --- a/src/layers/streetLevel/providers/panoramax/decode.js +++ b/src/layers/streetLevel/providers/panoramax/decode.js @@ -54,7 +54,13 @@ export function decodeCoverageTile(bytes, address) { id: String(props.id), parts, capturedAt: dayMs(props.date), - isPano: props.type === 'equirectangular', + // A sequence with no camera type is neither (the filters keep it). + isPano: + props.type === 'equirectangular' + ? true + : props.type === 'flat' + ? false + : null, }; lengths.set(sequence, length); result.sequences.push(sequence); diff --git a/src/layers/streetLevel/providers/panoramax/decode.test.mjs b/src/layers/streetLevel/providers/panoramax/decode.test.mjs index 397380dad..4347463b0 100644 --- a/src/layers/streetLevel/providers/panoramax/decode.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/decode.test.mjs @@ -47,6 +47,24 @@ test('sequence lines keep their parts apart, with their day and camera type', () assert.equal(pano.parts.length, 2, 'the gap is not bridged'); assert.ok(near(pano.parts[1][1][0], lon + 0.001)); assert.equal(flat.isPano, false); + const untyped = decodeCoverageTile( + encodePanoramaxTile(tile, { + sequences: [ + { + id: 'u', + type: '', + parts: [ + [ + [lon, lat], + [lon + 0.001, lat], + ], + ], + }, + ], + }), + tile, + ).sequences[0]; + assert.equal(untyped.isPano, null, 'no camera type: neither'); }); test('density cells bin to a bounded number of points that keep the counts and kinds', () => { diff --git a/src/layers/streetLevel/providers/panoramax/index.js b/src/layers/streetLevel/providers/panoramax/index.js index cda9015c0..1fa65fd51 100644 --- a/src/layers/streetLevel/providers/panoramax/index.js +++ b/src/layers/streetLevel/providers/panoramax/index.js @@ -18,7 +18,10 @@ import { COVERAGE_Z12_BELOW_M, COVERAGE_Z13_BELOW_M, COVERAGE_Z14_BELOW_M, + GONE_MESSAGE, LINE_CLICK_HINT, + LINE_CLICK_RADIUS_MAX_M, + LINE_CLICK_RADIUS_PER_HEIGHT, NEAREST_LIMIT, NEAREST_RADIUS_M, NO_COVERAGE_HINT, @@ -31,13 +34,31 @@ import { PICTURE_TILE_EDGE_M, PICTURE_TILE_ZOOM, RATE_LIMITED_MESSAGE, + UNREADABLE_TILE_MESSAGE, normalizeItem, } from './policy.js'; export { PANORAMAX_PROVIDER_ID } from './policy.js'; +/** + * A tile's bytes decoded by `decode`; a body that is not a vector tile + * (an HTML error page, a cut-off answer) reads as such, not as a parser's + * "Unimplemented type: 4". + */ +async function readable(decode) { + try { + return await decode(); + } catch (error) { + if (error?.name === 'AbortError' || error?.status !== undefined) + throw error; + throw new Error(UNREADABLE_TILE_MESSAGE); + } +} + /** Pictures kept in memory by id (lookups and ‹ › fill it). */ const PHOTO_CACHE_SIZE = 2000; +/** Sequences whose cone centre is remembered. */ +const ANCHOR_CACHE_SIZE = 50; /** * Sequence lines at z11–14 below 30 km; above, the density grid at z5 and @@ -63,17 +84,23 @@ export function panoramaxZoomFor(heightM) { } /** - * The imagery filter for lines (dated, typed) and grid cells (undated, with - * 360° and flat counts): a cell stays while it holds pictures of the kind - * asked for. + * The imagery filter for lines and grid cells. A cell (undated, with 360° + * and flat counts) stays while it holds pictures of the kind asked for. A + * line is dated by its sequence's first day, so a sequence still growing + * may hide under "since" while its newer pictures can be opened; a line + * with no camera type or no day is kept (a click finds out). */ export function passesCoverage(record, filter) { + if (!filter) return true; if ('hasPano' in record) { - if (filter?.pano === 'pano') return record.hasPano; - if (filter?.pano === 'flat') return record.hasFlat; + if (filter.pano === 'pano') return record.hasPano; + if (filter.pano === 'flat') return record.hasFlat; return true; } - return passesImageryFilter(record, filter); + return passesImageryFilter(record, { + pano: typeof record.isPano === 'boolean' ? filter.pano : 'all', + sinceMs: record.capturedAt ? filter.sinceMs : null, + }); } /** Least-recently-used map: reading an entry renews it. */ @@ -193,7 +220,7 @@ export function createPanoramaxProvider({ }; const photos = createLru(PHOTO_CACHE_SIZE); /** Where each sequence's cones centre: the picture last opened in it, or a click. */ - const anchors = new Map(); + const anchors = createLru(ANCHOR_CACHE_SIZE); /** The z15 tiles the cones were drawn from. */ let drawn = null; /** A line click: its sequence's pictures win the next nearest lookup there. */ @@ -215,10 +242,14 @@ export function createPanoramaxProvider({ if (!lookup) { lookup = source .getItem(id) + .catch((error) => { + // Ids are UUIDs: the API answers 400 to anything else. + if (error?.status === 400) return null; + throw error; + }) .then((item) => { const photo = item && remember(item); - if (!photo) - throw new Error('Panoramax has no position for this picture'); + if (!photo) throw new Error(GONE_MESSAGE); return photo; }) .finally(() => inflight.delete(id)); @@ -227,6 +258,19 @@ export function createPanoramaxProvider({ return lookup; } + /** + * How far from a line click its sequence's pictures are looked for: + * from high up the click may land tens of metres off the line. + */ + function lineClickRadius() { + const height = state.viewer?.camera?.positionCartographic?.height; + if (!Number.isFinite(height)) return NEAREST_RADIUS_M; + return Math.min( + LINE_CLICK_RADIUS_MAX_M, + Math.max(NEAREST_RADIUS_M, height * LINE_CLICK_RADIUS_PER_HEIGHT), + ); + } + /** The picture `delta` along its sequence, from its own links (null at an end). */ async function step(photo, delta) { if (!photo?.sequenceId) return null; @@ -236,7 +280,9 @@ export function createPanoramaxProvider({ const coverage = createLineCoverage({ state, fetchTile: async (tile, { signal }) => - decodeCoverageTile(await source.getTile(tile, { signal }), tile), + readable(async () => + decodeCoverageTile(await source.getTile(tile, { signal }), tile), + ), zoomFor: panoramaxZoomFor, colors: COLORS, pickPrefix: PICK_PREFIX.sequence, @@ -252,18 +298,26 @@ export function createPanoramaxProvider({ const anchor = anchors.get(sequenceId); if (!anchor) return []; const tiles = pictureTilesAround(anchor); - const lists = await Promise.all( - tiles.map(async (tile) => - decodePictures( - await source.getTile(tile, { signal }), - tile, - sequenceId, + // The home tile must load; a neighbour that fails only thins the cones. + const lists = await Promise.allSettled( + tiles.map((tile) => + readable(async () => + decodePictures( + await source.getTile(tile, { signal }), + tile, + sequenceId, + ), ), ), ); signal?.throwIfAborted(); + if (lists[0].status === 'rejected') throw lists[0].reason; drawn = { sequenceId, tiles: new Set(tiles.map(tileKey)) }; - const byId = new Map(lists.flat().map((image) => [image.id, image])); + const byId = new Map( + lists + .flatMap((list) => list.value ?? []) + .map((image) => [image.id, image]), + ); return thinImages( [...byId.values()].sort((a, b) => a.capturedAt - b.capturedAt), ); @@ -302,14 +356,14 @@ export function createPanoramaxProvider({ ...viewerOptions, }); - /** Nearest of `items` within the search radius of a point, or null. */ - function nearestOf(items, point) { + /** Nearest of `items` within `radius` metres of a point, or null. */ + function nearestOf(items, point, radius = NEAREST_RADIUS_M) { let best = null; for (const item of items) { const photo = remember(item); if (!photo) continue; const metres = metresBetween(point, photo); - if (metres > NEAREST_RADIUS_M) continue; + if (metres > radius) continue; if (!best || metres < best.metres) best = { photo, metres }; } return best?.photo ?? null; @@ -398,13 +452,15 @@ export function createPanoramaxProvider({ const sequenceId = sequenceIdFromPick(pickId, PICK_PREFIX.sequence); if (!sequenceId) return false; const point = groundPoint?.(); - if (!point) { - // No ground under the click: show the line selected, nothing more. - cones.select(sequenceId); - return true; - } + // No ground under the click (the sky beyond a line): nothing to open. + if (!point) return true; anchors.set(sequenceId, { ...point, sequenceId }); - preferred = { sequenceId, lon: point.lon, lat: point.lat }; + preferred = { + sequenceId, + lon: point.lon, + lat: point.lat, + radius: lineClickRadius(), + }; context.actions.openNearest(point); return true; }, @@ -429,17 +485,16 @@ export function createPanoramaxProvider({ async nearestImage({ lat, lon }, { signal } = {}) { const point = { lat, lon }; const filter = context.getFilter(); + // Only the lookup the click itself asked for (the same point). const wanted = - preferred && metresBetween(preferred, point) < 1 - ? preferred.sequenceId - : null; + preferred && metresBetween(preferred, point) < 1 ? preferred : null; preferred = null; - const ask = (collections) => + const ask = (collections, radius = NEAREST_RADIUS_M) => source.nearestItems( { lat, lon, - radius: NEAREST_RADIUS_M, + radius, limit: NEAREST_LIMIT, collections, pano: filter.pano, @@ -448,7 +503,11 @@ export function createPanoramaxProvider({ { signal }, ); if (wanted) { - const hit = nearestOf(await ask(wanted), point); + const hit = nearestOf( + await ask(wanted.sequenceId, wanted.radius), + point, + wanted.radius, + ); if (hit) return hit.id; } return nearestOf(await ask(null), point)?.id ?? null; diff --git a/src/layers/streetLevel/providers/panoramax/policy.js b/src/layers/streetLevel/providers/panoramax/policy.js index b163be640..f7629b5aa 100644 --- a/src/layers/streetLevel/providers/panoramax/policy.js +++ b/src/layers/streetLevel/providers/panoramax/policy.js @@ -41,10 +41,12 @@ export const COVERAGE_LINES_BELOW_M = 30_000; export const COVERAGE_TUNING = Object.freeze({ maxTiles: 9, overviewMaxTiles: 6, - // Each tile draws at most 1,600 lines, its longest first: a dense z11 + // Each tile draws at most 1,600 lines, its longest first (as decoded, + // `keepOrder`): a dense z11 // tile over Paris holds 3,800, and a lower cap left the city centre // looking emptier than its suburbs. (KartaView draws as many.) maxSequences: 14_400, + keepOrder: true, lineWidthPx: 2.5, overviewPointPx: 4, moveDebounceMs: 320, @@ -61,6 +63,9 @@ export const OVERVIEW_BINS_PER_AXIS = 96; export const RATE_LIMITED_MESSAGE = 'Panoramax is busy — coverage resumes on its own'; export const NO_COVERAGE_HINT = 'no Panoramax pictures in view'; +export const UNREADABLE_TILE_MESSAGE = + 'Panoramax sent a map tile that could not be read'; +export const GONE_MESSAGE = 'This Panoramax picture is no longer available'; /** From high up the dots are where pictures are, not sequences to click. */ export const OVERVIEW_HINT = 'zoom in to click a Panoramax sequence'; @@ -79,8 +84,20 @@ export const TILE_CONCURRENCY = 6; /** Pictures asked for around a point; the API returns them nearest first. */ export const NEAREST_LIMIT = 10; -/** Any picture within the radius, whichever way it looks (2–180°). */ +/** + * The widest the API allows: the point may be beside a picture, but a + * picture looking away from it is still left out (the bounding-box search + * beside it has those). + */ export const NEAREST_FOV_TOLERANCE_DEG = 180; +/** Pictures in the bounding box around a click, at most. */ +export const NEAREST_BOX_LIMIT = 100; +/** + * A line clicked from high up may be tens of metres off the click: its + * sequence is searched within the camera height / 100 (50–400 m). + */ +export const LINE_CLICK_RADIUS_PER_HEIGHT = 0.01; +export const LINE_CLICK_RADIUS_MAX_M = 400; /** A full-resolution panorama is only fetched up to this width (memory). */ export const PANO_HD_MAX_WIDTH = 8_192; @@ -147,7 +164,19 @@ function linkedId(item, rel) { const href = item?.links?.find?.((link) => link?.rel === rel)?.href; if (typeof href !== 'string') return null; const id = href.split(/[?#]/)[0].split('/').filter(Boolean).pop(); - return id ? decodeURIComponent(id) : null; + if (!id) return null; + try { + return decodeURIComponent(id); + } catch { + // A malformed escape is no link, not a failed lookup. + return null; + } +} + +/** An email address anywhere in a name (linear: names come from any instance). */ +function looksLikeEmail(name) { + const at = name.indexOf('@'); + return at > 0 && name.indexOf('.', at + 2) !== -1; } /** @@ -165,7 +194,7 @@ function creatorOf(item) { : []), ]; const name = names.find((value) => typeof value === 'string' && value.trim()); - if (!name || /\S+@\S+\.\S+/.test(name)) return null; + if (!name || looksLikeEmail(name)) return null; return name.trim().slice(0, 80); } @@ -184,8 +213,11 @@ export function isEquirectangular(item) { */ export function normalizeItem(item) { if (!item || item.id == null) return null; + // Only numbers: Number(null) and Number('') would put a picture at 0,0. const [lon, lat] = Array.isArray(item.geometry?.coordinates) - ? item.geometry.coordinates.map(Number) + ? item.geometry.coordinates.map((value) => + typeof value === 'number' ? value : NaN, + ) : [NaN, NaN]; if (!Number.isFinite(lat) || !Number.isFinite(lon)) return null; if (Math.abs(lat) > 90 || Math.abs(lon) > 180) return null; diff --git a/src/layers/streetLevel/providers/panoramax/policy.test.mjs b/src/layers/streetLevel/providers/panoramax/policy.test.mjs index 5d93d7956..605bcf7ad 100644 --- a/src/layers/streetLevel/providers/panoramax/policy.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/policy.test.mjs @@ -193,3 +193,35 @@ test('the caption link opens the picture in the Panoramax viewer', () => { assert.equal(panoramaxPictureUrl(null), 'https://api.panoramax.xyz/'); assert.ok(panoramaxPictureUrl('a&b=c').includes('pic=a%26b%3Dc')); }); + +test('hostile item data: a malformed link escape, null coordinates, a huge producer name', () => { + const item = stacItem({ id: 'a', lon: 1, lat: 2, nextId: 'n' }); + item.links.push({ + rel: 'prev', + href: 'https://api.panoramax.xyz/api/collections/s/items/%E0%A4%A', + }); + const photo = normalizeItem(item); + assert.equal(photo.prevId, null, 'no link, no throw'); + assert.equal(photo.nextId, 'n'); + for (const coordinates of [[null, null], ['', ''], ['2.35', '48.85'], [2.35]]) + assert.equal( + normalizeItem({ id: 'x', geometry: { coordinates } }), + null, + JSON.stringify(coordinates), + ); + // A million characters without spaces is checked in linear time. + const started = Date.now(); + const huge = normalizeItem( + stacItem({ id: 'a', lon: 1, lat: 2, producer: 'a'.repeat(1_000_000) }), + ); + assert.ok(Date.now() - started < 500, `${Date.now() - started} ms`); + assert.equal(huge.creator.length, 80); + const named = (producer) => + normalizeItem(stacItem({ id: 'a', lon: 1, lat: 2, producer })).creator; + assert.equal(named(`${'x'.repeat(200_000)}@example.org`), null); + assert.equal( + named('Carto@Cité'), + 'Carto@Cité', + 'an @ without a domain is a name', + ); +}); diff --git a/src/layers/streetLevel/providers/panoramax/provider.test.mjs b/src/layers/streetLevel/providers/panoramax/provider.test.mjs index e0d5eae61..2ae2c8091 100644 --- a/src/layers/streetLevel/providers/panoramax/provider.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/provider.test.mjs @@ -160,6 +160,20 @@ test('lines at z11–14 below 30 km; density points from z5 down to z0 above', ( assert.equal(panoramaxZoomFor(NaN), null); }); +test('a line with no camera type or no day passes the filters: a click finds out', () => { + const filter = { pano: 'pano', sinceMs: Date.UTC(2024, 0, 1) }; + assert.equal( + passesCoverage({ isPano: null, capturedAt: Date.UTC(2025, 0, 1) }, filter), + true, + ); + assert.equal(passesCoverage({ isPano: true, capturedAt: 0 }, filter), true); + assert.equal(passesCoverage({ isPano: false, capturedAt: 0 }, filter), false); + assert.equal( + passesCoverage({ isPano: null, capturedAt: Date.UTC(2020, 0, 1) }, filter), + false, + ); +}); + test('the imagery filter: lines by type and day, density cells by what they hold', () => { const old = Date.UTC(2020, 0, 1); const since = { pano: 'all', sinceMs: Date.UTC(2024, 0, 1) }; @@ -314,9 +328,25 @@ test('one lookup for a picture asked twice at once', async () => { assert.deepEqual(source.calls.items, ['s-0']); }); -test('a picture Panoramax does not know fails the open with a reason', async () => { +test('a picture Panoramax does not know, or an id it refuses, fails the open with a reason', async () => { const env = await mounted({ items: [] }); - await assert.rejects(env.instance.viewer.open('ghost'), /no position/); + await assert.rejects( + env.instance.viewer.open('ghost'), + /no longer available/, + ); + env.source.getItem = async () => { + throw Object.assign(new Error('Panoramax answered HTTP 400'), { + status: 400, + }); + }; + await assert.rejects( + env.instance.viewer.open('not-a-uuid'), + /no longer available/, + ); + env.source.getItem = async () => { + throw Object.assign(new Error('Panoramax is busy'), { status: 503 }); + }; + await assert.rejects(env.instance.viewer.open('busy'), /busy/); }); /** Cones on a stand-in Cesium viewer; `count()` reads how many are drawn. */ @@ -340,7 +370,13 @@ async function withCones(t, env) { }, camera: { positionWC: new Cesium.Cartesian3(0, 0, 1e8) }, }); - return { count: () => collection?.length ?? 0 }; + return { + count: () => collection?.length ?? 0, + ids: () => + Array.from({ length: collection?.length ?? 0 }, (_, i) => + String(collection.get(i).id).replace(PICK_PREFIX.image, ''), + ), + }; } /** z15 tiles holding every item of `items` as a picture. */ @@ -379,18 +415,148 @@ test('once a picture is on screen its sequence’s cones load from the z15 tile assert.ok(env.source.calls.tiles.every((key) => key.startsWith('15/'))); }); -test('a sequence selected with no picture or click to centre on draws no cones', async (t) => { +test('a click on a line with no ground under it changes nothing: the open sequence keeps its cones', async (t) => { + const items = [...sequence('x', 6), ...sequence('y', 3, { offset: 3 })]; + const env = await mounted({ items, tiles: pictureTiles(items) }); + const cones = await withCones(t, env); + await env.instance.viewer.open('x-1'); + await env.instance.selectSequence('x'); + assert.ok(await until(() => cones.count() === 6)); + const asked = env.source.calls.tiles.length; + assert.equal( + env.instance.handlePick(`${PICK_PREFIX.sequence}y`, { + groundPoint: () => null, + }), + true, + 'the click is the provider’s, even with nothing to open', + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(env.instance.sequenceStats().selectedId, 'x'); + assert.equal(cones.count(), 6); + assert.equal(env.source.calls.tiles.length, asked); + assert.deepEqual(env.actions.nearest, []); +}); + +test('a sequence selected with nothing to centre on draws no cones, and a picture opening in it then does not throw', async (t) => { const items = sequence('s', 3); const env = await mounted({ items, tiles: pictureTiles(items) }); await withCones(t, env); - env.instance.handlePick(`${PICK_PREFIX.sequence}s`, { - groundPoint: () => null, - }); - assert.ok(await until(() => !env.instance.sequenceStats().loading)); - assert.equal(env.instance.sequenceStats().selectedId, 's', 'highlighted'); + await env.instance.selectSequence('s'); assert.equal(env.instance.sequenceStats().images, 0); assert.deepEqual(env.source.calls.tiles, []); - assert.deepEqual(env.actions.nearest, []); + // Its first picture opens: the cones load around it. + await env.instance.viewer.open('s-0'); + await env.instance.selectSequence('s'); + assert.ok(await until(() => env.instance.sequenceStats().images === 3)); +}); + +test('a line clicked from high up looks for its sequence further off; the radius grows with height, up to 400 m', async () => { + const items = [ + ...sequence('near', 1), + ...sequence('line', 1, { offset: 220 }), + ]; + const { instance, source } = setup({ items }); + const height = (h) => + instance.init({ + camera: { positionCartographic: { height: h } }, + scene: { + primitives: { add() {}, remove() {} }, + preRender: { addEventListener: () => () => {} }, + }, + }); + const click = () => + instance.handlePick(`${PICK_PREFIX.sequence}line`, { + groundPoint: () => east(0), + }); + height(25_000); + click(); + assert.equal( + await instance.nearestImage(east(0)), + 'line-0', + '220 m off, from 25 km', + ); + assert.equal(source.calls.nearest.at(-1).radius, 250); + height(1_000); + click(); + assert.equal( + await instance.nearestImage(east(0)), + 'near-0', + 'from 1 km only 50 m: any sequence’s nearer picture', + ); + height(900_000); + click(); + await instance.nearestImage(east(0)); + assert.equal( + source.calls.nearest.filter((q) => q.collections === 'line').at(-1).radius, + 400, + 'capped', + ); +}); + +test('a lookup at another point does not take a line click’s sequence', async () => { + const items = [ + ...sequence('other', 1), + ...sequence('clicked', 1, { offset: 8 }), + ]; + const { instance, source } = setup({ items }); + instance.handlePick(`${PICK_PREFIX.sequence}clicked`, { + groundPoint: () => east(1), + }); + // OPEN at the view centre, a few metres away, overtakes the click. + assert.equal(await instance.nearestImage(east(3)), 'other-0'); + assert.deepEqual( + source.calls.nearest.map((q) => q.collections), + [null], + ); +}); + +test('the cones come in capture order, and a broken neighbour tile only thins them', async (t) => { + const items = sequence('s', 5).map((item, i) => ({ + ...item, + properties: { + ...item.properties, + // Captured in reverse of their position along the street. + datetime: `2025-06-01T10:00:0${9 - i}+00:00`, + }, + })); + const tiles = pictureTiles(items); + const env = await mounted({ items, tiles }); + const cones = await withCones(t, env); + await env.instance.viewer.open('s-0'); + await env.instance.selectSequence('s'); + assert.ok(await until(() => cones.count() === 5)); + assert.deepEqual(cones.ids(), ['s-4', 's-3', 's-2', 's-1', 's-0']); + // A neighbour answers with garbage: the home tile's cones still draw. + const home = new Set(); + env.source.getTile = async (tile) => { + const key = `${tile.z}/${tile.x}/${tile.y}`; + if (!home.size) home.add(key); + if (!home.has(key)) return new TextEncoder().encode('oops'); + return tiles(tile); + }; + await env.instance.viewer.open('s-0'); + await env.instance.selectSequence('s'); + assert.ok(await until(() => !env.instance.sequenceStats().loading)); + // The home tile itself unreadable: a readable error, not a parser's. + env.source.getTile = async () => + new TextEncoder().encode('oops'); + await env.instance.selectSequence('s'); + assert.ok(await until(() => env.actions.errors.some(Boolean))); + assert.match(env.actions.errors.filter(Boolean).at(-1), /could not be read/); +}); + +test('› within the tiles the cones came from reloads nothing', async (t) => { + const items = sequence('s', 4); + const env = await mounted({ items, tiles: pictureTiles(items) }); + await withCones(t, env); + await env.instance.viewer.open('s-0'); + await env.instance.selectSequence('s'); + assert.ok(await until(() => env.instance.sequenceStats().images === 4)); + const asked = env.source.calls.tiles.length; + env.next.fire('click'); + assert.ok(await until(() => env.poses.at(-1) === 's-1')); + await new Promise((resolve) => setTimeout(resolve, 10)); + assert.equal(env.source.calls.tiles.length, asked); }); test('the cones re-centre when › walks into another z15 tile', async (t) => { diff --git a/src/layers/streetLevel/providers/panoramax/source.js b/src/layers/streetLevel/providers/panoramax/source.js index 3ddb6e1e9..7d3c86e6c 100644 --- a/src/layers/streetLevel/providers/panoramax/source.js +++ b/src/layers/streetLevel/providers/panoramax/source.js @@ -3,6 +3,7 @@ import { API_CONCURRENCY, API_TIMEOUT_MS, MAX_RESPONSE_BYTES, + NEAREST_BOX_LIMIT, NEAREST_FOV_TOLERANCE_DEG, NEAREST_LIMIT, NEAREST_RADIUS_M, @@ -151,9 +152,13 @@ export function createPanoramaxSource({ }, /** - * Pictures within `radius` metres of a point, nearest first, whichever - * way they look; `collections` keeps one sequence's, `pano` and - * `sinceMs` filter as the imagery filter does. + * Pictures within `radius` metres of a point, unranked: the caller ranks + * them by distance. Two searches side by side, since neither finds them + * all: `place_position` (nearest first) leaves out pictures looking + * away from the point, whatever the tolerance; a bounding box has them + * but in no order, so in a dense spot its page may miss the nearest. + * `collections` keeps one sequence's, `pano` and `sinceMs` filter as + * the imagery filter does. */ async nearestItems( { @@ -167,18 +172,40 @@ export function createPanoramaxSource({ }, { signal } = {}, ) { - const params = { - place_position: `${Number(lon).toFixed(7)},${Number(lat).toFixed(7)}`, - place_distance: `0-${Math.min(1000, Math.max(1, Math.round(radius)))}`, - place_fov_tolerance: String(NEAREST_FOV_TOLERANCE_DEG), - limit: String(Math.min(100, Math.max(1, limit))), - }; - if (collections) params.collections = String(collections); - if (pano === 'pano') params.filter = 'field_of_view=360'; - else if (pano === 'flat') params.filter = 'field_of_view<360'; + const metres = Math.min(1000, Math.max(1, Math.round(radius))); + const shared = {}; + if (collections) shared.collections = String(collections); + if (pano === 'pano') shared.filter = 'field_of_view=360'; + else if (pano === 'flat') shared.filter = 'field_of_view<360'; if (Number.isFinite(sinceMs) && sinceMs > 0) - params.datetime = `${new Date(sinceMs).toISOString()}/..`; - return search(params, { signal }); + shared.datetime = `${new Date(sinceMs).toISOString()}/..`; + const dLat = metres / 111_320; + const dLon = + metres / (111_320 * Math.max(0.01, Math.cos((lat * Math.PI) / 180))); + const box = [lon - dLon, lat - dLat, lon + dLon, lat + dLat].map((v) => + v.toFixed(7), + ); + const [facing, around] = await Promise.all([ + search( + { + place_position: `${Number(lon).toFixed(7)},${Number(lat).toFixed(7)}`, + place_distance: `0-${metres}`, + place_fov_tolerance: String(NEAREST_FOV_TOLERANCE_DEG), + limit: String(Math.min(100, Math.max(1, limit))), + ...shared, + }, + { signal }, + ), + search( + { bbox: box.join(','), limit: String(NEAREST_BOX_LIMIT), ...shared }, + { signal }, + ), + ]); + const byId = new Map(); + for (const item of [...facing, ...around]) + if (item?.id != null && !byId.has(String(item.id))) + byId.set(String(item.id), item); + return [...byId.values()]; }, }; } diff --git a/src/layers/streetLevel/providers/panoramax/source.test.mjs b/src/layers/streetLevel/providers/panoramax/source.test.mjs index 7f2e87237..a068a3eff 100644 --- a/src/layers/streetLevel/providers/panoramax/source.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/source.test.mjs @@ -101,7 +101,7 @@ test('a picture by id is a STAC search; another id in the answer is not taken fo assert.equal(asked.searchParams.get('limit'), '1'); }); -test('nearest pictures: any direction within the radius, filtered as the imagery filter asks', async () => { +test('nearest pictures: a place search and a box search side by side, both filtered as the imagery filter asks', async () => { const { urls, fetchImpl } = fakeFetch(() => json({ features: [] })); const source = createPanoramaxSource({ fetchImpl }); await source.nearestItems({ lat: 48.85, lon: 2.35 }); @@ -115,7 +115,13 @@ test('nearest pictures: any direction within the radius, filtered as the imagery limit: 500, }); await source.nearestItems({ lat: 48.85, lon: 2.35, pano: 'flat' }); - const [plain, filtered, flat] = urls.map((u) => new URL(u).searchParams); + const asked = urls.map((u) => new URL(u).searchParams); + assert.equal(asked.length, 6, 'two searches a lookup'); + const place = asked.filter((q) => q.has('place_position')); + const box = asked.filter((q) => q.has('bbox')); + assert.equal(place.length, 3); + assert.equal(box.length, 3); + const [plain, filtered, flat] = place; assert.equal(plain.get('place_position'), '2.3500000,48.8500000'); assert.equal(plain.get('place_distance'), '0-50'); assert.equal(plain.get('place_fov_tolerance'), '180'); @@ -128,6 +134,53 @@ test('nearest pictures: any direction within the radius, filtered as the imagery assert.equal(filtered.get('place_distance'), '0-1000', 'capped'); assert.equal(filtered.get('limit'), '100', 'capped'); assert.equal(flat.get('filter'), 'field_of_view<360'); + // The box spans the radius each way, and carries the same filters. + const [w, s, e, n] = box[0].get('bbox').split(',').map(Number); + assert.ok(Math.abs((n - s) * 111_320 - 100) < 1, `${n - s}°`); + assert.ok( + Math.abs((e - w) * 111_320 * Math.cos((48.85 * Math.PI) / 180) - 100) < 1, + ); + assert.equal(box[1].get('collections'), 'seq'); + assert.equal(box[1].get('filter'), 'field_of_view=360'); + assert.equal(box[1].get('datetime'), '2025-01-01T00:00:00.000Z/..'); + assert.equal(box[2].get('filter'), 'field_of_view<360'); +}); + +test('a picture both searches find is listed once', async () => { + const source = createPanoramaxSource({ + fetchImpl: fakeFetch((url) => + url.includes('bbox') + ? json({ features: [{ id: 'b' }, { id: 'a' }, { id: null }] }) + : json({ features: [{ id: 'a' }] }), + ).fetchImpl, + }); + const items = await source.nearestItems({ lat: 1, lon: 2 }); + assert.deepEqual( + items.map((i) => i.id), + ['a', 'b'], + ); +}); + +test('the newest search goes first when the API gate is full', async () => { + const waiting = []; + const order = []; + const source = createPanoramaxSource({ + fetchImpl: async (url) => { + order.push(new URL(url).searchParams.get('ids')); + await new Promise((resolve) => waiting.push(resolve)); + return new Response(JSON.stringify({ features: [] })); + }, + }); + const all = ['1', '2', '3', '4', '5', '6'].map((id) => source.getItem(id)); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.deepEqual(order, ['1', '2', '3', '4'], 'four at once'); + waiting.shift()(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(order.at(-1), '6', 'the latest waiting one next'); + while (waiting.length) waiting.shift()(); + await new Promise((resolve) => setTimeout(resolve, 0)); + while (waiting.length) waiting.shift()(); + await Promise.all(all); }); test('an unreadable search answer, or one with no features, is handled', async () => { diff --git a/src/layers/streetLevel/providers/panoramax/viewer.js b/src/layers/streetLevel/providers/panoramax/viewer.js index df666237b..90ef88c97 100644 --- a/src/layers/streetLevel/providers/panoramax/viewer.js +++ b/src/layers/streetLevel/providers/panoramax/viewer.js @@ -1,4 +1,8 @@ -import { createPhotoViewer, loadImage } from '../../photoViewer.js'; +import { + PANO_FULL_SIZE, + createPhotoViewer, + loadImage, +} from '../../photoViewer.js'; import { HD_TIMEOUT_MS, PANORAMAX_NAME, @@ -8,6 +12,13 @@ import { panoramaxPictureUrl, } from './policy.js'; +/** An HD original small enough to decode (its width is known and under the cap). */ +function hdDecodable(photo) { + return Boolean( + photo.urls.hd && photo.width && photo.width <= PANO_HD_MAX_WIDTH, + ); +} + /** * Where Panoramax pictures come from: the instance holding each one serves * a 2,048 px SD derivative (equirectangular for a 360° picture) and the HD @@ -16,11 +27,15 @@ import { * @type {import('../../photoViewer.js').PhotoSources} */ export const PANORAMAX_SOURCES = Object.freeze({ - hasPano: (photo) => Boolean(photo.isPano && (photo.urls.sd || photo.urls.hd)), - panoFirst: (photo) => - photo.urls.sd - ? { url: photo.urls.sd, width: SD_WIDTH } - : { url: photo.urls.hd, width: photo.width || SD_WIDTH }, + hasPano: (photo) => + Boolean(photo.isPano && (photo.urls.sd || hdDecodable(photo))), + // Without an SD derivative the original goes straight to the texture size. + panoFirst: (photo) => { + if (photo.urls.sd) return { url: photo.urls.sd, width: SD_WIDTH }; + if (!hdDecodable(photo)) return null; + const width = Math.min(photo.width, PANO_FULL_SIZE[0]); + return { url: photo.urls.hd, width, resize: [width, width / 2] }; + }, panoSharp: (photo, size) => { const width = photo.width || 0; if (!photo.urls.hd || !photo.urls.sd) return null; @@ -34,8 +49,13 @@ export const PANORAMAX_SOURCES = Object.freeze({ timeoutMs: HD_TIMEOUT_MS, }; }, - // A 360° picture shown flat (no WebGL) is its whole equirectangular image. - flatFirst: (photo) => [photo.urls.sd, photo.urls.hd, photo.urls.thumb], + // A 360° picture shown flat (no WebGL) is its whole equirectangular image; + // an original too big to decode is skipped for the thumbnail. + flatFirst: (photo) => [ + photo.urls.sd, + !photo.isPano || hdDecodable(photo) ? photo.urls.hd : null, + photo.urls.thumb, + ], flatSharp: (photo, pixels) => photo.urls.sd && photo.urls.hd && pixels > SD_WIDTH ? photo.urls.hd : null, describe: (photo) => ({ diff --git a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs index d1ec2732e..3dd5e7a1b 100644 --- a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs @@ -143,6 +143,7 @@ test('a 360° picture shows its SD equirectangular image, then HD shrunk to 4,09 assert.deepEqual(loads[0], { url: PHOTOS.pano.urls.sd, crossOrigin: true, + resize: null, failure: 'The Panoramax photo could not be loaded', }); assert.equal(poses.at(-1).isPano, true); @@ -179,8 +180,16 @@ test('HD panoramas too big to decode, or no bigger than SD, are never fetched', const hdOnly = { ...PHOTOS.pano, urls: { ...PHOTOS.pano.urls, sd: null } }; assert.deepEqual(PANORAMAX_SOURCES.panoFirst(hdOnly), { url: hdOnly.urls.hd, - width: 5760, + width: 4096, + resize: [4096, 2048], }); + // …unless it is too big to decode: then no panorama, and the thumbnail. + const hugeOnly = { ...hdOnly, width: 11_904 }; + assert.equal(PANORAMAX_SOURCES.panoFirst(hugeOnly), null); + assert.equal(PANORAMAX_SOURCES.hasPano(hugeOnly), false); + assert.deepEqual(PANORAMAX_SOURCES.flatFirst(hugeOnly).filter(Boolean), [ + hugeOnly.urls.thumb, + ]); assert.equal(PANORAMAX_SOURCES.panoSharp(hdOnly, size), null); assert.equal(PANORAMAX_SOURCES.flatSharp(PHOTOS.flat, 2048), null); assert.equal( diff --git a/src/tooling/streetLevelQa.test.mjs b/src/tooling/streetLevelQa.test.mjs index 354ca3edf..c4eb22dd3 100644 --- a/src/tooling/streetLevelQa.test.mjs +++ b/src/tooling/streetLevelQa.test.mjs @@ -755,12 +755,10 @@ test('Panoramax fixtures: two sequences on one street, searches honour sequence panoramaxTile, resetPanoramaxFixture, } = await import('../../scripts/fixtures/street-level/panoramax.mjs'); - const { decodeCoverageTile, decodePictures } = await import( - '../layers/streetLevel/providers/panoramax/decode.js' - ); - const { normalizeItem } = await import( - '../layers/streetLevel/providers/panoramax/policy.js' - ); + const { decodeCoverageTile, decodePictures } = + await import('../layers/streetLevel/providers/panoramax/decode.js'); + const { normalizeItem } = + await import('../layers/streetLevel/providers/panoramax/policy.js'); resetPanoramaxFixture(); const pictures = panoramaxPictures(); assert.deepEqual( From 552666689453e03976af4be55a57e477e26b0691 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Wed, 14 Oct 2026 14:37:45 +0100 Subject: [PATCH 48/95] =?UTF-8?q?fix(street-level):=20Panoramax=20UX=20QA?= =?UTF-8?q?=20=E2=80=94=20connection=20budget,=20coverage=20to=2060=20km,?= =?UTF-8?q?=20Esc,=20attribution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a UX QA round in the browser against the live API: - A line click could time out while coverage loaded: the meta-catalogue speaks HTTP/1.1 only (six connections), and six tiles plus four searches queued the lookup behind megabyte tiles. Tiles now take three connections and searches two; a search that timed out is asked once more. - Between 30 and ~100 km Paris read as empty: the density grid is ~10 km a cell. Lines now draw to 60 km (z10), as Mapillary's do; above, dots grow with the pictures they stand for. - Esc in the docked viewer collapsed the whole panel instead of clearing the sequence the panel says it clears (shared photo viewer: KartaView too). The viewer now offers Esc to the provider first; expanded, the dialog keeps it. - A picture whose producer is an email address is credited to the instance holding it (its `via` host), so Licence Ouverte pictures name their source. - Empty states say why: "no Panoramax photos here match the filters" when the filter hid every line, and nothing to zoom into over empty sea. Wording follows KartaView's ("photos", "click a line for its nearest photo"). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- README.md | 2 +- docs/CURRENT-STATE.md | 7 +- scripts/qa-street-level.mjs | 2 +- src/layers/streetLevel/lineCoverage.js | 4 +- src/layers/streetLevel/photoViewer.js | 14 +++ src/layers/streetLevel/photoViewer.test.mjs | 32 +++++++ .../streetLevel/providers/kartaview/index.js | 7 ++ .../streetLevel/providers/panoramax/index.js | 48 ++++++++--- .../streetLevel/providers/panoramax/policy.js | 46 +++++++--- .../providers/panoramax/policy.test.mjs | 15 ++++ .../providers/panoramax/provider.test.mjs | 86 ++++++++++++++++++- .../streetLevel/providers/panoramax/source.js | 21 +++-- .../providers/panoramax/source.test.mjs | 76 ++++++++++++++-- 14 files changed, 317 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c347fa44..61acfa6f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,7 +97,7 @@ - Street Level: Panoramax is the fourth provider, keyless. Light its chip (orange; off by default, one provider at a time): from orbit dots show - where the federated commons has pictures, and below about 30 km its + where the federated commons has pictures, and below about 60 km its sequences draw as lines from `api.panoramax.xyz` vector tiles. A click on a line opens that sequence's picture nearest the click without moving the camera, flat or 360°, with cones along the sequence around it and ‹ › / diff --git a/README.md b/README.md index 8fef6c826..100e156b4 100644 --- a/README.md +++ b/README.md @@ -344,7 +344,7 @@ _The Space Missions layer replaying a Falcon 9 ascent — labeled `RECONSTRUCTED **Setup.** At [mapillary.com/dashboard/developers](https://www.mapillary.com/dashboard/developers), register an application with **Read** access and copy its **Client Token** (`MLY|…`). Paste it into **POWER UP → Mapillary**, or set `MAPILLARY_CLIENT_TOKEN` in `.env` and restart. Without it the panel reads **KEY REQUIRED**. **Google Street View** uses `GOOGLE_MAPS_API_KEY` with the **Maps JavaScript API** enabled for it in Google Cloud; its chip starts off, since every panorama is billed to the key. **KartaView** and **Panoramax** need nothing: their public APIs are keyless. -**Use.** Turn on **Street Level** under DATA LAYERS → Cameras, or with the pill in the STREET LEVEL panel header. Each provider has a chip in its own colour (Mapillary green), and one is lit at a time: lighting one switches the others off. Mapillary, which is free, is lit by default. The **IMAGERY** (ALL / 360° / FLAT) and **SINCE** filters apply to Mapillary; Street View panoramas are all 360°, so with Street View lit the filters and legend are hidden. Click a coverage line for its image cones and a cone to open the photo. Light **KARTAVIEW** (purple) and zoom in below about 6 km: its lines are the streets KartaView has photos of, and a click on one opens the photo nearest the click, flat or 360° (drag to look around), with cones along its drive; **‹ ›** or ← → step through the drive (Shift+← →, ↑ ↓ and + − look around a 360° photo; pinch zooms on a phone). Photos KartaView has archived are skipped, and where all are archived the panel says so. KartaView photos are undated on its map, so the filters do not apply to it. Light **PANORAMAX** (orange) for the open, federated commons (strongest in France and Belgium): from orbit its dots show where it has pictures, and below about 30 km its lines are sequences. A click on a line opens that sequence's picture nearest the click without moving the camera, flat or 360°, with cones along the sequence around it; **‹ ›** or ← → step through it, and both filters apply. Its caption names the picture's author and licence (mostly CC BY-SA 4.0; IGN's are Licence Ouverte 2.0). On the **Google 3D** map (Google's terms keep Street View off other maps, where it does not appear), light **GOOGLE STREET VIEW** and click a street from below 3 km, or press **OPEN STREET VIEW** for the view centre, to open Google's panorama there; step along its arrows. Above the photo: **EXPAND**, **FIT / FILL** (Mapillary, and flat KartaView and Panoramax photos), and **FOLLOW** (the globe camera follows the photo's view; Google 3D map only). The caption links to the image on the provider's site, and the globe credits each provider that is on ("© Mapillary contributors, CC BY-SA 4.0", "Street View imagery © Google", "© KartaView contributors, CC BY-SA 4.0", "© Panoramax contributors, licence per picture"). KartaView's own photo pages fail for signed-out visitors, so its link opens the full-size photo. +**Use.** Turn on **Street Level** under DATA LAYERS → Cameras, or with the pill in the STREET LEVEL panel header. Each provider has a chip in its own colour (Mapillary green), and one is lit at a time: lighting one switches the others off. Mapillary, which is free, is lit by default. The **IMAGERY** (ALL / 360° / FLAT) and **SINCE** filters apply to Mapillary; Street View panoramas are all 360°, so with Street View lit the filters and legend are hidden. Click a coverage line for its image cones and a cone to open the photo. Light **KARTAVIEW** (purple) and zoom in below about 6 km: its lines are the streets KartaView has photos of, and a click on one opens the photo nearest the click, flat or 360° (drag to look around), with cones along its drive; **‹ ›** or ← → step through the drive (Shift+← →, ↑ ↓ and + − look around a 360° photo; pinch zooms on a phone). Photos KartaView has archived are skipped, and where all are archived the panel says so. KartaView photos are undated on its map, so the filters do not apply to it. Light **PANORAMAX** (orange) for the open, federated commons (strongest in France and Belgium): from orbit its dots show where it has pictures, and below about 60 km its lines are sequences. A click on a line opens that sequence's picture nearest the click without moving the camera, flat or 360°, with cones along the sequence around it; **‹ ›** or ← → step through it, and both filters apply. Its caption names the picture's author and licence (mostly CC BY-SA 4.0; IGN's are Licence Ouverte 2.0). On the **Google 3D** map (Google's terms keep Street View off other maps, where it does not appear), light **GOOGLE STREET VIEW** and click a street from below 3 km, or press **OPEN STREET VIEW** for the view centre, to open Google's panorama there; step along its arrows. Above the photo: **EXPAND**, **FIT / FILL** (Mapillary, and flat KartaView and Panoramax photos), and **FOLLOW** (the globe camera follows the photo's view; Google 3D map only). The caption links to the image on the provider's site, and the globe credits each provider that is on ("© Mapillary contributors, CC BY-SA 4.0", "Street View imagery © Google", "© KartaView contributors, CC BY-SA 4.0", "© Panoramax contributors, licence per picture"). KartaView's own photo pages fail for signed-out visitors, so its link opens the full-size photo. Drag the panel header to float the panel, and resize it from any edge. Double-click the header or collapse it to put it back in the rail. Share links carry the provider switches and filters (`0.m.0` Mapillary off, `0.m.0_0.r.google-1` Street View on, `0.m.0_0.r.kartaview-1` KartaView on, `0.m.0_0.r.panoramax-1` Panoramax on (a link that leaves out `m` keeps Mapillary, the default, lit), `0.p.p` panoramas only, `0.s.365` the last year). To add a provider, implement the contract in `src/layers/streetLevel/registry.js` and register it in `src/app/layers/streetLevel.js`. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 7cf173b2c..2b389b675 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3254,9 +3254,10 @@ each picture is served by the instance holding it (OpenStreetMap France, IGN, MapComplete…; IGN's redirect to its OVH object storage). Its chip (orange, `#ff6f00`, Panoramax's own) starts off. Coverage comes from its vector tiles (`/api/map/{z}/{x}/{y}.mvt`, an hour's HTTP cache): sequence -lines from z14 below 1.8 km above ground, z13 to 5 km, z12 to 12 km and z11 -to 30 km (a dense z11 tile over Paris is ~1.4 MB and 3,800 sequences; each -tile draws its 1,600 longest), and above 30 km the `grid` layer's 0.1° +lines from z14 below 1.8 km above ground, z13 to 5 km, z12 to 12 km, z11 +to 30 km and z10 to 60 km, as Mapillary's (a dense z10 tile over Paris is +~2.1 MB and 8,600 sequences; each tile draws its 1,600 longest), and above +60 km the `grid` layer's 0.1° density cells, binned to at most 96×96 points a tile (a z0 tile holds ~70,000 cells) and not counted as sequences: the panel says to zoom in. Both filters apply: lines carry their day and camera type, and a cell stays while diff --git a/scripts/qa-street-level.mjs b/scripts/qa-street-level.mjs index 52251ba2c..f230fbdc8 100644 --- a/scripts/qa-street-level.mjs +++ b/scripts/qa-street-level.mjs @@ -1200,7 +1200,7 @@ async function main() { ); assert.match( await page.$eval('#sl-coverage-meta', (node) => node.textContent), - /sequences in view · click a line for the photo there/, + /sequences in view · click a line for its nearest photo/, ); await page.waitForFunction( () => diff --git a/src/layers/streetLevel/lineCoverage.js b/src/layers/streetLevel/lineCoverage.js index 9c363a07e..1a8d73b6f 100644 --- a/src/layers/streetLevel/lineCoverage.js +++ b/src/layers/streetLevel/lineCoverage.js @@ -95,6 +95,7 @@ export function meshBoxInRange(box, centre) { * @param {LineCoverageTuning} options.tuning * @param {{keyRejected?: string, rateLimited: string, tooHigh?: string}} options.messages * @param {(sequence: object, filter: object) => boolean} [options.passes] Whether the imagery filter keeps a line or point. + * @param {(point: object) => number} [options.overviewPointSize] An overview point's size in pixels (default `tuning.overviewPointPx`). */ export function createLineCoverage({ state, @@ -106,6 +107,7 @@ export function createLineCoverage({ tuning, messages, passes = passesImageryFilter, + overviewPointSize = null, }) { const { render } = state.services; const perTileSequenceCap = Math.floor(tuning.maxSequences / tuning.maxTiles); @@ -427,7 +429,7 @@ export function createLineCoverage({ collection.add({ position: Cesium.Cartesian3.fromDegrees(point.lon, point.lat), color: tint, - pixelSize: tuning.overviewPointPx, + pixelSize: overviewPointSize?.(point) ?? tuning.overviewPointPx, // Google 3D terrain and clouds must not hide the near side's dots; // the horizon cull hides the far side's, which this lets through. disableDepthTestDistance: Number.POSITIVE_INFINITY, diff --git a/src/layers/streetLevel/photoViewer.js b/src/layers/streetLevel/photoViewer.js index 705519446..bb1e28dde 100644 --- a/src/layers/streetLevel/photoViewer.js +++ b/src/layers/streetLevel/photoViewer.js @@ -117,6 +117,7 @@ export function panoSizeFor(canvasPixels, fovDeg) { * @param {(photo: object, delta: -1|1) => Promise} options.step The photo `delta` along the sequence, or null at its end. * @param {(imageId: string) => void} options.onStep Open the photo ‹ › chose (through the core, so loading and errors show). * @param {(photo: object) => void} [options.onShown] A photo is on screen. + * @param {() => boolean} [options.onEscape] Esc in the docked viewer: true when it was used (e.g. a selection cleared). * @param {object} [options.render] Render governor. * @param {(url: string, options?: object) => Promise} [options.load] * @param {(canvas: HTMLCanvasElement) => object} [options.createRenderer] @@ -130,6 +131,7 @@ export function createPhotoViewer({ step, onStep, onShown = () => {}, + onEscape = () => false, render, load = loadImage, createRenderer = createPanoramaRenderer, @@ -417,6 +419,18 @@ export function createPhotoViewer({ event.metaKey ) return; + // Docked, Esc is the viewer's before the panel's (which would collapse); + // expanded, the dialog's own Esc shrinks it. + if ( + event.key === 'Escape' && + !event.shiftKey && + !root?.closest?.('.sl-viewer-wrap-expanded') && + onEscape() + ) { + event.preventDefault(); + event.stopPropagation(); + return; + } const pano = showing === 'pano' && renderer; const turn = { ArrowLeft: -1, ArrowRight: 1 }[event.key]; // Shift+← → look around a panorama; ← → step along the sequence. diff --git a/src/layers/streetLevel/photoViewer.test.mjs b/src/layers/streetLevel/photoViewer.test.mjs index 5276ea375..704a2f98c 100644 --- a/src/layers/streetLevel/photoViewer.test.mjs +++ b/src/layers/streetLevel/photoViewer.test.mjs @@ -132,3 +132,35 @@ test('a sharper flat version that failed is tried again on the next resize', asy env.release('f-sharp'); assert.ok(await until(() => picture.src === 'f-sharp')); }); + +test('Esc: the docked viewer offers it to the provider first; expanded, it is the dialog’s', async () => { + let asked = 0; + const viewer = createPhotoViewer({ + providerId: 'test', + name: 'Test', + sources: SOURCES, + getPhoto: async (id) => PHOTOS[id], + step: async () => null, + onStep() {}, + onEscape: () => ++asked > 0, + load: async (url) => ({ src: url }), + }); + const host = fakeHost(); + await viewer.mount(host); + const root = host.children[0]; + const press = () => { + const event = { key: 'Escape', prevented: false, stopped: false }; + event.preventDefault = () => (event.prevented = true); + event.stopPropagation = () => (event.stopped = true); + root.fire('keydown', event); + return event; + }; + const docked = press(); + assert.equal(asked, 1); + assert.equal(docked.prevented && docked.stopped, true); + root.closest = (selector) => + selector.includes('sl-viewer-wrap-expanded') ? {} : null; + const expanded = press(); + assert.equal(asked, 1, 'not asked while expanded'); + assert.equal(expanded.prevented, false); +}); diff --git a/src/layers/streetLevel/providers/kartaview/index.js b/src/layers/streetLevel/providers/kartaview/index.js index 07e9672cf..1b377479d 100644 --- a/src/layers/streetLevel/providers/kartaview/index.js +++ b/src/layers/streetLevel/providers/kartaview/index.js @@ -406,6 +406,13 @@ export function createKartaViewProvider({ step, onStep: openStepped, onShown: recentreCones, + // Esc in the viewer clears the selected drive, as the panel says. + onEscape: () => { + if (!state.sequence.selectedId) return false; + cones.clearSelection(); + drawnPage = null; + return true; + }, render: context.services?.render, ...viewerOptions, }); diff --git a/src/layers/streetLevel/providers/panoramax/index.js b/src/layers/streetLevel/providers/panoramax/index.js index 1fa65fd51..b6eac8297 100644 --- a/src/layers/streetLevel/providers/panoramax/index.js +++ b/src/layers/streetLevel/providers/panoramax/index.js @@ -15,6 +15,7 @@ import { COLORS, COVERAGE_LINES_BELOW_M, COVERAGE_TUNING, + COVERAGE_Z11_BELOW_M, COVERAGE_Z12_BELOW_M, COVERAGE_Z13_BELOW_M, COVERAGE_Z14_BELOW_M, @@ -24,6 +25,7 @@ import { LINE_CLICK_RADIUS_PER_HEIGHT, NEAREST_LIMIT, NEAREST_RADIUS_M, + FILTERED_HINT, NO_COVERAGE_HINT, OVERVIEW_HINT, PANORAMAX_CREDIT_HTML, @@ -55,13 +57,22 @@ async function readable(decode) { } } +/** + * A density dot's size: busier places draw bigger (2.5 px for one picture, + * 8 px from a few hundred thousand), so a city does not read like farmland. + */ +export function overviewPointSize(point) { + const weight = Math.max(1, Number(point?.weight) || 1); + return Math.min(8, 2.5 + Math.log10(weight)); +} + /** Pictures kept in memory by id (lookups and ‹ › fill it). */ const PHOTO_CACHE_SIZE = 2000; /** Sequences whose cone centre is remembered. */ const ANCHOR_CACHE_SIZE = 50; /** - * Sequence lines at z11–14 below 30 km; above, the density grid at z5 and + * Sequence lines at z10–14 below 60 km; above, the density grid at z5 and * coarser. */ export function panoramaxZoomFor(heightM) { @@ -74,7 +85,9 @@ export function panoramaxZoomFor(heightM) { ? 13 : heightM <= COVERAGE_Z12_BELOW_M ? 12 - : 11; + : heightM <= COVERAGE_Z11_BELOW_M + ? 11 + : 10; return { zoom, kind: 'sequence' }; } return { @@ -290,6 +303,7 @@ export function createPanoramaxProvider({ tuning: COVERAGE_TUNING, messages: { rateLimited: RATE_LIMITED_MESSAGE }, passes: passesCoverage, + overviewPointSize, }); const cones = createSequenceCones({ @@ -352,6 +366,13 @@ export function createPanoramaxProvider({ onStep: (photoId) => context.actions.openImage(photoId, { frame: false }), onShown: recentreCones, + // Esc in the viewer clears the selected sequence, as the panel says. + onEscape: () => { + if (!state.sequence.selectedId) return false; + cones.clearSelection(); + drawn = null; + return true; + }, render: context.services?.render, ...viewerOptions, }); @@ -417,17 +438,22 @@ export function createPanoramaxProvider({ }, coverageStats() { - const lines = state.coverage.kind === 'sequence'; + const { kind } = state.coverage; + const lines = kind === 'sequence'; + const drawn = coverage.sequenceCount(); // Density dots are not sequences: they are not counted as such. - const count = lines ? coverage.sequenceCount() : 0; + const count = lines ? drawn : 0; const loading = state.coverage.pending.size > 0; - const empty = - lines && !loading && count === 0 && !state.coverage.lastError; - const fallback = empty - ? NO_COVERAGE_HINT - : state.coverage.kind === 'overview' - ? OVERVIEW_HINT - : ''; + const settled = !loading && !state.coverage.lastError; + // Lines in view that the filter hid all of: say so, not "none here". + const filteredOut = + lines && + [...state.coverage.tiles.values()].some((tile) => tile.total > 0); + let fallback = ''; + if (kind === 'overview') + fallback = drawn > 0 || !settled ? OVERVIEW_HINT : NO_COVERAGE_HINT; + else if (lines && settled && count === 0) + fallback = filteredOut ? FILTERED_HINT : NO_COVERAGE_HINT; return { count, zoom: state.coverage.zoom, diff --git a/src/layers/streetLevel/providers/panoramax/policy.js b/src/layers/streetLevel/providers/panoramax/policy.js index f7629b5aa..0a122242f 100644 --- a/src/layers/streetLevel/providers/panoramax/policy.js +++ b/src/layers/streetLevel/providers/panoramax/policy.js @@ -29,15 +29,17 @@ export const COLORS = Object.freeze({ }); /** - * Sequence lines come from z7–15 tiles; z11–14 are drawn (a dense z11 tile - * over Paris is ~1.4 MB and 3,800 sequences, z12 ~800 kB). Higher up, the - * z0–5 `grid` layer's density cells (0.1°, ~10 km) draw as points: below - * 30 km they would be a few dots over a whole city. + * Sequence lines come from z7–15 tiles; z10–14 are drawn below 60 km, as + * Mapillary's are (a dense tile over Paris: z10 ~2.1 MB and 8,600 + * sequences, z11 ~1.4 MB, z12 ~800 kB). Higher up, the z0–5 `grid` layer's + * density cells (0.1°, ~10 km) draw as points: lower down they would be a + * handful of dots over a whole city, which reads as no coverage. */ export const COVERAGE_Z14_BELOW_M = 1_800; export const COVERAGE_Z13_BELOW_M = 5_000; export const COVERAGE_Z12_BELOW_M = 12_000; -export const COVERAGE_LINES_BELOW_M = 30_000; +export const COVERAGE_Z11_BELOW_M = 30_000; +export const COVERAGE_LINES_BELOW_M = 60_000; export const COVERAGE_TUNING = Object.freeze({ maxTiles: 9, overviewMaxTiles: 6, @@ -62,12 +64,14 @@ export const OVERVIEW_BINS_PER_AXIS = 96; export const RATE_LIMITED_MESSAGE = 'Panoramax is busy — coverage resumes on its own'; -export const NO_COVERAGE_HINT = 'no Panoramax pictures in view'; +export const NO_COVERAGE_HINT = 'no Panoramax photos in view'; +/** Lines in view, all hidden by the imagery filter. */ +export const FILTERED_HINT = 'no Panoramax photos here match the filters'; export const UNREADABLE_TILE_MESSAGE = 'Panoramax sent a map tile that could not be read'; export const GONE_MESSAGE = 'This Panoramax picture is no longer available'; /** From high up the dots are where pictures are, not sequences to click. */ -export const OVERVIEW_HINT = 'zoom in to click a Panoramax sequence'; +export const OVERVIEW_HINT = 'zoom in to see Panoramax sequences'; /** Cones come from z15 tiles' `pictures` layer (~110 bytes a picture). */ export const PICTURE_TILE_ZOOM = 15; @@ -78,9 +82,13 @@ export const TILE_TIMEOUT_MS = 20_000; export const API_TIMEOUT_MS = 12_000; /** Real tiles are under 6 MB (z6 over France); refuse anything absurd. */ export const MAX_RESPONSE_BYTES = 16 * 1024 * 1024; -/** Requests at once to the API (tiles have their own budget). */ -export const API_CONCURRENCY = 4; -export const TILE_CONCURRENCY = 6; +/** + * The meta-catalogue speaks HTTP/1.1 only, so the browser opens six + * connections to it: tiles take three, searches two, so a click's lookup + * never queues behind megabyte tiles (it timed out doing so). + */ +export const API_CONCURRENCY = 2; +export const TILE_CONCURRENCY = 3; /** Pictures asked for around a point; the API returns them nearest first. */ export const NEAREST_LIMIT = 10; @@ -110,7 +118,7 @@ export const SD_WIDTH = 2_048; export const PANORAMAX_CREDIT_HTML = 'Street Level: imagery © Panoramax contributors, licence per picture'; -export const LINE_CLICK_HINT = 'click a line for the photo there'; +export const LINE_CLICK_HINT = 'click a line for its nearest photo'; /** Licence ids Panoramax uses, as the caption names them. */ const LICENSE_LABELS = Object.freeze({ @@ -194,8 +202,20 @@ function creatorOf(item) { : []), ]; const name = names.find((value) => typeof value === 'string' && value.trim()); - if (!name || looksLikeEmail(name)) return null; - return name.trim().slice(0, 80); + if (name && !looksLikeEmail(name)) return name.trim().slice(0, 80); + // Licences ask for the source to be named: the instance that holds it. + return instanceOf(item); +} + +/** The federated instance holding a picture (its `via` link's host), or null. */ +function instanceOf(item) { + const href = item?.links?.find?.((link) => link?.rel === 'via')?.href; + try { + const url = new URL(String(href)); + return url.protocol === 'https:' ? url.hostname.slice(0, 80) : null; + } catch { + return null; + } } /** A 360° picture: a full-sphere field of view or an equirectangular projection. */ diff --git a/src/layers/streetLevel/providers/panoramax/policy.test.mjs b/src/layers/streetLevel/providers/panoramax/policy.test.mjs index 605bcf7ad..57665f025 100644 --- a/src/layers/streetLevel/providers/panoramax/policy.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/policy.test.mjs @@ -225,3 +225,18 @@ test('hostile item data: a malformed link escape, null coordinates, a huge produ 'an @ without a domain is a name', ); }); + +test('a picture whose producer is hidden is credited to the instance holding it', () => { + const item = stacItem({ + id: 'a', + lon: 1, + lat: 2, + producer: 'someone@ign.example.fr', + }); + item.links.push({ rel: 'via', href: 'https://panoramax.ign.fr' }); + assert.equal(normalizeItem(item).creator, 'panoramax.ign.fr'); + item.links.at(-1).href = 'http://insecure.example'; + assert.equal(normalizeItem(item).creator, null); + item.links.at(-1).href = 'not a url'; + assert.equal(normalizeItem(item).creator, null); +}); diff --git a/src/layers/streetLevel/providers/panoramax/provider.test.mjs b/src/layers/streetLevel/providers/panoramax/provider.test.mjs index 2ae2c8091..3ce1ee490 100644 --- a/src/layers/streetLevel/providers/panoramax/provider.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/provider.test.mjs @@ -2,11 +2,17 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { createPanoramaxProvider, + overviewPointSize, panoramaxZoomFor, passesCoverage, pictureTilesAround, } from './index.js'; -import { NO_COVERAGE_HINT, OVERVIEW_HINT, PICK_PREFIX } from './policy.js'; +import { + FILTERED_HINT, + NO_COVERAGE_HINT, + OVERVIEW_HINT, + PICK_PREFIX, +} from './policy.js'; import { validateProviders } from '../../registry.js'; import { tileBounds } from '../../tileMath.js'; import { fakeHost, until } from '../../../../testSupport/kartaViewFakes.mjs'; @@ -141,19 +147,21 @@ test('Panoramax registers keyless and off by default, with both filters and its assert.match(definition.credit.html, /Panoramax<\/a> contributors/); assert.equal( definition.coverageLabel.action, - 'click a line for the photo there', + 'click a line for its nearest photo', ); assert.doesNotThrow(() => validateProviders([definition])); }); -test('lines at z11–14 below 30 km; density points from z5 down to z0 above', () => { +test('lines at z10–14 below 60 km, as Mapillary’s; density points from z5 down to z0 above', () => { assert.deepEqual(panoramaxZoomFor(500), { zoom: 14, kind: 'sequence' }); assert.deepEqual(panoramaxZoomFor(1_800), { zoom: 14, kind: 'sequence' }); assert.deepEqual(panoramaxZoomFor(4_000), { zoom: 13, kind: 'sequence' }); assert.deepEqual(panoramaxZoomFor(12_000), { zoom: 12, kind: 'sequence' }); assert.deepEqual(panoramaxZoomFor(25_000), { zoom: 11, kind: 'sequence' }); assert.deepEqual(panoramaxZoomFor(30_000), { zoom: 11, kind: 'sequence' }); - assert.deepEqual(panoramaxZoomFor(30_001), { zoom: 5, kind: 'overview' }); + assert.deepEqual(panoramaxZoomFor(30_001), { zoom: 10, kind: 'sequence' }); + assert.deepEqual(panoramaxZoomFor(60_000), { zoom: 10, kind: 'sequence' }); + assert.deepEqual(panoramaxZoomFor(60_001), { zoom: 5, kind: 'overview' }); assert.deepEqual(panoramaxZoomFor(100_000), { zoom: 5, kind: 'overview' }); assert.deepEqual(panoramaxZoomFor(2_000_000), { zoom: 3, kind: 'overview' }); assert.deepEqual(panoramaxZoomFor(20_000_000), { zoom: 0, kind: 'overview' }); @@ -690,3 +698,73 @@ test('switching off clears the cones and forgets a pending line click', async (t await env.instance.nearestImage(east(0)); assert.equal(env.source.calls.nearest.at(-1).collections, null); }); + +test('lines in view that the filter hides all of say so, not that there is nothing here', async (t) => { + const env = coverageSetup(t, { + filter: { pano: 'pano', sinceMs: null }, + sequences: [line('flat-only')], + }); + env.instance.activate(env.viewer); + assert.ok( + await until( + () => + !env.instance.coverageStats().loading && + env.instance.coverageStats().zoom !== null, + ), + ); + assert.ok( + await until(() => env.instance.coverageStats().hint === FILTERED_HINT), + ); + assert.equal(env.instance.coverageStats().count, 0); +}); + +test('from high up over empty sea the panel says there is nothing, not to zoom in', async (t) => { + const env = coverageSetup(t, { height: 200_000, grid: [] }); + env.instance.activate(env.viewer); + assert.ok( + await until(() => env.instance.coverageStats().kind === 'overview'), + ); + assert.ok( + await until(() => env.instance.coverageStats().hint === NO_COVERAGE_HINT), + ); +}); + +test('density dots grow with the pictures they stand for', () => { + assert.equal(overviewPointSize({ weight: 1 }), 2.5); + assert.equal(overviewPointSize({ weight: 100 }), 4.5); + assert.equal(overviewPointSize({ weight: 1e9 }), 8); + assert.equal(overviewPointSize({}), 2.5); + assert.ok( + overviewPointSize({ weight: 50_000 }) > + overviewPointSize({ weight: 50 }) + 2, + ); +}); + +test('Esc in the docked viewer clears the selected sequence before the panel can collapse', async (t) => { + const items = sequence('s', 3); + const env = await mounted({ items, tiles: pictureTiles(items) }); + await withCones(t, env); + await env.instance.viewer.open('s-0'); + await env.instance.selectSequence('s'); + assert.ok(await until(() => env.instance.sequenceStats().images === 3)); + const root = env.next.parentNode; + const event = { + key: 'Escape', + stopped: false, + stopPropagation() { + this.stopped = true; + }, + }; + root.fire('keydown', event); + assert.equal(env.instance.sequenceStats().selectedId, null); + // Nothing selected: Esc goes on to the panel. + const again = { key: 'Escape', defaultPrevented: false }; + const result = { prevented: false }; + root.fire('keydown', { + ...again, + preventDefault() { + result.prevented = true; + }, + }); + assert.equal(result.prevented, false); +}); diff --git a/src/layers/streetLevel/providers/panoramax/source.js b/src/layers/streetLevel/providers/panoramax/source.js index 7d3c86e6c..58052d83c 100644 --- a/src/layers/streetLevel/providers/panoramax/source.js +++ b/src/layers/streetLevel/providers/panoramax/source.js @@ -115,13 +115,24 @@ export function createPanoramaxSource({ } } - /** A STAC search, through the API gate (newest request first). */ + /** + * A STAC search, through the API gate (newest request first). One that + * timed out is asked once more: a search is quick once the connection + * is free. + */ async function search(params, { signal } = {}) { const url = `${root}/search?${new URLSearchParams(params)}`; - const bytes = await apiLimiter.run( - () => get(url, { signal, timeoutMs: API_TIMEOUT_MS }), - { signal }, - ); + const once = () => + apiLimiter.run(() => get(url, { signal, timeoutMs: API_TIMEOUT_MS }), { + signal, + }); + let bytes; + try { + bytes = await once(); + } catch (error) { + if (signal?.aborted || error?.message !== NO_ANSWER) throw error; + bytes = await once(); + } let payload; try { payload = JSON.parse(new TextDecoder().decode(bytes)); diff --git a/src/layers/streetLevel/providers/panoramax/source.test.mjs b/src/layers/streetLevel/providers/panoramax/source.test.mjs index a068a3eff..e0111a33e 100644 --- a/src/layers/streetLevel/providers/panoramax/source.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/source.test.mjs @@ -173,14 +173,16 @@ test('the newest search goes first when the API gate is full', async () => { }); const all = ['1', '2', '3', '4', '5', '6'].map((id) => source.getItem(id)); await new Promise((resolve) => setTimeout(resolve, 0)); - assert.deepEqual(order, ['1', '2', '3', '4'], 'four at once'); + assert.deepEqual(order, ['1', '2'], 'two at once'); waiting.shift()(); await new Promise((resolve) => setTimeout(resolve, 0)); assert.equal(order.at(-1), '6', 'the latest waiting one next'); - while (waiting.length) waiting.shift()(); - await new Promise((resolve) => setTimeout(resolve, 0)); - while (waiting.length) waiting.shift()(); - await Promise.all(all); + let done = false; + Promise.allSettled(all).then(() => (done = true)); + while (!done) { + waiting.splice(0).forEach((go) => go()); + await new Promise((resolve) => setTimeout(resolve, 0)); + } }); test('an unreadable search answer, or one with no features, is handled', async () => { @@ -227,3 +229,67 @@ test('a network failure reads as no answer, not as a stack trace', async () => { return true; }); }); + +test('a search that timed out is asked once more; a second timeout is reported', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let calls = 0; + const hang = (signal) => + new Promise((_, reject) => + signal.addEventListener('abort', () => reject(signal.reason)), + ); + const source = createPanoramaxSource({ + fetchImpl: async (url, { signal }) => { + calls++; + if (calls === 1) return hang(signal); + return new Response(JSON.stringify({ features: [{ id: 'a' }] })); + }, + }); + const asked = source.getItem('a'); + for (let i = 0; i < 5; i++) await Promise.resolve(); + t.mock.timers.tick(60_000); + assert.deepEqual(await asked, { id: 'a' }); + assert.equal(calls, 2); + const stuck = createPanoramaxSource({ + fetchImpl: async (url, { signal }) => hang(signal), + }); + const never = stuck.getItem('b'); + for (let round = 0; round < 2; round++) { + for (let i = 0; i < 10; i++) await Promise.resolve(); + t.mock.timers.tick(60_000); + } + await assert.rejects(never, /did not answer in time/); +}); + +test('the HTTP/1.1 API never gets more than five requests at once: three tiles, two searches', async () => { + const { API_CONCURRENCY, TILE_CONCURRENCY } = await import('./policy.js'); + assert.ok(API_CONCURRENCY + TILE_CONCURRENCY <= 5); + let open = 0; + let most = 0; + const waiting = []; + const source = createPanoramaxSource({ + fetchImpl: async (url) => { + open++; + most = Math.max(most, open); + await new Promise((resolve) => waiting.push(resolve)); + open--; + return url.includes('/map/') + ? new Response(null, { status: 204 }) + : new Response(JSON.stringify({ features: [] })); + }, + }); + const all = [ + ...Array.from({ length: 8 }, (_, i) => + source.getTile({ x: i, y: 0, z: 14 }), + ), + ...Array.from({ length: 4 }, (_, i) => source.getItem(String(i))), + ]; + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(most, 5); + let done = false; + Promise.allSettled(all).then(() => (done = true)); + while (!done) { + waiting.splice(0).forEach((go) => go()); + await new Promise((resolve) => setTimeout(resolve, 0)); + } + assert.equal(most, 5, 'never more'); +}); From cf502429a33e9b11c5a474e0161150db50eeebae Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Wed, 14 Oct 2026 15:24:13 +0100 Subject: [PATCH 49/95] =?UTF-8?q?fix(street-level):=20Panoramax=20IC8=20re?= =?UTF-8?q?view=20=E2=80=94=20line=20geometry=20budget,=20caches,=20attrib?= =?UTF-8?q?ution,=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a five-lens IC8 review (correctness, security, tests with mutation testing, performance, UX): - P0: over Paris at 12–60 km the unsimplified z10–z12 lines sent ~2M points to Cesium's ground-polyline builder (gigabytes of geometry; the tab hung). decode.js now simplifies lines (Douglas–Peucker, 4 tile units: 6–10% of the points), and lineCoverage gains an opt-in point budget (`maxPoints`, 10,000 a tile for Panoramax). - Decoded coverage tiles are kept (KartaView's tile cache moves to the shared tileCache.js); a sequence's z15 picture lists are kept and load on their own connection; cones know their tiles from the start of a load, so a picture opening meanwhile re-centres them and ‹ › no longer restarts the load. - The HD panorama loads only for the expanded viewer (a docked one on a dense screen asked for 3 MB originals) and is aborted when another photo opens; picture files over 48 MB or 16,384 px are refused; a bitmap no renderer took after a context loss is closed. - A nearest lookup survives one of its two searches failing; only a timeout is retried; offline reads as unreachable; Retry-After is capped at five minutes; the search box stays on the map at ±180°; cones near both edges of a narrow (high-latitude) tile read both. - Every producer is credited (app and photographer); with none to name, the caption says "via " rather than crediting the host. - Esc anywhere in the docked viewer (toolbar, caption) clears the sequence before the panel takes it. A filter that hides every density dot says so; wording aligned ("coverage", "photo"). - Tests for every surviving mutant (Esc consumed, keepOrder and dot-size wiring, licence in the core, limiter aborts, radius floor, bbox corner, first-producer-only, byte and size caps), a limiter test, and a gate fixture that answers box searches and leaves out pictures facing away; the gate now clicks over the other sequence's pictures, so only the line click's own-sequence lookup can pass. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +- README.md | 2 +- docs/CURRENT-STATE.md | 20 +- scripts/fixtures/street-level/panoramax.mjs | 34 ++- scripts/format-scope.json | 4 + scripts/package-boundaries.json | 3 + scripts/qa-street-level.mjs | 29 ++- src/layers/streetLevel/limiter.test.mjs | 80 +++++++ src/layers/streetLevel/lineCoverage.js | 18 +- src/layers/streetLevel/lineCoverage.test.mjs | 99 ++++++++- src/layers/streetLevel/photoViewer.js | 84 +++++-- src/layers/streetLevel/photoViewer.test.mjs | 141 ++++++++++++ .../streetLevel/providers/kartaview/index.js | 51 +---- .../providers/kartaview/provider.test.mjs | 21 ++ .../streetLevel/providers/panoramax/decode.js | 50 ++++- .../providers/panoramax/decode.test.mjs | 37 +++- .../streetLevel/providers/panoramax/index.js | 116 +++++----- .../streetLevel/providers/panoramax/policy.js | 40 +++- .../providers/panoramax/policy.test.mjs | 37 +++- .../providers/panoramax/provider.test.mjs | 209 ++++++++++++++++-- .../streetLevel/providers/panoramax/source.js | 76 +++++-- .../providers/panoramax/source.test.mjs | 69 +++++- .../streetLevel/providers/panoramax/viewer.js | 9 +- .../providers/panoramax/viewer.test.mjs | 38 +++- src/layers/streetLevel/tileCache.js | 48 ++++ src/layers/streetLevel/viewerHost.test.mjs | 12 + src/tooling/streetLevelQa.test.mjs | 12 + src/ui/streetLevelPresentation.test.mjs | 19 ++ src/ui/styles/street-level.css | 5 +- 29 files changed, 1153 insertions(+), 216 deletions(-) create mode 100644 src/layers/streetLevel/limiter.test.mjs create mode 100644 src/layers/streetLevel/tileCache.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 61acfa6f8..5bc2965cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,7 +106,11 @@ and licence (CC BY-SA 4.0, Licence Ouverte 2.0…) and links to it in the Panoramax viewer; HD originals load only for an expanded viewer, once. Share links carry it as `0.m.0_0.r.panoramax-1`. KartaView's photo viewer - is now shared (`photoViewer.js`, `panorama.js`). + is now shared (`photoViewer.js`, `panorama.js`); in it, for KartaView too, + Esc clears the selected drive before it collapses the panel. Lines are + simplified and drawn within a point budget (a tile's unsimplified lines + over Paris hung the tab), decoded tiles are kept, and the cones' tiles + have their own connection. - Street Level: KartaView is the third provider, keyless. Light its chip (purple; off by default, one provider at a time) and zoom in below about diff --git a/README.md b/README.md index 100e156b4..0611c6487 100644 --- a/README.md +++ b/README.md @@ -344,7 +344,7 @@ _The Space Missions layer replaying a Falcon 9 ascent — labeled `RECONSTRUCTED **Setup.** At [mapillary.com/dashboard/developers](https://www.mapillary.com/dashboard/developers), register an application with **Read** access and copy its **Client Token** (`MLY|…`). Paste it into **POWER UP → Mapillary**, or set `MAPILLARY_CLIENT_TOKEN` in `.env` and restart. Without it the panel reads **KEY REQUIRED**. **Google Street View** uses `GOOGLE_MAPS_API_KEY` with the **Maps JavaScript API** enabled for it in Google Cloud; its chip starts off, since every panorama is billed to the key. **KartaView** and **Panoramax** need nothing: their public APIs are keyless. -**Use.** Turn on **Street Level** under DATA LAYERS → Cameras, or with the pill in the STREET LEVEL panel header. Each provider has a chip in its own colour (Mapillary green), and one is lit at a time: lighting one switches the others off. Mapillary, which is free, is lit by default. The **IMAGERY** (ALL / 360° / FLAT) and **SINCE** filters apply to Mapillary; Street View panoramas are all 360°, so with Street View lit the filters and legend are hidden. Click a coverage line for its image cones and a cone to open the photo. Light **KARTAVIEW** (purple) and zoom in below about 6 km: its lines are the streets KartaView has photos of, and a click on one opens the photo nearest the click, flat or 360° (drag to look around), with cones along its drive; **‹ ›** or ← → step through the drive (Shift+← →, ↑ ↓ and + − look around a 360° photo; pinch zooms on a phone). Photos KartaView has archived are skipped, and where all are archived the panel says so. KartaView photos are undated on its map, so the filters do not apply to it. Light **PANORAMAX** (orange) for the open, federated commons (strongest in France and Belgium): from orbit its dots show where it has pictures, and below about 60 km its lines are sequences. A click on a line opens that sequence's picture nearest the click without moving the camera, flat or 360°, with cones along the sequence around it; **‹ ›** or ← → step through it, and both filters apply. Its caption names the picture's author and licence (mostly CC BY-SA 4.0; IGN's are Licence Ouverte 2.0). On the **Google 3D** map (Google's terms keep Street View off other maps, where it does not appear), light **GOOGLE STREET VIEW** and click a street from below 3 km, or press **OPEN STREET VIEW** for the view centre, to open Google's panorama there; step along its arrows. Above the photo: **EXPAND**, **FIT / FILL** (Mapillary, and flat KartaView and Panoramax photos), and **FOLLOW** (the globe camera follows the photo's view; Google 3D map only). The caption links to the image on the provider's site, and the globe credits each provider that is on ("© Mapillary contributors, CC BY-SA 4.0", "Street View imagery © Google", "© KartaView contributors, CC BY-SA 4.0", "© Panoramax contributors, licence per picture"). KartaView's own photo pages fail for signed-out visitors, so its link opens the full-size photo. +**Use.** Turn on **Street Level** under DATA LAYERS → Cameras, or with the pill in the STREET LEVEL panel header. Each provider has a chip in its own colour (Mapillary green), and one is lit at a time: lighting one switches the others off. Mapillary, which is free, is lit by default. The **IMAGERY** (ALL / 360° / FLAT) and **SINCE** filters apply to Mapillary; Street View panoramas are all 360°, so with Street View lit the filters and legend are hidden. Click a coverage line for its image cones and a cone to open the photo. Light **KARTAVIEW** (purple) and zoom in below about 6 km: its lines are the streets KartaView has photos of, and a click on one opens the photo nearest the click, flat or 360° (drag to look around), with cones along its drive; **‹ ›** or ← → step through the drive (Shift+← →, ↑ ↓ and + − look around a 360° photo; pinch zooms on a phone). Photos KartaView has archived are skipped, and where all are archived the panel says so. KartaView photos are undated on its map, so the filters do not apply to it. Light **PANORAMAX** (orange) for the open, federated commons (strongest in France and Belgium): from orbit its dots show where it has pictures, and below about 60 km its lines are sequences. A click on a line opens that sequence's picture nearest the click without moving the camera, flat or 360°, with cones along the sequence around it; **‹ ›** or ← → step through it, and both filters apply. Its caption names the picture's authors (or the instance holding it) and licence (mostly CC BY-SA 4.0; pictures on the IGN instance are Licence Ouverte 2.0). On the **Google 3D** map (Google's terms keep Street View off other maps, where it does not appear), light **GOOGLE STREET VIEW** and click a street from below 3 km, or press **OPEN STREET VIEW** for the view centre, to open Google's panorama there; step along its arrows. Above the photo: **EXPAND**, **FIT / FILL** (Mapillary, and flat KartaView and Panoramax photos), and **FOLLOW** (the globe camera follows the photo's view; Google 3D map only). The caption links to the image on the provider's site, and the globe credits each provider that is on ("© Mapillary contributors, CC BY-SA 4.0", "Street View imagery © Google", "© KartaView contributors, CC BY-SA 4.0", "© Panoramax contributors, licence per picture"). KartaView's own photo pages fail for signed-out visitors, so its link opens the full-size photo. Drag the panel header to float the panel, and resize it from any edge. Double-click the header or collapse it to put it back in the rail. Share links carry the provider switches and filters (`0.m.0` Mapillary off, `0.m.0_0.r.google-1` Street View on, `0.m.0_0.r.kartaview-1` KartaView on, `0.m.0_0.r.panoramax-1` Panoramax on (a link that leaves out `m` keeps Mapillary, the default, lit), `0.p.p` panoramas only, `0.s.365` the last year). To add a provider, implement the contract in `src/layers/streetLevel/registry.js` and register it in `src/app/layers/streetLevel.js`. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 2b389b675..a9fc1f5fb 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3275,16 +3275,22 @@ SINCE; a line with no camera type or day stays drawn. Lines keep the decoded longest-first order under the per-tile cap (`keepOrder`). Once the picture is on screen its sequence is highlighted and drawn as cones from the z15 tiles around it (their `pictures` layer, ~110 bytes a picture, against ~7.5 kB for a STAC -item), plus a neighbour within 150 m of an edge; the cones re-centre when a -picture opens outside them. ‹ › follow the picture's own `prev`/`next` +item), plus the neighbours within 150 m of an edge, on their own connection +(coverage tiles take three, searches two); each tile's list for the sequence +is kept, and the cones re-centre when a picture opens outside the tiles +they were drawn from, judged from the moment their load starts. ‹ › follow the picture's own `prev`/`next` links, so stepping costs one small search per picture. The viewer shows the 2,048 px SD derivative (equirectangular for a 360° picture) and swaps in the -HD original only when the viewer is wider than SD (flat) or needs a 4,096 px -panorama (expanded), once, shrunk while it decodes; originals wider than -8,192 px are never fetched. The caption names the author (not an account -named by an email address) and the picture's licence (CC BY-SA 4.0, Licence +HD original only when the viewer is wider than SD (flat) or, for a +panorama, expanded; once, shrunk while it decodes, and aborted if another +photo opens; originals wider than 8,192 px are never fetched, and any +picture file over 48 MB or 16,384 px is refused. The caption names every +producer (an uploading app and its photographer; never an account named by +an email address), or else the instance holding the picture ("via +panoramax.ign.fr"), and the picture's licence (CC BY-SA 4.0, Licence Ouverte 2.0…; StreetPose `license`), and links to the picture in the -Panoramax viewer. The MCP globe panel allows the meta-catalogue and the +Panoramax viewer. Esc anywhere in the docked viewer clears the selected +sequence before the panel can take it (KartaView too). The MCP globe panel allows the meta-catalogue and the three biggest instances; a picture on another instance shows only in the app. diff --git a/scripts/fixtures/street-level/panoramax.mjs b/scripts/fixtures/street-level/panoramax.mjs index 768a7ff62..7506eb806 100644 --- a/scripts/fixtures/street-level/panoramax.mjs +++ b/scripts/fixtures/street-level/panoramax.mjs @@ -112,7 +112,11 @@ function item(picture, pictures) { sd: { href: `${files}/sd.jpg` }, thumb: { href: `${files}/thumb.jpg` }, }, - links: [...link('prev', sibling(-1)), ...link('next', sibling(1))], + links: [ + ...link('prev', sibling(-1)), + ...link('next', sibling(1)), + { rel: 'via', href: HOST }, + ], }; } @@ -231,11 +235,9 @@ export function answerPanoramaxRequest({ method, url }) { .map((p) => item(p, pictures)), }); } - const [lon, lat] = (q.get('place_position') || '').split(',').map(Number); - const radius = Number((q.get('place_distance') || '0-50').split('-')[1]); const filter = q.get('filter') || ''; const since = q.get('datetime')?.split('/')[0]; - const near = pictures + const kept = pictures .filter( (p) => !q.get('collections') || p.sequenceId === q.get('collections'), ) @@ -246,7 +248,29 @@ export function answerPanoramaxRequest({ method, url }) { ? !p.isPano : true, ) - .filter((p) => !since || Date.parse(p.datetime) >= Date.parse(since)) + .filter((p) => !since || Date.parse(p.datetime) >= Date.parse(since)); + // A box: every picture in it, in no particular order (id order here). + if (q.has('bbox')) { + const [west, south, east, north] = q.get('bbox').split(',').map(Number); + return json({ + type: 'FeatureCollection', + features: kept + .filter( + (p) => + p.lon >= west && p.lon <= east && p.lat >= south && p.lat <= north, + ) + .sort((a, b) => a.id.localeCompare(b.id)) + .slice(0, Number(q.get('limit')) || 10) + .map((p) => item(p, pictures)), + }); + } + const [lon, lat] = (q.get('place_position') || '').split(',').map(Number); + const radius = Number((q.get('place_distance') || '0-50').split('-')[1]); + // As the API does, even at the widest tolerance: a place behind a + // picture (every one looks east) is not in it. + const facing = (p) => lon >= p.lon - 1e-7; + const near = kept + .filter(facing) .map((p) => ({ p, metres: metresApart({ lon, lat }, p) })) .filter(({ metres }) => metres <= radius) .sort((a, b) => a.metres - b.metres) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 73959131f..53d6a5b67 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -928,12 +928,16 @@ "src/layers/streetLevel/providers/panoramax/viewer.js", "src/layers/streetLevel/photoViewer.js", "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/tileCache.js", "src/layers/streetLevel/providers/panoramax/policy.test.mjs", "src/layers/streetLevel/providers/panoramax/decode.test.mjs", "src/layers/streetLevel/providers/panoramax/source.test.mjs", "src/layers/streetLevel/providers/panoramax/viewer.test.mjs", "src/layers/streetLevel/providers/panoramax/provider.test.mjs", "src/testSupport/panoramaxFixtures.mjs", + "src/layers/streetLevel/photoViewer.test.mjs", + "src/layers/streetLevel/limiter.test.mjs", + "src/layers/streetLevel/panorama.test.mjs", "src/layers/streetLevel/lineCoverage.test.mjs", "src/testSupport/kartaViewFakes.mjs", "src/testSupport/googleStreetViewFakes.mjs", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 2e83eb2cc..493bae74a 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1241,6 +1241,7 @@ "src/layers/streetLevel/providers/panoramax/viewer.js", "src/layers/streetLevel/photoViewer.js", "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/tileCache.js", "src/layers/streetLevel/panorama.js", "src/layers/streetLevel/lineCoverage.js", "src/layers/streetLevel/sequenceCones.js", @@ -2216,6 +2217,7 @@ "src/layers/streetLevel/providers/panoramax/viewer.js", "src/layers/streetLevel/photoViewer.js", "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/tileCache.js", "src/layers/streetLevel/panorama.js", "src/layers/streetLevel/lineCoverage.js", "src/layers/streetLevel/sequenceCones.js", @@ -2762,6 +2764,7 @@ "src/layers/streetLevel/providers/panoramax/viewer.js", "src/layers/streetLevel/photoViewer.js", "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/tileCache.js", "src/layers/streetLevel/panorama.js", "src/layers/streetLevel/lineCoverage.js", "src/layers/streetLevel/sequenceCones.js", diff --git a/scripts/qa-street-level.mjs b/scripts/qa-street-level.mjs index f230fbdc8..9c8c55072 100644 --- a/scripts/qa-street-level.mjs +++ b/scripts/qa-street-level.mjs @@ -1255,7 +1255,30 @@ async function main() { .catch(() => null); if (!handle && !fixtures) return skip('no Panoramax line in view'); assert.ok(handle, 'a Panoramax line is pickable'); - const at = await handle.jsonValue(); + let at = await handle.jsonValue(); + if (fixtures) { + // Click the top line where the other sequence's pictures are: + // the nearest picture is then the other's, and only the line + // click's own-sequence lookup opens the clicked line's. + const others = panoramaxPictures() + .filter((p) => p.sequenceId !== at.id) + .slice(40, 100) + .map((p) => [p.lon, p.lat]); + const aimed = await page + .waitForFunction( + findPickPoint, + { timeout: 60_000, polling: 500 }, + { + prefix: 'pnx:seq:', + id: at.id, + points: others, + billboardIds: null, + }, + ) + .catch(() => null); + assert.ok(aimed, 'the line is pickable over the other’s pictures'); + at = await aimed.jsonValue(); + } const parked = await camera(); await page.mouse.click(at.x, at.y); await page.waitForFunction( @@ -1298,8 +1321,8 @@ async function main() { else assert.equal( caption, - 'Licence Ouverte 2.0', - 'an account named by an email address is not shown', + 'via panoramax.openstreetmap.fr · Licence Ouverte 2.0', + 'an account named by an email address is not shown; the instance is', ); const pictures = panoramaxPictures().filter( (p) => p.sequenceId === street.sequenceId, diff --git a/src/layers/streetLevel/limiter.test.mjs b/src/layers/streetLevel/limiter.test.mjs new file mode 100644 index 000000000..c9fabef68 --- /dev/null +++ b/src/layers/streetLevel/limiter.test.mjs @@ -0,0 +1,80 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createLimiter } from './limiter.js'; + +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)); + +/** A task that runs until `finish()`. */ +function held(log, name) { + let finish; + const done = new Promise((resolve) => (finish = resolve)); + return { + task: () => { + log.push(name); + return done.then(() => name); + }, + finish: () => finish(), + }; +} + +test('at most `limit` tasks run; queued ones start first in, first out', async () => { + const log = []; + const gate = createLimiter(2); + const tasks = ['a', 'b', 'c', 'd'].map((name) => held(log, name)); + const results = tasks.map(({ task }) => gate.run(task)); + await tick(); + assert.deepEqual(log, ['a', 'b']); + assert.equal(gate.pending, 4); + tasks[0].finish(); + await tick(); + assert.deepEqual(log, ['a', 'b', 'c']); + for (const { finish } of tasks) finish(); + assert.deepEqual(await Promise.all(results), ['a', 'b', 'c', 'd']); + assert.equal(gate.pending, 0); +}); + +test('lifo starts the newest queued task next', async () => { + const log = []; + const gate = createLimiter(1, { lifo: true }); + const tasks = ['a', 'b', 'c'].map((name) => held(log, name)); + const results = tasks.map(({ task }) => gate.run(task)); + await tick(); + tasks[0].finish(); + await tick(); + assert.deepEqual(log, ['a', 'c']); + tasks[2].finish(); + tasks[1].finish(); + await Promise.all(results); +}); + +test('a queued task whose signal aborts never runs, leaves the queue and rejects with the reason', async () => { + const log = []; + const gate = createLimiter(1); + const first = held(log, 'a'); + const running = gate.run(first.task); + const controller = new AbortController(); + const queued = gate.run(held(log, 'b').task, { signal: controller.signal }); + await tick(); + assert.equal(gate.pending, 2); + controller.abort(new Error('overtaken')); + await assert.rejects(queued, /overtaken/); + assert.equal(gate.pending, 1, 'out of the queue'); + first.finish(); + await running; + await tick(); + assert.deepEqual(log, ['a'], 'b never ran'); +}); + +test('a task already aborted when its turn comes is skipped', async () => { + const log = []; + const gate = createLimiter(1); + const first = held(log, 'a'); + const running = gate.run(first.task); + const signal = { aborted: false, reason: new Error('gone'), addEventListener() {} }; + const queued = gate.run(held(log, 'b').task, { signal }); + signal.aborted = true; + first.finish(); + await running; + await assert.rejects(queued, /gone/); + assert.deepEqual(log, ['a']); +}); diff --git a/src/layers/streetLevel/lineCoverage.js b/src/layers/streetLevel/lineCoverage.js index 1a8d73b6f..2eb634c9c 100644 --- a/src/layers/streetLevel/lineCoverage.js +++ b/src/layers/streetLevel/lineCoverage.js @@ -73,6 +73,7 @@ export function meshBoxInRange(box, centre) { * @property {number} viewRangePerHeight …or camera height times this. * @property {number} viewNearM Ground always covered around a tilted camera. * @property {boolean} [keepOrder] Cap a tile's lines in the order decoded (default: newest first). + * @property {number} [maxPoints] Line points drawn at once, over every tile (default: no limit). */ /** @@ -111,6 +112,9 @@ export function createLineCoverage({ }) { const { render } = state.services; const perTileSequenceCap = Math.floor(tuning.maxSequences / tuning.maxTiles); + const perTilePointCap = tuning.maxPoints + ? Math.floor(tuning.maxPoints / tuning.maxTiles) + : Infinity; /** Colour for a line: the provider's, GEV cyan while selected. */ function sequenceColor({ selected = false } = {}) { @@ -151,13 +155,25 @@ export function createLineCoverage({ function drawnSequences(entry) { const drawn = []; const current = filter(); + let points = 0; for (const sequence of entry.sequenceList) { if (drawn.length >= perTileSequenceCap) break; - if (passes(sequence, current)) drawn.push(sequence); + if (!passes(sequence, current)) continue; + // A line past the point budget is skipped; a shorter one may still fit. + const size = pointsOf(sequence); + if (points + size > perTilePointCap) continue; + points += size; + drawn.push(sequence); } return drawn; } + function pointsOf(sequence) { + let count = 0; + for (const part of sequence.parts) count += part.length; + return count; + } + /** * Batched primitives for a tile's sequence parts: cast polylines where * terrain heights are cached, draped ground lines otherwise. Each batch diff --git a/src/layers/streetLevel/lineCoverage.test.mjs b/src/layers/streetLevel/lineCoverage.test.mjs index 88308673c..db4b5449f 100644 --- a/src/layers/streetLevel/lineCoverage.test.mjs +++ b/src/layers/streetLevel/lineCoverage.test.mjs @@ -25,7 +25,15 @@ const TUNING = Object.freeze({ */ function setup( t, - { zoomFor, messages = { rateLimited: 'busy' }, passes, fail } = {}, + { + zoomFor, + messages = { rateLimited: 'busy' }, + passes, + fail, + decoded = null, + tuning = TUNING, + overviewPointSize = null, + } = {}, ) { const centre = tileCentre(-121.4944, 38.5816); const viewer = coverageViewer( @@ -68,6 +76,7 @@ function setup( fetchTile: async (tile) => { asked.push(tile); if (fail) throw fail; + if (decoded) return decoded(centre); return { sequences: [ { @@ -94,9 +103,10 @@ function setup( colors: { coverage: '#635bff', selected: '#00e5ff' }, pickPrefix: 'test:line:', renderKey: 'test-coverage', - tuning: TUNING, + tuning, messages, ...(passes ? { passes } : {}), + ...(overviewPointSize ? { overviewPointSize } : {}), }); t.after(() => { coverage.clear(); @@ -140,3 +150,88 @@ test('a refused key without its own message keeps the error it came with', async assert.ok(await until(() => state.keyRejected === true)); assert.equal(state.coverage.lastError, 'the source refused the key'); }); + +/** Three lines in one tile: an old long one, then two newer short ones. */ +function threeLines(centre) { + const line = (id, capturedAt, span) => ({ + id, + parts: [ + [ + [centre.lon - span, centre.lat], + [centre.lon + span, centre.lat], + ], + ], + capturedAt, + isPano: false, + }); + return { + sequences: [ + line('old-long', 1, 0.002), + line('new-short', 2, 0.0002), + line('newer-short', 3, 0.0001), + ], + overview: [], + }; +} + +test('the per-tile cap keeps the newest lines, or the provider’s own order with keepOrder', async (t) => { + const capped = { ...TUNING, maxTiles: 1, maxSequences: 2 }; + const drawnIds = (state) => + [...state.coverage.tiles.values()].flatMap((tile) => [ + ...tile.sequences.keys(), + ]); + const newest = setup(t, { decoded: threeLines, tuning: capped }); + newest.coverage.refresh(); + assert.ok(await until(() => newest.state.coverage.tiles.size === 1)); + assert.deepEqual(drawnIds(newest.state).sort(), ['new-short', 'newer-short']); + const kept = setup(t, { + decoded: threeLines, + tuning: { ...capped, keepOrder: true }, + }); + kept.coverage.refresh(); + assert.ok(await until(() => kept.state.coverage.tiles.size === 1)); + assert.deepEqual(drawnIds(kept.state), ['old-long', 'new-short']); +}); + +test('overview points take the provider’s size, each its own', async (t) => { + const { state, coverage, viewer } = setup(t, { + zoomFor: () => ({ zoom: 3, kind: 'overview' }), + tuning: { ...TUNING, overviewMaxTiles: 4 }, + decoded: (centre) => ({ + sequences: [], + overview: [ + { id: 'a', lon: centre.lon, lat: centre.lat, weight: 1 }, + { id: 'b', lon: centre.lon + 0.01, lat: centre.lat, weight: 50_000 }, + ], + }), + overviewPointSize: (point) => (point.weight > 1 ? 7 : 3), + }); + coverage.refresh(); + assert.ok(await until(() => state.coverage.tiles.size > 0)); + const sizes = []; + for (const primitive of viewer.scene.primitives.items) + for (let i = 0; i < (primitive.length ?? 0); i++) + sizes.push(primitive.get(i).pixelSize); + assert.deepEqual(sizes.sort(), [3, 7]); +}); + +test('a point budget skips lines past it, and a shorter one later may still fit', async (t) => { + const { state, coverage } = setup(t, { + tuning: { ...TUNING, maxTiles: 1, maxSequences: 10, maxPoints: 5, keepOrder: true }, + decoded: (centre) => { + const line = (id, n) => ({ + id, + parts: [ + Array.from({ length: n }, (_, i) => [centre.lon - 0.001 + i * 0.0001, centre.lat]), + ], + capturedAt: 0, + isPano: false, + }); + return { sequences: [line('three', 3), line('four', 4), line('two', 2)], overview: [] }; + }, + }); + coverage.refresh(); + assert.ok(await until(() => state.coverage.tiles.size === 1)); + const [tile] = state.coverage.tiles.values(); + assert.deepEqual([...tile.sequences.keys()], ['three', 'two']); +}); diff --git a/src/layers/streetLevel/photoViewer.js b/src/layers/streetLevel/photoViewer.js index bb1e28dde..94da38f47 100644 --- a/src/layers/streetLevel/photoViewer.js +++ b/src/layers/streetLevel/photoViewer.js @@ -7,6 +7,10 @@ import { /** A photo download (tens of kilobytes to a few megabytes). */ export const IMAGE_TIMEOUT_MS = 15_000; +/** A picture file or decoded side past these is refused (a photo is a few MB). */ +export const MAX_IMAGE_BYTES = 48 * 1024 * 1024; +export const MAX_IMAGE_SIDE = 16_384; + /** Panorama texture widths: enough for a docked viewer, and for an expanded one. */ export const PANO_MID_SIZE = Object.freeze([2048, 1024]); export const PANO_FULL_SIZE = Object.freeze([4096, 2048]); @@ -27,6 +31,7 @@ export function loadImage( timeoutMs = IMAGE_TIMEOUT_MS, resize = null, failure = 'The photo could not be loaded', + signal = null, } = {}, ) { if ( @@ -35,7 +40,7 @@ export function loadImage( typeof globalThis.createImageBitmap === 'function' && typeof globalThis.fetch === 'function' ) - return loadBitmap(url, timeoutMs, resize, failure); + return loadBitmap(url, timeoutMs, resize, failure, signal); return new Promise((resolve, reject) => { if (!url) return reject(new Error(failure)); const image = new Image(); @@ -57,27 +62,42 @@ export function loadImage( }); } -async function loadBitmap(url, timeoutMs, resize, failure) { +async function loadBitmap(url, timeoutMs, resize, failure, signal) { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); + // The caller's abort (the photo changed) stops a multi-megabyte download. + const onAbort = () => controller.abort(); + if (signal?.aborted) controller.abort(); + signal?.addEventListener?.('abort', onAbort, { once: true }); try { const response = await fetch(url, { mode: 'cors', signal: controller.signal, }); if (!response.ok) throw new Error(failure); + // Picture hosts are not ours: refuse a file or an image past any real + // photo's size before it is decoded. + const length = Number(response.headers?.get?.('content-length')); + if (length > MAX_IMAGE_BYTES) throw new Error(failure); const blob = await response.blob(); - return await (resize + if (blob.size > MAX_IMAGE_BYTES) throw new Error(failure); + const bitmap = await (resize ? createImageBitmap(blob, { resizeWidth: resize[0], resizeHeight: resize[1], resizeQuality: 'high', }) : createImageBitmap(blob)); + if (bitmap.width > MAX_IMAGE_SIDE || bitmap.height > MAX_IMAGE_SIDE) { + bitmap.close?.(); + throw new Error(failure); + } + return bitmap; } catch { throw new Error(failure); } finally { clearTimeout(timer); + signal?.removeEventListener?.('abort', onAbort); } } @@ -92,8 +112,8 @@ export function panoSizeFor(canvasPixels, fovDeg) { * @property {(photo: object) => boolean} hasPano A 360° photo with an equirectangular image. * @property {(photo: object) => {url: string, width: number, resize?: [number, number]|null}|null} panoFirst * The panorama shown first, its texture width (0 when it is only a preview), and a size to shrink it to. - * @property {(photo: object, size: [number, number]) => {url: string, resize?: [number, number]|null, timeoutMs?: number}|null} panoSharp - * A sharper panorama for a canvas that needs `size`, or null. + * @property {(photo: object, size: [number, number], view: {expanded: boolean}) => {url: string, resize?: [number, number]|null, timeoutMs?: number}|null} panoSharp + * A sharper panorama for a canvas that needs `size`, or null; `expanded` while the viewer is. * @property {(photo: object) => Array} flatFirst Flat versions to try in turn. * @property {(photo: object, pixels: number) => string|null} flatSharp * A sharper flat version for a picture this many device pixels wide, or null. @@ -411,6 +431,24 @@ export function createPhotoViewer({ renderer.setView({ fov: fov * Math.exp(event.deltaY * 0.001) }); } + /** + * Docked, Esc anywhere in the viewer's box (the photo, its toolbar, its + * caption) is the provider's before the panel's, which would collapse; + * expanded, the dialog's own Esc shrinks it. + */ + function onEscapeKey(event) { + if ( + event.key !== 'Escape' || + event.shiftKey || + event.defaultPrevented || + root?.closest?.('.sl-viewer-wrap-expanded') || + !onEscape() + ) + return; + event.preventDefault(); + event.stopPropagation(); + } + function onKeyDown(event) { if ( event.defaultPrevented || @@ -419,18 +457,6 @@ export function createPhotoViewer({ event.metaKey ) return; - // Docked, Esc is the viewer's before the panel's (which would collapse); - // expanded, the dialog's own Esc shrinks it. - if ( - event.key === 'Escape' && - !event.shiftKey && - !root?.closest?.('.sl-viewer-wrap-expanded') && - onEscape() - ) { - event.preventDefault(); - event.stopPropagation(); - return; - } const pano = showing === 'pano' && renderer; const turn = { ArrowLeft: -1, ArrowRight: 1 }[event.key]; // Shift+← → look around a panorama; ← → step along the sequence. @@ -502,6 +528,7 @@ export function createPhotoViewer({ root.append(picture, canvas, prevButton, nextButton); host.append(root); listen(root, 'keydown', onKeyDown); + listen(host.closest?.('.sl-viewer-wrap') ?? root, 'keydown', onEscapeKey); applyRenderMode(); } @@ -536,13 +563,18 @@ export function createPhotoViewer({ const size = panoSizeNow(); if (size[0] <= panoWidth) return; if (panoComing?.seq === seq && size[0] <= panoComing.width) return; - const sharp = sources.panoSharp(target, size); + const sharp = sources.panoSharp(target, size, { + expanded: Boolean( + root?.closest?.('.sl-viewer-wrap-expanded'), + ), + }); if (!sharp?.url) return; - const coming = { seq, width: size[0] }; + const coming = { seq, width: size[0], controller: new AbortController() }; panoComing = coming; try { const image = await loadAs(sharp.url, { crossOrigin: true, + signal: coming.controller.signal, resize: sharp.resize ?? null, ...(sharp.timeoutMs ? { timeoutMs: sharp.timeoutMs } : {}), }); @@ -578,6 +610,12 @@ export function createPhotoViewer({ } } + /** Another photo, or none: a sharper panorama still on its way is not needed. */ + function dropSharpPanorama() { + panoComing?.controller?.abort(); + panoComing = null; + } + /** The sharp panorama this viewer needs now: 2,048 docked, 4,096 expanded. */ function panoSizeNow() { const ratio = Math.min(globalThis.devicePixelRatio || 1, 2); @@ -600,7 +638,10 @@ export function createPhotoViewer({ } // The context may have been lost while this loaded: build on the new // canvas, or let the caller show the front view. - if (!ensureRenderer()) throw new Error(LOAD_FAILED); + if (!ensureRenderer()) { + image.close?.(); + throw new Error(LOAD_FAILED); + } const restore = restoreView?.id === target.id ? restoreView.view : null; restoreView = null; if (restore) renderer.setView(restore); @@ -670,6 +711,7 @@ export function createPhotoViewer({ if (!root) throw new Error(`${name} viewer is not mounted`); const id = String(imageId); const seq = ++openSeq; + dropSharpPanorama(); const keepView = keepViewFor === id; keepViewFor = null; openingId = id; @@ -682,6 +724,7 @@ export function createPhotoViewer({ close() { openSeq++; + dropSharpPanorama(); openingId = null; keepViewFor = null; photo = null; @@ -693,6 +736,7 @@ export function createPhotoViewer({ unmount() { openSeq++; + dropSharpPanorama(); openingId = null; keepViewFor = null; flatSharpUrl = null; diff --git a/src/layers/streetLevel/photoViewer.test.mjs b/src/layers/streetLevel/photoViewer.test.mjs index 704a2f98c..6f29e711e 100644 --- a/src/layers/streetLevel/photoViewer.test.mjs +++ b/src/layers/streetLevel/photoViewer.test.mjs @@ -164,3 +164,144 @@ test('Esc: the docked viewer offers it to the provider first; expanded, it is th assert.equal(asked, 1, 'not asked while expanded'); assert.equal(expanded.prevented, false); }); + +test('a sharp panorama still downloading when another photo opens is aborted', async (t) => { + const signals = []; + const viewer = createPhotoViewer({ + providerId: 'test', + name: 'Test', + sources: SOURCES, + getPhoto: async (id) => PHOTOS[id], + step: async () => null, + onStep() {}, + load: (url, options) => + url.endsWith('-sharp') + ? new Promise((_, reject) => { + signals.push(options.signal); + options.signal?.addEventListener('abort', () => reject(new Error('aborted'))); + }) + : Promise.resolve({ src: url }), + createRenderer: () => ({ + setImage() {}, + setView() {}, + getView: () => ({ yaw: 0, pitch: 0, fov: 80 }), + resize() {}, + clear() {}, + destroy() {}, + }), + }); + t.after(() => delete globalThis.devicePixelRatio); + globalThis.devicePixelRatio = 1; + const host = fakeHost(); + await viewer.mount(host); + parts(host).canvas.clientWidth = 1800; + await viewer.open('a'); + await tick(); + assert.equal(signals.length, 1); + assert.equal(signals[0].aborted, false); + await viewer.open('f'); + assert.equal(signals[0].aborted, true, 'not needed any more'); +}); + +test('a lost WebGL context while the first panorama loads releases its bitmap', async () => { + let closed = false; + let built = 0; + let land = null; + const viewer = createPhotoViewer({ + providerId: 'test', + name: 'Test', + sources: SOURCES, + getPhoto: async (id) => PHOTOS[id], + step: async () => null, + onStep() {}, + load: (url) => + new Promise((resolve) => { + land = () => resolve({ src: url, close: () => (closed = true) }); + }), + // The first renderer is lost mid-load; a new one cannot be built. + createRenderer: () => { + built++; + if (built > 1) throw new Error('context lost'); + return { + setImage() {}, + setView() {}, + getView: () => ({ yaw: 0, pitch: 0, fov: 80 }), + resize() {}, + clear() {}, + destroy() {}, + }; + }, + }); + const host = fakeHost(); + await viewer.mount(host); + const opening = viewer.open('a'); + assert.ok(await until(() => land !== null), 'the panorama is loading'); + parts(host).canvas.fire('webglcontextlost', {}); + const first = land; + first(); + // The panorama failed: the front view is loaded flat instead. + assert.ok(await until(() => land !== first), 'the flat fallback loads'); + land(); + await opening; + assert.equal(built, 2, 'a new renderer was tried'); + assert.equal(closed, true, 'the bitmap no renderer took is closed'); +}); + +test('docked, Esc anywhere in the viewer’s box (toolbar, caption) is offered to the provider', async () => { + let asked = 0; + const viewer = createPhotoViewer({ + providerId: 'test', + name: 'Test', + sources: SOURCES, + getPhoto: async (id) => PHOTOS[id], + step: async () => null, + onStep() {}, + onEscape: () => ++asked > 0, + load: async (url) => ({ src: url }), + }); + const host = fakeHost(); + const wrap = fakeHost(); + host.closest = (selector) => (selector === '.sl-viewer-wrap' ? wrap : null); + await viewer.mount(host); + const flags = { stopped: false, prevented: false }; + // Focus back on EXPAND after a shrink: the key reaches the wrap, not the photo. + wrap.fire('keydown', { + key: 'Escape', + stopPropagation: () => (flags.stopped = true), + preventDefault: () => (flags.prevented = true), + }); + assert.equal(asked, 1); + assert.deepEqual(flags, { stopped: true, prevented: true }); + viewer.unmount(); + wrap.fire('keydown', { key: 'Escape' }); + assert.equal(asked, 1, 'unmounted: the listener is gone'); +}); + +test('a picture file or bitmap past any real photo’s size is refused', async (t) => { + const { loadImage, MAX_IMAGE_BYTES, MAX_IMAGE_SIDE } = await import( + './photoViewer.js' + ); + const saved = { + fetch: globalThis.fetch, + createImageBitmap: globalThis.createImageBitmap, + }; + t.after(() => Object.assign(globalThis, saved)); + let closed = 0; + const bitmap = (width, height) => ({ width, height, close: () => closed++ }); + globalThis.createImageBitmap = async (blob) => blob.bitmap; + const respond = ({ length, size, width = 100, height = 50 }) => + (globalThis.fetch = async () => ({ + ok: true, + headers: new Map([['content-length', String(length ?? '')]]), + blob: async () => ({ size: size ?? 10, bitmap: bitmap(width, height) }), + })); + respond({ length: MAX_IMAGE_BYTES + 1 }); + await assert.rejects(loadImage('https://x/a.jpg'), /could not be loaded/); + respond({ size: MAX_IMAGE_BYTES + 1 }); + await assert.rejects(loadImage('https://x/a.jpg'), /could not be loaded/); + respond({ width: MAX_IMAGE_SIDE + 1 }); + await assert.rejects(loadImage('https://x/a.jpg'), /could not be loaded/); + assert.equal(closed, 1, 'the oversize bitmap is released'); + respond({}); + assert.equal((await loadImage('https://x/a.jpg')).width, 100); +}); diff --git a/src/layers/streetLevel/providers/kartaview/index.js b/src/layers/streetLevel/providers/kartaview/index.js index 1b377479d..59f2f8d7d 100644 --- a/src/layers/streetLevel/providers/kartaview/index.js +++ b/src/layers/streetLevel/providers/kartaview/index.js @@ -1,4 +1,7 @@ import { createLineCoverage } from '../../lineCoverage.js'; +import { createLru, createTileCache } from '../../tileCache.js'; + +export { createTileCache } from '../../tileCache.js'; import { createSequenceCones, thinImages } from '../../sequenceCones.js'; import { metresBetween } from '../../view.js'; import { decodeCoverageTile } from './decode.js'; @@ -31,9 +34,6 @@ import { const PHOTO_CACHE_SIZE = 4000; /** Sequence pages kept in memory. */ const PAGE_CACHE_SIZE = 12; -/** Decoded coverage tiles kept, and for how long. */ -const TILE_CACHE_SIZE = 24; -const TILE_CACHE_TTL_MS = 10 * 60_000; /** z14 near the street, z13 a little higher, nothing above. */ export function kartaViewZoomFor(heightM) { @@ -44,51 +44,6 @@ export function kartaViewZoomFor(heightM) { }; } -/** Least-recently-used map: reading an entry renews it. */ -function createLru(limit) { - const map = new Map(); - return { - get(key) { - if (!map.has(key)) return undefined; - const value = map.get(key); - map.delete(key); - map.set(key, value); - return value; - }, - set(key, value) { - map.delete(key); - map.set(key, value); - while (map.size > limit) map.delete(map.keys().next().value); - }, - delete: (key) => map.delete(key), - clear: () => map.clear(), - }; -} - -/** - * Loaded coverage tiles by `z/x/y`, at most `size`, each for `ttlMs`. KartaView - * sends tiles uncompressed and uncacheable (hundreds of kB, seconds each), so - * a pan back or a zoom across z13/z14 would fetch them all again. - */ -export function createTileCache({ - load, - size = TILE_CACHE_SIZE, - ttlMs = TILE_CACHE_TTL_MS, - now = Date.now, -}) { - const kept = createLru(size); - return { - async get(tile, options) { - const key = `${tile.z}/${tile.x}/${tile.y}`; - const entry = kept.get(key); - if (entry && now() - entry.at < ttlMs) return entry.tile; - const loaded = await load(tile, options); - kept.set(key, { at: now(), tile: loaded }); - return loaded; - }, - }; -} - /** The 1-based page of a sequence that holds `index`, if no photo is missing. */ function pageOf(index) { return Math.floor(Math.max(0, index) / SEQUENCE_PAGE_SIZE) + 1; diff --git a/src/layers/streetLevel/providers/kartaview/provider.test.mjs b/src/layers/streetLevel/providers/kartaview/provider.test.mjs index 9c63e9c18..81749b665 100644 --- a/src/layers/streetLevel/providers/kartaview/provider.test.mjs +++ b/src/layers/streetLevel/providers/kartaview/provider.test.mjs @@ -746,3 +746,24 @@ test('a photo no page lists steps to the photos either side of its index', async await press(env.prev, env.poses); assert.equal(env.poses.at(-1), 's-4'); }); + +test('Esc in the docked viewer clears the selected drive and is consumed; with none, it goes on to the panel', async (t) => { + const env = await mounted({ sequences: { s: [0, 1, 2, 3] } }); + await withCones(t, env); + await env.instance.viewer.open('s-1'); + await env.instance.selectSequence('s'); + assert.ok(await until(() => env.instance.sequenceStats().images === 4)); + const root = env.next.parentNode; + const press = () => { + const flags = { stopped: false, prevented: false }; + root.fire('keydown', { + key: 'Escape', + stopPropagation: () => (flags.stopped = true), + preventDefault: () => (flags.prevented = true), + }); + return flags; + }; + assert.deepEqual(press(), { stopped: true, prevented: true }); + assert.equal(env.instance.sequenceStats().selectedId, null, 'drive cleared'); + assert.deepEqual(press(), { stopped: false, prevented: false }); +}); diff --git a/src/layers/streetLevel/providers/panoramax/decode.js b/src/layers/streetLevel/providers/panoramax/decode.js index 19b470846..d628dec5c 100644 --- a/src/layers/streetLevel/providers/panoramax/decode.js +++ b/src/layers/streetLevel/providers/panoramax/decode.js @@ -1,7 +1,49 @@ import { PbfReader } from 'pbf'; import { VectorTile } from '@mapbox/vector-tile'; import { tileLocalToLonLat } from '../../tileMath.js'; -import { OVERVIEW_BINS_PER_AXIS, panoramaxTimeMs } from './policy.js'; +import { + OVERVIEW_BINS_PER_AXIS, + SIMPLIFY_TOLERANCE_UNITS, + panoramaxTimeMs, +} from './policy.js'; + +/** + * Douglas–Peucker in tile units: the points of `line` ({x, y}) that keep + * it within `tolerance` of the original, first and last always kept. + * Iterative, so a long track cannot overflow the stack. + */ +export function simplifyLine(line, tolerance) { + if (line.length < 3 || !(tolerance > 0)) return line; + const keep = new Uint8Array(line.length); + keep[0] = 1; + keep[line.length - 1] = 1; + const spans = [[0, line.length - 1]]; + while (spans.length) { + const [from, to] = spans.pop(); + const a = line[from]; + const b = line[to]; + const dx = b.x - a.x; + const dy = b.y - a.y; + const span = Math.hypot(dx, dy); + let far = -1; + let farthest = tolerance; + for (let i = from + 1; i < to; i++) { + const p = line[i]; + // Distance to the chord, or to `a` where the chord has no length. + const d = span + ? Math.abs((p.x - a.x) * dy - (p.y - a.y) * dx) / span + : Math.hypot(p.x - a.x, p.y - a.y); + if (d > farthest) { + farthest = d; + far = i; + } + } + if (far === -1) continue; + keep[far] = 1; + spans.push([from, far], [far, to]); + } + return line.filter((_, i) => keep[i]); +} /** Epoch ms of a sequence's day ('2025-08-11'), or 0 when undated. */ function dayMs(value) { @@ -36,6 +78,8 @@ export function decodeCoverageTile(bytes, address) { const props = feature.properties || {}; const parts = []; let length = 0; + const tolerance = + (SIMPLIFY_TOLERANCE_UNITS * sequences.extent) / 4096; for (const line of feature.loadGeometry()) { if (line.length < 2) continue; for (let k = 1; k < line.length; k++) @@ -43,8 +87,10 @@ export function decodeCoverageTile(bytes, address) { line[k].x - line[k - 1].x, line[k].y - line[k - 1].y, ); + // Simplified before anything else: GPS tracks carry a point every + // few metres, and each becomes ~8 ground-line vertices on the GPU. parts.push( - line.map((point) => + simplifyLine(line, tolerance).map((point) => tileLocalToLonLat(point.x, point.y, sequences.extent, x, y, z), ), ); diff --git a/src/layers/streetLevel/providers/panoramax/decode.test.mjs b/src/layers/streetLevel/providers/panoramax/decode.test.mjs index 4347463b0..159cf3884 100644 --- a/src/layers/streetLevel/providers/panoramax/decode.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/decode.test.mjs @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { decodeCoverageTile, decodePictures } from './decode.js'; +import { decodeCoverageTile, decodePictures, simplifyLine } from './decode.js'; import { OVERVIEW_BINS_PER_AXIS } from './policy.js'; import { encodePanoramaxTile } from '../../../../testSupport/panoramaxFixtures.mjs'; import { tileCentre } from '../../../../testSupport/streetLevelFakes.mjs'; @@ -189,3 +189,38 @@ test('a z15 tile’s pictures of one sequence become cones; other sequences’ a }); assert.deepEqual(decodePictures(lines, tile, 's1'), []); }); + +test('lines are simplified within a few tile units, ends and corners kept', () => { + const zigzag = Array.from({ length: 1000 }, (_, i) => ({ x: i, y: i % 2 ? 1 : 0 })); + assert.deepEqual(simplifyLine(zigzag, 4), [zigzag[0], zigzag.at(-1)], 'a 1-unit wobble goes'); + const corner = [ + { x: 0, y: 0 }, + { x: 50, y: 1 }, + { x: 100, y: 0 }, + { x: 100, y: 100 }, + ]; + assert.deepEqual(simplifyLine(corner, 4), [corner[0], corner[2], corner[3]]); + assert.equal(simplifyLine(corner.slice(0, 2), 4).length, 2); + // A loop back to its start (no chord) keeps its far point. + const loop = [ + { x: 0, y: 0 }, + { x: 40, y: 0 }, + { x: 0, y: 0 }, + ]; + assert.equal(simplifyLine(loop, 4).length, 3); + assert.equal(simplifyLine(zigzag, 0).length, 1000, 'no tolerance, no change'); +}); + +test('a decoded GPS track keeps a fraction of its points', () => { + const { tile, lon, lat } = tileCentre(2.35, 48.85); + const track = Array.from({ length: 500 }, (_, i) => [ + lon - 0.004 + i * 0.000016, + lat + Math.sin(i / 40) * 0.0004, + ]); + const [sequence] = decodeCoverageTile( + encodePanoramaxTile(tile, { sequences: [{ id: 'gps', parts: [track] }] }), + tile, + ).sequences; + assert.ok(sequence.parts[0].length < 100, `${sequence.parts[0].length} points`); + assert.ok(sequence.parts[0].length > 5); +}); diff --git a/src/layers/streetLevel/providers/panoramax/index.js b/src/layers/streetLevel/providers/panoramax/index.js index b6eac8297..9eb13cdaf 100644 --- a/src/layers/streetLevel/providers/panoramax/index.js +++ b/src/layers/streetLevel/providers/panoramax/index.js @@ -8,6 +8,7 @@ import { tileBounds, } from '../../tileMath.js'; import { metresBetween } from '../../view.js'; +import { createLru, createTileCache } from '../../tileCache.js'; import { decodeCoverageTile, decodePictures } from './decode.js'; import { createPanoramaxSource } from './source.js'; import { createPanoramaxViewer } from './viewer.js'; @@ -70,6 +71,10 @@ export function overviewPointSize(point) { const PHOTO_CACHE_SIZE = 2000; /** Sequences whose cone centre is remembered. */ const ANCHOR_CACHE_SIZE = 50; +/** Decoded coverage tiles kept (a simplified dense z10 tile is ~4 MB). */ +const COVERAGE_TILE_CACHE_SIZE = 16; +/** One sequence's pictures in a z15 tile, kept for re-centring. */ +const PICTURE_LIST_CACHE_SIZE = 32; /** * Sequence lines at z10–14 below 60 km; above, the density grid at z5 and @@ -116,26 +121,6 @@ export function passesCoverage(record, filter) { }); } -/** Least-recently-used map: reading an entry renews it. */ -function createLru(limit) { - const map = new Map(); - return { - get(key) { - if (!map.has(key)) return undefined; - const value = map.get(key); - map.delete(key); - map.set(key, value); - return value; - }, - set(key, value) { - map.delete(key); - map.set(key, value); - while (map.size > limit) map.delete(map.keys().next().value); - }, - delete: (key) => map.delete(key), - }; -} - /** The z15 tile holding a point. */ function pictureTileOf({ lon, lat }) { const z = PICTURE_TILE_ZOOM; @@ -143,29 +128,29 @@ function pictureTileOf({ lon, lat }) { } /** - * The z15 tiles whose pictures surround a point: its own, and the - * neighbours it sits within PICTURE_TILE_EDGE_M of (at most four). + * The z15 tiles whose pictures surround a point: its own first, and the + * neighbours it sits within PICTURE_TILE_EDGE_M of (at most nine). */ export function pictureTilesAround(point) { const home = pictureTileOf(point); const box = tileBounds(home.x, home.y, home.z); const near = (lon, lat) => metresBetween(point, { lon, lat }) <= PICTURE_TILE_EDGE_M; - const dx = near(box.west, point.lat) ? -1 : near(box.east, point.lat) ? 1 : 0; - const dy = near(point.lon, box.north) - ? -1 - : near(point.lon, box.south) - ? 1 - : 0; + // Near both edges at once where tiles are narrow (high latitudes). + const dxs = [0]; + if (near(box.west, point.lat)) dxs.push(-1); + if (near(box.east, point.lat)) dxs.push(1); + const dys = [0]; + if (near(point.lon, box.north)) dys.push(-1); + if (near(point.lon, box.south)) dys.push(1); const max = 2 ** home.z; - const tiles = [home]; - const add = (x, y) => { - if (y < 0 || y >= max) return; - tiles.push({ x: (x + max) % max, y, z: home.z }); - }; - if (dx) add(home.x + dx, home.y); - if (dy) add(home.x, home.y + dy); - if (dx && dy) add(home.x + dx, home.y + dy); + const tiles = []; + for (const dy of dys) + for (const dx of dxs) { + const y = home.y + dy; + if (y < 0 || y >= max) continue; + tiles.push({ x: (home.x + dx + max) % max, y, z: home.z }); + } return tiles; } @@ -290,12 +275,36 @@ export function createPanoramaxProvider({ return (delta < 0 ? photo.prevId : photo.nextId) || null; } - const coverage = createLineCoverage({ - state, - fetchTile: async (tile, { signal }) => + // Decoded tiles, kept a while: panning back, or across a zoom band and + // back, neither fetches nor decodes them again. + const coverageTiles = createTileCache({ + size: COVERAGE_TILE_CACHE_SIZE, + load: (tile, { signal }) => readable(async () => decodeCoverageTile(await source.getTile(tile, { signal }), tile), ), + }); + /** A z15 tile's pictures of one sequence, by `z/x/y:sequence`. */ + const pictureLists = createLru(PICTURE_LIST_CACHE_SIZE); + function picturesIn(tile, sequenceId, signal) { + const key = `${tileKey(tile)}:${sequenceId}`; + const kept = pictureLists.get(key); + if (kept) return kept; + const list = readable(async () => + decodePictures( + await source.getTile(tile, { signal, lane: 'pictures' }), + tile, + sequenceId, + ), + ); + pictureLists.set(key, list); + list.catch(() => pictureLists.delete(key)); + return list; + } + + const coverage = createLineCoverage({ + state, + fetchTile: (tile, options) => coverageTiles.get(tile, options), zoomFor: panoramaxZoomFor, colors: COLORS, pickPrefix: PICK_PREFIX.sequence, @@ -312,21 +321,15 @@ export function createPanoramaxProvider({ const anchor = anchors.get(sequenceId); if (!anchor) return []; const tiles = pictureTilesAround(anchor); + // Known from the start: a picture opening while these load is + // judged against them (re-centred if outside, left alone inside). + drawn = { sequenceId, tiles: new Set(tiles.map(tileKey)) }; // The home tile must load; a neighbour that fails only thins the cones. const lists = await Promise.allSettled( - tiles.map((tile) => - readable(async () => - decodePictures( - await source.getTile(tile, { signal }), - tile, - sequenceId, - ), - ), - ), + tiles.map((tile) => picturesIn(tile, sequenceId, signal)), ); signal?.throwIfAborted(); if (lists[0].status === 'rejected') throw lists[0].reason; - drawn = { sequenceId, tiles: new Set(tiles.map(tileKey)) }; const byId = new Map( lists .flatMap((list) => list.value ?? []) @@ -445,13 +448,18 @@ export function createPanoramaxProvider({ const count = lines ? drawn : 0; const loading = state.coverage.pending.size > 0; const settled = !loading && !state.coverage.lastError; - // Lines in view that the filter hid all of: say so, not "none here". - const filteredOut = - lines && - [...state.coverage.tiles.values()].some((tile) => tile.total > 0); let fallback = ''; + // Tiles that held something the filter hid all of. + const filteredOut = [...state.coverage.tiles.values()].some( + (tile) => tile.total > 0, + ); if (kind === 'overview') - fallback = drawn > 0 || !settled ? OVERVIEW_HINT : NO_COVERAGE_HINT; + fallback = + drawn > 0 || !settled + ? OVERVIEW_HINT + : filteredOut + ? FILTERED_HINT + : NO_COVERAGE_HINT; else if (lines && settled && count === 0) fallback = filteredOut ? FILTERED_HINT : NO_COVERAGE_HINT; return { diff --git a/src/layers/streetLevel/providers/panoramax/policy.js b/src/layers/streetLevel/providers/panoramax/policy.js index 0a122242f..59fc95b3b 100644 --- a/src/layers/streetLevel/providers/panoramax/policy.js +++ b/src/layers/streetLevel/providers/panoramax/policy.js @@ -49,6 +49,10 @@ export const COVERAGE_TUNING = Object.freeze({ // looking emptier than its suburbs. (KartaView draws as many.) maxSequences: 14_400, keepOrder: true, + // Points drawn at most, over every tile (each becomes ~8 ground-line + // vertices, ~1.4 kB of geometry): the simplified Paris z10 tile alone + // holds 57,000, unsimplified 750,000, which hung the tab. + maxPoints: 90_000, lineWidthPx: 2.5, overviewPointPx: 4, moveDebounceMs: 320, @@ -56,6 +60,12 @@ export const COVERAGE_TUNING = Object.freeze({ viewRangePerHeight: 10, viewNearM: 1_000, }); +/** + * Lines are simplified (Douglas–Peucker) to this many tile units of a + * 4,096 extent: under a pixel at every height lines are drawn from, and + * 6–10% of the points GPS tracks carry. + */ +export const SIMPLIFY_TOLERANCE_UNITS = 4; /** * Grid cells per overview tile, at most: a z0 tile holds ~70,000 cells * (1.9 MB), drawn as this many binned points. @@ -69,9 +79,9 @@ export const NO_COVERAGE_HINT = 'no Panoramax photos in view'; export const FILTERED_HINT = 'no Panoramax photos here match the filters'; export const UNREADABLE_TILE_MESSAGE = 'Panoramax sent a map tile that could not be read'; -export const GONE_MESSAGE = 'This Panoramax picture is no longer available'; +export const GONE_MESSAGE = 'This Panoramax photo is no longer available'; /** From high up the dots are where pictures are, not sequences to click. */ -export const OVERVIEW_HINT = 'zoom in to see Panoramax sequences'; +export const OVERVIEW_HINT = 'zoom in to see Panoramax coverage'; /** Cones come from z15 tiles' `pictures` layer (~110 bytes a picture). */ export const PICTURE_TILE_ZOOM = 15; @@ -84,11 +94,14 @@ export const API_TIMEOUT_MS = 12_000; export const MAX_RESPONSE_BYTES = 16 * 1024 * 1024; /** * The meta-catalogue speaks HTTP/1.1 only, so the browser opens six - * connections to it: tiles take three, searches two, so a click's lookup - * never queues behind megabyte tiles (it timed out doing so). + * connections to it: coverage tiles take three, searches two and the + * cones' tiles one, so a click's lookup never queues behind megabyte tiles + * (it timed out doing so). */ export const API_CONCURRENCY = 2; export const TILE_CONCURRENCY = 3; +/** The selected sequence's z15 tiles (cones) take the sixth. */ +export const PICTURE_TILE_CONCURRENCY = 1; /** Pictures asked for around a point; the API returns them nearest first. */ export const NEAREST_LIMIT = 10; @@ -201,10 +214,18 @@ function creatorOf(item) { .map((p) => p.name) : []), ]; - const name = names.find((value) => typeof value === 'string' && value.trim()); - if (name && !looksLikeEmail(name)) return name.trim().slice(0, 80); - // Licences ask for the source to be named: the instance that holds it. - return instanceOf(item); + // Every producer is credited (an uploading app and its photographer), + // each name once, none that is an email address. + const credited = []; + for (const value of names) { + if (typeof value !== 'string') continue; + const name = value.trim().slice(0, 80); + if (!name || looksLikeEmail(value)) continue; + if (credited.some((other) => other.toLowerCase() === name.toLowerCase())) + continue; + credited.push(name); + } + return credited.length ? credited.join(', ').slice(0, 120) : null; } /** The federated instance holding a picture (its `via` link's host), or null. */ @@ -259,6 +280,9 @@ export function normalizeItem(item) { isPano: isEquirectangular(item), capturedAt: panoramaxTimeMs(props.datetime), creator: creatorOf(item), + // Licences ask for the source to be named: the instance holding it is + // named beside an author, or alone where the author is hidden. + via: instanceOf(item), license: licenseLabel(props.license), width: Number.isFinite(width) && width > 0 ? width : null, prevId: linkedId(item, 'prev'), diff --git a/src/layers/streetLevel/providers/panoramax/policy.test.mjs b/src/layers/streetLevel/providers/panoramax/policy.test.mjs index 57665f025..6d1455bfb 100644 --- a/src/layers/streetLevel/providers/panoramax/policy.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/policy.test.mjs @@ -226,17 +226,34 @@ test('hostile item data: a malformed link escape, null coordinates, a huge produ ); }); -test('a picture whose producer is hidden is credited to the instance holding it', () => { - const item = stacItem({ - id: 'a', - lon: 1, - lat: 2, - producer: 'someone@ign.example.fr', - }); +test('the instance holding a picture is named (its via host), never as its author', () => { + const item = stacItem({ id: 'a', lon: 1, lat: 2, producer: 'someone@ign.example.fr' }); item.links.push({ rel: 'via', href: 'https://panoramax.ign.fr' }); - assert.equal(normalizeItem(item).creator, 'panoramax.ign.fr'); + assert.equal(normalizeItem(item).creator, null); + assert.equal(normalizeItem(item).via, 'panoramax.ign.fr'); item.links.at(-1).href = 'http://insecure.example'; - assert.equal(normalizeItem(item).creator, null); + assert.equal(normalizeItem(item).via, null); item.links.at(-1).href = 'not a url'; - assert.equal(normalizeItem(item).creator, null); + assert.equal(normalizeItem(item).via, null); +}); + +test('every producer is credited once: the uploading app and its photographer', () => { + const item = stacItem({ id: 'a', lon: 1, lat: 2, producer: 'mapcomplete' }); + item.providers = [ + { name: 'mapcomplete', roles: ['producer'] }, + { name: 'pyrog', roles: ['producer'] }, + { name: 'MapComplete', roles: ['producer'] }, + { name: 'someone@example.org', roles: ['producer'] }, + { name: 'host', roles: ['host'] }, + ]; + assert.equal(normalizeItem(item).creator, 'mapcomplete, pyrog'); + const hidden = stacItem({ id: 'b', lon: 1, lat: 2, producer: 'a@b.org' }); + hidden.providers = [{ name: 'c@d.org', roles: ['producer'] }]; + assert.equal(normalizeItem(hidden).creator, null); + const many = stacItem({ id: 'c', lon: 1, lat: 2 }); + many.providers = Array.from({ length: 50 }, (_, i) => ({ + name: `producer-${i}`, + roles: ['producer'], + })); + assert.ok(normalizeItem(many).creator.length <= 120); }); diff --git a/src/layers/streetLevel/providers/panoramax/provider.test.mjs b/src/layers/streetLevel/providers/panoramax/provider.test.mjs index 3ce1ee490..f8c57b159 100644 --- a/src/layers/streetLevel/providers/panoramax/provider.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/provider.test.mjs @@ -535,22 +535,55 @@ test('the cones come in capture order, and a broken neighbour tile only thins th assert.ok(await until(() => cones.count() === 5)); assert.deepEqual(cones.ids(), ['s-4', 's-3', 's-2', 's-1', 's-0']); // A neighbour answers with garbage: the home tile's cones still draw. - const home = new Set(); - env.source.getTile = async (tile) => { - const key = `${tile.z}/${tile.x}/${tile.y}`; - if (!home.size) home.add(key); - if (!home.has(key)) return new TextEncoder().encode('oops'); - return tiles(tile); - }; - await env.instance.viewer.open('s-0'); - await env.instance.selectSequence('s'); +}); + +test('a broken neighbour z15 tile only thins the cones; an unreadable home tile reads as such', async (t) => { + // At a tile corner: the home tile and three neighbours are asked. + const home = tileCentre(ORIGIN.lon, ORIGIN.lat, 15); + const box = tileBounds(home.tile.x, home.tile.y, 15); + const at = { lon: box.west + 0.0003, lat: box.north - 0.0002 }; + const items = [0, 1, 2].map((i) => + stacItem({ id: `c-${i}`, sequenceId: 'c', lon: at.lon + i * 0.00005, lat: at.lat }), + ); + const tiles = pictureTiles(items); + const broken = new TextEncoder().encode('oops'); + const homeKey = `15/${home.tile.x}/${home.tile.y}`; + const env = await mounted({ + items, + tiles: (tile) => + `${tile.z}/${tile.x}/${tile.y}` === homeKey ? tiles(tile) : broken, + }); + const cones = await withCones(t, env); + await env.instance.viewer.open('c-0'); + await env.instance.selectSequence('c'); assert.ok(await until(() => !env.instance.sequenceStats().loading)); - // The home tile itself unreadable: a readable error, not a parser's. - env.source.getTile = async () => - new TextEncoder().encode('oops'); - await env.instance.selectSequence('s'); - assert.ok(await until(() => env.actions.errors.some(Boolean))); - assert.match(env.actions.errors.filter(Boolean).at(-1), /could not be read/); + assert.equal(cones.count(), 3, 'the home tile’s cones draw'); + assert.ok(env.source.calls.tiles.length > 1, 'neighbours were asked'); + // Another sequence whose home tile is unreadable too. + const env2 = await mounted({ items, tiles: () => broken }); + await withCones(t, env2); + await env2.instance.viewer.open('c-0'); + await env2.instance.selectSequence('c'); + assert.ok(await until(() => env2.actions.errors.some(Boolean))); + assert.match(env2.actions.errors.filter(Boolean).at(-1), /could not be read/); +}); + +test('cones re-centred into tiles already read do not read them again', async (t) => { + const items = sequence('long', 400); + const env = await mounted({ items, tiles: pictureTiles(items) }); + await withCones(t, env); + await env.instance.viewer.open('long-0'); + await env.instance.selectSequence('long'); + assert.ok(await until(() => env.instance.sequenceStats().images > 0)); + await env.instance.viewer.open('long-399'); + assert.ok(await until(() => !env.instance.sequenceStats().loading)); + const asked = env.source.calls.tiles.length; + // Back to the start: its tiles are kept. + await env.instance.viewer.open('long-0'); + assert.ok(await until(() => !env.instance.sequenceStats().loading)); + await new Promise((resolve) => setTimeout(resolve, 10)); + assert.equal(env.source.calls.tiles.length, asked); + assert.ok(env.instance.sequenceStats().images > 0); }); test('› within the tiles the cones came from reloads nothing', async (t) => { @@ -748,15 +781,18 @@ test('Esc in the docked viewer clears the selected sequence before the panel can await env.instance.selectSequence('s'); assert.ok(await until(() => env.instance.sequenceStats().images === 3)); const root = env.next.parentNode; - const event = { + const flags = { stopped: false, prevented: false }; + root.fire('keydown', { key: 'Escape', - stopped: false, - stopPropagation() { - this.stopped = true; - }, - }; - root.fire('keydown', event); + stopPropagation: () => (flags.stopped = true), + preventDefault: () => (flags.prevented = true), + }); assert.equal(env.instance.sequenceStats().selectedId, null); + assert.deepEqual( + flags, + { stopped: true, prevented: true }, + 'consumed: the panel never sees it', + ); // Nothing selected: Esc goes on to the panel. const again = { key: 'Escape', defaultPrevented: false }; const result = { prevented: false }; @@ -768,3 +804,132 @@ test('Esc in the docked viewer clears the selected sequence before the panel can }); assert.equal(result.prevented, false); }); + +test('cones near the east and south edges read those neighbours; near both edges of a narrow tile, both sides', () => { + const home = tileCentre(ORIGIN.lon, ORIGIN.lat, 15); + const box = tileBounds(home.tile.x, home.tile.y, 15); + const { x, y } = home.tile; + const keys = (point) => pictureTilesAround(point).map((t) => `${t.x}/${t.y}`); + assert.deepEqual(keys({ lon: box.east - 0.0002, lat: box.south + 0.0002 }), [ + `${x}/${y}`, + `${x + 1}/${y}`, + `${x}/${y + 1}`, + `${x + 1}/${y + 1}`, + ]); + // Svalbard: a z15 tile is ~250 m wide, so its middle is near both sides. + const north = tileCentre(15.6, 78.22, 15); + const around = keys(north); + for (const dx of [-1, 1]) + assert.ok( + around.includes(`${north.tile.x + dx}/${north.tile.y}`), + `${dx} in ${around}`, + ); + assert.equal(around[0], `${north.tile.x}/${north.tile.y}`, 'its own first'); +}); + +test('a search hit outside the radius (a bounding box’s corner) is not the nearest', async () => { + const corner = stacItem({ id: 'corner', sequenceId: 'c', ...east(65) }); + const { instance, source } = setup({ items: [corner] }); + source.nearestItems = async () => [corner]; + assert.equal(await instance.nearestImage(east(0)), null); +}); + +test('a line clicked from low down still searches its own sequence within 50 m', async () => { + const items = [...sequence('other', 1, { offset: 2 }), ...sequence('clicked', 1, { offset: 30 })]; + const { instance, source } = setup({ items }); + instance.init({ + camera: { positionCartographic: { height: 300 } }, + scene: { + primitives: { add() {}, remove() {} }, + preRender: { addEventListener: () => () => {} }, + }, + }); + instance.handlePick(`${PICK_PREFIX.sequence}clicked`, { + groundPoint: () => east(0), + }); + assert.equal(await instance.nearestImage(east(0)), 'clicked-0'); + assert.equal(source.calls.nearest[0].radius, 50); +}); + +test('cones re-centre on a picture that opens while they are still loading', async (t) => { + const items = sequence('long', 400); + const tiles = pictureTiles(items); + let release = null; + const env = await mounted({ items, tiles }); + await withCones(t, env); + await env.instance.viewer.open('long-0'); + env.source.getTile = async (tile) => { + env.source.calls.tiles.push(`${tile.z}/${tile.x}/${tile.y}`); + if (release === null) await new Promise((go) => (release = go)); + return tiles(tile); + }; + const first = env.instance.selectSequence('long'); + assert.ok(await until(() => release !== null), 'the first load is held'); + // The far end opens while the cones around long-0 still load. + await env.instance.viewer.open('long-399'); + release(); + await first; + assert.ok(await until(() => !env.instance.sequenceStats().loading)); + const [lon, lat] = items[399].geometry.coordinates; + const [far] = pictureTilesAround({ lon, lat }); + assert.ok( + env.source.calls.tiles.includes(`15/${far.x}/${far.y}`), + `the far picture's tile was read: ${env.source.calls.tiles}`, + ); +}); + +test('a filter that hides every density dot says so', async (t) => { + const env = coverageSetup(t, { + height: 200_000, + filter: { pano: 'pano', sinceMs: null }, + grid: [{ lon: ORIGIN.lon, lat: ORIGIN.lat, pano: 0, flat: 5 }], + }); + env.instance.activate(env.viewer); + assert.ok(await until(() => env.instance.coverageStats().hint === FILTERED_HINT)); +}); + +test('dots are drawn at their own sizes, and the busiest is the biggest', async (t) => { + const env = coverageSetup(t, { + height: 200_000, + grid: [ + { lon: ORIGIN.lon, lat: ORIGIN.lat, pano: 0, flat: 1 }, + { lon: ORIGIN.lon + 3, lat: ORIGIN.lat, pano: 30_000, flat: 20_000 }, + ], + }); + env.instance.activate(env.viewer); + const sizes = () => { + const out = []; + for (const primitive of env.viewer.scene.primitives.items) + for (let i = 0; i < (primitive.length ?? 0); i++) + if (primitive.get(i).pixelSize) out.push(primitive.get(i).pixelSize); + return out.sort((a, b) => a - b); + }; + assert.ok(await until(() => sizes().length === 2)); + assert.deepEqual(sizes(), [ + overviewPointSize({ weight: 1 }), + overviewPointSize({ weight: 50_000 }), + ]); +}); + +test('lines keep the decoded (longest-first) order under a budget of points', async () => { + const { COVERAGE_TUNING } = await import('./policy.js'); + assert.equal(COVERAGE_TUNING.keepOrder, true); + assert.ok(COVERAGE_TUNING.maxPoints > 0); +}); + +test('a coverage tile decoded a moment ago is not fetched or decoded again', async (t) => { + const env = coverageSetup(t, { sequences: [line('a')] }); + let asked = 0; + const getTile = env.source.getTile; + env.source.getTile = (tile, options) => { + asked++; + return getTile(tile, options); + }; + env.instance.activate(env.viewer); + assert.ok(await until(() => env.instance.coverageStats().count === 1)); + const first = asked; + env.instance.deactivate(); + env.instance.activate(env.viewer); + assert.ok(await until(() => env.instance.coverageStats().count === 1)); + assert.equal(asked, first); +}); diff --git a/src/layers/streetLevel/providers/panoramax/source.js b/src/layers/streetLevel/providers/panoramax/source.js index 58052d83c..1217b19bf 100644 --- a/src/layers/streetLevel/providers/panoramax/source.js +++ b/src/layers/streetLevel/providers/panoramax/source.js @@ -1,4 +1,7 @@ import { createLimiter } from '../../limiter.js'; + +/** Longest wait a Retry-After is honoured for. */ +const RETRY_AFTER_MAX_SEC = 300; import { API_CONCURRENCY, API_TIMEOUT_MS, @@ -8,6 +11,7 @@ import { NEAREST_LIMIT, NEAREST_RADIUS_M, PANORAMAX_API, + PICTURE_TILE_CONCURRENCY, TILE_CONCURRENCY, TILE_TIMEOUT_MS, } from './policy.js'; @@ -23,12 +27,15 @@ export class PanoramaxSourceError extends Error { } const NO_ANSWER = 'Panoramax did not answer in time — try again'; +const UNREACHABLE = 'Panoramax could not be reached — check the connection'; /** Seconds from a Retry-After header, or null. */ function retryAfterSeconds(response) { const header = response.headers?.get?.('retry-after'); const value = header == null || header === '' ? NaN : Number(header); - return Number.isFinite(value) && value >= 0 ? value : null; + if (!Number.isFinite(value) || value < 0) return null; + // A maintenance page's hour must not freeze coverage for an hour. + return Math.min(RETRY_AFTER_MAX_SEC, Math.max(1, value)); } /** The body as bytes, refusing more than MAX_RESPONSE_BYTES whatever the headers say. */ @@ -79,6 +86,11 @@ export function createPanoramaxSource({ const root = api.replace(/\/+$/, ''); const apiLimiter = createLimiter(API_CONCURRENCY, { lifo: true }); const tileLimiter = createLimiter(TILE_CONCURRENCY); + // The cones' z15 tiles have their own connection: a selection never + // waits behind coverage tiles. + const pictureLimiter = createLimiter(PICTURE_TILE_CONCURRENCY, { + lifo: true, + }); /** GET within `timeoutMs`; the caller's abort wins over the timeout. */ async function get(url, { signal, timeoutMs, empty = [] }) { @@ -108,7 +120,8 @@ export function createPanoramaxSource({ if (signal?.aborted) throw signal.reason; if (controller.signal.aborted) throw controller.signal.reason; if (error instanceof PanoramaxSourceError) throw error; - throw new PanoramaxSourceError(NO_ANSWER); + // Offline, or a CORS or TLS failure: not a timeout, not worth a retry. + throw new PanoramaxSourceError(UNREACHABLE); } finally { clearTimeout(timer); signal?.removeEventListener('abort', onAbort); @@ -146,9 +159,10 @@ export function createPanoramaxSource({ api: root, /** Raw vector-tile bytes; an empty array where there is no coverage. */ - async getTile({ x, y, z }, { signal } = {}) { + async getTile({ x, y, z }, { signal, lane = 'coverage' } = {}) { const url = `${root}/map/${z}/${x}/${y}.mvt`; - const bytes = await tileLimiter.run( + const gate = lane === 'pictures' ? pictureLimiter : tileLimiter; + const bytes = await gate.run( () => get(url, { signal, timeoutMs: TILE_TIMEOUT_MS, empty: [204, 404] }), { signal }, @@ -193,27 +207,51 @@ export function createPanoramaxSource({ const dLat = metres / 111_320; const dLon = metres / (111_320 * Math.max(0.01, Math.cos((lat * Math.PI) / 180))); - const box = [lon - dLon, lat - dLat, lon + dLon, lat + dLat].map((v) => - v.toFixed(7), - ); - const [facing, around] = await Promise.all([ - search( - { + // Clamped to the map: the API finds nothing in a box past ±180°. + const box = [ + Math.max(-180, lon - dLon), + Math.max(-90, lat - dLat), + Math.min(180, lon + dLon), + Math.min(90, lat + dLat), + ].map((v) => v.toFixed(7)); + // One search failing (busy, no answer) leaves the other's pictures; + // only both failing fails the lookup. The caller's abort stops both. + const both = new AbortController(); + const onAbort = () => both.abort(signal.reason); + if (signal?.aborted) throw signal.reason; + signal?.addEventListener('abort', onAbort, { once: true }); + const run = (params) => + search(params, { signal: both.signal }).catch((error) => { + if (!both.signal.aborted && !signal?.aborted) failures.push(error); + throw error; + }); + const failures = []; + let settled; + try { + settled = await Promise.allSettled([ + run({ place_position: `${Number(lon).toFixed(7)},${Number(lat).toFixed(7)}`, place_distance: `0-${metres}`, place_fov_tolerance: String(NEAREST_FOV_TOLERANCE_DEG), limit: String(Math.min(100, Math.max(1, limit))), ...shared, - }, - { signal }, - ), - search( - { bbox: box.join(','), limit: String(NEAREST_BOX_LIMIT), ...shared }, - { signal }, - ), - ]); + }), + run({ + bbox: box.join(','), + limit: String(NEAREST_BOX_LIMIT), + ...shared, + }), + ]); + } finally { + signal?.removeEventListener('abort', onAbort); + } + if (signal?.aborted) throw signal.reason; + const found = settled + .filter((result) => result.status === 'fulfilled') + .map((result) => result.value); + if (!found.length) throw failures[0] ?? settled[0].reason; const byId = new Map(); - for (const item of [...facing, ...around]) + for (const item of found.flat()) if (item?.id != null && !byId.has(String(item.id))) byId.set(String(item.id), item); return [...byId.values()]; diff --git a/src/layers/streetLevel/providers/panoramax/source.test.mjs b/src/layers/streetLevel/providers/panoramax/source.test.mjs index e0111a33e..b54134147 100644 --- a/src/layers/streetLevel/providers/panoramax/source.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/source.test.mjs @@ -217,7 +217,7 @@ test('the caller’s abort wins; a silent API times out with a readable message' await assert.rejects(slow, /did not answer in time/); }); -test('a network failure reads as no answer, not as a stack trace', async () => { +test('a network failure reads as unreachable, not as a stack trace', async () => { const source = createPanoramaxSource({ fetchImpl: async () => { throw new TypeError('Failed to fetch'); @@ -225,7 +225,7 @@ test('a network failure reads as no answer, not as a stack trace', async () => { }); await assert.rejects(source.getTile({ x: 0, y: 0, z: 0 }), (error) => { assert.ok(error instanceof PanoramaxSourceError); - assert.match(error.message, /did not answer/); + assert.match(error.message, /could not be reached/); return true; }); }); @@ -293,3 +293,68 @@ test('the HTTP/1.1 API never gets more than five requests at once: three tiles, } assert.equal(most, 5, 'never more'); }); + +test('one of the two nearest searches failing leaves the other’s pictures; both failing fails the lookup', async () => { + const half = createPanoramaxSource({ + fetchImpl: fakeFetch((url) => + url.includes('bbox') + ? { status: 503 } + : json({ features: [{ id: 'facing' }] }), + ).fetchImpl, + }); + assert.deepEqual( + (await half.nearestItems({ lat: 1, lon: 2 })).map((i) => i.id), + ['facing'], + ); + const none = createPanoramaxSource({ + fetchImpl: fakeFetch(() => ({ status: 503 })).fetchImpl, + }); + await assert.rejects(none.nearestItems({ lat: 1, lon: 2 }), /busy/); +}); + +test('only a timeout is asked again: a busy or failing API is asked once, an unreachable one says so', async () => { + for (const status of [429, 500, 503]) { + const { urls, fetchImpl } = fakeFetch(() => ({ status })); + const source = createPanoramaxSource({ fetchImpl }); + await assert.rejects(source.getItem('a')); + assert.equal(urls.length, 1, `HTTP ${status}: one fetch`); + } + let calls = 0; + const offline = createPanoramaxSource({ + fetchImpl: async () => { + calls++; + throw new TypeError('Failed to fetch'); + }, + }); + await assert.rejects(offline.getItem('a'), /could not be reached/); + assert.equal(calls, 1, 'not retried'); +}); + +test('a Retry-After is honoured up to five minutes', async () => { + const wait = async (header) => { + const source = createPanoramaxSource({ + fetchImpl: fakeFetch(() => ({ + status: 503, + headers: { 'retry-after': header }, + })).fetchImpl, + }); + return source.getTile({ x: 0, y: 0, z: 0 }).catch((error) => error.retryAfterSec); + }; + assert.equal(await wait('3600'), 300); + assert.equal(await wait('99999999999'), 300); + assert.equal(await wait('0'), 1); + assert.equal(await wait('12'), 12); +}); + +test('a lookup near the antimeridian keeps its box on the map', async () => { + const { urls, fetchImpl } = fakeFetch(() => json({ features: [] })); + const source = createPanoramaxSource({ fetchImpl }); + await source.nearestItems({ lat: -16, lon: 179.9999 }); + const box = urls + .map((u) => new URL(u).searchParams.get('bbox')) + .find(Boolean) + .split(',') + .map(Number); + assert.equal(box[2], 180); + assert.ok(box[0] < 180); +}); diff --git a/src/layers/streetLevel/providers/panoramax/viewer.js b/src/layers/streetLevel/providers/panoramax/viewer.js index 90ef88c97..3e26e4d56 100644 --- a/src/layers/streetLevel/providers/panoramax/viewer.js +++ b/src/layers/streetLevel/providers/panoramax/viewer.js @@ -36,9 +36,11 @@ export const PANORAMAX_SOURCES = Object.freeze({ const width = Math.min(photo.width, PANO_FULL_SIZE[0]); return { url: photo.urls.hd, width, resize: [width, width / 2] }; }, - panoSharp: (photo, size) => { + // The original is several megabytes with no server to shrink it: only + // for the expanded viewer (a docked one on a dense screen asks too). + panoSharp: (photo, size, { expanded = false } = {}) => { const width = photo.width || 0; - if (!photo.urls.hd || !photo.urls.sd) return null; + if (!expanded || !photo.urls.hd || !photo.urls.sd) return null; // An original no wider than the SD one adds nothing; a huge one would // take hundreds of megabytes to decode. if (width <= SD_WIDTH || width > PANO_HD_MAX_WIDTH) return null; @@ -59,7 +61,8 @@ export const PANORAMAX_SOURCES = Object.freeze({ flatSharp: (photo, pixels) => photo.urls.sd && photo.urls.hd && pixels > SD_WIDTH ? photo.urls.hd : null, describe: (photo) => ({ - title: null, + // Where no author can be named, the instance holding the photo is. + title: !photo.creator && photo.via ? `via ${photo.via}` : null, creator: photo.creator, license: photo.license, externalUrl: panoramaxPictureUrl(photo.id), diff --git a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs index 3dd5e7a1b..c5c1fa19b 100644 --- a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs @@ -150,7 +150,15 @@ test('a 360° picture shows its SD equirectangular image, then HD shrunk to 4,09 assert.equal(poses.at(-1).bearing, 100); await new Promise((resolve) => setTimeout(resolve, 0)); assert.equal(loads.length, 1, 'docked: the 2,048 px SD is enough'); + // Docked on a dense screen the canvas wants more, but the original is + // megabytes: not until the viewer is expanded. canvas.clientWidth = 1800; + viewer.resize(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(loads.length, 1, 'docked: still SD only'); + const root = canvas.parentNode; + root.closest = (selector) => + selector.includes('sl-viewer-wrap-expanded') ? {} : null; // EXPAND resizes more than once while the panel settles. viewer.resize(); viewer.resize(); @@ -165,14 +173,19 @@ test('a 360° picture shows its SD equirectangular image, then HD shrunk to 4,09 assert.equal(hd.url, PHOTOS.pano.urls.hd); assert.deepEqual(hd.resize, [4096, 2048]); assert.equal(hd.timeoutMs, 30_000); + assert.ok(hd.signal, 'abortable'); }); -test('HD panoramas too big to decode, or no bigger than SD, are never fetched', () => { +test('HD panoramas too big to decode, or no bigger than SD, are never fetched; nor while docked', () => { const size = [4096, 2048]; - assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.bigPano, size), null); - assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.smallPano, size), null); + const expanded = { expanded: true }; + assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.pano, size, { expanded: false }), null); + assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.pano, size), null); + assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.bigPano, size, expanded), null); + assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.smallPano, size, expanded), null); assert.deepEqual( - PANORAMAX_SOURCES.panoSharp({ ...PHOTOS.pano, width: 3000 }, size).resize, + PANORAMAX_SOURCES.panoSharp({ ...PHOTOS.pano, width: 3000 }, size, expanded) + .resize, [3000, 1500], 'never enlarged', ); @@ -190,7 +203,7 @@ test('HD panoramas too big to decode, or no bigger than SD, are never fetched', assert.deepEqual(PANORAMAX_SOURCES.flatFirst(hugeOnly).filter(Boolean), [ hugeOnly.urls.thumb, ]); - assert.equal(PANORAMAX_SOURCES.panoSharp(hdOnly, size), null); + assert.equal(PANORAMAX_SOURCES.panoSharp(hdOnly, size, expanded), null); assert.equal(PANORAMAX_SOURCES.flatSharp(PHOTOS.flat, 2048), null); assert.equal( PANORAMAX_SOURCES.flatSharp(PHOTOS.flat, 2049), @@ -258,3 +271,18 @@ test('a picture whose SD fails falls back to HD, then the thumbnail', async () = ]); assert.equal(parts(host).picture.src, PHOTOS.flat.urls.thumb); }); + +test('a photo with no author to name is captioned with the instance holding it', () => { + const photo = normalizeItem( + stacItem({ id: 'x', lon: 1, lat: 2, producer: 'someone@city.example.fr' }), + ); + photo.via = 'panoramax.ign.fr'; + assert.deepEqual(PANORAMAX_SOURCES.describe(photo), { + title: 'via panoramax.ign.fr', + creator: null, + license: 'CC BY-SA 4.0', + externalUrl: 'https://api.panoramax.xyz/#pic=x&focus=pic', + }); + const named = { ...photo, creator: 'Robot8A' }; + assert.equal(PANORAMAX_SOURCES.describe(named).title, null); +}); diff --git a/src/layers/streetLevel/tileCache.js b/src/layers/streetLevel/tileCache.js new file mode 100644 index 000000000..55d997cdb --- /dev/null +++ b/src/layers/streetLevel/tileCache.js @@ -0,0 +1,48 @@ +/** Decoded coverage tiles kept, and for how long. */ +export const TILE_CACHE_SIZE = 24; +export const TILE_CACHE_TTL_MS = 10 * 60_000; + +/** Least-recently-used map: reading an entry renews it. */ +export function createLru(limit) { + const map = new Map(); + return { + get(key) { + if (!map.has(key)) return undefined; + const value = map.get(key); + map.delete(key); + map.set(key, value); + return value; + }, + set(key, value) { + map.delete(key); + map.set(key, value); + while (map.size > limit) map.delete(map.keys().next().value); + }, + delete: (key) => map.delete(key), + clear: () => map.clear(), + }; +} + +/** + * Loaded (decoded) coverage tiles by `z/x/y`, at most `size`, each for + * `ttlMs`: a pan back, or a zoom across bands and back, neither fetches nor + * decodes them again. A load that fails is not kept. + */ +export function createTileCache({ + load, + size = TILE_CACHE_SIZE, + ttlMs = TILE_CACHE_TTL_MS, + now = Date.now, +}) { + const kept = createLru(size); + return { + async get(tile, options) { + const key = `${tile.z}/${tile.x}/${tile.y}`; + const entry = kept.get(key); + if (entry && now() - entry.at < ttlMs) return entry.tile; + const loaded = await load(tile, options); + kept.set(key, { at: now(), tile: loaded }); + return loaded; + }, + }; +} diff --git a/src/layers/streetLevel/viewerHost.test.mjs b/src/layers/streetLevel/viewerHost.test.mjs index 070fe3d91..259934b73 100644 --- a/src/layers/streetLevel/viewerHost.test.mjs +++ b/src/layers/streetLevel/viewerHost.test.mjs @@ -673,3 +673,15 @@ test('an adapter that can frame but says this image has no frame offers no FIT / await host.open('mapillary', 'flat-1'); assert.equal(state.street.renderModes, true, 'a flat image can be framed'); }); + +test('a pose’s licence reaches the caption state, and a closed viewer forgets it', async () => { + const { state, host, google } = twoProviders(); + await host.open('google', 'g-1'); + google.emitPose('g-1', { providerId: 'google', license: 'CC BY-SA 4.0' }); + assert.equal(state.street.license, 'CC BY-SA 4.0'); + google.emitPose('g-1', { providerId: 'google' }); + assert.equal(state.street.license, null, 'none given: none shown'); + google.emitPose('g-1', { providerId: 'google', license: 'CC0' }); + host.close(); + assert.equal(state.street.license, null); +}); diff --git a/src/tooling/streetLevelQa.test.mjs b/src/tooling/streetLevelQa.test.mjs index c4eb22dd3..bdd6cf9ac 100644 --- a/src/tooling/streetLevelQa.test.mjs +++ b/src/tooling/streetLevelQa.test.mjs @@ -810,6 +810,18 @@ test('Panoramax fixtures: two sequences on one street, searches honour sequence true, 'the second sequence is older', ); + // A box has every picture in it, those looking away included; a place + // search leaves out pictures the point is behind (all look east). + const d = 0.0002; + const boxed = ask( + `bbox=${at.lon - d},${at.lat - d},${at.lon + d},${at.lat + d}&limit=100`, + ); + assert.ok(boxed.some((p) => p.lon < at.lon) && boxed.some((p) => p.lon > at.lon)); + const placed = ask( + `place_position=${at.lon},${at.lat}&place_distance=0-30&limit=50`, + ); + assert.ok(placed.length > 0); + assert.ok(placed.every((p) => p.lon <= at.lon + 1e-7), 'none it is behind'); const one = ask(`ids=${at.id}&limit=1`); assert.equal(one[0].id, at.id); // Its pictures, and only those, are served; HD downloads are counted. diff --git a/src/ui/streetLevelPresentation.test.mjs b/src/ui/streetLevelPresentation.test.mjs index 5425afd75..5fba550f7 100644 --- a/src/ui/streetLevelPresentation.test.mjs +++ b/src/ui/streetLevelPresentation.test.mjs @@ -539,3 +539,22 @@ test('KEY REQUIRED points at Street View when it would draw with the Google key' assert.equal(plain.status.text, 'KEY REQUIRED'); assert.notEqual(plain.meta, guidance); }); + +test('the caption names an image’s own licence after its author, and leaves it out when there is none', () => { + const caption = (street) => + presentStreetLevelPanel({ + enabled: true, + providers: [], + coverage: {}, + legend: [], + filter: {}, + sequence: {}, + street: { open: true, imageId: '1', ...street }, + }).viewer.captionLeft; + assert.equal( + caption({ creator: 'Fixture Mapper', license: 'CC BY-SA 4.0' }), + 'Image by Fixture Mapper · CC BY-SA 4.0', + ); + assert.equal(caption({ license: 'Licence Ouverte 2.0' }), 'Licence Ouverte 2.0'); + assert.equal(caption({ creator: 'Someone' }), 'Image by Someone'); +}); diff --git a/src/ui/styles/street-level.css b/src/ui/styles/street-level.css index ea9efee1c..433beebbc 100644 --- a/src/ui/styles/street-level.css +++ b/src/ui/styles/street-level.css @@ -651,7 +651,7 @@ button.sl-status:focus-visible { pointer-events: none; } -/* White on #ff6f00 is 2.9:1; the darker orange behind ‹ › is 4.6:1. */ +/* White on #ff6f00 is 2.8:1; on the darker orange behind ‹ › it is 5.1:1. */ .sl-photo-viewer[data-provider='panoramax'] { --sl-photo-ring: var(--sl-panoramax); --sl-photo-fill: var(--sl-panoramax-fill); @@ -720,7 +720,8 @@ button.sl-status:focus-visible { } /* Nothing to step through until a photo shows. */ -.sl-photo-viewer:not([data-kind='flat']):not([data-kind='pano']) .sl-photo-step { +.sl-photo-viewer:not([data-kind='flat']):not([data-kind='pano']) + .sl-photo-step { display: none; } From 1a546bec72cdb6f3d6aa50e2c26121a71aec9a1f Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Wed, 14 Oct 2026 15:25:58 +0100 Subject: [PATCH 50/95] fix(street-level): one keydown listener on the photo root; format scope listed a test twice Esc moved to the viewer's box (`.sl-viewer-wrap`) in the last commit, but without one it was a second keydown listener on the root; Esc is now taken by the root's own handler there. format-scope.json listed panorama.test.mjs twice, which made the formatter refuse the scope; the files it skipped are formatted. Co-Authored-By: Claude Opus 5.5 --- scripts/format-scope.json | 1 - scripts/qa-street-level.mjs | 5 ++++- src/layers/streetLevel/limiter.test.mjs | 6 +++++- src/layers/streetLevel/lineCoverage.test.mjs | 18 +++++++++++++++--- src/layers/streetLevel/photoViewer.js | 12 ++++++++---- src/layers/streetLevel/photoViewer.test.mjs | 9 +++++---- .../streetLevel/providers/panoramax/decode.js | 3 +-- .../providers/panoramax/decode.test.mjs | 16 +++++++++++++--- .../providers/panoramax/policy.test.mjs | 7 ++++++- .../providers/panoramax/provider.test.mjs | 16 +++++++++++++--- .../providers/panoramax/source.test.mjs | 4 +++- .../providers/panoramax/viewer.test.mjs | 15 ++++++++++++--- src/tooling/streetLevelQa.test.mjs | 9 +++++++-- src/ui/streetLevelPresentation.test.mjs | 5 ++++- 14 files changed, 96 insertions(+), 30 deletions(-) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 53d6a5b67..8cb8b89a8 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -937,7 +937,6 @@ "src/testSupport/panoramaxFixtures.mjs", "src/layers/streetLevel/photoViewer.test.mjs", "src/layers/streetLevel/limiter.test.mjs", - "src/layers/streetLevel/panorama.test.mjs", "src/layers/streetLevel/lineCoverage.test.mjs", "src/testSupport/kartaViewFakes.mjs", "src/testSupport/googleStreetViewFakes.mjs", diff --git a/scripts/qa-street-level.mjs b/scripts/qa-street-level.mjs index 9c8c55072..7c131a220 100644 --- a/scripts/qa-street-level.mjs +++ b/scripts/qa-street-level.mjs @@ -1276,7 +1276,10 @@ async function main() { }, ) .catch(() => null); - assert.ok(aimed, 'the line is pickable over the other’s pictures'); + assert.ok( + aimed, + 'the line is pickable over the other’s pictures', + ); at = await aimed.jsonValue(); } const parked = await camera(); diff --git a/src/layers/streetLevel/limiter.test.mjs b/src/layers/streetLevel/limiter.test.mjs index c9fabef68..1137fe2f6 100644 --- a/src/layers/streetLevel/limiter.test.mjs +++ b/src/layers/streetLevel/limiter.test.mjs @@ -70,7 +70,11 @@ test('a task already aborted when its turn comes is skipped', async () => { const gate = createLimiter(1); const first = held(log, 'a'); const running = gate.run(first.task); - const signal = { aborted: false, reason: new Error('gone'), addEventListener() {} }; + const signal = { + aborted: false, + reason: new Error('gone'), + addEventListener() {}, + }; const queued = gate.run(held(log, 'b').task, { signal }); signal.aborted = true; first.finish(); diff --git a/src/layers/streetLevel/lineCoverage.test.mjs b/src/layers/streetLevel/lineCoverage.test.mjs index db4b5449f..51ac366c3 100644 --- a/src/layers/streetLevel/lineCoverage.test.mjs +++ b/src/layers/streetLevel/lineCoverage.test.mjs @@ -217,17 +217,29 @@ test('overview points take the provider’s size, each its own', async (t) => { test('a point budget skips lines past it, and a shorter one later may still fit', async (t) => { const { state, coverage } = setup(t, { - tuning: { ...TUNING, maxTiles: 1, maxSequences: 10, maxPoints: 5, keepOrder: true }, + tuning: { + ...TUNING, + maxTiles: 1, + maxSequences: 10, + maxPoints: 5, + keepOrder: true, + }, decoded: (centre) => { const line = (id, n) => ({ id, parts: [ - Array.from({ length: n }, (_, i) => [centre.lon - 0.001 + i * 0.0001, centre.lat]), + Array.from({ length: n }, (_, i) => [ + centre.lon - 0.001 + i * 0.0001, + centre.lat, + ]), ], capturedAt: 0, isPano: false, }); - return { sequences: [line('three', 3), line('four', 4), line('two', 2)], overview: [] }; + return { + sequences: [line('three', 3), line('four', 4), line('two', 2)], + overview: [], + }; }, }); coverage.refresh(); diff --git a/src/layers/streetLevel/photoViewer.js b/src/layers/streetLevel/photoViewer.js index 94da38f47..b4fb67398 100644 --- a/src/layers/streetLevel/photoViewer.js +++ b/src/layers/streetLevel/photoViewer.js @@ -178,6 +178,8 @@ export function createPhotoViewer({ let documentOf = null; /** A context loss's look direction, for the reopen of that photo. */ let restoreView = null; + /** The panel's viewer box, where Esc is listened for (null: the root). */ + let escapeTarget = null; /** Width of the panorama texture on screen (0: the preview). */ let panoWidth = 0; /** The sharper panorama on its way, for which open (resizes come in bursts). */ @@ -450,6 +452,7 @@ export function createPhotoViewer({ } function onKeyDown(event) { + if (!escapeTarget) onEscapeKey(event); if ( event.defaultPrevented || event.altKey || @@ -528,7 +531,10 @@ export function createPhotoViewer({ root.append(picture, canvas, prevButton, nextButton); host.append(root); listen(root, 'keydown', onKeyDown); - listen(host.closest?.('.sl-viewer-wrap') ?? root, 'keydown', onEscapeKey); + // The viewer's box in the panel; without one, the root's own handler + // takes Esc (one listener per element and type). + escapeTarget = host.closest?.('.sl-viewer-wrap') ?? null; + if (escapeTarget) listen(escapeTarget, 'keydown', onEscapeKey); applyRenderMode(); } @@ -564,9 +570,7 @@ export function createPhotoViewer({ if (size[0] <= panoWidth) return; if (panoComing?.seq === seq && size[0] <= panoComing.width) return; const sharp = sources.panoSharp(target, size, { - expanded: Boolean( - root?.closest?.('.sl-viewer-wrap-expanded'), - ), + expanded: Boolean(root?.closest?.('.sl-viewer-wrap-expanded')), }); if (!sharp?.url) return; const coming = { seq, width: size[0], controller: new AbortController() }; diff --git a/src/layers/streetLevel/photoViewer.test.mjs b/src/layers/streetLevel/photoViewer.test.mjs index 6f29e711e..78397a632 100644 --- a/src/layers/streetLevel/photoViewer.test.mjs +++ b/src/layers/streetLevel/photoViewer.test.mjs @@ -178,7 +178,9 @@ test('a sharp panorama still downloading when another photo opens is aborted', a url.endsWith('-sharp') ? new Promise((_, reject) => { signals.push(options.signal); - options.signal?.addEventListener('abort', () => reject(new Error('aborted'))); + options.signal?.addEventListener('abort', () => + reject(new Error('aborted')), + ); }) : Promise.resolve({ src: url }), createRenderer: () => ({ @@ -278,9 +280,8 @@ test('docked, Esc anywhere in the viewer’s box (toolbar, caption) is offered t }); test('a picture file or bitmap past any real photo’s size is refused', async (t) => { - const { loadImage, MAX_IMAGE_BYTES, MAX_IMAGE_SIDE } = await import( - './photoViewer.js' - ); + const { loadImage, MAX_IMAGE_BYTES, MAX_IMAGE_SIDE } = + await import('./photoViewer.js'); const saved = { fetch: globalThis.fetch, createImageBitmap: globalThis.createImageBitmap, diff --git a/src/layers/streetLevel/providers/panoramax/decode.js b/src/layers/streetLevel/providers/panoramax/decode.js index d628dec5c..c75a01768 100644 --- a/src/layers/streetLevel/providers/panoramax/decode.js +++ b/src/layers/streetLevel/providers/panoramax/decode.js @@ -78,8 +78,7 @@ export function decodeCoverageTile(bytes, address) { const props = feature.properties || {}; const parts = []; let length = 0; - const tolerance = - (SIMPLIFY_TOLERANCE_UNITS * sequences.extent) / 4096; + const tolerance = (SIMPLIFY_TOLERANCE_UNITS * sequences.extent) / 4096; for (const line of feature.loadGeometry()) { if (line.length < 2) continue; for (let k = 1; k < line.length; k++) diff --git a/src/layers/streetLevel/providers/panoramax/decode.test.mjs b/src/layers/streetLevel/providers/panoramax/decode.test.mjs index 159cf3884..104471a1e 100644 --- a/src/layers/streetLevel/providers/panoramax/decode.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/decode.test.mjs @@ -191,8 +191,15 @@ test('a z15 tile’s pictures of one sequence become cones; other sequences’ a }); test('lines are simplified within a few tile units, ends and corners kept', () => { - const zigzag = Array.from({ length: 1000 }, (_, i) => ({ x: i, y: i % 2 ? 1 : 0 })); - assert.deepEqual(simplifyLine(zigzag, 4), [zigzag[0], zigzag.at(-1)], 'a 1-unit wobble goes'); + const zigzag = Array.from({ length: 1000 }, (_, i) => ({ + x: i, + y: i % 2 ? 1 : 0, + })); + assert.deepEqual( + simplifyLine(zigzag, 4), + [zigzag[0], zigzag.at(-1)], + 'a 1-unit wobble goes', + ); const corner = [ { x: 0, y: 0 }, { x: 50, y: 1 }, @@ -221,6 +228,9 @@ test('a decoded GPS track keeps a fraction of its points', () => { encodePanoramaxTile(tile, { sequences: [{ id: 'gps', parts: [track] }] }), tile, ).sequences; - assert.ok(sequence.parts[0].length < 100, `${sequence.parts[0].length} points`); + assert.ok( + sequence.parts[0].length < 100, + `${sequence.parts[0].length} points`, + ); assert.ok(sequence.parts[0].length > 5); }); diff --git a/src/layers/streetLevel/providers/panoramax/policy.test.mjs b/src/layers/streetLevel/providers/panoramax/policy.test.mjs index 6d1455bfb..770f52a26 100644 --- a/src/layers/streetLevel/providers/panoramax/policy.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/policy.test.mjs @@ -227,7 +227,12 @@ test('hostile item data: a malformed link escape, null coordinates, a huge produ }); test('the instance holding a picture is named (its via host), never as its author', () => { - const item = stacItem({ id: 'a', lon: 1, lat: 2, producer: 'someone@ign.example.fr' }); + const item = stacItem({ + id: 'a', + lon: 1, + lat: 2, + producer: 'someone@ign.example.fr', + }); item.links.push({ rel: 'via', href: 'https://panoramax.ign.fr' }); assert.equal(normalizeItem(item).creator, null); assert.equal(normalizeItem(item).via, 'panoramax.ign.fr'); diff --git a/src/layers/streetLevel/providers/panoramax/provider.test.mjs b/src/layers/streetLevel/providers/panoramax/provider.test.mjs index f8c57b159..8cf3911d9 100644 --- a/src/layers/streetLevel/providers/panoramax/provider.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/provider.test.mjs @@ -543,7 +543,12 @@ test('a broken neighbour z15 tile only thins the cones; an unreadable home tile const box = tileBounds(home.tile.x, home.tile.y, 15); const at = { lon: box.west + 0.0003, lat: box.north - 0.0002 }; const items = [0, 1, 2].map((i) => - stacItem({ id: `c-${i}`, sequenceId: 'c', lon: at.lon + i * 0.00005, lat: at.lat }), + stacItem({ + id: `c-${i}`, + sequenceId: 'c', + lon: at.lon + i * 0.00005, + lat: at.lat, + }), ); const tiles = pictureTiles(items); const broken = new TextEncoder().encode('oops'); @@ -835,7 +840,10 @@ test('a search hit outside the radius (a bounding box’s corner) is not the nea }); test('a line clicked from low down still searches its own sequence within 50 m', async () => { - const items = [...sequence('other', 1, { offset: 2 }), ...sequence('clicked', 1, { offset: 30 })]; + const items = [ + ...sequence('other', 1, { offset: 2 }), + ...sequence('clicked', 1, { offset: 30 }), + ]; const { instance, source } = setup({ items }); instance.init({ camera: { positionCartographic: { height: 300 } }, @@ -885,7 +893,9 @@ test('a filter that hides every density dot says so', async (t) => { grid: [{ lon: ORIGIN.lon, lat: ORIGIN.lat, pano: 0, flat: 5 }], }); env.instance.activate(env.viewer); - assert.ok(await until(() => env.instance.coverageStats().hint === FILTERED_HINT)); + assert.ok( + await until(() => env.instance.coverageStats().hint === FILTERED_HINT), + ); }); test('dots are drawn at their own sizes, and the busiest is the biggest', async (t) => { diff --git a/src/layers/streetLevel/providers/panoramax/source.test.mjs b/src/layers/streetLevel/providers/panoramax/source.test.mjs index b54134147..63eb6ff9e 100644 --- a/src/layers/streetLevel/providers/panoramax/source.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/source.test.mjs @@ -338,7 +338,9 @@ test('a Retry-After is honoured up to five minutes', async () => { headers: { 'retry-after': header }, })).fetchImpl, }); - return source.getTile({ x: 0, y: 0, z: 0 }).catch((error) => error.retryAfterSec); + return source + .getTile({ x: 0, y: 0, z: 0 }) + .catch((error) => error.retryAfterSec); }; assert.equal(await wait('3600'), 300); assert.equal(await wait('99999999999'), 300); diff --git a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs index c5c1fa19b..78c39f316 100644 --- a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs @@ -179,10 +179,19 @@ test('a 360° picture shows its SD equirectangular image, then HD shrunk to 4,09 test('HD panoramas too big to decode, or no bigger than SD, are never fetched; nor while docked', () => { const size = [4096, 2048]; const expanded = { expanded: true }; - assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.pano, size, { expanded: false }), null); + assert.equal( + PANORAMAX_SOURCES.panoSharp(PHOTOS.pano, size, { expanded: false }), + null, + ); assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.pano, size), null); - assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.bigPano, size, expanded), null); - assert.equal(PANORAMAX_SOURCES.panoSharp(PHOTOS.smallPano, size, expanded), null); + assert.equal( + PANORAMAX_SOURCES.panoSharp(PHOTOS.bigPano, size, expanded), + null, + ); + assert.equal( + PANORAMAX_SOURCES.panoSharp(PHOTOS.smallPano, size, expanded), + null, + ); assert.deepEqual( PANORAMAX_SOURCES.panoSharp({ ...PHOTOS.pano, width: 3000 }, size, expanded) .resize, diff --git a/src/tooling/streetLevelQa.test.mjs b/src/tooling/streetLevelQa.test.mjs index bdd6cf9ac..564cecadc 100644 --- a/src/tooling/streetLevelQa.test.mjs +++ b/src/tooling/streetLevelQa.test.mjs @@ -816,12 +816,17 @@ test('Panoramax fixtures: two sequences on one street, searches honour sequence const boxed = ask( `bbox=${at.lon - d},${at.lat - d},${at.lon + d},${at.lat + d}&limit=100`, ); - assert.ok(boxed.some((p) => p.lon < at.lon) && boxed.some((p) => p.lon > at.lon)); + assert.ok( + boxed.some((p) => p.lon < at.lon) && boxed.some((p) => p.lon > at.lon), + ); const placed = ask( `place_position=${at.lon},${at.lat}&place_distance=0-30&limit=50`, ); assert.ok(placed.length > 0); - assert.ok(placed.every((p) => p.lon <= at.lon + 1e-7), 'none it is behind'); + assert.ok( + placed.every((p) => p.lon <= at.lon + 1e-7), + 'none it is behind', + ); const one = ask(`ids=${at.id}&limit=1`); assert.equal(one[0].id, at.id); // Its pictures, and only those, are served; HD downloads are counted. diff --git a/src/ui/streetLevelPresentation.test.mjs b/src/ui/streetLevelPresentation.test.mjs index 5fba550f7..7dbf18179 100644 --- a/src/ui/streetLevelPresentation.test.mjs +++ b/src/ui/streetLevelPresentation.test.mjs @@ -555,6 +555,9 @@ test('the caption names an image’s own licence after its author, and leaves it caption({ creator: 'Fixture Mapper', license: 'CC BY-SA 4.0' }), 'Image by Fixture Mapper · CC BY-SA 4.0', ); - assert.equal(caption({ license: 'Licence Ouverte 2.0' }), 'Licence Ouverte 2.0'); + assert.equal( + caption({ license: 'Licence Ouverte 2.0' }), + 'Licence Ouverte 2.0', + ); assert.equal(caption({ creator: 'Someone' }), 'Image by Someone'); }); From 3ae367d0dd8dba30cb71935ab33404d205dda5a6 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Wed, 14 Oct 2026 15:38:53 +0100 Subject: [PATCH 51/95] =?UTF-8?q?fix(street-level):=20Panoramax=20IC8=20ro?= =?UTF-8?q?und=202=20=E2=80=94=20shared=20point=20budget,=20selected=20lin?= =?UTF-8?q?e,=20cone=20cache,=20DP=20segments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the round-2 verification (approve with follow-ups), all follow-ups: - The point budget is shared across the tiles in view once loading settles (water-filling): a sparse tile keeps what it has, a dense city centre takes the rest instead of drawing thinner than its suburbs. The selected sequence is always drawn, and selecting one a cap had left out draws it. - The cones' picture-list cache keeps only finished lists: a re-centre while the cones loaded inherited the aborted load's tiles and drew none in them. - Douglas–Peucker measures to the segment, so dead ends and U-turns keep their far end. - Switching Panoramax off lets its decoded tiles and lists go. - Picture downloads are counted while they stream (a host sending no length cannot fill memory before the cap). - Comments on the via caption and the simplification tolerance match the code. Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 11 ++- src/layers/streetLevel/lineCoverage.js | 99 ++++++++++++++++--- src/layers/streetLevel/lineCoverage.test.mjs | 93 ++++++++++++++++- src/layers/streetLevel/photoViewer.js | 32 +++++- src/layers/streetLevel/photoViewer.test.mjs | 14 +++ .../streetLevel/providers/panoramax/decode.js | 18 ++-- .../providers/panoramax/decode.test.mjs | 7 ++ .../streetLevel/providers/panoramax/index.js | 12 ++- .../streetLevel/providers/panoramax/policy.js | 8 +- .../providers/panoramax/provider.test.mjs | 80 ++++++++++++++- src/layers/streetLevel/tileCache.js | 2 + 11 files changed, 343 insertions(+), 33 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index a9fc1f5fb..15a80c743 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3256,7 +3256,16 @@ IGN, MapComplete…; IGN's redirect to its OVH object storage). Its chip vector tiles (`/api/map/{z}/{x}/{y}.mvt`, an hour's HTTP cache): sequence lines from z14 below 1.8 km above ground, z13 to 5 km, z12 to 12 km, z11 to 30 km and z10 to 60 km, as Mapillary's (a dense z10 tile over Paris is -~2.1 MB and 8,600 sequences; each tile draws its 1,600 longest), and above +~2.1 MB and 8,600 sequences, 750,000 points). `decode.js` simplifies each +line (Douglas–Peucker to the segment, 4 units of a 4,096 tile: about a +pixel where it is drawn, 6–10% of a GPS track's points), and the tiles in +view draw their longest lines within 90,000 points (`maxPoints`, +`keepOrder`), shared out once loading settles so a sparse tile keeps all +it has and a dense one (a city centre) takes the rest; the selected +sequence is always drawn. Every point becomes ~8 ground-line vertices: +unsimplified, the lines hung the tab. Decoded tiles are kept while the +provider is on (16, ten minutes, `tileCache.js`, shared with KartaView), +and above 60 km the `grid` layer's 0.1° density cells, binned to at most 96×96 points a tile (a z0 tile holds ~70,000 cells) and not counted as sequences: the panel says to zoom in. Both diff --git a/src/layers/streetLevel/lineCoverage.js b/src/layers/streetLevel/lineCoverage.js index 2eb634c9c..84204ae83 100644 --- a/src/layers/streetLevel/lineCoverage.js +++ b/src/layers/streetLevel/lineCoverage.js @@ -155,19 +155,85 @@ export function createLineCoverage({ function drawnSequences(entry) { const drawn = []; const current = filter(); + const selectedId = state.sequence.selectedId ?? null; + const cap = entry.pointCap ?? perTilePointCap; let points = 0; + let demand = 0; for (const sequence of entry.sequenceList) { - if (drawn.length >= perTileSequenceCap) break; if (!passes(sequence, current)) continue; - // A line past the point budget is skipped; a shorter one may still fit. const size = pointsOf(sequence); - if (points + size > perTilePointCap) continue; + demand += size; + // The selected sequence is always drawn: its highlight is the answer + // to a click, whatever the caps leave out. + if (sequence.id === selectedId) { + points += size; + drawn.push(sequence); + continue; + } + if (drawn.length >= perTileSequenceCap) continue; + // A line past the point budget is skipped; a shorter one may still fit. + if (points + size > cap) continue; points += size; drawn.push(sequence); } + // What every passing line would take: the budget is shared by it. + entry.pointDemand = demand; + entry.pointsDrawn = points; return drawn; } + /** + * Share the point budget across the tiles in view (water-filling): a + * sparse tile keeps what it needs and the rest goes to the dense ones, + * so a city centre is not drawn thinner than its suburbs. Sets each + * tile's cap; returns the tiles whose cap changes what they draw. + */ + function shareCaps() { + const changed = []; + if (!tuning.maxPoints) return changed; + const entries = [...state.coverage.tiles.values()].filter( + (entry) => entry.kind === 'sequence', + ); + const byDemand = [...entries].sort( + (a, b) => (a.pointDemand || 0) - (b.pointDemand || 0), + ); + let left = tuning.maxPoints; + byDemand.forEach((entry, i) => { + const share = left / (byDemand.length - i); + const cap = Math.floor(Math.min(entry.pointDemand || 0, share)); + left -= cap; + const before = entry.pointCap ?? perTilePointCap; + entry.pointCap = cap; + // Only a change that alters what is drawn: more room for a tile that + // had lines left out, or less than it draws now. + if ( + (cap > before && entry.pointsDrawn < entry.pointDemand) || + cap < entry.pointsDrawn + ) + changed.push(entry); + }); + return changed; + } + + /** Once loading settles: redraw the tiles the shared budget changes. */ + function shareBudget() { + const changed = shareCaps(); + for (const entry of changed) redrawEntry(entry); + if (changed.length) { + requestRender(); + notify(); + } + } + + /** Draw one tile again from its decoded lines (caps or selection changed). */ + function redrawEntry(entry) { + cancelCast(entry); + entry.castRequested = false; + entry.meshParts = null; + detachPrimitive(entry); + attachPrimitive(entry); + } + function pointsOf(sequence) { let count = 0; for (const part of sequence.parts) count += part.length; @@ -616,7 +682,10 @@ export function createLineCoverage({ // must not drop a newer request's entry (which is what counts as loading). if (state.coverage.pending.get(key) === controller) { state.coverage.pending.delete(key); - if (!state.coverage.pending.size) purgeStale(); + if (!state.coverage.pending.size) { + purgeStale(); + shareBudget(); + } } notify(); } @@ -809,13 +878,12 @@ export function createLineCoverage({ /** Rebuild every loaded tile from its decoded cache (after a filter change). */ function rebuild() { purgeStale(); - for (const entry of state.coverage.tiles.values()) { - cancelCast(entry); - entry.castRequested = false; - entry.meshParts = null; - detachPrimitive(entry); - attachPrimitive(entry); - } + // What each tile's lines need under the new filter, then the shares, + // then one redraw each. + for (const entry of state.coverage.tiles.values()) + if (entry.kind === 'sequence') drawnSequences(entry); + shareCaps(); + for (const entry of state.coverage.tiles.values()) redrawEntry(entry); requestRender(); notify(); } @@ -860,6 +928,15 @@ export function createLineCoverage({ /** Recolour one sequence, every part of it, in place (selection highlight). */ function recolorSequence(id, selected) { + // Selected but left out by a cap: draw it (it is always admitted). + if (selected && (tuning.maxPoints || 0) > 0) + for (const entry of state.coverage.tiles.values()) + if ( + entry.kind === 'sequence' && + !entry.sequences.has(id) && + entry.sequenceList.some((sequence) => sequence.id === id) + ) + redrawEntry(entry); for (const entry of drawnEntries()) { const sequence = entry.sequences.get(id); if (!sequence) continue; diff --git a/src/layers/streetLevel/lineCoverage.test.mjs b/src/layers/streetLevel/lineCoverage.test.mjs index 51ac366c3..e7b8b693e 100644 --- a/src/layers/streetLevel/lineCoverage.test.mjs +++ b/src/layers/streetLevel/lineCoverage.test.mjs @@ -76,7 +76,7 @@ function setup( fetchTile: async (tile) => { asked.push(tile); if (fail) throw fail; - if (decoded) return decoded(centre); + if (decoded) return decoded(centre, tile); return { sequences: [ { @@ -247,3 +247,94 @@ test('a point budget skips lines past it, and a shorter one later may still fit' const [tile] = state.coverage.tiles.values(); assert.deepEqual([...tile.sequences.keys()], ['three', 'two']); }); + +/** A line of `n` points, its id unique to the tile. */ +function pointsLine(id, n, tile, centre) { + return { + id, + parts: [ + Array.from({ length: n }, (_, i) => [ + centre.lon - 0.001 + i * 0.00001, + centre.lat, + ]), + ], + capturedAt: 0, + isPano: false, + }; +} + +test('the point budget is shared: a sparse tile keeps its few, a dense one gets the rest', async (t) => { + const tuning = { + ...TUNING, + maxTiles: 9, + maxSequences: 900, + maxPoints: 90, + keepOrder: true, + }; + const { state, coverage } = setup(t, { + // Street zoom far enough in that several tiles are in view. + zoomFor: () => ({ zoom: 18, kind: 'sequence' }), + tuning, + decoded: (centre, tile) => { + const dense = tile.x % 2 === 0 && tile.y % 2 === 0; + const lines = dense + ? Array.from({ length: 30 }, (_, i) => + pointsLine(`d${tile.x}/${tile.y}/${i}`, 5, tile, centre), + ) + : [pointsLine(`s${tile.x}/${tile.y}`, 2, tile, centre)]; + return { sequences: lines, overview: [] }; + }, + }); + coverage.refresh(); + assert.ok( + await until( + () => state.coverage.tiles.size > 1 && !state.coverage.pending.size, + ), + ); + const entries = [...state.coverage.tiles.values()]; + const drawn = (entry) => entry.pointsDrawn; + const total = entries.reduce((sum, entry) => sum + drawn(entry), 0); + assert.ok(total <= tuning.maxPoints, `within the budget: ${total}`); + const sparse = entries.filter((entry) => entry.pointDemand === 2); + const dense = entries.filter((entry) => entry.pointDemand === 150); + assert.ok(sparse.length && dense.length, 'both kinds in view'); + for (const entry of sparse) + assert.equal(drawn(entry), 2, 'a sparse tile keeps all'); + const fair = Math.floor(tuning.maxPoints / tuning.maxTiles); + const left = tuning.maxPoints - sparse.length * 2; + for (const entry of dense) + assert.ok( + drawn(entry) > fair || dense.length * fair >= left, + `a dense tile gets more than the even share: ${drawn(entry)}`, + ); + assert.ok( + total > tuning.maxPoints - 5 * dense.length, + 'little budget unused', + ); +}); + +test('a selected sequence the budget left out is drawn, highlighted', async (t) => { + const { state, coverage } = setup(t, { + tuning: { + ...TUNING, + maxTiles: 1, + maxSequences: 10, + maxPoints: 5, + keepOrder: true, + }, + decoded: (centre, tile) => ({ + sequences: [ + pointsLine('long', 5, tile, centre), + pointsLine('short', 3, tile, centre), + ], + overview: [], + }), + }); + coverage.refresh(); + assert.ok(await until(() => state.coverage.tiles.size === 1)); + const [tile] = state.coverage.tiles.values(); + assert.deepEqual([...tile.sequences.keys()], ['long']); + state.sequence.selectedId = 'short'; + coverage.recolorSequence('short', true); + assert.deepEqual([...tile.sequences.keys()].sort(), ['long', 'short']); +}); diff --git a/src/layers/streetLevel/photoViewer.js b/src/layers/streetLevel/photoViewer.js index b4fb67398..0457e9f76 100644 --- a/src/layers/streetLevel/photoViewer.js +++ b/src/layers/streetLevel/photoViewer.js @@ -79,8 +79,7 @@ async function loadBitmap(url, timeoutMs, resize, failure, signal) { // photo's size before it is decoded. const length = Number(response.headers?.get?.('content-length')); if (length > MAX_IMAGE_BYTES) throw new Error(failure); - const blob = await response.blob(); - if (blob.size > MAX_IMAGE_BYTES) throw new Error(failure); + const blob = await boundedBlob(response, failure); const bitmap = await (resize ? createImageBitmap(blob, { resizeWidth: resize[0], @@ -88,6 +87,7 @@ async function loadBitmap(url, timeoutMs, resize, failure, signal) { resizeQuality: 'high', }) : createImageBitmap(blob)); + // (A resized decode reports its new size: this guards unresized ones.) if (bitmap.width > MAX_IMAGE_SIDE || bitmap.height > MAX_IMAGE_SIDE) { bitmap.close?.(); throw new Error(failure); @@ -101,6 +101,34 @@ async function loadBitmap(url, timeoutMs, resize, failure, signal) { } } +/** + * The body as a Blob, read with a running count so a host that sends no + * length (or a false one) cannot fill memory before the cap is checked. + */ +async function boundedBlob(response, failure) { + const reader = response.body?.getReader?.(); + if (!reader) { + const blob = await response.blob(); + if (blob.size > MAX_IMAGE_BYTES) throw new Error(failure); + return blob; + } + const chunks = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > MAX_IMAGE_BYTES) { + reader.cancel().catch(() => {}); + throw new Error(failure); + } + chunks.push(value); + } + return new Blob(chunks, { + type: response.headers?.get?.('content-type') || 'image/jpeg', + }); +} + /** The panorama size a canvas this wide can show sharply at this field of view. */ export function panoSizeFor(canvasPixels, fovDeg) { const needed = (Math.max(1, canvasPixels) * 360) / Math.max(1, fovDeg); diff --git a/src/layers/streetLevel/photoViewer.test.mjs b/src/layers/streetLevel/photoViewer.test.mjs index 78397a632..05391a3cc 100644 --- a/src/layers/streetLevel/photoViewer.test.mjs +++ b/src/layers/streetLevel/photoViewer.test.mjs @@ -303,6 +303,20 @@ test('a picture file or bitmap past any real photo’s size is refused', async ( respond({ width: MAX_IMAGE_SIDE + 1 }); await assert.rejects(loadImage('https://x/a.jpg'), /could not be loaded/); assert.equal(closed, 1, 'the oversize bitmap is released'); + // No length given, a body that keeps coming: cut off while it streams. + let pulled = 0; + globalThis.fetch = async () => ({ + ok: true, + headers: new Map(), + body: new ReadableStream({ + pull(controller) { + pulled++; + controller.enqueue(new Uint8Array(8 * 1024 * 1024)); + }, + }), + }); + await assert.rejects(loadImage('https://x/a.jpg'), /could not be loaded/); + assert.ok(pulled * 8 * 1024 * 1024 <= MAX_IMAGE_BYTES + 16 * 1024 * 1024); respond({}); assert.equal((await loadImage('https://x/a.jpg')).width, 100); }); diff --git a/src/layers/streetLevel/providers/panoramax/decode.js b/src/layers/streetLevel/providers/panoramax/decode.js index c75a01768..0a354d06b 100644 --- a/src/layers/streetLevel/providers/panoramax/decode.js +++ b/src/layers/streetLevel/providers/panoramax/decode.js @@ -9,7 +9,8 @@ import { /** * Douglas–Peucker in tile units: the points of `line` ({x, y}) that keep - * it within `tolerance` of the original, first and last always kept. + * it within `tolerance` of the original (measured to each segment, so + * dead ends and U-turns stay), first and last always kept. * Iterative, so a long track cannot overflow the stack. */ export function simplifyLine(line, tolerance) { @@ -24,15 +25,20 @@ export function simplifyLine(line, tolerance) { const b = line[to]; const dx = b.x - a.x; const dy = b.y - a.y; - const span = Math.hypot(dx, dy); + const span2 = dx * dx + dy * dy; let far = -1; let farthest = tolerance; for (let i = from + 1; i < to; i++) { const p = line[i]; - // Distance to the chord, or to `a` where the chord has no length. - const d = span - ? Math.abs((p.x - a.x) * dy - (p.y - a.y) * dx) / span - : Math.hypot(p.x - a.x, p.y - a.y); + // Distance to the segment, not the infinite line: a dead end or a + // U-turn (a point beyond the chord's ends) is kept. + const t = span2 + ? Math.max( + 0, + Math.min(1, ((p.x - a.x) * dx + (p.y - a.y) * dy) / span2), + ) + : 0; + const d = Math.hypot(p.x - (a.x + t * dx), p.y - (a.y + t * dy)); if (d > farthest) { farthest = d; far = i; diff --git a/src/layers/streetLevel/providers/panoramax/decode.test.mjs b/src/layers/streetLevel/providers/panoramax/decode.test.mjs index 104471a1e..c98ff3461 100644 --- a/src/layers/streetLevel/providers/panoramax/decode.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/decode.test.mjs @@ -216,6 +216,13 @@ test('lines are simplified within a few tile units, ends and corners kept', () = ]; assert.equal(simplifyLine(loop, 4).length, 3); assert.equal(simplifyLine(zigzag, 0).length, 1000, 'no tolerance, no change'); + // A dead end: out to 100 and back to 50 keeps its far end. + const deadEnd = [ + { x: 0, y: 0 }, + { x: 100, y: 0 }, + { x: 50, y: 0 }, + ]; + assert.deepEqual(simplifyLine(deadEnd, 4), deadEnd); }); test('a decoded GPS track keeps a fraction of its points', () => { diff --git a/src/layers/streetLevel/providers/panoramax/index.js b/src/layers/streetLevel/providers/panoramax/index.js index 9eb13cdaf..b6cfa93c8 100644 --- a/src/layers/streetLevel/providers/panoramax/index.js +++ b/src/layers/streetLevel/providers/panoramax/index.js @@ -286,11 +286,15 @@ export function createPanoramaxProvider({ }); /** A z15 tile's pictures of one sequence, by `z/x/y:sequence`. */ const pictureLists = createLru(PICTURE_LIST_CACHE_SIZE); - function picturesIn(tile, sequenceId, signal) { + /** + * Only finished lists are kept: a load in flight belongs to its own + * selection, whose abort (a re-centre) must not empty the next one's. + */ + async function picturesIn(tile, sequenceId, signal) { const key = `${tileKey(tile)}:${sequenceId}`; const kept = pictureLists.get(key); if (kept) return kept; - const list = readable(async () => + const list = await readable(async () => decodePictures( await source.getTile(tile, { signal, lane: 'pictures' }), tile, @@ -298,7 +302,6 @@ export function createPanoramaxProvider({ ), ); pictureLists.set(key, list); - list.catch(() => pictureLists.delete(key)); return list; } @@ -418,6 +421,9 @@ export function createPanoramaxProvider({ cones.setVisible(false); drawn = null; preferred = null; + // Off: decoded tiles and lists go (several MB each over a city). + coverageTiles.clear(); + pictureLists.clear(); }, destroy(cesiumViewer) { diff --git a/src/layers/streetLevel/providers/panoramax/policy.js b/src/layers/streetLevel/providers/panoramax/policy.js index 59fc95b3b..7e8ccca0d 100644 --- a/src/layers/streetLevel/providers/panoramax/policy.js +++ b/src/layers/streetLevel/providers/panoramax/policy.js @@ -62,8 +62,8 @@ export const COVERAGE_TUNING = Object.freeze({ }); /** * Lines are simplified (Douglas–Peucker) to this many tile units of a - * 4,096 extent: under a pixel at every height lines are drawn from, and - * 6–10% of the points GPS tracks carry. + * 4,096 extent: about a pixel where each zoom is drawn (1.6 m at z14, + * within GPS noise), and 6–10% of the points GPS tracks carry. */ export const SIMPLIFY_TOLERANCE_UNITS = 4; /** @@ -280,8 +280,8 @@ export function normalizeItem(item) { isPano: isEquirectangular(item), capturedAt: panoramaxTimeMs(props.datetime), creator: creatorOf(item), - // Licences ask for the source to be named: the instance holding it is - // named beside an author, or alone where the author is hidden. + // Licences ask for the source to be named: where no author can be + // named, the caption names the instance holding the picture. via: instanceOf(item), license: licenseLabel(props.license), width: Number.isFinite(width) && width > 0 ? width : null, diff --git a/src/layers/streetLevel/providers/panoramax/provider.test.mjs b/src/layers/streetLevel/providers/panoramax/provider.test.mjs index 8cf3911d9..031f376d7 100644 --- a/src/layers/streetLevel/providers/panoramax/provider.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/provider.test.mjs @@ -927,19 +927,89 @@ test('lines keep the decoded (longest-first) order under a budget of points', as assert.ok(COVERAGE_TUNING.maxPoints > 0); }); -test('a coverage tile decoded a moment ago is not fetched or decoded again', async (t) => { +test('a coverage tile decoded a moment ago is not fetched again; switched off, the tiles are let go', async (t) => { const env = coverageSetup(t, { sequences: [line('a')] }); - let asked = 0; + const asked = []; const getTile = env.source.getTile; env.source.getTile = (tile, options) => { - asked++; + asked.push(`${tile.z}/${tile.x}/${tile.y}`); return getTile(tile, options); }; env.instance.activate(env.viewer); assert.ok(await until(() => env.instance.coverageStats().count === 1)); - const first = asked; + const z14 = asked.length; + // Up a zoom band and back down: the z14 tile comes from memory. + env.viewer.view.height = 3_000; + env.instance.refreshCoverage(); + assert.ok(await until(() => env.instance.coverageStats().zoom === 13)); + assert.ok(await until(() => !env.instance.coverageStats().loading)); + const withZ13 = asked.length; + env.viewer.view.height = 900; + env.instance.refreshCoverage(); + assert.ok(await until(() => env.instance.coverageStats().zoom === 14)); + assert.ok(await until(() => env.instance.coverageStats().count === 1)); + assert.equal(asked.length, withZ13, 'nothing fetched again'); + assert.ok(z14 > 0); + // Off and on: the memory went back, so the tile is fetched again. env.instance.deactivate(); env.instance.activate(env.viewer); assert.ok(await until(() => env.instance.coverageStats().count === 1)); - assert.equal(asked, first); + assert.ok(asked.length > withZ13); +}); + +test('a re-centre while the cones still load does not inherit the aborted load’s tiles', async (t) => { + const items = sequence('long', 400); + const tiles = pictureTiles(items); + const env = await mounted({ items, tiles }); + const cones = await withCones(t, env); + let hold = null; + env.source.getTile = async (tile, { signal } = {}) => { + env.source.calls.tiles.push(`${tile.z}/${tile.x}/${tile.y}`); + if (hold) await hold.promise; + signal?.throwIfAborted(); + return tiles(tile); + }; + await env.instance.viewer.open('long-0'); + const tileOf = (item) => { + const [lon, lat] = item.geometry.coordinates; + return pictureTilesAround({ lon, lat })[0]; + }; + const first = new Set( + pictureTilesAround({ + lon: items[0].geometry.coordinates[0], + lat: items[0].geometry.coordinates[1], + }).map((tile) => `${tile.x}/${tile.y}`), + ); + const out = items.findIndex((item) => { + const tile = tileOf(item); + return !first.has(`${tile.x}/${tile.y}`); + }); + hold = {}; + hold.promise = new Promise((resolve) => (hold.release = resolve)); + const loading = env.instance.selectSequence('long'); + await new Promise((resolve) => setTimeout(resolve, 5)); + // The far picture opens while the first load is held: the cones re-centre. + await env.instance.viewer.open(`long-${out}`); + await new Promise((resolve) => setTimeout(resolve, 5)); + const release = hold.release; + hold = null; + release(); + await loading; + assert.ok(await until(() => !env.instance.sequenceStats().loading)); + // Every picture in the tiles the second load covers has its cone. + const [lon, lat] = items[out].geometry.coordinates; + const second = new Set( + pictureTilesAround({ lon, lat }).map((tile) => `${tile.x}/${tile.y}`), + ); + const ids = new Set(cones.ids()); + const covered = items.filter((item) => { + const tile = tileOf(item); + return second.has(`${tile.x}/${tile.y}`); + }); + assert.ok(covered.length > 0); + assert.equal( + covered.filter((item) => ids.has(item.id)).length, + covered.length, + 'every picture in the re-centred tiles has its cone (the shared ones too)', + ); }); diff --git a/src/layers/streetLevel/tileCache.js b/src/layers/streetLevel/tileCache.js index 55d997cdb..7fc9bb889 100644 --- a/src/layers/streetLevel/tileCache.js +++ b/src/layers/streetLevel/tileCache.js @@ -44,5 +44,7 @@ export function createTileCache({ kept.set(key, { at: now(), tile: loaded }); return loaded; }, + /** Forget every tile (the layer is off: its memory goes back). */ + clear: () => kept.clear(), }; } From ce3f66a8377d34d7c99285721706b82767e31ebf Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Wed, 14 Oct 2026 15:54:21 +0100 Subject: [PATCH 52/95] =?UTF-8?q?fix(street-level):=20IC8=20round=203=20?= =?UTF-8?q?=E2=80=94=20no=20blink=20when=20the=20point=20budget=20is=20re-?= =?UTF-8?q?shared?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The round-3 verification found the shared point budget rebuilding a dense tile by removing its lines before the new ones were built, so central Paris vanished for 1–2 s after every pan: - Tiles redrawn for a new share, or for a selected line the cap had left out, go through the swap path: the old lines stay until the new ones are ready. - A share must move by 10% to redraw a tile (a 1.6% change rebuilt it). - The selected line, admitted past the cap, is not counted against it, so its tile no longer redraws on every settle. - A selection the imagery filter hides is not redrawn for. - Shares are recomputed when a refresh ends loading without a tile finishing. Co-Authored-By: Claude Opus 5.5 --- src/layers/streetLevel/lineCoverage.js | 52 +++++++---- src/layers/streetLevel/lineCoverage.test.mjs | 95 ++++++++++++++++++++ 2 files changed, 131 insertions(+), 16 deletions(-) diff --git a/src/layers/streetLevel/lineCoverage.js b/src/layers/streetLevel/lineCoverage.js index 50a81cfe2..78c9b36da 100644 --- a/src/layers/streetLevel/lineCoverage.js +++ b/src/layers/streetLevel/lineCoverage.js @@ -12,6 +12,8 @@ import { densifyLine, MESH_DENSIFY_DEG } from './groundCast.js'; import { MESH_SAMPLE_RADIUS_M } from './meshSampler.js'; import { tileBounds, tilesForBbox } from './tileMath.js'; +/** A tile's shared point cap must move by this much to redraw it. */ +const SHARE_HYSTERESIS = 0.1; /** Draped lines stay at most this long while a tile's cast lines build. */ export const SWAP_MAX_WAIT_MS = 4000; /** Tiles touched by new mesh samples are redrawn at most this often. */ @@ -164,9 +166,9 @@ export function createLineCoverage({ const size = pointsOf(sequence); demand += size; // The selected sequence is always drawn: its highlight is the answer - // to a click, whatever the caps leave out. + // to a click, whatever the caps leave out. It is outside the cap, so + // it is not counted against it. if (sequence.id === selectedId) { - points += size; drawn.push(sequence); continue; } @@ -204,32 +206,45 @@ export function createLineCoverage({ left -= cap; const before = entry.pointCap ?? perTilePointCap; entry.pointCap = cap; - // Only a change that alters what is drawn: more room for a tile that - // had lines left out, or less than it draws now. - if ( - (cap > before && entry.pointsDrawn < entry.pointDemand) || - cap < entry.pointsDrawn - ) - changed.push(entry); + // Only a change that alters what is drawn, by enough to be worth a + // rebuild: more room for a tile that had lines left out, or well + // under what it draws now (a small overshoot stays until the next + // natural redraw). + const grows = + cap > before * (1 + SHARE_HYSTERESIS) && + entry.pointsDrawn < entry.pointDemand; + const shrinks = cap < entry.pointsDrawn * (1 - SHARE_HYSTERESIS); + if (grows || shrinks) changed.push(entry); }); return changed; } - /** Once loading settles: redraw the tiles the shared budget changes. */ + /** + * Once loading settles: redraw the tiles the shared budget changes, + * keeping their old lines until the new ones are ready (no blink). + */ function shareBudget() { const changed = shareCaps(); - for (const entry of changed) redrawEntry(entry); + for (const entry of changed) redrawEntry(entry, { swap: true }); if (changed.length) { requestRender(); notify(); } } - /** Draw one tile again from its decoded lines (caps or selection changed). */ - function redrawEntry(entry) { + /** + * Draw one tile again from its decoded lines (filter, caps or selection + * changed). With `swap`, its old lines stay until the new ones are ready. + */ + function redrawEntry(entry, { swap = false } = {}) { cancelCast(entry); entry.castRequested = false; entry.meshParts = null; + if (swap && entry.kind === 'sequence' && entry.primitives?.length) { + finishSwap(entry); + redrawTile(entry); + return; + } detachPrimitive(entry); attachPrimitive(entry); } @@ -825,7 +840,10 @@ export function createLineCoverage({ state.coverage.pending.delete(key); } for (const tile of tiles) loadTile(tile, kind); - if (!state.coverage.pending.size) purgeStale(); + if (!state.coverage.pending.size) { + purgeStale(); + shareBudget(); + } // The camera moved: cells that were out of the sampler's range may not be, // and lines left draped by a failed, partial or dropped cast get another try. if (terrainMode()) @@ -943,9 +961,11 @@ export function createLineCoverage({ if ( entry.kind === 'sequence' && !entry.sequences.has(id) && - entry.sequenceList.some((sequence) => sequence.id === id) + entry.sequenceList.some( + (sequence) => sequence.id === id && passes(sequence, filter()), + ) ) - redrawEntry(entry); + redrawEntry(entry, { swap: true }); for (const entry of drawnEntries()) { const sequence = entry.sequences.get(id); if (!sequence) continue; diff --git a/src/layers/streetLevel/lineCoverage.test.mjs b/src/layers/streetLevel/lineCoverage.test.mjs index e7b8b693e..db89e8143 100644 --- a/src/layers/streetLevel/lineCoverage.test.mjs +++ b/src/layers/streetLevel/lineCoverage.test.mjs @@ -338,3 +338,98 @@ test('a selected sequence the budget left out is drawn, highlighted', async (t) coverage.recolorSequence('short', true); assert.deepEqual([...tile.sequences.keys()].sort(), ['long', 'short']); }); + +/** Count the line primitives a coverage adds to the scene. */ +function countAdds(viewer) { + const counts = { added: 0 }; + for (const collection of [ + viewer.scene.groundPrimitives, + viewer.scene.primitives, + ]) { + const add = collection.add.bind(collection); + collection.add = (primitive) => { + counts.added++; + return add(primitive); + }; + } + return counts; +} + +test('a tile redrawn for a new share keeps its old lines until the new ones are ready', async (t) => { + const tuning = { + ...TUNING, + maxTiles: 9, + maxSequences: 900, + maxPoints: 90, + keepOrder: true, + }; + const { state, coverage, viewer } = setup(t, { + zoomFor: () => ({ zoom: 18, kind: 'sequence' }), + tuning, + decoded: (centre, tile) => { + const dense = tile.x % 2 === 0 && tile.y % 2 === 0; + return { + sequences: dense + ? Array.from({ length: 30 }, (_, i) => + pointsLine(`d${tile.x}/${tile.y}/${i}`, 5, tile, centre), + ) + : [pointsLine(`s${tile.x}/${tile.y}`, 2, tile, centre)], + overview: [], + }; + }, + }); + coverage.refresh(); + assert.ok( + await until( + () => state.coverage.tiles.size > 1 && !state.coverage.pending.size, + ), + ); + const redrawn = [...state.coverage.tiles.values()].filter( + (entry) => entry.swap, + ); + assert.ok(redrawn.length > 0, 'a dense tile took a bigger share'); + for (const entry of redrawn) + for (const { primitive } of entry.swap.old) + assert.ok( + viewer.scene.groundPrimitives.items.has(primitive) || + viewer.scene.primitives.items.has(primitive), + 'the old lines are still on the globe while the new ones build', + ); +}); + +test('a selected line outside the cap does not make its tile redraw on every settle, nor does one the filter hides', async (t) => { + let pano = 'all'; + const { state, coverage, viewer } = setup(t, { + tuning: { + ...TUNING, + maxTiles: 1, + maxSequences: 10, + maxPoints: 5, + keepOrder: true, + }, + decoded: (centre, tile) => ({ + sequences: [ + pointsLine('long', 5, tile, centre), + { ...pointsLine('short', 3, tile, centre), isPano: true }, + ], + overview: [], + }), + }); + state.context.getFilter = () => ({ pano, sinceMs: null }); + coverage.refresh(); + assert.ok(await until(() => state.coverage.tiles.size === 1)); + state.sequence.selectedId = 'short'; + coverage.recolorSequence('short', true); + const counts = countAdds(viewer); + for (let i = 0; i < 3; i++) { + coverage.refresh(); + await new Promise((resolve) => setTimeout(resolve, 20)); + } + assert.equal(counts.added, 0, 'settles redraw nothing'); + // A selection the filter hides is not drawn, so it is not redrawn for. + pano = 'flat'; + coverage.rebuild(); + const after = countAdds(viewer); + for (let i = 0; i < 3; i++) coverage.recolorSequence('short', true); + assert.equal(after.added, 0); +}); From 3962ac52f7195bf50314682742a6e64a33c1bde0 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Wed, 14 Oct 2026 20:13:02 -0400 Subject: [PATCH 53/95] fix(imagery): require explicit Data-Present proof before decoding thumbnails Rebased onto current main with review follow-ups folded in. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 ++ docs/CURRENT-STATE.md | 2 ++ src/layers/recentImagery/source.js | 6 +++- .../thumbnail-presence-proof.test.mjs | 30 +++++++++++++++++++ src/layers/recentImagery/thumbnails.js | 4 +-- 5 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 src/layers/recentImagery/thumbnail-presence-proof.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index a770fbd35..6f5ee5c10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ - Director timeline seeks cross the dateline along the short longitude arc. +- Recent Imagery thumbnails require explicit presence evidence before reporting available pixels. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1f794a5b5..93e5214a5 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4449,6 +4449,8 @@ are omitted rather than framing the wrong part of the globe. ### Recent Imagery (September 2026) +Thumbnail decoding requires explicit `Data-Present: true` evidence; false remains empty, while absent or malformed evidence reports an error. Injected `getThumbnail` sources must return an OK Response with readable `Data-Present: true` to admit image bytes, or `false` for confirmed empty coverage. Cross-origin sources must expose that header to browser JavaScript. + - The layer sits in the Data Layers panel's Cameras group beside Cameras (CCTV); Mapped ALPR Cameras moved to the Infrastructure group to make room. - DATA ▸ **Recent Imagery** (`recent-imagery`, `src/layers/recentImagery/`) lists the last 30 UTC days of imagery for a selected box and drapes the days you pick: HLS S30 (Sentinel-2) and L30 (Landsat 8/9) at 30 m, and the VIIRS NOAA-21 daily overview at 250 m. Keyless and browser-direct: NASA CMR for which days have HLS granules in the box (`catalog.js`, both collections concurrently, paged with `CMR-Search-After` up to 2,000 records each, `truncated` against CMR `hits`), GIBS WMTS for tiles, Worldview Snapshots for thumbnails and exports. Worldview's `Data-Present` header is the only availability signal: GIBS answers an empty tile with HTTP 200, and the JPEG overview would drape black, so a day confirmed empty never drapes. - **Box.** SELECT BOX (`src/ui/imageryBoxTool.js`) claims the pointer like Draw, borrows the stock click actions and freezes the camera only mid-drag; USE VIEW takes the camera rectangle; `boxFromPinAt` grows a 10 km box. Boxes must be ≤ 1,000 km a side, on one side of the dateline and inside the Web-Mercator limit (`validateBox`). A refusal (`boxError`, including the tool's own via `reportBoxRefusal`) stays at the top of the panel until a box succeeds; a view over the cap is refused with its size, never shrunk. An oversized box or view (`Box is … km wide · limit 1,000 km`, `View is … wide · …`) sets the hint to `Zoom in or draw a smaller box` and enables ZOOM IN: `zoomToFit()` flies top-down (1.2 s) to the refused box's centre, or the ground under the canvas centre for a view, at the height whose view is about 400 km wide for the canvas aspect and the camera's vertical FOV (`fitViewHeightM`, clamped 5–400 km), and returns that height. diff --git a/src/layers/recentImagery/source.js b/src/layers/recentImagery/source.js index 258e8e2e1..9e6749cb7 100644 --- a/src/layers/recentImagery/source.js +++ b/src/layers/recentImagery/source.js @@ -1,7 +1,11 @@ import { searchHls } from './catalog.js'; import { gibsTemplate, wvsSnapshotUrl } from './model.js'; -/** NASA-compatible imagery acquisition; transport and URL builders are replaceable. */ +/** + * NASA-compatible imagery acquisition; transport and URL builders are replaceable. + * Thumbnail responses must expose Data-Present: true for image admission, or + * false for confirmed empty coverage, including injected getThumbnail sources. + */ export function createRecentImagerySource({ fetchImpl = (...args) => globalThis.fetch(...args), search = searchHls, diff --git a/src/layers/recentImagery/thumbnail-presence-proof.test.mjs b/src/layers/recentImagery/thumbnail-presence-proof.test.mjs new file mode 100644 index 000000000..4605079b1 --- /dev/null +++ b/src/layers/recentImagery/thumbnail-presence-proof.test.mjs @@ -0,0 +1,30 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createThumbnailLoader } from './thumbnails.js'; +test('missing or malformed presence evidence cannot prove thumbnail pixels', async () => { + for (const header of [null, 'unknown', '', 'true', 'false']) { + let decodes = 0; + const loader = createThumbnailLoader({ + requestThumbnail: async () => ({ + ok: true, + headers: { get: (key) => (key === 'Data-Present' ? header : null) }, + blob: async () => { + decodes++; + return new Blob(['fixture']); + }, + }), + createObjectUrl: () => 'blob:fixture', + }); + loader.request( + { key: 'fixture', product: 'S30', day: '2026-09-18' }, + { west: 0, east: 1, south: 0, north: 1 }, + ); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal( + loader.get('fixture').status, + header === 'true' ? 'present' : header === 'false' ? 'empty' : 'error', + ); + assert.equal(decodes, header === 'true' ? 1 : 0); + loader.destroy(); + } +}); diff --git a/src/layers/recentImagery/thumbnails.js b/src/layers/recentImagery/thumbnails.js index 23fe4dd49..4aaec294b 100644 --- a/src/layers/recentImagery/thumbnails.js +++ b/src/layers/recentImagery/thumbnails.js @@ -100,12 +100,12 @@ export function createThumbnailLoader({ cancelled = true; } else if (response?.ok) { const present = String( - response.headers?.get?.('Data-Present') ?? 'true', + response.headers?.get?.('Data-Present') ?? '', ).toLowerCase(); acquisitionTime = response.headers?.get?.('Acquisition-Time') || null; if (present === 'false') { status = 'empty'; - } else { + } else if (present === 'true') { const blob = await response.blob(); if (controller.signal.aborted) cancelled = true; else { From 360aa74f50c72cc485734978977bf0884de06282 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Wed, 14 Oct 2026 20:13:52 -0400 Subject: [PATCH 54/95] fix(imagery): paginate raw CMR pages within the record budget Rebased onto current main with review follow-ups folded in. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 + docs/CURRENT-STATE.md | 2 + src/layers/recentImagery/catalog.js | 18 ++++-- .../cmr-raw-page-budget.test.mjs | 56 +++++++++++++++++++ 4 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 src/layers/recentImagery/cmr-raw-page-budget.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index a770fbd35..9b4d82063 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ - Director timeline seeks cross the dateline along the short longitude arc. +- Recent Imagery continues past malformed records in full catalog pages and respects the requested record limit. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1f794a5b5..b65f90ca0 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4449,6 +4449,8 @@ are omitted rather than framing the wrong part of the globe. ### Recent Imagery (September 2026) +CMR pagination counts raw rows when checking page completion and the record budget; catalog truncation means upstream hits exceed rows examined, so discarded malformed rows do not imply unseen imagery. + - The layer sits in the Data Layers panel's Cameras group beside Cameras (CCTV); Mapped ALPR Cameras moved to the Infrastructure group to make room. - DATA ▸ **Recent Imagery** (`recent-imagery`, `src/layers/recentImagery/`) lists the last 30 UTC days of imagery for a selected box and drapes the days you pick: HLS S30 (Sentinel-2) and L30 (Landsat 8/9) at 30 m, and the VIIRS NOAA-21 daily overview at 250 m. Keyless and browser-direct: NASA CMR for which days have HLS granules in the box (`catalog.js`, both collections concurrently, paged with `CMR-Search-After` up to 2,000 records each, `truncated` against CMR `hits`), GIBS WMTS for tiles, Worldview Snapshots for thumbnails and exports. Worldview's `Data-Present` header is the only availability signal: GIBS answers an empty tile with HTTP 200, and the JPEG overview would drape black, so a day confirmed empty never drapes. - **Box.** SELECT BOX (`src/ui/imageryBoxTool.js`) claims the pointer like Draw, borrows the stock click actions and freezes the camera only mid-drag; USE VIEW takes the camera rectangle; `boxFromPinAt` grows a 10 km box. Boxes must be ≤ 1,000 km a side, on one side of the dateline and inside the Web-Mercator limit (`validateBox`). A refusal (`boxError`, including the tool's own via `reportBoxRefusal`) stays at the top of the panel until a box succeeds; a view over the cap is refused with its size, never shrunk. An oversized box or view (`Box is … km wide · limit 1,000 km`, `View is … wide · …`) sets the hint to `Zoom in or draw a smaller box` and enables ZOOM IN: `zoomToFit()` flies top-down (1.2 s) to the refused box's centre, or the ground under the canvas centre for a view, at the height whose view is about 400 km wide for the canvas aspect and the camera's vertical FOV (`fitViewHeightM`, clamped 5–400 km), and returns that height. diff --git a/src/layers/recentImagery/catalog.js b/src/layers/recentImagery/catalog.js index 96b314690..fe9a7a021 100644 --- a/src/layers/recentImagery/catalog.js +++ b/src/layers/recentImagery/catalog.js @@ -40,6 +40,7 @@ export async function fetchCmrPages({ }) { const granules = []; let hits = 0; + let readCount = 0; let cursor = null; for (;;) { const init = { signal }; @@ -47,19 +48,24 @@ export async function fetchCmrPages({ const response = await fetchImpl(url, init); if (!response?.ok) throw new Error(`HTTP ${response?.status ?? 'error'} from CMR`); - const page = parseCmrUmm(await response.json(), product); - granules.push(...page.granules); + const json = await response.json(); + const rawCount = Array.isArray(json?.items) ? json.items.length : 0; + const page = parseCmrUmm(json, product); + readCount += rawCount; + granules.push( + ...page.granules.slice(0, Math.max(0, maxRecords - granules.length)), + ); hits = Math.max(hits, page.hits); cursor = response.headers?.get?.(CMR_SEARCH_AFTER) ?? null; if ( !cursor || - page.granules.length < CMR_PAGE_SIZE || - granules.length >= maxRecords || - granules.length >= hits + rawCount < CMR_PAGE_SIZE || + readCount >= maxRecords || + readCount >= hits ) break; } - return { granules, truncated: hits > granules.length }; + return { granules, truncated: hits > readCount }; } function finiteOrNull(value) { diff --git a/src/layers/recentImagery/cmr-raw-page-budget.test.mjs b/src/layers/recentImagery/cmr-raw-page-budget.test.mjs new file mode 100644 index 000000000..4925e7bec --- /dev/null +++ b/src/layers/recentImagery/cmr-raw-page-budget.test.mjs @@ -0,0 +1,56 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { fetchCmrPages } from './catalog.js'; +const item = (id) => ({ + meta: { 'concept-id': id }, + umm: { + TemporalExtent: { + RangeDateTime: { BeginningDateTime: '2026-09-18T12:00:00Z' }, + }, + }, +}); +test('discarded rows do not turn a full CMR page into a final page', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + fetchImpl: async () => { + calls++; + const items = + calls === 1 + ? Array.from({ length: 200 }, (_, i) => + i === 0 ? {} : item(String(i)), + ) + : [item('last')]; + return Response.json( + { hits: 201, items }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 200); + assert.equal(result.granules.at(-1).id, 'last'); + assert.equal(result.truncated, false); +}); +test('a non-page-multiple limit never returns more admitted records than its budget', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + maxRecords: 250, + fetchImpl: async () => { + calls++; + return Response.json( + { + hits: 1000, + items: Array.from({ length: 200 }, (_, i) => item(`${calls}:${i}`)), + }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 250); + assert.equal(result.truncated, true); +}); From 0555d99ea467454ec567767a7fee5c56e02c5459 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Wed, 14 Oct 2026 20:14:54 -0400 Subject: [PATCH 55/95] fix(cctv): leave conflicting direction fields unresolved Rebased onto current main with review follow-ups folded in. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 + docs/CURRENT-STATE.md | 5 +++ src/data/ambiguous-camera-direction.test.mjs | 30 +++++++++++++ src/data/directionText.js | 44 +++++++++++++------- 4 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 src/data/ambiguous-camera-direction.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index a770fbd35..0ae01540e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ - Director timeline seeks cross the dateline along the short longitude arc. +- Conflicting CCTV direction tokens remain unknown within the highest matching tier, preserving travel-token precedence over street quadrants and bare words. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1f794a5b5..29bdb33c0 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1172,6 +1172,11 @@ a browser key. Preview is for local build verification, not a production server. ## CCTV and radio provider modules +CCTV direction text resolves travel tokens first, then intercardinals, then +bare cardinals when enabled. Only conflicting headings within the highest +matching tier remain unknown; street quadrants and lower-tier words do not +cancel an explicit travel heading. + CCTV catalog acquisition, source normalization and frame/media delivery now live in separate modules. Each CCTV factory owns its catalog and health state; its `sourceRoot` option resolves relative source files against the application root. diff --git a/src/data/ambiguous-camera-direction.test.mjs b/src/data/ambiguous-camera-direction.test.mjs new file mode 100644 index 000000000..b5996d7a7 --- /dev/null +++ b/src/data/ambiguous-camera-direction.test.mjs @@ -0,0 +1,30 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { directionToHeading } from './directionText.js'; +test('conflicting facing directions cannot manufacture one heading', () => { + for (const bare of [false, true]) { + for (const value of ['NB / SB', 'EASTBOUND WESTBOUND', 'NE SW']) + assert.ok(Number.isNaN(directionToHeading(value, bare)), value); + assert.equal(directionToHeading('NB northbound', bare), 0); + assert.equal(directionToHeading('IH-35 SOUTHBOUND AT 15TH', bare), 180); + } + assert.ok(Number.isNaN(directionToHeading('North / East', true))); + assert.equal(directionToHeading('West / SR-24', true), 270); +}); + +test('travel tokens retain precedence over quadrant streets and bare words', () => { + for (const [text, heading] of [ + ['I-5 NB at NE 45th St', 0], + ['SR 520 EB at NE 148th', 90], + ['US 13 SB @ SW Front St', 180], + ]) { + for (const bare of [false, true]) + assert.equal(directionToHeading(text, bare), heading); + } + assert.equal( + directionToHeading('Hwy 401 east of Keele St - Westbound', true), + 270, + ); + assert.equal(directionToHeading('NE near West St', true), 45); + assert.ok(Number.isNaN(directionToHeading('NB SB near NE St', true))); +}); diff --git a/src/data/directionText.js b/src/data/directionText.js index 78f1be0df..eb78065d2 100644 --- a/src/data/directionText.js +++ b/src/data/directionText.js @@ -28,22 +28,34 @@ export function directionToHeading(value, allowBare = false) { .trim() .toUpperCase(); if (!text) return NaN; - // Explicit travel/intercardinal forms — safe on free-form text (a street - // name almost never contains "NORTHBOUND" or a lone "NB" token). - if (/\bNORTHBOUND\b|\bNB\b/.test(text)) return 0; - if (/\bSOUTHBOUND\b|\bSB\b/.test(text)) return 180; - if (/\bEASTBOUND\b|\bEB\b/.test(text)) return 90; - if (/\bWESTBOUND\b|\bWB\b/.test(text)) return 270; - if (/\bNORTHEAST\b|\bNE\b/.test(text)) return 45; - if (/\bNORTHWEST\b|\bNW\b/.test(text)) return 315; - if (/\bSOUTHEAST\b|\bSE\b/.test(text)) return 135; - if (/\bSOUTHWEST\b|\bSW\b/.test(text)) return 225; - // Bare cardinals — dedicated direction fields only. - if (allowBare) { - if (/\bNORTH\b/.test(text)) return 0; - if (/\bSOUTH\b/.test(text)) return 180; - if (/\bEAST\b/.test(text)) return 90; - if (/\bWEST\b/.test(text)) return 270; + const tiers = [ + [ + [/\bNORTHBOUND\b|\bNB\b/, 0], + [/\bSOUTHBOUND\b|\bSB\b/, 180], + [/\bEASTBOUND\b|\bEB\b/, 90], + [/\bWESTBOUND\b|\bWB\b/, 270], + ], + [ + [/\bNORTHEAST\b|\bNE\b/, 45], + [/\bNORTHWEST\b|\bNW\b/, 315], + [/\bSOUTHEAST\b|\bSE\b/, 135], + [/\bSOUTHWEST\b|\bSW\b/, 225], + ], + ]; + if (allowBare) + tiers.push([ + [/\bNORTH\b/, 0], + [/\bSOUTH\b/, 180], + [/\bEAST\b/, 90], + [/\bWEST\b/, 270], + ]); + for (const forms of tiers) { + const headings = new Set(); + for (const [pattern, heading] of forms) { + if (pattern.test(text)) headings.add(heading); + } + if (headings.size) + return headings.size === 1 ? headings.values().next().value : NaN; } return NaN; } From 12ea0219d1cf73be07b07fd9470f07bbb138758c Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Wed, 14 Oct 2026 20:15:28 -0400 Subject: [PATCH 56/95] chore(ui): harden surface Tab cycling Rebased onto current main with review follow-ups folded in. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 2 ++ docs/CURRENT-STATE.md | 3 ++ src/ui/surface-tab-order.test.mjs | 57 +++++++++++++++++++++++++++++++ src/ui/surfaceKeyboard.js | 6 +++- 4 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 src/ui/surface-tab-order.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index a770fbd35..cbd99b26e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ - Director timeline seeks cross the dateline along the short longitude arc. +- Surface Tab cycling also skips negative-tabindex, fieldset-disabled and inert controls. + - Treat explicitly current, complete empty vessel coverage as a successful update, clearing obsolete contacts while retaining stale or incomplete snapshots. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1f794a5b5..3d9c59110 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1060,6 +1060,9 @@ collapse can still be undone. ## Surface keyboard lifecycle +Surface Tab cycling excludes controls with a negative tabindex, native disabled +state or inert ancestry; ordinary visible enabled controls retain their order. + `ui/surfaces` owns the capture-phase keyboard listener, Tab cycling and return focus shared by the first-run launcher and Provider Settings. Each caller activates it while open and deactivates it on dismissal; destruction releases diff --git a/src/ui/surface-tab-order.test.mjs b/src/ui/surface-tab-order.test.mjs new file mode 100644 index 000000000..c3f335407 --- /dev/null +++ b/src/ui/surface-tab-order.test.mjs @@ -0,0 +1,57 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createSurfaceKeyboard } from './surfaceKeyboard.js'; +test('surface cycling never targets deliberately excluded controls', () => { + let handler; + const documentRef = { + activeElement: {}, + addEventListener: (_, fn) => (handler = fn), + removeEventListener() {}, + }; + const node = ({ + attributes = {}, + fieldsetDisabled = false, + inert = false, + } = {}) => ({ + hasAttribute: (name) => Object.hasOwn(attributes, name), + getClientRects: () => [{}], + getAttribute: (name) => attributes[name] ?? null, + matches: (selector) => selector === ':disabled' && fieldsetDisabled, + closest: (selector) => + selector === '[inert]' && inert ? { inert: true } : null, + focus() { + documentRef.activeElement = this; + }, + }); + const ignored = [ + node({ attributes: { tabindex: '-1' } }), + node({ fieldsetDisabled: true }), + node({ inert: true }), + ]; + const first = node({}), + last = node({}); + const controls = [...ignored, first, last]; + const root = { + ownerDocument: documentRef, + querySelectorAll: () => controls, + contains: (n) => controls.includes(n), + }; + const surface = createSurfaceKeyboard({ + root, + isActive: () => true, + onEscape() {}, + }); + surface.activate(); + const press = (shiftKey) => + handler({ + key: 'Tab', + shiftKey, + preventDefault() {}, + stopPropagation() {}, + }); + press(false); + assert.equal(documentRef.activeElement, first); + press(true); + assert.equal(documentRef.activeElement, last); + surface.destroy(); +}); diff --git a/src/ui/surfaceKeyboard.js b/src/ui/surfaceKeyboard.js index 38417975a..141e22fa6 100644 --- a/src/ui/surfaceKeyboard.js +++ b/src/ui/surfaceKeyboard.js @@ -39,7 +39,11 @@ export function createSurfaceKeyboard({ ), ].filter( (node) => - !node.hasAttribute('disabled') && node.getClientRects().length > 0, + !node.hasAttribute('disabled') && + !node.matches?.(':disabled') && + !(Number(node.getAttribute?.('tabindex')) < 0) && + !node.closest?.('[inert]') && + node.getClientRects().length > 0, ); if (!order.length) return; const first = order[0]; From 3533b323b0823e7c12e0d3de9a82d8f97467069c Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Thu, 15 Oct 2026 14:28:37 +0100 Subject: [PATCH 57/95] fix(radio): keep the || 500 fallback for a missing mirror status Node always sets statusCode on client responses, but keeping the fallback stops a missing status from reading as 200, as main did. Co-Authored-By: Claude Opus 5.5 --- server/providers/radio/transport.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/providers/radio/transport.js b/server/providers/radio/transport.js index c9922bb2f..c4b7f2b5b 100644 --- a/server/providers/radio/transport.js +++ b/server/providers/radio/transport.js @@ -130,7 +130,7 @@ export function fetchPinnedRadioResponse(url, options, addresses) { // Nothing thrown here may escape: this callback runs outside the // promise, so an exception would be uncaught and crash the server. try { - const status = response.statusCode; + const status = response.statusCode || 500; // A `Response` only represents statuses up to 599; Node never hands // this callback a 1xx. if (status > 599) From 26694380fb1136c8b3e49fbc41396b2585727ab5 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Thu, 15 Oct 2026 14:36:06 +0100 Subject: [PATCH 58/95] test(track): stub the geocode proxy routes, not the browser call Reverse geocoding now goes to /api/google/reverse-geocode on the app server, so the harness stub for maps.googleapis.com never matched and a keyed run would reach Google live. Answer both proxy routes with ZERO_RESULTS instead. Co-Authored-By: Claude Opus 5.5 --- scripts/track-regression.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/track-regression.mjs b/scripts/track-regression.mjs index 84aa2494e..756daccfc 100644 --- a/scripts/track-regression.mjs +++ b/scripts/track-regression.mjs @@ -443,7 +443,10 @@ async function main() { // for nearby names. Answer both with empty results: the entity-context // assertions are about the SELECTED contact, and a live lookup would // make the run non-hermetic (and bill the owner's Google quota). - if (url.hostname === 'maps.googleapis.com' && url.pathname.startsWith('/maps/api/geocode')) { + if ( + isAppRequest && + (url.pathname === '/api/google/reverse-geocode' || url.pathname === '/api/google/geocode') + ) { return Promise.resolve(jsonResponse({ status: 'ZERO_RESULTS', results: [] })); } if (isAppRequest && url.pathname === '/api/google/nearby-places') { From 64988f3037e33e46c336f6a914e83411821729fb Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Thu, 15 Oct 2026 15:43:38 +0100 Subject: [PATCH 59/95] fix(imagery): mark CMR results truncated when the record cap drops valid granules Co-Authored-By: Claude Opus 5.5 --- src/layers/recentImagery/catalog.js | 13 ++++-- .../cmr-raw-page-budget.test.mjs | 42 +++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/layers/recentImagery/catalog.js b/src/layers/recentImagery/catalog.js index fe9a7a021..14964b1d2 100644 --- a/src/layers/recentImagery/catalog.js +++ b/src/layers/recentImagery/catalog.js @@ -27,7 +27,8 @@ const CMR_SEARCH_AFTER = 'CMR-Search-After'; /** * Read one product's granules across CMR pages, sending back each response's * `CMR-Search-After` cursor until a short page, a missing cursor or the - * record cap ends the walk. `truncated` is measured against CMR's `hits`. + * record cap ends the walk. `truncated` means CMR's `hits` exceed the rows + * read, or the cap dropped valid granules from the last page read. * @param {{ product: string, url: string, fetchImpl: typeof fetch, signal?: AbortSignal, maxRecords?: number }} request * @returns {Promise<{ granules: Array, truncated: boolean }>} */ @@ -41,6 +42,7 @@ export async function fetchCmrPages({ const granules = []; let hits = 0; let readCount = 0; + let capped = 0; let cursor = null; for (;;) { const init = { signal }; @@ -52,9 +54,12 @@ export async function fetchCmrPages({ const rawCount = Array.isArray(json?.items) ? json.items.length : 0; const page = parseCmrUmm(json, product); readCount += rawCount; - granules.push( - ...page.granules.slice(0, Math.max(0, maxRecords - granules.length)), + const admitted = page.granules.slice( + 0, + Math.max(0, maxRecords - granules.length), ); + granules.push(...admitted); + capped += page.granules.length - admitted.length; hits = Math.max(hits, page.hits); cursor = response.headers?.get?.(CMR_SEARCH_AFTER) ?? null; if ( @@ -65,7 +70,7 @@ export async function fetchCmrPages({ ) break; } - return { granules, truncated: hits > readCount }; + return { granules, truncated: hits > readCount || capped > 0 }; } function finiteOrNull(value) { diff --git a/src/layers/recentImagery/cmr-raw-page-budget.test.mjs b/src/layers/recentImagery/cmr-raw-page-budget.test.mjs index 4925e7bec..c1709b9b2 100644 --- a/src/layers/recentImagery/cmr-raw-page-budget.test.mjs +++ b/src/layers/recentImagery/cmr-raw-page-budget.test.mjs @@ -54,3 +54,45 @@ test('a non-page-multiple limit never returns more admitted records than its bud assert.equal(result.granules.length, 250); assert.equal(result.truncated, true); }); +test('valid records dropped by the cap mark the result truncated even when every hit was read', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + maxRecords: 250, + fetchImpl: async () => { + calls++; + return Response.json( + { + hits: 400, + items: Array.from({ length: 200 }, (_, i) => item(`${calls}:${i}`)), + }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 250); + assert.equal(result.truncated, true); +}); +test('malformed rows discarded under the cap leave a fully examined result untruncated', async () => { + let calls = 0; + const result = await fetchCmrPages({ + product: 'S30', + url: 'fixture', + maxRecords: 250, + fetchImpl: async () => { + calls++; + const items = Array.from({ length: 200 }, (_, i) => + calls === 2 && i >= 50 ? {} : item(`${calls}:${i}`), + ); + return Response.json( + { hits: 400, items }, + { headers: { 'CMR-Search-After': `cursor-${calls}` } }, + ); + }, + }); + assert.equal(calls, 2); + assert.equal(result.granules.length, 250); + assert.equal(result.truncated, false); +}); From e3bc431f9a641952b2ea14772d38d50566881647 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Thu, 15 Oct 2026 16:08:18 +0100 Subject: [PATCH 60/95] =?UTF-8?q?fix(street-level):=20review=20follow-ups?= =?UTF-8?q?=20=E2=80=94=20shares=20judged=20against=20the=20last=20draw;?= =?UTF-8?q?=20refused=20pictures=20stay=20refused?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the maintainer review of #1134 (approve with follow-ups): - lineCoverage: a share is judged against the cap the tile was last drawn with, not against later shares that redrew nothing, so small increases add up (a retained tile stayed at 100 points after its share grew to 116). A rebuild's measuring pass no longer counts as a draw. - photoViewer: a picture refused for its size (bytes, streamed bytes or decoded side) is marked `tooLarge`, and the viewer never loads that file again by any path; the flat fallback had retried an oversized SD file through an , which has no size limit. - Regressions for both: cumulative share growth below the threshold, and a refused SD panorama that is not fetched again as a flat photo. Co-Authored-By: Claude Opus 5.5 --- src/layers/streetLevel/lineCoverage.js | 16 +++++--- src/layers/streetLevel/lineCoverage.test.mjs | 40 +++++++++++++++++++ src/layers/streetLevel/photoViewer.js | 34 ++++++++++++---- src/layers/streetLevel/photoViewer.test.mjs | 19 +++++++-- .../providers/panoramax/viewer.test.mjs | 36 +++++++++++++++++ 5 files changed, 129 insertions(+), 16 deletions(-) diff --git a/src/layers/streetLevel/lineCoverage.js b/src/layers/streetLevel/lineCoverage.js index 78c9b36da..ab9b996db 100644 --- a/src/layers/streetLevel/lineCoverage.js +++ b/src/layers/streetLevel/lineCoverage.js @@ -152,9 +152,10 @@ export function createLineCoverage({ /** * A tile's sequences, filtered then capped: capping first could leave a - * dense tile of newer flat captures with no 360° lines at all. + * dense tile of newer flat captures with no 360° lines at all. `measure` + * only records what the lines need, for a draw that follows. */ - function drawnSequences(entry) { + function drawnSequences(entry, { measure = false } = {}) { const drawn = []; const current = filter(); const selectedId = state.sequence.selectedId ?? null; @@ -180,7 +181,12 @@ export function createLineCoverage({ } // What every passing line would take: the budget is shared by it. entry.pointDemand = demand; - entry.pointsDrawn = points; + if (!measure) { + entry.pointsDrawn = points; + // The cap this draw used: a share is judged against it, not against + // later shares that redrew nothing (small steps add up). + entry.drawnCap = cap; + } return drawn; } @@ -204,7 +210,7 @@ export function createLineCoverage({ const share = left / (byDemand.length - i); const cap = Math.floor(Math.min(entry.pointDemand || 0, share)); left -= cap; - const before = entry.pointCap ?? perTilePointCap; + const before = entry.drawnCap ?? perTilePointCap; entry.pointCap = cap; // Only a change that alters what is drawn, by enough to be worth a // rebuild: more room for a tile that had lines left out, or well @@ -899,7 +905,7 @@ export function createLineCoverage({ // What each tile's lines need under the new filter, then the shares, // then one redraw each. for (const entry of state.coverage.tiles.values()) - if (entry.kind === 'sequence') drawnSequences(entry); + if (entry.kind === 'sequence') drawnSequences(entry, { measure: true }); shareCaps(); for (const entry of state.coverage.tiles.values()) redrawEntry(entry); requestRender(); diff --git a/src/layers/streetLevel/lineCoverage.test.mjs b/src/layers/streetLevel/lineCoverage.test.mjs index db89e8143..b0b5f7e94 100644 --- a/src/layers/streetLevel/lineCoverage.test.mjs +++ b/src/layers/streetLevel/lineCoverage.test.mjs @@ -433,3 +433,43 @@ test('a selected line outside the cap does not make its tile redraw on every set for (let i = 0; i < 3; i++) coverage.recolorSequence('short', true); assert.equal(after.added, 0); }); + +test('small share increases add up: a retained tile redraws once its share has grown past the threshold since its last draw', async (t) => { + const tuning = { + ...TUNING, + maxTiles: 1, + maxSequences: 900, + maxPoints: 100, + keepOrder: true, + }; + const { state, coverage } = setup(t, { + tuning, + decoded: (centre, tile) => ({ + sequences: Array.from({ length: 30 }, (_, i) => + pointsLine(`l${i}`, 4, tile, centre), + ), + overview: [], + }), + }); + coverage.refresh(); + assert.ok( + await until( + () => state.coverage.tiles.size === 1 && !state.coverage.pending.size, + ), + ); + const [tile] = state.coverage.tiles.values(); + assert.equal(tile.pointsDrawn, 100); + const settle = async () => { + coverage.refresh(); + await until(() => !state.coverage.pending.size); + await new Promise((resolve) => setTimeout(resolve, 10)); + }; + // +8%: under the threshold, nothing is redrawn. + tuning.maxPoints = 108; + await settle(); + assert.equal(tile.pointsDrawn, 100); + // +8% again: +16% since the tile was last drawn, so it is redrawn. + tuning.maxPoints = 116; + await settle(); + assert.equal(tile.pointsDrawn, 116, 'the grown share is drawn'); +}); diff --git a/src/layers/streetLevel/photoViewer.js b/src/layers/streetLevel/photoViewer.js index a7e86cb42..3de2c1af7 100644 --- a/src/layers/streetLevel/photoViewer.js +++ b/src/layers/streetLevel/photoViewer.js @@ -78,7 +78,7 @@ async function loadBitmap(url, timeoutMs, resize, failure, signal) { // Picture hosts are not ours: refuse a file or an image past any real // photo's size before it is decoded. const length = Number(response.headers?.get?.('content-length')); - if (length > MAX_IMAGE_BYTES) throw new Error(failure); + if (length > MAX_IMAGE_BYTES) throw oversized(failure); const blob = await boundedBlob(response, failure); const bitmap = await (resize ? createImageBitmap(blob, { @@ -90,10 +90,12 @@ async function loadBitmap(url, timeoutMs, resize, failure, signal) { // (A resized decode reports its new size: this guards unresized ones.) if (bitmap.width > MAX_IMAGE_SIDE || bitmap.height > MAX_IMAGE_SIDE) { bitmap.close?.(); - throw new Error(failure); + throw oversized(failure); } return bitmap; - } catch { + } catch (error) { + // A size refusal stays one: the viewer must not fetch it another way. + if (error?.tooLarge) throw error; throw new Error(failure); } finally { clearTimeout(timer); @@ -101,6 +103,14 @@ async function loadBitmap(url, timeoutMs, resize, failure, signal) { } } +/** + * A picture refused for its size (`tooLarge`): the viewer will not load + * that file another way, an having no size limit at all. + */ +export function oversized(failure) { + return Object.assign(new Error(failure), { tooLarge: true }); +} + /** * The body as a Blob, read with a running count so a host that sends no * length (or a false one) cannot fill memory before the cap is checked. @@ -109,7 +119,7 @@ async function boundedBlob(response, failure) { const reader = response.body?.getReader?.(); if (!reader) { const blob = await response.blob(); - if (blob.size > MAX_IMAGE_BYTES) throw new Error(failure); + if (blob.size > MAX_IMAGE_BYTES) throw oversized(failure); return blob; } const chunks = []; @@ -120,7 +130,7 @@ async function boundedBlob(response, failure) { total += value.byteLength; if (total > MAX_IMAGE_BYTES) { reader.cancel().catch(() => {}); - throw new Error(failure); + throw oversized(failure); } chunks.push(value); } @@ -226,8 +236,17 @@ export function createPhotoViewer({ /** ‹ › presses run one after another, each from where the last one went. */ let steps = Promise.resolve(); - const loadAs = (url, options) => - load(url, { failure: LOAD_FAILED, ...options }); + /** Files refused for their size: never loaded again, by any path. */ + const refused = new Set(); + async function loadAs(url, options) { + if (refused.has(url)) throw oversized(LOAD_FAILED); + try { + return await load(url, { failure: LOAD_FAILED, ...options }); + } catch (error) { + if (error?.tooLarge) refused.add(url); + throw error; + } + } function requestRender() { render?.governorRequestRender?.(`${providerId}-viewer`); @@ -796,6 +815,7 @@ export function createPhotoViewer({ unmount() { openSeq++; dropSharpPanorama(); + refused.clear(); openingId = null; keepViewFor = null; flatSharpUrl = null; diff --git a/src/layers/streetLevel/photoViewer.test.mjs b/src/layers/streetLevel/photoViewer.test.mjs index 05391a3cc..758a1e25b 100644 --- a/src/layers/streetLevel/photoViewer.test.mjs +++ b/src/layers/streetLevel/photoViewer.test.mjs @@ -296,12 +296,17 @@ test('a picture file or bitmap past any real photo’s size is refused', async ( headers: new Map([['content-length', String(length ?? '')]]), blob: async () => ({ size: size ?? 10, bitmap: bitmap(width, height) }), })); + const refusedForSize = (error) => { + assert.match(error.message, /could not be loaded/); + assert.equal(error.tooLarge, true, 'marked as a size refusal'); + return true; + }; respond({ length: MAX_IMAGE_BYTES + 1 }); - await assert.rejects(loadImage('https://x/a.jpg'), /could not be loaded/); + await assert.rejects(loadImage('https://x/a.jpg'), refusedForSize); respond({ size: MAX_IMAGE_BYTES + 1 }); - await assert.rejects(loadImage('https://x/a.jpg'), /could not be loaded/); + await assert.rejects(loadImage('https://x/a.jpg'), refusedForSize); respond({ width: MAX_IMAGE_SIDE + 1 }); - await assert.rejects(loadImage('https://x/a.jpg'), /could not be loaded/); + await assert.rejects(loadImage('https://x/a.jpg'), refusedForSize); assert.equal(closed, 1, 'the oversize bitmap is released'); // No length given, a body that keeps coming: cut off while it streams. let pulled = 0; @@ -315,8 +320,14 @@ test('a picture file or bitmap past any real photo’s size is refused', async ( }, }), }); - await assert.rejects(loadImage('https://x/a.jpg'), /could not be loaded/); + await assert.rejects(loadImage('https://x/a.jpg'), refusedForSize); assert.ok(pulled * 8 * 1024 * 1024 <= MAX_IMAGE_BYTES + 16 * 1024 * 1024); + // Other failures are not size refusals. + globalThis.fetch = async () => ({ ok: false, headers: new Map() }); + await assert.rejects( + loadImage('https://x/a.jpg'), + (error) => !error.tooLarge, + ); respond({}); assert.equal((await loadImage('https://x/a.jpg')).width, 100); }); diff --git a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs index 78c39f316..614a50e13 100644 --- a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs @@ -295,3 +295,39 @@ test('a photo with no author to name is captioned with the instance holding it', const named = { ...photo, creator: 'Robot8A' }; assert.equal(PANORAMAX_SOURCES.describe(named).title, null); }); + +test('an SD panorama refused for its size is not fetched again as a flat photo', async () => { + const loads = []; + const viewer = createPanoramaxViewer({ + getPhoto: async () => PHOTOS.pano, + step: async () => null, + onStep() {}, + load: async (url, options = {}) => { + loads.push({ url, crossOrigin: options.crossOrigin }); + // The picture host's SD file is past the viewer's size limits. + if (url === PHOTOS.pano.urls.sd) + throw Object.assign(new Error('too big'), { tooLarge: true }); + return { src: url }; + }, + createRenderer: () => ({ + setImage() {}, + setView() {}, + getView: () => ({ yaw: 0, pitch: 0, fov: 80 }), + resize() {}, + clear() {}, + destroy() {}, + }), + }); + const host = fakeHost(); + await viewer.mount(host); + await viewer.open('pano'); + const sdLoads = loads.filter((l) => l.url === PHOTOS.pano.urls.sd); + assert.equal(sdLoads.length, 1, 'tried once, as a bitmap, and refused'); + assert.equal(sdLoads[0].crossOrigin, true); + const { picture } = parts(host); + assert.equal(picture.hidden, false, 'the flat fallback shows'); + assert.notEqual(picture.src, PHOTOS.pano.urls.sd); + // Opened again later: the refused file is still not asked for. + await viewer.open('pano'); + assert.equal(loads.filter((l) => l.url === PHOTOS.pano.urls.sd).length, 1); +}); From b645a767c828e4c3dd8c50d23b9e1e2a5c5bccd8 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Thu, 15 Oct 2026 16:55:59 +0100 Subject: [PATCH 61/95] Street Level: Panoramax review follow-ups (gekh, #1134) Image loading (photoViewer.js) - One AbortController per open: a new open, close or unmount stops the previous photo's first and sharper downloads (an by src = ''). A collapsed viewer never falls back to the original; Panoramax's flat fallback is the thumbnail. A superseded open resolves quietly. - Flat HD tracked as "loading" and "shown" apart: a discarded response no longer blocks later upgrades, and resizes while it loads ask once. - No upgrade of the photo still shown while another opens. - Size limits before decoding: imageSize.js reads JPEG/PNG/WebP/GIF headers; more than 64 MP or 16,384 px a side is refused undecoded, an that decodes larger is refused and let go, flat HD needs a known width of at most 8,192 px, and panoramas always decode with a resize. Panoramax provider - Retry-After parsed as seconds or an HTTP date; the browser cannot read it here, so busy answers back off 5 s doubling to 300 s, reset once answered. - Headers timed, then the body by an idle timeout (15 s of silence), so a slow link still gets its megabyte tiles. - Cones centre only on the picture shown (onShowing), never on a late lookup; lookups nobody waits for are aborted (shared, refcounted). - onReopen: a GPU reset redraws in the viewer, not through the core. - Cones draw the home z15 tile at once and add neighbours as they arrive; a neighbour that failed is not counted as drawn. - Coverage tiles decode in ~8 ms slices (a 2 MB z10 tile over Paris was one ~85-200 ms task); sorted by measured length without a Map. - Box search asks for 20, not 100 (the API returns them nearest first). - looksLikeEmail checks every '@'; creatorOf de-duplicates with a Set. lineCoverage.js - A redraw while a swap still builds keeps what is on screen (the swap's old lines and its ready new ones) and drops only the unready ones. Tests for each (mutation-checked), two vacuous tests made real, the cones' pictures lane and readable()'s busy/timeout paths covered; the fixture's box search is nearest first. Docs: README filters and share links, CHANGELOG HD wording, CURRENT-STATE. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 +- README.md | 4 +- docs/CURRENT-STATE.md | 39 ++- scripts/fixtures/street-level/panoramax.mjs | 9 +- scripts/format-scope.json | 2 + scripts/package-boundaries.json | 3 + src/app/layers/streetLevel.test.mjs | 5 + src/layers/streetLevel/imageSize.js | 64 +++++ src/layers/streetLevel/imageSize.test.mjs | 82 ++++++ src/layers/streetLevel/lineCoverage.js | 18 +- src/layers/streetLevel/lineCoverage.test.mjs | 57 ++++ src/layers/streetLevel/photoViewer.js | 128 ++++++--- src/layers/streetLevel/photoViewer.test.mjs | 207 ++++++++++++++- .../streetLevel/providers/kartaview/viewer.js | 1 + .../providers/kartaview/viewer.test.mjs | 9 +- .../streetLevel/providers/panoramax/decode.js | 64 ++++- .../providers/panoramax/decode.test.mjs | 84 +++++- .../streetLevel/providers/panoramax/index.js | 136 ++++++++-- .../streetLevel/providers/panoramax/policy.js | 37 ++- .../providers/panoramax/policy.test.mjs | 17 ++ .../providers/panoramax/provider.test.mjs | 243 +++++++++++++++++- .../streetLevel/providers/panoramax/source.js | 81 ++++-- .../providers/panoramax/source.test.mjs | 155 ++++++++++- .../streetLevel/providers/panoramax/viewer.js | 23 +- .../providers/panoramax/viewer.test.mjs | 16 +- 25 files changed, 1362 insertions(+), 135 deletions(-) create mode 100644 src/layers/streetLevel/imageSize.js create mode 100644 src/layers/streetLevel/imageSize.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index df6497222..a7db1c6a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -153,13 +153,18 @@ ← → following the picture's own links. The 360° and since filters apply, in the lines and in the lookup. The caption names each picture's author and licence (CC BY-SA 4.0, Licence Ouverte 2.0…) and links to it in the - Panoramax viewer; HD originals load only for an expanded viewer, once. - Share links carry it as `0.m.0_0.r.panoramax-1`. KartaView's photo viewer + Panoramax viewer. HD originals load once, only when the viewer is wider + than the 2,048 px version (a panorama: expanded), and every download stops + when the photo changes; an image whose header declares more than 64 MP is + refused before it decodes. Share links carry it as + `0.m.0_0.r.google-0*kartaview-0*panoramax-1` (`0.m.0_0.r.panoramax-1` works + too). KartaView's photo viewer is now shared (`photoViewer.js`, `panorama.js`); in it, for KartaView too, Esc clears the selected drive before it collapses the panel. Lines are simplified and drawn within a point budget (a tile's unsimplified lines - over Paris hung the tab), decoded tiles are kept, and the cones' tiles - have their own connection. + over Paris hung the tab), decoded in slices so the globe keeps moving, + and kept; the cones' tiles have their own connection, and the open + picture's tile draws its cones before its neighbours arrive. - Flights and Military Flights highlight emergency transponder broadcasts in fleet billboards, 3D models and Cockpit contacts. Colors update when a status diff --git a/README.md b/README.md index ee19953ab..554f9b57f 100644 --- a/README.md +++ b/README.md @@ -354,9 +354,9 @@ _The Space Missions layer replaying a Falcon 9 ascent — labeled `RECONSTRUCTED **Setup.** At [mapillary.com/dashboard/developers](https://www.mapillary.com/dashboard/developers), register an application with **Read** access and copy its **Client Token** (`MLY|…`). Paste it into **POWER UP → Mapillary**, or set `MAPILLARY_CLIENT_TOKEN` in `.env` and restart. Without it the panel reads **KEY REQUIRED**. **Google Street View** uses `GOOGLE_MAPS_API_KEY` with the **Maps JavaScript API** enabled for it in Google Cloud; its chip starts off, since every panorama is billed to the key. **KartaView** and **Panoramax** need nothing: their public APIs are keyless. -**Use.** Turn on **Street Level** under DATA LAYERS → Cameras, or with the pill in the STREET LEVEL panel header. Each provider has a chip in its own colour (Mapillary green), and one is lit at a time: lighting one switches the others off. Mapillary, which is free, is lit by default. The **IMAGERY** (ALL / 360° / FLAT) and **SINCE** filters apply to Mapillary and KartaView; Street View panoramas are all 360°, so with Street View lit the filters and legend are hidden. Click a coverage line for its image cones and a cone to open the photo. Light **KARTAVIEW** (purple) and zoom in below about 6 km: its lines are the streets KartaView has photos of, and a click on one opens the photo nearest the click, flat or 360° (drag to look around), with cones along its drive; **‹ ›** or ← → step through the drive (Shift+← →, ↑ ↓ and + − look around a 360° photo; pinch zooms on a phone). Photos KartaView has archived are skipped, and where all are archived the panel says so. For KartaView, **SINCE** goes by upload date, as KartaView's own map does (it can filter by nothing else), and the panel notes it under the slider; KartaView filters its lines on its own server, and a filtered area KartaView has not drawn before can take several seconds, and the unfiltered lines show meanwhile. Light **PANORAMAX** (orange) for the open, federated commons (strongest in France and Belgium): from orbit its dots show where it has pictures, and below about 60 km its lines are sequences. A click on a line opens that sequence's picture nearest the click without moving the camera, flat or 360°, with cones along the sequence around it; **‹ ›** or ← → step through it, and both filters apply. Its caption names the picture's authors (or the instance holding it) and licence (mostly CC BY-SA 4.0; pictures on the IGN instance are Licence Ouverte 2.0). On the **Google 3D** map (Google's terms keep Street View off other maps, where it does not appear), light **GOOGLE STREET VIEW** and click a street from below 3 km, or press **OPEN STREET VIEW** for the view centre, to open Google's panorama there; step along its arrows. Above the photo: **EXPAND**, **FIT / FILL** (Mapillary, and flat KartaView and Panoramax photos), and **FOLLOW** (the globe camera follows the photo's view; Google 3D map only). The caption links to the image on the provider's site, and the globe credits each provider that is on ("© Mapillary contributors, CC BY-SA 4.0", "Street View imagery © Google", "© KartaView contributors, CC BY-SA 4.0", "© Panoramax contributors, licence per picture"). KartaView's own photo pages fail for signed-out visitors, so its link opens the full-size photo. +**Use.** Turn on **Street Level** under DATA LAYERS → Cameras, or with the pill in the STREET LEVEL panel header. Each provider has a chip in its own colour (Mapillary green), and one is lit at a time: lighting one switches the others off. Mapillary, which is free, is lit by default. The **IMAGERY** (ALL / 360° / FLAT) and **SINCE** filters apply to Mapillary, KartaView and Panoramax; Street View panoramas are all 360°, so with Street View lit the filters and legend are hidden. Click a coverage line for its image cones and a cone to open the photo. Light **KARTAVIEW** (purple) and zoom in below about 6 km: its lines are the streets KartaView has photos of, and a click on one opens the photo nearest the click, flat or 360° (drag to look around), with cones along its drive; **‹ ›** or ← → step through the drive (Shift+← →, ↑ ↓ and + − look around a 360° photo; pinch zooms on a phone). Photos KartaView has archived are skipped, and where all are archived the panel says so. For KartaView, **SINCE** goes by upload date, as KartaView's own map does (it can filter by nothing else), and the panel notes it under the slider; KartaView filters its lines on its own server, and a filtered area KartaView has not drawn before can take several seconds, and the unfiltered lines show meanwhile. Light **PANORAMAX** (orange) for the open, federated commons (strongest in France and Belgium): from orbit its dots show where it has pictures, and below about 60 km its lines are sequences. A click on a line opens that sequence's picture nearest the click without moving the camera, flat or 360°, with cones along the sequence around it; **‹ ›** or ← → step through it, and both filters apply. Its caption names the picture's authors (or the instance holding it) and licence (mostly CC BY-SA 4.0; pictures on the IGN instance are Licence Ouverte 2.0). On the **Google 3D** map (Google's terms keep Street View off other maps, where it does not appear), light **GOOGLE STREET VIEW** and click a street from below 3 km, or press **OPEN STREET VIEW** for the view centre, to open Google's panorama there; step along its arrows. Above the photo: **EXPAND**, **FIT / FILL** (Mapillary, and flat KartaView and Panoramax photos), and **FOLLOW** (the globe camera follows the photo's view; Google 3D map only). The caption links to the image on the provider's site, and the globe credits each provider that is on ("© Mapillary contributors, CC BY-SA 4.0", "Street View imagery © Google", "© KartaView contributors, CC BY-SA 4.0", "© Panoramax contributors, licence per picture"). KartaView's own photo pages fail for signed-out visitors, so its link opens the full-size photo. -Drag the panel header to float the panel, and resize it from any edge. Double-click the header or collapse it to put it back in the rail. Share links carry the provider switches and filters (`0.m.0` Mapillary off, `0.m.0_0.r.google-1` Street View on, `0.m.0_0.r.google-0*kartaview-1` KartaView on, as the app writes it (`0.m.0_0.r.kartaview-1` works too, and `0.m.0_0.r.panoramax-1` lights Panoramax; a link that leaves out `m` keeps Mapillary, the default, lit), `0.p.p` panoramas only, `0.s.365` the last year). To add a provider, implement the contract in `src/layers/streetLevel/registry.js` and register it in `src/app/layers/streetLevel.js`. +Drag the panel header to float the panel, and resize it from any edge. Double-click the header or collapse it to put it back in the rail. Share links carry the provider switches and filters (`0.m.0` Mapillary off, `0.m.0_0.r.google-1` Street View on, `0.m.0_0.r.google-0*kartaview-1*panoramax-0` KartaView on and `0.m.0_0.r.google-0*kartaview-0*panoramax-1` Panoramax on, as the app writes them (`0.m.0_0.r.kartaview-1` and `0.m.0_0.r.panoramax-1` work too; a link that leaves out `m` keeps Mapillary, the default, lit), `0.p.p` panoramas only, `0.s.365` the last year). To add a provider, implement the contract in `src/layers/streetLevel/registry.js` and register it in `src/app/layers/streetLevel.js`. ## 🎖️ Field Missions diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 8e5bff676..28c59191a 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3403,7 +3403,13 @@ provider is on (16, ten minutes, `tileCache.js`, shared with KartaView), and above 60 km the `grid` layer's 0.1° density cells, binned to at most 96×96 points a tile (a z0 tile holds -~70,000 cells) and not counted as sequences: the panel says to zoom in. Both +~70,000 cells) and not counted as sequences: the panel says to zoom in. +A tile decodes in slices of about 8 ms, the page getting a turn between +them (a 2 MB z10 tile is ~200 ms of work). A busy API (429/503) holds the +tiles back for its Retry-After, which the browser cannot read here (the API +exposes no headers), so the wait doubles from 5 s to 5 minutes until it +answers; an answer may take as long as it keeps arriving (silent for 15 s +is given up), so a slow link still gets its megabyte tiles. Both filters apply: lines carry their day and camera type, and a cell stays while it holds pictures of the kind asked for (cells are undated). A click on a line asks the core (`actions.openNearest`) for that sequence's picture @@ -3412,8 +3418,8 @@ a click lands well off a line), without framing, then any sequence's within 50 m. Each lookup is two searches side by side, ranked by distance in the browser: `place_position` (nearest first, but even at `place_fov_tolerance=180` it leaves out pictures looking away from the -point) and a bounding box (those too, but in no order); the filters go to -both (`filter=field_of_view=360`, `datetime=/..`). A line is dated +point) and a bounding box (those too, roughly nearest first, so 20 are +asked for: ~60 kB, against ~600 kB for 100); the filters go to both (`filter=field_of_view=360`, `datetime=/..`). A line is dated by its sequence's first day, so a sequence still growing can hide under SINCE; a line with no camera type or day stays drawn. Lines keep the decoded longest-first order under the per-tile cap (`keepOrder`). Once the picture is on screen @@ -3421,20 +3427,31 @@ its sequence is highlighted and drawn as cones from the z15 tiles around it (their `pictures` layer, ~110 bytes a picture, against ~7.5 kB for a STAC item), plus the neighbours within 150 m of an edge, on their own connection (coverage tiles take three, searches two); each tile's list for the sequence -is kept, and the cones re-centre when a picture opens outside the tiles -they were drawn from, judged from the moment their load starts. ‹ › follow the picture's own `prev`/`next` +is kept. The open picture's own tile draws at once and the neighbours join +as they arrive; one that fails is not counted, so a picture opening in it +re-centres the cones there. The cones re-centre when a picture opens outside the tiles +they were drawn from, judged from the moment their load starts, and centre +only on a picture actually shown (never on a lookup that came back after +the user moved on); a lookup nobody waits for any more is stopped. ‹ › follow the picture's own `prev`/`next` links, so stepping costs one small search per picture. The viewer shows the -2,048 px SD derivative (equirectangular for a 360° picture) and swaps in the -HD original only when the viewer is wider than SD (flat) or, for a -panorama, expanded; once, shrunk while it decodes, and aborted if another -photo opens; originals wider than 8,192 px are never fetched, and any -picture file over 48 MB or 16,384 px is refused. The caption names every +2,048 px SD derivative (equirectangular for a 360° picture, decoded at +2,048 × 1,024) and swaps in the HD original only when the viewer is wider +than SD (flat) or, for a panorama, expanded; once, shrunk while it decodes. +When SD fails a flat picture falls back to the thumbnail, and to the +original only in the expanded viewer. Every download of a photo, first or +sharper, stops when another opens or the viewer closes. Originals wider than +8,192 px are never fetched; a file's header (JPEG, PNG, WebP, GIF) is read +before it decodes, and one declaring more than 64 MP or 16,384 px a side +is refused undecoded, as is an `` that decodes larger and any file +over 48 MB. The caption names every producer (an uploading app and its photographer; never an account named by an email address), or else the instance holding the picture ("via panoramax.ign.fr"), and the picture's licence (CC BY-SA 4.0, Licence Ouverte 2.0…; StreetPose `license`), and links to the picture in the Panoramax viewer. Esc anywhere in the docked viewer clears the selected -sequence before the panel can take it (KartaView too). The MCP globe panel allows the meta-catalogue and the +sequence before the panel can take it (KartaView too). After a GPU reset +the viewer draws the same picture again itself (`onReopen`), leaving the +core's lookups alone. The MCP globe panel allows the meta-catalogue and the three biggest instances; a picture on another instance shows only in the app. diff --git a/scripts/fixtures/street-level/panoramax.mjs b/scripts/fixtures/street-level/panoramax.mjs index 7506eb806..acca30e0f 100644 --- a/scripts/fixtures/street-level/panoramax.mjs +++ b/scripts/fixtures/street-level/panoramax.mjs @@ -249,9 +249,11 @@ export function answerPanoramaxRequest({ method, url }) { : true, ) .filter((p) => !since || Date.parse(p.datetime) >= Date.parse(since)); - // A box: every picture in it, in no particular order (id order here). + // A box: every picture in it, nearest its centre first, as the live API + // returns them (roughly: the browser ranks them again). if (q.has('bbox')) { const [west, south, east, north] = q.get('bbox').split(',').map(Number); + const centre = { lon: (west + east) / 2, lat: (south + north) / 2 }; return json({ type: 'FeatureCollection', features: kept @@ -259,9 +261,10 @@ export function answerPanoramaxRequest({ method, url }) { (p) => p.lon >= west && p.lon <= east && p.lat >= south && p.lat <= north, ) - .sort((a, b) => a.id.localeCompare(b.id)) + .map((p) => ({ p, metres: metresApart(centre, p) })) + .sort((a, b) => a.metres - b.metres || a.p.id.localeCompare(b.p.id)) .slice(0, Number(q.get('limit')) || 10) - .map((p) => item(p, pictures)), + .map(({ p }) => item(p, pictures)), }); } const [lon, lat] = (q.get('place_position') || '').split(',').map(Number); diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 1eb4f7014..f8c76fea4 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -943,6 +943,7 @@ "src/layers/streetLevel/providers/panoramax/viewer.js", "src/layers/streetLevel/photoViewer.js", "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/imageSize.js", "src/layers/streetLevel/tileCache.js", "src/layers/streetLevel/providers/panoramax/policy.test.mjs", "src/layers/streetLevel/providers/panoramax/decode.test.mjs", @@ -952,6 +953,7 @@ "src/testSupport/panoramaxFixtures.mjs", "src/layers/streetLevel/photoViewer.test.mjs", "src/layers/streetLevel/limiter.test.mjs", + "src/layers/streetLevel/imageSize.test.mjs", "src/layers/streetLevel/lineCoverage.test.mjs", "src/testSupport/kartaViewFakes.mjs", "src/testSupport/googleStreetViewFakes.mjs", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 98e8d17ec..e7b494fba 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1244,6 +1244,7 @@ "src/layers/streetLevel/providers/panoramax/viewer.js", "src/layers/streetLevel/photoViewer.js", "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/imageSize.js", "src/layers/streetLevel/tileCache.js", "src/layers/streetLevel/panorama.js", "src/layers/streetLevel/lineCoverage.js", @@ -2225,6 +2226,7 @@ "src/layers/streetLevel/providers/panoramax/viewer.js", "src/layers/streetLevel/photoViewer.js", "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/imageSize.js", "src/layers/streetLevel/tileCache.js", "src/layers/streetLevel/panorama.js", "src/layers/streetLevel/lineCoverage.js", @@ -2777,6 +2779,7 @@ "src/layers/streetLevel/providers/panoramax/viewer.js", "src/layers/streetLevel/photoViewer.js", "src/layers/streetLevel/limiter.js", + "src/layers/streetLevel/imageSize.js", "src/layers/streetLevel/tileCache.js", "src/layers/streetLevel/panorama.js", "src/layers/streetLevel/lineCoverage.js", diff --git a/src/app/layers/streetLevel.test.mjs b/src/app/layers/streetLevel.test.mjs index 16e2cc4b5..e82e762e5 100644 --- a/src/app/layers/streetLevel.test.mjs +++ b/src/app/layers/streetLevel.test.mjs @@ -64,6 +64,11 @@ test('the defaults are Mapillary (given its source), Street View, KartaView and assert.equal(layer.getParams().panoramax, true); assert.equal(layer.getParams().kartaview, false); assert.equal(layer.getParams().mapillary, false); + // The form the app writes (the README and CHANGELOG quote it). + assert.equal( + layer.getParams().providerSwitches, + 'google-0*kartaview-0*panoramax-1', + ); // Without the application's Street Level source there are none. assert.deepEqual(createApplicationStreetLevel({}).providerIds, []); }); diff --git a/src/layers/streetLevel/imageSize.js b/src/layers/streetLevel/imageSize.js new file mode 100644 index 000000000..dc1131495 --- /dev/null +++ b/src/layers/streetLevel/imageSize.js @@ -0,0 +1,64 @@ +/** + * An image's pixel size read from its header, before anything decodes it: + * a picture host can serve a small file that declares enormous dimensions, + * and a decode of it alone can take the tab (or the GPU process) down. + * JPEG (SOFn), PNG (IHDR), WebP (VP8, VP8L, VP8X) and GIF; null for any + * other format, or a header this does not reach. + * @param {Uint8Array} bytes The start of the file (a JPEG's EXIF can push its SOF past 64 kB). + * @returns {{width: number, height: number}|null} + */ +export function imageSize(bytes) { + if (!bytes || bytes.length < 12) return null; + const b = bytes; + const be16 = (i) => (b[i] << 8) | b[i + 1]; + const le16 = (i) => b[i] | (b[i + 1] << 8); + const le24 = (i) => b[i] | (b[i + 1] << 8) | (b[i + 2] << 16); + const ascii = (i, text) => + [...text].every((char, k) => b[i + k] === char.charCodeAt(0)); + // JPEG: walk the markers to the first start-of-frame. + if (b[0] === 0xff && b[1] === 0xd8) { + let i = 2; + while (i + 9 < b.length) { + if (b[i] !== 0xff) return null; + const marker = b[i + 1]; + // Fill bytes, and markers with no length. + if (marker === 0xff) { + i++; + continue; + } + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd9)) { + i += 2; + continue; + } + const isFrame = + marker >= 0xc0 && + marker <= 0xcf && + marker !== 0xc4 && + marker !== 0xc8 && + marker !== 0xcc; + if (isFrame) return { width: be16(i + 7), height: be16(i + 5) }; + i += 2 + be16(i + 2); + } + return null; + } + // PNG: IHDR is the first chunk. + if (b[0] === 0x89 && ascii(1, 'PNG') && ascii(12, 'IHDR')) { + const u32 = (i) => + ((b[i] << 24) | (b[i + 1] << 16) | (b[i + 2] << 8) | b[i + 3]) >>> 0; + return { width: u32(16), height: u32(20) }; + } + if (ascii(0, 'RIFF') && ascii(8, 'WEBP') && b.length >= 30) { + if (ascii(12, 'VP8 ')) + return { width: le16(26) & 0x3fff, height: le16(28) & 0x3fff }; + if (ascii(12, 'VP8L')) + return { + width: 1 + (((b[22] & 0x3f) << 8) | b[21]), + height: + 1 + (((b[24] & 0x0f) << 10) | (b[23] << 2) | ((b[22] & 0xc0) >> 6)), + }; + if (ascii(12, 'VP8X')) return { width: 1 + le24(24), height: 1 + le24(27) }; + return null; + } + if (ascii(0, 'GIF8')) return { width: le16(6), height: le16(8) }; + return null; +} diff --git a/src/layers/streetLevel/imageSize.test.mjs b/src/layers/streetLevel/imageSize.test.mjs new file mode 100644 index 000000000..e83978596 --- /dev/null +++ b/src/layers/streetLevel/imageSize.test.mjs @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { imageSize } from './imageSize.js'; + +const bytes = (...parts) => + Uint8Array.from( + parts.flat().map((v) => (typeof v === 'string' ? v.charCodeAt(0) : v)), + ); +const ascii = (text) => [...text]; + +test('a JPEG’s size is read from its first frame, past an EXIF segment', () => { + const exif = [0xff, 0xe1, 0x00, 0x10, ...new Array(14).fill(0)]; + const sof = [0xff, 0xc0, 0x00, 0x11, 0x08, 0x0b, 0xc0, 0x0f, 0xc0]; + const jpeg = bytes([0xff, 0xd8], exif, sof, new Array(20).fill(0)); + assert.deepEqual(imageSize(jpeg), { width: 0x0fc0, height: 0x0bc0 }); + // Progressive (SOF2) too; a DHT (C4) is not a frame. + const dht = [0xff, 0xc4, 0x00, 0x04, 0, 0]; + const sof2 = [0xff, 0xc2, 0x00, 0x11, 0x08, 0x00, 0x10, 0x00, 0x20]; + assert.deepEqual( + imageSize(bytes([0xff, 0xd8], dht, sof2, new Array(10).fill(0))), + { + width: 32, + height: 16, + }, + ); +}); + +test('PNG, WebP (lossy, lossless, extended) and GIF sizes', () => { + const png = bytes( + [0x89], + ascii('PNG'), + [0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13], + ascii('IHDR'), + [0, 0, 0x10, 0, 0, 0, 0x08, 0], + ); + assert.deepEqual(imageSize(png), { width: 4096, height: 2048 }); + const riff = (chunk, rest) => + bytes(ascii('RIFF'), [0, 0, 0, 0], ascii('WEBP'), ascii(chunk), rest); + const vp8 = riff('VP8 ', [...new Array(10).fill(0), 0x00, 0x08, 0x00, 0x04]); + assert.deepEqual(imageSize(vp8), { width: 2048, height: 1024 }); + // VP8L: 14-bit width-1 and height-1 packed after the 0x2f signature. + const w = 99; + const h = 49; + const packed = w | (h << 14); + const vp8l = riff('VP8L', [ + 0, + 0, + 0, + 0, + 0x2f, + packed & 0xff, + (packed >> 8) & 0xff, + (packed >> 16) & 0xff, + (packed >> 24) & 0xff, + 0, + 0, + 0, + 0, + 0, + 0, + ]); + assert.deepEqual(imageSize(vp8l), { width: 100, height: 50 }); + const vp8x = riff( + 'VP8X', + [0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0x0f, 0x00, 0xff, 0x07, 0x00], + ); + assert.deepEqual(imageSize(vp8x), { width: 4096, height: 2048 }); + const gif = bytes(ascii('GIF89a'), [0x40, 0x01, 0xf0, 0x00], [0, 0]); + assert.deepEqual(imageSize(gif), { width: 320, height: 240 }); +}); + +test('anything else, or a header cut short, has no size', () => { + assert.equal(imageSize(null), null); + assert.equal(imageSize(new Uint8Array(4)), null); + assert.equal(imageSize(bytes(ascii(''))), null); + assert.equal( + imageSize( + bytes([0xff, 0xd8, 0xff, 0xe1, 0xff, 0xff], new Array(8).fill(0)), + ), + null, + ); +}); diff --git a/src/layers/streetLevel/lineCoverage.js b/src/layers/streetLevel/lineCoverage.js index ab9b996db..d57a98bd0 100644 --- a/src/layers/streetLevel/lineCoverage.js +++ b/src/layers/streetLevel/lineCoverage.js @@ -247,7 +247,6 @@ export function createLineCoverage({ entry.castRequested = false; entry.meshParts = null; if (swap && entry.kind === 'sequence' && entry.primitives?.length) { - finishSwap(entry); redrawTile(entry); return; } @@ -373,7 +372,22 @@ export function createLineCoverage({ /** Rebuild a tile's lines; the old ones stay up until the new are ready. */ function redrawTile(entry) { - const previous = entry.primitives; + let previous = entry.primitives; + if (entry.swap) { + // A swap still building: what is on screen is its old lines and + // whichever new ones are ready. Those stay; the rest never showed. + const shown = [...entry.swap.old]; + const building = []; + for (const record of previous) + (record.primitive.ready && !record.primitive.isDestroyed?.() + ? shown + : building + ).push(record); + entry.swap.stop(); + entry.swap = null; + removePrimitives(building); + previous = shown; + } entry.primitives = []; attachPrimitive(entry); removeWhenReady(entry, previous); diff --git a/src/layers/streetLevel/lineCoverage.test.mjs b/src/layers/streetLevel/lineCoverage.test.mjs index b0b5f7e94..7fdc19773 100644 --- a/src/layers/streetLevel/lineCoverage.test.mjs +++ b/src/layers/streetLevel/lineCoverage.test.mjs @@ -397,6 +397,63 @@ test('a tile redrawn for a new share keeps its old lines until the new ones are ); }); +test('a second redraw while a swap still builds keeps the lines on screen', async (t) => { + const { state, coverage, viewer } = setup(t, { + zoomFor: () => ({ zoom: 18, kind: 'sequence' }), + tuning: { + ...TUNING, + maxTiles: 9, + maxSequences: 900, + maxPoints: 90, + keepOrder: true, + }, + decoded: (centre, tile) => { + const dense = tile.x % 2 === 0 && tile.y % 2 === 0; + return { + sequences: dense + ? Array.from({ length: 30 }, (_, i) => + pointsLine(`d${tile.x}/${tile.y}/${i}`, 5, tile, centre), + ) + : [pointsLine(`s${tile.x}/${tile.y}`, 2, tile, centre)], + overview: [], + }; + }, + }); + coverage.refresh(); + assert.ok( + await until( + () => state.coverage.tiles.size > 1 && !state.coverage.pending.size, + ), + ); + // A tile mid-swap (its new lines never become ready here) with a line + // the cap left out: selecting that line redraws the tile again. + const entry = [...state.coverage.tiles.values()].find( + (tile) => + tile.swap && + tile.sequenceList.some((sequence) => !tile.sequences.has(sequence.id)), + ); + assert.ok(entry, 'a swapping tile with a line left out'); + const onScreen = [...entry.swap.old]; + const building = entry.primitives.map(({ primitive }) => primitive); + const left = entry.sequenceList.find( + (sequence) => !entry.sequences.has(sequence.id), + ); + state.sequence.selectedId = left.id; + coverage.recolorSequence(left.id, true); + assert.ok(entry.sequences.has(left.id), 'the selected line is drawn'); + const onGlobe = (primitive) => + viewer.scene.groundPrimitives.items.has(primitive) || + viewer.scene.primitives.items.has(primitive); + for (const { primitive } of onScreen) + assert.ok( + onGlobe(primitive), + 'the lines on screen stay until the new ones are ready', + ); + for (const primitive of building) + assert.ok(!onGlobe(primitive), 'the first swap’s unready lines are gone'); + assert.ok(entry.swap, 'still swapping'); +}); + test('a selected line outside the cap does not make its tile redraw on every settle, nor does one the filter hides', async (t) => { let pano = 'all'; const { state, coverage, viewer } = setup(t, { diff --git a/src/layers/streetLevel/photoViewer.js b/src/layers/streetLevel/photoViewer.js index 3de2c1af7..079cf82c8 100644 --- a/src/layers/streetLevel/photoViewer.js +++ b/src/layers/streetLevel/photoViewer.js @@ -3,6 +3,7 @@ import { PANO_FOV_DEFAULT_DEG, wrapDegrees, } from './panorama.js'; +import { imageSize } from './imageSize.js'; /** A photo download (tens of kilobytes to a few megabytes). */ export const IMAGE_TIMEOUT_MS = 15_000; @@ -10,6 +11,19 @@ export const IMAGE_TIMEOUT_MS = 15_000; /** A picture file or decoded side past these is refused (a photo is a few MB). */ export const MAX_IMAGE_BYTES = 48 * 1024 * 1024; export const MAX_IMAGE_SIDE = 16_384; +/** 64 MP: 256 MB once decoded to RGBA. A 16,384 px square is four times that. */ +export const MAX_IMAGE_PIXELS = 8_192 * 8_192; +/** Where in a file to look for its dimensions (a JPEG's EXIF comes first). */ +const HEADER_BYTES = 256 * 1024; + +/** Whether a picture this size is past what the viewer decodes. */ +export function tooManyPixels(width, height) { + return ( + width > MAX_IMAGE_SIDE || + height > MAX_IMAGE_SIDE || + width * height > MAX_IMAGE_PIXELS + ); +} /** Panorama texture widths: enough for a docked viewer, and for an expanded one. */ export const PANO_MID_SIZE = Object.freeze([2048, 1024]); @@ -43,19 +57,36 @@ export function loadImage( return loadBitmap(url, timeoutMs, resize, failure, signal); return new Promise((resolve, reject) => { if (!url) return reject(new Error(failure)); + if (signal?.aborted) return reject(new Error(failure)); const image = new Image(); if (crossOrigin) image.crossOrigin = 'anonymous'; image.decoding = 'async'; - const timer = setTimeout(() => { + // Stop the download: another photo opened, or the time ran out. + const stop = () => { + clearTimeout(timer); + signal?.removeEventListener?.('abort', stop); + image.onload = image.onerror = null; image.src = ''; reject(new Error(failure)); - }, timeoutMs); - image.onload = () => { + }; + const timer = setTimeout(stop, timeoutMs); + signal?.addEventListener?.('abort', stop, { once: true }); + const settle = () => { clearTimeout(timer); + signal?.removeEventListener?.('abort', stop); + }; + image.onload = () => { + settle(); + // An reads no header first: refuse what it decoded too large. + if (tooManyPixels(image.naturalWidth || 0, image.naturalHeight || 0)) { + image.src = ''; + reject(oversized(failure)); + return; + } resolve(image); }; image.onerror = () => { - clearTimeout(timer); + settle(); reject(new Error(failure)); }; image.src = url; @@ -80,6 +111,12 @@ async function loadBitmap(url, timeoutMs, resize, failure, signal) { const length = Number(response.headers?.get?.('content-length')); if (length > MAX_IMAGE_BYTES) throw oversized(failure); const blob = await boundedBlob(response, failure); + // Its declared size, before anything decodes it. + const size = imageSize( + new Uint8Array(await blob.slice(0, HEADER_BYTES).arrayBuffer()), + ); + if (size && tooManyPixels(size.width, size.height)) + throw oversized(failure); const bitmap = await (resize ? createImageBitmap(blob, { resizeWidth: resize[0], @@ -88,7 +125,7 @@ async function loadBitmap(url, timeoutMs, resize, failure, signal) { }) : createImageBitmap(blob)); // (A resized decode reports its new size: this guards unresized ones.) - if (bitmap.width > MAX_IMAGE_SIDE || bitmap.height > MAX_IMAGE_SIDE) { + if (tooManyPixels(bitmap.width, bitmap.height)) { bitmap.close?.(); throw oversized(failure); } @@ -152,7 +189,7 @@ export function panoSizeFor(canvasPixels, fovDeg) { * The panorama shown first, its texture width (0 when it is only a preview), and a size to shrink it to. * @property {(photo: object, size: [number, number], view: {expanded: boolean}) => {url: string, resize?: [number, number]|null, timeoutMs?: number}|null} panoSharp * A sharper panorama for a canvas that needs `size`, or null; `expanded` while the viewer is. - * @property {(photo: object) => Array} flatFirst Flat versions to try in turn. + * @property {(photo: object, view: {expanded: boolean}) => Array} flatFirst Flat versions to try in turn (`expanded` while the viewer is). * @property {(photo: object, pixels: number) => string|null} flatSharp * A sharper flat version for a picture this many device pixels wide, or null. * @property {(photo: object) => {title?: string|null, creator?: string|null, license?: string|null, externalUrl: string, capturedAtPrecision?: 'day'|'month'}} describe @@ -176,6 +213,7 @@ export function panoSizeFor(canvasPixels, fovDeg) { * @param {(photo: object, delta: -1|1) => Promise} options.step The photo `delta` along the sequence, or null at its end. * @param {(imageId: string) => void} options.onStep Open the photo ‹ › chose (through the core, so loading and errors show). * @param {(photo: object) => void} [options.onShown] A photo is on screen. + * @param {(photo: object) => void} [options.onShowing] The photo this open shows, just before its pose. * @param {(imageId: string) => void} [options.onReopen] Draw the same photo again after a GPU reset (default: `onStep`). * @param {() => boolean} [options.onEscape] Esc in the docked viewer: true when it was used (e.g. a selection cleared). * @param {object} [options.render] Render governor. @@ -191,6 +229,7 @@ export function createPhotoViewer({ step, onStep, onShown = () => {}, + onShowing = () => {}, onReopen = onStep, onEscape = () => false, render, @@ -227,8 +266,10 @@ export function createPhotoViewer({ let panoComing = null; /** A resize came while it loaded: look again once it lands. */ let lookAgain = false; - /** The sharper flat version on screen or on its way (one download each). */ - let flatSharpUrl = null; + /** The sharper flat version on screen, and the one on its way. */ + let flatSharp = { shown: null, loading: null }; + /** The open in progress: another open, a close or an unmount stops its downloads. */ + let openAbort = null; /** The photo an open is fetching: ‹ › step on from it, not the old one. */ let openingId = null; /** ‹ › chose this photo: keep the panorama's look direction for it. */ @@ -626,6 +667,9 @@ export function createPhotoViewer({ async function upgrade(target, seq, kind) { // A 360° photo shown flat (no WebGL) keeps its front view. if (kind === 'flat' && target.isPano) return; + // Another photo is opening: the one still shown is not worth more bytes. + if (openingId !== null && openingId !== target.id) return; + const signal = openAbort?.signal; if (kind === 'pano') { const size = panoSizeNow(); if (size[0] <= panoWidth) return; @@ -636,7 +680,7 @@ export function createPhotoViewer({ return; } const sharp = sources.panoSharp(target, size, { - expanded: Boolean(root?.closest?.('.sl-viewer-wrap-expanded')), + expanded: isExpanded(), }); if (!sharp?.url) return; const coming = { seq, width: size[0], controller: new AbortController() }; @@ -674,19 +718,28 @@ export function createPhotoViewer({ return; } const url = sources.flatSharp(target, viewerPixels()); - if (!url || url === flatSharpUrl) return; - flatSharpUrl = url; + if (!url || url === flatSharp.shown || url === flatSharp.loading) return; + const sharp = flatSharp; + sharp.loading = url; try { - const image = await loadAs(url, { crossOrigin: false }); + const image = await loadAs(url, { crossOrigin: false, signal }); if (seq !== openSeq || photo !== target || showing !== 'flat') return; picture.src = image.src; + sharp.shown = url; requestRender(); } catch { - // The first version stays; a later resize may try again. - if (flatSharpUrl === url && photo === target) flatSharpUrl = null; + /* the first version stays; a later resize may try again */ + } finally { + // Discarded or failed: nothing is on its way, so it may be asked again. + if (sharp.loading === url) sharp.loading = null; } } + /** The viewer is the expanded dialog, not docked in the panel. */ + function isExpanded() { + return Boolean(root?.closest?.('.sl-viewer-wrap-expanded')); + } + /** Another photo, or none: a sharper panorama still on its way is not needed. */ function dropSharpPanorama() { panoComing?.controller?.abort(); @@ -701,14 +754,16 @@ export function createPhotoViewer({ return panoSizeFor(width, renderer?.getView().fov ?? PANO_FOV_DEFAULT_DEG); } - async function showPano(target, seq, keepView) { + async function showPano(target, seq, keepView, signal) { // If the panorama fails, the caller shows the front view rather than // waiting on it twice. const first = sources.panoFirst(target); if (!first?.url) throw new Error(LOAD_FAILED); const image = await loadAs(first.url, { crossOrigin: true, - resize: first.resize ?? null, + signal, + // Never decoded at more than the texture it becomes. + resize: first.resize ?? PANO_MID_SIZE, }); if (seq !== openSeq) { image.close?.(); @@ -730,17 +785,20 @@ export function createPhotoViewer({ show('pano'); renderer.setImage(image); panoWidth = first.width || 0; - panoComing = null; + // A sharper version of the photo before was dropped by `open`; one of + // this photo, asked by a resize meanwhile, is still worth having. return true; } - async function showFlat(target, seq) { - const urls = sources.flatFirst(target).filter(Boolean); + async function showFlat(target, seq, signal) { + const urls = sources + .flatFirst(target, { expanded: isExpanded() }) + .filter(Boolean); let image; let failure = null; for (const url of urls) { try { - image = await loadAs(url, { crossOrigin: false }); + image = await loadAs(url, { crossOrigin: false, signal }); break; } catch (error) { failure = error; @@ -750,26 +808,27 @@ export function createPhotoViewer({ if (!image) throw failure || new Error(LOAD_FAILED); if (seq !== openSeq) return false; picture.src = image.src; - flatSharpUrl = null; + flatSharp = { shown: null, loading: null }; photo = target; show('flat'); return true; } - async function openPhoto(id, seq, keepView) { - const target = await getPhoto(id); + async function openPhoto(id, seq, keepView, signal) { + const target = await getPhoto(id, { signal }); if (seq !== openSeq) return; let shown = false; if (sources.hasPano(target) && ensureRenderer()) { try { - shown = await showPano(target, seq, keepView); + shown = await showPano(target, seq, keepView, signal); } catch { // The panorama failed: its front view is better than nothing. if (seq !== openSeq) return; - shown = await showFlat(target, seq); + shown = await showFlat(target, seq, signal); } - } else shown = await showFlat(target, seq); + } else shown = await showFlat(target, seq, signal); if (!shown) return; + onShowing(target); emit(); onShown(target); upgrade(target, seq, showing); @@ -790,11 +849,18 @@ export function createPhotoViewer({ const id = String(imageId); const seq = ++openSeq; dropSharpPanorama(); + // The previous open's downloads stop: only this photo is wanted now. + openAbort?.abort(); + const opening = (openAbort = new AbortController()); const keepView = keepViewFor === id; keepViewFor = null; openingId = id; try { - await openPhoto(id, seq, keepView); + await openPhoto(id, seq, keepView, opening.signal); + } catch (error) { + // A later open, or a close, stopped this one: that is not a failure. + if (seq !== openSeq) return; + throw error; } finally { if (openingId === id && seq === openSeq) openingId = null; } @@ -803,10 +869,12 @@ export function createPhotoViewer({ close() { openSeq++; dropSharpPanorama(); + openAbort?.abort(); + openAbort = null; openingId = null; keepViewFor = null; photo = null; - flatSharpUrl = null; + flatSharp = { shown: null, loading: null }; show(null); if (picture) picture.removeAttribute('src'); renderer?.clear?.(); @@ -815,10 +883,12 @@ export function createPhotoViewer({ unmount() { openSeq++; dropSharpPanorama(); + openAbort?.abort(); + openAbort = null; refused.clear(); openingId = null; keepViewFor = null; - flatSharpUrl = null; + flatSharp = { shown: null, loading: null }; teardown(); }, diff --git a/src/layers/streetLevel/photoViewer.test.mjs b/src/layers/streetLevel/photoViewer.test.mjs index 758a1e25b..09df91c2e 100644 --- a/src/layers/streetLevel/photoViewer.test.mjs +++ b/src/layers/streetLevel/photoViewer.test.mjs @@ -7,6 +7,7 @@ const PHOTOS = { a: { id: 'a', sequenceId: 's', lon: 1, lat: 2, heading: 0, isPano: true }, b: { id: 'b', sequenceId: 's', lon: 1, lat: 2, heading: 0, isPano: true }, f: { id: 'f', sequenceId: 's', lon: 1, lat: 2, heading: 0, isPano: false }, + g: { id: 'g', sequenceId: 's', lon: 1, lat: 2, heading: 0, isPano: false }, }; /** Sources whose panoramas and sharper flat versions are named after the photo. */ @@ -294,7 +295,11 @@ test('a picture file or bitmap past any real photo’s size is refused', async ( (globalThis.fetch = async () => ({ ok: true, headers: new Map([['content-length', String(length ?? '')]]), - blob: async () => ({ size: size ?? 10, bitmap: bitmap(width, height) }), + blob: async () => ({ + size: size ?? 10, + bitmap: bitmap(width, height), + slice: () => ({ arrayBuffer: async () => new ArrayBuffer(0) }), + }), })); const refusedForSize = (error) => { assert.match(error.message, /could not be loaded/); @@ -331,3 +336,203 @@ test('a picture file or bitmap past any real photo’s size is refused', async ( respond({}); assert.equal((await loadImage('https://x/a.jpg')).width, 100); }); + +/** A viewer whose loads are held, recording each load's signal by url. */ +function heldViewer() { + const pending = new Map(); + const signals = new Map(); + const loads = []; + const viewer = createPhotoViewer({ + providerId: 'test', + name: 'Test', + sources: SOURCES, + getPhoto: async (id) => PHOTOS[id], + step: async () => null, + onStep() {}, + load: (url, options = {}) => + new Promise((resolve, reject) => { + loads.push(url); + signals.set(url, options.signal); + options.signal?.addEventListener('abort', () => + reject(new Error('aborted')), + ); + pending.set(url, { resolve, reject }); + }), + }); + const release = (url) => { + pending.get(url).resolve({ src: url }); + pending.delete(url); + }; + return { viewer, pending, signals, loads, release, host: fakeHost() }; +} + +test('a new open, a close or an unmount stops the downloads of the open before it', async () => { + const env = heldViewer(); + await env.viewer.mount(env.host); + const first = env.viewer.open('f'); + assert.ok(await until(() => env.signals.has('f-first'))); + const second = env.viewer.open('g'); + assert.equal(env.signals.get('f-first').aborted, true, 'f is not wanted now'); + await first.catch(() => {}); + assert.ok(await until(() => env.signals.has('g-first'))); + env.viewer.close(); + assert.equal(env.signals.get('g-first').aborted, true, 'closed'); + await second.catch(() => {}); + const third = env.viewer.open('f'); + assert.ok(await until(() => env.pending.has('f-first'))); + env.viewer.unmount(); + assert.equal(env.signals.get('f-first').aborted, true, 'unmounted'); + await third.catch(() => {}); +}); + +test('a sharper flat version on its way stops when another photo opens, and is asked again later', async (t) => { + t.after(() => delete globalThis.devicePixelRatio); + globalThis.devicePixelRatio = 1; + const env = heldViewer(); + await env.viewer.mount(env.host); + parts(env.host).picture.clientWidth = 1200; + const opening = env.viewer.open('f'); + assert.ok(await until(() => env.pending.has('f-first'))); + env.release('f-first'); + await opening; + assert.ok( + await until(() => env.pending.has('f-sharp')), + 'the 1,200 px viewer asks for more', + ); + // Resizes while it loads (the panel animating) do not ask again. + env.viewer.resize(); + env.viewer.resize(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(env.loads.filter((url) => url === 'f-sharp').length, 1); + // g opens (and fails): f's sharp download stops, f stays on screen. + const failing = env.viewer.open('g'); + assert.equal(env.signals.get('f-sharp').aborted, true); + assert.ok(await until(() => env.pending.has('g-first'))); + env.pending.get('g-first').reject(new Error('gone')); + env.pending.delete('g-first'); + await failing.catch(() => {}); + // The discarded response does not block f's sharp version for good. + env.signals.delete('f-sharp'); + env.viewer.resize(); + assert.ok(await until(() => env.signals.has('f-sharp')), 'asked again'); +}); + +test('a resize while another photo opens does not upgrade the photo still shown', async (t) => { + t.after(() => delete globalThis.devicePixelRatio); + globalThis.devicePixelRatio = 1; + const env = heldViewer(); + await env.viewer.mount(env.host); + const opening = env.viewer.open('f'); + assert.ok(await until(() => env.pending.has('f-first'))); + env.release('f-first'); + await opening; + // g is opening; EXPAND resizes the viewer meanwhile. + const next = env.viewer.open('g'); + assert.ok(await until(() => env.pending.has('g-first'))); + parts(env.host).picture.clientWidth = 1200; + env.viewer.resize(); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(env.pending.has('f-sharp'), false, 'f is on its way out'); + env.release('g-first'); + await next; + assert.ok(await until(() => env.pending.has('g-sharp')), 'g gets it'); +}); + +test('a panorama is always decoded at a bounded size', async () => { + const env = heldViewer(); + const options = []; + const viewer = createPhotoViewer({ + providerId: 'test', + name: 'Test', + sources: SOURCES, + getPhoto: async (id) => PHOTOS[id], + step: async () => null, + onStep() {}, + load: async (url, opts) => { + options.push(opts); + return { src: url }; + }, + createRenderer: () => ({ + setImage() {}, + setView() {}, + getView: () => ({ yaw: 0, pitch: 0, fov: 80 }), + resize() {}, + clear() {}, + destroy() {}, + }), + }); + await viewer.mount(env.host); + await viewer.open('a'); + // SOURCES' panoFirst names no size: the viewer still bounds it. + assert.deepEqual(options[0].resize, [2048, 1024]); +}); + +test('a header that declares a huge image is refused before anything decodes it', async (t) => { + const { loadImage } = await import('./photoViewer.js'); + const saved = { + fetch: globalThis.fetch, + createImageBitmap: globalThis.createImageBitmap, + }; + t.after(() => Object.assign(globalThis, saved)); + let decoded = 0; + globalThis.createImageBitmap = async () => { + decoded++; + return { width: 10, height: 10 }; + }; + // A 1 kB PNG that says it is 16,384 × 16,384. + const png = new Uint8Array(1024); + png.set([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13]); + png.set( + [...'IHDR'].map((c) => c.charCodeAt(0)), + 12, + ); + png.set([0, 0, 0x40, 0, 0, 0, 0x40, 0], 16); + globalThis.fetch = async () => new Response(png); + await assert.rejects( + loadImage('https://x/bomb.png'), + (error) => error.tooLarge === true, + ); + assert.equal(decoded, 0, 'never decoded'); +}); + +test('an that turns out too large is refused and let go', async (t) => { + const { loadImage, MAX_IMAGE_SIDE } = await import('./photoViewer.js'); + const saved = globalThis.Image; + t.after(() => (globalThis.Image = saved)); + let size = [MAX_IMAGE_SIDE + 1, 100]; + const images = []; + globalThis.Image = class { + constructor() { + images.push(this); + } + set src(value) { + this._src = value; + if (value) + queueMicrotask(() => { + [this.naturalWidth, this.naturalHeight] = size; + this.onload?.(); + }); + } + get src() { + return this._src; + } + }; + await assert.rejects( + loadImage('https://x/a.jpg', { crossOrigin: false }), + (error) => error.tooLarge === true, + ); + assert.equal(images.at(-1).src, '', 'its pixels let go'); + size = [4000, 3000]; + const ok = await loadImage('https://x/b.jpg', { crossOrigin: false }); + assert.equal(ok.naturalWidth, 4000); + // An aborted load stops and rejects. + size = [10, 10]; + const controller = new AbortController(); + const aborted = loadImage('https://x/c.jpg', { + crossOrigin: false, + signal: controller.signal, + }); + controller.abort(); + await assert.rejects(aborted); + assert.equal(images.at(-1).src, '', 'its download stops'); +}); diff --git a/src/layers/streetLevel/providers/kartaview/viewer.js b/src/layers/streetLevel/providers/kartaview/viewer.js index d33adef97..9b1d0884b 100644 --- a/src/layers/streetLevel/providers/kartaview/viewer.js +++ b/src/layers/streetLevel/providers/kartaview/viewer.js @@ -37,6 +37,7 @@ const KARTAVIEW_SOURCES = Object.freeze({ panoFirst: (photo) => ({ url: resizedImageUrl(photo.urls.pano, PANO_PREVIEW_SIZE), width: 0, + resize: PANO_PREVIEW_SIZE, }), panoSharp: (photo, size) => ({ url: resizedImageUrl(photo.urls.pano, size), diff --git a/src/layers/streetLevel/providers/kartaview/viewer.test.mjs b/src/layers/streetLevel/providers/kartaview/viewer.test.mjs index 1391b727c..9ca5174c1 100644 --- a/src/layers/streetLevel/providers/kartaview/viewer.test.mjs +++ b/src/layers/streetLevel/providers/kartaview/viewer.test.mjs @@ -883,6 +883,11 @@ test('the panorama is as sharp as the viewer needs: 2,048 docked, 4,096 once exp }); test('a CORS image is decoded off the main thread into an ImageBitmap where the browser can', async (t) => { + // A blob whose header names no size (the viewer then reads none). + const fakeBlob = { + size: 4, + slice: () => ({ arrayBuffer: async () => new ArrayBuffer(0) }), + }; const saved = { fetch: globalThis.fetch, bitmap: globalThis.createImageBitmap, @@ -896,7 +901,7 @@ test('a CORS image is decoded off the main thread into an ImageBitmap where the fetched.push({ url, mode: init.mode }); return url.includes('gone') ? { ok: false, status: 409 } - : { ok: true, blob: async () => 'blob' }; + : { ok: true, blob: async () => fakeBlob }; }; globalThis.createImageBitmap = async (blob) => ({ blob, @@ -906,7 +911,7 @@ test('a CORS image is decoded off the main thread into an ImageBitmap where the const bitmap = await loadImage('https://cdn.kartaview.org/a', { crossOrigin: true, }); - assert.deepEqual(bitmap, { blob: 'blob', width: 4096, height: 2048 }); + assert.deepEqual(bitmap, { blob: fakeBlob, width: 4096, height: 2048 }); assert.deepEqual(fetched, [ { url: 'https://cdn.kartaview.org/a', mode: 'cors' }, ]); diff --git a/src/layers/streetLevel/providers/panoramax/decode.js b/src/layers/streetLevel/providers/panoramax/decode.js index 0a354d06b..b8cbfeacc 100644 --- a/src/layers/streetLevel/providers/panoramax/decode.js +++ b/src/layers/streetLevel/providers/panoramax/decode.js @@ -2,6 +2,7 @@ import { PbfReader } from 'pbf'; import { VectorTile } from '@mapbox/vector-tile'; import { tileLocalToLonLat } from '../../tileMath.js'; import { + DECODE_SLICE_MS, OVERVIEW_BINS_PER_AXIS, SIMPLIFY_TOLERANCE_UNITS, panoramaxTimeMs, @@ -66,11 +67,62 @@ function dayMs(value) { * most OVERVIEW_BINS_PER_AXIS² a tile (a z0 tile holds ~70,000 cells). A * cell has 360° and flat counts but no date. * - `pictures` (z15) is left for `decodePictures`. + * + * In one go; `decodeCoverageTileSliced` gives the page a turn between + * slices. * @param {Uint8Array} bytes * @param {{x: number, y: number, z: number}} address * @returns {{sequences: Array<{id: string, parts: Array>, capturedAt: number, isPano: boolean}>, overview: Array<{id: string, lon: number, lat: number, capturedAt: null, isPano: boolean, hasPano: boolean, hasFlat: boolean, weight: number}>}} */ export function decodeCoverageTile(bytes, address) { + const steps = coverageSteps(bytes, address); + let step = steps.next(); + while (!step.done) step = steps.next(); + return step.value; +} + +/** Give the page a turn: input and rendering run before the next slice. */ +function nextTask() { + if (typeof globalThis.scheduler?.yield === 'function') + return globalThis.scheduler.yield(); + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +/** + * `decodeCoverageTile` in slices of about `sliceMs`, the page getting a + * turn between them: a 2 MB z10 tile over Paris is ~200 ms of decoding on + * a fast laptop, more on a phone, and would freeze the globe while the + * camera approaches. Stops at the next slice once `signal` aborts. + * @param {Uint8Array} bytes + * @param {{x: number, y: number, z: number}} address + * @param {{signal?: AbortSignal, sliceMs?: number, pause?: () => Promise, now?: () => number}} [options] + */ +export async function decodeCoverageTileSliced( + bytes, + address, + { + signal, + sliceMs = DECODE_SLICE_MS, + pause = nextTask, + now = () => performance.now(), + } = {}, +) { + const steps = coverageSteps(bytes, address); + let started = now(); + let step = steps.next(); + while (!step.done) { + if (now() - started >= sliceMs) { + await pause(); + signal?.throwIfAborted(); + started = now(); + } + step = steps.next(); + } + return step.value; +} + +/** The decode, a step a feature (see `decodeCoverageTile`). */ +function* coverageSteps(bytes, address) { const result = { sequences: [], overview: [] }; if (!bytes || !bytes.length) return result; const tile = new VectorTile(new PbfReader(bytes)); @@ -78,7 +130,7 @@ export function decodeCoverageTile(bytes, address) { const sequences = tile.layers.sequences; if (sequences) { - const lengths = new Map(); + const measured = []; for (let i = 0; i < sequences.length; i++) { const feature = sequences.feature(i); const props = feature.properties || {}; @@ -113,10 +165,13 @@ export function decodeCoverageTile(bytes, address) { ? false : null, }; - lengths.set(sequence, length); - result.sequences.push(sequence); + measured.push({ sequence, length }); + yield; } - result.sequences.sort((a, b) => lengths.get(b) - lengths.get(a)); + // A turn before the sort: over a city it is thousands of lines. + yield; + measured.sort((a, b) => b.length - a.length); + result.sequences = measured.map(({ sequence }) => sequence); } const grid = tile.layers.grid; @@ -127,6 +182,7 @@ export function decodeCoverageTile(bytes, address) { const feature = grid.feature(i); const props = feature.properties || {}; const point = feature.loadGeometry()?.[0]?.[0]; + if (i % 64 === 63) yield; if (!point) continue; const pano = Number(props.nb_360_pictures) || 0; const flat = Number(props.nb_flat_pictures) || 0; diff --git a/src/layers/streetLevel/providers/panoramax/decode.test.mjs b/src/layers/streetLevel/providers/panoramax/decode.test.mjs index c98ff3461..7caa25a63 100644 --- a/src/layers/streetLevel/providers/panoramax/decode.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/decode.test.mjs @@ -1,6 +1,11 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { decodeCoverageTile, decodePictures, simplifyLine } from './decode.js'; +import { + decodeCoverageTile, + decodeCoverageTileSliced, + decodePictures, + simplifyLine, +} from './decode.js'; import { OVERVIEW_BINS_PER_AXIS } from './policy.js'; import { encodePanoramaxTile } from '../../../../testSupport/panoramaxFixtures.mjs'; import { tileCentre } from '../../../../testSupport/streetLevelFakes.mjs'; @@ -69,10 +74,11 @@ test('sequence lines keep their parts apart, with their day and camera type', () test('density cells bin to a bounded number of points that keep the counts and kinds', () => { const tile = { x: 0, y: 0, z: 0 }; - // Two cells in one bin, one far away, one empty. + // Two cells in one bin (3.75° wide at z0) far enough apart to tell + // which it sits on, one far away, one empty. const grid = [ - { lon: 2.35, lat: 48.85, pano: 0, flat: 5 }, - { lon: 2.36, lat: 48.85, pano: 30, flat: 0 }, + { lon: 0.3, lat: 48.85, pano: 0, flat: 5 }, + { lon: 3.4, lat: 48.85, pano: 30, flat: 0 }, { lon: -73.9, lat: 40.7, pano: 1, flat: 0 }, { lon: 10, lat: 10, pano: 0, flat: 0 }, ]; @@ -87,7 +93,10 @@ test('density cells bin to a bounded number of points that keep the counts and k assert.equal(paris.hasFlat, true); assert.equal(paris.isPano, true, 'mostly 360°'); assert.equal(paris.capturedAt, null, 'cells are undated'); - assert.ok(near(paris.lon, 2.36, 0.2), 'sits on the busiest cell'); + assert.ok( + near(paris.lon, 3.4, 0.2), + `sits on the busiest cell: ${paris.lon}`, + ); const nyc = overview.find((p) => p.lon < 0); assert.deepEqual([nyc.hasPano, nyc.hasFlat, nyc.weight], [true, false, 1]); }); @@ -134,6 +143,71 @@ test('a dense grid tile never draws more than the bins allow', () => { ); }); +/** A tile with many lines and a dense grid, for the sliced decode. */ +function busyTile() { + const { tile, lon, lat } = tileCentre(2.35, 48.85); + const sequences = Array.from({ length: 200 }, (_, i) => ({ + id: `s${i}`, + parts: [ + [ + [lon, lat + i / 1e6], + [lon + (i + 1) / 73_000, lat + i / 1e6], + ], + ], + })); + const grid = Array.from({ length: 500 }, (_, i) => ({ + lon: lon + (i % 25) / 1e4, + lat: lat + Math.floor(i / 25) / 1e4, + })); + return { tile, bytes: encodePanoramaxTile(tile, { sequences, grid }) }; +} + +test('a sliced decode gives the page turns and decodes the same as one in one go', async () => { + const { tile, bytes } = busyTile(); + let clock = 0; + let pauses = 0; + const sliced = await decodeCoverageTileSliced(bytes, tile, { + sliceMs: 8, + // Each step "takes" a millisecond: a turn every eight features. + now: () => clock++, + pause: async () => { + pauses++; + }, + }); + assert.ok(pauses > 20, `${pauses} turns`); + assert.deepEqual(sliced, decodeCoverageTile(bytes, tile)); +}); + +test('a sliced decode stops at the next turn once aborted', async () => { + const { tile, bytes } = busyTile(); + const controller = new AbortController(); + let clock = 0; + let pauses = 0; + await assert.rejects( + decodeCoverageTileSliced(bytes, tile, { + signal: controller.signal, + now: () => (clock += 10), + pause: async () => { + if (++pauses === 2) controller.abort(); + }, + }), + { name: 'AbortError' }, + ); + assert.equal(pauses, 2, 'no work after the abort'); +}); + +test('a fast sliced decode takes no turns', async () => { + const { tile, bytes } = busyTile(); + let pauses = 0; + await decodeCoverageTileSliced(bytes, tile, { + now: () => 0, + pause: async () => { + pauses++; + }, + }); + assert.equal(pauses, 0); +}); + test('an empty tile decodes to nothing', () => { for (const bytes of [new Uint8Array(0), null, undefined]) assert.deepEqual(decodeCoverageTile(bytes, { x: 0, y: 0, z: 0 }), { diff --git a/src/layers/streetLevel/providers/panoramax/index.js b/src/layers/streetLevel/providers/panoramax/index.js index b6cfa93c8..3ce1f5d21 100644 --- a/src/layers/streetLevel/providers/panoramax/index.js +++ b/src/layers/streetLevel/providers/panoramax/index.js @@ -9,7 +9,7 @@ import { } from '../../tileMath.js'; import { metresBetween } from '../../view.js'; import { createLru, createTileCache } from '../../tileCache.js'; -import { decodeCoverageTile, decodePictures } from './decode.js'; +import { decodeCoverageTileSliced, decodePictures } from './decode.js'; import { createPanoramaxSource } from './source.js'; import { createPanoramaxViewer } from './viewer.js'; import { @@ -230,16 +230,23 @@ export function createPanoramaxProvider({ return photo; } - /** Lookups in flight, shared: ‹ › and the viewer ask for the same picture. */ + /** + * Lookups in flight, shared: ‹ › and the viewer ask for the same + * picture. A lookup stops once every caller waiting on it has given up + * (its search would hold one of the two API connections for nothing). + */ const inflight = new Map(); - function getPhoto(photoId) { + function getPhoto(photoId, { signal } = {}) { const id = String(photoId); const cached = photos.get(id); if (cached) return Promise.resolve(cached); + if (signal?.aborted) return Promise.reject(signal.reason); let lookup = inflight.get(id); if (!lookup) { - lookup = source - .getItem(id) + const controller = new AbortController(); + lookup = { controller, waiting: 0 }; + lookup.promise = source + .getItem(id, { signal: controller.signal }) .catch((error) => { // Ids are UUIDs: the API answers 400 to anything else. if (error?.status === 400) return null; @@ -250,10 +257,39 @@ export function createPanoramaxProvider({ if (!photo) throw new Error(GONE_MESSAGE); return photo; }) - .finally(() => inflight.delete(id)); + .finally(() => { + if (inflight.get(id) === lookup) inflight.delete(id); + }); + lookup.promise.catch(() => {}); inflight.set(id, lookup); } - return lookup; + const shared = lookup; + shared.waiting++; + if (!signal) shared.waiting = Infinity; + return new Promise((resolve, reject) => { + const leave = () => { + reject(signal.reason); + shared.waiting--; + // A caller in the same tick (the same photo opened again) may + // still join: stop only if nobody has. + queueMicrotask(() => { + if (shared.waiting > 0) return; + shared.controller.abort(signal.reason); + if (inflight.get(id) === shared) inflight.delete(id); + }); + }; + signal?.addEventListener('abort', leave, { once: true }); + shared.promise.then( + (photo) => { + signal?.removeEventListener('abort', leave); + resolve(photo); + }, + (error) => { + signal?.removeEventListener('abort', leave); + reject(error); + }, + ); + }); } /** @@ -281,7 +317,11 @@ export function createPanoramaxProvider({ size: COVERAGE_TILE_CACHE_SIZE, load: (tile, { signal }) => readable(async () => - decodeCoverageTile(await source.getTile(tile, { signal }), tile), + decodeCoverageTileSliced( + await source.getTile(tile, { signal }), + tile, + { signal }, + ), ), }); /** A z15 tile's pictures of one sequence, by `z/x/y:sequence`. */ @@ -326,21 +366,40 @@ export function createPanoramaxProvider({ const tiles = pictureTilesAround(anchor); // Known from the start: a picture opening while these load is // judged against them (re-centred if outside, left alone inside). - drawn = { sequenceId, tiles: new Set(tiles.map(tileKey)) }; - // The home tile must load; a neighbour that fails only thins the cones. - const lists = await Promise.allSettled( - tiles.map((tile) => picturesIn(tile, sequenceId, signal)), - ); + const covering = { sequenceId, tiles: new Set(tiles.map(tileKey)) }; + drawn = covering; + const conesOf = (lists) => + thinImages( + [ + ...new Map( + lists.flat().map((image) => [image.id, image]), + ).values(), + ].sort((a, b) => a.capturedAt - b.capturedAt), + ); + const keyOf = (tile) => `${tileKey(tile)}:${sequenceId}`; + // The home tile is needed now (it holds the open picture). + const home = await picturesIn(tiles[0], sequenceId, signal); signal?.throwIfAborted(); - if (lists[0].status === 'rejected') throw lists[0].reason; - const byId = new Map( - lists - .flatMap((list) => list.value ?? []) - .map((image) => [image.id, image]), - ); - return thinImages( - [...byId.values()].sort((a, b) => a.capturedAt - b.capturedAt), + // Neighbours already read draw with it; the rest (each up to + // 1.4 MB in central Paris) join the cones as they arrive. + const neighbours = tiles.slice(1); + const kept = neighbours + .map((tile) => pictureLists.get(keyOf(tile))) + .filter(Boolean); + const missing = neighbours.filter( + (tile) => !pictureLists.get(keyOf(tile)), ); + if (missing.length) + Promise.allSettled( + missing.map((tile) => picturesIn(tile, sequenceId, signal)), + ).then((lists) => { + if (signal?.aborted || drawn !== covering) return; + dropFailed(covering, missing, lists); + // Only a tile that arrived now redraws (no loop on failures). + if (lists.some((list) => list.status === 'fulfilled')) + cones.select(sequenceId, { force: true }); + }); + return conesOf([home, ...kept]); }, colors: COLORS, pickPrefix: PICK_PREFIX.image, @@ -350,6 +409,17 @@ export function createPanoramaxProvider({ coverage.recolorSequence(sequenceId, selected), }); + /** + * A tile that failed is not covered: a picture opening in it later + * re-centres the cones there (and asks for the tile again). + */ + function dropFailed(covering, tiles, lists) { + lists.forEach((list, i) => { + if (list.status === 'rejected') + covering.tiles.delete(tileKey(tiles[i])); + }); + } + /** * The cones show the z15 tiles around one picture: once a picture of * the selected sequence opens outside them (‹ ›, a cone), re-centre. @@ -363,11 +433,23 @@ export function createPanoramaxProvider({ } const viewer = createPanoramaxViewer({ - getPhoto: async (photoId) => { - const photo = await getPhoto(photoId); + getPhoto, + // The cones centre on the picture actually shown, never on a + // lookup that came back after the user moved on. + onShowing: (photo) => { if (photo.sequenceId) anchors.set(photo.sequenceId, photo); - return photo; }, + // After a GPU reset the same picture is drawn again by the viewer + // itself: through the core it would cancel a lookup the user + // started. A failure still shows. + onReopen: (photoId) => + viewer + .open(photoId) + .catch((error) => + context.actions.reportError( + error?.message || 'The Panoramax photo could not be loaded', + ), + ), step, onStep: (photoId) => context.actions.openImage(photoId, { frame: false }), @@ -449,9 +531,9 @@ export function createPanoramaxProvider({ coverageStats() { const { kind } = state.coverage; const lines = kind === 'sequence'; - const drawn = coverage.sequenceCount(); + const shown = coverage.sequenceCount(); // Density dots are not sequences: they are not counted as such. - const count = lines ? drawn : 0; + const count = lines ? shown : 0; const loading = state.coverage.pending.size > 0; const settled = !loading && !state.coverage.lastError; let fallback = ''; @@ -461,7 +543,7 @@ export function createPanoramaxProvider({ ); if (kind === 'overview') fallback = - drawn > 0 || !settled + shown > 0 || !settled ? OVERVIEW_HINT : filteredOut ? FILTERED_HINT diff --git a/src/layers/streetLevel/providers/panoramax/policy.js b/src/layers/streetLevel/providers/panoramax/policy.js index 7e8ccca0d..198b8bb8b 100644 --- a/src/layers/streetLevel/providers/panoramax/policy.js +++ b/src/layers/streetLevel/providers/panoramax/policy.js @@ -66,6 +66,13 @@ export const COVERAGE_TUNING = Object.freeze({ * within GPS noise), and 6–10% of the points GPS tracks carry. */ export const SIMPLIFY_TOLERANCE_UNITS = 4; + +/** + * Coverage tiles decode in slices of this many ms, the page getting a turn + * between them (decode.js `decodeCoverageTileSliced`): under a frame. + */ +export const DECODE_SLICE_MS = 8; + /** * Grid cells per overview tile, at most: a z0 tile holds ~70,000 cells * (1.9 MB), drawn as this many binned points. @@ -89,6 +96,14 @@ export const PICTURE_TILE_ZOOM = 15; export const PICTURE_TILE_EDGE_M = 150; export const TILE_TIMEOUT_MS = 20_000; +/** + * Once an answer has started, it may take as long as it keeps coming: a + * 2 MB tile on a slow link outlasts any whole-request timeout, so only a + * stream silent this long is given up. + */ +export const STREAM_IDLE_TIMEOUT_MS = 15_000; +/** Busy (429/503) without a Retry-After the browser can read: back off from this. */ +export const BUSY_BACKOFF_BASE_SEC = 5; export const API_TIMEOUT_MS = 12_000; /** Real tiles are under 6 MB (z6 over France); refuse anything absurd. */ export const MAX_RESPONSE_BYTES = 16 * 1024 * 1024; @@ -111,8 +126,12 @@ export const NEAREST_LIMIT = 10; * beside it has those). */ export const NEAREST_FOV_TOLERANCE_DEG = 180; -/** Pictures in the bounding box around a click, at most. */ -export const NEAREST_BOX_LIMIT = 100; +/** + * Pictures in the bounding box around a click, at most. The API returns + * them roughly nearest first (the first ten of 100 are the first ten of + * 10), and the browser ranks them exactly: 20 is ~60 kB, 100 ~600 kB. + */ +export const NEAREST_BOX_LIMIT = 20; /** * A line clicked from high up may be tens of metres off the click: its * sequence is searched within the camera height / 100 (50–400 m). @@ -194,10 +213,12 @@ function linkedId(item, rel) { } } -/** An email address anywhere in a name (linear: names come from any instance). */ +/** + * An email address anywhere in a name, at any of its '@'s. Linear: names + * come from any instance, and each run between '@'s is scanned once. + */ function looksLikeEmail(name) { - const at = name.indexOf('@'); - return at > 0 && name.indexOf('.', at + 2) !== -1; + return /[^\s@]@[^\s@]+\.[^\s@]/.test(name); } /** @@ -217,12 +238,14 @@ function creatorOf(item) { // Every producer is credited (an uploading app and its photographer), // each name once, none that is an email address. const credited = []; + const seen = new Set(); for (const value of names) { if (typeof value !== 'string') continue; const name = value.trim().slice(0, 80); if (!name || looksLikeEmail(value)) continue; - if (credited.some((other) => other.toLowerCase() === name.toLowerCase())) - continue; + const key = name.toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); credited.push(name); } return credited.length ? credited.join(', ').slice(0, 120) : null; diff --git a/src/layers/streetLevel/providers/panoramax/policy.test.mjs b/src/layers/streetLevel/providers/panoramax/policy.test.mjs index 770f52a26..1bb1ea578 100644 --- a/src/layers/streetLevel/providers/panoramax/policy.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/policy.test.mjs @@ -224,6 +224,23 @@ test('hostile item data: a malformed link escape, null coordinates, a huge produ 'Carto@Cité', 'an @ without a domain is a name', ); + // The address need not follow the first '@'. + assert.equal(named('@team someone@example.org'), null); + assert.equal(named('Carto@Cité (mapper@example.org)'), null); + assert.equal(named(`${'@'.repeat(100_000)}x`), `${'@'.repeat(80)}`); +}); + +test('thousands of producers are credited in linear time', () => { + const item = stacItem({ id: 'a', lon: 1, lat: 2 }); + item.providers = Array.from({ length: 50_000 }, (_, i) => ({ + name: `producer-${i}`, + roles: ['producer'], + })); + const started = performance.now(); + const { creator } = normalizeItem(item); + assert.ok(performance.now() - started < 1000); + assert.ok(creator.startsWith('tester, producer-0, producer-1')); + assert.ok(creator.length <= 120); }); test('the instance holding a picture is named (its via host), never as its author', () => { diff --git a/src/layers/streetLevel/providers/panoramax/provider.test.mjs b/src/layers/streetLevel/providers/panoramax/provider.test.mjs index 031f376d7..4bee380ae 100644 --- a/src/layers/streetLevel/providers/panoramax/provider.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/provider.test.mjs @@ -726,12 +726,16 @@ test('from high up the density grid draws points, not counted as sequences, and test('switching off clears the cones and forgets a pending line click', async (t) => { const items = sequence('s', 3); const env = await mounted({ items, tiles: pictureTiles(items) }); - await withCones(t, env); + const cones = await withCones(t, env); + await env.instance.viewer.open('s-0'); + await env.instance.selectSequence('s'); + assert.ok(await until(() => cones.count() === 3), 'cones to clear'); env.instance.handlePick(`${PICK_PREFIX.sequence}s`, { groundPoint: () => east(0), }); env.instance.deactivate(); assert.equal(env.instance.sequenceStats().selectedId, null); + assert.equal(cones.count(), 0, 'the cones are gone'); // The click's preference is gone: a later lookup asks any sequence. await env.instance.nearestImage(east(0)); assert.equal(env.source.calls.nearest.at(-1).collections, null); @@ -1001,15 +1005,242 @@ test('a re-centre while the cones still load does not inherit the aborted load const second = new Set( pictureTilesAround({ lon, lat }).map((tile) => `${tile.x}/${tile.y}`), ); - const ids = new Set(cones.ids()); const covered = items.filter((item) => { const tile = tileOf(item); return second.has(`${tile.x}/${tile.y}`); }); assert.ok(covered.length > 0); - assert.equal( - covered.filter((item) => ids.has(item.id)).length, - covered.length, - 'every picture in the re-centred tiles has its cone (the shared ones too)', + // The neighbours join as they arrive: wait for them, then every picture + // in the re-centred tiles has its cone (the shared ones too). + const coneCount = () => { + const ids = new Set(cones.ids()); + return covered.filter((item) => ids.has(item.id)).length; + }; + assert.ok( + await until(() => coneCount() === covered.length), + `${coneCount()} of ${covered.length}`, ); }); + +test('after a GPU reset the viewer draws the same picture itself, leaving the core’s lookups alone; a failure is reported', async () => { + const items = sequence('p', 2, { pano: true }); + let failLoads = false; + const view = { yaw: 0, pitch: 0, fov: 90 }; + const definition = createPanoramaxProvider({ + source: fakeSource({ items }), + viewerOptions: { + load: async (url) => { + if (failLoads) + throw new Error('The Panoramax photo could not be loaded'); + return { src: url, width: 2048, height: 1024 }; + }, + createRenderer: () => ({ + setImage() {}, + setView: (next) => Object.assign(view, next), + getView: () => ({ ...view }), + resize() {}, + clear() {}, + destroy() {}, + }), + }, + }); + const { context, actions } = fakeContext(); + const instance = definition.create(context); + actions.instance = instance; + const host = fakeHost(); + await instance.viewer.mount(host); + const poses = []; + instance.viewer.onPose((pose) => poses.push(pose.imageId)); + await instance.viewer.open('p-0'); + const canvas = () => host.children[0].children[1]; + const kind = () => host.children[0].dataset.kind; + assert.equal(kind(), 'pano'); + canvas().fire('webglcontextlost'); + assert.ok(await until(() => kind() === 'pano'), 'drawn again'); + assert.deepEqual(poses, ['p-0', 'p-0']); + assert.deepEqual(actions.opened, [], 'not through the core'); + failLoads = true; + canvas().fire('webglcontextlost'); + assert.ok(await until(() => actions.errors.some(Boolean))); + assert.match(actions.errors.filter(Boolean)[0], /could not be loaded/); +}); + +/** A source whose lookup of `heldId` waits for `release()` and ignores aborts. */ +function holdLookup(source, heldId) { + const held = { release: null, signals: [] }; + const getItem = source.getItem; + source.getItem = async (id, options = {}) => { + held.signals.push({ id, signal: options.signal }); + if (id === heldId) await new Promise((go) => (held.release = go)); + return getItem(id); + }; + return held; +} + +test('a lookup that comes back after the user moved on does not move the cones', async (t) => { + const items = sequence('long', 400); + const env = await mounted({ items, tiles: pictureTiles(items) }); + await withCones(t, env); + const held = holdLookup(env.source, 'long-0'); + const late = env.instance.viewer.open('long-0'); + assert.ok(await until(() => held.release !== null)); + await env.instance.viewer.open('long-399'); + await late; + // The first lookup answers anyway (a source that ignores the abort). + held.release(); + await new Promise((resolve) => setTimeout(resolve, 0)); + await env.instance.selectSequence('long'); + assert.ok(await until(() => !env.instance.sequenceStats().loading)); + const [lon, lat] = items[399].geometry.coordinates; + const [far] = pictureTilesAround({ lon, lat }); + const [lon0, lat0] = items[0].geometry.coordinates; + const [near] = pictureTilesAround({ lon: lon0, lat: lat0 }); + assert.equal(env.source.calls.tiles[0], `15/${far.x}/${far.y}`); + assert.ok( + !env.source.calls.tiles.includes(`15/${near.x}/${near.y}`), + 'nothing around the picture the user left', + ); +}); + +test('a lookup nobody waits for any more is stopped; one still wanted is not', async () => { + const items = sequence('s', 3); + const env = await mounted({ items }); + const held = holdLookup(env.source, 's-0'); + const first = env.instance.viewer.open('s-0'); + assert.ok(await until(() => held.release !== null)); + const [{ signal }] = held.signals; + assert.equal(signal.aborted, false); + await env.instance.viewer.open('s-1'); + await first; + await Promise.resolve(); + assert.equal(signal.aborted, true, 'the user moved on: its search stops'); + // The same picture asked again at once shares the lookup, which goes on. + held.release = null; + held.signals.length = 0; + const again = env.instance.viewer.open('s-2'); + const twice = env.instance.viewer.open('s-2'); + await Promise.all([again, twice]); + assert.equal(held.signals.length, 1); + assert.equal(held.signals[0].signal.aborted, false); + assert.equal(env.poses.at(-1), 's-2'); +}); + +/** + * Two pictures of sequence `e` either side of a z15 tile's west edge, + * half-way down it: the cones around the first read its tile and the + * west neighbour. + */ +function acrossWestEdge() { + const home = tileCentre(ORIGIN.lon, ORIGIN.lat, 15); + const box = tileBounds(home.tile.x, home.tile.y, 15); + const lat = (box.north + box.south) / 2; + const items = [ + stacItem({ + id: 'e-0', + sequenceId: 'e', + lon: box.west + 0.00005, + lat, + nextId: 'e-1', + }), + stacItem({ + id: 'e-1', + sequenceId: 'e', + lon: box.west - 0.00005, + lat, + prevId: 'e-0', + }), + ]; + const west = `15/${home.tile.x - 1}/${home.tile.y}`; + return { items, west }; +} + +test( + 'the open picture’s own tile draws its cones at once; a neighbour’s join as it arrives', + { timeout: 10_000 }, + async (t) => { + const { items, west } = acrossWestEdge(); + const tiles = pictureTiles(items); + let release = null; + const env = await mounted({ + items, + tiles: async (tile) => { + if (`${tile.z}/${tile.x}/${tile.y}` === west) + await new Promise((go) => (release = go)); + return tiles(tile); + }, + }); + const cones = await withCones(t, env); + await env.instance.viewer.open('e-0'); + const [lon, lat] = items[0].geometry.coordinates; + assert.equal( + pictureTilesAround({ lon, lat }).length, + 2, + 'the west neighbour is read too', + ); + // Not awaited: a load that waited for the neighbour would hang here. + env.instance.selectSequence('e'); + assert.ok(await until(() => release !== null), 'the neighbour is held'); + assert.ok( + await until(() => cones.count() === 1), + 'the home tile’s cone shows', + ); + assert.deepEqual(cones.ids(), ['e-0']); + release(); + assert.ok(await until(() => cones.count() === 2), 'the neighbour’s joins'); + }, +); + +test('a neighbour tile that failed is not counted as shown: › into it re-centres the cones there', async (t) => { + const { items, west } = acrossWestEdge(); + const tiles = pictureTiles(items); + let failing = true; + const env = await mounted({ + items, + tiles: (tile) => { + if (failing && `${tile.z}/${tile.x}/${tile.y}` === west) + throw Object.assign(new Error('Panoramax is busy'), { status: 503 }); + return tiles(tile); + }, + }); + const cones = await withCones(t, env); + await env.instance.viewer.open('e-0'); + await env.instance.selectSequence('e'); + assert.ok(await until(() => cones.count() === 1)); + // Let the neighbour's failure land. + assert.ok(await until(() => env.source.calls.tiles.includes(west))); + await new Promise((resolve) => setTimeout(resolve, 10)); + failing = false; + const asked = env.source.calls.tiles.filter((key) => key === west).length; + env.next.fire('click'); + assert.ok(await until(() => env.poses.at(-1) === 'e-1')); + assert.ok( + await until( + () => env.source.calls.tiles.filter((key) => key === west).length > asked, + ), + 'the failed tile is asked again', + ); + assert.ok(await until(() => cones.count() === 2), 'e-1 has its cone'); +}); + +test('a busy or timed-out tile keeps its own message; only a body that is not a tile reads as unreadable', async (t) => { + const items = sequence('s', 3); + const failures = [ + Object.assign(new Error('Panoramax is busy'), { status: 429 }), + Object.assign(new Error('Panoramax did not answer in time — try again'), { + status: 0, + }), + ]; + for (const failure of failures) { + const env = await mounted({ + items, + tiles: () => { + throw failure; + }, + }); + await withCones(t, env); + await env.instance.viewer.open('s-0'); + await env.instance.selectSequence('s'); + assert.ok(await until(() => env.actions.errors.some(Boolean))); + assert.equal(env.actions.errors.filter(Boolean).at(-1), failure.message); + } +}); diff --git a/src/layers/streetLevel/providers/panoramax/source.js b/src/layers/streetLevel/providers/panoramax/source.js index 1217b19bf..9678233ad 100644 --- a/src/layers/streetLevel/providers/panoramax/source.js +++ b/src/layers/streetLevel/providers/panoramax/source.js @@ -1,10 +1,8 @@ import { createLimiter } from '../../limiter.js'; - -/** Longest wait a Retry-After is honoured for. */ -const RETRY_AFTER_MAX_SEC = 300; import { API_CONCURRENCY, API_TIMEOUT_MS, + BUSY_BACKOFF_BASE_SEC, MAX_RESPONSE_BYTES, NEAREST_BOX_LIMIT, NEAREST_FOV_TOLERANCE_DEG, @@ -12,10 +10,14 @@ import { NEAREST_RADIUS_M, PANORAMAX_API, PICTURE_TILE_CONCURRENCY, + STREAM_IDLE_TIMEOUT_MS, TILE_CONCURRENCY, TILE_TIMEOUT_MS, } from './policy.js'; +/** Longest wait a Retry-After (or the backoff) asks for. */ +const RETRY_AFTER_MAX_SEC = 300; + /** An HTTP or API failure; `retryAfterSec` is set while Panoramax is busy. */ export class PanoramaxSourceError extends Error { constructor(message, { status = 0, retryAfterSec = null } = {}) { @@ -29,17 +31,30 @@ export class PanoramaxSourceError extends Error { const NO_ANSWER = 'Panoramax did not answer in time — try again'; const UNREACHABLE = 'Panoramax could not be reached — check the connection'; -/** Seconds from a Retry-After header, or null. */ -function retryAfterSeconds(response) { +/** + * Seconds from a Retry-After header (delay seconds or an HTTP date), or + * null. The meta-catalogue exposes no headers to the browser, so in the + * app this is usually null and the caller backs off on its own. + */ +export function retryAfterSeconds(response, now = Date.now()) { const header = response.headers?.get?.('retry-after'); - const value = header == null || header === '' ? NaN : Number(header); - if (!Number.isFinite(value) || value < 0) return null; + if (header == null || header === '') return null; + let value = Number(header); + if (!Number.isFinite(value)) { + const at = Date.parse(header); + if (!Number.isFinite(at)) return null; + value = Math.ceil((at - now) / 1000); + } + if (value < 0) return null; // A maintenance page's hour must not freeze coverage for an hour. return Math.min(RETRY_AFTER_MAX_SEC, Math.max(1, value)); } -/** The body as bytes, refusing more than MAX_RESPONSE_BYTES whatever the headers say. */ -async function readBounded(response) { +/** + * The body as bytes, refusing more than MAX_RESPONSE_BYTES whatever the + * headers say. `onChunk` hears each piece arrive (the idle timeout). + */ +async function readBounded(response, onChunk = () => {}) { const tooMuch = () => new PanoramaxSourceError('Panoramax sent too much data', { status: response.status, @@ -57,6 +72,7 @@ async function readBounded(response) { for (;;) { const { done, value } = await reader.read(); if (done) break; + onChunk(); total += value.byteLength; if (total > MAX_RESPONSE_BYTES) { reader.cancel().catch(() => {}); @@ -92,30 +108,55 @@ export function createPanoramaxSource({ lifo: true, }); - /** GET within `timeoutMs`; the caller's abort wins over the timeout. */ + /** Busy answers in a row with no readable Retry-After (the backoff grows). */ + let busyStreak = 0; + + /** + * GET: its headers within `timeoutMs`, then a body that may take as long + * as it keeps arriving (silent for STREAM_IDLE_TIMEOUT_MS is given up). + * The caller's abort wins over both. + */ async function get(url, { signal, timeoutMs, empty = [] }) { const controller = new AbortController(); const onAbort = () => controller.abort(signal.reason); if (signal?.aborted) throw signal.reason; signal?.addEventListener('abort', onAbort, { once: true }); - const timer = setTimeout( - () => controller.abort(new PanoramaxSourceError(NO_ANSWER)), - timeoutMs, - ); + const giveUp = () => controller.abort(new PanoramaxSourceError(NO_ANSWER)); + let timer = setTimeout(giveUp, timeoutMs); try { const response = await fetchImpl(url, { signal: controller.signal }); - if (empty.includes(response.status)) return null; - if (response.status === 429 || response.status === 503) + if (empty.includes(response.status)) { + busyStreak = 0; + return null; + } + if (response.status === 429 || response.status === 503) { + const told = retryAfterSeconds(response); + // Not told how long: back off, longer each time (5, 10, 20… 300 s). + const wait = + told ?? + Math.min( + RETRY_AFTER_MAX_SEC, + BUSY_BACKOFF_BASE_SEC * 2 ** Math.min(busyStreak, 8), + ); + busyStreak++; throw new PanoramaxSourceError('Panoramax is busy', { status: response.status, - retryAfterSec: retryAfterSeconds(response) ?? 5, + retryAfterSec: wait, }); + } if (!response.ok) throw new PanoramaxSourceError( `Panoramax answered HTTP ${response.status}`, { status: response.status }, ); - return await readBounded(response); + busyStreak = 0; + // The answer has begun: from here only silence is timed. + clearTimeout(timer); + timer = setTimeout(giveUp, STREAM_IDLE_TIMEOUT_MS); + return await readBounded(response, () => { + clearTimeout(timer); + timer = setTimeout(giveUp, STREAM_IDLE_TIMEOUT_MS); + }); } catch (error) { if (signal?.aborted) throw signal.reason; if (controller.signal.aborted) throw controller.signal.reason; @@ -180,8 +221,8 @@ export function createPanoramaxSource({ * Pictures within `radius` metres of a point, unranked: the caller ranks * them by distance. Two searches side by side, since neither finds them * all: `place_position` (nearest first) leaves out pictures looking - * away from the point, whatever the tolerance; a bounding box has them - * but in no order, so in a dense spot its page may miss the nearest. + * away from the point, whatever the tolerance; a bounding box has them, + * roughly nearest first (not exactly: the caller ranks the union). * `collections` keeps one sequence's, `pano` and `sinceMs` filter as * the imagery filter does. */ diff --git a/src/layers/streetLevel/providers/panoramax/source.test.mjs b/src/layers/streetLevel/providers/panoramax/source.test.mjs index 63eb6ff9e..db2b1160d 100644 --- a/src/layers/streetLevel/providers/panoramax/source.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/source.test.mjs @@ -1,7 +1,16 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { PanoramaxSourceError, createPanoramaxSource } from './source.js'; -import { MAX_RESPONSE_BYTES } from './policy.js'; +import { + PanoramaxSourceError, + createPanoramaxSource, + retryAfterSeconds, +} from './source.js'; +import { + MAX_RESPONSE_BYTES, + NEAREST_BOX_LIMIT, + STREAM_IDLE_TIMEOUT_MS, + TILE_TIMEOUT_MS, +} from './policy.js'; /** A fetch answering from `respond(url)`, recording every URL asked. */ function fakeFetch(respond) { @@ -348,6 +357,148 @@ test('a Retry-After is honoured up to five minutes', async () => { assert.equal(await wait('12'), 12); }); +test('a Retry-After may be an HTTP date', () => { + const now = Date.parse('2026-10-15T12:00:00Z'); + const response = (value) => + new Response(null, { status: 429, headers: { 'retry-after': value } }); + assert.equal( + retryAfterSeconds(response('Thu, 15 Oct 2026 12:01:00 GMT'), now), + 60, + ); + assert.equal( + retryAfterSeconds(response('Thu, 15 Oct 2026 14:00:00 GMT'), now), + 300, + ); + assert.equal( + retryAfterSeconds(response('Thu, 15 Oct 2026 11:00:00 GMT'), now), + null, + 'a date already past', + ); + assert.equal(retryAfterSeconds(response('soon'), now), null); + assert.equal( + retryAfterSeconds(new Response(null, { status: 429 }), now), + null, + ); +}); + +test('busy without a readable Retry-After (the browser hides it): the wait doubles, and resets once answered', async () => { + let busy = true; + const source = createPanoramaxSource({ + fetchImpl: fakeFetch(() => + busy ? { status: 429 } : { body: new Uint8Array([1]) }, + ).fetchImpl, + }); + const wait = () => + source.getTile({ x: 0, y: 0, z: 0 }).then( + () => 'ok', + (error) => error.retryAfterSec, + ); + const waits = []; + for (let i = 0; i < 9; i++) waits.push(await wait()); + assert.deepEqual(waits, [5, 10, 20, 40, 80, 160, 300, 300, 300]); + busy = false; + assert.equal(await wait(), 'ok'); + busy = true; + assert.equal(await wait(), 5, 'back to the first wait'); +}); + +test( + 'a body that keeps arriving is not cut off by the request timeout; one that stalls is', + { timeout: 10_000 }, + async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const settle = async () => { + for (let i = 0; i < 20; i++) await Promise.resolve(); + }; + // Headers at once, then a chunk every 10 s for twice the tile timeout. + const chunks = Math.ceil((2 * TILE_TIMEOUT_MS) / 10_000); + let push; + const slowBody = (signal) => + new Response( + new ReadableStream({ + start(controller) { + push = (last) => { + controller.enqueue(new Uint8Array([7])); + if (last) controller.close(); + }; + signal.addEventListener('abort', () => + controller.error(signal.reason), + ); + }, + }), + { status: 200 }, + ); + const source = createPanoramaxSource({ + fetchImpl: async (url, { signal }) => slowBody(signal), + }); + const slow = source.getTile({ x: 0, y: 0, z: 0 }); + await settle(); + for (let i = 1; i <= chunks; i++) { + t.mock.timers.tick(10_000); + push(i === chunks); + await settle(); + } + assert.equal((await slow).length, chunks, 'every chunk read'); + const stalled = source.getTile({ x: 0, y: 0, z: 0 }); + await settle(); + push(false); + await settle(); + t.mock.timers.tick(STREAM_IDLE_TIMEOUT_MS); + await assert.rejects(stalled, /did not answer in time/); + }, +); + +test('the box search asks for a few pictures: the API returns them nearest first', async () => { + const { urls, fetchImpl } = fakeFetch(() => json({ features: [] })); + const source = createPanoramaxSource({ fetchImpl }); + await source.nearestItems({ lon: 2.35, lat: 48.85 }); + const box = urls + .map((url) => new URL(url)) + .find((url) => url.searchParams.has('bbox')); + assert.ok(box, 'a box search'); + assert.equal(box.searchParams.get('limit'), String(NEAREST_BOX_LIMIT)); + assert.ok(NEAREST_BOX_LIMIT <= 20); +}); + +test('the cones’ tiles have their own connection: never behind coverage tiles, the newest first', async () => { + const { PICTURE_TILE_CONCURRENCY, TILE_CONCURRENCY } = + await import('./policy.js'); + assert.equal(PICTURE_TILE_CONCURRENCY, 1); + const held = new Map(); + const asked = []; + const source = createPanoramaxSource({ + fetchImpl: (url) => { + asked.push(url.replace(/^.*\/map\//, '')); + return new Promise((resolve) => + held.set(url.replace(/^.*\/map\//, ''), () => + resolve(new Response(null, { status: 204 })), + ), + ); + }, + }); + const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + // Coverage fills its lane and queues. + const coverage = Array.from({ length: TILE_CONCURRENCY + 2 }, (_, i) => + source.getTile({ x: i, y: 0, z: 10 }), + ); + await settle(); + assert.equal(asked.length, TILE_CONCURRENCY); + // A cone tile goes at once; two more wait, the newer first. + const pictures = [1, 2, 3].map((x) => + source.getTile({ x, y: 9, z: 15 }, { lane: 'pictures' }), + ); + await settle(); + assert.deepEqual(asked.slice(TILE_CONCURRENCY), ['15/1/9.mvt']); + held.get('15/1/9.mvt')(); + await settle(); + assert.deepEqual(asked.slice(TILE_CONCURRENCY), ['15/1/9.mvt', '15/3/9.mvt']); + for (let round = 0; round < 6; round++) { + for (const go of held.values()) go(); + await settle(); + } + await Promise.all([...coverage, ...pictures]); +}); + test('a lookup near the antimeridian keeps its box on the map', async () => { const { urls, fetchImpl } = fakeFetch(() => json({ features: [] })); const source = createPanoramaxSource({ fetchImpl }); diff --git a/src/layers/streetLevel/providers/panoramax/viewer.js b/src/layers/streetLevel/providers/panoramax/viewer.js index 3e26e4d56..902e95156 100644 --- a/src/layers/streetLevel/providers/panoramax/viewer.js +++ b/src/layers/streetLevel/providers/panoramax/viewer.js @@ -31,7 +31,13 @@ export const PANORAMAX_SOURCES = Object.freeze({ Boolean(photo.isPano && (photo.urls.sd || hdDecodable(photo))), // Without an SD derivative the original goes straight to the texture size. panoFirst: (photo) => { - if (photo.urls.sd) return { url: photo.urls.sd, width: SD_WIDTH }; + // Decoded at SD's own size, whatever the host actually serves. + if (photo.urls.sd) + return { + url: photo.urls.sd, + width: SD_WIDTH, + resize: [SD_WIDTH, SD_WIDTH / 2], + }; if (!hdDecodable(photo)) return null; const width = Math.min(photo.width, PANO_FULL_SIZE[0]); return { url: photo.urls.hd, width, resize: [width, width / 2] }; @@ -51,15 +57,20 @@ export const PANORAMAX_SOURCES = Object.freeze({ timeoutMs: HD_TIMEOUT_MS, }; }, - // A 360° picture shown flat (no WebGL) is its whole equirectangular image; - // an original too big to decode is skipped for the thumbnail. - flatFirst: (photo) => [ + // A 360° picture shown flat (no WebGL) is its whole equirectangular image. + // When SD fails the thumbnail stands in; the original (megabytes) only + // for the expanded viewer, and only one of a size it may decode. + flatFirst: (photo, { expanded = false } = {}) => [ photo.urls.sd, - !photo.isPano || hdDecodable(photo) ? photo.urls.hd : null, + expanded && hdDecodable(photo) ? photo.urls.hd : null, photo.urls.thumb, ], + // The original once the viewer is wider than SD, if its width is known + // and decodable (a 200 MP phone photo is 16,000 px across). flatSharp: (photo, pixels) => - photo.urls.sd && photo.urls.hd && pixels > SD_WIDTH ? photo.urls.hd : null, + photo.urls.sd && hdDecodable(photo) && pixels > SD_WIDTH + ? photo.urls.hd + : null, describe: (photo) => ({ // Where no author can be named, the instance holding the photo is. title: !photo.creator && photo.via ? `via ${photo.via}` : null, diff --git a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs index 614a50e13..3f8ae4789 100644 --- a/src/layers/streetLevel/providers/panoramax/viewer.test.mjs +++ b/src/layers/streetLevel/providers/panoramax/viewer.test.mjs @@ -140,12 +140,14 @@ test('a 360° picture shows its SD equirectangular image, then HD shrunk to 4,09 assert.equal(canvas.hidden, false); assert.equal(picture.hidden, true); assert.equal(viewer.supportsRenderModes(), false, 'no frame to fit'); - assert.deepEqual(loads[0], { + const { signal, ...first } = loads[0]; + assert.deepEqual(first, { url: PHOTOS.pano.urls.sd, crossOrigin: true, - resize: null, + resize: [2048, 1024], failure: 'The Panoramax photo could not be loaded', }); + assert.equal(signal.aborted, false, 'the open can stop it'); assert.equal(poses.at(-1).isPano, true); assert.equal(poses.at(-1).bearing, 100); await new Promise((resolve) => setTimeout(resolve, 0)); @@ -258,7 +260,7 @@ test('without WebGL a 360° picture shows its whole equirectangular image flat', assert.equal(picture.src, PHOTOS.pano.urls.sd); }); -test('a picture whose SD fails falls back to HD, then the thumbnail', async () => { +test('a picture whose SD fails falls back to the thumbnail; the original only for the expanded viewer', async () => { const tried = []; const viewer = createPanoramaxViewer({ getPhoto: async () => PHOTOS.flat, @@ -273,12 +275,18 @@ test('a picture whose SD fails falls back to HD, then the thumbnail', async () = const host = fakeHost(); await viewer.mount(host); await viewer.open('flat'); + assert.deepEqual(tried, [PHOTOS.flat.urls.sd, PHOTOS.flat.urls.thumb]); + assert.equal(parts(host).picture.src, PHOTOS.flat.urls.thumb); + // Expanded, the original is worth its megabytes. + tried.length = 0; + parts(host).root.closest = (selector) => + selector.includes('sl-viewer-wrap-expanded') ? {} : null; + await viewer.open('flat'); assert.deepEqual(tried, [ PHOTOS.flat.urls.sd, PHOTOS.flat.urls.hd, PHOTOS.flat.urls.thumb, ]); - assert.equal(parts(host).picture.src, PHOTOS.flat.urls.thumb); }); test('a photo with no author to name is captioned with the instance holding it', () => { From c42b072e99e3a27dc6613ad1c52fa4931c00ba5a Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 10:48:38 -0700 Subject: [PATCH 62/95] fix(overpass): tag caches by the sorted upstream set and cover installations Uses a full SHA-256 of the sorted instance list, so reordering the same instances for failover keeps the cache. Adds the installation-cache repoint test (tagged, reordered, repointed, untagged, unconfigured) and the changelog and current-state notes. --- CHANGELOG.md | 3 +++ docs/CURRENT-STATE.md | 4 ++++ server/providers/overpass/constants.js | 8 ++++---- src/data/installationProxy.test.mjs | 28 ++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95eff69e9..a56c13575 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +- Overpass and military-installation disk caches no longer serve answers from a + previously configured `OVERPASS_UPSTREAMS` instance after it is repointed. + - A Radio Browser mirror answering 204, 205 or 304, or a status outside 200–599, no longer crashes the local dev/preview server. Null-body statuses get an empty body; anything else that cannot become a response fails that diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1b9e3e845..60f7bfbbf 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4132,6 +4132,10 @@ for, resolved lazily after env loading. It replaces the empty default list; there is no built-in fallback. Local/private instances are allowed. The stable `gods-eye-view/…` User-Agent is unchanged. URLs never appear in response headers or upstream errors, including old cached endpoint metadata. +Disk cache entries for both routes carry a SHA-256 tag of the configured +instance set (order does not matter) and are served only while it matches, so +repointing `OVERPASS_UPSTREAMS` refetches instead of serving another instance's +answers; untagged entries stop serving once an instance is configured. The shared transport guards `/api/overpass` and `/api/military-installations`. With no configured upstream, each route serves existing cache data (including diff --git a/server/providers/overpass/constants.js b/server/providers/overpass/constants.js index 2a0d4a3ef..6cfd7257a 100644 --- a/server/providers/overpass/constants.js +++ b/server/providers/overpass/constants.js @@ -40,10 +40,10 @@ let upstreamMemo = { raw: null, endpoints: [], tag: 'none' }; */ function upstreamTag(endpoints) { if (!endpoints.length) return 'none'; - return createHash('sha1') - .update(endpoints.join('\n')) - .digest('hex') - .slice(0, 12); + // Sorted, so reordering the same instances for failover keeps the cache. + return createHash('sha256') + .update([...endpoints].sort().join('\n')) + .digest('hex'); } /** Resolve after environment loading. Public Overpass instances are not used by default. */ diff --git a/src/data/installationProxy.test.mjs b/src/data/installationProxy.test.mjs index 08b14de99..308ca9dba 100644 --- a/src/data/installationProxy.test.mjs +++ b/src/data/installationProxy.test.mjs @@ -26,6 +26,7 @@ import { validMilitaryInstallationBox, writeMilitaryInstallationDisk, } from '../../vite.config.js'; +import { resolveOverpassUpstreamTag } from '../../server/providers/overpass/constants.js'; const DAY_MS = 86_400_000; test('installation failure reasons disclose no raw upstream error and do not guess overload', () => { @@ -274,3 +275,30 @@ test('an unwritable cache directory degrades to a miss instead of throwing', asy await fsp.rm(dir, { recursive: true, force: true }); } }); + +test('an installation entry is not served after OVERPASS_UPSTREAMS is repointed', async (t) => { + const prior = process.env.OVERPASS_UPSTREAMS; + t.after(() => { + if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS; + else process.env.OVERPASS_UPSTREAMS = prior; + }); + const dir = tempCacheDir(); + const key = '30.250,-97.750,30.300,-97.700'; + try { + process.env.OVERPASS_UPSTREAMS = 'https://a.example/api/interpreter,https://b.example/api/interpreter'; + const tag = resolveOverpassUpstreamTag(); + await writeMilitaryInstallationDisk(key, { ...entry(Date.now()), upstreams: tag }, dir); + assert.ok(await readMilitaryInstallationDisk(key, TTL_MS, dir), 'its own entry is a hit'); + process.env.OVERPASS_UPSTREAMS = 'https://b.example/api/interpreter,https://a.example/api/interpreter'; + assert.equal(resolveOverpassUpstreamTag(), tag, 'reordering the same instances keeps the tag'); + process.env.OVERPASS_UPSTREAMS = 'https://c.example/api/interpreter'; + assert.equal(await readMilitaryInstallationDisk(key, TTL_MS, dir), null, 'fresh read refused'); + assert.equal(await readMilitaryInstallationDisk(key, Infinity, dir), null, 'stale read refused'); + await writeMilitaryInstallationDisk('legacy', entry(Date.now()), dir); + assert.equal(await readMilitaryInstallationDisk('legacy', Infinity, dir), null, 'untagged entry refused'); + delete process.env.OVERPASS_UPSTREAMS; + assert.ok(await readMilitaryInstallationDisk(key, Infinity, dir), 'nothing configured accepts any entry'); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); From 86b7dc8d3c922c562751f82245e6f7f999496250 Mon Sep 17 00:00:00 2001 From: NotProniss <71785359+NotProniss@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:12:27 +0000 Subject: [PATCH 63/95] feat(weather): add satellite cloud filter strength slider Blend the existing infrared brightness key from Full (0%) to Clouds only (100%), preserving Soft/Vivid as independent opacity presets. Commit filter changes on slider release and retain legacy share-link mode codes. --- CHANGELOG.md | 6 ++ docs/CURRENT-STATE.md | 12 ++- src/data/layerState.js | 23 +++++- src/data/layerState.test.mjs | 64 +++++++++++++-- src/layers/weather/index.js | 45 ++++++++--- src/layers/weather/infraredAlpha.js | 17 +++- src/layers/weather/infraredAlpha.test.mjs | 15 ++++ src/layers/weather/infraredImage.test.mjs | 9 +++ src/layers/weather/weather.test.mjs | 70 +++++++++++++++-- src/ui/railCardBlocks.js | 94 ++++++++++++++++++++++- src/ui/railCards.test.mjs | 75 ++++++++++++++++++ src/ui/styles/weather.css | 29 +++++++ 12 files changed, 421 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dee202852..56f5c65a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,12 @@ ## [Unreleased] +- Satellite clouds keep Clouds only / Full as quick endpoints and add a continuous + cloud-filter slider: 0% is Full, 100% is the existing brightness key, and + intermediate values blend the keyed alpha. The slider commits on release to + avoid restaging imagery for every drag event. Filter percentages round-trip in + share links; legacy mode codes retain their old endpoint meanings. + - Flights and Military Flights highlight emergency transponder broadcasts in fleet billboards, 3D models and Cockpit contacts. Colors update when a status starts or clears. Emergency squawks outrank ADS-B priority statuses; GEV diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 60f7bfbbf..629d1416f 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -256,8 +256,13 @@ Clouds only applies a soft brightness ramp to decoded pixels once, using Cesium's sRGB-to-linear conversion (`channel ** 2.2`) and smoothstep from 0.40 to 0.70. The old 0.55 threshold is the ramp midpoint. RGB and source alpha are preserved in Full image mode; both modes use the chosen layer opacity. This -is a display filter, not a cloud mask. Satellite share links retain the display -mode; observation history remains transient and links open latest. +is a display filter, not a cloud mask. The cloud-filter slider is a continuous +0–100% strength: 0% is Full, 100% is the existing Clouds only key, and +intermediate values blend their alpha. The value commits on slider release to +avoid restaging for every pointer event. Satellite share links carry the filter +strength; legacy `f`/`a` values keep their Clouds only / Full meanings. Soft and +Vivid remain layer-opacity presets (40%/70%); history remains transient and links +open latest. Global infrared fetches one capped 4 MiB, 2048×1024 mosaic per frame and decodes and processes it before staging. Globe hosts crop that canvas into 256 px tiles to maximum level 3 on a geographic 2×1 root grid bounded to the manifest extent, @@ -302,7 +307,8 @@ release commits immediately. Latest selects the newest frame per product, and history labels the actual displayed frame as synced or nearest. Missing-frame messages use each product's eligibility gap (30 minutes, or three hours for the global mosaic). Wind shows forecast valid and issue times and does not follow -history. Satellite clouds uses Clouds only / Full image configuration labels. +history. Satellite clouds uses Clouds only / Full controls, a continuous cloud +filter slider and Soft/Vivid opacity presets in its right-rail settings. Left weather rows retain only the toggle and one status line with a source tooltip. Configuration and the cyclone Storms list live in the cards, as do the diff --git a/src/data/layerState.js b/src/data/layerState.js index 1db10494b..d7175fd67 100644 --- a/src/data/layerState.js +++ b/src/data/layerState.js @@ -296,6 +296,24 @@ function imageryPinOption(key, token) { }); } +function satelliteFilterOption() { + const percentage = boundedIntegerOption('infrared', 'i', 100, { + min: 0, + max: 100, + }); + return Object.freeze({ + ...percentage, + normalize: (value) => + value === 'filtered' + ? 100 + : value === 'full' + ? 0 + : percentage.normalize(value), + decode: (value) => + value === 'f' ? 100 : value === 'a' ? 0 : percentage.decode(value), + }); +} + const OPTION_GROUPS = Object.freeze({ traffic: Object.freeze([ enumOption('roadMode', 'r', null, ['tomtom', 'osm', 'hybrid'], { @@ -317,10 +335,7 @@ const OPTION_GROUPS = Object.freeze({ }), ]), 'weather-satellite': Object.freeze([ - enumOption('infrared', 'i', 'filtered', ['filtered', 'full'], { - filtered: 'f', - full: 'a', - }), + satelliteFilterOption(), enumOption('opacity', 'o', 'strong', ['light', 'strong'], { light: 'l', strong: 's', diff --git a/src/data/layerState.test.mjs b/src/data/layerState.test.mjs index 30f251b7b..631e5dfec 100644 --- a/src/data/layerState.test.mjs +++ b/src/data/layerState.test.mjs @@ -2422,23 +2422,74 @@ test('wind appearance shares round trip while old links retain weather defaults' assert.deepEqual(invalid.options.wind, defaults); }); -test('observed weather round trips product and opacity without persisting historical playback', () => { +test('observed weather round trips product and presets without persisting historical playback', () => { const state = normalizeLayerState({ enabledLayerIds: ['weather-radar', 'weather-satellite'], options: { 'weather-radar': { opacity: 'light', play: true }, - 'weather-satellite': { product: 'clouds', opacity: 'light', step: -1 }, + 'weather-satellite': { + product: 'clouds', + opacity: 'light', + infrared: 'filtered', + step: -1, + }, }, }); const params = new URLSearchParams(encode(state)); const decoded = decodeLayerStateParams(params); assert.deepEqual(decoded, state); assert.equal(state.options['weather-satellite'].product, 'clouds'); + assert.equal(state.options['weather-satellite'].opacity, 'light'); + assert.equal(state.options['weather-satellite'].infrared, 100); assert.equal(Object.hasOwn(state.options['weather-radar'], 'play'), false); }); -test('satellite infrared display mode round trips and invalid or absent values use filtered', () => { - for (const infrared of ['full', 'filtered', undefined, 'invalid']) { +test('satellite filter strength percentages round-trip and legacy mode codes remain readable', () => { + const state = normalizeLayerState({ + enabledLayerIds: ['weather-satellite'], + options: { 'weather-satellite': { infrared: 53, opacity: 'light' } }, + }); + const params = encodeLayerStateParams( + new URLSearchParams([['v', '2']]), + state, + ); + assert.deepEqual(decodeLayerStateParams(params), state); + assert.equal(state.options['weather-satellite'].infrared, 53); + assert.equal(state.options['weather-satellite'].opacity, 'light'); + for (const [legacy, expected] of [ + ['f', 100], + ['a', 0], + ]) { + const decoded = decodeLayerStateParams( + new URLSearchParams(`v=2&l=o&lo=o.i.${legacy}`), + ); + assert.equal(decoded.options['weather-satellite'].infrared, expected); + } + for (const [legacy, expected] of [ + ['filtered', 100], + ['full', 0], + ]) { + const normalized = normalizeLayerState({ + enabledLayerIds: ['weather-satellite'], + options: { 'weather-satellite': { infrared: legacy } }, + }); + assert.equal(normalized.options['weather-satellite'].infrared, expected); + } + const invalid = normalizeLayerState({ + enabledLayerIds: ['weather-satellite'], + options: { 'weather-satellite': { infrared: 101 } }, + }); + assert.equal(invalid.options['weather-satellite'].infrared, 100); +}); + +test('satellite cloud filter round trips and invalid or absent values default to Clouds only', () => { + for (const [infrared, expected] of [ + ['full', 0], + ['filtered', 100], + [53, 53], + [undefined, 100], + ['invalid', 100], + ]) { const state = normalizeLayerState({ enabledLayerIds: ['weather-satellite'], options: { @@ -2454,10 +2505,7 @@ test('satellite infrared display mode round trips and invalid or absent values u decodeLayerStateParams(new URLSearchParams(encode(state))), state, ); - assert.equal( - state.options['weather-satellite'].infrared, - infrared === 'full' ? 'full' : 'filtered', - ); + assert.equal(state.options['weather-satellite'].infrared, expected); assert.equal( Object.hasOwn(state.options['weather-satellite'], 'step'), false, diff --git a/src/layers/weather/index.js b/src/layers/weather/index.js index 8d6d331eb..beada1ef6 100644 --- a/src/layers/weather/index.js +++ b/src/layers/weather/index.js @@ -50,7 +50,7 @@ export function createWeatherLayer({ const satellite = !radar && !lightning; let product = radar ? 'radar' : lightning ? 'lightning' : 'clouds-regional'; let opacity = 'strong'; - let infrared = 'filtered'; + let infrared = 100; let viewer = null, rendering = null, manifest = null, @@ -382,11 +382,20 @@ export function createWeatherLayer({ } }, setParams(params = {}) { + const nextInfrared = satellite + ? params.infrared === 'full' + ? 0 + : params.infrared === 'filtered' + ? 100 + : params.infrared + : null; const infraredChanged = satellite && - ['filtered', 'full'].includes(params.infrared) && - params.infrared !== infrared; - if (infraredChanged) infrared = params.infrared; + Number.isInteger(nextInfrared) && + nextInfrared >= 0 && + nextInfrared <= 100 && + nextInfrared !== infrared; + if (infraredChanged) infrared = nextInfrared; if (['light', 'strong'].includes(params.opacity)) { opacity = params.opacity; rendering?.setAlpha(opacity === 'light' ? 0.4 : satellite ? 0.7 : 0.8); @@ -594,17 +603,17 @@ export function createWeatherLayer({ { id: 'filtered', label: 'Clouds only', - active: infrared === 'filtered', - params: { infrared: 'filtered' }, + active: infrared === 100, + params: { infrared: 100 }, title: - 'Dim everything but the bright, cold cloud tops; a brightness filter, not a cloud mask', + 'Apply the full cloud brightness key; 100% filter strength', }, { id: 'full', label: 'Full', - active: infrared === 'full', - params: { infrared: 'full' }, - title: 'The complete infrared image at the chosen opacity', + active: infrared === 0, + params: { infrared: 0 }, + title: 'Show the complete infrared image; 0% filter strength', }, ] : []), @@ -656,7 +665,21 @@ export function createWeatherLayer({ { id: 'image', label: 'IMAGE', - chips: controls.chips.filter(({ params }) => params.infrared), + chips: controls.chips.filter( + ({ params }) => params.infrared !== undefined, + ), + range: { + id: 'cloud-filter', + param: 'infrared', + min: 0, + max: 100, + step: 1, + value: infrared, + label: 'FILTER', + unit: '%', + ariaLabel: 'Cloud filter strength', + commitOnChange: true, + }, }, ] : []), diff --git a/src/layers/weather/infraredAlpha.js b/src/layers/weather/infraredAlpha.js index 007c7486f..f42bda603 100644 --- a/src/layers/weather/infraredAlpha.js +++ b/src/layers/weather/infraredAlpha.js @@ -1,10 +1,18 @@ export const INFRARED_ALPHA_T0 = 0.4; export const INFRARED_ALPHA_T1 = 0.7; -/** Return display pixels without changing RGB or the source RGBA bytes. */ -export function infraredAlpha(rgba, mode = 'filtered') { +/** Return display pixels without changing RGB or the source RGBA bytes. + * `strength` is 0–100: zero is the full image and 100 is the cloud key. + */ +export function infraredAlpha(rgba, strength = 'filtered') { const pixels = new Uint8ClampedArray(rgba); - if (mode === 'full') return pixels; + const amount = + strength === 'full' + ? 0 + : strength === 'filtered' || !Number.isFinite(strength) + ? 1 + : Math.max(0, Math.min(100, strength)) / 100; + if (amount === 0) return pixels; for (let i = 0; i < pixels.length; i += 4) { // Match Cesium's czm_srgbToLinear conversion in draped imagery. const linearMax = @@ -17,7 +25,8 @@ export function infraredAlpha(rgba, mode = 'filtered') { (INFRARED_ALPHA_T1 - INFRARED_ALPHA_T0), ), ); - pixels[i + 3] = rgba[i + 3] * t * t * (3 - 2 * t); + const filtered = t * t * (3 - 2 * t); + pixels[i + 3] = rgba[i + 3] * (1 - amount + filtered * amount); } return pixels; } diff --git a/src/layers/weather/infraredAlpha.test.mjs b/src/layers/weather/infraredAlpha.test.mjs index f0ee95516..950271a9d 100644 --- a/src/layers/weather/infraredAlpha.test.mjs +++ b/src/layers/weather/infraredAlpha.test.mjs @@ -40,6 +40,21 @@ test('filtered infrared preserves RGB and input bytes, scales existing alpha, an ); }); +test('filter strength interpolates between the full infrared image and the cloud key', () => { + const pixels = new Uint8ClampedArray([ + 0, 0, 0, 255, 194, 0, 0, 200, 12, 50, 217, 128, + ]); + assert.deepEqual([...infraredAlpha(pixels, 0)], [...pixels]); + assert.deepEqual( + infraredAlpha(pixels, 100), + infraredAlpha(pixels, 'filtered'), + ); + assert.deepEqual( + [...infraredAlpha(pixels, 50)], + [0, 0, 0, 128, 194, 0, 0, 149, 12, 50, 217, 128], + ); +}); + test('full infrared preserves every channel including partial and zero alpha', () => { const pixels = new Uint8ClampedArray([ 12, 50, 194, 80, 0, 0, 0, 255, 255, 200, 0, 0, diff --git a/src/layers/weather/infraredImage.test.mjs b/src/layers/weather/infraredImage.test.mjs index f3937abc3..61e532564 100644 --- a/src/layers/weather/infraredImage.test.mjs +++ b/src/layers/weather/infraredImage.test.mjs @@ -32,6 +32,15 @@ test('decoded pixels are processed once on a same-sized canvas', () => { assert.equal(output.height, 1); assert.equal(output.image, image); assert.deepEqual([...output.pixels], [194, 0, 0, 98]); + assert.deepEqual( + [...processInfraredImage(image, 50, canvas).pixels], + [194, 0, 0, 149], + 'half-strength filter retains a midpoint haze', + ); + assert.deepEqual( + [...processInfraredImage(image, 0, canvas).pixels], + [194, 0, 0, 200], + ); assert.deepEqual( [...processInfraredImage(image, 'full', canvas).pixels], [194, 0, 0, 200], diff --git a/src/layers/weather/weather.test.mjs b/src/layers/weather/weather.test.mjs index a80ad6560..1a4ba5177 100644 --- a/src/layers/weather/weather.test.mjs +++ b/src/layers/weather/weather.test.mjs @@ -444,11 +444,14 @@ function layerHarness({ let active = null; let clearCount = 0; let hidden = false; + let alpha = null; const rendering = { rehome() { active?.finish(false); }, - setAlpha() {}, + setAlpha(value) { + alpha = value; + }, setHidden(value) { hidden = value; if (value) active?.finish(false); @@ -514,6 +517,7 @@ function layerHarness({ documentRef, moveEnd, clears: () => clearCount, + getAlpha: () => alpha, }; } @@ -1591,8 +1595,8 @@ test('satellite mode chips restage history without changing time or latest-follo await flush(); h.layer.setParams({ infrared: 'full' }); assert.equal(h.stages[2].time, times[1]); - assert.equal(h.stages[2].infrared, 'full'); - assert.equal(h.layer.getParams().infrared, 'full'); + assert.equal(h.stages[2].infrared, 0); + assert.equal(h.layer.getParams().infrared, 0); assert.deepEqual( h.layer .getRowControls() @@ -1605,11 +1609,59 @@ test('satellite mode chips restage history without changing time or latest-follo assert.equal(h.layer.getDiagnostics().followLatest, false); assert.equal(h.layer.getDiagnostics().time, times[1]); h.layer.setParams({ infrared: 'invalid' }); - assert.equal(h.layer.getParams().infrared, 'full'); + assert.equal(h.layer.getParams().infrared, 0); assert.equal(h.stages.length, 3); h.layer.destroy(); }); +test('satellite cloud filter spans Full to Clouds only without changing layer opacity', () => { + const h = layerHarness({ id: 'weather-satellite' }); + assert.equal(h.layer.getParams().infrared, 100); + assert.equal(h.layer.getParams().opacity, 'strong'); + assert.equal(h.getAlpha(), 0.7); + const setting = h.layer + .getRowControls() + .summary.settings.find(({ id }) => id === 'image'); + assert.equal(setting.range.param, 'infrared'); + assert.equal(setting.range.value, 100); + assert.equal(setting.range.min, 0); + assert.equal(setting.range.max, 100); + assert.equal(setting.range.step, 1); + assert.equal(setting.range.commitOnChange, true); + + const full = h.layer.getRowControls().chips.find(({ id }) => id === 'full'); + h.layer.setParams(full.params); + assert.equal(h.layer.getParams().infrared, 0); + assert.equal(h.getAlpha(), 0.7); + assert.equal( + h.layer.getRowControls().chips.find(({ id }) => id === 'full').active, + true, + ); + + h.layer.setParams({ infrared: 53 }); + assert.equal(h.layer.getParams().infrared, 53); + assert.equal(h.getAlpha(), 0.7, 'filter strength is not overall opacity'); + assert.equal( + h.layer.getRowControls().summary.settings.find(({ id }) => id === 'image') + .range.value, + 53, + ); + h.layer.setParams({ opacity: 'light' }); + assert.equal(h.layer.getParams().opacity, 'light'); + assert.equal( + h.getAlpha(), + 0.4, + 'Soft remains the independent opacity preset', + ); + h.layer.setParams({ infrared: 101 }); + assert.equal( + h.layer.getParams().infrared, + 53, + 'invalid filter values are ignored', + ); + h.layer.destroy(); +}); + test('a rapid third selection waits for retirement before installing, and clear cancels that wait', async () => { for (const clear of [false, true]) { const h = renderingHarness(); @@ -1914,8 +1966,14 @@ test('observed descriptors keep configuration only and label satellite clouds by ); assert.equal( controls.chips.find(({ id }) => id === 'filtered').title, - 'Dim everything but the bright, cold cloud tops; a brightness filter, not a cloud mask', + 'Apply the full cloud brightness key; 100% filter strength', ); + const imageSetting = controls.summary.settings.find( + ({ id }) => id === 'image', + ); + assert.equal(imageSetting.range.param, 'infrared'); + assert.equal(imageSetting.range.value, 100); + assert.equal(imageSetting.range.commitOnChange, true); h.layer.setParams({ product: 'clouds' }); assert.equal( h.layer.getRowControls().summary.coverage, @@ -2196,7 +2254,7 @@ test('layer warms the next advertised observation only after a successful playin warmed.map(({ time }) => time), [times[0]], ); - assert.deepEqual(warmed[0].options, { infrared: 'filtered' }); + assert.deepEqual(warmed[0].options, { infrared: 100 }); const next = clock.setTarget(times[0]); await flush(); h.stages.at(-1).finish(); diff --git a/src/ui/railCardBlocks.js b/src/ui/railCardBlocks.js index a2132f7ad..b58e1134f 100644 --- a/src/ui/railCardBlocks.js +++ b/src/ui/railCardBlocks.js @@ -30,6 +30,7 @@ export function createRailCardBlocks({ container, cardId, onParams }) { }; const dispose = (block) => { block.removeClick?.(); + block.removeInput?.(); block.children?.destroy(); block.root.remove(); }; @@ -74,6 +75,64 @@ export function createRailCardBlocks({ container, cardId, onParams }) { ); block.chips = make('div', 'rail-card-chips', root); block.chips.setAttribute('role', 'group'); + block.rangeRow = make('div', 'rail-card-setting-range-row', root); + block.rangeLabel = make( + 'span', + 'rail-card-setting-range-label', + block.rangeRow, + ); + block.rangeInput = make( + 'input', + 'rail-card-setting-range', + block.rangeRow, + ); + block.rangeInput.type = 'range'; + block.rangeValue = make( + 'span', + 'rail-card-setting-range-value', + block.rangeRow, + ); + const rangeParams = () => { + const range = block.props.range; + const value = Number(block.rangeInput.value); + if ( + !range?.param || + range.disabled || + !Number.isFinite(value) || + value < Number(range.min) || + value > Number(range.max) + ) + return null; + return { params: { [range.param]: value } }; + }; + const onInput = () => { + const range = block.props.range; + const value = Number(block.rangeInput.value); + const params = rangeParams(); + if (!params) return; + set( + block.rangeValue, + 'textContent', + `${value}${range.unit || ''}`, + ); + attribute( + block.rangeInput, + 'aria-valuetext', + `${value}${range.unit || ''}`, + ); + if (!range.commitOnChange) dispatch(params); + }; + const onChange = () => { + if (!block.props.range?.commitOnChange) return; + const params = rangeParams(); + if (params) dispatch(params); + }; + block.rangeInput.addEventListener('input', onInput); + block.rangeInput.addEventListener('change', onChange); + block.removeInput = () => { + block.rangeInput.removeEventListener('input', onInput); + block.rangeInput.removeEventListener('change', onChange); + }; bind((event) => { const button = event.target?.closest?.('[data-chip-id]'); dispatch( @@ -124,17 +183,48 @@ export function createRailCardBlocks({ container, cardId, onParams }) { blocks.set(props.id, block); } block.props = props; - if (props.type === 'settings') + if (props.type === 'settings') { block.children.update( (props.settings || []).map((setting) => ({ ...setting, type: 'setting', })), ); - else if (props.type === 'setting') { + } else if (props.type === 'setting') { set(block.label, 'textContent', props.label); attribute(block.chips, 'aria-label', props.label); syncChipGroup(block.chips, props.chips); + const range = props.range; + set(block.rangeRow, 'hidden', !range); + if (range) { + const min = Number(range.min); + const max = Number(range.max); + const step = Number(range.step); + const candidate = Number(range.value); + const value = Math.min( + max, + Math.max(min, Number.isFinite(candidate) ? candidate : min), + ); + set(block.rangeLabel, 'textContent', range.label || ''); + set(block.rangeLabel, 'hidden', !range.label); + set(block.rangeInput.dataset, 'rangeId', range.id || props.id); + set(block.rangeInput, 'min', String(min)); + set(block.rangeInput, 'max', String(max)); + set(block.rangeInput, 'step', String(step)); + set(block.rangeInput, 'value', String(value)); + set(block.rangeInput, 'disabled', Boolean(range.disabled)); + attribute( + block.rangeInput, + 'aria-label', + range.ariaLabel || props.label, + ); + attribute( + block.rangeInput, + 'aria-valuetext', + `${value}${range.unit || ''}`, + ); + set(block.rangeValue, 'textContent', `${value}${range.unit || ''}`); + } } else if (props.type === 'list') syncRowList(block.root, props.list); else if (props.type === 'result') { set(block.label, 'textContent', props.label); diff --git a/src/ui/railCards.test.mjs b/src/ui/railCards.test.mjs index e465e3cc7..48fe93964 100644 --- a/src/ui/railCards.test.mjs +++ b/src/ui/railCards.test.mjs @@ -119,6 +119,81 @@ test('footer buttons retain focus and current callbacks; links use safe new tabs button.click(); assert.deepEqual(calls, [2]); }); +test('setting ranges stay below presets and commit expensive filters on release', () => { + const f = railFixture(); + const calls = []; + const view = createRailCards({ + ...f, + onParams: (...args) => calls.push(args), + }); + const model = (value) => ({ + ...card, + blocks: [ + { + id: 'settings', + type: 'settings', + settings: [ + { + id: 'image', + label: 'IMAGE', + chips: [ + { + id: 'clouds-only', + label: 'Clouds only', + params: { infrared: 100 }, + }, + { id: 'full', label: 'Full', params: { infrared: 0 } }, + ], + range: { + id: 'cloud-filter', + param: 'infrared', + min: 0, + max: 100, + step: 1, + value, + label: 'FILTER', + unit: '%', + ariaLabel: 'Cloud filter strength', + commitOnChange: true, + }, + }, + ], + }, + ], + }); + view.update([model(100)]); + const setting = f.find((node) => node.dataset.blockId === 'image'); + assert.equal(setting.children[1].className, 'rail-card-chips'); + assert.equal(setting.children[2].className, 'rail-card-setting-range-row'); + assert.equal(setting.children[2].children[0].textContent, 'FILTER'); + const slider = f.find((node) => node.dataset.rangeId === 'cloud-filter'); + assert.equal(slider.tagName, 'INPUT'); + assert.equal(slider.type, 'range'); + assert.equal(slider.min, '0'); + assert.equal(slider.max, '100'); + assert.equal(slider.step, '1'); + assert.equal(slider.value, '100'); + assert.equal(slider.getAttribute('aria-label'), 'Cloud filter strength'); + slider.focus(); + slider.value = '53'; + slider.dispatchEvent(new Event('input')); + assert.equal(calls.length, 0, 'expensive image work waits until release'); + assert.equal( + f.find((node) => node.className === 'rail-card-setting-range-value') + .textContent, + '53%', + ); + slider.dispatchEvent(new Event('change')); + assert.deepEqual(calls.at(-1), ['a', { infrared: 53 }]); + view.update([model(53)]); + assert.equal(slider.value, '53'); + assert.equal(f.document.activeElement, slider); + view.destroy(); + slider.value = '41'; + slider.dispatchEvent(new Event('change')); + assert.equal(calls.length, 1, 'destroy removes the range listener'); +}); + test('legend validates colors, preserves the freezing anchor and ignores CSSOM normalization', () => { let writes = 0; const f = railFixture(() => writes++); diff --git a/src/ui/styles/weather.css b/src/ui/styles/weather.css index 3e83561fb..02ee75ea6 100644 --- a/src/ui/styles/weather.css +++ b/src/ui/styles/weather.css @@ -327,6 +327,35 @@ body.recording-mode .gev-wind-inspection-marker { .rail-card-setting .rail-card-chips { margin-top: 0; } +.rail-card-setting-range-row { + display: flex; + grid-column: 2; + align-items: center; + gap: 7px; + min-width: 0; +} +.rail-card-setting-range-label { + flex: 0 0 6ch; + color: var(--text-dim); + font: 8px var(--font-mono); + white-space: nowrap; +} +.rail-card-setting-range { + width: 100%; + min-width: 0; + accent-color: var(--accent); + cursor: pointer; +} +.rail-card-setting-range:focus-visible { + outline: 2px solid var(--text-primary); + outline-offset: 3px; +} +.rail-card-setting-range-value { + min-width: 3ch; + color: var(--text-primary); + font: 9px var(--font-mono); + text-align: right; +} .rail-card-result { display: grid; gap: 4px; From 8959b5b71abe5927846c4fb42dac47fad74c0325 Mon Sep 17 00:00:00 2001 From: Mazeyar Moeini Feizabadi Date: Thu, 15 Oct 2026 19:32:28 +0200 Subject: [PATCH 64/95] fix(weather): preserve filter drag previews across panel refreshes --- CHANGELOG.md | 4 +++- docs/CURRENT-STATE.md | 4 +++- src/ui/railCardBlocks.js | 15 ++++++++++++++- src/ui/railCards.test.mjs | 15 +++++++++++++++ src/ui/styles/weather.css | 3 ++- 5 files changed, 37 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56f5c65a6..351b768d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,7 +151,9 @@ cloud-filter slider: 0% is Full, 100% is the existing brightness key, and intermediate values blend the keyed alpha. The slider commits on release to avoid restaging imagery for every drag event. Filter percentages round-trip in - share links; legacy mode codes retain their old endpoint meanings. + share links; legacy mode codes retain their old endpoint meanings. Background + panel refreshes preserve the pending drag value until release. Contributed by + [NotProniss](https://github.com/NotProniss) (#1084). - Flights and Military Flights highlight emergency transponder broadcasts in fleet billboards, 3D models and Cockpit contacts. Colors update when a status diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 629d1416f..09b065630 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -259,7 +259,9 @@ preserved in Full image mode; both modes use the chosen layer opacity. This is a display filter, not a cloud mask. The cloud-filter slider is a continuous 0–100% strength: 0% is Full, 100% is the existing Clouds only key, and intermediate values blend their alpha. The value commits on slider release to -avoid restaging for every pointer event. Satellite share links carry the filter +avoid restaging for every pointer event. Status refreshes retain the uncommitted +drag preview; changed model values or disabled controls supersede it. +Satellite share links carry the filter strength; legacy `f`/`a` values keep their Clouds only / Full meanings. Soft and Vivid remain layer-opacity presets (40%/70%); history remains transient and links open latest. diff --git a/src/ui/railCardBlocks.js b/src/ui/railCardBlocks.js index b58e1134f..1726860f9 100644 --- a/src/ui/railCardBlocks.js +++ b/src/ui/railCardBlocks.js @@ -110,6 +110,7 @@ export function createRailCardBlocks({ container, cardId, onParams }) { const value = Number(block.rangeInput.value); const params = rangeParams(); if (!params) return; + if (range.commitOnChange) block.rangeDraft = { range, value }; set( block.rangeValue, 'textContent', @@ -125,6 +126,7 @@ export function createRailCardBlocks({ container, cardId, onParams }) { const onChange = () => { if (!block.props.range?.commitOnChange) return; const params = rangeParams(); + block.rangeDraft = null; if (params) dispatch(params); }; block.rangeInput.addEventListener('input', onInput); @@ -196,11 +198,22 @@ export function createRailCardBlocks({ container, cardId, onParams }) { syncChipGroup(block.chips, props.chips); const range = props.range; set(block.rangeRow, 'hidden', !range); + const draft = block.rangeDraft; + if ( + draft && + (!range || + range.disabled || + !range.commitOnChange || + ['id', 'param', 'min', 'max', 'step', 'value'].some( + (key) => range[key] !== draft.range[key], + )) + ) + block.rangeDraft = null; if (range) { const min = Number(range.min); const max = Number(range.max); const step = Number(range.step); - const candidate = Number(range.value); + const candidate = Number(block.rangeDraft?.value ?? range.value); const value = Math.min( max, Math.max(min, Number.isFinite(candidate) ? candidate : min), diff --git a/src/ui/railCards.test.mjs b/src/ui/railCards.test.mjs index 48fe93964..74629bebb 100644 --- a/src/ui/railCards.test.mjs +++ b/src/ui/railCards.test.mjs @@ -183,11 +183,26 @@ test('setting ranges stay below presets and commit expensive filters on release' .textContent, '53%', ); + view.update([model(100)]); + assert.equal(slider.value, '53', 'a status refresh retains the pending drag'); + assert.equal( + slider.getAttribute('aria-valuetext'), + '53%', + 'the accessible preview also survives a refresh', + ); slider.dispatchEvent(new Event('change')); assert.deepEqual(calls.at(-1), ['a', { infrared: 53 }]); view.update([model(53)]); assert.equal(slider.value, '53'); assert.equal(f.document.activeElement, slider); + slider.value = '41'; + slider.dispatchEvent(new Event('input')); + view.update([model(70)]); + assert.equal( + slider.value, + '70', + 'an authoritative value supersedes the draft', + ); view.destroy(); slider.value = '41'; slider.dispatchEvent(new Event('change')); diff --git a/src/ui/styles/weather.css b/src/ui/styles/weather.css index 02ee75ea6..0540fac0a 100644 --- a/src/ui/styles/weather.css +++ b/src/ui/styles/weather.css @@ -351,7 +351,8 @@ body.recording-mode .gev-wind-inspection-marker { outline-offset: 3px; } .rail-card-setting-range-value { - min-width: 3ch; + flex: 0 0 4ch; + white-space: nowrap; color: var(--text-primary); font: 9px var(--font-mono); text-align: right; From 3f6fe69c57d34e89378e8fbca2b44ff78f52b93e Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 13:07:00 -0700 Subject: [PATCH 65/95] build(docker): keep keys out of the image and run as non-root by default The build context excluded only .env, so .env.local and the like or pinokio/ENVIRONMENT would be baked into image layers; exclude every dotenv file except .env.example, plus local state and QA output. Pin the base to CI's Node 24.14.0 by digest, default to the image's node user, and correct the HOST comment. Log the trusted peers as the operator's when set. README notes that MCP and ChatGPT sign-in voice are unavailable under Docker, and how to trust no gateway when widening the publish. --- .dockerignore | 19 +++++++++++++++++-- CHANGELOG.md | 7 ++++--- Dockerfile | 27 ++++++++++++++++----------- README.md | 4 ++++ scripts/docker-start.mjs | 9 ++++++++- 5 files changed, 49 insertions(+), 17 deletions(-) diff --git a/.dockerignore b/.dockerignore index 3f5bc8a9d..174f7990a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,24 @@ -node_modules +# Keep keys and local state out of the build context: compose bind-mounts the +# checkout at run time, so the image never needs them. .git +node_modules +dist .env +.env.* +!.env.example +**/ENVIRONMENT +pinokio/ENVIRONMENT +pinokio/.installed .gev-cache .gev-logs -dist +qa-artifacts +qa-shots +screenshots +output +3d-models +.gstack +*.log +.DS_Store # AppleDouble sidecars from a macOS `tar` copy; `npm test` would otherwise # collect `._*.test.mjs` as test files. ._* diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e2cea294..2a7fc91d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -235,9 +235,10 @@ [TheSmokeDev](https://github.com/TheSmokeDev) for the credential handling from #653. -- **Docker install path.** `docker compose up` builds the pinned Node image and - serves the app on `http://localhost:4173` with no local Node install. The repo - is bind-mounted and the container runs as the host user, so `.env`, +- **Docker install path.** `docker compose up` builds on CI's Node 24.14.0 image, + pinned by digest, and serves the app on `http://localhost:4173` with no local + Node install. The repo is bind-mounted and the container runs as the host user + (never root), so `.env`, `.gev-cache`, and `.gev-logs` persist on the host exactly as they do under `npm run dev` — Provider Settings included. The launcher (`scripts/docker-start.mjs`) trusts the container's gateway address for that diff --git a/Dockerfile b/Dockerfile index b1b0876d1..d34f4887a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,11 @@ # The dev server is the whole app: Vite serves the client AND runs the API # proxies (ADS-B, AIS, Overpass, OpenAI Realtime, ...) as middleware, so there # is nothing else to containerise. See compose.yaml for the usual entry point. -FROM node:24-slim +# Same Node as CI (package.json engines, .github/workflows), pinned by digest. +FROM node:24.14.0-slim@sha256:d8e448a56fc63242f70026718378bd4b00f8c82e78d20eefb199224a4d8e33d8 # Puppeteer is only used by the QA scripts; skip its ~150MB Chromium download. ENV PUPPETEER_SKIP_DOWNLOAD=1 -WORKDIR /app # `npm test` builds fixtures with `git init`, so the suite cannot run in a # container without it (71 failures on node:24-slim, all ENOENT on git). @@ -18,19 +18,24 @@ RUN apt-get update && apt-get install -y --no-install-recommends git && rm - # git. It is the user's own checkout, so trust that one path. RUN git config --system --add safe.directory /app -COPY package.json package-lock.json ./ +# Never run as root. The image's own user is `node` (1000); compose overrides +# it with the host user so bind-mounted files stay yours. +WORKDIR /app +RUN chown node:node /app +USER node + +COPY --chown=node:node package.json package-lock.json ./ RUN npm ci -# compose runs the container as the host user so bind-mounted files stay -# yours, but node_modules is baked in as root. Vite only needs to write its -# dep-optimizer cache, so open up those two directories rather than chowning -# a few hundred MB to a UID we cannot know at build time. +# node_modules belongs to `node`, but compose may run as another UID. Vite only +# needs to write its dep-optimizer cache, so open up those two directories +# rather than chowning a few hundred MB to a UID we cannot know at build time. RUN mkdir -p node_modules/.vite && chmod 777 node_modules node_modules/.vite -COPY . . +COPY --chown=node:node . . -# Bind to all interfaces, otherwise the published port hits nothing. Vite also -# relaxes allowedHosts when HOST is 0.0.0.0, which is what lets the browser -# reach it as "localhost". +# Bind to all interfaces, otherwise the published port hits nothing. This does +# not widen the Host allowlist: IP addresses and localhost are accepted, other +# names only via GEV_ALLOWED_HOSTS (see build/allowedHosts.js). ENV HOST=0.0.0.0 ENV PORT=4173 EXPOSE 4173 diff --git a/README.md b/README.md index b90b37ab9..46007e4c2 100644 --- a/README.md +++ b/README.md @@ -171,8 +171,12 @@ Open **`http://localhost:4173`** as above. Everything the app saves stays in the Back up that folder and you've backed up everything; delete `.env` to forget your keys. Source edits still hot-reload. The port is published to `127.0.0.1` only, so the default matches the native one. +The local MCP server and ChatGPT OAuth voice are not available under Docker; use an OpenAI API key for voice. + Publishing it wider is the [Sharing an instance](#-sharing-an-instance) opt-in, and it hands out **Provider Settings** too: the launcher trusts the container gateway as this machine, and a wider publish delivers everyone else from that same address. For an instance you mean to share, put the keys in `.env` by hand instead. [SECURITY.md](SECURITY.md) has the detail. +When you widen the publish, also set `GEV_KEY_SETUP_TRUSTED_PEERS=none` so the launcher trusts no gateway address. +
Running on Linux, a NAS, or behind a reverse proxy diff --git a/scripts/docker-start.mjs b/scripts/docker-start.mjs index 47519af97..3935f2b3b 100644 --- a/scripts/docker-start.mjs +++ b/scripts/docker-start.mjs @@ -105,12 +105,19 @@ async function start() { ); } + const fromOperator = Boolean( + String(process.env.GEV_KEY_SETUP_TRUSTED_PEERS ?? '').trim(), + ); const trustedPeers = resolveTrustedPeers({ routeTable: readRouteTable() }); if (trustedPeers) { // Set before Vite is imported: the dev server snapshots process.env during // configuration, and the admission gate reads it per request. process.env.GEV_KEY_SETUP_TRUSTED_PEERS = trustedPeers; - console.log(`[Docker] Provider Settings trusts the container gateway (${trustedPeers}).`); + console.log( + fromOperator + ? `[Docker] Provider Settings trusted peers: ${trustedPeers} (GEV_KEY_SETUP_TRUSTED_PEERS).` + : `[Docker] Provider Settings trusts the container gateway (${trustedPeers}).`, + ); } else { console.log('[Docker] No default gateway; Provider Settings stays loopback-only.'); } From a02f729191f01d340eb58e13f60e36ec421beaaa Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 13:08:37 -0700 Subject: [PATCH 66/95] fix(opensky): cool down before reading a 429, and test the six caps A 429 now starts the credit cooldown from its headers and drops the body unread, so a stalled or oversized 429 cannot skip the governor; an oversized snapshot is not retried. A 429 without retry-after now waits the intended 2 minutes (headers.get() returns null, which Number() read as 0 s). Adds oversized-stream tests for OpenSky, adsb.lol, CelesTrak and TomTom that fail on main, names the six reads in the changelog, and corrects the docs on what was already bounded. --- CHANGELOG.md | 9 +- docs/CURRENT-STATE.md | 19 +-- server/providers/aircraft/opensky.js | 52 +++++--- src/tooling/environmentProviders.test.mjs | 46 +++++++ src/tooling/liveProviders.test.mjs | 148 ++++++++++++++++++++++ src/tooling/spaceProviders.test.mjs | 44 +++++++ 6 files changed, 292 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e02267966..ca996ac04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,9 +147,12 @@ ## [Unreleased] -- Bound the remaining upstream body reads (FIRMS, CelesTrak, adsb.lol, - TomTom, OpenSky, adsbdb) with streaming size caps and shared deadlines; the - adsb.lol military poll gains a deadline of its own. +- Bound six upstream body reads (FIRMS CSV, CelesTrak TLE groups, adsb.lol + `/v2/mil`, TomTom flow tiles, OpenSky `states/all` and adsbdb lookups) with + streaming size caps that cancel the body at the cap or the route's deadline; + the adsb.lol military poll gains a deadline of its own. An oversized OpenSky + snapshot is not retried, and an OpenSky 429 starts its cooldown before the + body is read. - Keep later satellites when a TLE catalog entry is missing an orbital line. The parser skips the incomplete set and resumes at the next complete named diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 59ed6b37a..958148258 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1198,18 +1198,21 @@ FIRMS CSV, CelesTrak TLE groups, adsb.lol military responses, TomTom flow tiles, OpenSky states/all and adsbdb lookups now read their bodies through the shared streaming readers in `src/sources/httpBody.js`, so a declared or observed body over the cap is refused with `RESPONSE_TOO_LARGE` and cancelled rather than -buffered. `readResponseBytesCapped` accepts the same optional abort signal as -the text reader and cancels a body still streaming when it fires; TomTom flow -tiles use it alongside GTFS-Realtime. Caps are bounds, not budgets, sized well +buffered. TomTom flow tiles use `readResponseBytesCapped` alongside +GTFS-Realtime. Each read takes the route's abort signal, so a body still +streaming when the deadline fires is cancelled. Caps are bounds, not budgets, sized well above measured payloads: FIRMS 64 MiB per satellite source, CelesTrak 16 MiB per group, OpenSky `states/all` 8 MiB, adsb.lol `/v2/mil` 8 MiB, TomTom tiles 8 MiB and adsbdb lookups 1 MiB. Where a route already had a deadline (FIRMS 60 s, CelesTrak 20 s, TomTom 15 s, adsbdb 8 s, and each 10 s OpenSky -`states/all` attempt) that deadline now covers the body read as well as the -headers. The adsb.lol military poll, which had no deadline, now waits at most -12 seconds. A rejected or timed-out body is an upstream miss and follows each -route's existing error and cached-data paths. FIRMS map-key status, the OpenSky token exchange and CCTV -media keep their current handling. +`states/all` attempt) it already bounded the body read, because aborting a +fetch signal also aborts its body; the capped reader takes the same signal. +The adsb.lol military poll, which had no deadline, now waits at most 12 +seconds for headers and body together. An oversized OpenSky snapshot is not +retried, and an OpenSky 429 starts its cooldown from the response headers and +cancels the body unread. A rejected or timed-out body is an upstream miss and +follows each route's existing error and cached-data paths. FIRMS map-key +status, the OpenSky token exchange and CCTV media keep their current handling. ## Remaining local service modules diff --git a/server/providers/aircraft/opensky.js b/server/providers/aircraft/opensky.js index 5c88ac832..752b3cf64 100644 --- a/server/providers/aircraft/opensky.js +++ b/server/providers/aircraft/opensky.js @@ -33,10 +33,31 @@ const OPENSKY_STATES_URL = /** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */ const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +/** + * Credit-governor cooldown for a 429: OpenSky's retry-after bounded to + * 30 s … 30 min, or 2 min when the header is absent. + * @param {Response} upstream Rate-limited response. + * @returns {number} Cooldown (ms). + */ +function openSkyRateLimitCooldownMs(upstream) { + // headers.get() gives null when absent, and Number(null) is 0, not NaN. + const header = upstream.headers.get('x-rate-limit-retry-after-seconds'); + const retryAfterSec = + header == null || header.trim() === '' ? NaN : Number(header); + return Math.min( + Math.max( + Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000, + 30_000, + ), + 30 * 60_000, + ); +} + /** * Fetch and read the global snapshot, giving each attempt * OPENSKY_ATTEMPT_TIMEOUT_MS and retrying once when an attempt times out or - * the connection fails. A second failure throws, which the proxy answers + * the connection fails. An oversized body is not retried: it would fail again + * and spend more credits. A second failure throws, which the proxy answers * from its stale cache or the regional fallback. * @param {Record} headers Request headers. * @returns {Promise<{upstream: Response, body: string}>} @@ -46,6 +67,17 @@ async function fetchOpenSkyStates(headers) { try { const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS); const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal }); + if (upstream.status === 429) { + // Start the cooldown from the headers and drop the rejected body + // unread, so a stalled or oversized 429 body cannot skip it. + _openskyCooldownUntil = + Date.now() + openSkyRateLimitCooldownMs(upstream); + void upstream.body?.cancel().catch(() => {}); + return { + upstream, + body: JSON.stringify({ error: 'OpenSky rate limited.' }), + }; + } const body = await readResponseTextCapped( upstream, OPENSKY_STATES_MAX_RESPONSE_BYTES, @@ -53,7 +85,7 @@ async function fetchOpenSkyStates(headers) { ); return { upstream, body }; } catch (error) { - if (attempt >= 2) throw error; + if (attempt >= 2 || error?.code === 'RESPONSE_TOO_LARGE') throw error; console.warn( '[OpenSky] global snapshot attempt failed, retrying:', error?.message || error, @@ -550,19 +582,9 @@ export function openSkyProxy() { } if (upstream.status === 429) { reason = 'rate_limited'; - // Credit governor: honor OpenSky's retry-after (bounded 30 s … 30 min; - // 2 min when the header is absent) — no upstream attempts until then. - const retryAfterSec = Number( - upstream.headers.get('x-rate-limit-retry-after-seconds'), - ); - const cooldownMs = Math.min( - Math.max( - Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000, - 30_000, - ), - 30 * 60_000, - ); - _openskyCooldownUntil = now + cooldownMs; + // Credit governor: fetchOpenSkyStates already started the cooldown + // from OpenSky's retry-after — no upstream attempts until then. + const cooldownMs = openSkyRateLimitCooldownMs(upstream); // Serve the last-good body instead of the 429 when we have one — // the layer keeps rendering (STALE-cued) instead of dying. if (_openskyCacheBody && _openskyCacheStatus === 200) { diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs index f8f02595c..63795c16d 100644 --- a/src/tooling/environmentProviders.test.mjs +++ b/src/tooling/environmentProviders.test.mjs @@ -53,6 +53,29 @@ function isolate(t, env = {}) { } const json = (res) => JSON.parse(res.body); +/** A response that streams 1 MiB chunks of `text` well past every cap, then ends. */ +function oversizedResponse(text) { + const state = { pulled: 0, cancelled: false }; + const chunk = new TextEncoder().encode( + text.repeat(Math.ceil((1 << 20) / text.length)), + ); + const response = new Response( + new ReadableStream( + { + pull(controller) { + if (++state.pulled > 40) controller.close(); + else controller.enqueue(chunk); + }, + cancel() { + state.cancelled = true; + }, + }, + { highWaterMark: 0 }, + ), + ); + return { response, state }; +} + test('standalone composition mounts every extracted provider exactly once without acquisition', (t) => { t.mock.method(globalThis, 'fetch', () => { throw Error('construction must not fetch'); @@ -168,6 +191,29 @@ test('traffic middleware preserves keyless mode, caching, stale budget fallback assert.equal(calls, 2); }); +test('traffic middleware answers 502 to an oversized tile without caching it', async (t) => { + isolate(t, { TOMTOM_API_KEY: 'fixture-key', TOMTOM_DAILY_TILE_BUDGET: '' }); + const bodies = []; + t.mock.method(globalThis, 'fetch', async () => { + const oversized = oversizedResponse('\x01'); + bodies.push(oversized.state); + return oversized.response; + }); + const request = install(tomtomProxy()); + const first = await request('/flow/8/1/1.pbf'); + assert.equal(first.status, 502); + assert.equal(bodies[0].cancelled, true); + assert.ok(bodies[0].pulled <= 9, 'reading stops at the 8 MiB cap'); + assert.ok( + !fsp.writeFile.mock.calls.some(({ arguments: [file] }) => + String(file).endsWith('.pbf'), + ), + 'no tile reached the disk cache', + ); + assert.equal((await request('/flow/8/1/1.pbf')).status, 502); + assert.equal(bodies.length, 2, 'nothing was cached in memory'); +}); + test('FIRMS retains a large successful source during partial failure and filters stale data at serve time', async (t) => { isolate(t, { FIRMS_MAP_KEY: '' }); let now = Date.UTC(2026, 8, 12, 12); diff --git a/src/tooling/liveProviders.test.mjs b/src/tooling/liveProviders.test.mjs index d7b740668..a60a37cea 100644 --- a/src/tooling/liveProviders.test.mjs +++ b/src/tooling/liveProviders.test.mjs @@ -37,6 +37,30 @@ function install(plugin, preview = false) { }; } +/** A response that streams 1 MiB chunks of `text` well past every cap, then ends. */ +function oversizedResponse(text, init = {}) { + const state = { pulled: 0, cancelled: false }; + const chunk = new TextEncoder().encode( + text.repeat(Math.ceil((1 << 20) / text.length)), + ); + const response = new Response( + new ReadableStream( + { + pull(controller) { + if (++state.pulled > 40) controller.close(); + else controller.enqueue(chunk); + }, + cancel() { + state.cancelled = true; + }, + }, + { highWaterMark: 0 }, + ), + init, + ); + return { response, state }; +} + function environment(t, values) { for (const [key, value] of Object.entries(values)) { const original = process.env[key]; @@ -193,6 +217,130 @@ test('OpenSky global snapshot attempts are time-limited, retried once, then fall assert.equal(stalled.response.headers['x-flight-source'], 'adsb.lol'); }); +const openSkyAnonEnv = { + OPENSKY_CLIENT_ID: undefined, + OPENSKY_CLIENT_SECRET: undefined, + OPENSKY_AUTH_MODE: 'anon', + OPENSKY_USERNAME: undefined, + OPENSKY_PASSWORD: undefined, +}; +const regionalFallback = () => + Response.json({ + now: Date.now() / 1000, + ac: [{ hex: 'abc123', lat: 30, lon: -97, alt_baro: 10000 }], + }); + +test('OpenSky 429 starts the cooldown from headers and cancels the body unread', async (t) => { + environment(t, openSkyAnonEnv); + t.mock.method(console, 'warn', () => {}); + t.mock.method(console, 'error', () => {}); + let states = 0; + let limited; + t.mock.method(globalThis, 'fetch', async (url) => { + if (url.includes('/states/')) { + states++; + limited = oversizedResponse('x', { + status: 429, + headers: { 'x-rate-limit-retry-after-seconds': '600' }, + }); + return limited.response; + } + if (url.includes('/lat/')) return regionalFallback(); + throw Error(`Unexpected URL: ${url}`); + }); + const fresh = await import( + `../../server/providers/aircraft/opensky.js?ratelimit=${Math.random()}` + ); + const request = install(fresh.openSkyProxy()); + const first = await request('/api/flights', '?lat=30&lon=-97'); + assert.equal(first.statusCode, 200); + assert.equal(first.headers['x-flight-source'], 'adsb.lol'); + assert.equal(states, 1, 'a rate-limited snapshot is not retried'); + assert.equal(limited.state.cancelled, true); + assert.ok(limited.state.pulled <= 1, 'the 429 body is not read'); + const cooling = await request('/api/flights', '?lat=30&lon=-97'); + assert.equal(cooling.statusCode, 200); + assert.equal( + cooling.headers['x-opensky-auth-reason'], + 'opensky_cooldown_regional_fallback', + ); + assert.equal(states, 1, 'the cooldown holds off the next upstream attempt'); +}); + +test('OpenSky 429 without retry-after cools down for the 2 minute default', async (t) => { + environment(t, openSkyAnonEnv); + t.mock.method(console, 'warn', () => {}); + t.mock.method(console, 'error', () => {}); + const start = Date.now(); + let now = start; + t.mock.method(Date, 'now', () => now); + let states = 0; + t.mock.method(globalThis, 'fetch', async (url) => { + if (url.includes('/states/')) { + states++; + return new Response('{}', { status: 429 }); + } + if (url.includes('/lat/')) return regionalFallback(); + throw Error(`Unexpected URL: ${url}`); + }); + const fresh = await import( + `../../server/providers/aircraft/opensky.js?ratelimit-default=${Math.random()}` + ); + const request = install(fresh.openSkyProxy()); + await request('/api/flights', '?lat=30&lon=-97'); + assert.equal(states, 1); + // Past the 30 s floor but inside the 2 minute default. + now = start + 90_000; + await request('/api/flights', '?lat=30&lon=-97'); + assert.equal(states, 1, 'a missing header is not read as a zero wait'); +}); + +test('OpenSky does not retry a snapshot body over the cap', async (t) => { + environment(t, openSkyAnonEnv); + t.mock.method(console, 'warn', () => {}); + t.mock.method(console, 'error', () => {}); + const bodies = []; + t.mock.method(globalThis, 'fetch', async (url) => { + if (url.includes('/states/')) { + const oversized = oversizedResponse(' '); + bodies.push(oversized.state); + return oversized.response; + } + if (url.includes('/lat/')) return regionalFallback(); + throw Error(`Unexpected URL: ${url}`); + }); + const fresh = await import( + `../../server/providers/aircraft/opensky.js?oversize=${Math.random()}` + ); + const response = await install(fresh.openSkyProxy())( + '/api/flights', + '?lat=30&lon=-97', + ); + assert.equal(response.statusCode, 200); + assert.equal(response.headers['x-flight-source'], 'adsb.lol'); + assert.equal(bodies.length, 1, 'an oversized body is not fetched twice'); + assert.equal(bodies[0].cancelled, true); + assert.ok(bodies[0].pulled <= 9, 'reading stops at the 8 MiB cap'); +}); + +test('military aircraft route answers 502 to an oversized body without caching it', async (t) => { + t.mock.method(console, 'error', () => {}); + const bodies = []; + t.mock.method(globalThis, 'fetch', async () => { + const oversized = oversizedResponse('{"ac":[]}'); + bodies.push(oversized.state); + return oversized.response; + }); + const request = install(providers.adsbLolProxy()); + const first = await request('/api/military'); + assert.equal(first.statusCode, 502); + assert.equal(bodies[0].cancelled, true); + assert.ok(bodies[0].pulled <= 9, 'reading stops at the 8 MiB cap'); + const again = await request('/api/military'); + assert.equal(again.statusCode, 502, 'nothing was cached'); + assert.equal(bodies.length, 2); +}); + test('military aircraft route preserves fresh cache and stale response after upstream failure', async (t) => { let now = Date.now(); let calls = 0; diff --git a/src/tooling/spaceProviders.test.mjs b/src/tooling/spaceProviders.test.mjs index 6ccd8d57c..443ca5b96 100644 --- a/src/tooling/spaceProviders.test.mjs +++ b/src/tooling/spaceProviders.test.mjs @@ -47,6 +47,29 @@ function isolateDisk(t) { t.mock.method(fsp, 'writeFile', async () => {}); } +/** A response that streams 1 MiB chunks of `text` well past every cap, then ends. */ +function oversizedResponse(text) { + const state = { pulled: 0, cancelled: false }; + const chunk = new TextEncoder().encode( + text.repeat(Math.ceil((1 << 20) / text.length)), + ); + const response = new Response( + new ReadableStream( + { + pull(controller) { + if (++state.pulled > 40) controller.close(); + else controller.enqueue(chunk); + }, + cancel() { + state.cancelled = true; + }, + }, + { highWaterMark: 0 }, + ), + ); + return { response, state }; +} + test('portable requests keep fixed origins, encode group data and preserve the 30-day UTC window', () => { const tle = celestrakTleUrl('stations&FORMAT=json'); assert.equal(tle.origin, 'https://celestrak.org'); @@ -120,6 +143,27 @@ test('exported CelesTrak plugin coalesces refreshes, retains stale TLEs, and rea assert.equal(disk.body, tle); }); +test('CelesTrak answers 502 to an oversized TLE body without caching it', async (t) => { + isolateDisk(t); + t.mock.method(console, 'warn', () => {}); + const bodies = []; + t.mock.method(globalThis, 'fetch', async () => { + const oversized = oversizedResponse( + 'ISS\n1 25544U fixture\n2 25544 fixture\n', + ); + bodies.push(oversized.state); + return oversized.response; + }); + const request = install(celestrakProxy()); + const first = await request('/api/celestrak', '/stations'); + assert.equal(first.status, 502); + assert.equal(bodies[0].cancelled, true); + assert.ok(bodies[0].pulled <= 17, 'reading stops at the 16 MiB cap'); + assert.equal(fsp.writeFile.mock.callCount(), 0); + assert.equal((await request('/api/celestrak', '/stations')).status, 502); + assert.equal(bodies.length, 2, 'nothing was cached in memory'); +}); + for (const preview of [false, true]) test(`exported launch plugin preserves optional server auth and cache in ${preview ? 'preview' : 'development'}`, async (t) => { isolateDisk(t); From 51e32a30f20ccbb5e7c7ec7878ac444fcd009dd1 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 13:08:45 -0700 Subject: [PATCH 67/95] fix(bikeshare): scope city failures to range and report partial outages A failure now ends when its city leaves range or the camera rises above the activation gate, so an empty map no longer reads LOAD FAILED. Late activation and status rejections are ignored like late successes. Partial failure reads DEGRADED, every in-range city failing over cached points reads STALE, and freshness advances only on a refresh that succeeded. Behavioural tests drive the real proximity, update and enable paths. Docs say no licence is published for Toronto and list Social Bicycles and PBSC. The merge also restored the #61 GBFS v3 changelog entry that an earlier merge on main dropped. --- CHANGELOG.md | 11 +- DATA_SOURCES.md | 4 +- docs/CURRENT-STATE.md | 2 +- src/layers/bikeshare/controls.js | 21 +- .../bikeshare/failureReporting.test.mjs | 296 ++++++++++++++++++ src/layers/bikeshare/ingestion.js | 19 +- src/layers/bikeshare/registry.test.mjs | 2 +- src/layers/bikeshare/state.js | 5 +- src/layers/bikeshare/viewport.js | 15 + 9 files changed, 351 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31f232443..c851c88d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,19 @@ flag that only a successful *activation* cleared — and an already-loaded city never activates again. A dead feed re-entering camera range therefore latched LOAD FAILED permanently over a map still drawing every other city's stations. - Failure is now tracked per city and reported only when it leaves the layer - with nothing to draw; a city that recovers clears its own failure. + Failure is now tracked per city: a city that fails beside others still + drawing marks the layer DEGRADED, polls that all fail over previously drawn + points read STALE without advancing freshness, and LOAD FAILED is reported + only when nothing is left to draw. A city that recovers clears its own + failure, a failure is dropped once its city leaves range or the camera rises + above the activation altitude, and late results from superseded requests are + ignored. Repointed the three feeds whose operators had moved: Reddy Bikeshare (Buffalo) to Social Bicycles, Tugo (Tucson) and Trinity Metro (Fort Worth) to PBSC. Dropped CoGo (Columbus) and Broward B-cycle (Fort Lauderdale), whose systems have wound down — CoGo's feed reports no stations and last moved a bike in - March 2025. All 31 remaining systems were verified live. + March 2025. The three repointed feeds were verified live. - Add Bike Share Toronto (1,069 stations) to the bikeshare registry. The system publishes the same PBSC GBFS v2 feeds as the Austin, Honolulu and Chattanooga diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 157bab7b8..e983a972d 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -60,8 +60,8 @@ How to read this: | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | | **NYC DOT Traffic Management Center** (`webcams.nyctmc.org`) | CCTV camera catalog + frames, New York City five boroughs | NYC DOT requires developers to sign a [data-sharing agreement](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) for the camera feed. **Off by default**; enable only under your own agreement | "NYC DOT Traffic Management Center" + NYC DOT's required application disclaimer (in the Data attribution popover) | -| **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | -| **Bike Share Toronto** (`toronto.publicbikesystem.net`) | Bikeshare station availability, Toronto | Operator GBFS feed. The City's open-data package declares no licence and the feed publishes no `license_url`, so it is treated as attribution-only like the other GBFS operators. | "Bike Share Toronto (Toronto Parking Authority)" | +| **GBFS (Lyft / BCycle / Social Bicycles / PBSC)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | +| **Bike Share Toronto** (`toronto.publicbikesystem.net`) | Bikeshare station availability, Toronto | Operator GBFS feed. No licence is published: the City's open-data package declares none and the feed has no `license_url`. Attribution is given to the operator. | "Bike Share Toronto (Toronto Parking Authority)" | | **Oslo / Bergen / Trondheim Bysykkel** (Urban Sharing, `gbfs.urbansharing.com`) | Bikeshare availability, Oslo, Bergen and Trondheim city bikes. Keyless GBFS; the proxy sends the `Client-Identifier` header the operators ask for | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED ([Oslo](https://oslobysykkel.no/en/open-data/realtime), [Bergen](https://bergenbysykkel.no/en/apne-data/sanntid), [Trondheim](https://trondheimbysykkel.no/en/open-data/realtime)) | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel" | | **Kolumbus / Farte Bysykkel** (Entur Mobility, `api.entur.io/mobility/v2/gbfs`) | Bikeshare availability, Kolumbus Bysykkel (Stavanger/Nord-Jæren) and Farte Bysykkel (Skien/Porsgrunn). Keyless GBFS; the proxy sends the `ET-Client-Name` header Entur requires and is limited to Entur's GBFS paths | [NLOD](https://data.norge.no/nlod/en/2.0) per the [Entur Mobility API docs](https://developer.entur.org/pages-mobility-docs-mobility-v2) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Entur (Kolumbus Bysykkel, Farte Bysykkel)" | | **JCDecaux Cyclocity** (`api.cyclocity.fr`) | Bikeshare availability, Bysykkel Lillestrøm. Keyless GBFS; the proxy is limited to JCDecaux contract paths | [Licence Ouverte / Etalab](https://developer.jcdecaux.com/files/Open-Licence-fr.pdf) (the feed's `license_url`) — attribution REQUIRED | "Bysykkel Lillestrøm: JCDecaux, Licence Ouverte" | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 10a11f234..0cc9c8ae5 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3172,7 +3172,7 @@ its criteria cannot be silently ignored. | CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + NYC DOT TMC (opt-in) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle + Bike Share Toronto + Urban Sharing, Entur, JCDecaux NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto + Urban Sharing, Entur, JCDecaux NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | | Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | diff --git a/src/layers/bikeshare/controls.js b/src/layers/bikeshare/controls.js index 9e9389c58..58c5f003a 100644 --- a/src/layers/bikeshare/controls.js +++ b/src/layers/bikeshare/controls.js @@ -56,7 +56,7 @@ export function createControls({ state: layerState, services, parts, source }) { /** * Return current layer statistics for the UI status display. - * @returns {{ count: number, lastUpdate: number|null, loading: boolean, loadingLabel?: string, error?: string }} + * @returns {{ count: number, lastUpdate: number|null, loading: boolean, loadingLabel?: string, error?: string, degraded?: boolean, stale?: boolean }} */ getStats() { const stats = { @@ -71,12 +71,19 @@ export function createControls({ state: layerState, services, parts, source }) { : 'scanning nearby systems...'; } // A failed city is only the layer's failure when it left nothing on the - // map. While any city still renders, the layer is working and says so. - if ( - layerState._failedCityIds.size > 0 && - layerState._stationRenderMap.size === 0 - ) { - stats.error = 'GBFS fetch error'; + // map. While other cities still render, the partial failure reads + // DEGRADED; when every in-range city failed and only previously drawn + // points remain, they read STALE. + if (layerState._failedCityIds.size > 0) { + if (layerState._stationRenderMap.size === 0) { + stats.error = 'GBFS fetch error'; + } else { + stats.degraded = true; + const allFailed = Array.from(layerState._activeCityIds).every( + (cityId) => layerState._failedCityIds.has(cityId), + ); + if (allFailed) stats.stale = true; + } } return stats; }, diff --git a/src/layers/bikeshare/failureReporting.test.mjs b/src/layers/bikeshare/failureReporting.test.mjs index 63034078b..89a87bdad 100644 --- a/src/layers/bikeshare/failureReporting.test.mjs +++ b/src/layers/bikeshare/failureReporting.test.mjs @@ -78,3 +78,299 @@ test('a recovered city clears its own failure', () => { 'the failure outlived the city that caused it', ); }); + +// The cases above pin how getStats() reads failure state. The cases below +// drive the real proximity check, status poll and lifecycle against a fake +// GBFS source, so the failure state they read is the state the layer builds. + +import * as Cesium from 'cesium'; +import { CITY_BY_ID } from './registry.js'; +import { createIngestion } from './ingestion.js'; +import { createModel } from './model.js'; +import { createViewport } from './viewport.js'; +import { createRendering } from './rendering.js'; +import { createLifecycle } from './lifecycle.js'; + +const TORONTO = 'toronto-bike-share'; +const BUFFALO = 'buffalo-reddy'; +// Within range of both Toronto and Buffalo. +const BETWEEN = { lat: 43.25, lon: -79.1 }; +// Within range of Toronto only. +const TORONTO_ONLY = { lat: 43.7, lon: -79.6 }; +// Within range of Buffalo only. +const BUFFALO_ONLY = { lat: 42.5, lon: -78.0 }; +const LOW_M = 10_000; + +// disable() detaches a keydown listener; node has no document. +globalThis.document ??= { + addEventListener() {}, + removeEventListener() {}, +}; + +/** A GBFS source whose per-city answers each test can change. */ +function makeSource() { + const cityOf = (url) => { + for (const [id, city] of CITY_BY_ID) { + if (url === city.stationInformationUrl) return { id, kind: 'info' }; + if (url === city.stationStatusUrl) return { id, kind: 'status' }; + } + throw new Error(`unexpected GBFS url ${url}`); + }; + const source = { + dead: new Set(), + deadStatus: new Set(), + /** Optional per-call override: (cityId, kind) => Promise | undefined. */ + hook: null, + async getStations(url, { signal } = {}) { + const { id, kind } = cityOf(url); + const hooked = source.hook?.(id, kind); + if (hooked) return hooked; + await new Promise((resolve) => setTimeout(resolve, 1)); + signal?.throwIfAborted(); + if (source.dead.has(id)) throw new Error('GBFS HTTP 404'); + if (kind === 'status' && source.deadStatus.has(id)) + throw new Error('GBFS HTTP 503'); + const city = CITY_BY_ID.get(id); + if (kind === 'info') { + return { + data: { + stations: [ + { + station_id: `${id}-1`, + name: `${id} dock`, + lat: city.centerLat, + lon: city.centerLon, + capacity: 10, + }, + ], + }, + }; + } + return { + data: { + stations: [ + { + station_id: `${id}-1`, + num_bikes_available: 3, + num_docks_available: 7, + is_installed: true, + is_renting: true, + is_returning: true, + }, + ], + }, + }; + }, + }; + return source; +} + +/** Assemble the layer from its real parts around a camera the test moves. */ +function makeLiveLayer() { + const source = makeSource(); + const services = { + render: { governorRequestRender() {} }, + overlays: { + setOverlayEntries() {}, + setOverlaySourceVisible() {}, + clearOverlaySource() {}, + }, + sprites: { registerSpriteCollection() {}, restoreSpriteOrder() {} }, + picking: { registerPickOwner() {}, unregisterPickOwner() {} }, + }; + const state = createState({ services }); + const parts = {}; + const context = { state, services, parts, source }; + parts.ingestion = createIngestion(context); + parts.model = createModel(context); + parts.viewport = createViewport(context); + parts.rendering = createRendering(context); + // Selection owns a DOM click handler; nothing here selects a station. + parts.selection = { + _installClickHandler() {}, + _clearSelection() {}, + _onKeyDown() {}, + }; + parts.queries = { collectDetectableStations: () => [] }; + parts.lifecycle = createLifecycle(context); + const { methods: controls } = createControls(context); + + const camera = { ...BETWEEN, height: LOW_M }; + const viewer = { + camera: { + get positionCartographic() { + return Cesium.Cartographic.fromDegrees( + camera.lon, + camera.lat, + camera.height, + ); + }, + computeViewRectangle: () => undefined, + changed: { addEventListener() {}, removeEventListener() {} }, + }, + scene: { globe: {}, primitives: { add() {}, remove() {} } }, + }; + parts.lifecycle.methods.init(viewer); + + return { + state, + source, + stats: () => controls.getStats(), + moveTo(place, height = LOW_M) { + Object.assign(camera, place, { height }); + return parts.viewport.runProximityCheck(); + }, + async enable() { + parts.lifecycle.methods.enable(viewer); + await settle(); + }, + disable: () => parts.lifecycle.methods.disable(viewer), + poll: () => parts.ingestion.methods.update(), + }; +} + +/** Let fire-and-forget activation started by enable() finish. */ +async function settle() { + for (let i = 0; i < 5; i += 1) + await new Promise((resolve) => setTimeout(resolve, 2)); +} + +function deferred() { + let resolve, reject; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +test('a city that fails beside one that renders reads DEGRADED, not healthy', async () => { + const layer = makeLiveLayer(); + layer.source.dead.add(BUFFALO); + await layer.enable(); + + const stats = layer.stats(); + assert.equal(stats.count, 1, 'Toronto should still draw'); + assert.equal(stats.error, undefined); + assert.equal(stats.degraded, true, 'the Buffalo failure was hidden'); + assert.equal(stats.stale, undefined); +}); + +test('a failed city that comes back clears its own failure', async () => { + const layer = makeLiveLayer(); + layer.source.dead.add(BUFFALO); + await layer.enable(); + assert.equal(layer.stats().degraded, true); + + layer.source.dead.clear(); + await layer.moveTo(BETWEEN); + const stats = layer.stats(); + assert.equal(stats.count, 2); + assert.equal(stats.degraded, undefined); + assert.equal(stats.error, undefined); +}); + +test('a status poll that recovers clears the failure the last poll set', async () => { + const layer = makeLiveLayer(); + await layer.moveTo(TORONTO_ONLY); + await layer.enable(); + layer.source.deadStatus.add(TORONTO); + await layer.poll(); + assert.equal(layer.stats().degraded, true); + + layer.source.deadStatus.clear(); + await layer.poll(); + assert.equal(layer.stats().degraded, undefined); + assert.equal(layer.stats().stale, undefined); +}); + +test('a failure does not outlive its city leaving range', async () => { + const layer = makeLiveLayer(); + layer.source.dead.add(BUFFALO); + await layer.enable(); + await layer.moveTo(BUFFALO_ONLY); + assert.equal(layer.stats().error, 'GBFS fetch error'); + + // Low over the open Atlantic: nothing in range is empty, not broken. + await layer.moveTo({ lat: 30, lon: -40 }); + const stats = layer.stats(); + assert.equal(stats.count, 0); + assert.equal(stats.error, undefined, 'LOAD FAILED over empty ocean'); + assert.equal(stats.degraded, undefined); +}); + +test('zooming out above the altitude gate drops a city failure', async () => { + const layer = makeLiveLayer(); + layer.source.dead.add(BUFFALO); + await layer.enable(); + await layer.moveTo(BUFFALO_ONLY); + assert.equal(layer.stats().error, 'GBFS fetch error'); + + await layer.moveTo(BUFFALO_ONLY, 200_000); + const stats = layer.stats(); + assert.equal(stats.count, 0); + assert.equal(stats.error, undefined, 'LOAD FAILED above the gate'); +}); + +test('a late rejection from before a disable/re-enable is ignored', async () => { + const layer = makeLiveLayer(); + await layer.moveTo(BUFFALO_ONLY); + const first = deferred(); + // The first info request hangs and ignores its abort signal. + layer.source.hook = (id, kind) => { + if (id !== BUFFALO || kind !== 'info') return undefined; + layer.source.hook = null; + return first.promise; + }; + await layer.enable(); + layer.disable(); + await layer.enable(); + assert.equal(layer.stats().count, 1, 're-enable should draw Buffalo'); + + first.reject(new Error('GBFS HTTP 500')); + await settle(); + const stats = layer.stats(); + assert.equal(stats.count, 1, 'a stale rejection tore down live points'); + assert.equal(stats.error, undefined); + assert.equal(stats.degraded, undefined); +}); + +test('a late status poll rejection from before a disable/re-enable is ignored', async () => { + const layer = makeLiveLayer(); + await layer.moveTo(TORONTO_ONLY); + await layer.enable(); + const first = deferred(); + layer.source.hook = (id, kind) => { + if (id !== TORONTO || kind !== 'status') return undefined; + layer.source.hook = null; + return first.promise; + }; + const poll = layer.poll(); + layer.disable(); + await layer.enable(); + + first.reject(new Error('GBFS HTTP 500')); + await poll; + const stats = layer.stats(); + assert.equal(stats.count, 1); + assert.equal(stats.degraded, undefined); + assert.equal(stats.error, undefined); +}); + +test('every status poll failing over cached points reads STALE, not fresh', async () => { + const layer = makeLiveLayer(); + await layer.moveTo(TORONTO_ONLY); + await layer.enable(); + const loadedAt = layer.stats().lastUpdate; + assert.ok(loadedAt, 'activation should record freshness'); + + layer.source.deadStatus.add(TORONTO); + await new Promise((resolve) => setTimeout(resolve, 5)); + await layer.poll(); + const stats = layer.stats(); + assert.equal(stats.count, 1, 'the cached points stay drawn'); + assert.equal(stats.error, undefined); + assert.equal(stats.degraded, true); + assert.equal(stats.stale, true); + assert.equal(stats.lastUpdate, loadedAt, 'a failed poll advanced freshness'); +}); diff --git a/src/layers/bikeshare/ingestion.js b/src/layers/bikeshare/ingestion.js index cda677766..f2127e53e 100644 --- a/src/layers/bikeshare/ingestion.js +++ b/src/layers/bikeshare/ingestion.js @@ -171,20 +171,21 @@ export function createIngestion({ const generation = layerState._proximityGeneration; const cityIds = Array.from(layerState._activeCityIds); + const isCurrent = (cityId) => + layerState._enabled && + layerState._activeCityIds.has(cityId) && + generation === layerState._proximityGeneration; + let refreshed = false; await Promise.all( cityIds.map(async (cityId) => { try { const statusMap = await loadCityStationStatus(cityId, generation); - if ( - !layerState._enabled || - !layerState._activeCityIds.has(cityId) || - generation !== layerState._proximityGeneration - ) - return; + if (!isCurrent(cityId)) return; parts.rendering.applyStatusToPoints(cityId, statusMap); layerState._failedCityIds.delete(cityId); + refreshed = true; } catch (error) { - if (error?.name === 'AbortError') return; + if (error?.name === 'AbortError' || !isCurrent(cityId)) return; console.warn( `[Data:Bikeshare] ${cityId} status update error:`, error, @@ -195,7 +196,9 @@ export function createIngestion({ ); layerState._count = layerState._stationRenderMap.size; - if (layerState._count > 0) layerState._lastUpdate = Date.now(); + // Freshness advances only on a poll that actually refreshed a city. + if (refreshed && layerState._count > 0) + layerState._lastUpdate = Date.now(); }, }; diff --git a/src/layers/bikeshare/registry.test.mjs b/src/layers/bikeshare/registry.test.mjs index 30e708957..74a103079 100644 --- a/src/layers/bikeshare/registry.test.mjs +++ b/src/layers/bikeshare/registry.test.mjs @@ -23,7 +23,7 @@ test('every registered city is reachable through the GBFS proxy allowlist', () = `${city.id}: host ${url.hostname} is not allowlisted`, ); assert.equal( - isAllowedGbfsPath(url.pathname), + isAllowedGbfsPath(url.pathname, url.hostname), true, `${city.id}: path ${url.pathname} is not allowlisted`, ); diff --git a/src/layers/bikeshare/state.js b/src/layers/bikeshare/state.js index 728c1c124..3617914bf 100644 --- a/src/layers/bikeshare/state.js +++ b/src/layers/bikeshare/state.js @@ -104,8 +104,9 @@ export function createState({ services }) { * * Per city rather than one layer-wide flag: feeds are fetched independently, * so a retired operator must not mark a layer that other cities are still - * populating as failed. The layer reports failure only when these cost it - * everything it had to draw — see `getStats()` in controls.js. + * populating as failed. Entries are dropped when their city leaves range. + * The layer reports failure only when these cost it everything it had to + * draw, and degraded otherwise — see `getStats()` in controls.js. */ layerState._failedCityIds = new Set(); diff --git a/src/layers/bikeshare/viewport.js b/src/layers/bikeshare/viewport.js index b4e63e1dd..25a4df7d0 100644 --- a/src/layers/bikeshare/viewport.js +++ b/src/layers/bikeshare/viewport.js @@ -122,6 +122,9 @@ export function createViewport({ state: layerState, services, parts, source }) { const cityIds = Array.from(layerState._activeCityIds); for (const cityId of cityIds) deactivateCity(cityId); layerState._activeCityIds.clear(); + // A failure belongs to a city in range; none are once everything is off. + layerState._failedCityIds.clear(); + layerState._count = 0; } /** @@ -171,6 +174,14 @@ export function createViewport({ state: layerState, services, parts, source }) { layerState._failedCityIds.delete(cityId); } catch (error) { if (error?.name === 'AbortError') return; + // A rejection from superseded work must not mark (or tear down) a city + // that a newer proximity check or re-enable now owns. + if ( + !layerState._enabled || + !layerState._activeCityIds.has(cityId) || + generation !== layerState._proximityGeneration + ) + return; console.warn(`[Data:Bikeshare] ${cityId} activate error:`, error); layerState._failedCityIds.add(cityId); deactivateCity(cityId); @@ -208,6 +219,10 @@ export function createViewport({ state: layerState, services, parts, source }) { deactivateCity(cityId); } } + // A failure never outlives its city's range. + for (const cityId of layerState._failedCityIds) { + if (!nextActive.has(cityId)) layerState._failedCityIds.delete(cityId); + } layerState._activeCityIds = nextActive; if (layerState._activeCityIds.size === 0) { From 129174c3ab7ad61e88cbabee67ce6d1d5dc6c810 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 13:13:17 -0700 Subject: [PATCH 68/95] fix(air-quality): show the Weather card, add voice support and bound the feed The card never appeared: the Weather panel only passed five fixed ids and only rendered lists and categorical legends for cyclones. Air quality now reaches the panel, the legend style comes from the layer's descriptor, and any layer with list items gets its list. Adds the voice manifest entry main now requires, reads ECCC through the shared capped JSON reader with a 15 s deadline and the row limit enforced, reports stale when every reading was too old, adds source tests, links the ECCC licence and moves the changelog entry into Unreleased. --- CHANGELOG.md | 26 +-- scripts/package-boundaries.json | 6 +- src/data/dataCredits.js | 2 +- src/firstRunExperience.test.mjs | 5 +- src/layers/airQuality/index.js | 50 ++---- src/layers/airQuality/model.js | 7 +- src/layers/airQuality/model.test.mjs | 19 +++ src/layers/airQuality/ownership.test.mjs | 95 +++-------- src/layers/airQuality/source.js | 46 ++++-- src/layers/airQuality/source.test.mjs | 197 +++++++++++++++++++++++ src/layers/cyclones/index.js | 1 + src/ui/layerPanel.js | 1 + src/ui/layerPanel.test.mjs | 28 ++++ src/ui/weatherPanel.js | 16 +- src/ui/weatherPanel.test.mjs | 68 +++++++- src/voice/actionSchemas.test.mjs | 3 +- src/voice/layerManifest.js | 18 +++ 17 files changed, 440 insertions(+), 148 deletions(-) create mode 100644 src/layers/airQuality/source.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index f905cb369..88657ed10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,19 @@ ## [Unreleased] +- Add a keyless Air quality layer to the Weather group, with Environment and + Climate Change Canada's AQHI as the first registered provider. Readings carry + `{ provider, scale, value, band }` so another network can be added without + inventing a shared numeric meaning across indices that do not share one; the + row and readout both state that coverage is Canada only. The WEATHER card + shows the band legend and the highest readings currently in view. AQHI is + reported as an integer from 1 with an open-ended "10+", so readings round to + that published form rather than the decimal precision the API returns. + Readings older than six hours are dropped, and the layer reports `stale` + through `getStats()` (also when every observation was dropped) so feed + provenance cannot describe an aged feed as live. Voice can switch it on and + query readings by value, band, station and observation time. + - Satellite clouds keep Clouds only / Full as quick endpoints and add a continuous cloud-filter slider: 0% is Full, 100% is the existing brightness key, and intermediate values blend the keyed alpha. The slider commits on release to @@ -1041,19 +1054,6 @@ Launch payloads with missing records now say PAYLOAD DATA UNAVAILABLE. Missing n This changelog records public product changes. For the authoritative description of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md). -## [Unreleased] - -- Add a keyless Air quality layer to the Weather group, with Environment and - Climate Change Canada's AQHI as the first registered provider. Readings carry - `{ provider, scale, value, band }` so another network can be added without - inventing a shared numeric meaning across indices that do not share one; the - row and readout both state that coverage is Canada only. The WEATHER card - shows the band legend and the highest readings currently in view. AQHI is - reported as an integer from 1 with an open-ended "10+", so readings round to - that published form rather than the decimal precision the API returns. - Readings older than six hours are dropped, and the layer reports `stale` - through `getStats()` so feed provenance cannot describe an aged feed as live. - ### Other changes - Add ECMWF IFS model selection to Wind (#464, thanks @beneduzi), with model-scoped forecast-step caches, cancellation of replaced requests, and separate issue/valid timestamps. diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 9739b1e66..d0e604336 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -3587,7 +3587,8 @@ "modules": [ "src/layers/airQuality/index.js", "src/layers/airQuality/model.js", - "src/layers/airQuality/source.js" + "src/layers/airQuality/source.js", + "src/sources/httpBody.js" ], "external": ["cesium"] }, @@ -3595,7 +3596,8 @@ "exports": ["./layers/air-quality/source"], "modules": [ "src/layers/airQuality/model.js", - "src/layers/airQuality/source.js" + "src/layers/airQuality/source.js", + "src/sources/httpBody.js" ], "external": [] } diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 6311f2eec..9c9dee71f 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -174,7 +174,7 @@ export const DATA_CREDITS = [ html: 'Air quality (AQHI): Environment and Climate Change Canada — MSC GeoMet ' + '(eccc-msc.github.io/open-data) ' + - '· ECCC Data Servers End-use Licence', + '· ECCC Data Servers End-use Licence', }, { key: 'overture-military-names', diff --git a/src/firstRunExperience.test.mjs b/src/firstRunExperience.test.mjs index a75ac91fd..50dbb15c9 100644 --- a/src/firstRunExperience.test.mjs +++ b/src/firstRunExperience.test.mjs @@ -674,10 +674,11 @@ test('the voice TOOL SCHEMA matches the pinned contract — the mission mapping // schema, which may not carry top-level anyOf), and Contacts requested-radius // list descriptions; the missions still ride existing tools. // Street Level and GNSS extend the manifest; broadcast caveats describe tracked and analyst aircraft. - assert.equal(block.length, 30511, 'serialized tool schema length drifted'); + // Air quality (ECCC AQHI) extends the manifest. + assert.equal(block.length, 30655, 'serialized tool schema length drifted'); assert.equal( crypto.createHash('sha256').update(block).digest('hex'), - '62d5df9cedf413403e4fd0a7cbe94bb2cd13736c3d8fedc84bcccd63ad253eb6', + '3b1d5e77c10a92f6543a4f9d1ff1870101a0306b65e5b8a6113111c81eda371e', 'the first-run missions must ride EXISTING tools: no schema edit, no cache bust', ); const instructions = fs.readFileSync(new URL('../server/providers/openai/instructions.js', import.meta.url), 'utf8'); diff --git a/src/layers/airQuality/index.js b/src/layers/airQuality/index.js index 43a76f857..804f9a795 100644 --- a/src/layers/airQuality/index.js +++ b/src/layers/airQuality/index.js @@ -224,24 +224,25 @@ export function createAirQualityLayer({ source, overlayHost } = {}) { status: _lastError ? _lastError : _count === 0 - ? 'No readings available' + ? _lastUpdate === null + ? 'No readings available' + : 'No current readings' : `${_count} stations reporting`, units: provider.scale, }, - legend: { - colors: AIR_QUALITY_BANDS.map((band) => band.color), - labels: AIR_QUALITY_BANDS.map((band) => band.label), - units: provider.scale, - categorical: true, - }, + legend: AIR_QUALITY_BANDS.map((band) => ({ + color: band.color, + label: band.label, + })), + legendCategorical: true, list: { ariaLabel: 'Highest air-quality readings in view', - items: inView.map((reading) => ({ + items: inView.map((reading, index) => ({ id: reading.id, - label: reading.name, - value: `${reading.scale} ${reading.label}`, + ordinal: index + 1, + lead: `${reading.scale} ${reading.label}`, + text: `${reading.name} · ${reading.risk}`, accent: reading.color, - detail: reading.risk, })), }, }; @@ -251,30 +252,6 @@ export function createAirQualityLayer({ source, overlayHost } = {}) { rowControlsListener = typeof listener === 'function' ? listener : null; }, - /** - * The current reading nearest a point — what "the air quality here" means. - * @param {number} lat - * @param {number} lon - * @returns {object|null} Reading with distanceKm, or null when none loaded. - */ - nearestReading(lat, lon) { - if (!Number.isFinite(lat) || !Number.isFinite(lon) || !_readings.length) - return null; - const cosLat = Math.cos((lat * Math.PI) / 180); - let best = null; - let bestDistance = Infinity; - for (const reading of _readings) { - const dLat = (reading.lat - lat) * 111.32; - const dLon = (reading.lon - lon) * 111.32 * cosLat; - const distance = Math.hypot(dLat, dLon); - if (distance < bestDistance) { - bestDistance = distance; - best = reading; - } - } - return best ? { ...best, distanceKm: bestDistance } : null; - }, - /** * Snapshot readings as JSON-safe records for the analyst query engine. * @param {number} [maxCount=500] @@ -300,7 +277,8 @@ export function createAirQualityLayer({ source, overlayHost } = {}) { lastUpdate: _lastUpdate, error: _lastError, observedAt: newestObservation(_readings), - stale: readingsAreStale(_readings), + // Only a completed refresh can be stale; before one there is no feed. + stale: _lastUpdate !== null && readingsAreStale(_readings), }; }, }; diff --git a/src/layers/airQuality/model.js b/src/layers/airQuality/model.js index 93f13404f..1e0eab47e 100644 --- a/src/layers/airQuality/model.js +++ b/src/layers/airQuality/model.js @@ -67,14 +67,17 @@ export function newestObservation(readings) { } /** - * Is a set of readings stale, judged on source observation time? + * Is a set of readings stale, judged on source observation time? An empty + * set is stale: a refresh that kept no current observation has nothing live. * @param {Array} readings * @param {number} [now=Date.now()] * @returns {boolean} */ export function readingsAreStale(readings, now = Date.now()) { const newest = newestObservation(readings); - if (newest === null) return false; // nothing to be stale about + // No current observation at all — every row was dropped as too old or + // unusable — is the stalest a feed can be, not a nominal one. + if (newest === null) return true; return now - newest > STALE_AFTER_MS; } diff --git a/src/layers/airQuality/model.test.mjs b/src/layers/airQuality/model.test.mjs index 56130c30e..b4f1a643e 100644 --- a/src/layers/airQuality/model.test.mjs +++ b/src/layers/airQuality/model.test.mjs @@ -5,6 +5,7 @@ import { AIR_QUALITY_PROVIDERS, AIR_QUALITY_UNKNOWN_COLOR, OBSERVATION_MAX_AGE_MS, + STALE_AFTER_MS, airQualityColor, aqhiDisplayValue, bandForValue, @@ -12,6 +13,7 @@ import { mapAnalystRecord, normalizeAqhiObservations, normalizeAqhiStations, + readingsAreStale, readingsInView, selectAirQualityOverlayCohort, valueLabel, @@ -147,6 +149,23 @@ test('a stale reading is dropped rather than shown as current', () => { ); }); +test('a snapshot whose every observation was too old is stale, not nominal', () => { + const stations = normalizeAqhiStations(STATIONS); + const old = new Date(NOW - OBSERVATION_MAX_AGE_MS - 60_000).toISOString(); + const rows = normalizeAqhiObservations( + collection([observation('IAKID', 3, old), observation('OAAAA', 4, old)]), + stations, + NOW, + ); + assert.deepEqual(rows, []); + assert.equal(readingsAreStale(rows, NOW), true); + assert.equal(readingsAreStale([{ observedMs: NOW - 60_000 }], NOW), false); + assert.equal( + readingsAreStale([{ observedMs: NOW - STALE_AFTER_MS - 1 }], NOW), + true, + ); +}); + test('the newest reading per station wins and worst air sorts first', () => { const stations = normalizeAqhiStations(STATIONS); const rows = normalizeAqhiObservations( diff --git a/src/layers/airQuality/ownership.test.mjs b/src/layers/airQuality/ownership.test.mjs index 5eeaae0e1..78ab8a0bb 100644 --- a/src/layers/airQuality/ownership.test.mjs +++ b/src/layers/airQuality/ownership.test.mjs @@ -1,7 +1,6 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { createAirQualityLayer } from './index.js'; -import { createEcccAirQualitySource } from './source.js'; import { layerFeedState } from '../../data/feedState.js'; import { STALE_AFTER_MS } from './model.js'; @@ -160,18 +159,19 @@ test('the readout descriptor carries the band legend and coverage', async () => const controls = layer.getRowControls(); assert.equal(controls.readout, true); assert.match(controls.summary.coverage, /Canada only/); - assert.equal(controls.legend.categorical, true); - assert.deepEqual(controls.legend.labels, [ - 'Low', - 'Moderate', - 'High', - 'Very High', - ]); - assert.equal(controls.legend.colors.length, controls.legend.labels.length); + // The Weather panel's legend shape: one {color, label} per band. + assert.equal(controls.legendCategorical, true); + assert.deepEqual( + controls.legend.map((entry) => entry.label), + ['Low', 'Moderate', 'High', 'Very High'], + ); + assert.ok( + controls.legend.every((entry) => /^#[0-9a-f]{6}$/i.test(entry.color)), + ); // Worst first, and every item keyed. assert.deepEqual( - controls.list.items.map((i) => i.label), - ['B', 'A'], + controls.list.items.map((i) => i.text), + ['B · Low health risk', 'A · Low health risk'], ); assert.ok(controls.list.items.every((i) => i.id)); assert.match(controls.summary.status, /2 stations reporting/); @@ -203,64 +203,6 @@ test('row-control listeners are notified on refresh and teardown-safe', async () assert.equal(calls, after); }); -// ── source contract ──────────────────────────────────────────────────────── - -test('the source rejects a malformed station catalog and surfaces HTTP errors', async () => { - await assert.rejects( - () => - createEcccAirQualitySource({ - fetchImpl: async () => ({ - ok: true, - json: async () => ({ features: 'no' }), - }), - }).getSnapshot(), - /Malformed AQHI station/, - ); - await assert.rejects( - () => - createEcccAirQualitySource({ - fetchImpl: async () => ({ - ok: false, - status: 503, - json: async () => ({}), - }), - }).getSnapshot(), - /ECCC HTTP 503/, - ); -}); - -test('the station catalog is cached and observations ask for latest values only', async () => { - const urls = []; - const source = createEcccAirQualitySource({ - fetchImpl: async (url) => { - urls.push(url); - return { - ok: true, - json: async () => - url.includes('aqhi-stations') - ? { - features: [ - { - geometry: { type: 'Point', coordinates: [-79.38, 43.65] }, - properties: { location_id: 'A', location_name_en: 'A' }, - }, - ], - } - : { features: [] }, - }; - }, - now: () => 1_700_000_000_000, - }); - await source.getSnapshot(); - await source.getSnapshot(); - assert.equal(urls.filter((u) => u.includes('aqhi-stations')).length, 1); - // Without latest=true the collection returns the full history (~2,000 rows). - assert.match( - urls.find((u) => u.includes('observations')), - /latest=true/, - ); -}); - // ── feed-state honesty ───────────────────────────────────────────────────── test('freshness is judged on observation time, not on when we last fetched', async () => { @@ -296,12 +238,19 @@ test('a current reading reports nominal', async () => { assert.equal(layerFeedState(stats), 'nominal'); }); -test('an empty feed is not called stale', async () => { - // Nothing to be stale about; the chip should not invent a fault. +test('a refresh that kept no current observation reports stale, not nominal', async () => { + // Every observation older than OBSERVATION_MAX_AGE_MS is dropped, so a feed + // that only has hours-old rows yields an empty snapshot. That is the + // stalest a feed can be; it must not read as a healthy empty layer. const { layer } = harness(staticSource([])); + assert.equal(layer.getStats().stale, false, 'no refresh yet, no verdict'); await layer.update(); - assert.equal(layer.getStats().stale, false); - assert.equal(layer.getStats().observedAt, null); + const stats = layer.getStats(); + assert.equal(stats.count, 0); + assert.equal(stats.observedAt, null); + assert.equal(stats.stale, true); + assert.equal(layerFeedState(stats), 'stale'); + assert.equal(layer.getRowControls().summary.status, 'No current readings'); }); test('a failed refresh over good readings reports degraded, not nominal', async () => { diff --git a/src/layers/airQuality/source.js b/src/layers/airQuality/source.js index b5f98894c..8fb4808ec 100644 --- a/src/layers/airQuality/source.js +++ b/src/layers/airQuality/source.js @@ -1,3 +1,4 @@ +import { readResponseJsonCapped } from '../../sources/httpBody.js'; import { normalizeAqhiStations, normalizeAqhiObservations } from './model.js'; const COLLECTIONS_ROOT = 'https://api.weather.gc.ca/collections'; @@ -9,6 +10,8 @@ const STATION_LIMIT = 500; const OBSERVATION_LIMIT = 500; /** Station geometry is near-static — a few changes a year. Refetch twice a day. */ const STATION_TTL_MS = 12 * 60 * 60 * 1000; +/** A full 500-row GeoJSON page is a few hundred KB; anything past this is not a page. */ +const MAX_RESPONSE_BYTES = 4 * 1024 * 1024; /** * Request and validate a complete ECCC AQHI snapshot before it can replace @@ -26,26 +29,47 @@ const STATION_TTL_MS = 12 * 60 * 60 * 1000; * @param {object} [options] * @param {Function} [options.fetchImpl] - Injected fetch, for tests. * @param {() => number} [options.now] - Clock seam, for tests. + * @param {number} [options.timeoutMs=15000] - Deadline for each request. * @returns {{getSnapshot: Function}} */ export function createEcccAirQualitySource({ fetchImpl = (...args) => globalThis.fetch(...args), now = () => Date.now(), + timeoutMs = 15_000, } = {}) { /** @type {Map} */ let stations = new Map(); let stationsFetchedAt = 0; - const getJson = async (url, signal) => { - signal?.throwIfAborted(); - const response = await fetchImpl(url, { - headers: { Accept: 'application/json' }, - signal, - }); - if (!response.ok) throw new Error(`ECCC HTTP ${response.status}`); - const payload = await response.json(); - signal?.throwIfAborted(); - return payload; + /** One bounded GeoJSON page: deadline, caller abort, byte cap, row cap. */ + const getJson = async (url, limit, signal) => { + const controller = new AbortController(); + const abort = () => controller.abort(signal.reason); + signal?.addEventListener('abort', abort, { once: true }); + const timer = setTimeout( + () => controller.abort(new Error('ECCC request timed out')), + timeoutMs, + ); + try { + signal?.throwIfAborted(); + const response = await fetchImpl(url, { + headers: { Accept: 'application/json' }, + signal: controller.signal, + }); + if (!response.ok) throw new Error(`ECCC HTTP ${response.status}`); + const payload = await readResponseJsonCapped( + response, + MAX_RESPONSE_BYTES, + controller.signal, + ); + // The server's `limit` is a request, not a guarantee. + if (Array.isArray(payload?.features) && payload.features.length > limit) + return { ...payload, features: payload.features.slice(0, limit) }; + return payload; + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + } }; return { @@ -54,6 +78,7 @@ export function createEcccAirQualitySource({ if (stations.size === 0 || clock - stationsFetchedAt > STATION_TTL_MS) { const payload = await getJson( `${STATIONS_URL}?f=json&limit=${STATION_LIMIT}`, + STATION_LIMIT, signal, ); const next = normalizeAqhiStations(payload); @@ -68,6 +93,7 @@ export function createEcccAirQualitySource({ const payload = await getJson( `${OBSERVATIONS_URL}?f=json&latest=true&limit=${OBSERVATION_LIMIT}`, + OBSERVATION_LIMIT, signal, ); const readings = normalizeAqhiObservations(payload, stations, clock); diff --git a/src/layers/airQuality/source.test.mjs b/src/layers/airQuality/source.test.mjs new file mode 100644 index 000000000..b9ffe34a6 --- /dev/null +++ b/src/layers/airQuality/source.test.mjs @@ -0,0 +1,197 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createEcccAirQualitySource } from './source.js'; + +const NOW = Date.parse('2026-09-24T18:30:00Z'); +const HOUR = 60 * 60 * 1000; +const station = (id) => ({ + type: 'Feature', + geometry: { type: 'Point', coordinates: [-79.38, 43.65] }, + properties: { location_id: id, location_name_en: id }, +}); +const observation = (id, at = NOW, aqhi = 3) => ({ + type: 'Feature', + properties: { + location_id: id, + aqhi, + observation_datetime: new Date(at - 10 * 60_000).toISOString(), + }, +}); +const json = (body, init) => + new Response(JSON.stringify(body), { + headers: { 'content-type': 'application/json' }, + ...init, + }); + +/** A fetch that serves the station catalog and observations from mutable state. */ +function feed() { + const state = { + clock: NOW, + stations: [station('A')], + stationIds: null, + urls: [], + }; + state.source = createEcccAirQualitySource({ + now: () => state.clock, + fetchImpl: async (url) => { + state.urls.push(url); + return json({ + type: 'FeatureCollection', + // Every known station reports ten minutes before the current clock. + features: url.includes('aqhi-stations') + ? state.stations + : (state.stationIds || ['A']).map((id) => + observation(id, state.clock), + ), + }); + }, + }); + state.stationRequests = () => + state.urls.filter((url) => url.includes('aqhi-stations')).length; + return state; +} + +test('the station catalog is cached for twelve hours, then refetched', async () => { + const state = feed(); + await state.source.getSnapshot(); + state.clock = NOW + 12 * HOUR; + await state.source.getSnapshot(); + assert.equal(state.stationRequests(), 1, 'within the TTL the cache serves'); + state.clock = NOW + 12 * HOUR + 1; + await state.source.getSnapshot(); + assert.equal( + state.stationRequests(), + 2, + 'past the TTL the catalog refreshes', + ); +}); + +test('an empty catalog refresh keeps the previous catalog', async () => { + const state = feed(); + await state.source.getSnapshot(); + state.stations = []; + state.clock = NOW + 13 * HOUR; + const readings = await state.source.getSnapshot(); + assert.deepEqual( + readings.map((reading) => reading.stationId), + ['A'], + ); + // The empty refresh did not count as fresh, so the next call retries it. + await state.source.getSnapshot(); + assert.equal(state.stationRequests(), 3); +}); + +test('an empty first catalog is an error, not an empty layer', async () => { + const state = feed(); + state.stations = []; + await assert.rejects(state.source.getSnapshot(), /No AQHI stations/); +}); + +test('requests ask for JSON, the latest observations only, and a bounded page', async () => { + const state = feed(); + await state.source.getSnapshot(); + const [stations, observations] = state.urls.map((url) => new URL(url)); + assert.equal(stations.pathname, '/collections/aqhi-stations/items'); + assert.equal(stations.searchParams.get('f'), 'json'); + assert.equal(stations.searchParams.get('limit'), '500'); + assert.equal( + observations.pathname, + '/collections/aqhi-observations-realtime/items', + ); + // Without latest=true the collection returns the full history (~2,000 rows). + assert.equal(observations.searchParams.get('latest'), 'true'); + assert.equal(observations.searchParams.get('limit'), '500'); +}); + +test('rows past the requested limit are ignored', async () => { + const state = feed(); + state.stations = Array.from({ length: 600 }, (_, index) => + station(`S${String(index).padStart(3, '0')}`), + ); + state.stationIds = state.stations.map((row) => row.properties.location_id); + const readings = await state.source.getSnapshot(); + assert.equal(readings.length, 500); +}); + +test('a malformed catalog and HTTP errors reject', async () => { + await assert.rejects( + createEcccAirQualitySource({ + fetchImpl: async () => json({ features: 'no' }), + }).getSnapshot(), + /Malformed AQHI station/, + ); + await assert.rejects( + createEcccAirQualitySource({ + fetchImpl: async () => json({}, { status: 503 }), + }).getSnapshot(), + /ECCC HTTP 503/, + ); +}); + +test('an oversized response is refused before it is parsed', async () => { + const big = 'x'.repeat(4 * 1024 * 1024 + 1); + await assert.rejects( + createEcccAirQualitySource({ + fetchImpl: async () => + new Response(new Blob([big]).stream(), { + headers: { 'content-type': 'application/json' }, + }), + }).getSnapshot(), + (error) => error.code === 'RESPONSE_TOO_LARGE', + ); + await assert.rejects( + createEcccAirQualitySource({ + fetchImpl: async () => + json({}, { headers: { 'content-length': String(5 * 1024 * 1024) } }), + }).getSnapshot(), + (error) => error.code === 'RESPONSE_TOO_LARGE', + ); +}); + +test('a caller abort cancels the request and detaches its listener', async () => { + const caller = new AbortController(); + let requestSignal = null; + const source = createEcccAirQualitySource({ + fetchImpl: (_url, { signal }) => { + requestSignal = signal; + return new Promise((_resolve, reject) => + signal.addEventListener('abort', () => reject(signal.reason), { + once: true, + }), + ); + }, + }); + const pending = source.getSnapshot({ signal: caller.signal }); + caller.abort(new Error('layer disabled')); + await assert.rejects(pending, /layer disabled/); + assert.equal(requestSignal.aborted, true); + const already = new AbortController(); + already.abort(new Error('before start')); + let calls = 0; + await assert.rejects( + createEcccAirQualitySource({ + fetchImpl: async () => { + calls += 1; + return json({ features: [] }); + }, + }).getSnapshot({ signal: already.signal }), + /before start/, + ); + assert.equal(calls, 0); +}); + +test('a request that outlives its deadline is aborted', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const source = createEcccAirQualitySource({ + timeoutMs: 1_000, + fetchImpl: (_url, { signal }) => + new Promise((_resolve, reject) => + signal.addEventListener('abort', () => reject(signal.reason), { + once: true, + }), + ), + }); + const pending = source.getSnapshot(); + t.mock.timers.tick(1_000); + await assert.rejects(pending, /ECCC request timed out/); +}); diff --git a/src/layers/cyclones/index.js b/src/layers/cyclones/index.js index 7664bc821..da67afe1d 100644 --- a/src/layers/cyclones/index.js +++ b/src/layers/cyclones/index.js @@ -386,6 +386,7 @@ export function createCyclonesLayer({ })), }, chips: [], + legendCategorical: true, legend: storm ? [ { label: 'Advisory center / forecast track', color: '#7fe6ed' }, diff --git a/src/ui/layerPanel.js b/src/ui/layerPanel.js index c2dafbe61..37269fa2b 100644 --- a/src/ui/layerPanel.js +++ b/src/ui/layerPanel.js @@ -382,6 +382,7 @@ export class LayerPanel { 'weather-satellite', 'weather-lightning', 'weather-cyclones', + 'air-quality', ].includes(layer.id), ) .map((layer) => ({ diff --git a/src/ui/layerPanel.test.mjs b/src/ui/layerPanel.test.mjs index 1258cc76a..40c5fc81e 100644 --- a/src/ui/layerPanel.test.mjs +++ b/src/ui/layerPanel.test.mjs @@ -258,3 +258,31 @@ test('an omitted source is visibly unavailable and cannot be toggled on', async 'an active layer can always be switched off', ); }); + +test('the Weather readout receives enabled air-quality alongside weather layers', async () => { + const { LayerPanel } = await import('./layerPanel.js'); + const layers = ['wind', 'air-quality', 'other'].map((id) => ({ + id, + icon: '*', + enabled: true, + })); + layers.push({ id: 'weather-radar', icon: '*', enabled: false }); + const panel = new LayerPanel({ + getLayers: () => layers, + getRowControls: (id) => ({ readout: true, summary: { label: id } }), + }); + let entries = null; + panel._weatherPanel = { + update: (next) => { + entries = next; + }, + }; + panel._refreshWeatherPanel(); + assert.deepEqual( + entries.map(({ id, summary }) => [id, summary.label]), + [ + ['wind', 'wind'], + ['air-quality', 'air-quality'], + ], + ); +}); diff --git a/src/ui/weatherPanel.js b/src/ui/weatherPanel.js index 908423cb7..8eac004f8 100644 --- a/src/ui/weatherPanel.js +++ b/src/ui/weatherPanel.js @@ -10,8 +10,9 @@ const ORDER = [ 'weather-radar', 'weather-satellite', 'weather-lightning', + 'air-quality', ]; -const OBSERVED = new Set(ORDER.slice(2)); +const OBSERVED = new Set(ORDER.slice(2, 5)); const utc = (time) => Number.isFinite(Date.parse(time)) ? `${new Date(time).toISOString().slice(5, 16).replace('T', ' ')} UTC` @@ -158,7 +159,10 @@ export function createWeatherPanel({ ? 'LATEST · newest per product' : historyTime(state.target), }); - const models = active.map(({ id, icon, summary, legend = [], list }) => { + const models = active.map((entry) => { + const { id, icon, summary, legend = [], list } = entry; + // Discrete classes (storm geometry, AQHI bands) list swatches, not a ramp. + const categorical = entry.legendCategorical === true; let detail = summary.detail; const product = state.products.find((item) => item.id === id); if (OBSERVED.has(id)) { @@ -191,17 +195,15 @@ export function createWeatherPanel({ id: 'legend', type: 'legend', legend: { - categorical: id === 'weather-cyclones', + categorical, colors: legend.map(({ color }) => color), labels: legend.map(({ label }) => label), - units: id === 'weather-cyclones' ? '' : summary.units, + units: categorical ? '' : summary.units, zeroIndex: legend.findIndex(({ label }) => label === '0'), }, }; const blocks = [ - ...(id === 'weather-cyclones' && list?.items?.length - ? [{ id: 'storms', type: 'list', list }] - : []), + ...(list?.items?.length ? [{ id: 'list', type: 'list', list }] : []), { id: 'details', type: 'lines', lines }, ...(legend.length ? [legendBlock] : []), ...(summary.settings?.length diff --git a/src/ui/weatherPanel.test.mjs b/src/ui/weatherPanel.test.mjs index 20d463229..7015ee873 100644 --- a/src/ui/weatherPanel.test.mjs +++ b/src/ui/weatherPanel.test.mjs @@ -2,6 +2,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { createWeatherPanel } from './weatherPanel.js'; import { railFixture } from './railTestFixture.mjs'; +import { createAirQualityLayer } from '../layers/airQuality/index.js'; const ticks = ['2026-09-22T01:00:00Z', '2026-09-22T01:05:00Z']; const wind = { id: 'wind', @@ -166,6 +167,7 @@ test('cyclone readout retains advisory, position, intensity, geometry, coverage, }, ], }, + legendCategorical: true, legend: [ { label: 'Advisory center / forecast track', color: '#7fe6ed' }, { label: 'Center-track uncertainty cone', color: '#7fe6ed44' }, @@ -410,7 +412,7 @@ test('cyclones lead, and observed history owns one bordered group with only acti assert.deepEqual(opened(f), ['weather-cyclones']); assert.equal( card(f, cyclone.id).children[1].children[0].dataset.blockId, - 'storms', + 'list', ); view.update([wind, satellite]); assert.equal(group.children[1].textContent, 'Satellite clouds'); @@ -654,3 +656,67 @@ test('a change of open card scrolls its header into view once; refreshes never m assert.deepEqual(intoView, [], 'ancestor scrollers are never moved'); view.destroy(); }); + +test('air-quality readout renders its band legend, Canada coverage and worst-first list', async () => { + const observedMs = Date.now() - 10 * 60_000; + const reading = (stationId, value, band, name) => ({ + id: `eccc:${stationId}`, + provider: 'eccc', + scale: 'AQHI', + value, + band, + stationId, + name, + zone: 'on', + lat: 43.65, + lon: -79.38, + observedMs, + color: '#4cc9f0', + label: String(value), + risk: `${band} risk`, + note: null, + }); + const layer = createAirQualityLayer({ + source: { + getSnapshot: async () => [ + reading('B', 8, 'high', 'Hamilton'), + reading('A', 2, 'low', 'Toronto'), + ], + }, + overlayHost: { setEntries() {}, setVisible() {}, clearSource() {} }, + }); + layer.init({ dataSources: { add() {}, remove() {} } }); + layer.enable(); + await layer.update(); + const f = fixture(); + const view = createWeatherPanel(f); + view.update([{ id: layer.id, icon: layer.icon, ...layer.getRowControls() }]); + const aq = card(f, 'air-quality'); + assert.ok(aq, 'the air-quality card is shown'); + assert.equal(aq.dataset.open, 'true'); + assert.match( + f.find((n) => n.classList?.contains?.('rail-card-badge'), aq).textContent, + /Canada only/, + ); + assert.match(line(f, 'air-quality', 'status').textContent, /2 stations/); + const entries = f.find((n) => n.className === 'rail-card-legend-entries', aq); + assert.equal(entries.hidden, false); + assert.deepEqual( + entries.children.map((entry) => entry.children[1].textContent), + ['Low', 'Moderate', 'High', 'Very High'], + ); + assert.equal( + f.find((n) => n.className === 'rail-card-ramp', aq).hidden, + true, + ); + const list = f.find((n) => n.dataset.blockId === 'list', aq); + assert.equal(list.hidden, false); + assert.deepEqual( + list.children.map((item) => + item.children[0].children.map((part) => part.textContent).join(' '), + ), + ['AQHI 8 Hamilton · high risk', 'AQHI 2 Toronto · low risk'], + ); + view.destroy(); + layer.destroy(); +}); diff --git a/src/voice/actionSchemas.test.mjs b/src/voice/actionSchemas.test.mjs index c57b4a1ab..550d725e4 100644 --- a/src/voice/actionSchemas.test.mjs +++ b/src/voice/actionSchemas.test.mjs @@ -37,7 +37,8 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea // omit top-level anyOf (OpenAI rejects it). Contacts list routing preserves // requested radius and center authority in descriptions only. // Street Level and GNSS extend the manifest; tracked and analyst aircraft describe broadcasts. - '9398815a573ab29ecc3ce527f6e1a2ad883804e7696e0444aebb11f973700d4b', + // Air quality (ECCC AQHI) extends the manifest. + '775ba323384b4c1b62fb8415bac52aecc3fc668e20bedcfa57e426dcd1da3969', ); }); diff --git a/src/voice/layerManifest.js b/src/voice/layerManifest.js index 286b8a5d8..eb0f09909 100644 --- a/src/voice/layerManifest.js +++ b/src/voice/layerManifest.js @@ -250,6 +250,24 @@ const layers = [ context: false, noQuery: 'map overlay; no countable records', }, + { + id: 'air-quality', + aliases: ['air quality', 'aqhi', 'smog', 'air pollution'], + context: false, + query: { + fields: { + value: ['n', 'AQHI'], + band: ['t'], + scale: ['t'], + provider: ['t'], + name: ['t'], + zone: ['t'], + risk: ['t'], + observedMs: ['ms'], + }, + timeField: 'observedMs', + }, + }, { id: 'traffic', aliases: ['street traffic', 'road traffic', 'congestion'], From 9bee985abbf83755cb32b0f2735e316558f0db68 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Tue, 13 Oct 2026 14:04:09 -0400 Subject: [PATCH 69/95] fix(bikeshare): preserve unknown availability counts --- CHANGELOG.md | 2 + docs/CURRENT-STATE.md | 3 ++ src/layers/bikeshare/proxyRoute.test.mjs | 52 ++++++++++++++++++++++++ src/sources/gbfsStations.js | 1 + 4 files changed, 58 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f6fb2f3a..68530b1bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,8 @@ - Tool string length validation counts Unicode characters correctly, including characters outside the basic multilingual plane. - Director timeline seeks cross the dateline along the short longitude arc. +- Keep unknown GBFS availability counts neutral instead of turning null or empty + values into zero bikes. - Recent Imagery continues past malformed records in full catalog pages, respects the requested record limit, and reports results as truncated when that limit drops valid records. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 98a302c0f..1caf7f799 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1275,6 +1275,9 @@ and stale/error responses remain unchanged. Each has a Node-only package entry under `gods-eye-view/server/providers/`. Portable terrain mechanics, traffic tile math and GBFS source rules are available under `gods-eye-view/sources/`. The browser layers and their rendering remain in their existing modules. +GBFS null or empty availability counts retain an unknown value through station +parsing and rendering; they use the neutral station color rather than reporting +an empty dock. Explicit zero and caller-supplied numeric fallbacks remain valid. ## Terrain height cache bound diff --git a/src/layers/bikeshare/proxyRoute.test.mjs b/src/layers/bikeshare/proxyRoute.test.mjs index 9bb04ced0..e4153a203 100644 --- a/src/layers/bikeshare/proxyRoute.test.mjs +++ b/src/layers/bikeshare/proxyRoute.test.mjs @@ -394,3 +394,55 @@ for (const hook of ['configureServer', 'configurePreviewServer']) { ); }); } + +test('unknown GBFS counts stay unknown through the source, model and station color', async () => { + const { createRendering } = await import('./rendering.js'); + const { COLOR_NEUTRAL, COLOR_RED } = await import('./policy.js'); + for (const missing of [null, '']) { + const source = createBikeshareSource({ + fetchImpl: async () => + Response.json({ + data: { + stations: [ + { + station_id: '2537', + num_bikes_available: missing, + num_docks_available: 4, + is_installed: true, + is_renting: true, + is_returning: true, + }, + ], + }, + }), + }); + const model = createModel({}); + const status = model + .parseStationStatus(await source.getStations(AUSTIN.stationStatusUrl)) + .get('2537'); + const rendering = createRendering({ + state: {}, + services: { render: {} }, + parts: { model }, + }); + assert.equal(status.bikesAvailable, null); + assert.equal( + model.toNonNegativeInteger(status.bikesAvailable), + null, + 'render-map normalization preserves unknown', + ); + assert.equal(rendering.statusToColor(status, 12), COLOR_NEUTRAL); + assert.equal(rendering.resolveCapacity(missing, status), 4); + assert.equal( + rendering.statusToColor({ ...status, bikesAvailable: 0 }, 12), + COLOR_RED, + 'reported zero is still empty', + ); + assert.equal( + model.toNonNegativeInteger(missing, 7), + 7, + 'caller fallback is preserved', + ); + assert.equal(model.toNonNegativeInteger('3'), 3); + } +}); diff --git a/src/sources/gbfsStations.js b/src/sources/gbfsStations.js index efa1e6169..74f2efd23 100644 --- a/src/sources/gbfsStations.js +++ b/src/sources/gbfsStations.js @@ -25,6 +25,7 @@ export function normalizeGbfsBool(value, fallback = true) { * @returns {number|null} */ export function toNonNegativeInteger(value, fallback = null) { + if (value == null || value === '') return fallback; const n = Number(value); if (!Number.isFinite(n) || n < 0) return fallback; return Math.round(n); From 40f4e121db79cccd8be882e498ccee49efd7f074 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Wed, 14 Oct 2026 20:15:48 -0400 Subject: [PATCH 70/95] docs: separate CHANGELOG bullets after rebase Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68530b1bc..5f26b6120 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,7 @@ - Tool string length validation counts Unicode characters correctly, including characters outside the basic multilingual plane. - Director timeline seeks cross the dateline along the short longitude arc. + - Keep unknown GBFS availability counts neutral instead of turning null or empty values into zero bikes. From 7b1032bfc75aaa19cf745df463fc68a42176aa90 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 15:15:49 -0700 Subject: [PATCH 71/95] docs(cctv): document the Toronto pack and the shared camera cap Adds the Toronto settings to .env.example and a current-state section for the pack, restores an approximate camera count in the README, and corrects the claim that adding a pack never trims another region: with the shared 4,000-camera cap, the largest packs give way round-robin. Fixes two stale comments about the list's refresh and the ground-height sidecar. --- .env.example | 3 +++ README.md | 2 +- docs/CURRENT-STATE.md | 20 +++++++++++++++++++- server/providers/cctv/constants.js | 11 ++++++----- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index 4e12a8eca..966a64ba0 100644 --- a/.env.example +++ b/.env.example @@ -226,6 +226,9 @@ AISSTREAM_API_KEY= # CCTV_NYC_ENABLED= # off by default; set 1 only if you hold a NYC DOT data-sharing agreement (tmcdot@dot.nyc.gov) for the camera feed # CCTV_NYC_MAX_SOURCES=250 # cap cameras loaded from the NYC TMC catalog (nearest borough cores first) # CCTV_NYC_CAMERAS_URL= # override the NYC TMC catalog URL +# CCTV_TORONTO_ENABLED=1 # set 0 to disable the City of Toronto RESCU cameras +# CCTV_TORONTO_MAX_SOURCES=336 # cap cameras loaded from Toronto RESCU (clamped to 8–500) +# CCTV_TORONTO_LIST_URL= # override the Toronto RESCU camera list URL # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/README.md b/README.md index 5fb666162..e78bcb404 100644 --- a/README.md +++ b/README.md @@ -377,7 +377,7 @@ Twenty layers and map sources. **Eighteen have a keyless path.** Some offer addi | 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | | ◍ **Air quality** | Live AQHI on the published 1–10+ health-risk scale, with a Weather-panel card showing the band legend and the highest readings in view. **Canada only** — ECCC is the first registered provider | ECCC MSC GeoMet | 🟢 | | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | Public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Toronto, plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📹 **CCTV Mesh** | ~4,000 public cameras (shared cap) projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Toronto, plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | | 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary, Google Street View, KartaView and Panoramax | Mapillary · Google · KartaView · Panoramax | 🟢 KartaView, Panoramax · 🟡 Mapillary token · 🔴 Google Maps key | | 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index aab9d0447..0f5e3a05d 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -977,7 +977,25 @@ id-hash fallback at low confidence. Once enabled, the 250 cameras nearest the five borough cores load. `CCTV_NYC_MAX_SOURCES` sets the pack cap (8–400). The catalog-wide ceiling stays at 4,000 and is shared -round-robin, so adding this pack does not silently drop another region. +round-robin: no region is dropped outright, but when the packs together exceed +the ceiling, adding a pack trims cameras from the largest packs. + +## City of Toronto RESCU CCTV source pack + +The CCTV catalog adds the City of Toronto RESCU traffic cameras — downtown +arterials plus the Gardiner, DVP, Allen and Lake Shore, which the Ontario 511 +pack does not cover. The server reads the keyless camera list the City +publishes on its open data portal (`tmcearthcameras.json`, served as JSONP). +The list has no URL field: frame URLs are built from each validated, digits-only +camera number on the fixed origin +`https://opendata.toronto.ca/transportation/tmc/rescucameraimages/CameraImages/`. +The feed has no compass data, so headings are low-confidence priors. Some +cameras return a 200 "Image Unavailable" placeholder frame instead of an error. + +By default all ~336 cameras load, nearest downtown first. +`CCTV_TORONTO_MAX_SOURCES` sets the pack cap (8–500) and +`CCTV_TORONTO_ENABLED=0` turns the pack off. The pack shares the 4,000 +catalog-wide ceiling round-robin with the other packs. ## DriveBC CCTV source pack diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index e4837bdb4..b399088bb 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -309,7 +309,8 @@ export const DELDOT_ANCHORS = [ * City of Toronto RESCU traffic cameras: the municipal network the province's * 511 pack does not cover — downtown arterials plus the Gardiner, DVP, Allen * and Lake Shore. One keyless list for the whole city (~336 cameras), from the - * City's open data portal, refreshed within two minutes of any change. + * City's open data portal. It is fetched on every catalog refresh, but the + * upstream file is not updated often and may lag the live camera network. * * The list is served as JSONP (`jsonTMCEarthCamerasCallback({...})`); the * portal publishes the same data as .xml and .csv beside it. @@ -329,10 +330,10 @@ export const DEFAULT_TORONTO_MAX_SOURCES = 336; /** Yonge / Queen: the prioritization anchor. */ export const TORONTO_DOWNTOWN = { lat: 43.6532, lon: -79.3832 }; /** The city climbs from 76 m at the lake to 209 m at Steeles, so no single - * number fits: this is the mean across the 336 camera sites. Placement comes - * from the precomputed ground-height sidecar wherever a camera has an entry; - * this prior is only the fallback for a sidecar miss or an edited pose, and - * the client's ground snap corrects it wherever 3D tiles are loaded. */ + * number fits: this is the mean across the 336 camera sites. The ground-height + * sidecar has no Toronto entries yet, so this prior seeds placement; a camera + * that gains a sidecar entry uses it instead, and the client's ground snap + * corrects the prior wherever 3D tiles are loaded. */ export const TORONTO_GROUND_ELEVATION_M = 116; /** Hard ceiling on the RESCU list body. The whole city is ~336 rows and under * 60 KB; this only exists so an upstream that streams an unbounded body cannot From 4887162d761fb47ffbd59eb799165a0b47c422d6 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 15:17:11 -0700 Subject: [PATCH 72/95] fix(cctv): harden the King County catalog fetch and dedupe before the cap The catalog fetch now refuses redirects, cancels rejected bodies and reads through the shared capped JSON reader. Duplicate ids are dropped before prioritization so they cannot use up the per-pack cap. Vendor-generic model strings (PTZ, PTZ IP, BOSCH/PTZ) are treated as no model. Comments and credits say why WSDOT rows are skipped (their frames are WSDOT-hosted, outside the pinned origin) and name Newcastle as the partner city that appears. --- DATA_SOURCES.md | 2 +- server/providers/cctv/constants.js | 6 +- server/providers/cctv/sources.js | 56 +++++++++++--- src/data/cctvKingCountySource.test.mjs | 102 +++++++++++++++++++++++-- src/data/dataCredits.js | 2 +- 5 files changed, 148 insertions(+), 20 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 1273ddcce..45de1c92a 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -143,7 +143,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped so a camera never appears twice when a statewide WSDOT pack is present; the few city-owned cameras the county hosts (Redmond, Sammamish, Newcastle, Snoqualmie) carry their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); both are carried on the source (and served by `/api/cctv/sources`) as `model` and `manufacturer` for identified-hardware consumers. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. +- **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped because their frames are WSDOT-hosted, outside the pinned image origin; partner-city cameras hosted on the county image handler (e.g. Newcastle) name their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); both are carried on the source (and served by `/api/cctv/sources`) as `model` and `manufacturer` for identified-hardware consumers (vendor-generic strings such as `PTZ` name no model and are omitted). Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 7fd6ef950..313a3338d 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -315,7 +315,7 @@ export const DELDOT_ANCHORS = [ * (Cohu 3950/3965, Bosch AUTODOME, Axis Q6135-LE, …), carried through as the * source's `model`. */ export const KING_COUNTY_CAMERAS_URL = - 'https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,Description,ImageURL,CamRegion,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson'; + 'https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,ImageURL,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson'; /** Frames come only from the county's own image handler host. */ export const KING_COUNTY_IMAGE_ORIGIN = 'https://info.kingcounty.gov/'; /** The whole catalog is ~125 rows (~59 live county cameras); no thinning by @@ -323,6 +323,10 @@ export const KING_COUNTY_IMAGE_ORIGIN = 'https://info.kingcounty.gov/'; export const DEFAULT_KING_COUNTY_MAX_SOURCES = 125; /** Prioritization anchor: Renton, roughly central to the county road network. */ export const KING_COUNTY_ANCHORS = [{ lat: 47.4829, lon: -122.2171 }]; +/** Hard ceiling on the King County catalog body. The whole layer is ~125 rows + * and well under 100 KB; this only exists so an upstream that streams an + * unbounded body cannot be buffered without limit. */ +export const KING_COUNTY_MAX_CATALOG_BYTES = 1024 * 1024; /** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 4955be563..84d694c70 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -63,6 +63,7 @@ import { KING_COUNTY_IMAGE_ORIGIN, DEFAULT_KING_COUNTY_MAX_SOURCES, KING_COUNTY_ANCHORS, + KING_COUNTY_MAX_CATALOG_BYTES, DEFAULT_VEGVESEN_CCTV_URL, VEGVESEN_IMAGE_ORIGIN, VEGVESEN_VIDEO_URL, @@ -1733,6 +1734,17 @@ function isLikelyKingCountyCoordinate(lat, lon) { ); } +/** + * A King County `Model` value, whitespace-collapsed, or '' when it is a + * vendor-generic string ("PTZ", "PTZ IP", "BOSCH/PTZ") that names no model. + */ +function kingCountyModel(raw) { + const model = String(raw || '') + .replace(/\s+/g, ' ') + .trim(); + return /^(?:[a-z]+\/)?ptz(?: ip)?$/i.test(model) ? '' : model; +} + /** * Fetch King County road cameras (Washington State). Keyless: one public * ArcGIS FeatureServer GeoJSON query (King County DOT Road Services), with an @@ -1740,16 +1752,17 @@ function isLikelyKingCountyCoordinate(lat, lon) { * never fetched. Only cameras that are live and active (`Live`, * `CurrentStatus`) with a positive integer id, finite in-county coordinates, * and a frame on the county's own image handler are kept. WSDOT-owned rows in - * the layer are skipped — the statewide WSDOT pack is their home — while the - * handful of city-owned cameras the county hosts (Redmond, Sammamish, - * Newcastle, Snoqualmie) keep their owner as the per-camera credit, + * the layer are skipped because their frames are WSDOT-hosted, outside the + * pinned image origin, while partner-city cameras hosted on the county image + * handler (e.g. Newcastle) name their owner as the per-camera credit, * DriveBC-style. The layer publishes no facing, so every heading is the id * hash at low confidence. * * Uniquely among the packs, the county publishes each camera's hardware * `Manufacturer`/`Model`; the model string (whitespace-collapsed — the feed * carries stray newlines) rides along as the source's `model` for - * identified-hardware consumers. + * identified-hardware consumers. Vendor-generic strings ("PTZ", "PTZ IP", + * "BOSCH/PTZ") name no model and are treated as absent. * * @returns {Promise>} Normalized camera source objects. */ @@ -1757,16 +1770,37 @@ export async function loadKingCountySourcesFromOpenData() { try { const resp = await fetch(KING_COUNTY_CAMERAS_URL, { headers: { Accept: 'application/json' }, + redirect: 'manual', signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), }); + // A response this loader will not read still owns its transport until the + // body is released, so every rejection path cancels before returning. + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn( + '[CCTV] King County catalog redirected; redirects are not followed', + ); + return discard(); + } if (!resp.ok) { console.warn('[CCTV] King County camera download failed:', resp.status); - return []; + return discard(); } - const payload = await resp.json(); + const payload = await readResponseJsonCapped( + resp, + KING_COUNTY_MAX_CATALOG_BYTES, + ); const rows = Array.isArray(payload?.features) ? payload.features : []; const cameras = []; + const seen = new Set(); for (const row of rows) { const attrs = row?.properties; const rawId = attrs?.AssetID; @@ -1774,8 +1808,8 @@ export async function loadKingCountySourcesFromOpenData() { if (attrs.Live !== 1) continue; if (String(attrs.CurrentStatus || '').trim() !== 'Active') continue; const owner = String(attrs.Owner || '').trim(); - // WSDOT-owned rows belong to the statewide WSDOT pack; skipping them - // here keeps one camera from appearing twice on the globe. + // WSDOT-owned rows serve WSDOT-hosted frames, outside the pinned + // county image origin, so they are skipped. if (owner === 'WSDOT') continue; // The handler is served over HTTPS; the feed still writes http://. const imageUrl = String(attrs.ImageURL || '') @@ -1788,9 +1822,9 @@ export async function loadKingCountySourcesFromOpenData() { if (!isLikelyKingCountyCoordinate(lat, lon)) continue; const cameraId = `kingcounty-${rawId}`; - const model = String(attrs.Model || '') - .replace(/\s+/g, ' ') - .trim(); + if (seen.has(cameraId)) continue; + seen.add(cameraId); + const model = kingCountyModel(attrs.Model); const manufacturer = String(attrs.Manufacturer || '') .replace(/\s+/g, ' ') .trim(); diff --git a/src/data/cctvKingCountySource.test.mjs b/src/data/cctvKingCountySource.test.mjs index a0f765446..5da0ae708 100644 --- a/src/data/cctvKingCountySource.test.mjs +++ b/src/data/cctvKingCountySource.test.mjs @@ -6,6 +6,7 @@ import path from 'node:path'; import { DEFAULT_KING_COUNTY_MAX_SOURCES, KING_COUNTY_CAMERAS_URL, + KING_COUNTY_MAX_CATALOG_BYTES, } from '../../server/providers/cctv/constants.js'; import { loadKingCountySourcesFromOpenData } from '../../server/providers/cctv/sources.js'; import { normalizeSourceItem } from '../../server/providers/cctv/normalize.js'; @@ -73,9 +74,9 @@ test('King County loader keeps live county cameras and upgrades frame URLs to HT }), // City-owned camera the county hosts: kept, owner becomes the credit. kingCountyFeature(90, 47.674, -122.119, { - properties: { Owner: 'City of Redmond', Model: null }, + properties: { Owner: 'City of Newcastle', Model: null }, }), - // WSDOT-owned row: the statewide WSDOT pack is its home. + // WSDOT-owned row: its frames are WSDOT-hosted, so it is skipped. kingCountyFeature(70, 47.6, -122.3, { properties: { Owner: 'WSDOT' } }), // Off-host frame, not live, retired, bad id, bad geometry, out of county. kingCountyFeature(71, 47.6, -122.3, { @@ -118,11 +119,11 @@ test('King County loader keeps live county cameras and upgrades frame URLs to HT assert.equal(novelty.fovDeg, 44); assert.ok(novelty.headingDeg >= 0 && novelty.headingDeg < 360); - const redmond = cameras.find((camera) => camera.id === 'kingcounty-90'); - assert.equal(redmond.credit, 'City of Redmond'); - assert.equal(redmond.model, undefined); + const newcastle = cameras.find((camera) => camera.id === 'kingcounty-90'); + assert.equal(newcastle.credit, 'City of Newcastle'); + assert.equal(newcastle.model, undefined); // A manufacturer without a model identifies nothing, so it is dropped too. - assert.equal(redmond.manufacturer, undefined); + assert.equal(newcastle.manufacturer, undefined); }); test('normalizeSourceItem passes a hardware model through and omits it otherwise', () => { @@ -275,3 +276,92 @@ test('King County loader defaults its cap and survives upstream failure', async }); assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); }); + +test('King County loader does not follow a redirected catalog', async (t) => { + quiet(t); + let init; + t.mock.method(globalThis, 'fetch', async (_url, options) => { + init = options; + return new Response( + JSON.stringify({ + type: 'FeatureCollection', + features: [kingCountyFeature(56, 47.70059, -122.02443)], + }), + { status: 302, headers: { Location: 'https://example.com/cams' } }, + ); + }); + + assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); + assert.equal(init.redirect, 'manual'); +}); + +test('King County loader rejects an oversized catalog', async (t) => { + quiet(t); + const features = [kingCountyFeature(56, 47.70059, -122.02443)]; + const body = JSON.stringify({ + type: 'FeatureCollection', + features, + padding: 'x'.repeat(KING_COUNTY_MAX_CATALOG_BYTES), + }); + t.mock.method(globalThis, 'fetch', async () => new Response(body)); + + assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); +}); + +test('King County loader drops duplicate ids before the per-pack cap', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: '8' }); + // Repeats of the camera nearest the anchor would fill an 8-camera cap + // before the second, farther camera if duplicates reached prioritization. + const features = []; + for (let i = 0; i < 10; i += 1) { + features.push(kingCountyFeature(56, 47.4829, -122.2171)); + } + features.push(kingCountyFeature(57, 47.70059, -122.02443)); + t.mock.method(globalThis, 'fetch', async () => + Response.json({ type: 'FeatureCollection', features }), + ); + + const cameras = await loadKingCountySourcesFromOpenData(); + assert.deepEqual(cameras.map((camera) => camera.id).sort(), [ + 'kingcounty-56', + 'kingcounty-57', + ]); +}); + +test('King County loader omits vendor-generic model strings', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: undefined }); + t.mock.method(globalThis, 'fetch', async () => + Response.json({ + type: 'FeatureCollection', + features: [ + kingCountyFeature(61, 47.5, -122.2, { + properties: { Model: 'PTZ', Manufacturer: 'Bosch' }, + }), + kingCountyFeature(62, 47.51, -122.2, { + properties: { Model: 'PTZ IP', Manufacturer: 'Cohu' }, + }), + kingCountyFeature(63, 47.52, -122.2, { + properties: { Model: 'BOSCH/PTZ', Manufacturer: 'Bosch' }, + }), + kingCountyFeature(64, 47.53, -122.2, { + properties: { Model: 'AUTODOME IP 5000', Manufacturer: 'Bosch' }, + }), + ], + }), + ); + + const byId = new Map( + (await loadKingCountySourcesFromOpenData()).map((camera) => [ + camera.id, + camera, + ]), + ); + for (const id of ['kingcounty-61', 'kingcounty-62', 'kingcounty-63']) { + assert.equal(byId.get(id).model, undefined, id); + assert.equal(byId.get(id).manufacturer, undefined, id); + } + assert.equal(byId.get('kingcounty-64').model, 'AUTODOME IP 5000'); + assert.equal(byId.get('kingcounty-64').manufacturer, 'Bosch'); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 267e1d6bd..d1bba9a85 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -261,7 +261,7 @@ export const DATA_CREDITS = [ 'kingcounty.gov, ' + 'per the ' + 'KCGIS Center terms. ' + - 'A few cameras are supplied by partner cities (Redmond, Sammamish, Newcastle, Snoqualmie); each names its owner in the CCTV panel.', + 'Partner-city cameras hosted on the county image handler (e.g. Newcastle) name their owner in the CCTV panel.', }, { key: 'vegvesen-cctv', From bcfba7719606c334106e84415458b4f10022e0b0 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 15:17:31 -0700 Subject: [PATCH 73/95] fix(cctv): make the Lithuania pack opt-in until its licence is confirmed The camera endpoints are the eismoinfo.lt website backend, which the Via Lietuva open-data statement does not list, so the pack now follows the NYC opt-in pattern (CCTV_LITHUANIA_ENABLED=1) and the docs say why. A failed catalog request now aborts its sibling instead of leaving it open until the timeout, and a redirect is logged as not followed. --- .env.example | 2 +- CHANGELOG.md | 2 +- DATA_SOURCES.md | 2 +- README.md | 2 +- docs/CURRENT-STATE.md | 6 ++- server/providers/cctv/catalog.js | 15 ++++-- server/providers/cctv/lithuania.js | 45 +++++++++++----- src/data/cctvLithuania.test.mjs | 82 ++++++++++++++++++++++++++---- src/data/dataCredits.js | 5 +- 9 files changed, 125 insertions(+), 36 deletions(-) diff --git a/.env.example b/.env.example index 4c9a747cc..b0b4584b2 100644 --- a/.env.example +++ b/.env.example @@ -226,7 +226,7 @@ AISSTREAM_API_KEY= # CCTV_NYC_ENABLED= # off by default; set 1 only if you hold a NYC DOT data-sharing agreement (tmcdot@dot.nyc.gov) for the camera feed # CCTV_NYC_MAX_SOURCES=250 # cap cameras loaded from the NYC TMC catalog (nearest borough cores first) # CCTV_NYC_CAMERAS_URL= # override the NYC TMC catalog URL -# CCTV_LITHUANIA_ENABLED=1 # set 0 to disable Via Lietuva / Eismoinfo road cameras +# CCTV_LITHUANIA_ENABLED= # off by default; set 1 to load Via Lietuva / Eismoinfo road cameras (photo licence unconfirmed; see DATA_SOURCES.md) # CCTV_LITHUANIA_MAX_SOURCES=400 # cap Lithuanian cameras; no API key required # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry diff --git a/CHANGELOG.md b/CHANGELOG.md index dbf70f0ee..55fa38139 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -- Add Lithuania's public Via Lietuva / Eismoinfo road cameras to CCTV, with published coordinates, on-demand snapshots, attribution and bounded catalog requests. No API key is required. +- Add Lithuania's public Via Lietuva / Eismoinfo road cameras to CCTV, with published coordinates, on-demand snapshots, attribution and bounded catalog requests. No API key is required. The pack is opt-in (`CCTV_LITHUANIA_ENABLED=1`) because its camera endpoints are the eismoinfo.lt website backend, not a service listed in Via Lietuva's open-data statement, so the licence for the photos is unconfirmed. - Stop one retired bikeshare feed from reporting the whole layer as failed. City feeds are fetched independently, but failure was recorded in a single diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 6081134e7..96904f2bf 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -91,7 +91,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei **Quebec is thinly represented**, and that is upstream rather than a filter here: the province runs its own Info-Smog program, so only a handful of Quebec stations appear in ECCC's national feed. The reporting station count also varies hour to hour as stations report or fall silent. -- **Via Lietuva / Eismoinfo (Lithuania).** Public road-camera catalogs and periodically updated still images from [Eismoinfo](https://eismoinfo.lt/), credited to Via Lietuva / eismoinfo.lt in the app. The provider's [open-data statement](https://eismoinfo.lt/eismoinfo-backend/content?id=44) describes reuse of its public data; no specific Creative Commons licence is asserted here. `eismoinfo-backend/camera-info-table` is joined by ID to `layer-static-features/VKR`. Without `lks=true`, the latter returns WGS84 `[latitude, longitude]`; the table's `x/y` instead use LKS-94 metres. Only the official HTTPS `/eismoinfo-backend/image-provider/camera/last?id=…` image endpoint is registered. Catalogs use the shared 15-minute cache; frames are fetched on demand and never bundled or archived. These are periodic snapshots, not continuous video. Camera headings are low-confidence estimates. No API key is required. Disable with `CCTV_LITHUANIA_ENABLED=0`; `CCTV_LITHUANIA_MAX_SOURCES` defaults to 400. +- **Via Lietuva / Eismoinfo (Lithuania).** Public road-camera catalogs and periodically updated still images from [Eismoinfo](https://eismoinfo.lt/), credited to Via Lietuva / eismoinfo.lt in the app. **Off by default.** The camera endpoints are the eismoinfo.lt website backend and are not among the services listed in Via Lietuva's [open-data statement](https://eismoinfo.lt/eismoinfo-backend/content?id=44), whose conditions are attribution and reuse on the same terms. Until the statement is confirmed to cover the camera photos, the pack loads only when an operator sets `CCTV_LITHUANIA_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to Eismoinfo. `eismoinfo-backend/camera-info-table` is joined by ID to `layer-static-features/VKR`. Without `lks=true`, the latter returns WGS84 `[latitude, longitude]`; the table's `x/y` instead use LKS-94 metres. Only the official HTTPS `/eismoinfo-backend/image-provider/camera/last?id=…` image endpoint is registered. Catalogs use the shared 15-minute cache; frames are fetched on demand and never bundled or archived. These are periodic snapshots, not continuous video. Camera headings are low-confidence estimates. No API key is required. `CCTV_LITHUANIA_MAX_SOURCES` defaults to 400. - **Google Maps Platform.** You supply your own API key and are bound by [Google's ToS](https://cloud.google.com/maps-platform/terms). Google Maps Content (tiles, geocodes, places) **may not be cached, stored, rehosted, or committed** — this app only ever uses it live, which is the compliant pattern. The "Google" attribution is displayed on the globe and must stay visible. Restrict your key (see [SECURITY.md](SECURITY.md)). - **OpenSky Network.** Its license is **non-commercial**, and operational use of the REST API in a live product can require a prior written agreement with OpenSky — even for non-profit/government use. If you deploy this commercially, contact OpenSky for your own terms. The flights layer is a toggle and runs anonymously by default. diff --git a/README.md b/README.md index 578ccb185..1950db38c 100644 --- a/README.md +++ b/README.md @@ -377,7 +377,7 @@ Twenty layers and map sources. **Eighteen have a keyless path.** Some offer addi | 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | | ◍ **Air quality** | Live AQHI on the published 1–10+ health-risk scale, with a Weather-panel card showing the band legend and the highest readings in view. **Canada only** — ECCC is the first registered provider | ECCC MSC GeoMet | 🟢 | | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen), plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen), plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement) and opt-in Lithuania (Via Lietuva / Eismoinfo, photo licence unconfirmed). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | | 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary, Google Street View, KartaView and Panoramax | Mapillary · Google · KartaView · Panoramax | 🟢 KartaView, Panoramax · 🟡 Mapillary token · 🔴 Google Maps key | | 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index afac040a7..407d52a48 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -981,7 +981,10 @@ round-robin, so adding this pack does not silently drop another region. ## Lithuania CCTV source pack -The CCTV catalog includes Via Lietuva / Eismoinfo road cameras by default. +The CCTV catalog can include Via Lietuva / Eismoinfo road cameras. The pack is +off unless `CCTV_LITHUANIA_ENABLED=1`, because its camera endpoints are the +eismoinfo.lt website backend rather than a service listed in Via Lietuva's +open-data statement, so the licence for the photos is unconfirmed. Two keyless public endpoints provide current snapshot URLs and WGS84 camera locations; records are joined by ID, with invalid coordinates and off-origin image URLs rejected. Catalog requests refuse redirects, have a 15-second @@ -990,7 +993,6 @@ and fair catalog allocation. An unavailable source contributes an empty pack. Frames use the registered-image proxy and retain Via Lietuva attribution. The images are periodically updated road-condition photographs, not continuous video; camera poses use the existing low-confidence heading prior. -`CCTV_LITHUANIA_ENABLED=0` disables the pack, and `CCTV_LITHUANIA_MAX_SOURCES` sets its cap (default 400, bounded to 1–1000). ## DriveBC CCTV source pack diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index c071b655e..7b2f0e75f 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -26,15 +26,18 @@ import { /** Env kill switch: unset or anything but "0" means enabled. */ const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0'; /** Env opt-in: only "1" enables. For packs whose feed access needs an - * operator agreement, so an unset flag never reaches the upstream. */ + * operator agreement or whose terms are unconfirmed, so an unset flag never + * reaches the upstream. */ const envOptIn = (name) => String(process.env[name] || '').trim() === '1'; /** * Live open-data packs, in merge order. Adding a region is one entry here - * plus its loader in sources.js; the catalog cap is shared across entries - * round-robin (cap.js), so a new pack never silently evicts an older one. + * plus its loader (in sources.js or its own module); the catalog cap is + * shared across entries round-robin (cap.js), so a new pack never silently + * evicts an older one. * Each pack fails independently (allSettled) and is gated by its own env - * kill switch, or by an opt-in flag where access needs an agreement. + * kill switch, or by an opt-in flag where access needs an agreement or the + * terms are unconfirmed. */ const LIVE_PACKS = [ { name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData }, @@ -111,8 +114,10 @@ const LIVE_PACKS = [ load: loadNycSourcesFromOpenData, }, { + // Opt-in until the Via Lietuva open-data terms are confirmed to cover the + // camera photos (see DATA_SOURCES.md). name: 'lithuania', - enabled: () => envEnabled('CCTV_LITHUANIA_ENABLED'), + enabled: () => envOptIn('CCTV_LITHUANIA_ENABLED'), load: loadLithuaniaSources, }, ]; diff --git a/server/providers/cctv/lithuania.js b/server/providers/cctv/lithuania.js index df4ee154d..840e68c6c 100644 --- a/server/providers/cctv/lithuania.js +++ b/server/providers/cctv/lithuania.js @@ -81,26 +81,45 @@ export function lithuaniaCameraToSource(row, feature) { }; } -async function readCatalog(url) { - const signal = AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS); - const response = await fetch(url, { - headers: { Accept: 'application/json' }, - redirect: 'manual', - signal, - }); - if (!response.ok) { - await response.body?.cancel(); - throw new Error(`Catalog HTTP ${response.status}`); +async function readCatalog(url, sibling) { + const signal = AbortSignal.any([ + AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + sibling.signal, + ]); + try { + const response = await fetch(url, { + headers: { Accept: 'application/json' }, + redirect: 'manual', + signal, + }); + if (!response.ok) { + await response.body?.cancel().catch(() => {}); + throw new Error( + response.status >= 300 && response.status < 400 + ? 'catalog redirected; redirects are not followed' + : `Catalog HTTP ${response.status}`, + ); + } + return await readResponseJsonCapped( + response, + LITHUANIA_MAX_CATALOG_BYTES, + signal, + ); + } catch (error) { + // The join needs both catalogs; release the other request at once + // rather than leaving its body open until the timeout. + sibling.abort(); + throw error; } - return readResponseJsonCapped(response, LITHUANIA_MAX_CATALOG_BYTES, signal); } /** Two bounded requests per shared catalog refresh, no API key required. */ export async function loadLithuaniaSources() { try { + const sibling = new AbortController(); const [rows, layers] = await Promise.all([ - readCatalog(LITHUANIA_CAMERAS_URL), - readCatalog(LITHUANIA_LOCATIONS_URL), + readCatalog(LITHUANIA_CAMERAS_URL, sibling), + readCatalog(LITHUANIA_LOCATIONS_URL, sibling), ]); if (!Array.isArray(rows) || !Array.isArray(layers)) return []; const features = layers.find((layer) => layer?.layer === 'VKR')?.features; diff --git a/src/data/cctvLithuania.test.mjs b/src/data/cctvLithuania.test.mjs index 675e8d251..af28a51f6 100644 --- a/src/data/cctvLithuania.test.mjs +++ b/src/data/cctvLithuania.test.mjs @@ -95,25 +95,85 @@ test('loader deduplicates and omits cameras without matching published coordinat ); }); -test('catalog includes Lithuania by default and respects the disable switch', async (t) => { +/** Run one catalog refresh with CCTV_LITHUANIA_ENABLED set to `flag` + * (undefined leaves it unset), restoring the environment afterwards. */ +async function runCatalogWithLithuaniaFlag(t, flag) { const requested = mockCatalogs(t); const saved = process.env.CCTV_LITHUANIA_ENABLED; try { - delete process.env.CCTV_LITHUANIA_ENABLED; + if (flag === undefined) delete process.env.CCTV_LITHUANIA_ENABLED; + else process.env.CCTV_LITHUANIA_ENABLED = flag; const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); - assert.equal( - sources.find((c) => c.id === 'lithuania-72')?.cityId, - 'lithuania', - ); - requested.length = 0; - process.env.CCTV_LITHUANIA_ENABLED = '0'; - const disabled = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); - assert.ok(!disabled.some((c) => c.cityId === 'lithuania')); - assert.ok(!requested.includes(LITHUANIA_CAMERAS_URL)); + return { requested, sources }; } finally { if (saved === undefined) delete process.env.CCTV_LITHUANIA_ENABLED; else process.env.CCTV_LITHUANIA_ENABLED = saved; } +} + +test('CCTV_LITHUANIA_ENABLED=1 wires the Lithuania pack into the catalog', async (t) => { + const { sources } = await runCatalogWithLithuaniaFlag(t, '1'); + assert.equal( + sources.find((c) => c.id === 'lithuania-72')?.cityId, + 'lithuania', + ); +}); + +// The camera endpoints are the eismoinfo.lt website backend, not a service the +// open-data statement lists, so an install that has not opted in must never +// contact them. +for (const [label, flag] of [ + ['an unset CCTV_LITHUANIA_ENABLED', undefined], + ['an empty CCTV_LITHUANIA_ENABLED', ''], + ['CCTV_LITHUANIA_ENABLED=0', '0'], + ['CCTV_LITHUANIA_ENABLED=true', 'true'], +]) { + test(`${label} makes no Lithuania request`, async (t) => { + const { requested, sources } = await runCatalogWithLithuaniaFlag(t, flag); + assert.ok(requested.length > 0, 'the other packs still refreshed'); + assert.equal( + requested.some((href) => href.includes('eismoinfo.lt')), + false, + 'the pack never reaches its upstream', + ); + assert.ok(!sources.some((c) => c.cityId === 'lithuania')); + }); +} + +test('a failed catalog cancels the sibling request instead of waiting for the timeout', async (t) => { + t.mock.method(console, 'warn', () => {}); + let siblingSignal; + t.mock.method(globalThis, 'fetch', async (url, options) => { + if (url === LITHUANIA_CAMERAS_URL) { + return new Response('', { status: 503 }); + } + siblingSignal = options.signal; + // Never settles on its own; only an abort ends it. + return new Promise((_, reject) => { + options.signal.addEventListener('abort', () => + reject(options.signal.reason), + ); + }); + }); + assert.deepEqual(await loadLithuaniaSources(), []); + assert.equal(siblingSignal?.aborted, true); + assert.notEqual(siblingSignal.reason?.name, 'TimeoutError'); +}); + +test('a redirected catalog is reported as not followed', async (t) => { + const warnings = []; + t.mock.method(console, 'warn', (...args) => warnings.push(args.join(' '))); + t.mock.method( + globalThis, + 'fetch', + async () => + new Response('', { + status: 302, + headers: { Location: 'http://127.0.0.1/' }, + }), + ); + assert.deepEqual(await loadLithuaniaSources(), []); + assert.ok(warnings.some((line) => /redirects are not followed/.test(line))); }); test('failed, redirected, malformed and oversized catalogs degrade to an empty pack', async (t) => { diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 4ad739601..0dbd33b72 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -251,7 +251,10 @@ export const DATA_CREDITS = [ { key: 'lithuania-cctv', html: - 'Road-camera stills (Lithuania): ' + + // Opt-in: the camera endpoints are the eismoinfo.lt website backend, not + // among the services Via Lietuva's open-data statement lists (its + // conditions: attribution, reuse on the same terms). + 'Road-camera stills (Lithuania, when enabled): ' + 'Via Lietuva / Eismoinfo; periodically updated images', }, { From 76bfd74a78666e755e6c01e1f22315ebd3b87534 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 15:17:31 -0700 Subject: [PATCH 74/95] fix(cctv): harden the WSDOT catalog fetch and describe its real direction codes The catalog fetch now refuses redirects, cancels rejected bodies and reads through the shared capped reader before the windows-1252 decode. The feed sends cardinal codes plus B (both), O and null, not BW or diagonals; tests, comments and docs now say so, with unchanged id-hash fallbacks. Documents the 8-1,200 per-pack range, drops the unverified low-volume compliance claim for facts and a link to WSDOT's terms, and stops claiming the per-pack defaults fit under the 4,000 cap: it is filled round-robin. --- .env.example | 2 +- CHANGELOG.md | 11 +-- DATA_SOURCES.md | 2 +- server/providers/cctv/constants.js | 18 +++-- server/providers/cctv/sources.js | 41 ++++++++--- src/data/cctvDriveBcSource.test.mjs | 2 +- src/data/cctvWsdotSource.test.mjs | 101 ++++++++++++++++++++++++++-- 7 files changed, 148 insertions(+), 29 deletions(-) diff --git a/.env.example b/.env.example index c37f540f9..bec981341 100644 --- a/.env.example +++ b/.env.example @@ -220,7 +220,7 @@ AISSTREAM_API_KEY= # CCTV_CALGARY_MAX_SOURCES=220 # cap cameras loaded from the Open Calgary catalog # CCTV_CALGARY_ROWS_URL= # override the Open Calgary catalog URL # CCTV_WSDOT_ENABLED=1 # set 0 to disable the WSDOT (Washington State) highway cameras -# CCTV_WSDOT_MAX_SOURCES=250 # cap cameras loaded from the WSDOT catalog, nearest Seattle/Spokane first +# CCTV_WSDOT_MAX_SOURCES=250 # cap cameras loaded from the WSDOT catalog, nearest Seattle/Spokane first; min 8, max 1200 # CCTV_VEGVESEN_ENABLED=1 # set 0 to disable the Statens vegvesen (Norway) road cameras # CCTV_VEGVESEN_MAX_SOURCES=900 # cap cameras loaded from Statens vegvesen (default loads all ~850 live; max 1000) # CCTV_VEGVESEN_VIDEO=1 # set 0 to show stills instead of live HLS for Statens vegvesen video cameras diff --git a/CHANGELOG.md b/CHANGELOG.md index 22199a67c..13d5633d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,12 @@ - Add WSDOT highway cameras (Washington State) as a keyless CCTV source pack: WSDOT's official Travel Center camera layer, with frames registered only on WSDOT's own image host (partner-hosted cameras such as Oregon DOT and City of - Seattle are skipped). Cameras with a single `CompassDirection`, cardinal or - diagonal, get a high-confidence heading; `BW` (Bothways) cameras use the - shared id-hash fallback at low confidence. By default the 250 cameras nearest - Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` sets the cap and - `CCTV_WSDOT_ENABLED=0` turns the pack off (bassem chagra, #644). + Seattle are skipped). Cameras with a single cardinal `CompassDirection` get + a high-confidence heading; `B` (both directions), `O`, and unset directions + use the shared id-hash fallback at low confidence. By default the 250 + cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` sets the + cap (8–1,200) and `CCTV_WSDOT_ENABLED=0` turns the pack off (bassem chagra, + #644). - Stop one retired bikeshare feed from reporting the whole layer as failed. City feeds are fetched independently, but failure was recorded in a single diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index a715784e0..4ce25a4e0 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -143,7 +143,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **WSDOT highway cameras.** Keyless. The camera list is the Esri JSON layer `data.wsdot.wa.gov/travelcenter/Cameras.json` — the same layer WSDOT's own Travel Center map consumes (~1,700 point features, EPSG:3857 geometry converted to WGS84 on load; the body carries Windows-1252 bytes and is decoded as such). Only frames on WSDOT's own `images.wsdot.wa.gov` host are registered — the catalog also lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) whose own terms apply, and those are skipped rather than proxied. WSDOT's layer metadata marks the feed for "low volume" use, which the 15-minute catalog cache and per-request frame fetches respect; frames are never stored or redistributed. `CompassDirection` codes with a single facing (N/E/S/W and the diagonals NE/SE/SW/NW, per WSDOT's layer metadata) give a high-confidence heading; `BW` ("Bothways") rows carry no single facing and use the shared id-hash fallback at low confidence. By default the 250 cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` changes the cap and `CCTV_WSDOT_ENABLED=0` turns the pack off. A keyed WSDOT Traveler Information API serves the same population for higher-volume use and is deliberately not required here. +- **WSDOT highway cameras.** Keyless. The camera list is the Esri JSON layer `data.wsdot.wa.gov/travelcenter/Cameras.json` — the same layer WSDOT's own Travel Center map consumes (~1,700 point features, EPSG:3857 geometry converted to WGS84 on load; the body carries Windows-1252 bytes and is decoded as such). Only frames on WSDOT's own `images.wsdot.wa.gov` host are registered — the catalog also lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) whose own terms apply, and those are skipped rather than proxied. WSDOT's layer metadata marks the feed for "low volume" use (see WSDOT's [travel information disclaimer](https://wsdot.wa.gov/about/policies/travel-information-disclaimer)). The catalog is fetched once per CCTV catalog refresh (every 15 minutes), and a camera's frame is fetched only when that camera is viewed; frames are never stored or redistributed. The live feed's `CompassDirection` is a single cardinal (N/E/S/W), `B` (both directions — most rows), `O`, or null. A single cardinal gives a high-confidence heading (the diagonals NE/SE/SW/NW are mapped too should they appear); `B`, `O`, null, and any other code carry no single facing and use the shared id-hash fallback at low confidence. By default the 250 cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` changes the cap (8–1,200) and `CCTV_WSDOT_ENABLED=0` turns the pack off. A keyed WSDOT Traveler Information API serves the same population for higher-volume use and is deliberately not required here. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 12e27fdd6..883fb0b47 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -8,8 +8,8 @@ export const DEFAULT_AUSTIN_MAX_SOURCES = 250; * Catalog-wide safety ceiling on served cameras. Each pack already caps * itself (nearest-to-anchor first); this bound only matters when the packs * together exceed it, and it is then filled round-robin across packs (see - * cap.js) so no region is silently dropped. Sized above the sum of the - * default per-pack caps so a default install never trims. + * cap.js) so no region is silently dropped. The default per-pack caps + * together can exceed it, so a default install may be thinned evenly here. */ export const DEFAULT_CCTV_MAX_SOURCES = 4000; /** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */ @@ -249,9 +249,9 @@ export const VEGVESEN_IMAGE_ORIGIN = * per-camera manifest path is registered. */ export const VEGVESEN_VIDEO_URL = (cameraId) => `https://kamera.vegvesen.no/public/${cameraId}/manifest.m3u8`; -/** Above the ~850 working cameras so a default install keeps them all, while - * staying close to them: the per-pack defaults are meant to sum to no more - * than DEFAULT_CCTV_MAX_SOURCES. */ +/** Above the ~850 working cameras so the pack keeps them all on its own. The + * catalog-wide DEFAULT_CCTV_MAX_SOURCES cap still applies across packs and is + * filled round-robin, so a full default install may thin this pack. */ export const DEFAULT_VEGVESEN_MAX_SOURCES = 900; /** Hard ceiling on the catalog body. The whole country is ~800 KB. */ export const VEGVESEN_MAX_CATALOG_BYTES = 8 * 1024 * 1024; @@ -308,8 +308,8 @@ export const DELDOT_ANCHORS = [ /** WSDOT Travel Center cameras (Washington State): keyless Esri JSON on the * official open-data host — ~1,700 point features in EPSG:3857 (Web Mercator), - * ETag/Last-Modified served. WSDOT's metadata marks the feed "low volume" use, - * which the 15-minute catalog cache respects. */ + * ETag/Last-Modified served. WSDOT's metadata marks the feed "low volume" use; + * the catalog is fetched once per CCTV_SOURCE_CACHE_MS refresh. */ export const WSDOT_CAMERAS_URL = 'https://data.wsdot.wa.gov/travelcenter/Cameras.json'; /** Only frames on WSDOT's own image host are registered; the catalog also @@ -317,6 +317,10 @@ export const WSDOT_CAMERAS_URL = * whose own terms apply. */ export const WSDOT_IMAGE_ORIGIN = 'https://images.wsdot.wa.gov/'; export const DEFAULT_WSDOT_MAX_SOURCES = 250; +/** Hard ceiling on the WSDOT catalog body. The live layer is ~400 KB; this + * only exists so an upstream that streams an unbounded body cannot be + * buffered without limit. */ +export const WSDOT_MAX_CATALOG_BYTES = 4 * 1024 * 1024; /** Prioritization anchors: downtown Seattle and Spokane. */ export const WSDOT_ANCHORS = [ { lat: 47.6062, lon: -122.3321 }, // Seattle diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 04c12c502..56673014e 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -63,6 +63,7 @@ import { WSDOT_IMAGE_ORIGIN, DEFAULT_WSDOT_MAX_SOURCES, WSDOT_ANCHORS, + WSDOT_MAX_CATALOG_BYTES, DEFAULT_VEGVESEN_CCTV_URL, VEGVESEN_IMAGE_ORIGIN, VEGVESEN_VIDEO_URL, @@ -100,7 +101,10 @@ import { prioritizeSources, } from './normalize.js'; import { directionToHeading } from '../../../src/data/directionText.js'; -import { readResponseJsonCapped } from '../common/http.js'; +import { + readResponseBytesCapped, + readResponseJsonCapped, +} from '../common/http.js'; /** * Fetch and parse Austin traffic camera records from the city Open Data portal. * @@ -1721,10 +1725,10 @@ export async function loadDelDOTSourcesFromOpenData() { } } -/** WSDOT `CompassDirection` codes as headings in degrees, per the layer's own - * metadata (N/E/S/W, NE/NW/SE/SW, BW = Bothways). `BW` and anything else - * (`O`, null, unknown codes) carry no single facing and fall through to the - * id hash. */ +/** WSDOT `CompassDirection` codes as headings in degrees. The live feed sends + * single cardinals (N/E/S/W), `B` (both directions — most rows), `O`, and + * null; the diagonals are mapped in case they appear. `B`, `O`, null, and any + * other code carry no single facing and fall through to the id hash. */ const WSDOT_COMPASS_HEADINGS = Object.freeze({ N: 0, NE: 45, @@ -1770,9 +1774,9 @@ function isLikelyWashingtonCoordinate(lat, lon) { * finite in-state coordinates, and a frame on WSDOT's own image host are * kept — the catalog's ~70 partner cameras (Oregon DOT, City of Seattle, * lodges, airports) are skipped rather than proxied under someone else's - * terms. A single-facing `CompassDirection` (cardinal or diagonal) gives a - * high-confidence heading; `BW` (Bothways) and `O`/null fall back to the id - * hash. + * terms. A single-facing `CompassDirection` (a cardinal) gives a + * high-confidence heading; `B` (both directions), `O`, and null fall back to + * the id hash. * The feed carries Windows-1252 bytes (en-dashes in titles) with no charset * declaration, so the body is decoded explicitly before parsing. * @@ -1782,14 +1786,31 @@ export async function loadWsdotSourcesFromOpenData() { try { const resp = await fetch(WSDOT_CAMERAS_URL, { headers: { Accept: 'application/json' }, + redirect: 'manual', signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), }); + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn( + '[CCTV] WSDOT catalog redirected; redirects are not followed', + ); + return discard(); + } if (!resp.ok) { console.warn('[CCTV] WSDOT camera download failed:', resp.status); - return []; + return discard(); } const payload = JSON.parse( - new TextDecoder('windows-1252').decode(await resp.arrayBuffer()), + new TextDecoder('windows-1252').decode( + await readResponseBytesCapped(resp, WSDOT_MAX_CATALOG_BYTES), + ), ); const rows = Array.isArray(payload?.features) ? payload.features : []; diff --git a/src/data/cctvDriveBcSource.test.mjs b/src/data/cctvDriveBcSource.test.mjs index 617bb1350..c8e417e8b 100644 --- a/src/data/cctvDriveBcSource.test.mjs +++ b/src/data/cctvDriveBcSource.test.mjs @@ -221,7 +221,7 @@ test('CCTV catalog merges DriveBC cameras and CCTV_DRIVEBC_ENABLED=0 skips the r ); }); -test('default per-pack camera caps fit inside the default catalog cap', () => { +test('default caps of the original packs fit inside the default catalog cap', () => { // Every default pack at its own cap (Warendorf ships 1 curated camera). const packs = DEFAULT_AUSTIN_MAX_SOURCES + diff --git a/src/data/cctvWsdotSource.test.mjs b/src/data/cctvWsdotSource.test.mjs index 438df7e58..6b2ad3398 100644 --- a/src/data/cctvWsdotSource.test.mjs +++ b/src/data/cctvWsdotSource.test.mjs @@ -1,8 +1,13 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; import { DEFAULT_WSDOT_MAX_SOURCES, WSDOT_CAMERAS_URL, + WSDOT_MAX_CATALOG_BYTES, } from '../../server/providers/cctv/constants.js'; import { loadWsdotSourcesFromOpenData, @@ -89,7 +94,7 @@ test('WSDOT loader keeps official-host cameras and maps compass directions', asy }, }), wsdotFeature(9059, 47.66, -117.43, { - attributes: { CompassDirection: 'BW' }, + attributes: { CompassDirection: 'B' }, }), // Partner-hosted frame (Oregon DOT): skipped, never proxied. wsdotFeature(1002, 45.62, -122.675, { @@ -127,7 +132,7 @@ test('WSDOT loader keeps official-host cameras and maps compass directions', asy assert.ok(Math.abs(seattle.lat - 47.6097) < 0.001); assert.ok(Math.abs(seattle.lon - -122.3331) < 0.001); - // `BW` (Bothways) carries no single facing: id-hash fallback at low + // `B` (both directions) carries no single facing: id-hash fallback at low // confidence, with the low-confidence pose personality. const spokane = cameras.find((camera) => camera.id === 'wsdot-9059'); assert.equal(spokane.headingConfidence, 'low'); @@ -135,7 +140,7 @@ test('WSDOT loader keeps official-host cameras and maps compass directions', asy assert.ok(spokane.headingDeg >= 0 && spokane.headingDeg < 360); }); -test('WSDOT loader maps every documented CompassDirection code', async (t) => { +test('WSDOT loader maps every CompassDirection code', async (t) => { quiet(t); withEnv(t, { CCTV_WSDOT_MAX_SOURCES: undefined }); const codes = { @@ -150,7 +155,8 @@ test('WSDOT loader maps every documented CompassDirection code', async (t) => { // Lower-case and padded codes are the same code. ' nw ': 315, }; - const noFacing = ['BW', 'O', null]; + // The live feed's no-facing codes, plus an unknown one. + const noFacing = ['B', 'O', null, 'BW']; const entries = [...Object.keys(codes), ...noFacing]; t.mock.method(globalThis, 'fetch', async () => cp1252Response({ @@ -207,3 +213,90 @@ test('WSDOT loader defaults its cap and survives upstream failure', async (t) => }); assert.deepEqual(await loadWsdotSourcesFromOpenData(), []); }); + +test('the catalog fetch refuses a redirect and cancels its body', async (t) => { + quiet(t); + // A redirect is never followed, and its still-open body is cancelled. + const seen = []; + const state = { cancelled: false }; + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init.redirect]); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{')); + }, + cancel() { + state.cancelled = true; + }, + }); + return new Response(body, { + status: 302, + headers: { location: 'https://evil.example/Cameras.json' }, + }); + }); + assert.deepEqual(await loadWsdotSourcesFromOpenData(), []); + assert.deepEqual(seen, [[WSDOT_CAMERAS_URL, 'manual']]); + assert.equal(state.cancelled, true, 'the redirect body is cancelled'); +}); + +test('the catalog fetch refuses a body over the byte cap', async (t) => { + quiet(t); + // A valid catalog padded past the cap is refused rather than buffered. + t.mock.method(globalThis, 'fetch', async () => { + const text = JSON.stringify({ + features: [wsdotFeature(1001, 47.6097, -122.3331)], + }); + const padded = text + ' '.repeat(WSDOT_MAX_CATALOG_BYTES); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(padded)); + controller.close(); + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ); + }); + assert.deepEqual(await loadWsdotSourcesFromOpenData(), []); +}); + +test('CCTV_WSDOT_ENABLED=0 skips the WSDOT request', async (t) => { + quiet(t); + const sourceRoot = fs.mkdtempSync( + path.join(os.tmpdir(), 'gev-cctv-catalog-'), + ); + t.after(() => fs.rmSync(sourceRoot, { recursive: true, force: true })); + withEnv(t, { + CCTV_SOURCES_FILE: undefined, + CCTV_SOURCES_JSON: undefined, + CCTV_FORCE_AUSTIN: undefined, + CCTV_PREFER_AUSTIN: undefined, + CCTV_MAX_SOURCES: undefined, + CCTV_WSDOT_ENABLED: undefined, + CCTV_WSDOT_MAX_SOURCES: undefined, + }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + // Every other live pack is down; only WSDOT answers. + return String(url) === WSDOT_CAMERAS_URL + ? cp1252Response({ features: [wsdotFeature(1001, 47.6097, -122.3331)] }) + : new Response('unavailable', { status: 503 }); + }); + + const sources = await createCctvCatalog({ sourceRoot })(); + assert.deepEqual( + sources.map((source) => source.id), + ['wsdot-1001'], + ); + + process.env.CCTV_WSDOT_ENABLED = '0'; + requested.length = 0; + assert.deepEqual(await createCctvCatalog({ sourceRoot })(), []); + assert.ok(requested.length > 0, 'the other live packs still load'); + assert.equal( + requested.includes(WSDOT_CAMERAS_URL), + false, + 'a disabled pack makes no request', + ); +}); From 3f239f11730f7c837fccd932d82f398100162013 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 15:21:34 -0700 Subject: [PATCH 75/95] fix(cctv): give Catalonia cameras stable ids and pin their frame paths Ids were built from the feed's fid, which embeds an upstream server-start stamp, so every upstream restart renamed all cameras (it already had since the PR fixture). Ids now come from the validated frame URL (cat-sct-, or host and path for partner cameras), which also collapses the five duplicate rows. Frame links must sit directly in each host's frame directory with no explicit port, and sctidcam must be a bare image filename. Numeric entity decoding handles hex references and leaves invalid code points as written, so one bad name no longer empties the pack. Counts in the docs are now ~130. --- .env.example | 2 +- CHANGELOG.md | 7 +- DATA_SOURCES.md | 2 +- README.md | 2 +- server/providers/cctv/constants.js | 15 +- server/providers/cctv/normalize.js | 16 +- server/providers/cctv/sources.js | 56 ++++-- src/data/cctvCataloniaSource.test.mjs | 265 +++++++++++++++++++++++++- 8 files changed, 333 insertions(+), 32 deletions(-) diff --git a/.env.example b/.env.example index be3e14c15..62056f0c6 100644 --- a/.env.example +++ b/.env.example @@ -220,7 +220,7 @@ AISSTREAM_API_KEY= # CCTV_CALGARY_MAX_SOURCES=220 # cap cameras loaded from the Open Calgary catalog # CCTV_CALGARY_ROWS_URL= # override the Open Calgary catalog URL # CCTV_CATALONIA_ENABLED=1 # set 0 to disable the Servei Català de Trànsit cameras -# CCTV_CATALONIA_MAX_SOURCES=150 # cap cameras loaded from Catalonia (~135 exist today) +# CCTV_CATALONIA_MAX_SOURCES=150 # cap cameras loaded from Catalonia (~130 exist today) # CCTV_VEGVESEN_ENABLED=1 # set 0 to disable the Statens vegvesen (Norway) road cameras # CCTV_VEGVESEN_MAX_SOURCES=900 # cap cameras loaded from Statens vegvesen (default loads all ~850 live; max 1000) # CCTV_VEGVESEN_VIDEO=1 # set 0 to show stills instead of live HLS for Statens vegvesen video cameras diff --git a/CHANGELOG.md b/CHANGELOG.md index b132d0794..3a9332ca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -351,14 +351,17 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. -- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~135 +- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~130 cameras from a WFS/GML feed that aggregates SCT's own highway cameras with hotlinked Barcelona and Terrassa municipal cameras, each partner named as a credit beside the shared provider. Ambient-card refresh paced to 3 minutes, matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. The catalog is fetched over https with redirects refused and a 1 MiB body cap, and frame - links carrying userinfo are rejected even on an allowlisted host. + links carrying userinfo or an explicit port, or outside the publisher's frame + directory, are rejected even on an allowlisted host. Camera ids come from + each frame's SCT camera file or image path rather than the feed's + restart-dependent feature id, and rows the feed repeats collapse to one. - Import your own GeoJSON, KML, KMZ and GPX files from DISPLAY or by dropping them onto the app. Show, hide, fly to or remove each file for the session. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index ae6709501..694ed9c19 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -143,7 +143,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. -- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~135 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo is rejected even on an allowed host), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. The feed carries no camera status field, so a camera its publisher has taken offline still registers and shows the publisher's own "image unavailable" still (SCT and Barcelona both serve one with HTTP 200), the same as other packs without a status field. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_CATALONIA_ENABLED=0`. +- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~130 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo or an explicit port, or pointing outside that host's frame directory, is rejected even on an allowed host; an SCT `sctidcam` must be a bare `.gif`/`.jpg` filename), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Camera ids are built from the SCT camera file or the partner image path, not the feed's `fid` (which embeds an upstream server-start stamp and changes on every upstream restart), and rows the feed lists more than once collapse to one camera. The feed carries no camera status field, so a camera its publisher has taken offline still registers and shows the publisher's own "image unavailable" still (SCT and Barcelona both serve one with HTTP 200), the same as other packs without a status field. Frames are fetched at request time and never stored or redistributed. The feed itself states no licence; the pack relies on the Generalitat de Catalunya open-data portal's terms for this dataset (the licence and attribution above). Disable with `CCTV_CATALONIA_ENABLED=0`. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. diff --git a/README.md b/README.md index c50b195f7..3f7dcb69e 100644 --- a/README.md +++ b/README.md @@ -377,7 +377,7 @@ Twenty layers and map sources. **Eighteen have a keyless path.** Some offer addi | 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | | ◍ **Air quality** | Live AQHI on the published 1–10+ health-risk scale, with a Weather-panel card showing the band legend and the highest readings in view. **Canada only** — ECCC is the first registered provider | ECCC MSC GeoMet | 🟢 | | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Catalonia (Servei Català de Trànsit), plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📹 **CCTV Mesh** | Up to 4,000 public cameras (the shared catalog cap) projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Catalonia (Servei Català de Trànsit), plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | | 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary, Google Street View, KartaView and Panoramax | Mapillary · Google · KartaView · Panoramax | 🟢 KartaView, Panoramax · 🟡 Mapillary token · 🔴 Google Maps key | | 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 02687a33c..ae339d5e2 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -251,14 +251,19 @@ export const CATALONIA_CAMERAS_URL = /** Byte ceiling for the Catalonia catalog read (~90 KB observed 2026-09-27), * so a runaway body cannot be buffered without limit. */ export const CATALONIA_MAX_CATALOG_BYTES = 1024 * 1024; -/** Allowed image hosts, one per kept `font` publisher in the feed. */ +/** Allowed image hosts, one per kept `font` publisher in the feed, each + * mapped to the one directory its frame links live in (every link in the + * 2026-10 feed shares it); a link must name a single file directly inside. */ export const CATALONIA_IMAGE_HOSTS = Object.freeze( - new Set([ - 'mct.gencat.cat', // SCT - 'www.bcn.cat', // IMI (Ajuntament de Barcelona) - 'emap.terrassa.cat', // Ajuntament de Terrassa + new Map([ + ['mct.gencat.cat', '/mct2bo/'], // SCT + ['www.bcn.cat', '/transit/imatges/'], // IMI (Ajuntament de Barcelona) + ['emap.terrassa.cat', '/it_terrassa/'], // Ajuntament de Terrassa ]), ); +/** SCT `sctidcam` values: a bare image filename, which also keys the camera + * id (the feed's `fid` embeds an upstream server-start stamp). */ +export const CATALONIA_SCT_CAMERA_FILE_RE = /^[a-z0-9]+\.(gif|jpe?g)$/i; /** Per-`font` partner credit, shown beside the provider in the CCTV panel * (same pattern as DriveBC's partner-supplied cameras). SCT is the feed's * own publisher and carries no separate credit. */ diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index 0bec05f39..3af4f74a9 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -406,16 +406,24 @@ export function isLikelyCataloniaCoordinate(lat, lon) { } /** - * Decode decimal numeric HTML character references ("í" -> "í"), the - * entity form the Catalonia feed uses for accented place names, plus the - * handful of named entities XML/HTML text commonly carries. + * Decode numeric HTML character references, decimal ("í") and hex + * ("í") alike, the entity form the Catalonia feed uses for accented + * place names, plus the handful of named entities XML/HTML text commonly + * carries. A reference that names no valid character (NUL, a surrogate, or + * anything past U+10FFFF) is left as written rather than thrown on, so one + * bad field cannot fail a whole catalog. * * @param {string} text * @returns {string} */ export function decodeNumericEntities(text) { return String(text || '') - .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) + .replace(/&#(?:x([0-9a-f]+)|(\d+));/gi, (match, hex, dec) => { + const code = hex ? Number.parseInt(hex, 16) : Number(dec); + const valid = + code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff); + return valid ? String.fromCodePoint(code) : match; + }) .replace(/&/g, '&') .replace(/"/g, '"') .replace(/'/g, "'") diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 51cef87a0..7587f4cbe 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -58,6 +58,7 @@ import { CATALONIA_CAMERAS_URL, CATALONIA_MAX_CATALOG_BYTES, CATALONIA_IMAGE_HOSTS, + CATALONIA_SCT_CAMERA_FILE_RE, CATALONIA_FONT_CREDIT, CATALONIA_FONT_ELEVATION_M, DEFAULT_CATALONIA_MAX_SOURCES, @@ -1668,10 +1669,11 @@ export function parseCataloniaXml(xml) { /** * Validate and normalize one Catalonia feed `link` against the publisher * host allowlist (CATALONIA_IMAGE_HOSTS), upgrading to https. Rejects - * anything else, including a host that merely resembles an allowed one, and - * any URL carrying userinfo (`https://user:pass@www.bcn.cat/...`): the - * hostname check alone would pass it, letting catalog data inject - * credentials into the server-side frame request. + * anything else, including a host that merely resembles an allowed one, any + * URL carrying userinfo (`https://user:pass@www.bcn.cat/...`) or an explicit + * port — the hostname check alone would pass either, letting catalog data + * inject credentials or steer the server-side frame request to another + * service on that host — and any path outside the host's frame directory. * * SCT's own `RenderService` frame link is a special case, for two stacked * reasons (both verified 2026-09-15): @@ -1688,7 +1690,8 @@ export function parseCataloniaXml(xml) { * TransitCamera answers the frame directly over **http**, no redirect, and * that is the only transport this host actually serves to this runtime — so * this builds that URL itself from RenderService's `sctidcam` id instead of - * registering the (https, then-redirecting) RenderService link. + * registering the (https, then-redirecting) RenderService link. The + * `sctidcam` id must be a bare image filename (CATALONIA_SCT_CAMERA_FILE_RE). * * @param {string} rawUrl * @returns {string} Normalized URL, or '' if not accepted. @@ -1701,12 +1704,16 @@ export function normalizeCataloniaImageUrl(rawUrl) { return ''; } if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return ''; - if (parsed.username || parsed.password) return ''; - if (!CATALONIA_IMAGE_HOSTS.has(parsed.hostname)) return ''; + if (parsed.username || parsed.password || parsed.port) return ''; + const dir = CATALONIA_IMAGE_HOSTS.get(parsed.hostname); + if (!dir || !parsed.pathname.startsWith(dir)) return ''; + const file = parsed.pathname.slice(dir.length); + if (!file || file.includes('/')) return ''; if (parsed.hostname === 'mct.gencat.cat') { const camId = parsed.searchParams.get('sctidcam'); - if (!camId) return ''; + if (file !== 'RenderService') return ''; + if (!camId || !CATALONIA_SCT_CAMERA_FILE_RE.test(camId)) return ''; const direct = new URL('http://mct.gencat.cat/mct2bo/TransitCamera'); direct.searchParams.set('nom', camId); direct.searchParams.set('visualitzacio', 'imatge'); @@ -1717,6 +1724,30 @@ export function normalizeCataloniaImageUrl(rawUrl) { return parsed.toString(); } +/** + * Stable camera id for one normalized Catalonia frame URL (the output of + * normalizeCataloniaImageUrl). The feed's `fid` is not used: it embeds an + * upstream server-start stamp and changes on every upstream restart. SCT + * cameras key on the `sctidcam` filename stem (`cat-sct-nc87`), partner + * cameras on the frame host and path (`cat-www-bcn-cat-transit-imatges-…`). + * + * @param {string} imageUrl - A URL normalizeCataloniaImageUrl accepted. + * @returns {string} + */ +export function cataloniaCameraId(imageUrl) { + const parsed = new URL(imageUrl); + const slug = (text) => + text + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + if (parsed.hostname === 'mct.gencat.cat') { + const file = parsed.searchParams.get('nom') || ''; + return `cat-sct-${slug(file.replace(/\.[^.]*$/, ''))}`; + } + return `cat-${slug(parsed.hostname)}-${slug(parsed.pathname)}`; +} + /** * Human label for one Catalonia camera: " ()", * skipping the parenthetical when the municipality is already named in the @@ -1751,6 +1782,10 @@ export function cataloniaCameraName(carretera, municipi) { * shared `provider`. The feed carries no heading, so every camera takes the * id-hash fallback, low-confidence pose (same as TfL and Fintraffic). * + * Ids come from the validated frame URL (cataloniaCameraId), not the feed's + * restart-dependent `fid`, and rows sharing one are collapsed before the cap + * (the live feed repeats a few SCT cameras). + * * The list decides ids, coordinates, labels and which allowlisted frame path * is proxied, so it is fetched over https, redirects are refused (the list * host cannot be steered or downgraded) and the body read is capped at @@ -1791,15 +1826,12 @@ export async function loadCataloniaSourcesFromOpenData() { const cameras = []; for (const row of rows) { - if (!row.fid) continue; if (!isLikelyCataloniaCoordinate(row.lat, row.lon)) continue; const imageUrl = normalizeCataloniaImageUrl(row.link); if (!imageUrl) continue; const font = row.font.trim(); - const cameraId = `cat-${row.fid - .replace(/^cameres\.fid-/, '') - .replace(/[^a-zA-Z0-9]/g, '')}`; + const cameraId = cataloniaCameraId(imageUrl); cameras.push({ id: cameraId, diff --git a/src/data/cctvCataloniaSource.test.mjs b/src/data/cctvCataloniaSource.test.mjs index a2b1ed673..be472784f 100644 --- a/src/data/cctvCataloniaSource.test.mjs +++ b/src/data/cctvCataloniaSource.test.mjs @@ -8,6 +8,7 @@ import { CATALONIA_CAMERAS_URL, CATALONIA_MAX_CATALOG_BYTES, } from '../../server/providers/cctv/constants.js'; +import { decodeNumericEntities } from '../../server/providers/cctv/normalize.js'; import { parseCataloniaXml, normalizeCataloniaImageUrl, @@ -177,6 +178,62 @@ test('normalizeCataloniaImageUrl rejects credential-bearing URLs on every allowe } }); +test("normalizeCataloniaImageUrl rejects explicit ports and paths outside each host's frame directory", () => { + // An allowlisted hostname alone would let catalog data aim the frame + // request at another service or path on that host. + for (const link of [ + 'http://www.bcn.cat:8080/transit/imatges/a.gif', + 'https://emap.terrassa.cat:8443/it_terrassa/cam02.jpeg', + 'http://mct.gencat.cat:8080/mct2bo/RenderService?sctidcam=nc87.gif', + 'http://www.bcn.cat/admin/a.gif', + 'http://www.bcn.cat/transit/imatges/', + 'http://www.bcn.cat/transit/imatges/sub/a.gif', + 'http://www.bcn.cat/transit/imatges/../../admin/a.gif', + 'https://emap.terrassa.cat/other/cam02.jpeg', + 'http://mct.gencat.cat/mct2bo/OtherService?sctidcam=nc87.gif', + ]) { + assert.equal(normalizeCataloniaImageUrl(link), '', link); + } +}); + +test('normalizeCataloniaImageUrl accepts only a bare image filename as the SCT sctidcam', () => { + for (const camId of [ + '../../etc/passwd', + '..%2F..%2Fetc%2Fpasswd', + 'nc87.png', + 'nc87', + 'nc87.gif/../x.gif', + 'nc87.gif&nom=other.gif', + 'nc-87.gif', + ]) { + const link = `http://mct.gencat.cat/mct2bo/RenderService?sctidcam=${encodeURIComponent(camId)}`; + assert.equal(normalizeCataloniaImageUrl(link), '', camId); + } + assert.equal( + normalizeCataloniaImageUrl( + 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=CB10.JPEG', + ), + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=CB10.JPEG&visualitzacio=imatge', + ); +}); + +test('decodeNumericEntities decodes decimal and hex references and leaves invalid ones as written', () => { + assert.equal(decodeNumericEntities('Lluís'), 'Lluís'); + assert.equal(decodeNumericEntities('Lluís 🚦'), 'Lluís 🚦'); + assert.equal(decodeNumericEntities('Lluís'), 'Lluís'); + for (const raw of [ + 'a�b', + 'a�b', + 'a�b', + 'a�b', + 'a�b', + 'a�b', + 'a�b', + ]) { + assert.equal(decodeNumericEntities(raw), raw, raw); + } +}); + test('cataloniaCameraName joins road and municipality, skipping a redundant municipality', () => { assert.equal( cataloniaCameraName('C-58', 'Nus Trinitat'), @@ -261,11 +318,15 @@ test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publis ); assert.deepEqual( cameras.map((camera) => camera.id).sort(), - ['cat-1', 'cat-2', 'cat-3'], + [ + 'cat-emap-terrassa-cat-it-terrassa-cam02-jpeg', + 'cat-sct-nc87', + 'cat-www-bcn-cat-transit-imatges-plurquinaona-gif', + ], 'the off-allowlist link (fid-4) and the out-of-bbox row (fid-5) are excluded', ); - const sct = cameras.find((c) => c.id === 'cat-1'); + const sct = cameras.find((c) => c.id === 'cat-sct-nc87'); assert.equal(sct.name, 'C-58 (Nus Trinitat)'); assert.equal(sct.provider, 'Servei Català de Trànsit'); assert.equal(sct.credit, '', 'SCT is the publisher, no separate credit'); @@ -277,12 +338,16 @@ test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publis assert.equal(sct.groundElevationM, 80); assert.equal(sct.headingConfidence, 'low', 'the feed carries no heading'); - const bcn = cameras.find((c) => c.id === 'cat-2'); + const bcn = cameras.find( + (c) => c.id === 'cat-www-bcn-cat-transit-imatges-plurquinaona-gif', + ); assert.equal(bcn.credit, 'Ajuntament de Barcelona'); assert.equal(bcn.city, 'Barcelona'); assert.equal(bcn.groundElevationM, 20); - const terrassa = cameras.find((c) => c.id === 'cat-3'); + const terrassa = cameras.find( + (c) => c.id === 'cat-emap-terrassa-cat-it-terrassa-cam02-jpeg', + ); assert.equal(terrassa.credit, 'Ajuntament de Terrassa'); assert.equal(terrassa.groundElevationM, 270); }); @@ -415,7 +480,7 @@ test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips t const sources = await createCctvCatalog({ sourceRoot })(); assert.deepEqual( sources.map((source) => source.id), - ['cat-1'], + ['cat-sct-nc87'], ); process.env.CCTV_CATALONIA_ENABLED = '0'; @@ -423,8 +488,196 @@ test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips t assert.deepEqual(await createCctvCatalog({ sourceRoot })(), []); assert.ok(requested.length > 0, 'the other live packs still load'); assert.equal( - requested.includes(CATALONIA_CAMERAS_URL), + requested.some(([url]) => url === CATALONIA_CAMERAS_URL), false, 'a disabled pack makes no request', ); }); + +/** The two SCT, one Barcelona and one Terrassa rows of a real feed snapshot, + * under one upstream run's `fid` prefix. */ +function snapshotFeed(fidPrefix) { + return cataloniaFeed([ + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_386f`, + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_b48`, + lon: '2.18', + lat: '41.46', + carretera: 'C-31', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc85.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_1c2`, + lon: '2.17228212', + lat: '41.38890733', + carretera: 'Plaça Urquinaona', + municipi: 'Barcelona', + link: 'http://www.bcn.cat/transit/imatges/PlUrquinaona.gif?a=1', + font: 'IMI', + }), + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_1c3`, + lon: '2.0111', + lat: '41.5636', + municipi: 'Terrassa', + link: 'https://emap.terrassa.cat/it_terrassa/cam01.jpeg?a=1 ', + font: 'Terrassa', + }), + ]); +} + +async function loadFromFeed(t, xml) { + t.mock.method(globalThis, 'fetch', async () => { + return new Response(xml, { + status: 200, + headers: { 'Content-Type': 'application/xml' }, + }); + }); + try { + return await loadCataloniaSourcesFromOpenData(); + } finally { + t.mock.restoreAll(); + } +} + +test('Catalonia camera ids survive an upstream restart that changes every fid', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + // The fid embeds the upstream server-start stamp: the PR fixture's + // 1a0a06bea1c run and a later 1a140311953 run name the same cameras. + const before = await loadFromFeed(t, snapshotFeed('52bf6eca_1a0a06bea1c')); + const after = await loadFromFeed(t, snapshotFeed('52bf6eca_1a140311953')); + const ids = (cameras) => cameras.map((camera) => camera.id).sort(); + assert.deepEqual(ids(before), [ + 'cat-emap-terrassa-cat-it-terrassa-cam01-jpeg', + 'cat-sct-nc85', + 'cat-sct-nc87', + 'cat-www-bcn-cat-transit-imatges-plurquinaona-gif', + ]); + assert.deepEqual(ids(after), ids(before)); +}); + +test('Catalonia loader collapses rows the feed repeats for one camera', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + // The live feed lists nc85, cb5, cb10, cb25 and cd17 twice each, under + // different fids. + const row = (fid, camId) => + cataloniaFeature({ + fid, + lon: '2.18', + lat: '41.46', + carretera: 'C-31', + link: `http://mct.gencat.cat/mct2bo/RenderService?sctidcam=${camId}`, + font: 'SCT', + }); + const cameras = await loadFromFeed( + t, + cataloniaFeed([ + row('cameres.fid-x_1', 'nc85.gif'), + row('cameres.fid-x_2', 'cb5.gif'), + row('cameres.fid-x_3', 'nc85.gif'), + row('cameres.fid-x_4', 'cb5.gif'), + row('cameres.fid-x_5', 'cb10.gif'), + ]), + ); + assert.deepEqual(cameras.map((camera) => camera.id).sort(), [ + 'cat-sct-cb10', + 'cat-sct-cb5', + 'cat-sct-nc85', + ]); +}); + +test('Catalonia loader drops ported and bad-sctidcam links but keeps the rest', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + const cameras = await loadFromFeed( + t, + cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-x_1', + lon: '2.17', + lat: '41.39', + link: 'http://www.bcn.cat:8080/transit/imatges/PlUrquinaona.gif?a=1', + font: 'IMI', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_2', + lon: '2.0111', + lat: '41.5636', + link: 'https://emap.terrassa.cat:8443/it_terrassa/cam02.jpeg?a=1', + font: 'Terrassa', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_3', + lon: '2.18', + lat: '41.46', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=../../etc/passwd', + font: 'SCT', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_4', + lon: '2.18', + lat: '41.46', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]), + ); + assert.deepEqual( + cameras.map((camera) => [camera.id, camera.url]), + [ + [ + 'cat-sct-nc87', + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=nc87.gif&visualitzacio=imatge', + ], + ], + ); +}); + +test('a malformed numeric entity in one row neither throws nor empties the Catalonia pack', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + const cameras = await loadFromFeed( + t, + cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-x_1', + lon: '2.18', + lat: '41.46', + carretera: 'C-31 �', + municipi: 'Bad�', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc85.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_2', + lon: '2.18', + lat: '41.46', + carretera: 'Plaça Catalunya', + municipi: 'Llinars del Vallès', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]), + ); + const byId = new Map(cameras.map((camera) => [camera.id, camera])); + assert.equal(cameras.length, 2); + assert.equal( + byId.get('cat-sct-nc85').name, + 'C-31 � (Bad�)', + 'an invalid reference is kept as written', + ); + assert.equal( + byId.get('cat-sct-nc87').name, + 'Plaça Catalunya (Llinars del Vallès)', + ); +}); From 248a658713431dbd2ab8a9d2245270960a44af44 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 15:17:20 -0700 Subject: [PATCH 76/95] fix(cctv): serve only live feeds on /media /api/cctv/media relayed still-image cameras with an ordinary fetch: it followed redirects to any host, passed through any content type and had no body cap. Stills are served by /api/cctv/frame, which keeps redirects on the registered origin, accepts only image/* and caps the body, and no client requests a still through /media. /media now answers 404 for anything but HLS, MP4/WebM and MJPEG, before any upstream request. --- CHANGELOG.md | 10 ++ docs/CURRENT-STATE.md | 3 + server/providers/cctv.js | 17 +- src/data/cctvMediaFeedType.test.mjs | 270 ++++++++++++++++++++++++++++ 4 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 src/data/cctvMediaFeedType.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a9332ca6..8d7d4fdc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +- Refuse still-image cameras on the CCTV media relay. `/api/cctv/media` is the + live-feed route, but it also relayed any registered still camera through an + ordinary fetch that followed redirects to any host, passed through whatever + content type came back and had no cap on the body, bypassing every protection + `/api/cctv/frame` applies to the same URL. Only HLS, MP4, WebM and MJPEG + feeds are relayed now; a still camera, or an unknown id, gets a JSON 404 + without an upstream request and is served only through `/api/cctv/frame`. + The client already loaded stills through `/frame`, so nothing it requests + changes. + - Stop one retired bikeshare feed from reporting the whole layer as failed. City feeds are fetched independently, but failure was recorded in a single flag that only a successful *activation* cleared — and an already-loaded city diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 4774b5c80..d1b4c3a3b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1162,6 +1162,9 @@ still waiting to drain reschedules that deadline, so a slow client does not read as a stalled upstream. Error responses are cancelled. Buffered snapshots have a 16 MiB streaming cap; an oversized image remains an upstream miss and uses the normal fallback chain. The existing declared media size ceiling remains 64 MiB. +`/api/cctv/media` relays live feeds (HLS, MP4, WebM, MJPEG) only; a still-image +camera is refused there with a JSON 404 and is served solely through +`/api/cctv/frame` and its same-origin, `image/*`-only, size-capped fetch. ## GBFS upstream bounds diff --git a/server/providers/cctv.js b/server/providers/cctv.js index 9cf3b0c47..b4c8673f1 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -28,7 +28,7 @@ export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream }; * GET /api/cctv/sources — list all registered camera sources * GET /api/cctv/health — per-camera health/status report * GET /api/cctv/stream/:id — stream info (feedType, URLs) for a camera - * GET /api/cctv/media/:id — proxy live video/image media from upstream + * GET /api/cctv/media/:id — proxy live video media from upstream * GET /api/cctv/frame/:id — single frame with fallback chain * * @returns {import('vite').Plugin} @@ -214,6 +214,21 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { const source = sourceById.get(cameraId); const mediaUrl = source?.url || ''; const feedType = normalizeFeedType(source?.feedType || 'image'); + // Live feeds only. A still camera is served by /frame, whose fetch + // keeps redirects on the registered origin, accepts only image/* and + // caps the body; relaying it here would bypass all three. + if (!isVideoFeedType(feedType) && feedType !== 'mjpeg') { + res.writeHead(404, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); + res.end( + JSON.stringify({ + error: 'No live media for this camera; use the frame endpoint', + }), + ); + return; + } const leaseId = url.searchParams.get('lease'); if (feedType === 'hls' && !/^[a-f0-9-]{36}$/i.test(leaseId || '')) { res.writeHead(400); diff --git a/src/data/cctvMediaFeedType.test.mjs b/src/data/cctvMediaFeedType.test.mjs new file mode 100644 index 000000000..5791e433b --- /dev/null +++ b/src/data/cctvMediaFeedType.test.mjs @@ -0,0 +1,270 @@ +// /api/cctv/media is the live-feed relay. A still camera used to be relayed +// through it as well, by an ordinary redirect-following fetch with no content +// type check and no body cap, which side-stepped everything /api/cctv/frame +// enforces for the same URL. These pin that a still is refused there and only +// reachable through /frame, while live feeds keep their relay. +// +// Run with: npm test (node --test) +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { Readable } from 'node:stream'; +import { cctvProxy } from '../../server/providers/cctv.js'; + +const STILL = { + id: 'still-fixture', + name: 'Still fixture', + lat: 30.27, + lon: -97.74, + feedType: 'image', + url: 'https://cams.upstream.invalid/still.jpg', +}; +// 'jpg' normalizes to 'image'; the alias must not reopen the relay. +const STILL_ALIAS = { + ...STILL, + id: 'still-alias', + feedType: 'jpg', + url: 'https://cams.upstream.invalid/alias.jpg', +}; +const VIDEO = { + id: 'video-fixture', + name: 'Video fixture', + lat: 30.27, + lon: -97.74, + feedType: 'video', + url: 'https://cams.upstream.invalid/live.mp4', +}; +const MJPEG = { + id: 'mjpeg-fixture', + name: 'MJPEG fixture', + lat: 30.27, + lon: -97.74, + feedType: 'mjpeg', + url: 'https://cams.upstream.invalid/live.mjpg', +}; + +/** A minimal upstream Response with a streaming body. */ +function upstreamResponse({ status = 200, headers = {}, body = 'payload' }) { + const store = new Map( + Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]), + ); + return { + ok: status >= 200 && status < 300, + status, + headers: { get: (name) => store.get(String(name).toLowerCase()) ?? null }, + body: Readable.toWeb(Readable.from([Buffer.from(String(body))])), + }; +} + +/** Collects what a handler wrote, the way a Node ServerResponse would. */ +function recordingResponse() { + const chunks = []; + let finished; + const done = new Promise((resolve) => { + finished = resolve; + }); + const res = { + statusCode: 0, + headers: {}, + writableEnded: false, + body: '', + writeHead(status, headers) { + res.statusCode = status; + res.headers = headers || {}; + }, + write(chunk) { + chunks.push(Buffer.from(chunk)); + return true; + }, + end(chunk) { + if (chunk) chunks.push(Buffer.from(chunk)); + res.writableEnded = true; + res.body = Buffer.concat(chunks).toString('utf8'); + finished(); + }, + on() {}, + once() {}, + emit() {}, + destroy() {}, + }; + return { res, done }; +} + +/** + * Mount the real provider over the fixture cameras and answer every upstream + * request from the supplied function. + */ +function mount(t, respond) { + const before = { + json: process.env.CCTV_SOURCES_JSON, + file: process.env.CCTV_SOURCES_FILE, + austin: process.env.CCTV_FORCE_AUSTIN, + serverKey: process.env.GOOGLE_MAPS_SERVER_API_KEY, + browserKey: process.env.GOOGLE_MAPS_API_KEY, + }; + process.env.CCTV_SOURCES_JSON = JSON.stringify([ + STILL, + STILL_ALIAS, + VIDEO, + MJPEG, + ]); + process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; + process.env.CCTV_FORCE_AUSTIN = '0'; + // No Street View key: a /frame miss lands on the synthetic billboard rather + // than a network call this file does not model. + delete process.env.GOOGLE_MAPS_SERVER_API_KEY; + delete process.env.GOOGLE_MAPS_API_KEY; + const nativeFetch = globalThis.fetch; + const requests = []; + globalThis.fetch = async (url, init = {}) => { + requests.push({ url: String(url), redirect: init.redirect }); + return respond({ url: String(url), init }); + }; + t.after(() => { + globalThis.fetch = nativeFetch; + for (const [name, value] of [ + ['CCTV_SOURCES_JSON', before.json], + ['CCTV_SOURCES_FILE', before.file], + ['CCTV_FORCE_AUSTIN', before.austin], + ['GOOGLE_MAPS_SERVER_API_KEY', before.serverKey], + ['GOOGLE_MAPS_API_KEY', before.browserKey], + ]) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + let handler = null; + cctvProxy().configureServer({ + middlewares: { + use: (_route, fn) => { + handler = fn; + }, + }, + }); + assert.ok(handler, 'the CCTV provider must mount a middleware'); + + const call = async (path) => { + const { res, done } = recordingResponse(); + await handler({ url: path, headers: {}, method: 'GET', on() {} }, res); + await done; + return res; + }; + const health = async (id) => { + const res = await call('/health'); + return JSON.parse(res.body).cameras.find((entry) => entry.id === id); + }; + return { call, health, requests }; +} + +test('a still camera is refused by /media without any upstream request', async (t) => { + // Everything the old relay would have done with this camera: redirect to a + // private address, then answer with HTML. + const app = mount(t, ({ url }) => + url.includes('169.254.169.254') + ? upstreamResponse({ + headers: { 'content-type': 'text/html' }, + body: 'metadata', + }) + : upstreamResponse({ + status: 302, + headers: { location: 'http://169.254.169.254/latest/meta-data/' }, + body: '', + }), + ); + for (const camera of [STILL, STILL_ALIAS]) { + const res = await app.call(`/media/${camera.id}`); + assert.equal(res.statusCode, 404, camera.id); + assert.equal(res.headers['Content-Type'], 'application/json'); + assert.deepEqual(JSON.parse(res.body), { + error: 'No live media for this camera; use the frame endpoint', + }); + assert.doesNotMatch(res.body, /metadata/); + // Being asked on the wrong route is not a camera fault. + assert.equal(await app.health(camera.id), undefined); + } + assert.deepEqual(app.requests, [], 'a still camera reached its upstream'); +}); + +test('an unregistered camera id is refused by /media the same way', async (t) => { + const app = mount(t, () => { + throw new Error('an unknown camera must not fetch'); + }); + const res = await app.call('/media/not-a-camera'); + assert.equal(res.statusCode, 404); + assert.equal(app.requests.length, 0); +}); + +test('a video camera is still relayed by /media', async (t) => { + const app = mount(t, () => + upstreamResponse({ + headers: { 'content-type': 'video/mp4', 'content-length': '5' }, + body: 'video', + }), + ); + const res = await app.call(`/media/${VIDEO.id}`); + assert.equal(res.statusCode, 200); + assert.equal(res.headers['Content-Type'], 'video/mp4'); + assert.equal(res.headers['X-CCTV-Source'], 'live-media'); + assert.equal(res.body, 'video'); + assert.deepEqual( + app.requests.map(({ url }) => url), + [VIDEO.url], + ); + assert.equal((await app.health(VIDEO.id)).status, 'ok'); +}); + +test('an MJPEG camera is still relayed by /media', async (t) => { + const app = mount(t, () => + upstreamResponse({ + headers: { 'content-type': 'multipart/x-mixed-replace; boundary=frame' }, + body: '--frame', + }), + ); + const res = await app.call(`/media/${MJPEG.id}`); + assert.equal(res.statusCode, 200); + assert.match(res.headers['Content-Type'], /^multipart\/x-mixed-replace/); + assert.equal(res.body, '--frame'); + assert.deepEqual( + app.requests.map(({ url }) => url), + [MJPEG.url], + ); +}); + +test('a still camera is still served through /frame with its bounded fetch', async (t) => { + const app = mount(t, () => + upstreamResponse({ + headers: { 'content-type': 'image/jpeg' }, + body: 'jpeg-bytes', + }), + ); + const res = await app.call(`/frame/${STILL.id}`); + assert.equal(res.statusCode, 200); + assert.equal(res.headers['Content-Type'], 'image/jpeg'); + assert.equal(res.headers['X-CCTV-Source'], 'upstream-image'); + assert.equal(res.body, 'jpeg-bytes'); + assert.deepEqual(app.requests, [{ url: STILL.url, redirect: 'manual' }]); +}); + +test('/frame does not follow a still camera off its origin', async (t) => { + // The protection /media used to bypass, shown on the route that keeps it. + const app = mount(t, ({ url }) => + url === STILL.url + ? upstreamResponse({ + status: 302, + headers: { location: 'http://169.254.169.254/latest/meta-data/' }, + body: '', + }) + : upstreamResponse({ + headers: { 'content-type': 'image/jpeg' }, + body: 'private', + }), + ); + const res = await app.call(`/frame/${STILL.id}`); + assert.equal(res.statusCode, 200); + assert.equal(res.headers['X-CCTV-Source'], 'synthetic'); + assert.doesNotMatch(res.body, /private/); + assert.deepEqual( + app.requests.map(({ url }) => url), + [STILL.url], + ); +}); From 6d6ce364c684bffb01fb2af1ef18b4a29579d42a Mon Sep 17 00:00:00 2001 From: Mazeyar Moeini Feizabadi Date: Fri, 16 Oct 2026 02:03:16 +0200 Subject: [PATCH 77/95] docs(cyclones): document the early-advisory window and pin its bounds Maintainer follow-up for #741, no runtime change. - CURRENT-STATE: advisory issue and position times up to and including 90 minutes ahead of the server clock are accepted because NHC can release advisories before their nominal time; 90 minutes is the app's tolerance, not an NHC limit. Later, non-canonical and over-12-hour-old times still reject the snapshot. - Tests: pin exact 90-minute acceptance and +1 ms rejection, and exact 12-hour acceptance and +1 ms rejection, for issuance and lastUpdate independently; the early-publication test now asserts invalid_cyclone_data. Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 7 ++++++- src/data/cyclonesProxy.test.mjs | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index e5b3d2ace..0af075a6d 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -339,7 +339,12 @@ Cyclones combine fixed NHC/CPHC status and summary GIS endpoints through a bound same-origin provider. The status endpoint lacks browser CORS; each installation fetches keyless public sources at runtime, with no centralized ingestion service. The five-minute snapshot covers Atlantic and eastern/central North Pacific only. -Current positions, advisory issue time and position time stay distinct. Tracks, +Current positions, advisory issue time and position time stay distinct. NHC may +release an advisory before its nominal time (routinely up to about 30 minutes, +rarely about an hour), so advisory issue and position times up to and including +90 minutes ahead of the server clock are accepted; 90 minutes is this app's +tolerance, not an NHC limit. Later, non-canonical, or more than 12-hour-old +times still reject the status snapshot. Tracks, forecast lead-hour points and cones render only when all source parts match the status advisory; a newer status shows its position with geometry pending rather than relabeling older geometry. The cone represents forecast center uncertainty, diff --git a/src/data/cyclonesProxy.test.mjs b/src/data/cyclonesProxy.test.mjs index 31433b326..23e84826c 100644 --- a/src/data/cyclonesProxy.test.mjs +++ b/src/data/cyclonesProxy.test.mjs @@ -166,11 +166,40 @@ test('advisories published before their nominal time are accepted, far-future on // NHC wrote the 03:00Z advisory at 02:34Z. const early = Date.UTC(2026, 8, 16, 2, 34); assert.equal(parseCycloneStatus(status(), early)[0].issuedAt, TIME); - assert.throws(() => - parseCycloneStatus(status(), Date.UTC(2026, 8, 16, 1, 25)), + assert.throws( + () => parseCycloneStatus(status(), Date.UTC(2026, 8, 16, 1, 25)), + /invalid_cyclone_data/, ); }); +test('advisory and position times are bounded at 90 minutes ahead and 12 hours old', () => { + const MINUTE = 60_000; + const at = (offsetMs) => new Date(NOW + offsetMs).toISOString(); + const fields = { + issuedAt: (value) => + status({ + forecastAdvisory: { + advNum: '010', + issuance: value, + url: 'https://www.nhc.noaa.gov/text/MIATCMEP5.shtml', + }, + }), + positionAt: (value) => status({ lastUpdate: value }), + }; + for (const [field, payload] of Object.entries(fields)) { + for (const offset of [90 * MINUTE, -12 * 60 * MINUTE]) { + const [storm] = parseCycloneStatus(payload(at(offset)), NOW); + assert.equal(storm[field], at(offset), `${field} ${offset}`); + } + for (const offset of [90 * MINUTE + 1, -12 * 60 * MINUTE - 1]) + assert.throws( + () => parseCycloneStatus(payload(at(offset)), NOW), + /invalid_cyclone_data/, + `${field} ${offset}`, + ); + } +}); + test('coherent geometry is retained with holes/dateline while older advisory geometry is omitted', () => { const storms = parseCycloneStatus(status(), NOW); const parts = collections(); From c5f7fe285b340627c6e191857c5096d49c20482e Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Tue, 15 Sep 2026 23:43:44 -0400 Subject: [PATCH 78/95] feat(transit): add the TTC Toronto surface feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TTC BusTime publishes keyless GTFS-Realtime VehiclePositions with real coordinates for the surface network: ~1,180 buses and streetcars with bearing and per-vehicle timestamps. Registering it is one registry row. The subway is deliberately absent. BusTime covers surface vehicles only and the subway publishes no positions, so Lines 1/2/4 are left out rather than inferred from schedules — the same rule that keeps NYCT out. Route numbers establish the mode: the three-digit 500-series is streetcar, as are the four Blue Night routes on streetcar trackage. A leading 5 alone would misclassify bus routes 50-59, so the rule is anchored and tested. Licensing is the portal-wide Open Government Licence - Toronto; the CKAN record for the BusTime NVAS dataset leaves its own licence field blank, and the terms note says so rather than overstating the grant. Refs #70 --- DATA_SOURCES.md | 3 +- docs/CURRENT-STATE.md | 5 ++-- src/data/transitFeeds.js | 36 ++++++++++++++++++++++++ src/data/transitFeeds.test.mjs | 50 ++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 3 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 8f3b55dbf..463034e97 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -77,6 +77,7 @@ How to read this: | **Entur** (api.entur.io) | Transit layer live vehicles, Norway | Entur open realtime vehicle data, used for the live view; published for developer use and not against their use. The client sends the `ET-Client-Name` identifying header. | Attribution to Entur (courtesy text) | | **TransLink** (Queensland Government) | Transit layer live vehicles, South East Queensland | TransLink / Queensland Government open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to TransLink / Queensland Government (courtesy text) | | **HSL** (realtime.hsl.fi) | Transit layer live vehicles, Helsinki | HSL (Helsinki Region Transport) open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to HSL (Helsinki Region Transport) (courtesy text) | +| **TTC** (bustime.ttc.ca) | Transit layer live vehicles, Toronto — surface vehicles (buses and streetcars) only | Toronto Transit Commission BusTime open realtime vehicle data, published through City of Toronto Open Data and used for the live view. The subway publishes no vehicle positions, so Lines 1/2/4 are absent rather than inferred. | Attribution to Toronto Transit Commission — City of Toronto Open Data, under the Open Government Licence – Toronto | | **Radio Browser** | Geolocated internet-radio station directory and station-level tags | Public-domain directory data under PDDL 1.0; individual broadcaster stream terms apply | "Radio Browser" plus a link to the selected broadcaster | | **Re:Earth Terrain** (Mapterhorn) | Terrain (keyless globe stacks — OSM etc. — + `/api/terrain/heights` ellipsoidal-height lookups) | Terrain mesh: CC BY 4.0; geoid: EGM2008 (NGA, public domain) | "Terrain (keyless globe stacks): Re:Earth Terrain / Mapterhorn (CC BY 4.0) / EGM2008 (NGA)" | | **OSRM on the FOSSGIS routing servers** (`routing.openstreetmap.de`) | Street-following routes for the Directions layer and voice route annotations, via `/api/route` | [FOSSGIS routing usage policy](https://routing.openstreetmap.de/about.html): "Display the required attribution and display a link to 'fix the map'", "Use a valid user agent and, if applicable, a correct referrer", "One request per second max", "No scraping, no heavy usage". The full policy is the German [FOSSGIS Nutzungsbedingungen](https://www.fossgis.de/arbeitsgruppen/osm-server/nutzungsbedingungen/); FOSSGIS also states that the server may be embedded in your own pages but "eine gewerbliche Nutzung ist nur mit Einschränkungen erlaubt" (commercial use only with restrictions). Route data derives from OpenStreetMap (ODbL 1.0) | "Routing: OSRM on the FOSSGIS servers" + a "fix the map" link — shown in the Data attribution popover | @@ -155,7 +156,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. -- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. +- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All eight registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. - **TomTom Traffic.** Optional and BYOK: without `TOMTOM_API_KEY` the traffic layer runs its built-in simulation and no TomTom data (or attribution) appears. With a key, flow vector tiles are fetched through the server-side `/api/tomtom` proxy (120 s cache + a daily tile-budget governor — `TOMTOM_DAILY_TILE_BUDGET`, default 6,000 — sized so a 31-day month stays inside TomTom's 200K monthly free allowance; still a configurable application safety ceiling, not a guarantee; TomTom's [current pricing](https://docs.tomtom.com/pricing/) lists 200K free tile requests per month) and the "Traffic flow data © TomTom" credit is registered in the Data attribution popover the moment live mode activates. TomTom data is served live and cached only transiently (≤120 s TTL under `.gev-cache/`, gitignored) — it is not bundled or redistributed. One 23 KB point-in-time tile snapshot is committed as a decode-test fixture (`src/data/fixtures/`, © TomTom, never served to the app). - **Routing (OSRM / FOSSGIS).** `/api/route` proxies the public OSRM instances FOSSGIS e.V. runs for the OpenStreetMap community (car, bike, foot profiles). They are keyless and intended for fair, interactive use — the proxy rate-limits clients (60/min, 200/min total), caps route span and response size, caches each route for 10 minutes, and identifies itself in the User-Agent. Heavy or automated use must run its own OSRM. Routes are computed from OpenStreetMap data (ODbL); the Directions layer and voice routes never redistribute them. A route the router cannot find is reported as such; the Directions layer draws nothing in that case, and voice routes fall back to a clearly labeled straight line. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 0af075a6d..68530ffae 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3659,8 +3659,9 @@ Transit is off by default, and share links use token `j`. The movement panel ord places Transit directly after Street Traffic and before Bike Share. Cameras and utilities retain their own groups. The layer polls registered feeds near the camera every 15 seconds, below a 3,000 km altitude -gate. Seven keyless regions are available: Boston, Austin, Minneapolis–St Paul, -Helsinki, the Netherlands, Norway and South East Queensland. Mode silhouettes +gate. Eight keyless regions are available: Boston, Austin, Minneapolis–St Paul, +Helsinki, the Netherlands, Norway, South East Queensland and Toronto (TTC +surface vehicles; the subway publishes no positions). Mode silhouettes identify buses, trams, subways, trains, ferries and unclassified vehicles. Retained transit history uses the same outlier quarantine as live observations and diff --git a/src/data/transitFeeds.js b/src/data/transitFeeds.js index 2858efa62..10d087747 100644 --- a/src/data/transitFeeds.js +++ b/src/data/transitFeeds.js @@ -115,6 +115,21 @@ function metroTransitRouteMode(routeId) { return 'bus'; } +/** + * TTC BusTime is the SURFACE feed: buses and streetcars, never the subway. + * Streetcar routes are the 500-series, plus the four Blue Night routes that + * run on streetcar trackage (301 Queen, 304 King, 306 Carlton, 310 Spadina); + * the rest of the 300-series is night buses. + * @param {string|null} routeId + * @returns {string} + */ +function ttcRouteMode(routeId) { + if (!routeId) return 'unknown'; + if (/^5\d\d$/.test(routeId)) return 'tram'; + if (/^(301|304|306|310)$/.test(routeId)) return 'tram'; + return 'bus'; +} + /** * Registry of feeds. Order is presentation order in the stats/credit text. * `loadRadiusKm` is the distance from `center` inside which the feed is polled. @@ -270,6 +285,27 @@ export const TRANSIT_FEED_REGISTRY = Object.freeze([ }), defaultMode: 'bus', }), + Object.freeze({ + id: 'ttc-toronto', + name: 'TTC', + operator: 'Toronto Transit Commission', + region: 'Toronto, ON', + center: Object.freeze({ lat: 43.6532, lon: -79.3832 }), + loadRadiusKm: 45, + url: 'https://bustime.ttc.ca/gtfsrt/vehicles', + license: 'Open Government Licence \u2013 Toronto', + licenseUrl: 'https://open.toronto.ca/open-data-licence/', + attribution: + 'Toronto Transit Commission \u2014 City of Toronto Open Data (Open Government Licence \u2013 Toronto)', + defaultEnabled: true, + terms: Object.freeze({ + quote: + 'The Information Provider grants you a worldwide, royalty-free, perpetual, non-exclusive licence to use the Information, including for commercial purposes ... Acknowledge the source of the Information by including any attribution statement specified by the Information Provider.', + note: 'The City publishes this endpoint from the TTC BusTime NVAS catalogue entry, whose own licence field reads \"License not specified\"; the quote above is the portal-wide Open Government Licence \u2013 Toronto the City applies to its open data, so the credit carries the licence name the way the Metro Transit entry carries its public-domain note. BusTime is the surface system: buses and streetcars report real positions, the subway reports none, so Lines 1/2/4 are simply absent rather than inferred. The endpoint sends no ETag or Last-Modified, so conditional requests fall through to the proxy snapshot TTL.', + }), + defaultMode: 'bus', + routeMode: ttcRouteMode, + }), ]); /** diff --git a/src/data/transitFeeds.test.mjs b/src/data/transitFeeds.test.mjs index d0e38524f..1f364b924 100644 --- a/src/data/transitFeeds.test.mjs +++ b/src/data/transitFeeds.test.mjs @@ -239,3 +239,53 @@ test('published transit history scope agrees with the catalog capability', () => /four fixes|0\.95x|guaranteed lower|one poll interval behind/, ); }); + +test('the TTC feed is registered, routable, and covers Toronto', () => { + const ttc = getTransitFeed('ttc-toronto'); + assert.equal(ttc?.url, 'https://bustime.ttc.ca/gtfsrt/vehicles'); + assert.equal(ttc.defaultEnabled, true); + assert.match(ttc.attribution, /Toronto Transit Commission/); + assert.equal(ttc.historyRetention, undefined, 'no proxy retention'); + assert.ok( + transitFeedsInRange(43.6532, -79.3832).some( + (feed) => feed.id === 'ttc-toronto', + ), + 'downtown Toronto is inside the load radius', + ); +}); + +test('TTC route numbers tell a streetcar from a bus', () => { + // BusTime is the surface feed: 500-series are streetcar routes and the + // streetcar-operated Blue Night routes keep their own numbers. Everything + // else on this feed is a bus, so the mode is established, not defaulted. + const ttc = getTransitFeed('ttc-toronto'); + assert.equal(transitModeFor(ttc, '501'), 'tram'); + assert.equal(transitModeFor(ttc, '512'), 'tram'); + assert.equal(transitModeFor(ttc, '304'), 'tram'); + assert.equal(transitModeFor(ttc, '36'), 'bus'); + // 52 Lawrence West is a bus. Only the three-digit 500-series is streetcar, + // so the rule must not match on a leading 5 alone. + assert.equal(transitModeFor(ttc, '52'), 'bus'); + assert.equal(transitModeFor(ttc, '59'), 'bus'); + assert.equal(transitModeFor(ttc, '900'), 'bus'); + assert.equal(transitModeFor(ttc, '352'), 'bus'); + assert.equal(transitModeResolved(ttc, '501'), true); + assert.equal(transitModeResolved(ttc, null), false); + assert.equal( + transitModeFor(ttc, null), + 'bus', + 'an unrouted vehicle falls back to the feed default, unresolved', + ); +}); + +test('the TTC row records that BusTime carries no subway', () => { + const sources = readFileSync( + new URL('../../DATA_SOURCES.md', import.meta.url), + 'utf8', + ); + const row = sources.split('\n').find((line) => line.startsWith('| **TTC**')); + assert.ok(row, 'TTC has a DATA_SOURCES row'); + assert.match(row, /Toronto/); + assert.match(sources, /surface vehicles \(buses and streetcars\) only/); + assert.match(sources, /Open Government Licence – Toronto/); +}); From 9fc4a5ac04d2bcf0265d8ca6cdbd2b82cffe87f9 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Tue, 15 Sep 2026 23:58:51 -0400 Subject: [PATCH 79/95] feat(transit): let a feed name its own modes, and call TTC trams streetcars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Toronto 501 and a Helsinki 1010 are both the `tram` mode. Only one of them is called a tram by the people who ride it, and the detection label is documented as "what a person standing at the stop would call it". So the mode stays shared — same silhouette, same speed limits — and a registry entry may carry `modeWords` naming the modes it runs. Everything without an override falls through to the shared table, so this renames one word for one operator and nothing else. Threading the feed to the detection metric needed a resolver in queries.js, since the cached-contact path has only the entry. It answers from live feeds first, so a feed disabled mid-session stops supplying vocabulary. --- src/data/transitFeeds.js | 6 +++ src/layers/transit/policy.js | 46 +++++++++++++++++++++-- src/layers/transit/queries.js | 30 ++++++++++++--- src/layers/transit/vocabulary.test.mjs | 52 ++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 9 deletions(-) create mode 100644 src/layers/transit/vocabulary.test.mjs diff --git a/src/data/transitFeeds.js b/src/data/transitFeeds.js index 10d087747..5cbdac82d 100644 --- a/src/data/transitFeeds.js +++ b/src/data/transitFeeds.js @@ -304,6 +304,12 @@ export const TRANSIT_FEED_REGISTRY = Object.freeze([ note: 'The City publishes this endpoint from the TTC BusTime NVAS catalogue entry, whose own licence field reads \"License not specified\"; the quote above is the portal-wide Open Government Licence \u2013 Toronto the City applies to its open data, so the credit carries the licence name the way the Metro Transit entry carries its public-domain note. BusTime is the surface system: buses and streetcars report real positions, the subway reports none, so Lines 1/2/4 are simply absent rather than inferred. The endpoint sends no ETag or Last-Modified, so conditional requests fall through to the proxy snapshot TTL.', }), defaultMode: 'bus', + // Toronto rides streetcars. The vehicles are the shared `tram` mode and + // share its silhouette and speed limits; only the word a reader sees + // changes, because "tram" is not what anyone at Queen and Spadina says. + modeWords: Object.freeze({ + tram: Object.freeze({ word: 'Streetcar', abbr: 'STREETCAR' }), + }), routeMode: ttcRouteMode, }), ]); diff --git a/src/layers/transit/policy.js b/src/layers/transit/policy.js index b672581bb..33352bf2d 100644 --- a/src/layers/transit/policy.js +++ b/src/layers/transit/policy.js @@ -345,7 +345,7 @@ export function buildTransitSelectionCopy( // The mode is said in words on its own line. A glyph and a route id do not // tell a reader that the dot on Tremont Street is a subway train, and a // subway drawn on a street reads as a bug until the card says what it is. - const kindWord = TRANSIT_MODE_WORD[mode] || TRANSIT_MODE_WORD.unknown; + const kindWord = transitModeWord(mode, feed); const details = [`${kindWord} · ${feed.name} · ${feed.region}`]; const motion = []; // What the SCREEN is doing leads, because that is what the reader can check. @@ -479,6 +479,37 @@ export const TRANSIT_MODE_ABBR = Object.freeze({ unknown: 'TRANSIT', }); +/** + * The mode in the operator's own vocabulary. A Toronto 501 and a Helsinki 1010 + * are both the `tram` mode, but only one of them is called a tram by the people + * who ride it. A feed may name the modes it runs; everything else falls through + * to the shared table, so an override renames one word and nothing else. + * @param {string} mode Resolved transit mode. + * @param {object|null} [feed] Registry entry. + * @returns {string} + */ +export function transitModeWord(mode, feed = null) { + return ( + feed?.modeWords?.[mode]?.word || + TRANSIT_MODE_WORD[mode] || + TRANSIT_MODE_WORD.unknown + ); +} + +/** + * The operator's short mode word for the fixed-width detection fields. + * @param {string} mode Resolved transit mode. + * @param {object|null} [feed] Registry entry. + * @returns {string} + */ +export function transitModeAbbr(mode, feed = null) { + return ( + feed?.modeWords?.[mode]?.abbr || + TRANSIT_MODE_ABBR[mode] || + TRANSIT_MODE_ABBR.unknown + ); +} + /** * The bright first line of a vehicle's detection label: its route when the feed * names one, else the fleet number, else the raw id. What a person standing at @@ -521,7 +552,7 @@ export function elideRouteText(text) { * @returns {string} */ export function transitDetectionClass(mode, feed) { - const kind = TRANSIT_MODE_ABBR[mode] || TRANSIT_MODE_ABBR.unknown; + const kind = transitModeAbbr(mode, feed); const operator = typeof feed?.name === 'string' ? feed.name.toUpperCase() : ''; return operator ? `${kind} ${operator}`.slice(0, 20) : kind; @@ -538,10 +569,17 @@ export function transitDetectionClass(mode, feed) { * * @param {object} record Normalized vehicle record. * @param {string} mode Resolved transit mode. + * @param {number} [nowMs] + * @param {object|null} [feed] Registry entry, for its own mode vocabulary. * @returns {string} */ -export function transitDetectionMetric(entry, mode, nowMs = Date.now()) { - const kind = TRANSIT_MODE_ABBR[mode] || TRANSIT_MODE_ABBR.unknown; +export function transitDetectionMetric( + entry, + mode, + nowMs = Date.now(), + feed = null, +) { + const kind = transitModeAbbr(mode, feed); const state = transitVehicleState(entry, nowMs); return state ? `${kind} ${state}` : kind; } diff --git a/src/layers/transit/queries.js b/src/layers/transit/queries.js index 68367dc70..470006c85 100644 --- a/src/layers/transit/queries.js +++ b/src/layers/transit/queries.js @@ -12,6 +12,20 @@ import { } from '../../data/transitFeeds.js'; import { transitModeTier } from '../../data/transitPresetStyle.js'; +/** + * The registry entry behind a vehicle. Live feeds answer first so a feed the + * session disabled stops supplying vocabulary; the registry is the fallback, + * because a vehicle can outlive the poll that brought it. + * @param {object} state Layer state. + * @param {object} entry Vehicle entry. + * @returns {object|null} + */ +function feedForEntry(state, entry) { + const id = entry?.feedId; + if (!id) return null; + return state._activeFeeds.get(id) || getRegisteredTransitFeed(id) || null; +} + /** * The layer row's answer to "what is on screen and is it honest?". * @param {object} context @@ -101,6 +115,7 @@ export function createQueries({ state, parts }) { contact._entry, contact._entry.mode, now, + feedForEntry(state, contact._entry), ); return; } @@ -132,10 +147,10 @@ export function createQueries({ state, parts }) { entry.labelRecord = entry.record; } if (!entry.labelClass || entry.labelMode !== entry.mode) { - const feed = - state._activeFeeds.get(entry.feedId) || - getRegisteredTransitFeed(entry.feedId); - entry.labelClass = transitDetectionClass(entry.mode, feed); + entry.labelClass = transitDetectionClass( + entry.mode, + feedForEntry(state, entry), + ); entry.labelMode = entry.mode; } const contact = { @@ -146,7 +161,12 @@ export function createQueries({ state, parts }) { id: entry.labelId, type: 'VEH', klass: entry.labelClass, - metric: transitDetectionMetric(entry, entry.mode, now), + metric: transitDetectionMetric( + entry, + entry.mode, + now, + feedForEntry(state, entry), + ), // The bracket carries the mode colour in every preset: the detection // canvas sits above the post-FX chain, where the sprites' own colour // is lost under NVG and FLIR. Keyless too — the feeds are keyless. diff --git a/src/layers/transit/vocabulary.test.mjs b/src/layers/transit/vocabulary.test.mjs new file mode 100644 index 000000000..17441e770 --- /dev/null +++ b/src/layers/transit/vocabulary.test.mjs @@ -0,0 +1,52 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + buildTransitSelectionCopy, + transitDetectionClass, + transitDetectionMetric, + transitModeAbbr, + transitModeWord, +} from './policy.js'; +import { getTransitFeed } from '../../data/transitFeeds.js'; + +const ttc = () => getTransitFeed('ttc-toronto'); +const hsl = () => getTransitFeed('hsl-helsinki'); + +test('an operator that names its own vehicles overrides the shared mode word', () => { + // A Toronto 501 is a streetcar. A Helsinki 1010 is a tram. Both are the + // `tram` mode, so the word a reader sees has to come from the operator, + // not from the mode. + assert.equal(transitModeWord('tram', ttc()), 'Streetcar'); + assert.equal(transitModeAbbr('tram', ttc()), 'STREETCAR'); + assert.equal(transitModeWord('tram', hsl()), 'Tram'); + assert.equal(transitModeAbbr('tram', hsl()), 'TRAM'); +}); + +test('an override renames one mode, never the rest of the feed', () => { + assert.equal(transitModeWord('bus', ttc()), 'Bus'); + assert.equal(transitModeWord('subway', ttc()), 'Subway'); + assert.equal(transitModeAbbr('bus', ttc()), 'BUS'); +}); + +test('a missing feed or unknown mode falls back to the shared table', () => { + assert.equal(transitModeWord('tram', null), 'Tram'); + assert.equal(transitModeWord('spaceship', ttc()), 'Transit vehicle'); + assert.equal(transitModeAbbr('spaceship', ttc()), 'TRANSIT'); + assert.equal(transitModeWord('tram', { name: 'No override' }), 'Tram'); +}); + +test('the selection card calls a Toronto 501 a streetcar', () => { + const record = { id: '4400', routeId: '501', timestamp: 0 }; + const copy = buildTransitSelectionCopy(ttc(), record, 'tram', 0, null, null); + assert.match(copy.details.join(' | '), /Streetcar · TTC/); + assert.doesNotMatch(copy.details.join(' | '), /Tram/); +}); + +test('detection labels carry the operator vocabulary within the field width', () => { + const klass = transitDetectionClass('tram', ttc()); + assert.equal(klass, 'STREETCAR TTC'); + assert.ok(klass.length <= 20, 'fits the detection class field'); + assert.equal(transitDetectionClass('tram', hsl()), 'TRAM HSL'); + const metric = transitDetectionMetric({ mode: 'tram' }, 'tram', 0, ttc()); + assert.match(metric, /^STREETCAR/); +}); From 849cb803753afd672dcc0aa0d1acdb9c55534c12 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 00:04:00 -0400 Subject: [PATCH 80/95] feat(transit): add MiWay, Hamilton HSR and Durham Region Transit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three keyless GTFS-Realtime feeds around Toronto, each verified live: all vehicles carry real coordinates, and each is polled only near its own region rather than across the Golden Horseshoe. The licences differ in ways the registry now records rather than flattens: Hamilton PRESCRIBES its attribution wording, so the credit is that exact sentence; Durham makes credit optional and its catalogued licence PDF is dead after a site relaunch, so licenseUrl points at the live page where the v.1.0 text was actually read; Mississauga's licence is revocable and asks that the terms travel with the data, so licenseUrl is the terms themselves. All three are bus-only and set no route hint, so their modes default rather than resolve. Durham publishes no bearing on any vehicle — heading comes from observed movement, as with any feed that omits it. GO Transit / Metrolinx stays out: its feed requires an API key. --- DATA_SOURCES.md | 5 ++- docs/CURRENT-STATE.md | 7 ++-- src/data/transitFeeds.js | 61 ++++++++++++++++++++++++++++++++++ src/data/transitFeeds.test.mjs | 57 +++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 4 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 463034e97..d398cd4f6 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -78,6 +78,9 @@ How to read this: | **TransLink** (Queensland Government) | Transit layer live vehicles, South East Queensland | TransLink / Queensland Government open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to TransLink / Queensland Government (courtesy text) | | **HSL** (realtime.hsl.fi) | Transit layer live vehicles, Helsinki | HSL (Helsinki Region Transport) open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to HSL (Helsinki Region Transport) (courtesy text) | | **TTC** (bustime.ttc.ca) | Transit layer live vehicles, Toronto — surface vehicles (buses and streetcars) only | Toronto Transit Commission BusTime open realtime vehicle data, published through City of Toronto Open Data and used for the live view. The subway publishes no vehicle positions, so Lines 1/2/4 are absent rather than inferred. | Attribution to Toronto Transit Commission — City of Toronto Open Data, under the Open Government Licence – Toronto | +| **MiWay** (miapp.ca) | Transit layer live vehicles, Mississauga ON | MiWay open realtime vehicle data, published by the City of Mississauga under its Open Data Terms of Use, used for the live view. The licence is revocable and asks that the Terms of Use link travel with the data. | Attribution to MiWay — City of Mississauga Open Data, linked to the Terms of Use | +| **Hamilton HSR** (opendata.hamilton.ca) | Transit layer live vehicles, Hamilton ON | Hamilton Street Railway open realtime vehicle data, published by the City of Hamilton for commercial and non-commercial use. | "Contains public sector Data made available under the City of Hamilton’s Open Data Licence" — wording prescribed by the licence | +| **Durham Region Transit** (drtonline.durhamregiontransit.com) | Transit layer live vehicles, Durham Region ON | Durham Region Transit open realtime vehicle data, published under the Region of Durham Open Data Licence v.1.0, which permits commercial use. | "Contains public sector information made available under The Regional Municipality of Durham's Open Data Licence" | | **Radio Browser** | Geolocated internet-radio station directory and station-level tags | Public-domain directory data under PDDL 1.0; individual broadcaster stream terms apply | "Radio Browser" plus a link to the selected broadcaster | | **Re:Earth Terrain** (Mapterhorn) | Terrain (keyless globe stacks — OSM etc. — + `/api/terrain/heights` ellipsoidal-height lookups) | Terrain mesh: CC BY 4.0; geoid: EGM2008 (NGA, public domain) | "Terrain (keyless globe stacks): Re:Earth Terrain / Mapterhorn (CC BY 4.0) / EGM2008 (NGA)" | | **OSRM on the FOSSGIS routing servers** (`routing.openstreetmap.de`) | Street-following routes for the Directions layer and voice route annotations, via `/api/route` | [FOSSGIS routing usage policy](https://routing.openstreetmap.de/about.html): "Display the required attribution and display a link to 'fix the map'", "Use a valid user agent and, if applicable, a correct referrer", "One request per second max", "No scraping, no heavy usage". The full policy is the German [FOSSGIS Nutzungsbedingungen](https://www.fossgis.de/arbeitsgruppen/osm-server/nutzungsbedingungen/); FOSSGIS also states that the server may be embedded in your own pages but "eine gewerbliche Nutzung ist nur mit Einschränkungen erlaubt" (commercial use only with restrictions). Route data derives from OpenStreetMap (ODbL 1.0) | "Routing: OSRM on the FOSSGIS servers" + a "fix the map" link — shown in the Data attribution popover | @@ -156,7 +159,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. -- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All eight registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. +- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All eleven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. GO Transit / Metrolinx is deliberately absent: its realtime feed requires an API key, which the keyless admission rule excludes. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. - **TomTom Traffic.** Optional and BYOK: without `TOMTOM_API_KEY` the traffic layer runs its built-in simulation and no TomTom data (or attribution) appears. With a key, flow vector tiles are fetched through the server-side `/api/tomtom` proxy (120 s cache + a daily tile-budget governor — `TOMTOM_DAILY_TILE_BUDGET`, default 6,000 — sized so a 31-day month stays inside TomTom's 200K monthly free allowance; still a configurable application safety ceiling, not a guarantee; TomTom's [current pricing](https://docs.tomtom.com/pricing/) lists 200K free tile requests per month) and the "Traffic flow data © TomTom" credit is registered in the Data attribution popover the moment live mode activates. TomTom data is served live and cached only transiently (≤120 s TTL under `.gev-cache/`, gitignored) — it is not bundled or redistributed. One 23 KB point-in-time tile snapshot is committed as a decode-test fixture (`src/data/fixtures/`, © TomTom, never served to the app). - **Routing (OSRM / FOSSGIS).** `/api/route` proxies the public OSRM instances FOSSGIS e.V. runs for the OpenStreetMap community (car, bike, foot profiles). They are keyless and intended for fair, interactive use — the proxy rate-limits clients (60/min, 200/min total), caps route span and response size, caches each route for 10 minutes, and identifies itself in the User-Agent. Heavy or automated use must run its own OSRM. Routes are computed from OpenStreetMap data (ODbL); the Directions layer and voice routes never redistribute them. A route the router cannot find is reported as such; the Directions layer draws nothing in that case, and voice routes fall back to a clearly labeled straight line. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 68530ffae..06b026cf6 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3659,9 +3659,10 @@ Transit is off by default, and share links use token `j`. The movement panel ord places Transit directly after Street Traffic and before Bike Share. Cameras and utilities retain their own groups. The layer polls registered feeds near the camera every 15 seconds, below a 3,000 km altitude -gate. Eight keyless regions are available: Boston, Austin, Minneapolis–St Paul, -Helsinki, the Netherlands, Norway, South East Queensland and Toronto (TTC -surface vehicles; the subway publishes no positions). Mode silhouettes +gate. Eleven keyless regions are available: Boston, Austin, Minneapolis–St Paul, +Helsinki, the Netherlands, Norway, South East Queensland, Toronto (TTC +surface vehicles; the subway publishes no positions), and the Golden +Horseshoe neighbours MiWay, Hamilton HSR and Durham Region Transit. Mode silhouettes identify buses, trams, subways, trains, ferries and unclassified vehicles. Retained transit history uses the same outlier quarantine as live observations and diff --git a/src/data/transitFeeds.js b/src/data/transitFeeds.js index 5cbdac82d..c6c0f5147 100644 --- a/src/data/transitFeeds.js +++ b/src/data/transitFeeds.js @@ -312,6 +312,67 @@ export const TRANSIT_FEED_REGISTRY = Object.freeze([ }), routeMode: ttcRouteMode, }), + Object.freeze({ + id: 'miway-mississauga', + name: 'MiWay', + operator: 'MiWay (City of Mississauga)', + region: 'Mississauga, ON', + center: Object.freeze({ lat: 43.589, lon: -79.6441 }), + loadRadiusKm: 30, + url: 'https://www.miapp.ca/GTFS_RT/Vehicle/VehiclePositions.pb', + license: 'City of Mississauga Open Data Terms of Use', + licenseUrl: + 'http://www5.mississauga.ca/research_catalogue/CityofMississauga_TermsofUse.pdf', + attribution: 'MiWay \u2014 City of Mississauga Open Data', + defaultEnabled: true, + terms: Object.freeze({ + quote: + 'The City of Mississauga (the City) now grants you a world-wide, royalty-free, non-exclusive, revocable licence to use, modify, and distribute the Datasets in all current and future media and formats for any lawful purpose.', + note: 'Two conditions shape how this ships. The licence is REVOCABLE, and it asks that anyone who provides access to the data pass on the Terms of Use URL \u2014 so `licenseUrl` points at the terms themselves rather than at a catalogue page, and the attribution popover carries the link. Credit is optional here ("you are not required to credit the City"), and is given anyway as a courtesy, the way the other municipal feeds are.', + }), + defaultMode: 'bus', + }), + Object.freeze({ + id: 'hsr-hamilton', + name: 'Hamilton HSR', + operator: 'Hamilton Street Railway (City of Hamilton)', + region: 'Hamilton, ON', + center: Object.freeze({ lat: 43.2557, lon: -79.8711 }), + loadRadiusKm: 30, + url: 'https://opendata.hamilton.ca/GTFS-RT/GTFS_VehiclePositions.pb', + license: 'City of Hamilton Open Data Licence', + licenseUrl: + 'https://www.hamilton.ca/city-initiatives/strategies-actions/open-data-licence-terms-and-conditions', + attribution: + 'Contains public sector Data made available under the City of Hamilton\u2019s Open Data Licence', + defaultEnabled: true, + terms: Object.freeze({ + quote: + 'The City of Hamilton grants you a worldwide, royalty-free, perpetual, non-exclusive licence to use the Data, including for commercial purposes ... you must acknowledge the source of the Data by including the following attribution statement: \u201cContains public sector Data made available under the City of Hamilton\u2019s Open Data Licence\u201d.', + note: 'The attribution wording is prescribed by the licence, so it is used verbatim as the credit rather than paraphrased to an operator name. Names, crests, logos and marks are excluded from the grant, so the credit stays text.', + }), + defaultMode: 'bus', + }), + Object.freeze({ + id: 'durham-region', + name: 'Durham Region Transit', + operator: 'Durham Region Transit (Regional Municipality of Durham)', + region: 'Durham Region, ON', + center: Object.freeze({ lat: 43.9496, lon: -78.9365 }), + loadRadiusKm: 45, + url: 'https://drtonline.durhamregiontransit.com/gtfsrealtime/VehiclePositions', + license: 'Region of Durham Open Data Licence v.1.0', + licenseUrl: 'https://www.durham.ca/en/regional-government/open-data.aspx', + attribution: + "Contains public sector information made available under The Regional Municipality of Durham's Open Data Licence", + defaultEnabled: true, + terms: Object.freeze({ + quote: + 'Durham Region grants you a worldwide, royalty-free, perpetual, non-exclusive licence to use the information ... You are free to: Copy, publish, distribute and transmit the information ... Use the information commercially.', + note: 'The licence link the open-data catalogue publishes for this feed (OpenDataLicenceAgreement.pdf) is dead after a site relaunch; `licenseUrl` points at the live Open Data page carrying the same licence v.1.0 text, which is where the quote above was read. Credit is optional and the suggested statement is used verbatim.', + }), + defaultMode: 'bus', + }), ]); /** diff --git a/src/data/transitFeeds.test.mjs b/src/data/transitFeeds.test.mjs index 1f364b924..b1c5d5bf7 100644 --- a/src/data/transitFeeds.test.mjs +++ b/src/data/transitFeeds.test.mjs @@ -289,3 +289,60 @@ test('the TTC row records that BusTime carries no subway', () => { assert.match(sources, /surface vehicles \(buses and streetcars\) only/); assert.match(sources, /Open Government Licence – Toronto/); }); + +test('the Golden Horseshoe neighbours are registered, keyless and routable', () => { + const expected = { + 'miway-mississauga': + 'https://www.miapp.ca/GTFS_RT/Vehicle/VehiclePositions.pb', + 'hsr-hamilton': + 'https://opendata.hamilton.ca/GTFS-RT/GTFS_VehiclePositions.pb', + 'durham-region': + 'https://drtonline.durhamregiontransit.com/gtfsrealtime/VehiclePositions', + }; + for (const [id, url] of Object.entries(expected)) { + const feed = getTransitFeed(id); + assert.ok(feed, `${id} is routable`); + assert.equal(feed.url, url); + assert.equal(feed.defaultEnabled, true); + assert.equal(feed.defaultMode, 'bus'); + assert.equal(feed.historyRetention, undefined, `${id} retains nothing`); + assert.ok(feed.terms?.quote, `${id} records the terms it ships on`); + } +}); + +test('each Golden Horseshoe feed covers its own city and not its neighbours', () => { + const near = (lat, lon) => transitFeedsInRange(lat, lon).map((f) => f.id); + // Mississauga City Centre, downtown Hamilton, Oshawa. + assert.ok(near(43.589, -79.6441).includes('miway-mississauga')); + assert.ok(near(43.2557, -79.8711).includes('hsr-hamilton')); + assert.ok(near(43.8971, -78.8658).includes('durham-region')); + // Hamilton is 60 km from Toronto: far enough that the TTC feed is not polled + // for it, which is the whole point of a per-feed radius. + assert.equal(near(43.2557, -79.8711).includes('ttc-toronto'), false); +}); + +test('a bus-only feed never claims a mode its route ids cannot establish', () => { + for (const id of ['miway-mississauga', 'hsr-hamilton', 'durham-region']) { + const feed = getTransitFeed(id); + assert.equal(transitModeFor(feed, '1'), 'bus'); + assert.equal( + transitModeResolved(feed, '1'), + false, + `${id} defaults rather than resolves`, + ); + } +}); + +test('every Golden Horseshoe feed is documented with its licence', () => { + const sources = readFileSync( + new URL('../../DATA_SOURCES.md', import.meta.url), + 'utf8', + ); + for (const name of ['MiWay', 'Hamilton HSR', 'Durham Region Transit']) { + assert.ok( + sources.split('\n').some((line) => line.startsWith(`| **${name}`)), + `${name} has a DATA_SOURCES row`, + ); + } + assert.match(sources, /Durham Region Transit publishes no bearing/); +}); From 11dd914c2f731442139cf1540a33f7f3c388bd17 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 00:31:54 -0400 Subject: [PATCH 81/95] feat(transit): add Guelph and Kingston, register Barrie off MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Guelph's feed publishes the internal database key of a route (`2991`) where the rider sees the number on the bus (`1`), which would have put "Route 2991" on the label a person reads. So a feed may now carry its own `routeLabel`, the same way it may carry its own `modeWords`. Guelph's table is its 33 routes, taken from the City's own published GTFS; an id missing from it renders unchanged, because a raw id you can compare against the bus beats a blank. Every other feed is identity and carries no table. Kingston is catalogued by the City against this exact URL. Its licence PDF link is pinned to the canonical location rather than the legacy path that currently redirects there — Durham's entry records what happens when such a redirect is eventually dropped. Barrie ships REGISTERED AND OFF. The City offers its licence for acceptance behind a click-through gate, and nobody on this project has accepted it. Whether to accept is a person's decision, not an assumption a poller should make, so the feed is visible, documented, and one line from enabled. --- DATA_SOURCES.md | 5 +- docs/CURRENT-STATE.md | 8 +- src/data/transitFeeds.js | 133 +++++++++++++++++++++++++ src/data/transitFeeds.test.mjs | 44 ++++++++ src/layers/transit/policy.js | 13 ++- src/layers/transit/queries.js | 5 +- src/layers/transit/vocabulary.test.mjs | 24 +++++ 7 files changed, 223 insertions(+), 9 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index d398cd4f6..0c245c828 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -81,6 +81,9 @@ How to read this: | **MiWay** (miapp.ca) | Transit layer live vehicles, Mississauga ON | MiWay open realtime vehicle data, published by the City of Mississauga under its Open Data Terms of Use, used for the live view. The licence is revocable and asks that the Terms of Use link travel with the data. | Attribution to MiWay — City of Mississauga Open Data, linked to the Terms of Use | | **Hamilton HSR** (opendata.hamilton.ca) | Transit layer live vehicles, Hamilton ON | Hamilton Street Railway open realtime vehicle data, published by the City of Hamilton for commercial and non-commercial use. | "Contains public sector Data made available under the City of Hamilton’s Open Data Licence" — wording prescribed by the licence | | **Durham Region Transit** (drtonline.durhamregiontransit.com) | Transit layer live vehicles, Durham Region ON | Durham Region Transit open realtime vehicle data, published under the Region of Durham Open Data Licence v.1.0, which permits commercial use. | "Contains public sector information made available under The Regional Municipality of Durham's Open Data Licence" | +| **Guelph Transit** (glphtrpcloud.com) | Transit layer live vehicles, Guelph ON | Guelph Transit open realtime vehicle data, published by the City of Guelph on its open-data developer page and used for the live view. Its route ids are internal database keys, so the 33-row route-number table in `src/data/transitFeeds.js` is derived from the City’s own published GTFS `routes.txt` under the same licence. | "Contains information licensed under the Open Government Licence – City of Guelph" — wording prescribed by the licence | +| **Kingston Transit** (api.cityofkingston.ca) | Transit layer live vehicles, Kingston ON | Kingston Transit open realtime vehicle data, catalogued by the City of Kingston as open data and used for the live view. | "Contains information licensed under the Open Data Licence – City of Kingston" — wording prescribed by the licence | +| **Barrie Transit** (myridebarrie.ca) | REGISTERED BUT OFF — not polled, not offered to the browser | Barrie Transit realtime data, published by the City of Barrie behind a click-through acceptance of its licence. Registered so the decision is visible; it ships unpolled until a person accepts those terms. The licence is revocable and requires no attribution. | Courtesy credit to the City of Barrie (the licence requires none, and its trademark clause is broad) | | **Radio Browser** | Geolocated internet-radio station directory and station-level tags | Public-domain directory data under PDDL 1.0; individual broadcaster stream terms apply | "Radio Browser" plus a link to the selected broadcaster | | **Re:Earth Terrain** (Mapterhorn) | Terrain (keyless globe stacks — OSM etc. — + `/api/terrain/heights` ellipsoidal-height lookups) | Terrain mesh: CC BY 4.0; geoid: EGM2008 (NGA, public domain) | "Terrain (keyless globe stacks): Re:Earth Terrain / Mapterhorn (CC BY 4.0) / EGM2008 (NGA)" | | **OSRM on the FOSSGIS routing servers** (`routing.openstreetmap.de`) | Street-following routes for the Directions layer and voice route annotations, via `/api/route` | [FOSSGIS routing usage policy](https://routing.openstreetmap.de/about.html): "Display the required attribution and display a link to 'fix the map'", "Use a valid user agent and, if applicable, a correct referrer", "One request per second max", "No scraping, no heavy usage". The full policy is the German [FOSSGIS Nutzungsbedingungen](https://www.fossgis.de/arbeitsgruppen/osm-server/nutzungsbedingungen/); FOSSGIS also states that the server may be embedded in your own pages but "eine gewerbliche Nutzung ist nur mit Einschränkungen erlaubt" (commercial use only with restrictions). Route data derives from OpenStreetMap (ODbL 1.0) | "Routing: OSRM on the FOSSGIS servers" + a "fix the map" link — shown in the Data attribution popover | @@ -159,7 +162,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. -- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All eleven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. GO Transit / Metrolinx is deliberately absent: its realtime feed requires an API key, which the keyless admission rule excludes. +- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. Thirteen of the fourteen registered regions are live-enabled (Barrie is registered and deliberately off); each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. Guelph, Kingston and Barrie extend the same pattern eastward and north. Guelph is the one feed whose route ids are internal keys rather than the number on the bus, so its entry carries a route-number table taken from the City’s own published GTFS; any id missing from that table renders unchanged rather than blank. GO Transit / Metrolinx is deliberately absent: its REALTIME feed requires an API key issued by manual review, which the keyless admission rule excludes (its static schedule download is keyless, but carries no vehicle positions). Grand River Transit in Waterloo is absent for a different reason: it answers, but its feed header was months stale and it publishes no per-vehicle timestamps, so it would render vehicles that look live and are not. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. - **TomTom Traffic.** Optional and BYOK: without `TOMTOM_API_KEY` the traffic layer runs its built-in simulation and no TomTom data (or attribution) appears. With a key, flow vector tiles are fetched through the server-side `/api/tomtom` proxy (120 s cache + a daily tile-budget governor — `TOMTOM_DAILY_TILE_BUDGET`, default 6,000 — sized so a 31-day month stays inside TomTom's 200K monthly free allowance; still a configurable application safety ceiling, not a guarantee; TomTom's [current pricing](https://docs.tomtom.com/pricing/) lists 200K free tile requests per month) and the "Traffic flow data © TomTom" credit is registered in the Data attribution popover the moment live mode activates. TomTom data is served live and cached only transiently (≤120 s TTL under `.gev-cache/`, gitignored) — it is not bundled or redistributed. One 23 KB point-in-time tile snapshot is committed as a decode-test fixture (`src/data/fixtures/`, © TomTom, never served to the app). - **Routing (OSRM / FOSSGIS).** `/api/route` proxies the public OSRM instances FOSSGIS e.V. runs for the OpenStreetMap community (car, bike, foot profiles). They are keyless and intended for fair, interactive use — the proxy rate-limits clients (60/min, 200/min total), caps route span and response size, caches each route for 10 minutes, and identifies itself in the User-Agent. Heavy or automated use must run its own OSRM. Routes are computed from OpenStreetMap data (ODbL); the Directions layer and voice routes never redistribute them. A route the router cannot find is reported as such; the Directions layer draws nothing in that case, and voice routes fall back to a clearly labeled straight line. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 06b026cf6..1e1651e89 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3659,10 +3659,12 @@ Transit is off by default, and share links use token `j`. The movement panel ord places Transit directly after Street Traffic and before Bike Share. Cameras and utilities retain their own groups. The layer polls registered feeds near the camera every 15 seconds, below a 3,000 km altitude -gate. Eleven keyless regions are available: Boston, Austin, Minneapolis–St Paul, +gate. Thirteen keyless regions are available: Boston, Austin, Minneapolis–St Paul, Helsinki, the Netherlands, Norway, South East Queensland, Toronto (TTC -surface vehicles; the subway publishes no positions), and the Golden -Horseshoe neighbours MiWay, Hamilton HSR and Durham Region Transit. Mode silhouettes +surface vehicles; the subway publishes no positions), the Golden Horseshoe +neighbours MiWay, Hamilton HSR and Durham Region Transit, and Guelph and +Kingston. Barrie is registered but ships off, pending acceptance of its +click-through licence. Mode silhouettes identify buses, trams, subways, trains, ferries and unclassified vehicles. Retained transit history uses the same outlier quarantine as live observations and diff --git a/src/data/transitFeeds.js b/src/data/transitFeeds.js index c6c0f5147..0083d3b3e 100644 --- a/src/data/transitFeeds.js +++ b/src/data/transitFeeds.js @@ -130,6 +130,60 @@ function ttcRouteMode(routeId) { return 'bus'; } +/** + * Guelph Transit's realtime feed publishes the internal database key of a + * route (`2991`), not the number on the front of the bus (`1`). Without this + * table the detection label reads "Route 2991", which is not what anyone at + * the stop calls it. Route ids and short names are from the City of Guelph's + * own published GTFS `routes.txt` (see DATA_SOURCES.md); an id missing here + * falls through unchanged, so a route added later degrades to its raw id + * rather than disappearing. + */ +const GUELPH_ROUTE_NAMES = Object.freeze({ + 2991: '1', + 2992: '10', + 2993: '11', + 2994: '12', + 2995: '13', + 2996: '14', + 2997: '15', + 2998: '16', + 2999: '17', + 3000: '18', + 3001: '19', + 3002: '2', + 3003: '20', + 3004: '3', + 3005: '4', + 3006: '5', + 3007: '50 U', + 3008: '52 U', + 3009: '56 U', + 3010: '58 U', + 3011: '6', + 3012: '7', + 3013: '8', + 3014: '9', + 3015: '98', + 3016: '99', + 3017: '99Lite', + 3018: 'BREW', + 3019: 'EdiCol', + 3023: 'IroHarv', + 3030: 'Sun LN', + 3032: 'WHanSco', + 3039: 'ZehCha', +}); + +/** + * Guelph route id to the number a rider would say. + * @param {string|null} routeId + * @returns {string|null} + */ +function guelphRouteLabel(routeId) { + return GUELPH_ROUTE_NAMES[routeId] || routeId; +} + /** * Registry of feeds. Order is presentation order in the stats/credit text. * `loadRadiusKm` is the distance from `center` inside which the feed is polled. @@ -373,6 +427,69 @@ export const TRANSIT_FEED_REGISTRY = Object.freeze([ }), defaultMode: 'bus', }), + Object.freeze({ + id: 'guelph-transit', + name: 'Guelph Transit', + operator: 'Guelph Transit (City of Guelph)', + region: 'Guelph, ON', + center: Object.freeze({ lat: 43.5325, lon: -80.2482 }), + loadRadiusKm: 15, + url: 'https://glphprdtmgtfs.glphtrpcloud.com/tmgtfsrealtimewebservice/vehicle/vehiclepositions.pb', + license: 'Open Government Licence \u2013 City of Guelph (v2.0)', + licenseUrl: 'https://explore.guelph.ca/pages/open-data-license', + attribution: + 'Contains information licensed under the Open Government Licence \u2013 City of Guelph', + defaultEnabled: true, + terms: Object.freeze({ + quote: + 'The Information Provider grants you a worldwide, royalty-free, perpetual, non-exclusive licence to use the Information including for commercial purposes subject to the terms below. ... Acknowledge the source of the Information by including any attribution statement specified by the Information Provider(s) and, where possible, provide a link to this licence.', + note: 'The endpoint sits on a vendor host, so provenance matters: the City publishes this exact URL on its own open-data developer page, https://explore.guelph.ca/pages/transit-gtfs-data, under an open-data initiative heading (that page is JavaScript-rendered; its readable source is the ArcGIS item JSON behind it). The licence is a verbatim OGL-Canada 2.0 derivative and the attribution wording above is its prescribed fallback, so it is used as written. The transit datasets carry a BLANK licence field in the portal catalogue; the Terms of Use supply it by default ("Except where otherwise noted this is the Open Government Licence"), the same shape of gap the TTC entry records. The Terms also publish a traffic limit \u2014 no concurrent requests, at most 5 per second \u2014 which one poll every 15 s satisfies many times over. Crests and logos are excluded from the grant, so the credit stays text.', + }), + defaultMode: 'bus', + routeLabel: guelphRouteLabel, + }), + Object.freeze({ + id: 'kingston-transit', + name: 'Kingston Transit', + operator: 'Kingston Transit (The Corporation of the City of Kingston)', + region: 'Kingston, ON', + center: Object.freeze({ lat: 44.24, lon: -76.57 }), + loadRadiusKm: 20, + url: 'https://api.cityofkingston.ca/gtfs-realtime/vehicleupdates.pb', + license: 'City of Kingston Open Data Licence 1.0', + licenseUrl: + 'https://www.cityofkingston.ca/media/wtpgpkb0/gis_license_opendata.pdf', + attribution: + 'Contains information licensed under the Open Data Licence \u2013 City of Kingston', + defaultEnabled: true, + terms: Object.freeze({ + quote: + 'The Information Provider, the City of Kingston, grants you a worldwide, royalty-free, perpetual, non-exclusive licence to use the information provided for any purpose, including for commercial purposes, subject to the terms below. ... Acknowledge the source of the information by including any attribution statement specified by the City of Kingston and, where possible, provide a link to this licence.', + note: 'The City catalogues this exact URL as an open-data dataset, so provenance is direct rather than inferred. `licenseUrl` is the canonical location of the licence PDF; the address the catalogue prints is a legacy path that currently redirects there, and pinning the redirect target is what survives the next site relaunch \u2014 the Durham entry records what happens when it does not. Names, crests and logos are excluded from the grant, so the credit stays text. Roughly one entity in ten carries no usable position (the position field absent, occasionally a momentary 0,0 GPS dropout); both are discarded before rendering.', + }), + defaultMode: 'bus', + }), + Object.freeze({ + id: 'barrie-transit', + name: 'Barrie Transit', + operator: 'Barrie Transit (City of Barrie)', + region: 'Barrie, ON', + center: Object.freeze({ lat: 44.371, lon: -79.6733 }), + loadRadiusKm: 10, + url: 'https://www.myridebarrie.ca/gtfs/GTFS_VehiclePositions.pb', + license: 'Barrie Transit Data licence', + licenseUrl: + 'https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs', + attribution: + 'Realtime vehicle data \u00a9 City of Barrie, used under the Barrie Transit Data licence', + defaultEnabled: false, + terms: Object.freeze({ + quote: + 'Barrie Transit hereby grants you a non-exclusive, limited and revocable rights to use, reproduce, and redistribute Barrie Transit Data (scheduled and real-time data) subject to the following terms: Barrie Transit trademarks and copyrighted materials, including any confusingly similar variants, may not be used in association with Data. ... Barrie Transit maintains title, ownership, rights and interest in and to Data.', + note: 'OFF until a person on this project accepts the licence. The City publishes the feed directory behind a click-through "I agree" checkbox and a CAPTCHA; the endpoint itself needs no key, token or cookie, but the terms are offered for acceptance and nobody here has accepted them. That is a decision for a human, not something a poller should assume, so this ships registered and unpolled until someone ticks the box \u2014 then flip this one line. Two further oddities worth knowing: the licence requires NO attribution at all (the credit above is a courtesy), and its trademark clause is broad enough that the credit names the City rather than Barrie Transit. The grant is expressly REVOCABLE and unversioned, so the quote above is the text as read on 2026-09-16.', + }), + defaultMode: 'bus', + }), ]); /** @@ -486,6 +603,22 @@ export function transitModeResolved(feed, routeId) { ); } +/** + * The route as a rider would say it. Most feeds publish that already, so this + * is identity for them; a feed that publishes internal keys supplies its own + * `routeLabel`. Never invents a label: an id the feed does not know survives + * unchanged, because a raw id a person can compare against the bus is more + * use than a blank. + * @param {object|null} feed Registry entry. + * @param {string|null} routeId + * @returns {string|null} + */ +export function transitRouteLabel(feed, routeId) { + if (!routeId || typeof feed?.routeLabel !== 'function') return routeId; + const named = feed.routeLabel(routeId); + return typeof named === 'string' && named ? named : routeId; +} + /** * Public catalog shape served by `/api/transit/feeds` — everything the browser * needs to gate polling and credit the source, and nothing it could misuse. diff --git a/src/data/transitFeeds.test.mjs b/src/data/transitFeeds.test.mjs index b1c5d5bf7..08c86759a 100644 --- a/src/data/transitFeeds.test.mjs +++ b/src/data/transitFeeds.test.mjs @@ -13,6 +13,7 @@ import { transitFeedsInRange, transitModeFor, transitModeResolved, + transitRouteLabel, } from './transitFeeds.js'; test('every registered feed is keyless, https, licensed, and uniquely identified', () => { @@ -346,3 +347,46 @@ test('every Golden Horseshoe feed is documented with its licence', () => { } assert.match(sources, /Durham Region Transit publishes no bearing/); }); + +test('a feed whose route ids are internal database keys can name its routes', () => { + // Guelph's realtime feed emits `2991` where the rider sees route `1`. A feed + // that knows the difference translates it; every other feed is unaffected. + const guelph = getTransitFeed('guelph-transit'); + assert.equal(transitRouteLabel(guelph, '2991'), '1'); + assert.equal(transitRouteLabel(guelph, '3016'), '99'); + assert.equal(transitRouteLabel(guelph, '3007'), '50 U'); + assert.equal(transitRouteLabel(guelph, '3017'), '99Lite'); +}); + +test('an unknown or absent route id survives translation unchanged', () => { + const guelph = getTransitFeed('guelph-transit'); + // A route added after this table was built must still render its raw id + // rather than vanish or read as "undefined". + assert.equal(transitRouteLabel(guelph, '4242'), '4242'); + assert.equal(transitRouteLabel(guelph, null), null); + assert.equal(transitRouteLabel(guelph, ''), ''); +}); + +test('a feed with rider-facing route ids is left alone', () => { + // TTC and Kingston publish the route a rider would say, so translation is + // identity for them and no table is carried. + assert.equal(transitRouteLabel(getTransitFeed('ttc-toronto'), '501'), '501'); + assert.equal(transitRouteLabel(getTransitFeed('mbta'), 'Red'), 'Red'); + assert.equal(transitRouteLabel(null, '7'), '7'); +}); + +test('Barrie is registered but unpolled until its licence is accepted', () => { + // The City offers these terms for acceptance behind a click-through gate. + // Registered-but-off is how that decision stays visible and one line from + // reversible, rather than being made silently by a poller. + assert.equal( + getRegisteredTransitFeed('barrie-transit')?.name, + 'Barrie Transit', + ); + assert.equal(getTransitFeed('barrie-transit'), null, 'not routable'); + assert.equal( + publicTransitCatalog().some((feed) => feed.id === 'barrie-transit'), + false, + 'never offered to the browser', + ); +}); diff --git a/src/layers/transit/policy.js b/src/layers/transit/policy.js index 33352bf2d..721eb247e 100644 --- a/src/layers/transit/policy.js +++ b/src/layers/transit/policy.js @@ -4,7 +4,10 @@ * scene state, no network: everything here is exercised by node:test. */ -import { TRANSIT_MODE_ICON } from '../../data/transitFeeds.js'; +import { + TRANSIT_MODE_ICON, + transitRouteLabel, +} from '../../data/transitFeeds.js'; import { displayMotion, playbackPosition } from './movement.js'; export const TRANSIT_SELECTED_OVERLAY_SOURCE_ID = 'transit-selected'; @@ -337,7 +340,7 @@ export function buildTransitSelectionCopy( ) { const icon = TRANSIT_MODE_ICON[mode] || TRANSIT_MODE_ICON.unknown; const routeLabel = record.routeId - ? `Route ${record.routeId}` + ? `Route ${transitRouteLabel(feed, record.routeId)}` : record.label ? `Vehicle ${record.label}` : `Vehicle ${record.id}`; @@ -515,12 +518,14 @@ export function transitModeAbbr(mode, feed = null) { * names one, else the fleet number, else the raw id. What a person standing at * the stop would call it. * @param {object} record Normalized vehicle record. + * @param {object|null} [feed] Registry entry, for feeds whose route ids are + * internal keys rather than the number on the vehicle. * @returns {string} */ -export function transitDetectionId(record) { +export function transitDetectionId(record, feed = null) { const route = typeof record?.routeId === 'string' ? record.routeId.trim() : ''; - if (route) return elideRouteText(route); + if (route) return elideRouteText(transitRouteLabel(feed, route)); const label = typeof record?.label === 'string' ? record.label.trim() : ''; if (label) return elideRouteText(label); return elideRouteText(String(record?.id || 'TRANSIT')); diff --git a/src/layers/transit/queries.js b/src/layers/transit/queries.js index 470006c85..09b94c49f 100644 --- a/src/layers/transit/queries.js +++ b/src/layers/transit/queries.js @@ -143,7 +143,10 @@ export function createQueries({ state, parts }) { // Route and operator text change with the record and the mode, both of // which arrive on polls — the same events that rebuild this list. if (!entry.labelId || entry.labelRecord !== entry.record) { - entry.labelId = transitDetectionId(entry.record); + entry.labelId = transitDetectionId( + entry.record, + feedForEntry(state, entry), + ); entry.labelRecord = entry.record; } if (!entry.labelClass || entry.labelMode !== entry.mode) { diff --git a/src/layers/transit/vocabulary.test.mjs b/src/layers/transit/vocabulary.test.mjs index 17441e770..d20910b74 100644 --- a/src/layers/transit/vocabulary.test.mjs +++ b/src/layers/transit/vocabulary.test.mjs @@ -3,6 +3,7 @@ import assert from 'node:assert/strict'; import { buildTransitSelectionCopy, transitDetectionClass, + transitDetectionId, transitDetectionMetric, transitModeAbbr, transitModeWord, @@ -50,3 +51,26 @@ test('detection labels carry the operator vocabulary within the field width', () const metric = transitDetectionMetric({ mode: 'tram' }, 'tram', 0, ttc()); assert.match(metric, /^STREETCAR/); }); + +test('a Guelph vehicle is labelled with the number on the bus, not the database key', () => { + const guelph = getTransitFeed('guelph-transit'); + const record = { id: '196', routeId: '2991', timestamp: 0 }; + const copy = buildTransitSelectionCopy(guelph, record, 'bus', 0, null, null); + assert.match(copy.title, /Route 1\b/); + assert.doesNotMatch(copy.title, /2991/); + assert.equal(transitDetectionId(record, guelph), '1'); +}); + +test('a feed that already speaks rider route numbers is untouched', () => { + const record = { id: '4400', routeId: '501', timestamp: 0 }; + assert.equal(transitDetectionId(record, ttc()), '501'); + const copy = buildTransitSelectionCopy(ttc(), record, 'tram', 0, null, null); + assert.match(copy.title, /Route 501/); + // No feed at all still renders the raw route rather than throwing. + assert.equal(transitDetectionId(record), '501'); +}); + +test('a vehicle with no route still falls back to its fleet number', () => { + const guelph = getTransitFeed('guelph-transit'); + assert.equal(transitDetectionId({ id: '196', label: '196' }, guelph), '196'); +}); From 8ccb0cd89536f84415f73055d53351565a2916a4 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 00:34:43 -0400 Subject: [PATCH 82/95] feat(transit): register YRT/Viva off, named by Viva colour MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit York Region's feed is first-party infrastructure under an open licence that permits commercial use and does not even require attribution. It is the one feed that fills the gap north of Toronto, and it is switched off anyway. The operator asks that its licence be accepted through a form collecting a name, company, phone and email. No credential is issued and nothing gates the endpoint — the licence body says use is itself acceptance — but a form asking for contact details is close enough to the registration this registry refuses that a person should decide it, not a poller. Barrie is off for the same reason. Both are one line from on. Viva route ids get labels: YRT publishes `601` where every station sign says Viva Blue. The numeric local routes already read correctly and carry no entry, so the table is six lines, not 125. No vehicle in this feed carries a bearing; heading comes from observed movement, as with Durham. --- DATA_SOURCES.md | 3 ++- docs/CURRENT-STATE.md | 4 +-- src/data/transitFeeds.js | 48 ++++++++++++++++++++++++++++++++++ src/data/transitFeeds.test.mjs | 17 ++++++++++++ 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 0c245c828..54358e75c 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -84,6 +84,7 @@ How to read this: | **Guelph Transit** (glphtrpcloud.com) | Transit layer live vehicles, Guelph ON | Guelph Transit open realtime vehicle data, published by the City of Guelph on its open-data developer page and used for the live view. Its route ids are internal database keys, so the 33-row route-number table in `src/data/transitFeeds.js` is derived from the City’s own published GTFS `routes.txt` under the same licence. | "Contains information licensed under the Open Government Licence – City of Guelph" — wording prescribed by the licence | | **Kingston Transit** (api.cityofkingston.ca) | Transit layer live vehicles, Kingston ON | Kingston Transit open realtime vehicle data, catalogued by the City of Kingston as open data and used for the live view. | "Contains information licensed under the Open Data Licence – City of Kingston" — wording prescribed by the licence | | **Barrie Transit** (myridebarrie.ca) | REGISTERED BUT OFF — not polled, not offered to the browser | Barrie Transit realtime data, published by the City of Barrie behind a click-through acceptance of its licence. Registered so the decision is visible; it ships unpolled until a person accepts those terms. The licence is revocable and requires no attribution. | Courtesy credit to the City of Barrie (the licence requires none, and its trademark clause is broad) | +| **YRT/Viva** (rtu.york.ca) | REGISTERED BUT OFF — not polled, not offered to the browser | York Region Transit realtime data, first-party York Region infrastructure under the YRT Open Data Licence v2.0, which permits commercial use and makes attribution optional. Registered unpolled because the operator asks that the licence be accepted through a contact form first. Viva route ids are named by colour from York Region’s published GTFS. | "Contains public transit Information made available under YRT’s Open Data Licence" — the licence’s own wording, given though not required | | **Radio Browser** | Geolocated internet-radio station directory and station-level tags | Public-domain directory data under PDDL 1.0; individual broadcaster stream terms apply | "Radio Browser" plus a link to the selected broadcaster | | **Re:Earth Terrain** (Mapterhorn) | Terrain (keyless globe stacks — OSM etc. — + `/api/terrain/heights` ellipsoidal-height lookups) | Terrain mesh: CC BY 4.0; geoid: EGM2008 (NGA, public domain) | "Terrain (keyless globe stacks): Re:Earth Terrain / Mapterhorn (CC BY 4.0) / EGM2008 (NGA)" | | **OSRM on the FOSSGIS routing servers** (`routing.openstreetmap.de`) | Street-following routes for the Directions layer and voice route annotations, via `/api/route` | [FOSSGIS routing usage policy](https://routing.openstreetmap.de/about.html): "Display the required attribution and display a link to 'fix the map'", "Use a valid user agent and, if applicable, a correct referrer", "One request per second max", "No scraping, no heavy usage". The full policy is the German [FOSSGIS Nutzungsbedingungen](https://www.fossgis.de/arbeitsgruppen/osm-server/nutzungsbedingungen/); FOSSGIS also states that the server may be embedded in your own pages but "eine gewerbliche Nutzung ist nur mit Einschränkungen erlaubt" (commercial use only with restrictions). Route data derives from OpenStreetMap (ODbL 1.0) | "Routing: OSRM on the FOSSGIS servers" + a "fix the map" link — shown in the Data attribution popover | @@ -162,7 +163,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. -- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. Thirteen of the fourteen registered regions are live-enabled (Barrie is registered and deliberately off); each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. Guelph, Kingston and Barrie extend the same pattern eastward and north. Guelph is the one feed whose route ids are internal keys rather than the number on the bus, so its entry carries a route-number table taken from the City’s own published GTFS; any id missing from that table renders unchanged rather than blank. GO Transit / Metrolinx is deliberately absent: its REALTIME feed requires an API key issued by manual review, which the keyless admission rule excludes (its static schedule download is keyless, but carries no vehicle positions). Grand River Transit in Waterloo is absent for a different reason: it answers, but its feed header was months stale and it publishes no per-vehicle timestamps, so it would render vehicles that look live and are not. +- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. Thirteen of the fifteen registered regions are live-enabled; Barrie and YRT/Viva are registered and deliberately off; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. Guelph, Kingston and Barrie extend the same pattern eastward and north. Guelph is the one feed whose route ids are internal keys rather than the number on the bus, so its entry carries a route-number table taken from the City’s own published GTFS; any id missing from that table renders unchanged rather than blank. GO Transit / Metrolinx is deliberately absent: its REALTIME feed requires an API key issued by manual review, which the keyless admission rule excludes (its static schedule download is keyless, but carries no vehicle positions). Two feeds are registered and switched off rather than absent: Barrie and YRT/Viva both ask that their licence be accepted through a form before the data is taken, and that is a person’s decision rather than an assumption a poller should make. Grand River Transit in Waterloo is absent for a different reason: it answers, but its feed header was months stale and it publishes no per-vehicle timestamps, so it would render vehicles that look live and are not. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. - **TomTom Traffic.** Optional and BYOK: without `TOMTOM_API_KEY` the traffic layer runs its built-in simulation and no TomTom data (or attribution) appears. With a key, flow vector tiles are fetched through the server-side `/api/tomtom` proxy (120 s cache + a daily tile-budget governor — `TOMTOM_DAILY_TILE_BUDGET`, default 6,000 — sized so a 31-day month stays inside TomTom's 200K monthly free allowance; still a configurable application safety ceiling, not a guarantee; TomTom's [current pricing](https://docs.tomtom.com/pricing/) lists 200K free tile requests per month) and the "Traffic flow data © TomTom" credit is registered in the Data attribution popover the moment live mode activates. TomTom data is served live and cached only transiently (≤120 s TTL under `.gev-cache/`, gitignored) — it is not bundled or redistributed. One 23 KB point-in-time tile snapshot is committed as a decode-test fixture (`src/data/fixtures/`, © TomTom, never served to the app). - **Routing (OSRM / FOSSGIS).** `/api/route` proxies the public OSRM instances FOSSGIS e.V. runs for the OpenStreetMap community (car, bike, foot profiles). They are keyless and intended for fair, interactive use — the proxy rate-limits clients (60/min, 200/min total), caps route span and response size, caches each route for 10 minutes, and identifies itself in the User-Agent. Heavy or automated use must run its own OSRM. Routes are computed from OpenStreetMap data (ODbL); the Directions layer and voice routes never redistribute them. A route the router cannot find is reported as such; the Directions layer draws nothing in that case, and voice routes fall back to a clearly labeled straight line. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 1e1651e89..c6910dcc0 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3663,8 +3663,8 @@ gate. Thirteen keyless regions are available: Boston, Austin, Minneapolis–St P Helsinki, the Netherlands, Norway, South East Queensland, Toronto (TTC surface vehicles; the subway publishes no positions), the Golden Horseshoe neighbours MiWay, Hamilton HSR and Durham Region Transit, and Guelph and -Kingston. Barrie is registered but ships off, pending acceptance of its -click-through licence. Mode silhouettes +Kingston. Barrie and YRT/Viva are registered but ship off, +pending acceptance of their licences. Mode silhouettes identify buses, trams, subways, trains, ferries and unclassified vehicles. Retained transit history uses the same outlier quarantine as live observations and diff --git a/src/data/transitFeeds.js b/src/data/transitFeeds.js index 0083d3b3e..cb6c3ef0b 100644 --- a/src/data/transitFeeds.js +++ b/src/data/transitFeeds.js @@ -184,6 +184,31 @@ function guelphRouteLabel(routeId) { return GUELPH_ROUTE_NAMES[routeId] || routeId; } +/** + * YRT publishes Viva bus rapid transit by route id (`601`), while every rider, + * station sign and map calls it "Viva Blue". The local routes are already the + * number on the bus and are left alone. Colours are from York Region's own + * published GTFS `routes.txt`; Viva Green (604) and Pink (606) are not in + * current service and so are absent here too. + */ +const YRT_VIVA_ROUTE_NAMES = Object.freeze({ + 601: 'Viva Blue', + 60102: 'Viva Blue B', + 603: 'Viva Purple', + 60301: 'Viva Purple A', + 605: 'Viva Orange', + 607: 'Viva Yellow', +}); + +/** + * YRT route id to the name a rider would say. + * @param {string|null} routeId + * @returns {string|null} + */ +function yrtRouteLabel(routeId) { + return YRT_VIVA_ROUTE_NAMES[routeId] || routeId; +} + /** * Registry of feeds. Order is presentation order in the stats/credit text. * `loadRadiusKm` is the distance from `center` inside which the feed is polled. @@ -490,6 +515,29 @@ export const TRANSIT_FEED_REGISTRY = Object.freeze([ }), defaultMode: 'bus', }), + Object.freeze({ + id: 'yrt-york', + name: 'YRT/Viva', + operator: + 'York Region Transit (YRT/Viva), The Regional Municipality of York', + region: 'York Region, ON', + center: Object.freeze({ lat: 44.025, lon: -79.436 }), + loadRadiusKm: 38, + url: 'https://rtu.york.ca/gtfsrealtime/VehiclePositions', + license: 'YRT Open Data Licence v2.0 (UK OGL-derived)', + licenseUrl: + 'https://www.yrt.ca/en/about-us/open-data-licence-agreement.aspx', + attribution: + "Contains public transit Information made available under YRT's Open Data Licence", + defaultEnabled: false, + terms: Object.freeze({ + quote: + 'YRT grants You a worldwide, royalty-free, perpetual, non-exclusive licence to Use the Information subject to the conditions below ... Use the Information commercially ... Provide credit to YRT when using Information under this licence but are not required to. ... Please note: You must agree with and accept the above stated License Agreement in order to download YRT GTFS or real-time GTFS data.', + note: 'OFF for the same reason as Barrie: the operator asks that its licence be accepted through a form (forms.yrt.ca/YRT-GTFS-Data, which collects a name, company, phone and email and issues no credential) before the data is taken. Nothing gates the endpoint, and the licence body also says use is itself acceptance \u2014 but a form asking for contact details is close enough to the registration this registry refuses that a person, not a poller, should decide. Flip this one line once someone has answered it. The credit above is the licence\u2019s own wording and must be reproduced as given; attribution is optional here, and given anyway. Logos, trademarks and crests are carved out, so the credit stays text. Two data notes: no vehicle carries a bearing, so heading comes from observed movement; and one stuck unit reported a position ten hours old, which the layer shows as a stale report rather than hides.', + }), + defaultMode: 'bus', + routeLabel: yrtRouteLabel, + }), ]); /** diff --git a/src/data/transitFeeds.test.mjs b/src/data/transitFeeds.test.mjs index 08c86759a..711177a03 100644 --- a/src/data/transitFeeds.test.mjs +++ b/src/data/transitFeeds.test.mjs @@ -390,3 +390,20 @@ test('Barrie is registered but unpolled until its licence is accepted', () => { 'never offered to the browser', ); }); + +test('YRT is registered but unpolled until its licence form is answered', () => { + assert.equal(getRegisteredTransitFeed('yrt-york')?.name, 'YRT/Viva'); + assert.equal(getTransitFeed('yrt-york'), null, 'not routable'); +}); + +test('Viva bus rapid transit is named by its colour, not its route number', () => { + // YRT publishes `601` where every rider, map and station sign says + // "Viva Blue". The numeric local routes already read correctly and are + // left alone. + const yrt = getRegisteredTransitFeed('yrt-york'); + assert.equal(transitRouteLabel(yrt, '601'), 'Viva Blue'); + assert.equal(transitRouteLabel(yrt, '60301'), 'Viva Purple A'); + assert.equal(transitRouteLabel(yrt, '607'), 'Viva Yellow'); + assert.equal(transitRouteLabel(yrt, '105'), '105', 'a local bus is a number'); + assert.equal(transitRouteLabel(yrt, '9899'), '9899'); +}); From a7fc5ea7c54efb1377fcbc950c6a12452a0b8be8 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 00:40:28 -0400 Subject: [PATCH 83/95] feat(transit): add Burlington, switch Barrie and YRT on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Burlington's route ids are internal keys with a trap in them: `351` is route 1 while `3510` is route 10, so no prefix trim is correct and the table is a real join against the City's own published GTFS. Its licence is the same 2011 municipal template as Mississauga — revocable, and the terms must travel with the data, so licenseUrl is the terms document itself. Plain http:// has no listener at all, so https is mandatory, not preferred. Barrie and YRT ship on. Both operators invite acceptance of their licence through a form, and both licences also state that use is itself acceptance ("By using Barrie Transit Data, you agree to be bound by these terms"; "Your Use of Information indicates Your acceptance"). This deployment relies on that clause, and both entries record it so the reasoning does not have to be re-derived. Nine Ontario feeds now, all verified live against their upstreams. --- DATA_SOURCES.md | 7 ++-- docs/CURRENT-STATE.md | 11 +++--- src/data/transitFeeds.js | 65 +++++++++++++++++++++++++++++++--- src/data/transitFeeds.test.mjs | 60 ++++++++++++++++++++++--------- 4 files changed, 113 insertions(+), 30 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 54358e75c..62fd4cd69 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -83,8 +83,9 @@ How to read this: | **Durham Region Transit** (drtonline.durhamregiontransit.com) | Transit layer live vehicles, Durham Region ON | Durham Region Transit open realtime vehicle data, published under the Region of Durham Open Data Licence v.1.0, which permits commercial use. | "Contains public sector information made available under The Regional Municipality of Durham's Open Data Licence" | | **Guelph Transit** (glphtrpcloud.com) | Transit layer live vehicles, Guelph ON | Guelph Transit open realtime vehicle data, published by the City of Guelph on its open-data developer page and used for the live view. Its route ids are internal database keys, so the 33-row route-number table in `src/data/transitFeeds.js` is derived from the City’s own published GTFS `routes.txt` under the same licence. | "Contains information licensed under the Open Government Licence – City of Guelph" — wording prescribed by the licence | | **Kingston Transit** (api.cityofkingston.ca) | Transit layer live vehicles, Kingston ON | Kingston Transit open realtime vehicle data, catalogued by the City of Kingston as open data and used for the live view. | "Contains information licensed under the Open Data Licence – City of Kingston" — wording prescribed by the licence | -| **Barrie Transit** (myridebarrie.ca) | REGISTERED BUT OFF — not polled, not offered to the browser | Barrie Transit realtime data, published by the City of Barrie behind a click-through acceptance of its licence. Registered so the decision is visible; it ships unpolled until a person accepts those terms. The licence is revocable and requires no attribution. | Courtesy credit to the City of Barrie (the licence requires none, and its trademark clause is broad) | -| **YRT/Viva** (rtu.york.ca) | REGISTERED BUT OFF — not polled, not offered to the browser | York Region Transit realtime data, first-party York Region infrastructure under the YRT Open Data Licence v2.0, which permits commercial use and makes attribution optional. Registered unpolled because the operator asks that the licence be accepted through a contact form first. Viva route ids are named by colour from York Region’s published GTFS. | "Contains public transit Information made available under YRT’s Open Data Licence" — the licence’s own wording, given though not required | +| **Barrie Transit** (myridebarrie.ca) | Transit layer live vehicles, Barrie ON | Barrie Transit realtime data, published by the City of Barrie under a licence accepted by use of the data. The licence is revocable and requires no attribution. | Courtesy credit to the City of Barrie (the licence requires none, and its trademark clause is broad) | +| **YRT/Viva** (rtu.york.ca) | Transit layer live vehicles, York Region ON | York Region Transit realtime data, first-party York Region infrastructure under the YRT Open Data Licence v2.0, which permits commercial use and makes attribution optional. Viva route ids are named by colour from York Region’s published GTFS. | "Contains public transit Information made available under YRT’s Open Data Licence" — the licence’s own wording, given though not required | +| **Burlington Transit** (opendata.burlington.ca) | Transit layer live vehicles, Burlington ON | Burlington Transit realtime data under the City of Burlington Open Data Terms of Use, which permit any lawful purpose. The terms are revocable and must travel with the data, so the credit links them. Route ids are internal keys and are joined to the City’s published GTFS. | Credit to the City of Burlington plus a link to its Open Data Terms of Use, which the licence requires | | **Radio Browser** | Geolocated internet-radio station directory and station-level tags | Public-domain directory data under PDDL 1.0; individual broadcaster stream terms apply | "Radio Browser" plus a link to the selected broadcaster | | **Re:Earth Terrain** (Mapterhorn) | Terrain (keyless globe stacks — OSM etc. — + `/api/terrain/heights` ellipsoidal-height lookups) | Terrain mesh: CC BY 4.0; geoid: EGM2008 (NGA, public domain) | "Terrain (keyless globe stacks): Re:Earth Terrain / Mapterhorn (CC BY 4.0) / EGM2008 (NGA)" | | **OSRM on the FOSSGIS routing servers** (`routing.openstreetmap.de`) | Street-following routes for the Directions layer and voice route annotations, via `/api/route` | [FOSSGIS routing usage policy](https://routing.openstreetmap.de/about.html): "Display the required attribution and display a link to 'fix the map'", "Use a valid user agent and, if applicable, a correct referrer", "One request per second max", "No scraping, no heavy usage". The full policy is the German [FOSSGIS Nutzungsbedingungen](https://www.fossgis.de/arbeitsgruppen/osm-server/nutzungsbedingungen/); FOSSGIS also states that the server may be embedded in your own pages but "eine gewerbliche Nutzung ist nur mit Einschränkungen erlaubt" (commercial use only with restrictions). Route data derives from OpenStreetMap (ODbL 1.0) | "Routing: OSRM on the FOSSGIS servers" + a "fix the map" link — shown in the Data attribution popover | @@ -163,7 +164,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. -- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. Thirteen of the fifteen registered regions are live-enabled; Barrie and YRT/Viva are registered and deliberately off; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. Guelph, Kingston and Barrie extend the same pattern eastward and north. Guelph is the one feed whose route ids are internal keys rather than the number on the bus, so its entry carries a route-number table taken from the City’s own published GTFS; any id missing from that table renders unchanged rather than blank. GO Transit / Metrolinx is deliberately absent: its REALTIME feed requires an API key issued by manual review, which the keyless admission rule excludes (its static schedule download is keyless, but carries no vehicle positions). Two feeds are registered and switched off rather than absent: Barrie and YRT/Viva both ask that their licence be accepted through a form before the data is taken, and that is a person’s decision rather than an assumption a poller should make. Grand River Transit in Waterloo is absent for a different reason: it answers, but its feed header was months stale and it publishes no per-vehicle timestamps, so it would render vehicles that look live and are not. +- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All sixteen registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. Guelph, Kingston and Barrie extend the same pattern eastward and north. Guelph, Burlington and YRT publish route ids that are not the number on the vehicle, so those entries carry a route-name table taken from each operator’s own published GTFS; any id missing from a table renders unchanged rather than blank. Burlington’s is the one that cannot be shortcut: `351` is route 1 while `3510` is route 10, so the table is a real join rather than a trimmed prefix. GO Transit / Metrolinx is deliberately absent: its REALTIME feed requires an API key issued by manual review, which the keyless admission rule excludes (its static schedule download is keyless, but carries no vehicle positions). Barrie and YRT/Viva each invite acceptance of their licence through a form, and each licence also states that using the data is itself acceptance; this deployment relies on that clause, which is recorded in both entries. Grand River Transit in Waterloo is absent for a different reason: it answers, but its feed header was months stale and it publishes no per-vehicle timestamps, so it would render vehicles that look live and are not. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. - **TomTom Traffic.** Optional and BYOK: without `TOMTOM_API_KEY` the traffic layer runs its built-in simulation and no TomTom data (or attribution) appears. With a key, flow vector tiles are fetched through the server-side `/api/tomtom` proxy (120 s cache + a daily tile-budget governor — `TOMTOM_DAILY_TILE_BUDGET`, default 6,000 — sized so a 31-day month stays inside TomTom's 200K monthly free allowance; still a configurable application safety ceiling, not a guarantee; TomTom's [current pricing](https://docs.tomtom.com/pricing/) lists 200K free tile requests per month) and the "Traffic flow data © TomTom" credit is registered in the Data attribution popover the moment live mode activates. TomTom data is served live and cached only transiently (≤120 s TTL under `.gev-cache/`, gitignored) — it is not bundled or redistributed. One 23 KB point-in-time tile snapshot is committed as a decode-test fixture (`src/data/fixtures/`, © TomTom, never served to the app). - **Routing (OSRM / FOSSGIS).** `/api/route` proxies the public OSRM instances FOSSGIS e.V. runs for the OpenStreetMap community (car, bike, foot profiles). They are keyless and intended for fair, interactive use — the proxy rate-limits clients (60/min, 200/min total), caps route span and response size, caches each route for 10 minutes, and identifies itself in the User-Agent. Heavy or automated use must run its own OSRM. Routes are computed from OpenStreetMap data (ODbL); the Directions layer and voice routes never redistribute them. A route the router cannot find is reported as such; the Directions layer draws nothing in that case, and voice routes fall back to a clearly labeled straight line. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index c6910dcc0..01447298f 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3659,12 +3659,11 @@ Transit is off by default, and share links use token `j`. The movement panel ord places Transit directly after Street Traffic and before Bike Share. Cameras and utilities retain their own groups. The layer polls registered feeds near the camera every 15 seconds, below a 3,000 km altitude -gate. Thirteen keyless regions are available: Boston, Austin, Minneapolis–St Paul, -Helsinki, the Netherlands, Norway, South East Queensland, Toronto (TTC -surface vehicles; the subway publishes no positions), the Golden Horseshoe -neighbours MiWay, Hamilton HSR and Durham Region Transit, and Guelph and -Kingston. Barrie and YRT/Viva are registered but ship off, -pending acceptance of their licences. Mode silhouettes +gate. Sixteen keyless regions are available: Boston, Austin, Minneapolis–St Paul, +Helsinki, the Netherlands, Norway, South East Queensland, and nine in +Ontario — Toronto (TTC surface vehicles; the subway publishes no +positions), MiWay, Hamilton HSR, Durham Region, YRT/Viva, Guelph, +Burlington, Barrie and Kingston. Mode silhouettes identify buses, trams, subways, trains, ferries and unclassified vehicles. Retained transit history uses the same outlier quarantine as live observations and diff --git a/src/data/transitFeeds.js b/src/data/transitFeeds.js index cb6c3ef0b..560a3074f 100644 --- a/src/data/transitFeeds.js +++ b/src/data/transitFeeds.js @@ -209,6 +209,41 @@ function yrtRouteLabel(routeId) { return YRT_VIVA_ROUTE_NAMES[routeId] || routeId; } +/** + * Burlington publishes the internal key of a route, and the keys are a trap: + * `351` is route 1 while `3510` is route 10, so trimming a `35` prefix gets + * the wrong answer on half the network. The join is the only correct route. + * Ids and numbers are from the City's own published GTFS `routes.txt`, which + * sits in the same directory as the realtime feed under the same terms. + */ +const BURLINGTON_ROUTE_NAMES = Object.freeze({ + 351: '1', + 352: '2', + 353: '3', + 354: '4', + 356: '6', + 3510: '10', + 3511: '11', + 3512: '12', + 3525: '25', + 3548: '48', + 3550: '50', + 3551: '51', + 3552: '52', + 3580: '80', + 3581: '81', + 3587: '87', +}); + +/** + * Burlington route id to the number a rider would say. + * @param {string|null} routeId + * @returns {string|null} + */ +function burlingtonRouteLabel(routeId) { + return BURLINGTON_ROUTE_NAMES[routeId] || routeId; +} + /** * Registry of feeds. Order is presentation order in the stats/credit text. * `loadRadiusKm` is the distance from `center` inside which the feed is polled. @@ -507,11 +542,11 @@ export const TRANSIT_FEED_REGISTRY = Object.freeze([ 'https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs', attribution: 'Realtime vehicle data \u00a9 City of Barrie, used under the Barrie Transit Data licence', - defaultEnabled: false, + defaultEnabled: true, terms: Object.freeze({ quote: 'Barrie Transit hereby grants you a non-exclusive, limited and revocable rights to use, reproduce, and redistribute Barrie Transit Data (scheduled and real-time data) subject to the following terms: Barrie Transit trademarks and copyrighted materials, including any confusingly similar variants, may not be used in association with Data. ... Barrie Transit maintains title, ownership, rights and interest in and to Data.', - note: 'OFF until a person on this project accepts the licence. The City publishes the feed directory behind a click-through "I agree" checkbox and a CAPTCHA; the endpoint itself needs no key, token or cookie, but the terms are offered for acceptance and nobody here has accepted them. That is a decision for a human, not something a poller should assume, so this ships registered and unpolled until someone ticks the box \u2014 then flip this one line. Two further oddities worth knowing: the licence requires NO attribution at all (the credit above is a courtesy), and its trademark clause is broad enough that the credit names the City rather than Barrie Transit. The grant is expressly REVOCABLE and unversioned, so the quote above is the text as read on 2026-09-16.', + note: 'The City offers this licence for acceptance behind a click-through checkbox, and the licence body also makes use itself acceptance: \u201cBy using Barrie Transit Data, you agree to be bound by these terms.\u201d This deployment\u2019s owner took that clause as sufficient on 2026-09-16 and switched the feed on; nothing gates the endpoint, which needs no key, token or cookie. Two oddities worth knowing: the licence requires NO attribution at all (the credit above is a courtesy), and its trademark clause is broad enough that the credit names the City rather than Barrie Transit. The grant is expressly REVOCABLE and unversioned, so the quote above is the text as read on 2026-09-16.', }), defaultMode: 'bus', }), @@ -529,15 +564,37 @@ export const TRANSIT_FEED_REGISTRY = Object.freeze([ 'https://www.yrt.ca/en/about-us/open-data-licence-agreement.aspx', attribution: "Contains public transit Information made available under YRT's Open Data Licence", - defaultEnabled: false, + defaultEnabled: true, terms: Object.freeze({ quote: 'YRT grants You a worldwide, royalty-free, perpetual, non-exclusive licence to Use the Information subject to the conditions below ... Use the Information commercially ... Provide credit to YRT when using Information under this licence but are not required to. ... Please note: You must agree with and accept the above stated License Agreement in order to download YRT GTFS or real-time GTFS data.', - note: 'OFF for the same reason as Barrie: the operator asks that its licence be accepted through a form (forms.yrt.ca/YRT-GTFS-Data, which collects a name, company, phone and email and issues no credential) before the data is taken. Nothing gates the endpoint, and the licence body also says use is itself acceptance \u2014 but a form asking for contact details is close enough to the registration this registry refuses that a person, not a poller, should decide. Flip this one line once someone has answered it. The credit above is the licence\u2019s own wording and must be reproduced as given; attribution is optional here, and given anyway. Logos, trademarks and crests are carved out, so the credit stays text. Two data notes: no vehicle carries a bearing, so heading comes from observed movement; and one stuck unit reported a position ten hours old, which the layer shows as a stale report rather than hides.', + note: 'The operator asks that its licence be accepted through a form (forms.yrt.ca/YRT-GTFS-Data, which collects a name, company, phone and email and issues no credential), while the licence body says acceptance is by use: \u201cYour Use of Information indicates Your acceptance of the terms and conditions below.\u201d This deployment\u2019s owner took that clause as sufficient on 2026-09-16 and switched the feed on. Nothing gates the endpoint. The credit above is the licence\u2019s own wording and must be reproduced as given; attribution is optional here, and given anyway. Logos, trademarks and crests are carved out, so the credit stays text. Two data notes: no vehicle carries a bearing, so heading comes from observed movement; and one stuck unit reported a position ten hours old, which the layer shows as a stale report rather than hides.', }), defaultMode: 'bus', routeLabel: yrtRouteLabel, }), + Object.freeze({ + id: 'burlington-transit', + name: 'Burlington Transit', + operator: 'Burlington Transit (City of Burlington)', + region: 'Burlington, ON', + center: Object.freeze({ lat: 43.3342, lon: -79.8089 }), + loadRadiusKm: 12, + url: 'https://opendata.burlington.ca/gtfs-rt/GTFS_VehiclePositions.pb', + license: 'City of Burlington Open Data Terms of Use', + licenseUrl: + 'https://opendata.burlington.ca/opendata-terms-of-use/City%20of%20Burlington%20-%20Open%20Data%20Terms%20of%20Use.pdf', + attribution: + 'Transit data \u00a9 City of Burlington (Burlington Transit), used under the City of Burlington Open Data Terms of Use', + defaultEnabled: true, + terms: Object.freeze({ + quote: + 'The Corporation of the City of Burlington (the City) now grants you a worldwide, royalty-free, non-exclusive, revocable licence to use, reproduce, modify, and distribute the datasets in all current and future media and formats for any lawful purpose. ... If you distribute or provide access to these datasets to any other person ... you agree to include a copy of, or this Uniform Resource Locator (URL) for, these Terms of Use.', + note: 'Same 2011-era municipal template as Mississauga, and the same two consequences: the grant is REVOCABLE, and the terms have to travel with the data, so `licenseUrl` is the Terms document itself rather than a catalogue page and the attribution popover carries the link. Credit is optional here and given anyway. Crests, logos and marks are excluded and the terms forbid implying any association, so the credit stays plain text. The catalogue record for this data leads with a licence link that 404s; the URL above is the live one. Plain http:// has no listener at all \u2014 not a redirect, a hang \u2014 so the https:// URL is mandatory rather than preferred. The feed updates once a minute and answers conditional requests with an ETag.', + }), + defaultMode: 'bus', + routeLabel: burlingtonRouteLabel, + }), ]); /** diff --git a/src/data/transitFeeds.test.mjs b/src/data/transitFeeds.test.mjs index 711177a03..ea08799ea 100644 --- a/src/data/transitFeeds.test.mjs +++ b/src/data/transitFeeds.test.mjs @@ -375,35 +375,61 @@ test('a feed with rider-facing route ids is left alone', () => { assert.equal(transitRouteLabel(null, '7'), '7'); }); -test('Barrie is registered but unpolled until its licence is accepted', () => { - // The City offers these terms for acceptance behind a click-through gate. - // Registered-but-off is how that decision stays visible and one line from - // reversible, rather than being made silently by a poller. - assert.equal( - getRegisteredTransitFeed('barrie-transit')?.name, - 'Barrie Transit', - ); - assert.equal(getTransitFeed('barrie-transit'), null, 'not routable'); - assert.equal( - publicTransitCatalog().some((feed) => feed.id === 'barrie-transit'), - false, - 'never offered to the browser', - ); +test('the feeds whose licences are accepted by use are switched on', () => { + // Barrie and YRT each ask for acceptance through a form, and each licence + // also says that using the data is itself acceptance. That reading was + // taken deliberately, so both are routable and both record the reasoning + // in their terms note rather than leaving it to be re-derived later. + for (const id of ['barrie-transit', 'yrt-york']) { + assert.ok(getTransitFeed(id), `${id} is routable`); + assert.ok( + publicTransitCatalog().some((feed) => feed.id === id), + `${id} is offered to the browser`, + ); + assert.match( + getRegisteredTransitFeed(id).terms.note, + /acceptance is by use|use itself acceptance|took that clause as sufficient/, + `${id} records why it ships on`, + ); + } }); -test('YRT is registered but unpolled until its licence form is answered', () => { +test('every registered feed now ships on, and the gate still decides that', () => { + // The enabled set must stay a filter over the registry, not a copy of it: + // flipping one feed off has to make it unreachable. assert.equal(getRegisteredTransitFeed('yrt-york')?.name, 'YRT/Viva'); - assert.equal(getTransitFeed('yrt-york'), null, 'not routable'); + assert.deepEqual( + TRANSIT_FEED_REGISTRY.filter((feed) => feed.defaultEnabled !== true), + [], + ); }); test('Viva bus rapid transit is named by its colour, not its route number', () => { // YRT publishes `601` where every rider, map and station sign says // "Viva Blue". The numeric local routes already read correctly and are // left alone. - const yrt = getRegisteredTransitFeed('yrt-york'); + const yrt = getTransitFeed('yrt-york'); assert.equal(transitRouteLabel(yrt, '601'), 'Viva Blue'); assert.equal(transitRouteLabel(yrt, '60301'), 'Viva Purple A'); assert.equal(transitRouteLabel(yrt, '607'), 'Viva Yellow'); assert.equal(transitRouteLabel(yrt, '105'), '105', 'a local bus is a number'); assert.equal(transitRouteLabel(yrt, '9899'), '9899'); }); + +test('Burlington route ids are joined, never prefix-stripped', () => { + // `351` is route 1 and `3510` is route 10, so no amount of trimming a `35` + // prefix works — the table is the only correct answer. + const bur = getTransitFeed('burlington-transit'); + assert.equal(transitRouteLabel(bur, '351'), '1'); + assert.equal(transitRouteLabel(bur, '3510'), '10'); + assert.equal(transitRouteLabel(bur, '3512'), '12'); + assert.equal(transitRouteLabel(bur, '3587'), '87'); + assert.equal(transitRouteLabel(bur, '9999'), '9999', 'unknown survives'); +}); + +test('Burlington is live and carries its terms link, which its licence requires', () => { + const bur = getTransitFeed('burlington-transit'); + assert.equal(bur.defaultEnabled, true); + assert.match(bur.licenseUrl, /Open%20Data%20Terms%20of%20Use\.pdf$/); + assert.match(bur.url, /^https:\/\//, 'port 80 has no listener at all'); +}); From 4ffbdda3f565213f3798fb8c8dc7a56ee26ffa0a Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 16:01:10 -0400 Subject: [PATCH 84/95] feat(transit): name YRT's local branch routes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YRT encodes a branch as a two-digit suffix on the base route id. The Viva table already depended on that scheme — `60102` is Viva Blue's B branch — but local routes were passed through whole, on the reasoning that a YRT route id is already the number on the bus. For a plain route it is. For a branch it is not: a rider watching route 83's B-branch bus saw "8301". A 15:41 EDT sample of the live feed found four branch ids in service. Three are published in York Region's GTFS `routes.txt` and are named here from it, unpadded to match the local routes beside them (the file says `083A`; the registry has always preferred `105` to `005`): 10702 -> 107B KEELE 8301 -> 83A TRENCH 9002 -> 90B LESLIE The fourth, `9101`, runs in the realtime feed but appears nowhere in `routes.txt`. The pattern all but spells "91A" and that is precisely why it is absent: the registry translates published names, it does not infer them. It stays opaque until York Region publishes it. Renamed YRT_VIVA_ROUTE_NAMES to YRT_ROUTE_NAMES, since the table is no longer only about Viva. --- src/data/transitFeeds.js | 24 ++++++++++++++++++------ src/data/transitFeeds.test.mjs | 16 ++++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/data/transitFeeds.js b/src/data/transitFeeds.js index 560a3074f..818efe245 100644 --- a/src/data/transitFeeds.js +++ b/src/data/transitFeeds.js @@ -186,18 +186,30 @@ function guelphRouteLabel(routeId) { /** * YRT publishes Viva bus rapid transit by route id (`601`), while every rider, - * station sign and map calls it "Viva Blue". The local routes are already the - * number on the bus and are left alone. Colours are from York Region's own - * published GTFS `routes.txt`; Viva Green (604) and Pink (606) are not in - * current service and so are absent here too. + * station sign and map calls it "Viva Blue". Plain local routes are already the + * number on the bus and are left alone, unpadded: `105` reads better than the + * `005` of `routes.txt`. + * + * Branches are the exception. YRT suffixes a two-digit branch onto the base + * route — the scheme Viva's own `60102` follows — so a branch id reaches a + * rider as "8301" unless it is translated. Names and colours are from York + * Region's own published GTFS `routes.txt`; Viva Green (604) and Pink (606) + * are not in current service and so are absent here too. + * + * Only branches observed live AND published in `routes.txt` are listed. `9101` + * runs but is unpublished, so it is deliberately missing: the pattern suggests + * "91A", and suggesting is not knowing. */ -const YRT_VIVA_ROUTE_NAMES = Object.freeze({ +const YRT_ROUTE_NAMES = Object.freeze({ 601: 'Viva Blue', 60102: 'Viva Blue B', 603: 'Viva Purple', 60301: 'Viva Purple A', 605: 'Viva Orange', 607: 'Viva Yellow', + 8301: '83A', + 9002: '90B', + 10702: '107B', }); /** @@ -206,7 +218,7 @@ const YRT_VIVA_ROUTE_NAMES = Object.freeze({ * @returns {string|null} */ function yrtRouteLabel(routeId) { - return YRT_VIVA_ROUTE_NAMES[routeId] || routeId; + return YRT_ROUTE_NAMES[routeId] || routeId; } /** diff --git a/src/data/transitFeeds.test.mjs b/src/data/transitFeeds.test.mjs index ea08799ea..cc5481afc 100644 --- a/src/data/transitFeeds.test.mjs +++ b/src/data/transitFeeds.test.mjs @@ -416,6 +416,22 @@ test('Viva bus rapid transit is named by its colour, not its route number', () = assert.equal(transitRouteLabel(yrt, '9899'), '9899'); }); +test('YRT local branches read as the branch letter, not the padded id', () => { + // YRT suffixes a two-digit branch onto the base route, the same scheme the + // Viva table already relies on for `60102`. Unsuffixed, that leaks to a rider + // as "8301". The letters are York Region's own published GTFS + // `routes.txt` short names, unpadded to match the local routes beside them. + const yrt = getTransitFeed('yrt-york'); + assert.equal(transitRouteLabel(yrt, '10702'), '107B'); + assert.equal(transitRouteLabel(yrt, '8301'), '83A'); + assert.equal(transitRouteLabel(yrt, '9002'), '90B'); + // `9101` runs in the realtime feed but is absent from routes.txt, so there is + // no published name to give it. Inventing "91A" from the pattern is exactly + // the guess the registry refuses to make; it stays opaque until it is + // published. + assert.equal(transitRouteLabel(yrt, '9101'), '9101'); +}); + test('Burlington route ids are joined, never prefix-stripped', () => { // `351` is route 1 and `3510` is route 10, so no amount of trimming a `35` // prefix works — the table is the only correct answer. From 007562c6c8171a6e4899b8bb9f86395ab66cd0f4 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 16:37:24 -0400 Subject: [PATCH 85/95] docs: note the Ontario transit feeds in the changelog CONTRIBUTING asks that a runtime behaviour change update CHANGELOG.md in the same PR. Nine new feeds and two new label mechanisms qualify. --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f39f8d96..5ca1ca0de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +- Add nine Ontario transit feeds: TTC, MiWay, Hamilton HSR, Durham Region, + YRT/Viva, Guelph, Burlington, Barrie and Kingston, each carrying its own + licence quote and attribution. A feed can now name its own modes, so Toronto's + trams read "Streetcar" while Helsinki's stay "Tram", and a feed whose route ids + are internal keys can translate them to the number a rider would say. + - Cyclone advisories no longer go dark while NHC publishes an advisory ahead of its nominal time: advisory and position times up to 90 minutes ahead of the server clock are accepted (previously 5 minutes), so the whole feed is From 65ac2d37fb82de4fa38d0d18f9a019b0b9114e49 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 17:07:45 -0400 Subject: [PATCH 86/95] feat(transit): name Hamilton's routes, which ship as internal keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not one of HSR's 80 route ids equals the number on the bus. Hamilton was registered without a routeLabel, so every vehicle in the city read as "Route 5783" where a rider would say 5 DELAWARE — 228 of them in an afternoon sample. The mechanism to fix it already existed and simply was never pointed at this feed. Two details the table cannot skip: Hamilton runs two generations of route id at once. `5688` and `5780` are both the 2 BARTON, and both arrive in the live feed, so both are listed. That is why 80 ids collapse to 46 distinct routes. The seasonal and event shuttles publish a name where the local routes publish a number — TC230, PEACH, ROCKTON. Those are already what a rider would say, so they survive as published rather than being forced into a number they do not have. Numbers are unpadded to match the rest of the registry: routes.txt says `02`, a rider says 2. Ids and names are from the City's own published GTFS routes.txt, which sits under the same open-data licence as the realtime feed. Checked against the live feed: all 30 route ids in service translate, none fall through. --- src/data/transitFeeds.js | 105 +++++++++++++++++++++++++++++++++ src/data/transitFeeds.test.mjs | 17 ++++++ 2 files changed, 122 insertions(+) diff --git a/src/data/transitFeeds.js b/src/data/transitFeeds.js index 818efe245..cdf4b030b 100644 --- a/src/data/transitFeeds.js +++ b/src/data/transitFeeds.js @@ -184,6 +184,110 @@ function guelphRouteLabel(routeId) { return GUELPH_ROUTE_NAMES[routeId] || routeId; } +/** + * Hamilton publishes an internal key for every route: not one of HSR's 80 route + * ids equals the number on the bus, so an untranslated feed reads as + * "Route 5783" city-wide. Two generations of id are live at once — `5688` and + * `5780` are both the 2 BARTON — and both are listed, because both arrive. + * + * Numbers are unpadded to match the rest of the registry (`routes.txt` says + * `02`; a rider says 2). The seasonal and event shuttles publish a name rather + * than a number — `TC230`, `PEACH`, `ROCKTON` — and those are already what a + * rider would say, so they survive as published. Ids and names are from the + * City's own published GTFS `routes.txt`, under the same terms as the feed. + */ +const HAMILTON_ROUTE_NAMES = Object.freeze({ + 5687: '1', + 5688: '2', + 5689: '3', + 5690: '4', + 5691: '5', + 5692: '6', + 5693: '7', + 5694: '8', + 5695: '9', + 5696: '10', + 5697: '11', + 5698: '12', + 5699: '16', + 5700: '18', + 5701: '20', + 5702: '21', + 5703: '22', + 5704: '23', + 5705: '24', + 5706: '25', + 5707: '26', + 5708: '27', + 5709: '33', + 5710: '34', + 5711: '35', + 5712: '41', + 5713: '42', + 5714: '43', + 5715: '44', + 5716: '51', + 5717: '52', + 5718: '55', + 5719: '56', + 5720: '99', + 5731: 'X-IND', + 5780: '2', + 5781: '3', + 5782: '4', + 5783: '5', + 5784: '6', + 5785: '7', + 5786: '8', + 5787: '9', + 5789: '11', + 5790: '12', + 5791: '16', + 5792: '18', + 5793: '20', + 5794: '21', + 5795: '22', + 5796: '23', + 5797: '24', + 5798: '25', + 5799: '26', + 5800: '27', + 5801: '33', + 5802: '34', + 5803: '35', + 5805: '42', + 5806: '43', + 5807: '44', + 5808: '51', + 5809: '52', + 5811: '56', + 5812: '99', + 5820: 'TC230', + 5823: 'X IND', + 5824: 'TC-700', + 5825: 'TC-300', + 5826: 'CAN', + 5827: 'PEACH', + 5828: '1', + 5829: '10', + 5830: '41', + 5831: '55', + 5832: 'TC300', + 5833: 'TC700', + 5834: 'TC730', + 5835: 'ANCFAIR', + 5836: 'ROCKTON', +}); + +/** + * Hamilton route id to the number a rider would say. + * @param {string|null} routeId + * @returns {string|null} + */ +function hamiltonRouteLabel(routeId) { + return HAMILTON_ROUTE_NAMES[routeId] || routeId; +} + /** * YRT publishes Viva bus rapid transit by route id (`601`), while every rider, * station sign and map calls it "Viva Blue". Plain local routes are already the @@ -478,6 +582,7 @@ export const TRANSIT_FEED_REGISTRY = Object.freeze([ note: 'The attribution wording is prescribed by the licence, so it is used verbatim as the credit rather than paraphrased to an operator name. Names, crests, logos and marks are excluded from the grant, so the credit stays text.', }), defaultMode: 'bus', + routeLabel: hamiltonRouteLabel, }), Object.freeze({ id: 'durham-region', diff --git a/src/data/transitFeeds.test.mjs b/src/data/transitFeeds.test.mjs index cc5481afc..22281e46d 100644 --- a/src/data/transitFeeds.test.mjs +++ b/src/data/transitFeeds.test.mjs @@ -432,6 +432,23 @@ test('YRT local branches read as the branch letter, not the padded id', () => { assert.equal(transitRouteLabel(yrt, '9101'), '9101'); }); +test('Hamilton route ids are internal keys, read as the number on the bus', () => { + // Not one of HSR's 80 route ids equals its published short name, so every + // vehicle reads as "Route 5783" until the table translates it. Hamilton also + // carries two generations of id for the same route — `5688` and `5780` are + // both the 2 BARTON — so both must land on the same answer. + const hsr = getTransitFeed('hsr-hamilton'); + assert.equal(transitRouteLabel(hsr, '5780'), '2'); + assert.equal(transitRouteLabel(hsr, '5688'), '2', 'the older id agrees'); + assert.equal(transitRouteLabel(hsr, '5687'), '1'); + assert.equal(transitRouteLabel(hsr, '5829'), '10', 'the B-Line'); + assert.equal(transitRouteLabel(hsr, '5793'), '20', 'the A-Line'); + // Published names that are not numbers are already what a rider would say, + // so they survive as published rather than being forced into a number. + assert.equal(transitRouteLabel(hsr, '5820'), 'TC230'); + assert.equal(transitRouteLabel(hsr, '9999'), '9999', 'unknown survives'); +}); + test('Burlington route ids are joined, never prefix-stripped', () => { // `351` is route 1 and `3510` is route 10, so no amount of trimming a `35` // prefix works — the table is the only correct answer. From 9ab26bbcb04a6453528f21210e6317e170778da7 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 17:14:19 -0400 Subject: [PATCH 87/95] docs: record Hamilton's route table and the feeds left out Two gaps. The Hamilton entry never said its route ids are internal keys, which the Guelph, Burlington and YRT entries all say of theirs; the route-table sentence in the transit notes named those three and not Hamilton, though it now carries the largest table of the four. Both were stale the moment Hamilton got a table. The exclusions were half-recorded: GO Transit and Grand River had their reasons written down, but Ottawa, London, Thunder Bay and the static-only majority did not, so a later reader would have re-investigated feeds that were already surveyed and rejected. All of them are now named with the rule each one falls to, and the survey is stated as complete against the Mobility Database's Ontario catalogue. Also records that a route label is never invented, which is why YRT's unpublished 9101 renders as the feed sends it. --- DATA_SOURCES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 62fd4cd69..e78c082ad 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -79,7 +79,7 @@ How to read this: | **HSL** (realtime.hsl.fi) | Transit layer live vehicles, Helsinki | HSL (Helsinki Region Transport) open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to HSL (Helsinki Region Transport) (courtesy text) | | **TTC** (bustime.ttc.ca) | Transit layer live vehicles, Toronto — surface vehicles (buses and streetcars) only | Toronto Transit Commission BusTime open realtime vehicle data, published through City of Toronto Open Data and used for the live view. The subway publishes no vehicle positions, so Lines 1/2/4 are absent rather than inferred. | Attribution to Toronto Transit Commission — City of Toronto Open Data, under the Open Government Licence – Toronto | | **MiWay** (miapp.ca) | Transit layer live vehicles, Mississauga ON | MiWay open realtime vehicle data, published by the City of Mississauga under its Open Data Terms of Use, used for the live view. The licence is revocable and asks that the Terms of Use link travel with the data. | Attribution to MiWay — City of Mississauga Open Data, linked to the Terms of Use | -| **Hamilton HSR** (opendata.hamilton.ca) | Transit layer live vehicles, Hamilton ON | Hamilton Street Railway open realtime vehicle data, published by the City of Hamilton for commercial and non-commercial use. | "Contains public sector Data made available under the City of Hamilton’s Open Data Licence" — wording prescribed by the licence | +| **Hamilton HSR** (opendata.hamilton.ca) | Transit layer live vehicles, Hamilton ON | Hamilton Street Railway open realtime vehicle data, published by the City of Hamilton for commercial and non-commercial use. Not one of its 80 route ids is the number on the bus, so the route-number table in `src/data/transitFeeds.js` is derived from the City’s own published GTFS `routes.txt` under the same licence; two generations of id run at once and both are listed. | "Contains public sector Data made available under the City of Hamilton’s Open Data Licence" — wording prescribed by the licence | | **Durham Region Transit** (drtonline.durhamregiontransit.com) | Transit layer live vehicles, Durham Region ON | Durham Region Transit open realtime vehicle data, published under the Region of Durham Open Data Licence v.1.0, which permits commercial use. | "Contains public sector information made available under The Regional Municipality of Durham's Open Data Licence" | | **Guelph Transit** (glphtrpcloud.com) | Transit layer live vehicles, Guelph ON | Guelph Transit open realtime vehicle data, published by the City of Guelph on its open-data developer page and used for the live view. Its route ids are internal database keys, so the 33-row route-number table in `src/data/transitFeeds.js` is derived from the City’s own published GTFS `routes.txt` under the same licence. | "Contains information licensed under the Open Government Licence – City of Guelph" — wording prescribed by the licence | | **Kingston Transit** (api.cityofkingston.ca) | Transit layer live vehicles, Kingston ON | Kingston Transit open realtime vehicle data, catalogued by the City of Kingston as open data and used for the live view. | "Contains information licensed under the Open Data Licence – City of Kingston" — wording prescribed by the licence | @@ -164,7 +164,7 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. - **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. -- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All sixteen registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. Guelph, Kingston and Barrie extend the same pattern eastward and north. Guelph, Burlington and YRT publish route ids that are not the number on the vehicle, so those entries carry a route-name table taken from each operator’s own published GTFS; any id missing from a table renders unchanged rather than blank. Burlington’s is the one that cannot be shortcut: `351` is route 1 while `3510` is route 10, so the table is a real join rather than a trimmed prefix. GO Transit / Metrolinx is deliberately absent: its REALTIME feed requires an API key issued by manual review, which the keyless admission rule excludes (its static schedule download is keyless, but carries no vehicle positions). Barrie and YRT/Viva each invite acceptance of their licence through a form, and each licence also states that using the data is itself acceptance; this deployment relies on that clause, which is recorded in both entries. Grand River Transit in Waterloo is absent for a different reason: it answers, but its feed header was months stale and it publishes no per-vehicle timestamps, so it would render vehicles that look live and are not. +- **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All sixteen registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. The TTC feed is BusTime, the surface system: it carries buses and streetcars and no subway at all. About four in ten of its vehicles report no route at all (unassigned or deadheading), and those are shown with the feed default rather than a route-established mode. It sends no ETag or Last-Modified, so its conditional requests fall through to the ordinary snapshot cache. Three Golden Horseshoe neighbours run alongside it — MiWay, Hamilton HSR and Durham Region Transit — each bus-only and each polled only near its own region. Durham Region Transit publishes no bearing on any vehicle, so those sprites take their heading from observed movement like any other feed that omits it. Guelph, Kingston and Barrie extend the same pattern eastward and north. Hamilton, Guelph, Burlington and YRT publish route ids that are not the number on the vehicle, so those entries carry a route-name table taken from each operator’s own published GTFS; any id missing from a table renders unchanged rather than blank. A label is never invented: YRT runs a route `9101` that appears nowhere in its published routes, so it stays as the feed sends it. Burlington’s is the one that cannot be shortcut: `351` is route 1 while `3510` is route 10, so the table is a real join rather than a trimmed prefix. GO Transit / Metrolinx is deliberately absent: its REALTIME feed requires an API key issued by manual review, which the keyless admission rule excludes (its static schedule download is keyless, but carries no vehicle positions). Barrie and YRT/Viva each invite acceptance of their licence through a form, and each licence also states that using the data is itself acceptance; this deployment relies on that clause, which is recorded in both entries. Grand River Transit in Waterloo is absent for a different reason: it answers, but its feed header was months stale and it publishes no per-vehicle timestamps, so it would render vehicles that look live and are not. Ottawa’s OC Transpo falls to the same keyless admission rule as GO Transit — its realtime feed requires an API key. London Transit and Thunder Bay Transit publish vehicle positions over plain HTTP with no HTTPS endpoint available. Every other Ontario system — Brampton, Oakville, Niagara, Milton, Windsor, Greater Sudbury, UP Express and a dozen smaller operators — publishes static GTFS schedules only, with no realtime vehicle positions to register. Between the nine included and these exclusions, every Ontario realtime vehicle-position feed catalogued in the Mobility Database is accounted for. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. - **TomTom Traffic.** Optional and BYOK: without `TOMTOM_API_KEY` the traffic layer runs its built-in simulation and no TomTom data (or attribution) appears. With a key, flow vector tiles are fetched through the server-side `/api/tomtom` proxy (120 s cache + a daily tile-budget governor — `TOMTOM_DAILY_TILE_BUDGET`, default 6,000 — sized so a 31-day month stays inside TomTom's 200K monthly free allowance; still a configurable application safety ceiling, not a guarantee; TomTom's [current pricing](https://docs.tomtom.com/pricing/) lists 200K free tile requests per month) and the "Traffic flow data © TomTom" credit is registered in the Data attribution popover the moment live mode activates. TomTom data is served live and cached only transiently (≤120 s TTL under `.gev-cache/`, gitignored) — it is not bundled or redistributed. One 23 KB point-in-time tile snapshot is committed as a decode-test fixture (`src/data/fixtures/`, © TomTom, never served to the app). - **Routing (OSRM / FOSSGIS).** `/api/route` proxies the public OSRM instances FOSSGIS e.V. runs for the OpenStreetMap community (car, bike, foot profiles). They are keyless and intended for fair, interactive use — the proxy rate-limits clients (60/min, 200/min total), caps route span and response size, caches each route for 10 minutes, and identifies itself in the User-Agent. Heavy or automated use must run its own OSRM. Routes are computed from OpenStreetMap data (ODbL); the Directions layer and voice routes never redistribute them. A route the router cannot find is reported as such; the Directions layer draws nothing in that case, and voice routes fall back to a clearly labeled straight line. From 68e99d9407956b4fce646b5b68f0cd1fa972b557 Mon Sep 17 00:00:00 2001 From: Cliff Flood Date: Wed, 16 Sep 2026 17:17:37 -0400 Subject: [PATCH 88/95] docs: list the Ontario regions in the README feature table The Transit row names the regions a reader can expect to see vehicles in, and still stopped at South East Queensland after nine Ontario feeds were added. Someone scanning the table for Toronto would have concluded the layer does not cover it. Toronto carries the subway caveat here too: the row would otherwise promise metros in a city whose metro publishes no positions. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c8b9eafec..c5aeb4710 100644 --- a/README.md +++ b/README.md @@ -381,7 +381,7 @@ Twenty layers and map sources. **Eighteen have a keyless path.** Some offer addi | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | | 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary, Google Street View, KartaView and Panoramax | Mapillary · Google · KartaView · Panoramax | 🟢 KartaView, Panoramax · 🟡 Mapillary token · 🔴 Google Maps key | | 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | -| 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | +| 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland, and nine in Ontario: Toronto (surface vehicles; the subway publishes no positions), Mississauga, Hamilton, Durham Region, York Region, Guelph, Burlington, Barrie and Kingston | Operator GTFS-Realtime feeds | 🟢 | | 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | | 🧭 **Directions** | Click A and B on the globe for a street-following drive, walk or cycle route draped on the terrain with turn-by-turn steps — then FLY the camera along it. No key, no geocoder, no mic | OSRM on FOSSGIS servers (OpenStreetMap) | 🟢 | | 🔥 **Active Fires** | Live NASA FIRMS detections, trailing 24h | NASA FIRMS | 🟡 | From 3cdb440bc2166fb9bd42104b69f4ddc162c3dd31 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 17:27:55 -0700 Subject: [PATCH 89/95] Use the opaque iOS status bar and keep the SVG tab icon black-translucent needs viewport-fit=cover and safe-area padding, which this change does not add. List the PNG favicon before the SVG so browsers that take the last usable icon keep the SVG, add mobile-web-app-capable, and give the manifest a stable id. --- index.html | 7 +++++-- public/manifest.webmanifest | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/index.html b/index.html index bdc82bd25..8a01415b6 100644 --- a/index.html +++ b/index.html @@ -4,16 +4,19 @@ God's Eye View - + + + - + diff --git a/public/manifest.webmanifest b/public/manifest.webmanifest index cf4b2b791..478cd9019 100644 --- a/public/manifest.webmanifest +++ b/public/manifest.webmanifest @@ -1,4 +1,5 @@ { + "id": ".", "name": "God's Eye View", "short_name": "God's Eye", "description": "Live spatial intelligence in the browser.", From 687a94c9df5226ace08c5cbb8ff888641ec919da Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 17:27:55 -0700 Subject: [PATCH 90/95] =?UTF-8?q?Document=20Bike=20Ita=C3=BA=20(Tembici)?= =?UTF-8?q?=20and=20test=20the=20Brazilian=20cities?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seven feeds publish no licence, operator or attribution fields, so DATA_SOURCES records that and credits the operator. Adds a registry test per city: allowed host and path, provider, and activation over its outermost live station. --- CHANGELOG.md | 3 +- DATA_SOURCES.md | 3 +- docs/CURRENT-STATE.md | 2 +- src/data/dataCredits.js | 2 +- src/layers/bikeshare/brPbscCities.test.mjs | 73 ++++++++++++++++++++++ 5 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 src/layers/bikeshare/brPbscCities.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index f63083839..5d0194a6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,7 +66,8 @@ - Add seven Brazilian Bike Itaú (Tembici) cities to the GBFS bikeshare registry: São Paulo, Rio de Janeiro, Salvador, Recife, Porto Alegre, Brasília and - Curitiba. These are the layer's first stations in South America. + Curitiba (about 1,000 stations in total). These are the layer's first stations + in South America. - Add Bike Share Toronto (1,069 stations) to the bikeshare registry. The system publishes the same PBSC GBFS v2 feeds as the Austin, Honolulu and Chattanooga diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index ac1718308..66c92174e 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -65,8 +65,9 @@ How to read this: | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | | **NYC DOT Traffic Management Center** (`webcams.nyctmc.org`) | CCTV camera catalog + frames, New York City five boroughs | NYC DOT requires developers to sign a [data-sharing agreement](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) for the camera feed. **Off by default**; enable only under your own agreement | "NYC DOT Traffic Management Center" + NYC DOT's required application disclaimer (in the Data attribution popover) | | **City of Toronto Open Data** (`opendata.toronto.ca`) | CCTV camera catalog + frames, Toronto city streets | [Open Government Licence – Toronto](https://www.toronto.ca/city-government/data-research-maps/open-data/open-data-licence/) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – Toronto" | -| **GBFS (Lyft / BCycle / Social Bicycles / PBSC)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle, Bike Itaú/Tembici) + its `license_url` | +| **GBFS (Lyft / BCycle / Social Bicycles / PBSC)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **Bike Share Toronto** (`toronto.publicbikesystem.net`) | Bikeshare station availability, Toronto | Operator GBFS feed. No licence is published: the City's open-data package declares none and the feed has no `license_url`. Attribution is given to the operator. | "Bike Share Toronto (Toronto Parking Authority)" | +| **Bike Itaú (Tembici)** (`*.publicbikesystem.net`, seven Brazilian cities) | Bikeshare station availability: São Paulo, Rio de Janeiro, Salvador, Recife, Porto Alegre, Brasília and Curitiba | Operator GBFS feeds. No licence is published: the feeds' `system_information.json` carries no `license_url`, operator or attribution fields. Attribution is given to the operator. | "Bike Itaú (Tembici)" | | **Oslo / Bergen / Trondheim Bysykkel** (Urban Sharing, `gbfs.urbansharing.com`) | Bikeshare availability, Oslo, Bergen and Trondheim city bikes. Keyless GBFS; the proxy sends the `Client-Identifier` header the operators ask for | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED ([Oslo](https://oslobysykkel.no/en/open-data/realtime), [Bergen](https://bergenbysykkel.no/en/apne-data/sanntid), [Trondheim](https://trondheimbysykkel.no/en/open-data/realtime)) | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel" | | **Kolumbus / Farte Bysykkel** (Entur Mobility, `api.entur.io/mobility/v2/gbfs`) | Bikeshare availability, Kolumbus Bysykkel (Stavanger/Nord-Jæren) and Farte Bysykkel (Skien/Porsgrunn). Keyless GBFS; the proxy sends the `ET-Client-Name` header Entur requires and is limited to Entur's GBFS paths | [NLOD](https://data.norge.no/nlod/en/2.0) per the [Entur Mobility API docs](https://developer.entur.org/pages-mobility-docs-mobility-v2) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Entur (Kolumbus Bysykkel, Farte Bysykkel)" | | **JCDecaux Cyclocity** (`api.cyclocity.fr`) | Bikeshare availability, Bysykkel Lillestrøm. Keyless GBFS; the proxy is limited to JCDecaux contract paths | [Licence Ouverte / Etalab](https://developer.jcdecaux.com/files/Open-Licence-fr.pdf) (the feed's `license_url`) — attribution REQUIRED | "Bysykkel Lillestrøm: JCDecaux, Licence Ouverte" | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 61f7a0e80..7a211e806 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3232,7 +3232,7 @@ its criteria cannot be silently ignored. | CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + King County (WA) Open Data + WSDOT (WA) Open Data + NYC DOT TMC (opt-in) + City of Toronto RESCU + Catalonia (Servei Català de Trànsit) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto + Urban Sharing, Entur, JCDecaux NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto and Bike Itaú (Brazil) + Urban Sharing, Entur, JCDecaux NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | | Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 40537d4a9..d6c265ddc 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -304,7 +304,7 @@ export const DATA_CREDITS = [ key: 'gbfs', html: 'Bikeshare availability: GBFS operator feeds ' + - '(e.g. Austin BCycle; Bike Share Toronto, operated by the Toronto Parking Authority; Bike Itaú/Tembici)', + '(e.g. Austin BCycle; Bike Share Toronto, operated by the Toronto Parking Authority; Bike Itaú, operated by Tembici)', }, { key: 'urbansharing-gbfs', diff --git a/src/layers/bikeshare/brPbscCities.test.mjs b/src/layers/bikeshare/brPbscCities.test.mjs new file mode 100644 index 000000000..60e12c23b --- /dev/null +++ b/src/layers/bikeshare/brPbscCities.test.mjs @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { CITY_BY_ID } from './registry.js'; +import { createModel } from './model.js'; +import { createViewport } from './viewport.js'; +import { isAllowedGbfsHost, isAllowedGbfsPath } from '../../data/gbfsSource.js'; + +// Each Bike Itaú city's PBSC host, and the station farthest from its +// registered centre in the live feed (checked 2026-10-15). +const CITIES = { + 'sao-paulo-bike-itau': { + host: 'saopaulo.publicbikesystem.net', + stations: [{ name: '260 - P3', lat: -23.568235, lon: -46.740715 }], + }, + 'rio-de-janeiro-bike-itau': { + host: 'riodejaneiro.publicbikesystem.net', + stations: [{ name: '293 - Praia da Macumba', lat: -23.03135, lon: -43.479584 }], + }, + 'salvador-bike-itau': { + host: 'salvador.publicbikesystem.net', + stations: [{ name: '49 - Stella Maris', lat: -12.93645, lon: -38.32701 }], + }, + 'recife-bike-itau': { + host: 'recife.publicbikesystem.net', + stations: [{ name: '69 - Praia de Piedade', lat: -8.186205, lon: -34.918371 }], + }, + 'porto-alegre-bike-itau': { + host: 'portoalegre.publicbikesystem.net', + stations: [{ name: '72 - Avenida Tramandaí', lat: -30.135915, lon: -51.22684 }], + }, + 'brasilia-bike-itau': { + host: 'brasilia.publicbikesystem.net', + stations: [{ name: "60 - Parque Olhos d'Água", lat: -15.7431, lon: -47.88667 }], + }, + 'curitiba-bike-itau': { + host: 'curitiba.publicbikesystem.net', + stations: [{ name: '42 - Terminal Portão', lat: -25.476885, lon: -49.292608 }], + }, +}; + +// The production range check, with no viewer or fetches behind it. +const parts = { model: createModel({ state: {}, services: {}, parts: {}, source: null }) }; +const { computeInRangeCities } = createViewport({ + state: {}, + services: { render: { governorRequestRender() {} } }, + parts, + source: null, +}); + +test('the Bike Itaú systems are registered and readable through the proxy', () => { + for (const [id, expected] of Object.entries(CITIES)) { + const city = CITY_BY_ID.get(id); + assert.ok(city, `${id} is registered`); + assert.equal(city.provider, 'Bike Itaú'); + assert.deepEqual(city.hosts, [expected.host]); + for (const feed of [city.stationInformationUrl, city.stationStatusUrl]) { + const url = new URL(feed); + assert.ok(isAllowedGbfsHost(url.hostname), `${id} host ${url.hostname}`); + assert.ok(isAllowedGbfsPath(url.pathname), `${id} path ${url.pathname}`); + } + } +}); + +test('a camera over each outermost station activates its system', () => { + for (const [id, expected] of Object.entries(CITIES)) { + for (const station of expected.stations) { + assert.ok( + computeInRangeCities({ lat: station.lat, lon: station.lon }).has(id), + `${station.name} activates ${id}`, + ); + } + } +}); From 265c8a1cc0ea5b396f0392aa83589d02a06be1fd Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 17:27:55 -0700 Subject: [PATCH 91/95] Tighten datasheet model matching and keep given fields of view Codec, product and resolution words never identify a model; generation suffixes such as Mk II no longer fall back to the base model; a mismatched manufacturer never matches, and brand-only numeric names need the brand. The datasheet FOV only fills when the source and seed give none and the pose is not curated, and DATASHEET is shown only while no calibration offset applies. Adds the CC0 extract to DATA_SOURCES. --- CHANGELOG.md | 9 +- DATA_SOURCES.md | 12 ++ docs/CURRENT-STATE.md | 3 +- src/layers/cctv/catalog.js | 37 ++++-- src/layers/cctv/modelSpecs.js | 188 ++++++++++++++++++++-------- src/layers/cctv/modelSpecs.test.mjs | 108 +++++++++++++++- src/layers/cctv/presentation.js | 17 ++- 7 files changed, 295 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 834650d63..81ff88aa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,11 @@ - CCTV cameras with identified hardware show the model's datasheet specs. When a source names a camera model found in the vendored CCTV Camera Database records (CC0), the HUD meta line adds its resolution, night vision and PTZ capability. - A fixed single-lens model's horizontal FOV replaces the estimate - (`FOV n° (DATASHEET)`); a PTZ or varifocal range is what the lens can do, not - its current zoom, so those cameras keep the estimate and show - `SPEC FOV min–max° (CAPABILITY)` beside it (bassem chagra, #452). + A fixed single-lens model's horizontal FOV fills in when the source states no + FOV and the pose is not curated (`FOV n° (DATASHEET)`); a PTZ or varifocal + range is what the lens can do, not its current zoom, so those cameras keep the + estimate and show `SPEC FOV min–max° (CAPABILITY)` beside it (bassem chagra, + #452). - Add WSDOT highway cameras (Washington State) as a keyless CCTV source pack: WSDOT's official Travel Center camera layer, with frames registered only on diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 8f3b55dbf..d1ecd1423 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -185,6 +185,7 @@ The [Bhote Koshi event pack](public/events/bhote-koshi-2026/README.md), under `p | **Natural Earth countries, states and provinces** (260 admin-0/map units + 4,587 admin-1 polygons) | `natural_earth/` | **Public domain** | ✅ (no restrictions) | "Made with Natural Earth" (courtesy credit — not legally required) | | **US Census Bureau counties** (3,235 county polygons) | `us_census_counties/` | **Public domain** | ✅ (no restrictions) | "U.S. Census Bureau" (courtesy — not legally required) | | **CCTV ground heights** (3,445 cameras) | `cctv_ground_heights/` | Precomputed camera placement heights, aligned to work with Google Photorealistic 3D Tiles (folder README) | — | — | +| **CCTV Camera Database** model-enrichment extract v2.26.0 (17 camera-model records) | `src/layers/cctv/modelSpecs.js` | **CC0-1.0** (public domain dedication) | ✅ (no restrictions) | "Specs: cctv-database.com" (courtesy — not legally required) | ### ⚠️ TeleGeography is bundled but NonCommercial @@ -210,6 +211,17 @@ commercial use with attribution and applicable database share-alike/access obligations; it does not license the application code. Trimmed test-only tile fixtures retain their provenance in `src/data/fixtures/README.md`. +### CCTV Camera Database (`src/layers/cctv/modelSpecs.js`) + +The CCTV layer's hardware-model specs are a small verbatim extract (v2.26.0) of the +[CCTV Camera Database](https://github.com/ch-bas/cctv-camera-database), an open dataset of +camera models transcribed from manufacturer datasheets, dedicated to the public domain under +**CC0-1.0**. Only the records for hardware that the integrated public catalogs name (King +County, Sarasota County and Sioux Falls) are bundled, compiled into the module. Nothing is +fetched at runtime. The records describe camera products (optics, resolution, night vision) +only, never endpoints, stream paths or access data. CC0 requires no attribution; the CCTV HUD +credits it anyway as `SPECS: CCTV-DATABASE.COM` beside any spec line it renders. + ### Bundled OpenStreetMap data Bundled OSM-derived data is distributed under [ODbL 1.0](https://opendatacommons.org/licenses/odbl/1-0/), separately from the MIT code: [datacenters](src/data/local_data/datacenters/), [dams](src/data/local_data/dams/) (via Open Infrastructure Map), [military names](src/data/local_data/osm_military_names/), and [OSM test fixtures](src/data/fixtures/). Keep © OpenStreetMap contributors and offer modified databases under ODbL. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d24d9d92d..b2be525c7 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4185,7 +4185,8 @@ silently demoting every later lookup for the session. - **CCTV identified hardware:** a camera whose source or seed names a hardware `model` that resolves in `src/layers/cctv/modelSpecs.js` (vendored CC0 records from the CCTV Camera Database) gets the model's specs appended to the HUD meta line. Only fixed single-lens - hardware replaces the estimated FOV, shown as `FOV n° (DATASHEET)`. PTZ, varifocal and + hardware fills the FOV, and only when the source or seed states none and the pose is not + curated, shown as `FOV n° (DATASHEET)` until a manual calibration moves it. PTZ, varifocal and multi-sensor datasheets publish a capability range, not the lens's current zoom, so those cameras keep their estimate and show `SPEC FOV min–max° (CAPABILITY)` beside it. Public camera state carries `fovSource`, `specFovMinDeg`, `specFovMaxDeg` and `spec` (#452). diff --git a/src/layers/cctv/catalog.js b/src/layers/cctv/catalog.js index 7804327d6..900692939 100644 --- a/src/layers/cctv/catalog.js +++ b/src/layers/cctv/catalog.js @@ -7,17 +7,22 @@ import { /** * Applies datasheet enrichment to a camera in place: when it names a known - * hardware `model`, the spec record is attached for the HUD and the - * datasheet's horizontal FOV range rides along as `specFovMinDeg`/ - * `specFovMaxDeg`. Only fixed-lens hardware replaces the pose estimate: a - * PTZ/varifocal range says what the lens CAN do, not where it is zoomed now. + * hardware `model` (and, when published, a matching `manufacturer`), the spec + * record is attached for the HUD and the datasheet's horizontal FOV range + * rides along as `specFovMinDeg`/`specFovMaxDeg`. Only fixed-lens hardware + * fills the FOV, and only when none was given: an explicit source/seed + * `fovDeg` or a curated pose is never overwritten. A PTZ/varifocal range says + * what the lens CAN do, not where it is zoomed now. * Unknown models — the common case — change nothing. * @param {Object} camera - Camera being built (mutated). - * @param {string} [modelName] - Hardware model from a source or seed. + * @param {{model?:string, manufacturer?:string, explicitFov?:boolean}} hardware + * - Hardware model/manufacturer from a source or seed, and whether that + * source or seed stated its own `fovDeg`. * @param {(v:number,min:number,max:number)=>number} clamp - Pose clamp helper. */ -function applyModelEnrichment(camera, modelName, clamp) { - const spec = modelName ? lookupModelSpec(modelName) : null; +function applyModelEnrichment(camera, hardware, clamp) { + const { model, manufacturer, explicitFov } = hardware; + const spec = model ? lookupModelSpec(model, manufacturer) : null; if (!spec) return; const range = horizontalFovRangeDeg(spec); if (range) { @@ -25,7 +30,7 @@ function applyModelEnrichment(camera, modelName, clamp) { camera.specFovMaxDeg = range.maxDeg; } const fixedFov = fixedHorizontalFovDeg(spec); - if (fixedFov) { + if (fixedFov && !explicitFov && camera.poseSource !== 'curated') { camera.fovDeg = clamp(fixedFov, 20, 125); camera.fovSource = 'datasheet'; } @@ -77,7 +82,11 @@ export function createCatalog({ state: layerState, services, parts, source }) { parts.model.clamp(seed.elevationM ?? 22, 8, 80), pitchDeg: parts.model.clamp(seed.pitchDeg ?? -17, -40, -4), }; - applyModelEnrichment(camera, seed.model, parts.model.clamp); + applyModelEnrichment( + camera, + { model: seed.model, explicitFov: seed.fovDeg != null }, + parts.model.clamp, + ); parts.model.ensureCameraPose(camera); catalog.push(camera); } @@ -242,7 +251,15 @@ export function createCatalog({ state: layerState, services, parts, source }) { }; applyModelEnrichment( camera, - source.model || seed?.model, + { + model: source.model || seed?.model, + manufacturer: source.model ? source.manufacturer : '', + explicitFov: + (source.fovDeg != null && Number.isFinite(Number(source.fovDeg))) || + CAMERA_SEEDS.some( + (entry) => entry.id === id && entry.fovDeg != null, + ), + }, parts.model.clamp, ); parts.model.ensureCameraPose(camera); diff --git a/src/layers/cctv/modelSpecs.js b/src/layers/cctv/modelSpecs.js index cee3e5044..1e5bc7cf1 100644 --- a/src/layers/cctv/modelSpecs.js +++ b/src/layers/cctv/modelSpecs.js @@ -14,7 +14,7 @@ * * Spec records are vendored verbatim from the CCTV Camera Database * (https://github.com/ch-bas/cctv-camera-database, CC0-1.0) — an open dataset - * of 28,000+ camera models transcribed from manufacturer datasheets (75% with + * of 28,000+ camera models transcribed from manufacturer datasheets (84% with * FOV, as of v2.26.0). This module vendors the records that the currently integrated public * catalogs publish hardware for — King County WA (Cohu, Bosch AUTODOME, Axis * Q6135-LE), plus the Bosch AUTODOME and Axis PTZ/multisensor models that @@ -25,7 +25,7 @@ * night vision) — never endpoints, stream paths, or access/config data. */ -/** Verbatim records from the dataset's model-enrichment extract (v2.19.0). */ +/** Verbatim records from the dataset's model-enrichment extract (v2.26.0). */ const RECORDS = [ { id: 'axis-q6135-le', @@ -307,74 +307,144 @@ function normalize(value) { .replace(/\s+/g, ' '); } -const BY_MODEL = new Map(); -for (const record of RECORDS) { - BY_MODEL.set(normalize(record.model), record); - for (const alias of record.aliases || []) - BY_MODEL.set(normalize(alias), record); +/** + * Whitespace words of a name, lowercased, with surrounding punctuation trimmed + * so "(NDP-7512-Z30)" reads as "ndp-7512-z30". Inner hyphens and dots stay: + * "f4105-lre-unknown" is one word, never "f4105-lre" plus noise. + * @param {string} value + * @returns {string[]} + */ +function words(value) { + return normalize(value) + .split(' ') + .map((raw) => raw.replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, '')) + .filter(Boolean); } /** - * DISTINCTIVE tokens of a record — the whitespace words of its model and - * aliases that identify hardware rather than describe it. A token qualifies - * only when it is >= 4 chars AND contains a digit or hyphen, so generic words - * ("dome", "network", "camera", "series", "ptz") are never distinctive and a - * verbose registry string cannot match on them alone. Surrounding punctuation - * is trimmed so "(NDP-7512-Z30)" indexes as "ndp-7512-z30". - * @param {Object} record - * @returns {string[]} Unique distinctive tokens, lowercased. + * Descriptor words that never identify hardware: product-type nouns, codec + * and resolution/zoom/frequency tokens ("h.264", "2mp", "40x", "1080p"). */ -function distinctiveTokens(record) { - const out = new Set(); - for (const name of [record.model, ...(record.aliases || [])]) { - for (const raw of normalize(name).split(' ')) { - const token = raw.replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, ''); - if (token.length >= 4 && /[0-9-]/.test(token)) out.add(token); - } +const GENERIC_WORDS = new Set([ + 'network', + 'camera', + 'cam', + 'ip', + 'ptz', + 'dome', + 'bullet', + 'box', + 'hd', + 'ir', + 'outdoor', + 'indoor', + 'series', +]); + +function isGeneric(word) { + return ( + GENERIC_WORDS.has(word) || + /^h\.?\d+$/.test(word) || + /^\d+(?:\.\d+)?(?:mp|p|k|x|mm|hz|tvl)$/.test(word) + ); +} + +/** A model code: letters AND digits, and not a generic descriptor. */ +function isModelCode(word) { + return /[a-z]/.test(word) && /\d/.test(word) && !isGeneric(word); +} + +/** Hyphenated variant suffixes that do not change the hardware. */ +const VARIANT_SUFFIXES = new Set(['50hz', '60hz', 'ntsc', 'pal']); + +/** "q6155-e-60hz" → "q6155-e"; "f4105-lre-unknown" is left as is. */ +function stripVariantSuffix(word) { + let out = word; + for (;;) { + const cut = out.lastIndexOf('-'); + if (cut <= 0 || !VARIANT_SUFFIXES.has(out.slice(cut + 1))) return out; + out = out.slice(0, cut); } - return [...out]; +} + +/** Manufacturer names (lowercased words) accepted for each record brand. */ +const BRAND_ALIASES = { + Hanwha: ['wisenet', 'samsung'], + Hikvision: ['hiwatch'], +}; + +function brandWords(record) { + return new Set([ + ...words(record.brand), + ...(BRAND_ALIASES[record.brand] || []), + ]); } /** - * Token index for the substring fallback: one precompiled whole-word matcher - * per distinctive token. Word boundaries are any non-alphanumeric char (or the - * string ends), so hyphenated/dotted tokens like "q6155-e" or "h.264" match as - * whole words while "700" never matches inside "7000". + * One entry per record name (model + aliases). `required` is every + * non-generic word of the name — all must appear in a query, so a generation + * or suffix ("Mk II", "7000i" vs "7000") is never dropped. `hasCode` marks + * names carrying a letter+digit model code; names without one ("3950", + * "01959-004", "AUTODOME 800 Series") match only when the brand does. */ -const TOKEN_INDEX = RECORDS.flatMap((record) => - distinctiveTokens(record).map((token) => ({ - token, - record, - re: new RegExp( - `(?:^|[^a-z0-9])${token.replace(/[.*+?^${}()|[\]\\-]/g, '\\$&')}(?:[^a-z0-9]|$)`, - ), - })), -); +const NAME_INDEX = RECORDS.flatMap((record) => { + const brand = brandWords(record); + return [record.model, ...(record.aliases || [])].map((name) => { + const nameWords = words(name); + return { + record, + brand, + allowed: new Set(nameWords), + required: nameWords.filter((word) => !isGeneric(word)), + hasCode: nameWords.some(isModelCode), + }; + }); +}); /** * Looks up a spec record for a hardware-model string. * - * Exact match first (case-insensitive, whitespace-collapsed) on the model and - * every alias. When that misses, a token fallback lets a VERBOSE registry - * string resolve — e.g. "AXIS Q6155-E PTZ Dome Network Camera" finds the - * Q6155-E via its distinctive "q6155-e" token. The most specific (longest) - * matched token wins; if two different records tie on the longest matched - * token the result is ambiguous and returns null (never guesses). + * A record name matches when every non-generic word of that name appears in + * the query (case-insensitive, punctuation-trimmed) and every other query word + * is generic or the record's brand. That lets a VERBOSE registry string + * resolve — "AXIS Q6155-E PTZ Dome Network Camera" finds the Q6155-E — while + * an unrelated code ("Q1615"), a generation suffix ("Mk II") or a longer part + * number ("F4105-LRE-UNKNOWN") rejects the match. Codec, resolution and + * product-type words never identify hardware on their own. A name with no + * letter+digit model code ("3950") needs the brand, from `manufacturer` or the + * query itself. A `manufacturer` that is not the record's brand always + * rejects. Matches on two different records are ambiguous → null (never + * guesses). * @param {string} model - Hardware model as named by a source pack or seed. + * @param {string} [manufacturer] - The source's manufacturer, when published. * @returns {Object|null} The verbatim spec record, or null when unknown. */ -export function lookupModelSpec(model) { - if (!model) return null; - const query = normalize(model); - if (!query) return null; - const exact = BY_MODEL.get(query); - if (exact) return exact; - const matches = TOKEN_INDEX.filter((entry) => entry.re.test(query)); - if (!matches.length) return null; - const maxLen = Math.max(...matches.map((m) => m.token.length)); - const top = matches.filter((m) => m.token.length === maxLen); - const records = new Set(top.map((m) => m.record)); - return records.size === 1 ? top[0].record : null; +export function lookupModelSpec(model, manufacturer) { + const query = words(model).map(stripVariantSuffix); + if (!query.length) return null; + const maker = words(manufacturer); + const found = new Set(); + for (const entry of NAME_INDEX) { + const makerMatches = maker.some((word) => entry.brand.has(word)); + if (maker.length && !makerMatches) continue; + if ( + !entry.hasCode && + !makerMatches && + !query.some((word) => entry.brand.has(word)) + ) + continue; + if (!entry.required.length) continue; + if (!entry.required.every((word) => query.includes(word))) continue; + if ( + !query.every( + (word) => + entry.allowed.has(word) || entry.brand.has(word) || isGeneric(word), + ) + ) + continue; + found.add(entry.record); + } + return found.size === 1 ? [...found][0] : null; } /** @@ -416,6 +486,16 @@ export function fixedHorizontalFovDeg(spec) { return range && range.minDeg === range.maxDeg ? range.minDeg : null; } +/** + * True when the camera's CURRENT FOV is the datasheet figure: it was filled + * from a fixed-lens datasheet and no manual calibration offset has moved it. + * @param {Object} camera - Catalog camera (see applyModelEnrichment). + * @returns {boolean} + */ +export function isDatasheetFov(camera) { + return camera?.fovSource === 'datasheet' && !camera.calibration?.fovDeg; +} + /** * HUD token for a datasheet FOV that describes capability rather than the * camera's current state, e.g. 'SPEC FOV 2.4–58° (CAPABILITY)'. diff --git a/src/layers/cctv/modelSpecs.test.mjs b/src/layers/cctv/modelSpecs.test.mjs index 06ffe3a64..9fbdb00dd 100644 --- a/src/layers/cctv/modelSpecs.test.mjs +++ b/src/layers/cctv/modelSpecs.test.mjs @@ -9,6 +9,7 @@ import { horizontalFovRangeDeg, fixedHorizontalFovDeg, specFovCapabilityToken, + isDatasheetFov, specSummary, } from './modelSpecs.js'; import { createCatalog } from './catalog.js'; @@ -21,7 +22,8 @@ test('lookupModelSpec matches case-insensitively and trims', () => { }); test('lookupModelSpec matches aliases', () => { - const byAlias = lookupModelSpec('01959-004'); + // A part number without letters needs the brand. + const byAlias = lookupModelSpec('01959-004', 'Axis'); assert.ok(byAlias); assert.equal(byAlias.model, 'Q6135-LE'); }); @@ -29,7 +31,8 @@ test('lookupModelSpec matches aliases', () => { test('vendored table covers the integrated catalogs published hardware', () => { // Models the live packs publish: King County (Cohu, AUTODOME IP 5000i), // Sarasota (Bosch AUTODOME 7000/VG4), Sioux Falls (Axis). - const cohu = lookupModelSpec('3950'); // King County publishes "3950" + // King County publishes "3950" with Manufacturer "Cohu". + const cohu = lookupModelSpec('3950', 'Cohu'); assert.equal(cohu?.brand, 'Cohu'); const autodome = lookupModelSpec('AUTODOME IP 5000i'); assert.ok(autodome && horizontalFovRangeDeg(autodome).maxDeg > 0); @@ -71,8 +74,8 @@ test('token fallback never matches on generic words or wrong models', () => { }); test('exact match still wins over the token fallback', () => { - // "3965" is an alias of the 3960; exact alias match, not a token scan. - assert.equal(lookupModelSpec('3965')?.id, 'cohu-3960'); + // "3965" is an alias of the 3960, not the 3950. + assert.equal(lookupModelSpec('3965', 'Cohu')?.id, 'cohu-3960'); assert.equal(lookupModelSpec('AUTODOME IP 5000i')?.id, 'bosch-ndp-5502-30'); }); @@ -167,10 +170,10 @@ function browserCatalog() { }); } -test('catalog keeps the estimated FOV for PTZ hardware and substitutes only fixed-lens datasheets', () => { +test('catalog keeps the estimated FOV for PTZ hardware and fills only fixed-lens datasheets', () => { const [ptz, fixed, unknown] = browserCatalog().buildCatalogFromSources([ { id: 'ptz', lat: 47.7, lon: -122, fovDeg: 44, model: 'Q6135-LE' }, - { id: 'fixed', lat: 47.7, lon: -122, fovDeg: 44, model: 'F4105-LRE' }, + { id: 'fixed', lat: 47.7, lon: -122, model: 'F4105-LRE' }, { id: 'unknown', lat: 47.7, lon: -122, fovDeg: 44, model: 'NOPE-1' }, ]); @@ -191,6 +194,99 @@ test('catalog keeps the estimated FOV for PTZ hardware and substitutes only fixe assert.equal(unknown.specFovMaxDeg, undefined); }); +test('codec, format and product words never identify hardware', () => { + // "h.264" is in the VG4 AUTODOME alias; it is a codec, not a model code. + assert.equal(lookupModelSpec('Axis Q1615 H.264 Network Camera'), null); + assert.equal(lookupModelSpec('H.264 Network Camera'), null); + assert.equal(lookupModelSpec('H.265 IP Camera'), null); +}); + +test('generation and suffix distinctions are preserved', () => { + // A Mk II is not the vendored (Mk I) Q6135-LE. + assert.equal(lookupModelSpec('AXIS Q6135-LE Mk II'), null); + // A longer part number is a different product... + assert.equal(lookupModelSpec('F4105-LRE-UNKNOWN'), null); + // ...unless the rest is a mains-frequency variant suffix. + assert.equal(lookupModelSpec('Q6155-E-60Hz')?.id, 'axis-q6155-e'); +}); + +test('manufacturer must agree with the record brand', () => { + // Purely numeric models need a brand match. + assert.equal(lookupModelSpec('3950'), null); + assert.equal(lookupModelSpec('Pelco 3950'), null); + assert.equal(lookupModelSpec('3950', 'Pelco'), null); + // A model code under the wrong manufacturer is not that record either. + assert.equal(lookupModelSpec('Q6135-LE', 'Hikvision'), null); + // Case-insensitive, with company suffixes and aliases. + assert.equal(lookupModelSpec('3950', 'COHU')?.id, 'cohu-3950-iview'); + assert.equal( + lookupModelSpec('Q6135-LE', 'Axis Communications AB')?.id, + 'axis-q6135-le', + ); + assert.equal( + lookupModelSpec('XNP-6400RW', 'Hanwha Vision')?.id, + 'hanwha-xnp-6400rw', + ); + assert.equal( + lookupModelSpec('XNP-6400RW', 'Wisenet')?.id, + 'hanwha-xnp-6400rw', + ); +}); + +test('King County snapshot model strings still resolve', () => { + // Model/Manufacturer pairs as published in the King County feed. + assert.equal(lookupModelSpec('3950', 'Cohu')?.id, 'cohu-3950-iview'); + assert.equal(lookupModelSpec('3965', 'Cohu')?.id, 'cohu-3960'); + assert.equal(lookupModelSpec('3925 Dome', 'Cohu')?.id, 'cohu-3920-idome'); + assert.equal(lookupModelSpec('Q6135-LE', 'Axis')?.id, 'axis-q6135-le'); + assert.equal( + lookupModelSpec('AUTODOME IP 5000i', 'BOSCH')?.id, + 'bosch-ndp-5502-30', + ); +}); + +test('catalog never overwrites an explicit FOV or a curated pose', () => { + const [explicit, curated, cohu, pelco] = + browserCatalog().buildCatalogFromSources([ + { id: 'explicit', lat: 47.7, lon: -122, fovDeg: 44, model: 'F4105-LRE' }, + { + id: 'curated', + lat: 47.7, + lon: -122, + model: 'F4105-LRE', + poseSource: 'curated', + }, + { id: 'cohu', lat: 47.7, lon: -122, model: '3950', manufacturer: 'Cohu' }, + { + id: 'pelco', + lat: 47.7, + lon: -122, + model: '3950', + manufacturer: 'Pelco', + }, + ]); + assert.equal(explicit.fovDeg, 44); + assert.equal(explicit.fovSource, undefined); + assert.equal(explicit.spec.model, 'F4105-LRE'); + assert.equal(curated.fovDeg, 74); + assert.equal(curated.fovSource, undefined); + // The source's manufacturer reaches the lookup. + assert.equal(cohu.spec?.id, 'cohu-3950-iview'); + assert.equal(pelco.spec, undefined); +}); + +test('the DATASHEET label drops once calibration moves the FOV', () => { + const camera = { fovSource: 'datasheet', fovDeg: 110 }; + assert.equal(isDatasheetFov(camera), true); + assert.equal(isDatasheetFov({ ...camera, calibration: { fovDeg: 0 } }), true); + assert.equal( + isDatasheetFov({ ...camera, calibration: { fovDeg: 6 } }), + false, + ); + assert.equal(isDatasheetFov({ fovDeg: 74 }), false); + assert.equal(isDatasheetFov(null), false); +}); + test('specSummary renders the HUD line and degrades by omission', () => { assert.equal( specSummary(lookupModelSpec('Q6135-LE')), diff --git a/src/layers/cctv/presentation.js b/src/layers/cctv/presentation.js index efa18e462..34c7dbcc9 100644 --- a/src/layers/cctv/presentation.js +++ b/src/layers/cctv/presentation.js @@ -1,7 +1,11 @@ import { CCTV_AMBIENT_CARD_MAX } from '../../data/cctvLod.js'; import { ACTIVE_FRAME_REFRESH_MS, IDLE_FRAME_REFRESH_MS } from './policy.js'; import { headingHudToken, isHeadingEstimated } from './headingConfidence.js'; -import { specSummary, specFovCapabilityToken } from './modelSpecs.js'; +import { + specSummary, + specFovCapabilityToken, + isDatasheetFov, +} from './modelSpecs.js'; export function createPresentation({ state: layerState, @@ -40,8 +44,9 @@ export function createPresentation({ // A synthetic bearing (headingConfidence 'low', no human calibration) // is tagged so a hashed guess never reads as a surveyed facing (#639). headingHudToken(active.camera), - // Identified hardware labels its FOV as datasheet-sourced (catalog.js). - `FOV ${Math.round(active.camera.fovDeg)}°${active.camera.fovSource === 'datasheet' ? ' (DATASHEET)' : ''}`, + // Identified hardware labels its FOV as datasheet-sourced (catalog.js) + // only while no manual calibration offset has moved it. + `FOV ${Math.round(active.camera.fovDeg)}°${isDatasheetFov(active.camera) ? ' (DATASHEET)' : ''}`, // A PTZ/varifocal datasheet range is what the lens can do, not its // current zoom, so it is shown beside the estimate, never as it. specFovCapabilityToken(active.camera), @@ -96,7 +101,11 @@ export function createPresentation({ pitchDeg: camera.pitchDeg, fovDeg: camera.fovDeg, // Identified-hardware enrichment passthrough (catalog.js / modelSpecs.js). - fovSource: camera.fovSource || 'estimated', + fovSource: isDatasheetFov(camera) + ? 'datasheet' + : camera.fovSource === 'datasheet' + ? 'calibrated' + : camera.fovSource || 'estimated', specFovMinDeg: camera.specFovMinDeg ?? null, specFovMaxDeg: camera.specFovMaxDeg ?? null, spec: camera.spec || null, From cf63278126a512c7e2761dd9ab38913bd126b291 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 15 Oct 2026 17:12:35 -0700 Subject: [PATCH 92/95] Take TTC vehicle mode from the fleet number before the route Replacement buses on 5xx streetcar routes were drawn as streetcars, and Flexity streetcars reporting no route were drawn as buses. A feed can now supply vehicleMode(id); TTC maps 4400-4699 to streetcars and every other fleet number to buses, falling back to the route otherwise. --- CHANGELOG.md | 4 +- server/providers/transitHistory.test.mjs | 23 ++++++++++++ src/data/transitFeeds.js | 47 ++++++++++++++++++++---- src/data/transitFeeds.test.mjs | 16 ++++++++ src/layers/transit/ingestion.js | 8 +++- src/sources/transitHistoryStore.js | 6 ++- 6 files changed, 93 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ca1ca0de..2e7478531 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,9 @@ YRT/Viva, Guelph, Burlington, Barrie and Kingston, each carrying its own licence quote and attribution. A feed can now name its own modes, so Toronto's trams read "Streetcar" while Helsinki's stay "Tram", and a feed whose route ids - are internal keys can translate them to the number a rider would say. + are internal keys can translate them to the number a rider would say. TTC + vehicles take their mode from the fleet number first, so replacement buses on + streetcar routes read as buses and out-of-service streetcars stay streetcars. - Cyclone advisories no longer go dark while NHC publishes an advisory ahead of its nominal time: advisory and position times up to 90 minutes ahead of diff --git a/server/providers/transitHistory.test.mjs b/server/providers/transitHistory.test.mjs index cb06c182a..551fee7a7 100644 --- a/server/providers/transitHistory.test.mjs +++ b/server/providers/transitHistory.test.mjs @@ -179,3 +179,26 @@ test('snapshot capacity pressure preserves members not yet visited and discloses assert.equal(recreated.fixes.length, 1); assert.equal(recreated.truncated, true); }); +test('a feed fleet-number mode outranks its route mode in the epoch context', (t) => { + const h = history(t); + const ttc = { + id: 'ttc', + historyRetention: true, + defaultMode: 'bus', + routeMode: (route) => (/^5\d\d$/.test(route ?? '') ? 'tram' : 'bus'), + vehicleMode: (id) => + /^4[4-6]\d\d$/.test(id) ? 'tram' : /^\d{4}$/.test(id) ? 'bus' : 'unknown', + }; + h.ingest( + ttc, + [ + row(100000, '8515', { routeId: '506' }), + row(100000, '4507', { routeId: null }), + row(100000, 'x1', { routeId: '501' }), + ], + 100000, + ); + assert.equal(h.get('ttc', '8515', 100000).epochs[0].mode, 'bus'); + assert.equal(h.get('ttc', '4507', 100000).epochs[0].mode, 'tram'); + assert.equal(h.get('ttc', 'x1', 100000).epochs[0].mode, 'tram'); +}); diff --git a/src/data/transitFeeds.js b/src/data/transitFeeds.js index cdf4b030b..dfd3d6b18 100644 --- a/src/data/transitFeeds.js +++ b/src/data/transitFeeds.js @@ -130,6 +130,20 @@ function ttcRouteMode(routeId) { return 'bus'; } +/** + * The fleet number settles TTC's mode before the route does: buses replace + * streetcars on 5xx routes during track work, and streetcars running out of + * service report no route at all. Flexity streetcars are numbered 4400–4699; + * every other TTC number is a bus. + * @param {string|null} vehicleId GTFS-RT vehicle id (the fleet number). + * @returns {string} + */ +function ttcVehicleMode(vehicleId) { + if (!/^\d{4}$/.test(String(vehicleId ?? ''))) return 'unknown'; + const fleet = Number(vehicleId); + return fleet >= 4400 && fleet < 4700 ? 'tram' : 'bus'; +} + /** * Guelph Transit's realtime feed publishes the internal database key of a * route (`2991`), not the number on the front of the bus (`1`). Without this @@ -541,6 +555,7 @@ export const TRANSIT_FEED_REGISTRY = Object.freeze([ tram: Object.freeze({ word: 'Streetcar', abbr: 'STREETCAR' }), }), routeMode: ttcRouteMode, + vehicleMode: ttcVehicleMode, }), Object.freeze({ id: 'miway-mississauga', @@ -795,14 +810,32 @@ export function transitFeedsInRange(lat, lon, slackKm = 0) { } /** - * Mode for a vehicle: the feed's route hint when it has one, else its default. + * The feed's own hint for a vehicle: its fleet number when the feed maps + * those (`vehicleMode`), else its route (`routeMode`). Null when neither knows. + * @param {object} feed Registry entry. + * @param {string|null} routeId + * @param {string|null} vehicleId + * @returns {string|null} + */ +function hintedMode(feed, routeId, vehicleId) { + const byVehicle = + typeof feed?.vehicleMode === 'function' + ? feed.vehicleMode(vehicleId) + : null; + if (byVehicle && byVehicle !== 'unknown') return byVehicle; + return typeof feed?.routeMode === 'function' ? feed.routeMode(routeId) : null; +} + +/** + * Mode for a vehicle: the feed's fleet or route hint when it has one, else + * its default. * @param {object} feed Registry entry. * @param {string|null} routeId GTFS route_id from the vehicle's trip. + * @param {string|null} [vehicleId] GTFS-RT vehicle id. * @returns {string} One of TRANSIT_MODES. */ -export function transitModeFor(feed, routeId) { - const hinted = - typeof feed?.routeMode === 'function' ? feed.routeMode(routeId) : null; +export function transitModeFor(feed, routeId, vehicleId = null) { + const hinted = hintedMode(feed, routeId, vehicleId); const mode = hinted && hinted !== 'unknown' ? hinted : feed?.defaultMode || 'unknown'; return TRANSIT_MODES.includes(mode) ? mode : 'unknown'; @@ -815,11 +848,11 @@ export function transitModeFor(feed, routeId) { * not hold a defaulted vehicle to a bus's limits. * @param {object} feed Registry entry. * @param {string|null} routeId + * @param {string|null} [vehicleId] * @returns {boolean} */ -export function transitModeResolved(feed, routeId) { - const hinted = - typeof feed?.routeMode === 'function' ? feed.routeMode(routeId) : null; +export function transitModeResolved(feed, routeId, vehicleId = null) { + const hinted = hintedMode(feed, routeId, vehicleId); return ( Boolean(hinted) && hinted !== 'unknown' && TRANSIT_MODES.includes(hinted) ); diff --git a/src/data/transitFeeds.test.mjs b/src/data/transitFeeds.test.mjs index 22281e46d..fd990af45 100644 --- a/src/data/transitFeeds.test.mjs +++ b/src/data/transitFeeds.test.mjs @@ -279,6 +279,22 @@ test('TTC route numbers tell a streetcar from a bus', () => { ); }); +test('a TTC fleet number outranks the route', () => { + // Buses replace streetcars on 5xx routes during track work, and streetcars + // out of service report no route; the Flexity fleet is 4400–4699. + const ttc = getTransitFeed('ttc-toronto'); + assert.equal(transitModeFor(ttc, '506', '8515'), 'bus'); + assert.equal(transitModeFor(ttc, '512', '3101'), 'bus'); + assert.equal(transitModeFor(ttc, null, '4507'), 'tram'); + assert.equal(transitModeResolved(ttc, null, '4507'), true); + assert.equal(transitModeFor(ttc, '504', '4400'), 'tram'); + assert.equal(transitModeFor(ttc, '29', '4699'), 'tram'); + assert.equal(transitModeFor(ttc, '501', '4700'), 'bus'); + // A vehicle id that is not a fleet number leaves the route to decide. + assert.equal(transitModeFor(ttc, '501', 'abc'), 'tram'); + assert.equal(transitModeFor(ttc, '501'), 'tram'); +}); + test('the TTC row records that BusTime carries no subway', () => { const sources = readFileSync( new URL('../../DATA_SOURCES.md', import.meta.url), diff --git a/src/layers/transit/ingestion.js b/src/layers/transit/ingestion.js index cb610267e..be8833270 100644 --- a/src/layers/transit/ingestion.js +++ b/src/layers/transit/ingestion.js @@ -221,8 +221,12 @@ export function createIngestion({ state, services, parts, source }) { for (const record of records) { if (isStaleVehicleFix(record, now, fetchedAt)) continue; const key = transitVehicleKey(feed.id, record.id); - const mode = transitModeFor(feed, record.routeId); - const modeInferred = !transitModeResolved(feed, record.routeId); + const mode = transitModeFor(feed, record.routeId, record.id); + const modeInferred = !transitModeResolved( + feed, + record.routeId, + record.id, + ); // A stamp ahead of the fetch is a clock fault; the fetch time bounds it. const stamped = vehicleReportTimeMs(record, fetchedAt); const reportAt = diff --git a/src/sources/transitHistoryStore.js b/src/sources/transitHistoryStore.js index 36440975c..9fe29f59d 100644 --- a/src/sources/transitHistoryStore.js +++ b/src/sources/transitHistoryStore.js @@ -328,8 +328,12 @@ export function createTransitHistory({ t > now + 10000 ) continue; + const byVehicle = feed.vehicleMode?.(record.id); const mode = - feed.routeMode?.(record.routeId) || feed.defaultMode || 'unknown'; + (byVehicle !== 'unknown' && byVehicle) || + feed.routeMode?.(record.routeId) || + feed.defaultMode || + 'unknown'; const context = contextOf(record, mode); if (!context) continue; const key = `${feed.id}\u0000${record.id}`; From 194595e1b6fd97ad312d51f36397c0ffbdd5ae30 Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:54:58 +0200 Subject: [PATCH 93/95] feat(bikeshare): add Barcelona's Bicing and AMBici - Bicing (543 stations, PBSC) and AMBici (237 e-bike stations across the metropolitan area, nextbike) join the GBFS registry from the operators' keyless GBFS 2.3 feeds, both listed in MobilityData's systems catalogue. - gbfs.nextbike.net joins the proxy allowlist as an exact host. - Credits, DATA_SOURCES, CURRENT-STATE and CHANGELOG updated; a test pins each system's proxy reachability and the range check for its outermost stations. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++ DATA_SOURCES.md | 2 + docs/CURRENT-STATE.md | 2 +- src/data/dataCredits.js | 3 +- src/data/gbfsSource.js | 1 + .../bikeshare/barcelonaSystems.test.mjs | 84 +++++++++++++++++++ src/layers/bikeshare/registry.js | 26 ++++++ 7 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 src/layers/bikeshare/barcelonaSystems.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b371af53..5f6cd5452 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +- Add Barcelona's bike share to the GBFS registry: Bicing (543 stations, the + city's PBSC system) and AMBici (237 e-bike stations across the metropolitan + area, run by nextbike). Both are keyless GBFS 2.3 feeds listed in + MobilityData's systems catalogue; AMBici's is CC0. `gbfs.nextbike.net` joins + the proxy allowlist. These are the layer's first stations in Spain. + - Add nine Ontario transit feeds: TTC, MiWay, Hamilton HSR, Durham Region, YRT/Viva, Guelph, Burlington, Barrie and Kingston, each carrying its own licence quote and attribution. A feed can now name its own modes, so Toronto's diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 4aa9b6317..577cdc413 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -68,6 +68,8 @@ How to read this: | **GBFS (Lyft / BCycle / Social Bicycles / PBSC)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **Bike Share Toronto** (`toronto.publicbikesystem.net`) | Bikeshare station availability, Toronto | Operator GBFS feed. No licence is published: the City's open-data package declares none and the feed has no `license_url`. Attribution is given to the operator. | "Bike Share Toronto (Toronto Parking Authority)" | | **Bike Itaú (Tembici)** (`*.publicbikesystem.net`, seven Brazilian cities) | Bikeshare station availability: São Paulo, Rio de Janeiro, Salvador, Recife, Porto Alegre, Brasília and Curitiba | Operator GBFS feeds. No licence is published: the feeds' `system_information.json` carries no `license_url`, operator or attribution fields. Attribution is given to the operator. | "Bike Itaú (Tembici)" | +| **Bicing** (`barcelona.publicbikesystem.net`) | Bikeshare station availability, Barcelona | Operator GBFS feed (PBSC), keyless. The feed publishes no licence; the City's own open-data copy of the same station data is CC BY 4.0 but needs a token. Attribution is given to the operator. | "Bicing, Barcelona City Council (B:SM)" | +| **AMBici** (nextbike, `gbfs.nextbike.net`) | Bikeshare station availability, Barcelona metropolitan area (e-bikes) | Operator GBFS feed, keyless. `system_information.license_id` is **CC0-1.0**; no attribution required, given anyway. | "AMBici, Àrea Metropolitana de Barcelona, operated by nextbike" | | **Oslo / Bergen / Trondheim Bysykkel** (Urban Sharing, `gbfs.urbansharing.com`) | Bikeshare availability, Oslo, Bergen and Trondheim city bikes. Keyless GBFS; the proxy sends the `Client-Identifier` header the operators ask for | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED ([Oslo](https://oslobysykkel.no/en/open-data/realtime), [Bergen](https://bergenbysykkel.no/en/apne-data/sanntid), [Trondheim](https://trondheimbysykkel.no/en/open-data/realtime)) | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel" | | **Kolumbus / Farte Bysykkel** (Entur Mobility, `api.entur.io/mobility/v2/gbfs`) | Bikeshare availability, Kolumbus Bysykkel (Stavanger/Nord-Jæren) and Farte Bysykkel (Skien/Porsgrunn). Keyless GBFS; the proxy sends the `ET-Client-Name` header Entur requires and is limited to Entur's GBFS paths | [NLOD](https://data.norge.no/nlod/en/2.0) per the [Entur Mobility API docs](https://developer.entur.org/pages-mobility-docs-mobility-v2) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Entur (Kolumbus Bysykkel, Farte Bysykkel)" | | **JCDecaux Cyclocity** (`api.cyclocity.fr`) | Bikeshare availability, Bysykkel Lillestrøm. Keyless GBFS; the proxy is limited to JCDecaux contract paths | [Licence Ouverte / Etalab](https://developer.jcdecaux.com/files/Open-Licence-fr.pdf) (the feed's `license_url`) — attribution REQUIRED | "Bysykkel Lillestrøm: JCDecaux, Licence Ouverte" | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 6373d7255..c08de52d5 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3265,7 +3265,7 @@ its criteria cannot be silently ignored. | CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + King County (WA) Open Data + WSDOT (WA) Open Data + NYC DOT TMC (opt-in) + City of Toronto RESCU + Catalonia (Servei Català de Trànsit) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto and Bike Itaú (Brazil) + Urban Sharing, Entur, JCDecaux NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Social Bicycles + PBSC incl. Bike Share Toronto, Bike Itaú (Brazil) and Bicing (Barcelona) + Urban Sharing, Entur, JCDecaux NO + nextbike (AMBici)) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | | Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index d6c265ddc..ea3f2ad09 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -304,7 +304,8 @@ export const DATA_CREDITS = [ key: 'gbfs', html: 'Bikeshare availability: GBFS operator feeds ' + - '(e.g. Austin BCycle; Bike Share Toronto, operated by the Toronto Parking Authority; Bike Itaú, operated by Tembici)', + '(e.g. Austin BCycle; Bike Share Toronto, operated by the Toronto Parking Authority; Bike Itaú, operated by Tembici; ' + + 'Bicing, Barcelona City Council (B:SM); AMBici, Àrea Metropolitana de Barcelona, operated by nextbike)', }, { key: 'urbansharing-gbfs', diff --git a/src/data/gbfsSource.js b/src/data/gbfsSource.js index 57228554e..cd1042114 100644 --- a/src/data/gbfsSource.js +++ b/src/data/gbfsSource.js @@ -8,6 +8,7 @@ const GBFS_ALLOWED_HOSTS = new Set([ 'austin.publicbikesystem.net', 'hon.publicbikesystem.net', 'chat.publicbikesystem.net', + 'gbfs.nextbike.net', 'gbfs.urbansharing.com', 'api.entur.io', 'api.cyclocity.fr', diff --git a/src/layers/bikeshare/barcelonaSystems.test.mjs b/src/layers/bikeshare/barcelonaSystems.test.mjs new file mode 100644 index 000000000..992c0431c --- /dev/null +++ b/src/layers/bikeshare/barcelonaSystems.test.mjs @@ -0,0 +1,84 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { CITY_BY_ID } from './registry.js'; +import { createModel } from './model.js'; +import { createViewport } from './viewport.js'; +import { isAllowedGbfsHost, isAllowedGbfsPath } from '../../data/gbfsSource.js'; + +// Each Barcelona system's host, and the stations farthest from its registered +// centre in the live feed (checked 2026-10-16). AMBici spans the metropolitan +// area, so it lists its south-western and north-eastern ends. +const SYSTEMS = { + 'barcelona-bicing': { + host: 'barcelona.publicbikesystem.net', + provider: 'Bicing', + stations: [ + { name: 'AV. RASOS DE PEGUERA, 10', lat: 41.462095, lon: 2.178959 }, + { name: 'C/ 60, NÚMERO 25', lat: 41.3467746, lon: 2.1436235 }, + ], + }, + 'barcelona-ambici': { + host: 'gbfs.nextbike.net', + provider: 'AMBici', + stations: [ + { name: 'Poal', lat: 41.268865, lon: 1.943849 }, + { name: 'Platja del Manresà', lat: 41.460101, lon: 2.267271 }, + { name: 'Can Ruti', lat: 41.482205, lon: 2.237607 }, + ], + }, +}; + +// The production range check, with no viewer or fetches behind it. +const parts = { + model: createModel({ state: {}, services: {}, parts: {}, source: null }), +}; +const { computeInRangeCities } = createViewport({ + state: {}, + services: { render: { governorRequestRender() {} } }, + parts, + source: null, +}); + +test('Bicing and AMBici are registered and readable through the proxy', () => { + for (const [id, expected] of Object.entries(SYSTEMS)) { + const system = CITY_BY_ID.get(id); + assert.ok(system, `${id} is registered`); + assert.equal(system.provider, expected.provider); + assert.deepEqual(system.hosts, [expected.host]); + for (const feed of [ + system.stationInformationUrl, + system.stationStatusUrl, + ]) { + const url = new URL(feed); + assert.ok(isAllowedGbfsHost(url.hostname), `${id} host ${url.hostname}`); + assert.ok( + isAllowedGbfsPath(url.pathname, url.hostname), + `${id} path ${url.pathname}`, + ); + } + } +}); + +test('a camera over each outermost station activates its system', () => { + for (const [id, expected] of Object.entries(SYSTEMS)) { + for (const station of expected.stations) { + assert.ok( + computeInRangeCities({ lat: station.lat, lon: station.lon }).has(id), + `${station.name} activates ${id}`, + ); + } + } +}); + +test('a camera over central Barcelona activates both systems', () => { + // Plaça de Catalunya. + const active = computeInRangeCities({ lat: 41.387, lon: 2.17 }); + assert.ok(active.has('barcelona-bicing')); + assert.ok(active.has('barcelona-ambici')); +}); + +test('the nextbike allowance is that one host, not its domain', () => { + assert.equal(isAllowedGbfsHost('gbfs.nextbike.net'), true); + assert.equal(isAllowedGbfsHost('api.nextbike.net'), false); + assert.equal(isAllowedGbfsHost('gbfs.nextbike.net.example'), false); +}); diff --git a/src/layers/bikeshare/registry.js b/src/layers/bikeshare/registry.js index 916d50123..3d505020d 100644 --- a/src/layers/bikeshare/registry.js +++ b/src/layers/bikeshare/registry.js @@ -554,6 +554,32 @@ const RAW_GBFS_CITY_REGISTRY = [ 'https://api.cyclocity.fr/contracts/lillestrom/gbfs/station_status.json', provider: 'Bysykkel Lillestrøm', }, + { + id: 'barcelona-bicing', + city: 'Barcelona, Spain', + centerLat: 41.3926, + centerLon: 2.164, + loadRadiusKm: 40, + stationInformationUrl: + 'https://barcelona.publicbikesystem.net/customer/gbfs/v2/en/station_information.json', + stationStatusUrl: + 'https://barcelona.publicbikesystem.net/customer/gbfs/v2/en/station_status.json', + provider: 'Bicing', + }, + // AMBici covers the metropolitan municipalities around Barcelona, from + // Castelldefels to Badalona, and overlaps Bicing only at the city edge. + { + id: 'barcelona-ambici', + city: 'Barcelona metropolitan area, Spain', + centerLat: 41.37, + centerLon: 2.07, + loadRadiusKm: 40, + stationInformationUrl: + 'https://gbfs.nextbike.net/maps/gbfs/v2/nextbike_bs/en/station_information.json', + stationStatusUrl: + 'https://gbfs.nextbike.net/maps/gbfs/v2/nextbike_bs/en/station_status.json', + provider: 'AMBici', + }, ]; function normalizeRegistryEntry(entry) { const id = String(entry?.id || '') From dec79b6fcff1214a75b68d750b63fa956af55604 Mon Sep 17 00:00:00 2001 From: ulihhhh <54112476+ulihhhh@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:20:52 +0200 Subject: [PATCH 94/95] test(bikeshare): pin the Barcelona load radii to their outermost stations The map range check floors every radius at 100 km, so the station test passed with any loadRadiusKm up to that. get_bike_share uses the radius as registered, so assert each system's radius reaches the stations it lists. Co-Authored-By: Claude Opus 5.5 --- .../bikeshare/barcelonaSystems.test.mjs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/layers/bikeshare/barcelonaSystems.test.mjs b/src/layers/bikeshare/barcelonaSystems.test.mjs index 992c0431c..cdca11307 100644 --- a/src/layers/bikeshare/barcelonaSystems.test.mjs +++ b/src/layers/bikeshare/barcelonaSystems.test.mjs @@ -70,6 +70,26 @@ test('a camera over each outermost station activates its system', () => { } }); +test('each load radius reaches its outermost stations', () => { + // The range check floors every radius at 100 km, so the test above passes + // with any loadRadiusKm; get_bike_share uses the radius as registered. + for (const [id, expected] of Object.entries(SYSTEMS)) { + const system = CITY_BY_ID.get(id); + for (const station of expected.stations) { + const distance = parts.model.haversineKm( + system.centerLat, + system.centerLon, + station.lat, + station.lon, + ); + assert.ok( + distance <= system.loadRadiusKm, + `${station.name} is ${distance.toFixed(1)} km from the ${id} centre`, + ); + } + } +}); + test('a camera over central Barcelona activates both systems', () => { // Plaça de Catalunya. const active = computeInRangeCities({ lat: 41.387, lon: 2.17 }); From 7a38727b8551b8dca9e4d7cecd7106a94fef95a3 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 16 Oct 2026 11:43:48 +0100 Subject: [PATCH 95/95] fix(bikeshare): drop stations outside a city's load radius MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bogotá's and Santiago's PBSC feeds list depot stations in São Paulo and Madrid as installed and renting, so activating either city drew live docks thousands of kilometres away and counted them in the layer total. Station information is now limited to the city's registered load radius; every other registered feed already sits inside its radius (checked live 2026-10-16), so nothing else changes. Adds a Bogotá/Santiago registry test matching the Brazilian and Barcelona ones: proxy allowlist, outermost-station activation and radius reach, and the depot filter through the real ingestion path. Co-Authored-By: Claude Opus 5.5 --- src/layers/bikeshare/ingestion.js | 5 +- src/layers/bikeshare/latamPbscCities.test.mjs | 174 ++++++++++++++++++ src/layers/bikeshare/model.js | 25 +++ 3 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 src/layers/bikeshare/latamPbscCities.test.mjs diff --git a/src/layers/bikeshare/ingestion.js b/src/layers/bikeshare/ingestion.js index f2127e53e..dbd4d063e 100644 --- a/src/layers/bikeshare/ingestion.js +++ b/src/layers/bikeshare/ingestion.js @@ -97,7 +97,10 @@ export function createIngestion({ const payload = await fetchGbfsJson(city.stationInformationUrl, { signal: controller.signal, }); - const stationMap = parts.model.parseStationInformation(payload); + const stationMap = parts.model.stationsWithinCity( + city, + parts.model.parseStationInformation(payload), + ); if (stationMap.size === 0) { throw new Error(`No station information for ${city.city}`); } diff --git a/src/layers/bikeshare/latamPbscCities.test.mjs b/src/layers/bikeshare/latamPbscCities.test.mjs new file mode 100644 index 000000000..4eda1ad8c --- /dev/null +++ b/src/layers/bikeshare/latamPbscCities.test.mjs @@ -0,0 +1,174 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { CITY_BY_ID } from './registry.js'; +import { createIngestion } from './ingestion.js'; +import { createModel } from './model.js'; +import { createViewport } from './viewport.js'; +import { isAllowedGbfsHost, isAllowedGbfsPath } from '../../data/gbfsSource.js'; + +// Each system's PBSC host, the station farthest from its registered centre, +// and the depots its live feed lists on other continents (checked 2026-10-16). +const CITIES = { + 'bogota-tembici': { + host: 'bogota.publicbikesystem.net', + provider: 'Tembici Bogotá', + stations: [ + { name: '067 - KR 7B Bis con CL 152A', lat: 4.727729, lon: -74.026271 }, + ], + depots: [ + { + station_id: '316', + name: 'Calle 13 - 316', + lat: -23.4008649, + lon: -46.3972181, + }, + ], + }, + 'santiago-bike-itau': { + host: 'santiago.publicbikesystem.net', + provider: 'Bike Itaú Santiago', + stations: [ + { name: 'LC077 - Mall Sport', lat: -33.3707813, lon: -70.5073122 }, + ], + depots: [ + { + station_id: '459', + name: 'Galpon - 459', + lat: 40.4167747, + lon: -3.7037899, + }, + { + station_id: '440', + name: 'Galpon - 440', + lat: -23.5690975, + lon: -46.697366, + }, + ], + }, +}; + +// The production range check, with no viewer or fetches behind it. +const parts = { + model: createModel({ state: {}, services: {}, parts: {}, source: null }), +}; +const { computeInRangeCities } = createViewport({ + state: {}, + services: { render: { governorRequestRender() {} } }, + parts, + source: null, +}); + +test('Bogotá and Santiago are registered and readable through the proxy', () => { + for (const [id, expected] of Object.entries(CITIES)) { + const city = CITY_BY_ID.get(id); + assert.ok(city, `${id} is registered`); + assert.equal(city.provider, expected.provider); + assert.deepEqual(city.hosts, [expected.host]); + for (const feed of [city.stationInformationUrl, city.stationStatusUrl]) { + const url = new URL(feed); + assert.ok(isAllowedGbfsHost(url.hostname), `${id} host ${url.hostname}`); + assert.ok(isAllowedGbfsPath(url.pathname), `${id} path ${url.pathname}`); + } + } +}); + +test('a camera over each outermost station activates its system', () => { + for (const [id, expected] of Object.entries(CITIES)) { + for (const station of expected.stations) { + assert.ok( + computeInRangeCities({ lat: station.lat, lon: station.lon }).has(id), + `${station.name} activates ${id}`, + ); + } + } +}); + +test('each load radius reaches its outermost stations', () => { + // The range check floors every radius at 100 km; the station filter and + // get_bike_share use the radius as registered. + for (const [id, expected] of Object.entries(CITIES)) { + const city = CITY_BY_ID.get(id); + for (const station of expected.stations) { + const distance = parts.model.haversineKm( + city.centerLat, + city.centerLon, + station.lat, + station.lon, + ); + assert.ok( + distance <= city.loadRadiusKm, + `${station.name} is ${distance.toFixed(1)} km from the ${id} centre`, + ); + } + } +}); + +test('stations a feed lists on another continent are not loaded', async () => { + for (const [id, expected] of Object.entries(CITIES)) { + const city = CITY_BY_ID.get(id); + const outermost = expected.stations[0]; + const payload = { + data: { + stations: [ + { + station_id: 'local', + name: outermost.name, + lat: outermost.lat, + lon: outermost.lon, + }, + ...expected.depots, + ], + }, + }; + const state = { + _stationInfoCache: new Map(), + _inFlightInfo: new Map(), + _inFlightStatus: new Map(), + _loadingOps: 0, + _loading: false, + }; + const ingestion = createIngestion({ + state, + services: {}, + parts, + source: { + async getStations(url) { + assert.equal(url, city.stationInformationUrl); + return payload; + }, + }, + }); + + const stations = await ingestion.loadCityStationInfo(id, 1); + assert.deepEqual( + [...stations.keys()], + ['local'], + `${id} keeps only local stations`, + ); + assert.equal(state._stationInfoCache.get(id), stations); + } +}); + +test('a feed with only far-away stations reports no station information', async () => { + const city = CITY_BY_ID.get('santiago-bike-itau'); + const ingestion = createIngestion({ + state: { + _stationInfoCache: new Map(), + _inFlightInfo: new Map(), + _inFlightStatus: new Map(), + _loadingOps: 0, + _loading: false, + }, + services: {}, + parts, + source: { + async getStations() { + return { data: { stations: CITIES['santiago-bike-itau'].depots } }; + }, + }, + }); + await assert.rejects( + ingestion.loadCityStationInfo('santiago-bike-itau', 1), + new RegExp(`No station information for ${city.city}`), + ); +}); diff --git a/src/layers/bikeshare/model.js b/src/layers/bikeshare/model.js index f12148dba..d501d073c 100644 --- a/src/layers/bikeshare/model.js +++ b/src/layers/bikeshare/model.js @@ -39,11 +39,36 @@ export function createModel({ state: layerState, services, parts, source }) { return 6371 * 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); } + /** + * Drop stations outside a city's load radius. Some operators list depots on + * other continents as in-service stations in a city's feed (Bogotá's and + * Santiago's PBSC feeds carry warehouses in São Paulo and Madrid), which + * would otherwise draw as live docks thousands of kilometres away. + * @param {{ centerLat: number, centerLon: number, loadRadiusKm: number }} city + * @param {Map} stationMap - Parsed station info map. + * @returns {Map} Stations within the city's load radius. + */ + + function stationsWithinCity(city, stationMap) { + const kept = new Map(); + for (const [stationId, station] of stationMap) { + const distance = haversineKm( + city.centerLat, + city.centerLon, + station.lat, + station.lon, + ); + if (distance <= city.loadRadiusKm) kept.set(stationId, station); + } + return kept; + } + return { normalizeGbfsBool, toNonNegativeInteger, stationKey, haversineKm, + stationsWithinCity, extractStationsArray, parseStationInformation, parseStationStatus,