diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..174f7990a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,24 @@ +# Keep keys and local state out of the build context: compose bind-mounts the +# checkout at run time, so the image never needs them. +.git +node_modules +dist +.env +.env.* +!.env.example +**/ENVIRONMENT +pinokio/ENVIRONMENT +pinokio/.installed +.gev-cache +.gev-logs +qa-artifacts +qa-shots +screenshots +output +3d-models +.gstack +*.log +.DS_Store +# AppleDouble sidecars from a macOS `tar` copy; `npm test` would otherwise +# collect `._*.test.mjs` as test files. +._* diff --git a/.env.example b/.env.example index 2b7ff95ab..a2f8afa24 100644 --- a/.env.example +++ b/.env.example @@ -18,7 +18,8 @@ # (they're used client-side) and WILL be visible in devtools. Restrict/scope them rather than trying to hide # them (see SECURITY.md). All other keys below stay server-side. -# Optional: direct Google Photorealistic 3D Tiles and GEV place search. +# Optional: direct Google Photorealistic 3D Tiles, GEV place search and Street +# View (Street View also needs the Maps JavaScript API enabled for the key). # Without it, a Cesium ion token can still load ion-hosted Google 3D; with # neither credential, the app starts on keyless Esri World Imagery with OSM # available in the map tray and as the automatic provider-failure fallback. @@ -55,8 +56,14 @@ GOOGLE_MAPS_SERVER_API_KEY= # CLIENT-EXPOSED — use a public assets:read token with URL restrictions. CESIUM_ION_TOKEN= -# Optional: OpenAI Realtime voice control. Do not prefix with VITE_. +# Optional: OpenAI Realtime voice control. API key is the default cloud auth. +# Provider Settings can instead opt voice into the signed-in local +# ChatGPT/Codex OAuth session; the browser still receives only a short-lived +# Realtime client secret. Do not prefix with VITE_. OPENAI_API_KEY= +# Optional: forbid API-key voice sessions. Select ChatGPT OAuth in Provider +# Settings; missing or expired Codex sign-in then fails closed. +# GEV_PREFER_CODEX_OAUTH=true OPENAI_REALTIME_MODEL=gpt-realtime-2 # Cheaper voice tier, used when the GEV MIC panel's MINI toggle is on. # Model ids and prices are external facts that drift — if OpenAI moves this id, @@ -89,6 +96,13 @@ OPENAI_HUD_SUMMARY_MODEL=gpt-5-nano # spend protection set a provider-side budget: OpenAI platform -> Settings -> # Limits (usage limits). # GEV_RATELIMIT_OPENAI_PER_MIN=30 # the default; uncomment only to change it +# Optional: exact socket addresses (comma-separated, no wildcards or prefixes) +# that Provider Settings treats as this machine in addition to loopback. Leave +# unset. The Docker launcher sets it to the container's gateway at start, +# because that is where Docker delivers your own browser from; it is safe only +# while the port is published on 127.0.0.1. Read SECURITY.md before setting it +# by hand. +# GEV_KEY_SETUP_TRUSTED_PEERS= # Optional: Mapillary imagery for the Street Level layer. Free client token from # https://www.mapillary.com/dashboard/developers (register an app, copy its @@ -205,10 +219,24 @@ AISSTREAM_API_KEY= # CCTV_CALGARY_ENABLED=1 # set 0 to disable the Open Calgary (Alberta) cameras # CCTV_CALGARY_MAX_SOURCES=220 # cap cameras loaded from the Open Calgary catalog # CCTV_CALGARY_ROWS_URL= # override the Open Calgary catalog URL +# CCTV_CATALONIA_ENABLED=1 # set 0 to disable the Servei Català de Trànsit cameras +# CCTV_CATALONIA_MAX_SOURCES=150 # cap cameras loaded from Catalonia (~130 exist today) +# CCTV_KINGCOUNTY_ENABLED=1 # set 0 to disable the King County (Washington) road cameras +# CCTV_KINGCOUNTY_MAX_SOURCES=125 # cap cameras loaded from the King County catalog +# CCTV_WSDOT_ENABLED=1 # set 0 to disable the WSDOT (Washington State) highway cameras +# CCTV_WSDOT_MAX_SOURCES=250 # cap cameras loaded from the WSDOT catalog, nearest Seattle/Spokane first; min 8, max 1200 # CCTV_VEGVESEN_ENABLED=1 # set 0 to disable the Statens vegvesen (Norway) road cameras # CCTV_VEGVESEN_MAX_SOURCES=900 # cap cameras loaded from Statens vegvesen (default loads all ~850 live; max 1000) # CCTV_VEGVESEN_VIDEO=1 # set 0 to show stills instead of live HLS for Statens vegvesen video cameras # CCTV_VEGVESEN_URL= # override the Statens vegvesen catalog URL +# CCTV_NYC_ENABLED= # off by default; set 1 only if you hold a NYC DOT data-sharing agreement (tmcdot@dot.nyc.gov) for the camera feed +# CCTV_NYC_MAX_SOURCES=250 # cap cameras loaded from the NYC TMC catalog (nearest borough cores first) +# CCTV_NYC_CAMERAS_URL= # override the NYC TMC catalog URL +# CCTV_TORONTO_ENABLED=1 # set 0 to disable the City of Toronto RESCU cameras +# CCTV_TORONTO_MAX_SOURCES=336 # cap cameras loaded from Toronto RESCU (clamped to 8–500) +# CCTV_TORONTO_LIST_URL= # override the Toronto RESCU camera list URL +# CCTV_LITHUANIA_ENABLED= # off by default; set 1 to load Via Lietuva / Eismoinfo road cameras (photo licence unconfirmed; see DATA_SOURCES.md) +# CCTV_LITHUANIA_MAX_SOURCES=400 # cap Lithuanian cameras; no API key required # CCTV_MAX_SOURCES=4000 # catalog-wide ceiling, shared round-robin across packs # CCTV_PREFER_AUSTIN=1 # prioritize Austin cameras in the registry # CCTV_FORCE_AUSTIN= # force live default sources when a source-pack JSON is configured diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 375bc9077..491f2468d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,7 +150,7 @@ jobs: run: npm run check:boundaries - name: Run focused onboarding tests - run: node --test src/setupDoctor.test.mjs src/pinokioEnvironment.test.mjs src/pinokioPreflight.test.mjs src/pinokioLauncherContract.test.mjs src/keySetupHardening.test.mjs src/mapStartup.test.mjs src/hudSummaryResponse.test.mjs + run: node --test src/setupDoctor.test.mjs src/pinokioEnvironment.test.mjs src/pinokioPreflight.test.mjs src/pinokioLauncherContract.test.mjs src/keySetupHardening.test.mjs src/mapStartup.test.mjs src/hudSummaryResponse.test.mjs src/codexOauthRealtime.test.mjs - name: Build production bundle run: npm run build diff --git a/CHANGELOG.md b/CHANGELOG.md index e3f5619c1..834650d63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,166 @@ its current zoom, so those cameras keep the estimate and show `SPEC FOV min–max° (CAPABILITY)` beside it (bassem chagra, #452). +- Add WSDOT highway cameras (Washington State) as a keyless CCTV source pack: + WSDOT's official Travel Center camera layer, with frames registered only on + WSDOT's own image host (partner-hosted cameras such as Oregon DOT and City of + Seattle are skipped). Cameras with a single cardinal `CompassDirection` get + a high-confidence heading; `B` (both directions), `O`, and unset directions + use the shared id-hash fallback at low confidence. By default the 250 + cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` sets the + cap (8–1,200) and `CCTV_WSDOT_ENABLED=0` turns the pack off (bassem chagra, + #644). + +- Refuse still-image cameras on the CCTV media relay. `/api/cctv/media` is the + live-feed route, but it also relayed any registered still camera through an + ordinary fetch that followed redirects to any host, passed through whatever + content type came back and had no cap on the body, bypassing every protection + `/api/cctv/frame` applies to the same URL. Only HLS, MP4, WebM and MJPEG + feeds are relayed now; a still camera, or an unknown id, gets a JSON 404 + without an upstream request and is served only through `/api/cctv/frame`. + The client already loaded stills through `/frame`, so nothing it requests + changes. + +- Add Lithuania's public Via Lietuva / Eismoinfo road cameras to CCTV, with published coordinates, on-demand snapshots, attribution and bounded catalog requests. No API key is required. The pack is opt-in (`CCTV_LITHUANIA_ENABLED=1`) because its camera endpoints are the eismoinfo.lt website backend, not a service listed in Via Lietuva's open-data statement, so the licence for the photos is unconfirmed. + +- Add King County road cameras (Washington State) as a keyless CCTV source + pack: one public King County DOT ArcGIS layer, fetched with an explicit field + list so its staff-name columns are never requested, and frames pinned to the + county's own image host and upgraded to HTTPS. WSDOT-owned rows are skipped, + and city-owned cameras the county hosts credit their owner. The layer + publishes no facing, so headings use the shared id-hash fallback at low + confidence. It does publish each camera's hardware, which is served on + `/api/cctv/sources` as `model` and `manufacturer`. `CCTV_KINGCOUNTY_MAX_SOURCES` + sets the cap and `CCTV_KINGCOUNTY_ENABLED=0` turns the pack off + (bassem chagra, #645). + +- Add City of Toronto RESCU traffic cameras as a keyless CCTV source pack: the + ~336 municipal cameras the Ontario 511 pack does not carry — downtown + arterials plus the Gardiner, DVP, Allen and Lake Shore. The list is the + City's own open data feed; frames are built from the documented + `loc####.jpg` convention and a digits-only camera number, so the frame proxy + is pinned to the portal origin by construction. The dataset publishes no + camera facing — its `D1`-`D4` fields describe static 2016 comparison images, + not the live pan-tilt-zoom frame — so headings use the shared id-hash + fallback at low confidence and are corrected with the calibration gizmo. + Carries the required Open Government Licence – Toronto attribution. Disable + with `CCTV_TORONTO_ENABLED=0`. + +- Stop one retired bikeshare feed from reporting the whole layer as failed. + City feeds are fetched independently, but failure was recorded in a single + flag that only a successful *activation* cleared — and an already-loaded city + never activates again. A dead feed re-entering camera range therefore latched + LOAD FAILED permanently over a map still drawing every other city's stations. + Failure is now tracked per city: a city that fails beside others still + drawing marks the layer DEGRADED, polls that all fail over previously drawn + points read STALE without advancing freshness, and LOAD FAILED is reported + only when nothing is left to draw. A city that recovers clears its own + failure, a failure is dropped once its city leaves range or the camera rises + above the activation altitude, and late results from superseded requests are + ignored. + + Repointed the three feeds whose operators had moved: Reddy Bikeshare (Buffalo) + to Social Bicycles, Tugo (Tucson) and Trinity Metro (Fort Worth) to PBSC. + Dropped CoGo (Columbus) and Broward B-cycle (Fort Lauderdale), whose systems + have wound down — CoGo's feed reports no stations and last moved a bike in + March 2025. The three repointed feeds were verified live. + +- Add Bike Share Toronto (1,069 stations) to the bikeshare registry. The system + publishes the same PBSC GBFS v2 feeds as the Austin, Honolulu and Chattanooga + entries, so it needs no new plumbing; a registry test now holds every city's + feed URLs against the proxy host and path allowlists, so a city the proxy + would refuse fails the suite instead of rendering an empty layer. + +- GBFS v3 station names use English translations when available, then the first + usable translation or short name. Empty or malformed names no longer appear as + `[object Object]` or interrupt station loading; legacy string names remain + supported in the globe and bike-share tools. Thanks to + [Kristóf Kékesi](https://github.com/KristofKekesi) (#61). + +- Overpass and military-installation disk caches no longer serve answers from a + previously configured `OVERPASS_UPSTREAMS` instance after it is repointed. + +- A Radio Browser mirror answering 204, 205 or 304, or a status outside + 200–599, no longer crashes the local dev/preview server. Null-body statuses + get an empty body; anything else that cannot become a response fails that + mirror so the next one is tried. + +- Snow flakes sway within a fixed range instead of drifting sideways faster the + longer the app runs; the Wind setting still scales the sway. + +- The right rail ignores panels that other CSS hides, such as during an event + view, so they no longer force one-panel-at-a-time layout or collapse visible + panels. Panels the rail collapses itself are still restored. + +- Coordinate searches refuse empty latitude or longitude components instead of navigating to an unintended location. + +- Bing Aerial and Bing Labels request HTTPS tiles on HTTP localhost through + their ion-backed providers, retaining ion and per-tile attribution without + widening the Content Security Policy. + +- Add New York City traffic cameras from the NYC DOT Traffic Management Center + as an opt-in CCTV pack. NYC DOT requires a data-sharing agreement for the + camera feed, so the pack is off unless `CCTV_NYC_ENABLED=1` is set by an + operator who holds one; otherwise no request reaches NYC DOT. NYC DOT's + requested disclaimer is in the Data attribution popover. The camera list is one keyless JSON request to + `webcams.nyctmc.org/api/cameras`; frames are stills synthesized from each + camera UUID on that same host, so a catalog field cannot steer the proxy + off-host. Offline cameras are dropped. Once enabled, the 250 cameras nearest + the five borough cores load and `CCTV_NYC_MAX_SOURCES` changes the cap. The catalog-wide ceiling is + unchanged; extra cameras share it round-robin with the other packs. + +- Street Level counts unique tiles before ranking so overlapping dateline halves + in polar views do not waste slots in the 16-tile overview budget. + +- Directions no longer shows a truncation notice for a route with exactly 200 + turns, and a roundabout at the 200th turn keeps its exit distance. + +- Weather records keep missing conditions unknown and refuse summaries without a measured temperature. + +- Tool string length validation counts Unicode characters correctly, including characters outside the basic multilingual plane. + +- Director timeline seeks cross the dateline along the short longitude arc. + +- Recent Imagery continues past malformed records in full catalog pages, respects the requested record limit, and reports results as truncated when that limit drops valid records. + +- Regional articles retain unknown publication times when compact source dates do not identify a real calendar instant. + +- Add Detroit (MoGo), Pittsburgh (POGOH), Aspen (WE-cycle) and Kona & Hilo + (HIBike) to the GBFS bikeshare registry, the layer's first stations in + Michigan. + +- Treat explicitly current, complete empty vessel coverage as a successful update, + clearing obsolete contacts while retaining stale or incomplete snapshots. + +- Custom Street Level provider switches survive share links and stored state + using URL-safe stable provider IDs and a shared boolean-switch codec; existing + Mapillary links keep their meaning. Explicit provider toggles reach durable + state without adopting unrelated live switch changes. + +- Directions and Recent Imagery accept replaceable acquisition sources, including + route transport, imagery catalog, thumbnails, tile templates and attribution. + Explicitly removing either source leaves its layer unavailable; standalone + defaults preserve the existing providers. + +- Catalog source availability no longer modifies layer objects; unavailable + tooltips use layer display names. + +- Street Level with no providers uses the shared unavailable state and refuses + activation before initialization, while keeping its layer identity visible. + +- Application source configuration accepts replacements and explicit removal. + Missing catalog feeds leave their layers unavailable without blocking startup + or making fallback requests; malformed supplied feeds remain configuration errors. + +- Street Level providers are optional in application composition. Omitting a + Mapillary source no longer prevents startup; callers can inject a different + provider list or an empty list without changing layer identities or share links. + +- Vector tile sources take their allowed tile origin from the configured + `tileJsonUrl`, so repointing only that URL at a mirror loads tiles from the + mirror or fails visibly instead of silently using OpenFreeMap. Thanks to + [daikaginza](https://github.com/daikaginza) (#935). + - MCP setup examples use the app's default port, `4173`. Thanks to [Huanyi Xie](https://github.com/xiehuanyi) (#934) and [shirunjies8-png](https://github.com/shirunjies8-png) (#959) for the same fix. @@ -71,6 +231,133 @@ ## [Unreleased] +- Add a keyless Air quality layer to the Weather group, with Environment and + Climate Change Canada's AQHI as the first registered provider. Readings carry + `{ provider, scale, value, band }` so another network can be added without + inventing a shared numeric meaning across indices that do not share one; the + row and readout both state that coverage is Canada only. The WEATHER card + shows the band legend and the highest readings currently in view. AQHI is + reported as an integer from 1 with an open-ended "10+", so readings round to + that published form rather than the decimal precision the API returns. + Readings older than six hours are dropped, and the layer reports `stale` + through `getStats()` (also when every observation was dropped) so feed + provenance cannot describe an aged feed as live. Voice can switch it on and + query readings by value, band, station and observation time. + +- Bound six upstream body reads (FIRMS CSV, CelesTrak TLE groups, adsb.lol + `/v2/mil`, TomTom flow tiles, OpenSky `states/all` and adsbdb lookups) with + streaming size caps that cancel the body at the cap or the route's deadline; + the adsb.lol military poll gains a deadline of its own. An oversized OpenSky + snapshot is not retried, and an OpenSky 429 starts its cooldown before the + body is read. + +- Keep later satellites when a TLE catalog entry is missing an orbital line. + The parser skips the incomplete set and resumes at the next complete named + set, instead of losing the rest of the group (#906). + +- Satellite clouds keep Clouds only / Full as quick endpoints and add a continuous + cloud-filter slider: 0% is Full, 100% is the existing brightness key, and + intermediate values blend the keyed alpha. The slider commits on release to + avoid restaging imagery for every drag event. Filter percentages round-trip in + share links; legacy mode codes retain their old endpoint meanings. Background + panel refreshes preserve the pending drag value until release. Contributed by + [NotProniss](https://github.com/NotProniss) (#1084). + +- Street Level: Panoramax is the fourth provider, keyless. Light its chip + (orange; off by default, one provider at a time): from orbit dots show + where the federated commons has pictures, and below about 60 km its + sequences draw as lines from `api.panoramax.xyz` vector tiles. A click on a + line opens that sequence's picture nearest the click without moving the + camera, flat or 360°, with cones along the sequence around it and ‹ › / + ← → following the picture's own links. The 360° and since filters apply, + in the lines and in the lookup. The caption names each picture's author + and licence (CC BY-SA 4.0, Licence Ouverte 2.0…) and links to it in the + Panoramax viewer. HD originals load once, only when the viewer is wider + than the 2,048 px version (a panorama: expanded), and every download stops + when the photo changes; an image whose header declares more than 64 MP is + refused before it decodes. Share links carry it as + `0.m.0_0.r.google-0*kartaview-0*panoramax-1` (`0.m.0_0.r.panoramax-1` works + too). KartaView's photo viewer + is now shared (`photoViewer.js`, `panorama.js`); in it, for KartaView too, + Esc clears the selected drive before it collapses the panel. Lines are + simplified and drawn within a point budget (a tile's unsimplified lines + over Paris hung the tab), decoded in slices so the globe keeps moving, + and kept; the cones' tiles have their own connection, and the open + picture's tile draws its cones before its neighbours arrive. + +- Flights and Military Flights highlight emergency transponder broadcasts in + fleet billboards, 3D models and Cockpit contacts. Colors update when a status + starts or clears. Emergency squawks outrank ADS-B priority statuses; GEV + reports minimum fuel and lifeguard without red tint. Cockpit names the code + or ADS-B status, and analyst/voice results retain broadcast provenance and + the latest known contact time. One omitted ADS-B status poll during a source + switch may retain the previous status if the squawk is unchanged; an explicit + clear takes effect immediately. Broadcasts are not confirmed incidents. + Contributed by [Leo Pechnicki](https://github.com/leopechnicki) (#856). + +- Street Level: KartaView is the third provider, keyless. Light its chip + (purple; off by default, one provider at a time) and zoom in below about + 6 km: the lines are the streets KartaView has photos of (z13–14 tiles from + `api.kartaview.org`). A click on a line opens the photo nearest the click + without moving the camera, flat or 360° (a small WebGL panorama: drag to + look, wheel to zoom), with cones along its drive and ‹ › / ← → to step + through it. The caption links to the full-size photo (KartaView's photo + pages fail for signed-out visitors), and the globe credits "© KartaView + contributors, CC BY-SA 4.0". Share links carry it in the provider + switches (`0.m.0_0.r.google-0*kartaview-1`). Archived photos (KartaView + answers 409) are skipped by the click and by ‹ ›. Shift+← →, ↑ ↓, + − and + pinch look around a 360° photo. The open photo's marker is a purple view + cone pointing along the drive (KartaView's own compass headings are often + 0 or far off). The IMAGERY (ALL / 360° / FLAT) and SINCE filters apply to + KartaView as to Mapillary: its lines are filtered by KartaView's own + server (its filtered PNG tiles, as a mask over the JSON lines), and the + photo a click opens, and the cones, by each photo's projection and upload + date. KartaView's SINCE is the upload date (all its server can filter by); + a note under the slider says so while KartaView is lit (`sinceBasis: + 'upload'` in the provider contract). + Mapillary's line coverage and sequence cones are now shared by both + (`lineCoverage.js`, `sequenceCones.js`). + +- Street Level: Google Street View is the second provider. On the Google 3D + map (Google's terms keep it off other maps), light its chip (off by + default: every panorama is billed to the Google Maps key, which needs the + Maps JavaScript API) and click a street, or press OPEN STREET VIEW, to + open Google's own panorama in + the panel, with its arrows, the address and month in the caption, a link + to Google Maps and FOLLOW. Share links carry it in the provider switches + (`r.google-1`). One provider is lit at a time (lighting one switches the + other off), Mapillary by default since it is free. Street View's panoramas + are all 360°, so the IMAGERY and SINCE filters and the legend apply to + Mapillary only and hide while GOOGLE STREET VIEW is lit; its globe marker + is a Google-blue view cone. + +- Add an experimental ChatGPT/Codex OAuth option for cloud voice in Provider + Settings. API-key voice remains the default. Local sign-in can start from + the auth button and reports completion, failure, timeout, or expired + credentials. OAuth credentials stay on the server; local routes require + loopback access and same-site requests. OAuth sessions retain usage data + and show unknown cost; caption transcriptions are counted rather than + priced into the API-key cap. The auth button keeps keyboard focus after a + mode change, and pending sign-in does not discard text in the key fields. + Thanks to [Shayan Khan](https://github.com/devv-shayan) for #621 and + [TheSmokeDev](https://github.com/TheSmokeDev) for the credential handling + from #653. + +- **Docker install path.** `docker compose up` builds on CI's Node 24.14.0 image, + pinned by digest, and serves the app on `http://localhost:4173` with no local + Node install. The repo is bind-mounted and the container runs as the host user + (never root), so `.env`, + `.gev-cache`, and `.gev-logs` persist on the host exactly as they do under + `npm run dev` — Provider Settings included. The launcher + (`scripts/docker-start.mjs`) trusts the container's gateway address for that + panel, since that is where Docker delivers the host's own browser from; the + opt-in behind it, `GEV_KEY_SETUP_TRUSTED_PEERS`, takes exact addresses only + and is unset (so loopback-only, unchanged) under every other launcher. + Thanks to [jklaze](https://github.com/jklaze) for finding and diagnosing the + container-to-host 403, and to [14traits](https://github.com/14traits), + [PoBruno](https://github.com/PoBruno) and [milu70hub](https://github.com/milu70hub) + for testing on Apple silicon, Docker Engine on Linux, and NAS hosts. + - Stdio servers from one install share a panel key so a page read from one process can make requests through another. Concurrent malformed-key repairs now select one winner; unavailable storage or a busy repair retains the @@ -95,6 +382,17 @@ panel, resize it from any edge or corner, and double-click the header to dock it again. The position persists across reloads. +- Add Norway's city bikes to the Bike Share layer: Oslo, Bergen and + Trondheim Bysykkel (Urban Sharing), Kolumbus Bysykkel in Stavanger and + Farte Bysykkel in Skien/Porsgrunn (Entur Mobility), and Bysykkel + Lillestrøm (JCDecaux), about 760 stations in all. Every feed is keyless + GBFS. `gbfs.urbansharing.com`, `api.entur.io` and `api.cyclocity.fr` join + the proxy allowlist; the last two are kept to their GBFS paths. The proxy + now sends `Client-Identifier: gods-eye-view` on every request and + `ET-Client-Name: gods-eye-view-bikeshare` to Entur, as the operators ask, + and accepts Entur's extensionless feed paths. Attributed under NLOD and + Licence Ouverte. `get_bike_share` covers the six systems too. + - Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera list is one keyless GeoJSON request to the agency's OGC API view of its DATEX 3.1 CCTV table, so no DATEX account is needed. Every working camera @@ -106,6 +404,53 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. +- Add Catalonia (Servei Català de Trànsit) as a keyless CCTV source pack: ~130 + cameras from a WFS/GML feed that aggregates SCT's own highway cameras with + hotlinked Barcelona and Terrassa municipal cameras, each partner named as a + credit beside the shared provider. Ambient-card refresh paced to 3 minutes, + matching the empirically observed upstream cadence. `CCTV_CATALONIA_MAX_SOURCES` + sets the cap and `CCTV_CATALONIA_ENABLED=0` turns the pack off. The catalog + is fetched over https with redirects refused and a 1 MiB body cap, and frame + links carrying userinfo or an explicit port, or outside the publisher's frame + directory, are rejected even on an allowlisted host. Camera ids come from + each frame's SCT camera file or image path rather than the feed's + restart-dependent feature id, and rows the feed repeats collapse to one. + +- Import your own GeoJSON, KML, KMZ and GPX files from DISPLAY or by dropping + them onto the app. Show, hide, fly to or remove each file for the session. + GeoJSON/GPX draw on the ground; KML retains its styles and explicit heights. + Large repeated-style exports use shared local pins, while recorded track + vertices and segment gaps stay intact. Imports respect newer camera gestures, + leave the shared clock alone, and report malformed or oversized files. + File bytes stay on the device; rendered overlays may appear in shared map + screenshots. Thanks to [Chase Mills](https://github.com/kapturner) (#925). + +- GNSS Integrity (Events, voice-addressable, share token `3`) draws navigation + accuracy scores from a keyless regional adsb.lol sample; aircraft-level analyst + queries are unavailable for these aggregate cells. Cells do not intercept + map clicks. Pan/zoom refreshes automatically; feed failures retain the display + and explain the next retry. Aircraft-reported quality includes MLAT-positioned + contacts after GPS loss, excludes TIS-B/MLAT-derived quality, and locates GPS-loss + evidence at the last good GNSS position when available. Scores discount one + bad aircraft per cell and retain 30 minutes of visited-view evidence; they + do not confirm jamming. See [method and limits](DATA_SOURCES.md#gnss-integrity). + Contributed by [Leo Pechnicki](https://github.com/leopechnicki) (#855). + +- Live HLS cameras play on a page opened over plain HTTP from another machine + (`HOST=0.0.0.0`). The client lease id came from `crypto.randomUUID`, which + browsers only expose in secure contexts, so starting a live feed threw + before playback. The id now falls back to a v4 UUID built from + `crypto.getRandomValues`, which the server's lease check accepts. + +- Move Google geocoding behind the local server. Forward and reverse lookups now + go through `/api/google/geocode` and `/api/google/reverse-geocode` with the + server key, joining Places and Street View. Google's Geocoding web service + refuses referrer-restricted keys, so the previous browser call left + `SECURITY.md`'s advice and working search mutually exclusive: the bundled + browser key had to be unrestricted for search to work at all. That key now + needs only the Map Tiles API and the Maps JavaScript API (Street View). Keyless sessions and the Photon/Nominatim + fallback order are unchanged. + ## [0.2.1] — 2026-10-02 — God's Eye View through Your Agent (with security fixes and other features) - Ship each bundled data pack once. The region, marine, admin-boundary, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 63f83e4c2..cde8582f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,8 +79,8 @@ if its layer is later removed. New layers first use the remaining unreserved single-character digits (`0` through `9`), then the two-character base-36 namespace (`00` through `zz`). On the -current published ledger, `0`, `1` and `2` are already owned, so the next -choices are `3` through `9`, then `00`. After rebasing onto the latest `main`, run +current published ledger, `0`, `1`, `2` and `3` are already owned, so the next +choices are `4` through `9`, then `00`. After rebasing onto the latest `main`, run `npm run layer-token:next -- `, add that result to both the permanent JSON reservation ledger and registry entry, and keep the exact registered-layer count assertion up to date. The token is assigned against the merge-time base, not @@ -98,7 +98,7 @@ remove only your still-unpublished layer's registry entry and ledger row so the local registry is valid again, then rerun `npm run layer-token:next -- `. For example, if PR A publishes `0` while PR B still provisionally uses `0`, PR B removes only its own provisional entry, reruns the helper, and receives -`3`. Put `3` on PR B's registry entry and ledger row, update its pinned count +`4`. Put `4` on PR B's registry entry and ledger row, update its pinned count and token tests, then rerun `npm run layer-token:check -- --base-ref origin/main`. An invalid duplicate ledger can fail during module import before the helper prints anything; never fix that by renaming or deleting PR A's published token. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 0b8eac358..8f3b55dbf 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -21,10 +21,13 @@ How to read this: | **NOAA nowCOAST lightning density** (`nowcoast.noaa.gov/geoserver/observations/lightning_detection/ows`) | Lightning density: 15-minute accumulated strike density on an approximately 8 km grid, Americas and Pacific. Keyless; layer metadata refreshed every 10 minutes, exact-time images cached for up to 24 hours | NOAA Level-5 derived product from Vaisala NLDN/GLD360; the [product description](https://ocean.weather.gov/lightning/lightning_pdd.php) permits public distribution of this density product, not of raw Vaisala detections | "NOAA/NWS nowCOAST; derived from Vaisala NLDN/GLD360" | | **NOAA NHC / CPHC advisories** (`www.nhc.noaa.gov/CurrentStorms.json` + the `mapservices.weather.noaa.gov` tropical weather summary MapServer) | Cyclone advisories: storm positions, forecast tracks, points and cones for the Atlantic and eastern/central North Pacific. Keyless through `/api/cyclones`; cached for 5 minutes | [NWS public-data terms](https://www.weather.gov/disclaimer); no endorsement implied | "NOAA/NWS NHC / CPHC" | | **Mapillary** (vector tiles, Graph API, MapillaryJS) | Street Level: coverage lines, image cones and the embedded viewer. Coverage tiles cached for 24 h | Imagery [CC BY-SA 4.0](https://www.mapillary.com/terms); API under the Mapillary terms with a client token | "© Mapillary contributors, CC BY-SA 4.0", shown on the globe while the provider is on | +| **Google Street View** (Maps JavaScript API) | Street Level, on the Google 3D map only (Google's terms): Google's panorama viewer and nearest-panorama lookups; panorama dates and addresses kept for the session only | Google Maps Platform ToS (your own key + billing; every panorama load is billed) | "Street View imagery © Google" on the globe while the provider is on, and Google's logo and terms in the viewer | +| **KartaView** (`api.kartaview.org` coverage tiles and photo API, `cdn.kartaview.org` image proxy, `storage*.openstreetcam.org` photos) | Street Level: coverage lines (z13–14 JSON tiles; with the imagery filter on, KartaView's filtered PNG tiles as a mask), nearest-photo lookups, the photo's sequence and the photo itself, fetched by the browser. Keyless; nothing cached beyond the session | Imagery [CC BY-SA 4.0](https://kartaview.org/terms) (KartaView terms §4); the public API needs no token | "© KartaView contributors, CC BY-SA 4.0", shown on the globe while the provider is on | +| **Panoramax** (`api.panoramax.xyz` meta-catalogue: vector tiles and STAC search; pictures from the federated instance holding each one, e.g. `panoramax.openstreetmap.fr`, `panoramax.ign.fr`) | Street Level: coverage lines and density points, nearest-picture lookups, a sequence's pictures as cones, and the picture itself, fetched by the browser. Keyless; tiles cached by the browser for an hour (the API's own header) | Per picture: mostly [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/), IGN's [Licence Ouverte 2.0](https://www.etalab.gouv.fr/licence-ouverte-open-licence/); the API needs no token | "© Panoramax contributors, licence per picture" on the globe while the provider is on; the caption names each picture's author and licence | | **OpenStreetMap** | Map and place data (roads, military areas and names, ALPR, place search, outlines); via vector tiles, hourly ALPR extract, bundled data, Nominatim and optional self-configured Overpass | [ODbL 1.0](https://opendatacommons.org/licenses/odbl/1-0/) | © OpenStreetMap contributors | | **Google Map Tiles API** (Photorealistic 3D Tiles) + Places/Geocoding | The 3D globe, voice scene context, and on-demand nearby installation search | Google Maps Platform ToS (proprietary, your own key + billing) | "Google" / "Google Maps" logo — **shown in-app**, required | | **OpenSky Network** | Primary worldwide live-flight snapshot | Non-commercial research/education license | Schäfer et al., _"Bringing Up OpenSky"_, IPSN 2014 + opensky-network.org | -| **adsb.lol point API** | Bounded live-flight fallback when OpenSky has no usable snapshot | ODbL 1.0 | adsb.lol contributors; `api.adsb.lol/v2/lat/{lat}/lon/{lon}/dist/{radius}` | +| **adsb.lol point API** | Bounded live-flight fallback when OpenSky has no usable snapshot; GNSS Integrity layer (aircraft-reported NIC/NACp and GPS-loss integrity facts, via `/api/gnss-integrity`, 60 s server cache) | ODbL 1.0 | adsb.lol contributors; `api.adsb.lol/v2/lat/{lat}/lon/{lon}/dist/{radius}` | | **adsb.lol** | Military flights + aircraft traces | ODbL 1.0 | "adsb.lol" (ODbL) | | **adsbdb** (`api.adsbdb.com`) | Aircraft type, model name and registration by ICAO hex for enriched flights; airline and origin/destination airports for the tracked flight's callsign | No licence or rate limit is published for the service. Its own credits: aircraft data from PlaneBase, ICAO-to-N-number conversion by Guillaume Michel, and flight route data that "is the work of David Taylor, Edinburgh and Jim Mason, Glasgow, and may not be copied, published, or incorporated into other databases without the explicit permission of David J Taylor, Edinburgh" | "adsbdb — api.adsbdb.com" (courtesy); route data credited to David Taylor, Edinburgh, and Jim Mason, Glasgow | | **AISStream.io** | Live vessels (AIS) | Free, beta, no formal ToS; AIS is a public broadcast | "AISStream.io" (courtesy) | @@ -32,6 +35,7 @@ How to read this: | **The Space Devs — Launch Library 2 v2.3** | Recent launch, payload, stage, and recovery metadata for Space Missions (30d) | [The Space Devs terms of use](https://github.com/TheSpaceDevs/Tutorials/blob/main/faqs/faq_TSD.md#terms-of-use): data may be used and shared in any form; avoid forwarding it without added value; attribution is encouraged (not mandatory). [Official API limits](https://ll.thespacedevs.com/docs/): 15 unauthenticated calls/hour; optional token | "Launch Library 2 — The Space Devs" (courtesy attribution) | | **Esri World Imagery** (ArcGIS Online tile service) | The keyless satellite basemap — the default landing when no Google/ion credential is configured, and the "Esri Satellite" map stack | [Esri Master Agreement](https://www.esri.com/en-us/legal/terms/full-master-agreement): the public World Imagery service is usable in public-facing apps with attribution; no key is required for this classic endpoint, but Esri governs and can change access — an app at scale should review current ArcGIS Location Platform terms | "Powered by Esri — Source: Esri, Maxar, Earthstar Geographics, and the GIS User Community" (provider carries the service's own credit line) | | **USGS** | Earthquakes | U.S. public domain | "Data courtesy of the U.S. Geological Survey" | +| **Environment and Climate Change Canada** (MSC GeoMet OGC API) | Air quality (AQHI) — Canadian station observations | [ECCC Data Servers End-use Licence](https://eccc-msc.github.io/open-data/licence/readme_en/) (open, attribution required, no endorsement implied) | "Contains information licensed under the Environment and Climate Change Canada Data Servers End-use Licence" | | **NASA GIBS** (WMTS tiles + Worldview Snapshots) and **NASA CMR** (granule catalog) | Recent Imagery layer: HLS S30/L30 (Sentinel-2, Landsat 8/9) and VIIRS true-colour tiles draped on the map, per-day thumbnails and PNG exports; the HLS granule catalog (dates, cloud, footprints) for a selected box | U.S. public domain (NASA data); keyless, browser-direct (CORS `*`); no bulk use — one 30-day catalog query per box, tiles bounded to the box, six tile requests in flight | "We acknowledge the use of imagery provided by services from NASA's Global Imagery Browse Services (GIBS), part of NASA's Earth Science Data and Information System (ESDIS)." — [gibs.earthdata.nasa.gov](https://gibs.earthdata.nasa.gov), [HLS](https://lpdaac.usgs.gov/products/hlss30v002/) | | **NIFC WFIGS** (Wildland Fire Interagency Geospatial Services, ArcGIS feature service) | Fire Perimeters layer: current interagency wildfire incident perimeters, containment, and incident facts | U.S. public domain (interagency wildland-fire data published through the [NIFC Open Data portal](https://data-nifc.opendata.arcgis.com/)); fetched keyless through `/api/fire-perimeters` (5-minute server cache) with `exceededTransferLimit` paging | "Wildfire perimeters: National Interagency Fire Center (WFIGS)" | | **InciWeb** (inciweb.wildfire.gov) | Per-incident "InciWeb ↗" card links to official incident pages (narratives, evacuation/closure notices) | U.S. government public incident information. Links are matched best-effort by incident name and state, with incident page origin and update times checked on selection. Same-origin `/api/fire-perimeters/inciweb` routes cache the catalog for 1 hour and publication metadata for 30 minutes; linked pages remain InciWeb content | "Incident information: InciWeb (inciweb.wildfire.gov)" | @@ -55,8 +59,17 @@ How to read this: | **Fintraffic / Digitraffic (weathercams)** | CCTV camera catalog + frames, Finland | [CC BY 4.0](https://www.digitraffic.fi/en/terms-of-service/) — attribution REQUIRED | "Fintraffic / digitraffic.fi, license CC BY 4.0" | | **DriveBC** (Government of British Columbia) | CCTV camera catalog + frames, British Columbia highways | [Open Government Licence – British Columbia](https://www2.gov.bc.ca/gov/content/data/open-data/open-government-licence-bc) — attribution REQUIRED | "DriveBC. Contains information licensed under the Open Government Licence – British Columbia" | | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | +| **Servei Català de Trànsit** (+ Ajuntament de Barcelona, Ajuntament de Terrassa) | CCTV camera catalog + frames, Catalonia | [Llicència oberta d'ús d'informació - Catalunya](https://web.gencat.cat/ca/generalitat/dades-indicadors/dades-obertes/llicencies) — attribution REQUIRED | "Servei Català de Trànsit"; partner-supplied cameras also name their owner | +| **King County Road Services** (`services.arcgis.com`, King County GIS) | CCTV camera catalog + frames, unincorporated King County WA road network | [KCGIS Center terms](https://kingcounty.gov/en/dept/kcit/data-information-services/gis-center/about/terms-conditions-copyrights) — attribution required; public ArcGIS layer | "King County Road Services" (partner-city cameras name their owner) | +| **WSDOT** (`data.wsdot.wa.gov`) | CCTV camera catalog + frames, Washington State highways | [WSDOT travel information disclaimer](https://wsdot.wa.gov/about/policies/travel-information-disclaimer) — public "low volume" feed; catalog cached, frames on WSDOT's own image host only | "Washington State Department of Transportation (Travel Center)" (courtesy) | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | -| **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | +| **NYC DOT Traffic Management Center** (`webcams.nyctmc.org`) | CCTV camera catalog + frames, New York City five boroughs | NYC DOT requires developers to sign a [data-sharing agreement](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) for the camera feed. **Off by default**; enable only under your own agreement | "NYC DOT Traffic Management Center" + NYC DOT's required application disclaimer (in the Data attribution popover) | +| **City of Toronto Open Data** (`opendata.toronto.ca`) | CCTV camera catalog + frames, Toronto city streets | [Open Government Licence – Toronto](https://www.toronto.ca/city-government/data-research-maps/open-data/open-data-licence/) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – Toronto" | +| **GBFS (Lyft / BCycle / Social Bicycles / PBSC)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | +| **Bike Share Toronto** (`toronto.publicbikesystem.net`) | Bikeshare station availability, Toronto | Operator GBFS feed. No licence is published: the City's open-data package declares none and the feed has no `license_url`. Attribution is given to the operator. | "Bike Share Toronto (Toronto Parking Authority)" | +| **Oslo / Bergen / Trondheim Bysykkel** (Urban Sharing, `gbfs.urbansharing.com`) | Bikeshare availability, Oslo, Bergen and Trondheim city bikes. Keyless GBFS; the proxy sends the `Client-Identifier` header the operators ask for | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED ([Oslo](https://oslobysykkel.no/en/open-data/realtime), [Bergen](https://bergenbysykkel.no/en/apne-data/sanntid), [Trondheim](https://trondheimbysykkel.no/en/open-data/realtime)) | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel" | +| **Kolumbus / Farte Bysykkel** (Entur Mobility, `api.entur.io/mobility/v2/gbfs`) | Bikeshare availability, Kolumbus Bysykkel (Stavanger/Nord-Jæren) and Farte Bysykkel (Skien/Porsgrunn). Keyless GBFS; the proxy sends the `ET-Client-Name` header Entur requires and is limited to Entur's GBFS paths | [NLOD](https://data.norge.no/nlod/en/2.0) per the [Entur Mobility API docs](https://developer.entur.org/pages-mobility-docs-mobility-v2) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Entur (Kolumbus Bysykkel, Farte Bysykkel)" | +| **JCDecaux Cyclocity** (`api.cyclocity.fr`) | Bikeshare availability, Bysykkel Lillestrøm. Keyless GBFS; the proxy is limited to JCDecaux contract paths | [Licence Ouverte / Etalab](https://developer.jcdecaux.com/files/Open-Licence-fr.pdf) (the feed's `license_url`) — attribution REQUIRED | "Bysykkel Lillestrøm: JCDecaux, Licence Ouverte" | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | | **Metro Transit** (Metropolitan Council) | Transit layer live vehicles, Minneapolis–St Paul | Metro Transit / Metropolitan Council open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to Metro Transit / Metropolitan Council (courtesy text) | @@ -74,9 +87,54 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei ### Notes on the live sources +- **Air quality (ECCC AQHI).** Keyless. Read from the MSC GeoMet OGC API (`api.weather.gc.ca`): the `aqhi-stations` catalog plus `aqhi-observations-realtime`. **Coverage is Canada only** — ECCC is the single registered provider, and the layer row and readout card both say so. Roughly 134 stations across ECCC's five administrative zones (Atlantic, Quebec, Ontario, Prairies & North, Pacific & Yukon). AQHI publishes hourly, so readings poll every 20 minutes and the station catalog, which changes a few times a year, is refetched only every 12 hours. Observations are requested with `latest=true`; without it the collection returns the full history, roughly 2,000 rows nationally. Readings older than six hours are dropped rather than displayed as current. + + Readings are stored as `{ provider, scale, value, band }` rather than a bare number, because air-quality indices are not interchangeable: AQHI runs 1-10+ on health risk, the US AQI runs 0-500 on concentration breakpoints, and Europe's EAQI is a five-step category. Naming the index a value belongs to is what lets another network be added later without inventing a shared numeric meaning that does not exist. + + AQHI is reported by ECCC as an integer from 1 to 10 with an open-ended "10+" above that. The API returns the underlying decimal (1.08, 2.32); the layer rounds to the published form rather than displaying a precision the index does not claim. + + **Quebec is thinly represented**, and that is upstream rather than a filter here: the province runs its own Info-Smog program, so only a handful of Quebec stations appear in ECCC's national feed. The reporting station count also varies hour to hour as stations report or fall silent. + +- **Via Lietuva / Eismoinfo (Lithuania).** Public road-camera catalogs and periodically updated still images from [Eismoinfo](https://eismoinfo.lt/), credited to Via Lietuva / eismoinfo.lt in the app. **Off by default.** The camera endpoints are the eismoinfo.lt website backend and are not among the services listed in Via Lietuva's [open-data statement](https://eismoinfo.lt/eismoinfo-backend/content?id=44), whose conditions are attribution and reuse on the same terms. Until the statement is confirmed to cover the camera photos, the pack loads only when an operator sets `CCTV_LITHUANIA_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to Eismoinfo. `eismoinfo-backend/camera-info-table` is joined by ID to `layer-static-features/VKR`. Without `lks=true`, the latter returns WGS84 `[latitude, longitude]`; the table's `x/y` instead use LKS-94 metres. Only the official HTTPS `/eismoinfo-backend/image-provider/camera/last?id=…` image endpoint is registered. Catalogs use the shared 15-minute cache; frames are fetched on demand and never bundled or archived. These are periodic snapshots, not continuous video. Camera headings are low-confidence estimates. No API key is required. `CCTV_LITHUANIA_MAX_SOURCES` defaults to 400. + - **Google Maps Platform.** You supply your own API key and are bound by [Google's ToS](https://cloud.google.com/maps-platform/terms). Google Maps Content (tiles, geocodes, places) **may not be cached, stored, rehosted, or committed** — this app only ever uses it live, which is the compliant pattern. The "Google" attribution is displayed on the globe and must stay visible. Restrict your key (see [SECURITY.md](SECURITY.md)). - **OpenSky Network.** Its license is **non-commercial**, and operational use of the REST API in a live product can require a prior written agreement with OpenSky — even for non-profit/government use. If you deploy this commercially, contact OpenSky for your own terms. The flights layer is a toggle and runs anonymously by default. - **adsb.lol flight fallback.** When OpenSky is unavailable and no last-good OpenSky response exists, the server requests a cached, capped 250 nm adsb.lol point snapshot around the current camera subpoint. This is regional observed context, not worldwide completeness; provenance is exposed in the Flights stats/context row. Military ICAOs remain reconciled through the existing dedicated military registry rather than duplicated. + + +- **GNSS integrity (adsb.lol).** A separate keyless point request samples a 250 nm + circle around the screen centre, snapped once to whole degrees. This does not + reuse the Flights fallback cache. Cache lifetime is 60 s; 429 backoff is capped + at one hour and other upstream failures cool down for 15 s. Future access may + require feeder keys under the [API terms](https://github.com/adsblol/api). + Airborne decoded ADS-B v1/v2 aircraft with a known 0–60 s position age qualify, + including MLAT-positioned aircraft still broadcasting their own quality. + TIS-B/MLAT-derived NIC and NACp are ignored. GPS-loss evidence uses + `gpsOkLat/gpsOkLon` when present, otherwise the current known position. + The proxy strips identity metadata and tags its response with classifier + `gev-nic-nacp-v2`; rows carry only integrity facts. + + The method has three separately attributed parts: + 1. **GEV classifier (unvalidated):** GPS-loss flag, aircraft-reported NACp < 8, + or aircraft-reported NIC < 7 except NIC 0 with NACp ≥ 10 (unknown containment). + The numerical minima come from US 14 CFR 91.227(c), not a published jamming + classifier. Readsb's [JSON contract](https://github.com/wiedehopf/readsb/blob/dev/README-json.md) + describes the quality field origins and GPS-loss flag. + 2. **GPSJAM aggregation:** score = `100 × max(0, bad − 1) / total`, with percent + bands < 2, 2–10, > 10. One bad aircraft is discounted. This is not the raw + share of bad aircraft. See the [GPSJAM FAQ](https://gpsjam.org/faq). + 3. **GEV window:** 30 minutes over visited views, at least three aircraft per + 0.5° cell. These bands were developed for daily totals; with fewer than ten + observations a cell can skip the middle band. Source timestamps date evidence + even through cache replays, and degraded evidence expires independently. + + Medium/high cells report navigation-integrity anomalies. Interference is one + possible explanation, not a detected or located emitter. Avionics faults and + manoeuvring military trainers can also lower reported accuracy. Blank areas + can be outside coverage or lack enough observations. Attribution: adsb.lol + contributors, ODbL 1.0; GPSJAM for aggregation only. + + - **adsbdb enrichment.** `/api/adsbdb` resolves an ICAO hex to aircraft type, model name and registration, and the tracked flight's callsign to an airline and an origin/destination pair. Fan-out is deliberately small because adsbdb is a free community API that publishes no quota: at most 4 requests in flight, dispatches dripped to roughly 5/s, and each key asked for at most once per session, with route lookups reserved for the tracked aircraft. The dev-server proxy caches each key for 24 hours in `.gev-cache/adsbdb.json`, negative answers included, so fresh entries survive server restarts. That gitignored cache remains local to the deployment and is not bundled with this repo. adsbdb publishes no general service license; its README says the route data may not be copied, published, or incorporated into other databases without the explicit permission of David J Taylor, Edinburgh. Deployers are responsible for obtaining any permission their use requires. Enrichment is best-effort and fail-silent — a flight that gets no answer keeps what the flight feed already gave it. - **Launch Library 2.** `/api/launches` makes a server-side rolling-30-day query against the supported v2.3 detailed launch endpoint, caches successful responses for 15 minutes in memory and on disk, and serves the last successful response during a throttle or transient outage. Anonymous access is limited to 15 calls/hour; deployments can provide `LL2_API_TOKEN` for authenticated access. The Space Devs' published terms permit using and sharing the API data in any form, ask users not to forward it without adding value, disclaim complete accuracy, and encourage—but do not require—attribution. This app keeps a courtesy credit. Payload and stage/recovery records are shown only when supplied. Failed launches expose their source status and never receive fallback orbit geometry or a live/estimated marker. LL2 supplies launch context and event timing, not continuous ascent telemetry or live orbital state. - **TfL JamCams.** The camera list comes from the keyless `api.tfl.gov.uk` endpoint (an optional `TFL_APP_KEY` raises its rate limit); frames come from TfL's public S3 bucket. The "Powered by TfL Open Data" attribution is required by TfL's terms and is registered in the Data attribution popover. @@ -90,7 +148,12 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei - **Warendorf webcam.** The Stadt Warendorf Marktplatz webcam, registered from the curated `config/cctv_sources.warendorf.json` (the Kreis Warendorf registration-office cameras from the contributing PR are indoor waiting rooms and were left out); only the municipal image host is proxied, frames are fetched at request time and never stored or redistributed. Disable with `CCTV_WARENDORF_ENABLED=0`. - **Live Traffic NSW cameras.** The camera list is the keyless public `data.livetraffic.com/cameras/traffic-cam.json` feed (Transport for NSW open data, CC BY 4.0); frames are stills on `webcams.transport.nsw.gov.au`, and only that host is registered. That image host answers non-browser clients with an HTML placeholder, so the proxy identifies as a browser for that one host and as itself everywhere else. Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_NSW_ENABLED=0`. - **Open Calgary traffic cameras.** Keyless. The camera list is the public Socrata dataset `k7p9-kppz` on `data.calgary.ca`; frames are stills on the city's own `trafficcam.calgary.ca` host, and only that host is registered. Most rows publish an `http://` frame URL and the host redirects those to HTTPS, so URLs are upgraded and pinned to that origin before registration. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `quadrant` field and the quadrant suffix on each camera name are Calgary's address grid, not a bearing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. The licence requires the attribution statement above. Disable with `CCTV_CALGARY_ENABLED=0`. +- **Catalonia (Servei Català de Trànsit) cameras.** The camera list is the keyless `https://www.gencat.cat/transit/opendata/cameres.xml` WFS/GML feed, fetched over https with redirects refused and the body read capped at 1 MiB, since the list decides ids, coordinates and which allowlisted frame path is proxied. It carries a `font` field naming the real publisher per camera; this pack keeps SCT's own highway cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras (~130 total), each confirmed to return a real frame, so frame URLs are checked against a small host allowlist (`mct.gencat.cat`, `www.bcn.cat`, `emap.terrassa.cat`) rather than one pinned origin (a link carrying `user:pass@` userinfo or an explicit port, or pointing outside that host's frame directory, is rejected even on an allowed host; an SCT `sctidcam` must be a bare `.gif`/`.jpg` filename), and every non-SCT camera carries its real owner as a `credit` shown beside the shared `provider`. The feed carries no compass heading, so every camera loads at low heading confidence (same as TfL and Fintraffic). SCT's own feed link (`RenderService`) needed a workaround: it always 302s to a second endpoint (`TransitCamera`) on a Location the upstream hardcodes as plain http even when requested over https — refused by `fetchWithinHost`'s same-origin (scheme-included) redirect policy — and separately, `mct.gencat.cat`'s TLS setup fails the handshake with Node's fetch outright (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though other HTTP clients complete it. The loader therefore builds the `TransitCamera` URL itself, over plain http, from `RenderService`'s `sctidcam` id, rather than registering the (broken) upstream link — the only transport this one host actually serves to this runtime. Camera ids are built from the SCT camera file or the partner image path, not the feed's `fid` (which embeds an upstream server-start stamp and changes on every upstream restart), and rows the feed lists more than once collapse to one camera. The feed carries no camera status field, so a camera its publisher has taken offline still registers and shows the publisher's own "image unavailable" still (SCT and Barcelona both serve one with HTTP 200), the same as other packs without a status field. Frames are fetched at request time and never stored or redistributed. The feed itself states no licence; the pack relies on the Generalitat de Catalunya open-data portal's terms for this dataset (the licence and attribution above). Disable with `CCTV_CATALONIA_ENABLED=0`. +- **King County road cameras.** Keyless. The camera list is one public ArcGIS FeatureServer GeoJSON query (King County DOT Road Services, "King County and WSDOT Traffic Cameras"). The `outFields` list is explicit on purpose: the layer also carries asset-administration columns — including staff names — that are never fetched. Only live, active cameras with frames on the county's own `info.kingcounty.gov` image handler are registered (the feed writes `http://`; the handler serves HTTPS, so URLs are upgraded and pinned before registration). WSDOT-owned rows in the layer are skipped because their frames are WSDOT-hosted, outside the pinned image origin; partner-city cameras hosted on the county image handler (e.g. Newcastle) name their owner as the per-camera credit. The layer publishes no facing, so headings use the shared id-hash fallback at low confidence. Uniquely, the county publishes each camera's hardware `Manufacturer`/`Model` (Cohu 3950/3965, Bosch AUTODOME, Axis PTZ domes); both are carried on the source (and served by `/api/cctv/sources`) as `model` and `manufacturer` for identified-hardware consumers (vendor-generic strings such as `PTZ` name no model and are omitted). Frames are fetched at request time and never stored or redistributed. Disable with `CCTV_KINGCOUNTY_ENABLED=0`. +- **WSDOT highway cameras.** Keyless. The camera list is the Esri JSON layer `data.wsdot.wa.gov/travelcenter/Cameras.json` — the same layer WSDOT's own Travel Center map consumes (~1,700 point features, EPSG:3857 geometry converted to WGS84 on load; the body carries Windows-1252 bytes and is decoded as such). Only frames on WSDOT's own `images.wsdot.wa.gov` host are registered — the catalog also lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) whose own terms apply, and those are skipped rather than proxied. WSDOT's layer metadata marks the feed for "low volume" use (see WSDOT's [travel information disclaimer](https://wsdot.wa.gov/about/policies/travel-information-disclaimer)). The catalog is fetched once per CCTV catalog refresh (every 15 minutes), and a camera's frame is fetched only when that camera is viewed; frames are never stored or redistributed. The live feed's `CompassDirection` is a single cardinal (N/E/S/W), `B` (both directions — most rows), `O`, or null. A single cardinal gives a high-confidence heading (the diagonals NE/SE/SW/NW are mapped too should they appear); `B`, `O`, null, and any other code carry no single facing and use the shared id-hash fallback at low confidence. By default the 250 cameras nearest Seattle and Spokane load; `CCTV_WSDOT_MAX_SOURCES` changes the cap (8–1,200) and `CCTV_WSDOT_ENABLED=0` turns the pack off. A keyed WSDOT Traveler Information API serves the same population for higher-volume use and is deliberately not required here. - **Statens vegvesen road cameras.** Keyless. The camera list is one GeoJSON request to the OGC API Features view of the official DATEX 3.1 CCTV site table (`ogckart-sn1.atlas.vegvesen.no/ogc/features/v1/collections/datex_3_1:CctvSimple/items`), so no DATEX account is needed; frames are stills on `kamera.atlas.vegvesen.no/api/images/`, and only that exact path per camera is registered. Cameras the feed reports as faulty are dropped. About 135 working cameras also publish live HLS (`kamera.vegvesen.no/public//manifest.m3u8`, MPEG-TS); those play as live video through the bounded in-memory HLS relay, with the still as the snapshot fallback, and only that exact manifest path per camera is registered. `CCTV_VEGVESEN_VIDEO=0` keeps every camera on stills. `orientationDescription` names the place a camera faces, not a bearing, so it goes into the label and headings use the id-hash fallback at low confidence. By default every live camera loads (~850); `CCTV_VEGVESEN_MAX_SOURCES` (8–1000) lowers the cap, keeping the cameras nearest Oslo, Bergen, Stavanger, Trondheim, Kristiansand, Bodø and Tromsø and `CCTV_VEGVESEN_ENABLED=0` turns the pack off. Frames are fetched at request time and never stored or redistributed. The NLOD attribution is registered in the Data attribution popover. +- **NYC DOT Traffic Management Center cameras.** Off by default. NYC DOT's [developer data feeds page](https://www.nyc.gov/html/dot/html/about/datafeeds.shtml) asks developers who want the camera feed to contact `tmcdot@dot.nyc.gov` and sign a data-sharing agreement. This project does not hold one that covers downstream installations, so the pack loads only when an operator who holds their own agreement sets `CCTV_NYC_ENABLED=1`; with the flag unset, empty or any other value, the server makes no request to NYC DOT. The endpoint needs no key. The camera list is the public `webcams.nyctmc.org/api/cameras` catalog (~980 cameras across the five boroughs); frames are stills at `webcams.nyctmc.org/api/cameras//image`, synthesized from a validated UUID rather than taken from the payload, and only that host is registered. Cameras the feed reports as offline are dropped. The catalog publishes no camera facing, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. Once enabled, the 250 cameras nearest the borough cores load; `CCTV_NYC_MAX_SOURCES` (8–400) changes the cap. Frames are fetched at request time and never stored or redistributed. NYC DOT asks every application that uses its data to carry its disclaimer; it is registered verbatim in the Data attribution popover. +- **City of Toronto RESCU cameras.** Keyless. The camera list is the City's public `tmcearthcameras.json` on `opendata.toronto.ca` (dataset "Traffic Cameras", Transportation Services), served as JSONP and unwrapped with an anchored pattern. This is the municipal network the Ontario 511 pack does not carry: downtown arterials plus the Gardiner, DVP, Allen and Lake Shore. The list publishes no frame URL, so frame URLs are BUILT from the documented `CameraImages/loc####.jpg` convention and a strictly-digits camera number, pinning the frame proxy to the portal origin by construction; the list fetch refuses redirects and bounds the body it buffers. Frames are fetched at request time and never stored or redistributed. The dataset publishes no camera facing: the `D1`-`D4` fields are the directions of the *comparison* images (static reference stills captured in 2016 under a different path), not a bearing for the live pan-tilt-zoom frame, so headings use the shared id-hash fallback at low confidence and are corrected with the in-app calibration gizmo. A camera that is temporarily out of service still answers 200 with a byte-identical "Image Unavailable" placeholder rather than an error (about 6% of the network at any moment), so such a camera reads as healthy; the frame it shows is the agency's, not a synthesized one. The licence requires the attribution statement above. Disable with `CCTV_TORONTO_ENABLED=0`. - **Camera frame content.** Public camera frames can contain people, vehicles and license plates, and the camera layer does nothing with them beyond putting them on the map. A successful upstream response is relayed as the provider served it: nothing in the camera pipeline blurs, redacts, enhances or restores it, and nothing there runs face, plate or object detection or otherwise recognises or classifies what is in the picture. (The ALPR layer described below maps where plate-reader cameras are, from locations contributors tagged in OpenStreetMap; it reads nothing from any camera frame.) When an upstream has no frame for a camera, what you see instead is a Google Street View still or the app's own placeholder card; the panel's source badge names which of them you are looking at, and a refresh that fails after a frame has arrived leaves the last good picture on screen rather than replacing it. The browser resamples each frame for display: onto a fixed 1920x1080 surface for the camera's projection plane in the scene, and onto a 192x108 canvas for the small ambient card. That scaling is the only change made to the picture, and the only pixels the camera layer reads back are a 64x36 downscale hashed to tell whether the picture has changed. Live streams are piped through; the still path reads the image into memory to pass it on — TxDOT wraps its JPEG in a JSON envelope, which is decoded there — and no frame is written to disk (the disk cache, `.gev-cache/`, holds feed JSON for other layers and never camera imagery). Outside that pipeline, the voice assistant is the one feature that sends imagery anywhere: when it answers a question about what is in view it may attach a downscaled screenshot of the rendered scene — including any camera plane on screen — to the request it sends to the voice model. Frames are fetched only from server-registered camera URLs — a client cannot hand the proxy a URL (see [SECURITY.md](SECURITY.md)). What the app does stays inside the line README draws: no named-person search, no face recognition, no tracking individuals. - **Transit (GTFS-Realtime).** The proxy requests only registered operator feeds, validates redirects, bounds responses, and uses conditional requests, short snapshot caching and failure backoff. All seven registered regions are live-enabled; each can be disabled independently. Vehicles use delayed playback between reports, with report age and waiting states shown separately. Session-local selected trails are available for every feed. MBTA additionally retains up to 15 minutes of recently observed positions in the running proxy, so available history can survive a browser reload; server restart clears it. History reads never fetch upstream, and other feeds have no proxy retention. Nothing is archived to disk or collected in the background. Operator text credits are included as a courtesy and do not imply endorsement. Heights are aligned to work with Google 3D tiles; subway markers are shown at street level because these feeds do not supply depth. - **Radio Browser.** `/api/radio/stations` discovers official API mirrors, makes bounded and coalesced healthy/geolocated HTTPS-station queries, caches the normalized public-domain directory for 45 minutes, and may serve the last good catalog for up to seven days during an outage. Refreshes must meet minimum accepted-query and station coverage before replacing a warm catalog; schema-valid responses whose rows all fail the product's health policy do not count as successful queries. A usable partial cold catalog is explicitly `DEGRADED`, and malformed or empty successful payloads are rejected atomically. Every directory and click-count request rejects redirects, validates all resolved addresses as globally routable (including reserved/documentation IPv4 and special/non-global IPv6 exclusions), and pins the TLS connection to a validated address. Only MP3/AAC non-HLS directory rows with public HTTPS stream targets are returned; favicons are intentionally omitted. Pressing play connects one browser audio element directly to the selected broadcaster and calls the directory's click counter through known-ID-only `POST /api/radio/click/:uuid`. GEV never proxies, caches, records, bundles, or redistributes audio. Radio Browser supplies station-level tags, not dependable current-song or upcoming-program metadata, so Radio filtering never claims either. Direct playback exposes the listener's IP address to the broadcaster, whose own stream terms apply. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..d34f4887a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +# syntax=docker/dockerfile:1 +# +# The dev server is the whole app: Vite serves the client AND runs the API +# proxies (ADS-B, AIS, Overpass, OpenAI Realtime, ...) as middleware, so there +# is nothing else to containerise. See compose.yaml for the usual entry point. +# Same Node as CI (package.json engines, .github/workflows), pinned by digest. +FROM node:24.14.0-slim@sha256:d8e448a56fc63242f70026718378bd4b00f8c82e78d20eefb199224a4d8e33d8 + +# Puppeteer is only used by the QA scripts; skip its ~150MB Chromium download. +ENV PUPPETEER_SKIP_DOWNLOAD=1 + +# `npm test` builds fixtures with `git init`, so the suite cannot run in a +# container without it (71 failures on node:24-slim, all ENOENT on git). +RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* +# The repo is bind-mounted, so inside the container it can show an owner other +# than the user the container runs as - always, on Docker Desktop. git then +# refuses it as "dubious ownership", which fails the tests that shell out to +# git. It is the user's own checkout, so trust that one path. +RUN git config --system --add safe.directory /app + +# Never run as root. The image's own user is `node` (1000); compose overrides +# it with the host user so bind-mounted files stay yours. +WORKDIR /app +RUN chown node:node /app +USER node + +COPY --chown=node:node package.json package-lock.json ./ +RUN npm ci +# node_modules belongs to `node`, but compose may run as another UID. Vite only +# needs to write its dep-optimizer cache, so open up those two directories +# rather than chowning a few hundred MB to a UID we cannot know at build time. +RUN mkdir -p node_modules/.vite && chmod 777 node_modules node_modules/.vite + +COPY --chown=node:node . . + +# Bind to all interfaces, otherwise the published port hits nothing. This does +# not widen the Host allowlist: IP addresses and localhost are accepted, other +# names only via GEV_ALLOWED_HOSTS (see build/allowedHosts.js). +ENV HOST=0.0.0.0 +ENV PORT=4173 +EXPOSE 4173 +# The launcher is `npm run dev` plus one step: it trusts the container's +# gateway address for Provider Settings, because that is where Docker delivers +# the host's own browser from (never 127.0.0.1). See scripts/docker-start.mjs. +CMD ["node", "scripts/docker-start.mjs"] diff --git a/README.md b/README.md index 5a4ab8b24..c8b9eafec 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,14 @@ locked dependencies, finds a free local port, and opens the app. Version 8.2 fixes the launcher installation issue; [details from the Pinokio maintainer](https://pinokio.co/posts/01m1m4p9xxm3qw7dnnpj2wr93g). +**Pinokio opens but the globe is very choppy?** Before treating that as a GPU +compatibility or minimum-hardware problem, copy the local server URL shown by +Pinokio and open the same URL in your normal hardware-accelerated browser. One +Pinokio 8.2.0 user reported severe stutter in Pinokio's embedded view that went +away when the same running server was opened in an external browser ([#365](https://github.com/bilawalsidhu/gods-eye-view/issues/365)). +That points to the embedded browser rather than the app server or your +hardware; it does not establish a minimum GPU requirement. + ### Path 2 — Terminal / coding agent Use **Node.js 24.x (24.14.0 or later) or 26.x**. The setup doctor warns about @@ -145,6 +153,72 @@ See [docs/PERFORMANCE.md](docs/PERFORMANCE.md). **macOS shortcut:** `./scripts/dev-fresh.sh` clears the Vite cache and pulls any configured keys straight from the Keychain. It starts keyless too. +### Path 3 — Docker + +No Node install needed; the container publishes the same dev server. + +```bash +git clone https://github.com/bilawalsidhu/gods-eye-view.git +cd gods-eye-view +docker compose up +``` + +Open **`http://localhost:4173`** as above. Everything the app saves stays in the folder you cloned, so `docker compose down` never loses anything: + +- `.env` - your API keys, whether saved from Provider Settings or typed in by hand +- `.gev-cache/` - downloaded map and satellite data, so later starts are faster +- `.gev-logs/` - logs + +Back up that folder and you've backed up everything; delete `.env` to forget your keys. Source edits still hot-reload. The port is published to `127.0.0.1` only, so the default matches the native one. + +The local MCP server and ChatGPT OAuth voice are not available under Docker; use an OpenAI API key for voice. + +Publishing it wider is the [Sharing an instance](#-sharing-an-instance) opt-in, and it hands out **Provider Settings** too: the launcher trusts the container gateway as this machine, and a wider publish delivers everyone else from that same address. For an instance you mean to share, put the keys in `.env` by hand instead. [SECURITY.md](SECURITY.md) has the detail. + +When you widen the publish, also set `GEV_KEY_SETUP_TRUSTED_PEERS=none` so the launcher trusts no gateway address. + +
+Running on Linux, a NAS, or behind a reverse proxy + +Run it as the user who should own the files it writes. `id` shows the numbers; for yourself on Linux: + +```bash +env UID=$(id -u) GID=$(id -g) docker compose up +``` + +On a NAS, clone onto the share you want the data on and use the share owner's ids: + +```bash +cd /volume1/docker +git clone https://github.com/bilawalsidhu/gods-eye-view.git +cd gods-eye-view +env UID=1026 GID=100 docker compose up -d +``` + +Keep the `env`: bash treats `UID` as read-only, so a plain `UID=1026` in front of a command is silently ignored. If the container can't write the checkout, it says so at start. Docker Desktop on macOS and Windows maps ownership for you. + +Behind a reverse proxy, put your keys in `.env` or pass them as environment variables - Provider Settings refuses proxied requests on purpose - and add the hostname you serve it on, for example `GEV_ALLOWED_HOSTS=globe.example.lan` (see [Sharing an instance](#-sharing-an-instance)). + +
+ +
+IPv6-only networks, rebuilding, and the test suites + +The launcher reads the IPv4 default gateway, so an IPv6-only Docker network +finds none: it logs `No default gateway; Provider Settings stays loopback-only` +and the panel refuses every save. Put your keys in `.env` by hand there, or name +the gateway yourself with `GEV_KEY_SETUP_TRUSTED_PEERS` after reading +[SECURITY.md](SECURITY.md). + +`node_modules` deliberately stays inside the image, so it survives in an +anonymous volume across restarts: after a `git pull` that changes dependencies, +recreate it with `docker compose down -v && docker compose up --build`. The +image also skips Puppeteer's Chromium download, which keeps it small but leaves +`npm run test:track` to the host — `npm run build` and `npm test` run fine in +the container. + +
+ ### Then power it up — in the app, not in a file Keys are upgrades, not prerequisites. When you want one, click the **POWER UP** @@ -235,7 +309,13 @@ _Why cockpit mode exists: you're riding a real aircraft over real terrain — an ## 🎙️ Talk to It -> Voice needs an **OpenAI key**. Without one the entire app still runs — the mic button just reports voice is unavailable. The same key drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. +> Voice can use an **OpenAI API key** or, on the same machine, an opt-in signed-in **ChatGPT/Codex OAuth** session. The API-key path remains the default. Selecting ChatGPT OAuth in Provider Settings reuses an existing local sign-in or starts the Codex browser sign-in flow and selects OAuth when it completes. The OpenAI key still drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. + +The OAuth option is experimental. OpenAI documents ChatGPT plan usage for the [Responses API](https://developers.openai.com/siwc/token-sharing-open-source/models-and-inference); a supported third-party Realtime contract for reusing Codex tokens has not been confirmed. A successful token mint does not establish billing or support terms. + +GEV reads Codex's file credentials but does not write or refresh them. It checks token shape and expiry, and reports expired sign-in as requiring re-authentication. The login button runs `codex login`, permits one pending attempt, and reports exit failures or a two-minute timeout. `CODEX_HOME` must match the directory that Codex uses. If `CODEX_AUTH_JSON` points elsewhere, sign in separately or fix that mismatch before using the button. Codex credential storage must use `auth.json`; Keychain-only and in-memory storage are not readable by GEV. On Windows, the login button may not be able to start an npm-installed Codex; run `codex login` yourself, then select OAuth. See [Codex credential storage](https://learn.chatgpt.com/docs/auth#credential-storage). + +For a live voice check, run `node scripts/qa-voice-auth.mjs http://127.0.0.1:4189 oauth microphone.wav` against your local dev server. Use a WAV with three seconds of leading silence followed by a short spoken reply request. Replace `oauth` with `api-key` to check the API-key path. The check verifies audio input, a completed spoken reply, usage, resource cleanup, and reconnect. It saves screenshots and credential-free results under `.gev-logs/voice-auth/`. Click **GEV MIC**, grant the microphone, and just talk. This is more than a voice-controlled remote: @@ -243,7 +323,7 @@ Click **GEV MIC**, grant the microphone, and just talk. This is more than a voic - **🎯 Entity Q&A.** Click any plane, ship, or datacenter and ask _"what's this?"_ It answers using the object's live telemetry. - **👁️ Visual grounding.** At street level, it reads a viewport screenshot to identify legible signage and building names, and is instructed never to hallucinate labels. - **🎬 Cinematic framing.** _"Show me the planes overhead"_ pulls the camera back, angles it, and frames the live traffic like a director. -- **🔒 Honest and secure.** The agent only confirms actions that succeeded. Your `OPENAI_API_KEY` never touches the browser; the client only gets a short-lived session token. +- **🔒 Honest and secure.** The agent only confirms actions that succeeded. Your `OPENAI_API_KEY` never touches the browser; the client only gets a short-lived session token. If you choose **USE CHATGPT OAUTH** in Provider Settings, the local server uses that sign-in only to mint the same kind of short-lived Realtime credential, and refuses the OAuth path for non-loopback clients. Thirty tools, four jobs — the commands below come straight from the product's voice test suite and tool playbook: @@ -283,20 +363,23 @@ _Ask for radio near anywhere and the globe starts broadcasting — every station ## 🛰️ What's on the Globe -Twenty layers and map sources. **Seventeen have a keyless path.** Some offer additional capabilities with a provider key. (🟢 no key · 🟡 free key · 🔴 metered.) +Twenty layers and map sources. **Eighteen have a keyless path.** Some offer additional capabilities with a provider key. (🟢 no key · 🟡 free key · 🔴 metered.) | Layer | What you get | Source | Auth | | --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------------- | | 🗺️ **Map Stack** | Esri satellite imagery, Google Photorealistic 3D, OSM, plus additional ion-hosted stacks | Esri / Google / Ion / OSM | 🟢 Esri satellite + OSM · 🟡 ion-hosted Google 3D + world terrain · 🔴 direct Google + place search | | ✈️ **Live Flights** | 11,000+ live aircraft + route history | OpenSky + adsb.lol | 🟢 (🟡 optional for more polling credits) | -| 🎖️ **Military Flights** | ADS-B military traffic in amber | adsb.lol | 🟢 | +| 🎖️ **Military Flights** | ADS-B military traffic in amber, emergency broadcasts in red | adsb.lol | 🟢 | +| 📡 **GNSS Integrity** | Regional navigation accuracy scores; anomalies do not confirm interference | adsb.lol, GPSJAM aggregation | 🟢 | +| 🔥 **Fire Perimeters** | Incident boundaries and reported containment | WFIGS / NIFC | 🟢 | | 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | | 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | | 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | +| ◍ **Air quality** | Live AQHI on the published 1–10+ health-risk scale, with a Weather-panel card showing the band legend and the highest readings in view. **Canada only** — ECCC is the first registered provider | ECCC MSC GeoMet | 🟢 | | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | -| 📹 **CCTV Mesh** | ~3,900 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | +| 📹 **CCTV Mesh** | ~4,000 public cameras (shared cap) projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary · Norway (Statens vegvesen) · Toronto · King County (WA) · Washington (WSDOT) · Catalonia (Servei Català de Trànsit), plus opt-in New York City (NYC DOT TMC, needs your own NYC DOT agreement) and opt-in Lithuania (Via Lietuva / Eismoinfo, photo licence unconfirmed). Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | -| 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary first; Panoramax, KartaView and Google Street View planned | Mapillary | 🟡 Mapillary client token | +| 📷 **Street Level** | Street-level photos behind one panel, one chip per provider: coverage on the streets, image cones, an embedded viewer and **FOLLOW**. Mapillary, Google Street View, KartaView and Panoramax | Mapillary · Google · KartaView · Panoramax | 🟢 KartaView, Panoramax · 🟡 Mapillary token · 🔴 Google Maps key | | 📻 **Radio** | Geolocated world radio with an **analog tuner** — drag the needle across up to 750 stations and the globe flies to each broadcaster | Radio Browser / broadcasters | 🟢 | | 🚌 **Transit** | Live buses, trams, metros, trains and ferries with delayed playback between reports, selected-vehicle trails, and mode-coloured DETECT labels — Boston, Austin, Minneapolis, Helsinki, the Netherlands, Norway, South East Queensland | Operator GTFS-Realtime feeds | 🟢 | | 🚲 **Bikeshare** | Live station availability | GBFS | 🟢 | @@ -334,13 +417,13 @@ _The Space Missions layer replaying a Falcon 9 ascent — labeled `RECONSTRUCTED ## 📷 Street Level -> Street-level photos from any provider in one panel. This build ships **Mapillary**, which needs a free client token. +> Street-level photos from any provider in one panel. This build ships **Mapillary** (a free client token), **Google Street View** (your Google Maps key), **KartaView** and **Panoramax** (no key). -**Setup.** At [mapillary.com/dashboard/developers](https://www.mapillary.com/dashboard/developers), register an application with **Read** access and copy its **Client Token** (`MLY|…`). Paste it into **POWER UP → Mapillary**, or set `MAPILLARY_CLIENT_TOKEN` in `.env` and restart. Without it the panel reads **KEY REQUIRED**. +**Setup.** At [mapillary.com/dashboard/developers](https://www.mapillary.com/dashboard/developers), register an application with **Read** access and copy its **Client Token** (`MLY|…`). Paste it into **POWER UP → Mapillary**, or set `MAPILLARY_CLIENT_TOKEN` in `.env` and restart. Without it the panel reads **KEY REQUIRED**. **Google Street View** uses `GOOGLE_MAPS_API_KEY` with the **Maps JavaScript API** enabled for it in Google Cloud; its chip starts off, since every panorama is billed to the key. **KartaView** and **Panoramax** need nothing: their public APIs are keyless. -**Use.** Turn on **Street Level** under DATA LAYERS → Cameras, or with the pill in the STREET LEVEL panel header. Each provider has a chip in its own colour (Mapillary green). The **IMAGERY** (ALL / 360° / FLAT) and **SINCE** filters apply to every provider. Click a coverage line for its image cones and a cone to open the photo. Above the photo: **EXPAND**, **FIT / FILL**, and **FOLLOW** (the globe camera follows the photo's view; Google 3D map only). The caption links to the image on the provider's site, and the globe credits "© Mapillary contributors, CC BY-SA 4.0" while Mapillary is on. +**Use.** Turn on **Street Level** under DATA LAYERS → Cameras, or with the pill in the STREET LEVEL panel header. Each provider has a chip in its own colour (Mapillary green), and one is lit at a time: lighting one switches the others off. Mapillary, which is free, is lit by default. The **IMAGERY** (ALL / 360° / FLAT) and **SINCE** filters apply to Mapillary, KartaView and Panoramax; Street View panoramas are all 360°, so with Street View lit the filters and legend are hidden. Click a coverage line for its image cones and a cone to open the photo. Light **KARTAVIEW** (purple) and zoom in below about 6 km: its lines are the streets KartaView has photos of, and a click on one opens the photo nearest the click, flat or 360° (drag to look around), with cones along its drive; **‹ ›** or ← → step through the drive (Shift+← →, ↑ ↓ and + − look around a 360° photo; pinch zooms on a phone). Photos KartaView has archived are skipped, and where all are archived the panel says so. For KartaView, **SINCE** goes by upload date, as KartaView's own map does (it can filter by nothing else), and the panel notes it under the slider; KartaView filters its lines on its own server, and a filtered area KartaView has not drawn before can take several seconds, and the unfiltered lines show meanwhile. Light **PANORAMAX** (orange) for the open, federated commons (strongest in France and Belgium): from orbit its dots show where it has pictures, and below about 60 km its lines are sequences. A click on a line opens that sequence's picture nearest the click without moving the camera, flat or 360°, with cones along the sequence around it; **‹ ›** or ← → step through it, and both filters apply. Its caption names the picture's authors (or the instance holding it) and licence (mostly CC BY-SA 4.0; pictures on the IGN instance are Licence Ouverte 2.0). On the **Google 3D** map (Google's terms keep Street View off other maps, where it does not appear), light **GOOGLE STREET VIEW** and click a street from below 3 km, or press **OPEN STREET VIEW** for the view centre, to open Google's panorama there; step along its arrows. Above the photo: **EXPAND**, **FIT / FILL** (Mapillary, and flat KartaView and Panoramax photos), and **FOLLOW** (the globe camera follows the photo's view; Google 3D map only). The caption links to the image on the provider's site, and the globe credits each provider that is on ("© Mapillary contributors, CC BY-SA 4.0", "Street View imagery © Google", "© KartaView contributors, CC BY-SA 4.0", "© Panoramax contributors, licence per picture"). KartaView's own photo pages fail for signed-out visitors, so its link opens the full-size photo. -Drag the panel header to float the panel, and resize it from any edge. Double-click the header or collapse it to put it back in the rail. Share links carry the provider switches and filters (`0.m.0` Mapillary off, `0.p.p` panoramas only, `0.s.365` the last year). To add a provider, implement the contract in `src/layers/streetLevel/registry.js` and register it in `src/app/layers/streetLevel.js`. +Drag the panel header to float the panel, and resize it from any edge. Double-click the header or collapse it to put it back in the rail. Share links carry the provider switches and filters (`0.m.0` Mapillary off, `0.m.0_0.r.google-1` Street View on, `0.m.0_0.r.google-0*kartaview-1*panoramax-0` KartaView on and `0.m.0_0.r.google-0*kartaview-0*panoramax-1` Panoramax on, as the app writes them (`0.m.0_0.r.kartaview-1` and `0.m.0_0.r.panoramax-1` work too; a link that leaves out `m` keeps Mapillary, the default, lit), `0.p.p` panoramas only, `0.s.365` the last year). To add a provider, implement the contract in `src/layers/streetLevel/registry.js` and register it in `src/app/layers/streetLevel.js`. ## 🎖️ Field Missions @@ -428,8 +511,8 @@ Seven keys. Five have a free tier, and the two 🔴 ones are metered: | | Key | Why | Get it | | --- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 🟡 | **Cesium ion** | 🗺️ Google Photorealistic 3D, world terrain, and additional ion-hosted imagery stacks. The free Community plan is for eligible individual, personal/non-commercial use and has quotas | [cesium.com/ion](https://cesium.com/ion) — use a public `assets:read` token and check current [pricing/eligibility](https://cesium.com/platform/cesium-ion/pricing/) | -| 🔴 | **Google Maps** | Direct Google Photorealistic 3D + Google place search ([Map Tiles API](https://developers.google.com/maps/documentation/tile)) | [Google Cloud Console](https://console.cloud.google.com/) — URL-restrict it | -| 🔴 | **OpenAI** | 🎙️ The voice experience + AI HUD summary. The mini model works; the standard model is noticeably smarter. Want Gemini or another provider behind the mic? PRs welcome | [platform.openai.com](https://platform.openai.com) — metered, see costs below | +| 🔴 | **Google Maps** | Direct Google Photorealistic 3D, Google place search and Street View ([Map Tiles API](https://developers.google.com/maps/documentation/tile); Street View also needs the Maps JavaScript API) | [Google Cloud Console](https://console.cloud.google.com/) — URL-restrict it | +| 🔴 | **OpenAI** | 🎙️ Voice + AI HUD summary through an API key. Provider Settings also offers **USE CHATGPT OAUTH** for cloud voice without removing the API-key option | [platform.openai.com](https://platform.openai.com) — API-key usage is metered, see costs below | | 🟡 | **AISStream** | 🚢 Live global ships | [aisstream.io](https://aisstream.io) — free signup | | 🟡 | **NASA FIRMS** | 🔥 Live active fires | [firms.modaps.eosdis.nasa.gov](https://firms.modaps.eosdis.nasa.gov/api/map_key/) — free | | 🟡 | **TomTom** | 🚦 Live flow speeds and congestion colors for the simulated traffic layer | [my.tomtom.com](https://my.tomtom.com/keys) — free tier available | @@ -508,7 +591,7 @@ Honest numbers, roughly, as of mid-2026 — always check the provider pricing pa | **🟢 Most layers** | **$0, no signup.** OpenSky anon, USGS, CelesTrak, adsb.lol, city CCTV, Radio Browser, GBFS, Launch Library 2, transit feeds, OSRM routing, NOAA and ECMWF weather, OpenStreetMap ALPR mapping, bundled datasets. | | **🟡 The free-key tier** | **$0 with a signup.** AISStream, FIRMS, TomTom, OpenSky, plus Cesium ion for eligible personal/non-commercial use. Provider quotas and eligibility still apply. | | **🗺️ Google 3D tiles** | **Free through an eligible Cesium ion Community account within its quota; metered through a direct Google key.** Use the direct route for Google place search or commercial deployment, verify current provider terms, and set budget alerts where billing is enabled. | -| **🔴 OpenAI voice** | **The one that costs real money — so the app meters it for you.** Realtime audio runs a few cents per active minute; an evening of heavy use is single-digit dollars. A live session-spend readout sits next to the mic, with an STD/MINI model toggle, a $2 warning, and a **$5 hard cap that ends the session**. The voice context window is kept deliberately short too. | +| **🔴 OpenAI voice** | API-key sessions keep the live spend readout, STD/MINI model toggle, $2 warning, and **$5 hard cap**. ChatGPT OAuth is an explicit choice in Provider Settings. Its usage is tracked without an API-dollar estimate or spend cap. The voice context window stays deliberately short. | Google's direct 3D route is surprisingly generous: the first 1,000 Photorealistic 3D Tiles sessions each month are currently free, and one root request supports diff --git a/SECURITY.md b/SECURITY.md index eb5fcd6e0..cc3f27cfe 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -18,21 +18,22 @@ The golden rule: **secret-bearing API keys stay on the server side.** The dev/pr | Key | Where it lives | How the browser uses it | |-----|----------------|--------------------------| | `OPENAI_API_KEY` | Server only | Browser fetches a short-lived **ephemeral** Realtime session token from `/api/realtime/token`; the real key never ships | +| ChatGPT sign-in through Codex (optional, experimental) | Codex's own `auth.json` (`CODEX_HOME`, default `~/.codex`); GEV reads it server-side and never writes or refreshes it | Only after **USE CHATGPT OAUTH** is selected in Provider Settings: the server mints the same ephemeral Realtime token with the ChatGPT access token, for loopback clients only | | `AISSTREAM_API_KEY` | Server only | Server holds the AISStream websocket; browser polls the same-origin `/api/vessels` cache | | OpenSky OAuth (`OPENSKY_CLIENT_ID/SECRET`) | Server only | Server mints + refreshes the token behind `/api/flights` | -| `GOOGLE_MAPS_SERVER_API_KEY` (optional, #33) | Server only | Server calls Places (`/api/google/nearby-places`, `/api/google/text-search`) and the Street View fallback with this key; falls back to `GOOGLE_MAPS_API_KEY` when unset | +| `GOOGLE_MAPS_SERVER_API_KEY` (optional, #33) | Server only | Server calls Places (`/api/google/nearby-places`, `/api/google/text-search`), Geocoding (`/api/google/geocode`, `/api/google/reverse-geocode`) and the Street View fallback with this key; falls back to `GOOGLE_MAPS_API_KEY` when unset | ### Three deliberately client-side keys — restrict them These are designed to be used directly in the browser (like a Mapbox public token). They are injected into the client bundle via Vite's `define`, so they **will** be visible in browser devtools. Scope and restrict them rather than trying to hide them: -1. **Google Maps API key** — loads Photorealistic 3D Tiles directly and powers GEV place search. **Restrict it** (HTTP referrer + API restriction to the required Google APIs) in the Google Cloud Console. An unrestricted key in a public deployment can be abused and billed to you. +1. **Google Maps API key** — loads Photorealistic 3D Tiles directly and powers Street View (Maps JavaScript API; add it to the key's API restrictions). **Restrict it** (HTTP referrer + API restriction to the required Google APIs) in the Google Cloud Console. An unrestricted key in a public deployment can be abused and billed to you. Restrict it to the **Map Tiles API and Maps JavaScript API only**: a referrer restriction applies to the Maps JavaScript API and the client SDKs, not to the web services, so a key carrying Geocoding under one is refused with `API keys with referer restrictions cannot be used with this API`. Search does not need it — geocoding goes through the server proxies above (#363). 2. **Cesium ion token** (`CESIUM_ION_TOKEN`, optional — for ion-hosted Google Photorealistic 3D Tiles, Bing world imagery, and world terrain) — used as `Cesium.Ion.defaultAccessToken` client-side. Use a public **`assets:read`** token with **URL restrictions** for any hosted deployment. The Community plan has eligibility and usage limits; a public token is not a secret, but it can still consume the account's quota. 3. **Mapillary client token** (`MAPILLARY_CLIENT_TOKEN`, optional, Street Level) — used by the MapillaryJS viewer and Graph API lookups in the browser, and by the server's coverage-tile proxy. Mapillary client tokens are meant for browser use. > The explicit browser `define` block in `build/vite.js` controls exactly what reaches the client: only these three keys. Everything else stays server-side. -**Places and Street View never needed to be on that list** (#33): they're called from the server-side proxies in the table above, which use `GOOGLE_MAPS_SERVER_API_KEY` when it's set. Splitting it from the browser-exposed key lets each key's Google Cloud restriction actually match what it does — the browser key referrer-restricted to the APIs the client loads, the server key IP-restricted (never a referrer, since it never leaves your server) to Places + Street View Static — instead of one key that has to be either over-permissioned or broken for one of its two jobs. A single shared `GOOGLE_MAPS_API_KEY` still works if you don't split them; it just has to cover every API both sides use. +**Places, Geocoding and the Street View Static fallback never needed to be on that list** (#33, #363): they're called from the server-side proxies in the table above, which use `GOOGLE_MAPS_SERVER_API_KEY` when it's set. Splitting it from the browser-exposed key lets each key's Google Cloud restriction actually match what it does — the browser key referrer-restricted to Map Tiles + Maps JavaScript API, the server key IP-restricted (never a referrer, since it never leaves your server) to Places + Geocoding + Street View Static — instead of one key that has to be either over-permissioned or broken for one of its jobs. A single shared `GOOGLE_MAPS_API_KEY` still works if you don't split them, but then it cannot carry a referrer restriction at all: the web services refuse one, and search would be the half that breaks. Never commit real keys. `.env` is gitignored; only `.env.example` (placeholder names) is tracked. On macOS `dev-fresh.sh` can read keys from the Keychain; plain Vite uses env vars or a local `.env`, and Pinokio uses its ignored app `ENVIRONMENT` file. @@ -73,7 +74,11 @@ The data proxies under `server/providers/` are written so the browser cannot tur - **Coalesced OAuth refresh** and cached successful responses only (OpenSky). - **Redacted debug logging.** The voice debug log (`.gev-logs/`, gitignored) strips API keys, bearer tokens, client secrets, and image data URLs before writing. - **Cross-site gate on cost/log endpoints.** The cost-bearing (`/api/realtime/token`, `/api/openai/hud-summary`, `/api/google/nearby-places`, `/api/google/text-search`), quota-bearing (`/api/mapillary/*`, with a per-IP limit on tiles) and log (`/api/realtime/debug-log`) endpoints refuse cross-site browser requests: a foreign or opaque `Origin`, a `Sec-Fetch-Site` other than `same-origin`/`none` (this is what blocks an ``/navigation that carries no `Origin`), or any reverse-proxy/CDN forwarding header returns `403`. Non-browser loopback tools (the repo QA harnesses POST the token endpoint with no `Origin`) and the explicit `HOST=0.0.0.0` LAN opt-in keep working — the gate deliberately does not require a loopback remote address; that stricter requirement belongs only to the credential panel (`admitKeySetupRequest`). The Host check (`allowedHosts`, applied by `hostCheckPlugin` ahead of every route, since Vite's own check runs only after plugin middleware) rejects a rebinding `Host` before any handler runs, in every mode, and the gate is defense in depth behind it: the LAN opt-in keeps that check and accepts only IP addresses, `localhost` and names listed in `GEV_ALLOWED_HOSTS`. **Limit:** a name you list there is trusted like the app's own address, so list only names you control. +- **Full Content-Security-Policy.** The dev and preview servers send a real CSP (`script-src 'self' 'unsafe-eval' blob:` plus the YouTube, Facebook and X scripts the Nepal event's embedded media loads and the Maps JavaScript API that Street View runs on, with a matching `frame-src` for their players and posts: no inline script and no other script origin; `blob:` covers the object URLs Cesium's bundled workers bootstrap through in the built app), alongside the existing `X-Frame-Options: DENY` and `frame-ancestors 'none'`. `'unsafe-eval'` stays because Knockout, bundled inside Cesium's widgets package, resolves the global object with `eval` at load time and the globe does not initialize without it; styles permit `'unsafe-inline'` and Google Fonts because Vite's dev client injects `'; +const notes = 'A local observation with supporting detail. '.repeat(120); +const placemarks = Array.from({ length: 5000 }, (_, i) => { + const lon = -122.6 + (i % 100) * 0.002; + const lat = 37.65 + Math.floor(i / 100) * 0.003; + return `Point ${i}#base${style}${notes}${lon},${lat},0`; +}); +const wrap = (body) => + `${body}`; +const large = wrap(placemarks.join('')); +fs.writeFileSync(file, large); +const bytes = fs.statSync(file).size; +if (bytes < 20 * 1024 * 1024 || bytes > 30 * 1024 * 1024) + throw new Error(`Fixture must be 20–30 MiB, got ${bytes} bytes`); +const small = wrap( + Array.from({ length: 6 }, (_, i) => { + const kind = i % 3; + const geometry = + kind === 0 + ? '-122.49,37.77,0' + : kind === 1 + ? '-122.50,37.77,0 -122.49,37.77,0' + : '-122.50,37.77,0 -122.49,37.77,0 -122.49,37.78,0 -122.50,37.77,0'; + const override = + kind === 0 + ? style + : kind === 1 + ? '' + : ''; + return `Small ${i}#base${override}${geometry}`; + }).join('') + + '' + + '' + + 'normal' + + ['named-a', 'named-b', 'paired'] + .map( + (id) => + `${id}#${id}-122.49,37.77,0`, + ) + .join(''), +); +const browser = await puppeteer.launch({ + headless: true, + args: + process.platform === 'darwin' + ? ['--use-gl=angle', '--use-angle=metal'] + : ['--no-sandbox', '--use-gl=angle', '--use-angle=swiftshader'], +}); +const page = await browser.newPage(); +await hookRenderErrors(page); +const errors = []; +const fileRequests = []; +page.on('pageerror', (e) => errors.push(e.message)); +await page.setRequestInterception(true); +page.on('request', (request) => { + if (new URL(request.url()).hostname === 'earthquake.usgs.gov') { + fileRequests.push(request.url()); + void request.abort(); + } else void request.continue(); +}); +let failures = 0; +const check = (name, ok, detail) => { + console.log(`[${ok ? 'PASS' : 'FAIL'}] ${name}`, detail ?? ''); + if (!ok) failures++; +}; +try { + await page.setViewport({ width: 1440, height: 900 }); + await page.goto(`${base}/?welcome=0`, { waitUntil: 'domcontentloaded' }); + await page.waitForFunction( + () => + window.__gevGeoImport && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 60000 }, + ); + await page.evaluate(async () => { + const { viewer, dataManager } = window.__godsEyeView; + const C = window.__CESIUM__; + for (const id of dataManager.layers.keys()) + await dataManager.setEnabled(id, false); + viewer.clock.shouldAnimate = false; + viewer.imageryLayers.removeAll(); + viewer.terrainProvider = new C.EllipsoidTerrainProvider(); + viewer.camera.cancelFlight(); + viewer.camera.setView({ + destination: C.Cartesian3.fromDegrees(-122.5, 37.72, 50000), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + window.__qaStyleSnapshot = (source) => { + const now = viewer.clock.currentTime; + const images = new Map(); + const imageValue = (p) => { + const value = p?.getValue(now); + if (value?.tagName !== 'CANVAS') return value; + if (!images.has(value)) images.set(value, value.toDataURL()); + return images.get(value); + }; + const color = (p) => p?.getValue(now)?.toCssColorString(); + const positions = (p) => p?.getValue(now)?.map((v) => [v.x, v.y, v.z]); + const renderedPins = new Map(); + const collection = source.importedBillboards; + if (collection) + for (let i = 0; i < collection.length; i++) { + const pin = collection.get(i); + renderedPins.set(pin.id, pin); + } + const actual = (e, key, fallback) => { + const pin = renderedPins.get(e); + return pin ? pin[key] : (e.billboard?.[key]?.getValue(now) ?? fallback); + }; + return (source.importedEntities || source.entities.values) + .map((e) => ({ + name: e.name, + point: renderedPins.get(e)?.position || e.position?.getValue(now), + scale: e.billboard ? actual(e, 'scale', 1) : undefined, + rotation: e.billboard ? actual(e, 'rotation', 0) : undefined, + image: renderedPins.get(e)?.image || imageValue(e.billboard?.image), + color: e.billboard + ? actual(e, 'color', C.Color.WHITE).toCssColorString() + : undefined, + width: e.polyline?.width?.getValue(now), + lineColor: color(e.polyline?.material?.color), + line: positions(e.polyline?.positions), + fill: e.polygon?.fill?.getValue(now), + outline: e.polygon?.outline?.getValue(now), + polygonColor: color(e.polygon?.material?.color), + polygon: e.polygon?.hierarchy + ?.getValue(now) + ?.positions?.map((v) => [v.x, v.y, v.z]), + })) + .sort((a, b) => a.name.localeCompare(b.name)); + }; + window.__qaUnsharedKml = async (text) => { + const doc = new DOMParser().parseFromString(text, 'application/xml'); + for (const el of [...doc.getElementsByTagName('*')]) + if ( + ['description', 'ExtendedData', 'BalloonStyle'].includes(el.localName) + ) + el.remove(); + const source = await C.KmlDataSource.load(doc, { + camera: viewer.scene.camera, + canvas: viewer.scene.canvas, + clampToGround: true, + }); + for (const e of source.importedEntities || source.entities.values) + e.label = undefined; + return source; + }; + }); + const equivalent = await page.evaluate(async (text) => { + const { viewer } = window.__godsEyeView; + const control = await window.__qaUnsharedKml(text); + const outcomes = await window.__gevGeoImport.importFiles([ + new File([text], 'style-equivalence.kml'), + ]); + const candidate = viewer.dataSources.get(viewer.dataSources.length - 1); + const result = { + ok: outcomes[0]?.ok, + same: + JSON.stringify(window.__qaStyleSnapshot(control)) === + JSON.stringify(window.__qaStyleSnapshot(candidate)), + entities: (candidate.importedEntities || candidate.entities.values) + .length, + }; + await window.__gevGeoImport.remove(outcomes[0]?.id); + return result; + }, small); + check( + 'shared inline and named styles preserve point, line and polygon overrides/inheritance', + equivalent.ok && equivalent.same && equivalent.entities === 9, + JSON.stringify(equivalent), + ); + + // The control bypasses the former style-count refusal solely in this test, + // to compare native unshared rendering against normalized rendering. + await page.evaluate(async (text) => { + const source = await window.__qaUnsharedKml(text); + window.__qaUnsharedSnapshot = JSON.stringify( + window.__qaStyleSnapshot(source), + ); + window.__qaControl = source; + await window.__godsEyeView.viewer.dataSources.add(source); + }, large); + const measure = async () => + page.evaluate(async () => { + const C = window.__CESIUM__, + { viewer } = window.__godsEyeView; + const scene = viewer.scene; + viewer.camera.cancelFlight(); + viewer.camera.setView({ + destination: C.Cartesian3.fromDegrees(-122.5, 37.72, 50000), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + const started = performance.now(), + samples = []; + let last, + frames = 0; + const remove = scene.postRender.addEventListener(() => { + const t = performance.now(); + if (t - started < 2000) return; + frames++; + if (last !== undefined) samples.push(t - last); + last = t; + }); + await new Promise((resolve) => { + let previous = performance.now(); + function tick(t) { + viewer.camera.rotate( + C.Cartesian3.UNIT_Z, + Math.min(t - previous, 100) * 0.0000002, + ); + previous = t; + scene.requestRender(); + if (t - started < 7000) requestAnimationFrame(tick); + else resolve(); + } + requestAnimationFrame(tick); + }); + remove(); + samples.sort((a, b) => a - b); + const gl = scene.context._gl, + debug = gl.getExtension('WEBGL_debug_renderer_info'); + return { + fps: frames / 5, + p50Ms: samples[Math.floor(samples.length * 0.5)], + p95Ms: samples[Math.floor(samples.length * 0.95)], + renderer: debug ? gl.getParameter(debug.UNMASKED_RENDERER_WEBGL) : null, + }; + }); + console.log('UNSHARED CONTROL', JSON.stringify(await measure())); + await page.evaluate(() => + window.__godsEyeView.viewer.dataSources.remove(window.__qaControl, true), + ); + + const loadPicked = async (filename, expected) => { + const before = await page.evaluate( + () => window.__godsEyeView.viewer.dataSources.length, + ); + const priorIds = await page.evaluate(() => + window.__gevGeoImport.list().map((f) => f.id), + ); + const started = performance.now(); + await (await page.$('#geo-import-input')).uploadFile(filename); + const name = path.basename(filename); + const loaded = await page + .waitForFunction( + ({ name, priorIds }) => + window.__gevGeoImport.list().some((f) => !priorIds.includes(f.id)) || + document + .getElementById('geo-import-hint') + .textContent.startsWith(`${name}:`), + { timeout: 120000 }, + { name, priorIds }, + ) + .then( + () => true, + () => false, + ); + const state = await page.evaluate(() => ({ + files: window.__gevGeoImport.list(), + hint: document.getElementById('geo-import-hint').textContent, + sources: window.__godsEyeView.viewer.dataSources.length, + })); + const item = state.files.find((f) => !priorIds.includes(f.id)); + check( + `picked ${name} loads`, + loaded && + (expected === null + ? item?.entityCount > 0 + : item?.entityCount === expected) && + state.sources === before + 1, + JSON.stringify({ + bytes: fs.statSync(filename).size, + loadMs: Math.round(performance.now() - started), + ...state, + }), + ); + return item; + }; + const item = await loadPicked(file, 5000); + if (item) { + const same = await page.evaluate(() => { + const viewer = window.__godsEyeView.viewer; + const source = viewer.dataSources.get(viewer.dataSources.length - 1); + return ( + JSON.stringify(window.__qaStyleSnapshot(source)) === + window.__qaUnsharedSnapshot + ); + }); + check('all 5,000 normalized graphics match their unshared control', same); + console.log('NORMALIZED CANDIDATE', JSON.stringify(await measure())); + await page.evaluate((id) => window.__gevGeoImport.remove(id), item.id); + } + if (supplied) { + const clockBefore = await page.evaluate(() => { + const v = window.__godsEyeView.viewer, + C = window.__CESIUM__; + return [ + C.JulianDate.toIso8601(v.clock.startTime), + C.JulianDate.toIso8601(v.clock.stopTime), + C.JulianDate.toIso8601(v.clock.currentTime), + v.clock.multiplier, + ]; + }); + const suppliedItem = await loadPicked(supplied, null); + const clockAfter = await page.evaluate(() => { + const v = window.__godsEyeView.viewer, + C = window.__CESIUM__; + return [ + C.JulianDate.toIso8601(v.clock.startTime), + C.JulianDate.toIso8601(v.clock.stopTime), + C.JulianDate.toIso8601(v.clock.currentTime), + v.clock.multiplier, + ]; + }); + check( + 'real dated KML leaves the shared clock unchanged', + JSON.stringify(clockBefore) === JSON.stringify(clockAfter), + ); + if (suppliedItem && process.argv.includes('--expect-local-pins')) { + const markers = await page.evaluate(() => { + const { viewer } = window.__godsEyeView, + C = window.__CESIUM__; + const source = viewer.dataSources.get(viewer.dataSources.length - 1); + const points = ( + source.importedEntities || source.entities.values + ).filter((e) => e.position && e.billboard); + const images = points.filter((e) => + e.billboard.image?.getValue(viewer.clock.currentTime), + ); + if (images.length) { + const at = C.Cartographic.fromCartesian( + images[0].position.getValue(viewer.clock.currentTime), + ); + viewer.camera.cancelFlight(); + viewer.camera.setView({ + destination: C.Cartesian3.fromRadians( + at.longitude, + at.latitude, + 100000, + ), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + } + return { points: points.length, images: images.length }; + }); + check( + 'supplied point markers have drawable images', + markers.points > 0 && markers.images === markers.points, + JSON.stringify(markers), + ); + if (markers.points) { + const yellowPixels = async () => { + const shot = await page.screenshot({ + clip: { x: 620, y: 350, width: 200, height: 200 }, + type: 'png', + encoding: 'base64', + }); + return page.evaluate(async (base64) => { + const bitmap = await createImageBitmap( + await (await fetch(`data:image/png;base64,${base64}`)).blob(), + ); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height), + context = canvas.getContext('2d'); + context.drawImage(bitmap, 0, 0); + const data = context.getImageData( + 0, + 0, + bitmap.width, + bitmap.height, + ).data; + let count = 0; + for (let i = 0; i < data.length; i += 4) + if (data[i] > 150 && data[i + 1] > 150 && data[i + 2] < 100) + count++; + return count; + }, shot); + }; + const setVisible = (visible) => + page.evaluate( + ({ id, visible }) => window.__gevGeoImport.setVisible(id, visible), + { id: suppliedItem.id, visible }, + ); + await setVisible(false); + await new Promise((resolve) => setTimeout(resolve, 1000)); + const hidden = await yellowPixels(); + await setVisible(true); + let shown = hidden; + const deadline = performance.now() + 10000; + while (performance.now() < deadline && shown <= hidden + 10) { + await new Promise((resolve) => setTimeout(resolve, 500)); + shown = await yellowPixels(); + } + check( + 'real USGS markers paint when shown and disappear when hidden', + shown > hidden + 10, + JSON.stringify({ hidden, shown }), + ); + await page.screenshot({ + path: path.join(directory, 'real-kml-visible.png'), + }); + } + } + if (suppliedItem) + await page.evaluate( + (id) => window.__gevGeoImport.remove(id), + suppliedItem.id, + ); + } + check( + 'file-directed USGS icon/metadata requests never leave the page', + fileRequests.length === 0, + fileRequests.length, + ); + errors.push(...(await readRenderErrors(page))); + check( + 'no uncaught page or render errors', + errors.length === 0, + JSON.stringify(errors), + ); + check( + 'all imported sources are removed', + await page.evaluate(() => window.__gevGeoImport.list().length === 0), + ); + console.log('Fixture retained at', file); +} finally { + await browser.close(); +} +process.exitCode = failures ? 1 : 0; diff --git a/scripts/qa-geo-import-regressions.mjs b/scripts/qa-geo-import-regressions.mjs new file mode 100644 index 000000000..ba5a040a2 --- /dev/null +++ b/scripts/qa-geo-import-regressions.mjs @@ -0,0 +1,564 @@ +#!/usr/bin/env node +/** Import regressions through the real UI controller and Cesium loaders. + * Run against dev and preview: QA_BASE_URL=http://localhost:4173 node scripts/qa-geo-import-regressions.mjs + */ +import puppeteer from 'puppeteer'; +import * as Cesium from 'cesium'; +import { hookRenderErrors, readRenderErrors } from './qa-browserEvidence.mjs'; + +const browser = await puppeteer.launch({ headless: true }); +const page = await browser.newPage(); +const meshBytes = Buffer.alloc(60); +[ + -2000, 2000, -2000, 2000, 2000, -2000, -2000, 2000, 2000, 2000, 2000, 2000, +].forEach((value, i) => meshBytes.writeFloatLE(value, i * 4)); +[0, 1, 2, 2, 1, 3].forEach((value, i) => + meshBytes.writeUInt16LE(value, 48 + i * 2), +); +const fixture = { + '/__qa_import_surface/tileset.json': { + asset: { version: '1.1' }, + geometricError: 0, + root: { + transform: Cesium.Matrix4.toArray( + Cesium.Transforms.eastNorthUpToFixedFrame( + Cesium.Cartesian3.fromDegrees(-122.5, 37.7), + ), + ), + boundingVolume: { box: [0, 0, 2000, 2000, 0, 0, 0, 2000, 0, 0, 0, 100] }, + geometricError: 0, + content: { uri: 'surface.gltf' }, + }, + }, + '/__qa_import_surface/surface.gltf': { + asset: { version: '2.0' }, + scene: 0, + scenes: [{ nodes: [0] }], + nodes: [{ mesh: 0 }], + meshes: [ + { + primitives: [{ attributes: { POSITION: 0 }, indices: 1, material: 0 }], + }, + ], + materials: [{ doubleSided: true }], + buffers: [ + { + byteLength: 60, + uri: `data:application/octet-stream;base64,${meshBytes.toString('base64')}`, + }, + ], + bufferViews: [ + { buffer: 0, byteOffset: 0, byteLength: 48, target: 34962 }, + { buffer: 0, byteOffset: 48, byteLength: 12, target: 34963 }, + ], + accessors: [ + { + bufferView: 0, + componentType: 5126, + count: 4, + type: 'VEC3', + min: [-2000, 2000, -2000], + max: [2000, 2000, 2000], + }, + { bufferView: 1, componentType: 5123, count: 6, type: 'SCALAR' }, + ], + }, +}; +await page.setRequestInterception(true); +page.on('request', (request) => { + const body = fixture[new URL(request.url()).pathname]; + if (body) + void request.respond({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(body), + }); + else void request.continue(); +}); +const errors = [], + requests = []; +page.on('pageerror', (e) => errors.push(e.message)); +page.on('console', (m) => { + if (/Ignoring clampToGround/.test(m.text())) errors.push(m.text()); +}); +await hookRenderErrors(page); +let failed = 0; +try { + await page.setViewport({ width: 390, height: 844 }); + await page.goto( + `${process.env.QA_BASE_URL || 'http://localhost:4173'}/?welcome=0`, + { waitUntil: 'domcontentloaded' }, + ); + await page.waitForFunction( + () => + window.__gevGeoImport && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 90000 }, + ); + await page.evaluate(async () => { + const { viewer, dataManager } = window.__godsEyeView, + C = window.__CESIUM__; + for (const id of dataManager.layers.keys()) + await dataManager.setEnabled(id, false); + viewer.imageryLayers.removeAll(); + viewer.terrainProvider = new C.EllipsoidTerrainProvider(); + viewer.clock.shouldAnimate = false; + viewer.camera.cancelFlight(); + viewer.camera.setView({ + destination: C.Cartesian3.fromDegrees(-122.5, 37.7, 100000), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + }); + page.on('request', (r) => { + if (!/^(data:|blob:)/.test(r.url())) requests.push(r.url()); + }); + const results = await page.evaluate(async () => { + const { viewer } = window.__godsEyeView, + C = window.__CESIUM__, + tool = window.__gevGeoImport; + const results = [], + now = C.JulianDate.clone(viewer.clock.currentTime); + const check = (name, pass, detail = '') => + results.push({ name, pass: Boolean(pass), detail }); + const wrap = (body, attrs = '') => + `${body}`; + const point = '-122.5,37.7,0'; + const load = async (value, name) => { + const content = + typeof value === 'object' && !ArrayBuffer.isView(value) + ? JSON.stringify(value) + : value; + const [outcome] = await tool.importFiles([new File([content], name)]); + return { + outcome, + source: outcome.ok + ? viewer.dataSources.get(viewer.dataSources.length - 1) + : null, + }; + }; + const remove = (r) => { + if (r.outcome.ok) tool.remove(r.outcome.id); + }; + let r = await load( + { + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + id: 'kartaview-owned', + properties: { + name: 'XYZ', + addProperty: 1, + 'marker-symbol': 'rail', + }, + geometry: { type: 'Point', coordinates: [-122.5, 37.7, 100] }, + }, + { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [-122.5, 37.7, 10], + [-122.4, 37.7, 10], + [-122.4, 37.8, 10], + [-122.5, 37.7, 10], + ], + ], + }, + }, + ], + }, + 'xyz.geojson', + ); + const pin = (r.source?.importedEntities || r.source?.entities.values).find( + (e) => e.billboard, + ), + polygon = (r.source?.importedEntities || r.source?.entities.values).find( + (e) => e.polygon, + ); + check( + 'XYZ ground footprint and isolated entity IDs', + r.outcome.ok && + pin.id !== 'kartaview-owned' && + Math.abs( + C.Cartographic.fromCartesian(pin.position.getValue(now)).height, + ) < 0.01 && + polygon.polygon.perPositionHeight?.getValue(now) !== true, + ); + check( + 'ground pins have no persistent clamping callbacks', + pin?.billboard.heightReference?.getValue(now) === + C.HeightReference.NONE && + pin.billboard.disableDepthTestDistance.getValue(now) === Infinity, + ); + remove(r); + r = await load( + wrap( + 'ground-122.5 37.7 0-122.4 37.8 0' + + 'absoluteabsolute-122.5 37.7 10000-122.4 37.8 10000' + + 'line-122.5,37.7 -122.4,37.8', + ), + 'tracks.kml', + ); + const find = (name) => + (r.source?.importedEntities || r.source?.entities.values).find( + (e) => e.name === name, + ); + check( + 'ground tracks and untessellated authored lines clamp', + find('ground')?.polyline.clampToGround?.getValue(now) && + find('line')?.polyline.clampToGround?.getValue(now), + ); + check( + 'explicit track altitude and original vertices survive', + find('absolute')?.polyline.positions.getValue(now).length === 2 && + find('absolute') + .polyline.positions.getValue(now) + .every( + (p) => + Math.abs(C.Cartographic.fromCartesian(p).height - 10000) < 0.01, + ), + ); + remove(r); + r = await load( + wrap( + ` #s ${point}`, + ), + 'local-style.kml', + ); + check('whitespace around a local style remains local', r.outcome.ok); + remove(r); + r = await load( + wrap( + `normalhttps://gev-qa-trap.invalid/style${point}`, + ), + 'incomplete-style.kml', + ); + check('removing external style does not hang the queue', r.outcome.ok); + remove(r); + r = await load( + wrap( + `0${point}`, + ), + 'hidden-folder.kml', + ); + check( + 'import show controls own previously hidden KML content', + r.outcome.ok && + (r.source.importedEntities || r.source.entities.values).every( + (e) => e.isShowing, + ), + ); + remove(r); + r = await load( + wrap( + 'ff00ff0037.837.7-122.4-122.5', + ), + 'colored.kml', + ); + check( + 'a colored overlay without an icon is drawable', + r.outcome.ok && + (r.source.importedEntities || r.source.entities.values).some( + (e) => e.rectangle, + ), + ); + remove(r); + const xml = wrap(`${point}`); + const utf16 = new Uint8Array(2 + xml.length * 2); + utf16.set([255, 254]); + for (let i = 0; i < xml.length; i++) { + utf16[2 + i * 2] = xml.charCodeAt(i) & 255; + utf16[3 + i * 2] = xml.charCodeAt(i) >> 8; + } + r = await load(utf16, 'utf16.kml'); + check('UTF-16 BOM is decoded', r.outcome.ok); + remove(r); + r = await load( + wrap( + `${point}`, + 'xmlns:gx="http://www.google.com/kml/ext/2.2" xsi:schemaLocation="test"', + ), + 'namespaces.kml', + ); + check( + 'ordinary duplicate and missing export namespaces are repaired', + r.outcome.ok, + ); + remove(r); + for (const [name, value] of [ + ['null geometry', { type: 'Feature', geometry: null }], + ['invalid coordinates', { type: 'Point', coordinates: ['bad', 37] }], + [ + 'missing feature type', + { + type: 'FeatureCollection', + features: [{ geometry: { type: 'Point', coordinates: [0, 0] } }], + }, + ], + [ + 'nested unsupported geometry', + { type: 'Feature', geometry: { type: 'Topology', arcs: [] } }, + ], + ]) { + r = await load(value, 'refused.geojson'); + check(`${name} is refused without stopping rendering`, !r.outcome.ok); + remove(r); + } + r = await load( + wrap( + 'NaN,37,0', + ), + 'invalid.kml', + ); + check('non-finite KML coordinates are refused', !r.outcome.ok); + remove(r); + r = await load( + { + type: 'LineString', + coordinates: [ + [-122.6, 37.7], + [-122.4, 37.7], + ], + }, + 'portrait.geojson', + ); + await new Promise((resolve) => setTimeout(resolve, 2300)); + const positions = (r.source?.importedEntities || + r.source?.entities.values)[0].polyline.positions.getValue(now); + const screen = positions?.map((p) => + C.SceneTransforms.worldToWindowCoordinates(viewer.scene, p), + ); + check( + 'portrait flight includes both route endpoints', + screen?.every( + (p) => p && p.x > 0 && p.x < innerWidth && p.y > 0 && p.y < innerHeight, + ), + screen?.map((p) => p && [p.x, p.y]), + ); + remove(r); + r = await load( + { + type: 'MultiPoint', + coordinates: [ + [-74, 40.7], + [139.7, 35.7], + ], + }, + 'intercontinental.geojson', + ); + await new Promise((resolve) => setTimeout(resolve, 2300)); + const intercontinental = ( + r.source?.importedEntities || r.source?.entities.values + ) + .filter((e) => e.position) + .map((e) => e.position.getValue(now)); + const occluder = new C.EllipsoidalOccluder( + C.Ellipsoid.WGS84, + viewer.camera.positionWC, + ); + check( + 'intercontinental points remain in frame and in front of the horizon', + intercontinental?.every((p) => { + const s = C.SceneTransforms.worldToWindowCoordinates(viewer.scene, p); + return ( + occluder.isPointVisible(p) && + s && + s.x > 0 && + s.x < innerWidth && + s.y > 0 && + s.y < innerHeight + ); + }), + ); + remove(r); + r = await load( + wrap( + 'absolute14.9,52.37,3000 5.12,52.09,30005.12,52.09', + ), + 'wall.kml', + ); + await new Promise((resolve) => setTimeout(resolve, 2300)); + check( + 'a wall contributes both endpoints to the flight', + r.outcome.ok && + (r.source.importedEntities || r.source.entities.values) + .find((e) => e.wall) + ?.wall.positions.getValue(now) + .every((p) => { + const s = C.SceneTransforms.worldToWindowCoordinates( + viewer.scene, + p, + ); + return ( + s && s.x > 0 && s.x < innerWidth && s.y > 0 && s.y < innerHeight + ); + }), + ); + remove(r); + r = await load( + { + type: 'MultiPoint', + coordinates: [ + [0, 0], + [180, 0], + ], + }, + 'antipodal.geojson', + ); + await new Promise((resolve) => setTimeout(resolve, 1800)); + check( + 'symmetric global extents keep a finite camera', + r.outcome.ok && + Number.isFinite(viewer.camera.positionCartographic.height), + ); + remove(r); + viewer.camera.cancelFlight(); + viewer.camera.setView({ + destination: C.Cartesian3.fromDegrees(-122.5, 37.7, 100000), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + const mesh = await C.Cesium3DTileset.fromUrl( + `${location.origin}/__qa_import_surface/tileset.json`, + ); + viewer.scene.primitives.add(mesh); + // This fixture installs a mesh outside the app's map-source lifecycle. + // Let the new camera/tiles reach a rendered frame before testing sampling. + await new Promise((resolve) => { + let frames = 0; + const remove = viewer.scene.postRender.addEventListener(() => { + if (++frames >= 2) { + remove(); + resolve(); + } else viewer.scene.requestRender(); + }); + viewer.scene.requestRender(); + }); + const fixtureDeadline = performance.now() + 5000; + while (!mesh.tilesLoaded && performance.now() < fixtureDeadline) { + viewer.scene.requestRender(); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + const globeShown = viewer.scene.globe.show; + viewer.scene.globe.show = false; + + r = await load( + { type: 'Point', coordinates: [-122.5, 37.7, 0] }, + 'mesh-point.geojson', + ); + const meshPoint = (r.source?.importedEntities || r.source?.entities.values) + .find((e) => e.billboard) + ?.position.getValue(now); + const meshHeight = + meshPoint && C.Cartographic.fromCartesian(meshPoint).height; + check( + 'native 3D Tiles ground sampling keeps pins on the rendered mesh', + r.outcome.ok && Math.abs(meshHeight - 2000) < 2, + { + meshHeight, + tilesLoaded: mesh.tilesLoaded, + globeShown: viewer.scene.globe.show, + }, + ); + tool.flyTo(r.outcome.id); + await new Promise((resolve) => setTimeout(resolve, 2300)); + const meshScreen = + meshPoint && + C.SceneTransforms.worldToWindowCoordinates(viewer.scene, meshPoint); + check( + 'a high-surface arrival frames the elevated pin', + meshScreen && + Math.abs(meshScreen.x - innerWidth / 2) < 20 && + Math.abs(meshScreen.y - innerHeight / 2) < 20, + { + screen: meshScreen && [meshScreen.x, meshScreen.y], + camera: { + height: viewer.camera.positionCartographic.height, + pitch: viewer.camera.pitch, + }, + meshHeight, + }, + ); + const groundSource = r.source; + const elevated = await load( + wrap( + 'absolute-122.5,37.7,5000', + ), + 'elevated-pin.kml', + ); + await new Promise((resolve) => setTimeout(resolve, 2600)); + check( + 'a higher imported pin never becomes the sampled ground', + elevated.outcome.ok && + Math.abs( + C.Cartographic.fromCartesian( + groundSource.importedBillboards.get(0).position, + ).height - 2000, + ) < 2, + ); + remove(elevated); + remove(r); + viewer.scene.primitives.remove(mesh); + viewer.scene.globe.show = globeShown; + const before = tool.list().length; + const transfer = new DataTransfer(); + transfer.items.add( + new File( + [JSON.stringify({ type: 'Point', coordinates: [-122.5, 37.7] })], + 'panel-drop.geojson', + ), + ); + const event = new DragEvent('drop', { + bubbles: true, + cancelable: true, + dataTransfer: transfer, + }); + document.getElementById('geo-import-button').dispatchEvent(event); + for (let i = 0; i < 100 && tool.list().length === before; i++) + await new Promise((resolve) => setTimeout(resolve, 20)); + check( + 'a panel drop imports and prevents the browser default', + event.defaultPrevented && tool.list().length === before + 1, + ); + for (const file of tool.list()) tool.remove(file.id); + check( + 'the shared clock stays exactly unchanged', + C.JulianDate.equals(now, viewer.clock.currentTime), + ); + check('no file sources remain', tool.list().length === 0); + return results; + }); + for (const r of results) { + console.log( + `[${r.pass ? 'PASS' : 'FAIL'}] ${r.name}`, + JSON.stringify(r.detail), + ); + if (!r.pass) failed++; + } + const renderErrors = await readRenderErrors(page); + if (errors.length || renderErrors.length) { + failed++; + console.log( + '[FAIL] browser/render errors', + JSON.stringify({ errors, renderErrors }), + ); + } + // Terrain belongs to the base map; imported files may sample that provider. + const unexpected = requests.filter( + (url) => + new URL(url).pathname !== '/logo.svg' && + !new URL(url).pathname.startsWith('/__qa_import_surface/') && + !/\/api\/|\.terrain(?:\?|$)|reearth|cesium|arcgis|openstreetmap|\.pbf(?:\?|$)/i.test( + url, + ), + ); + console.log( + `[${unexpected.length ? 'FAIL' : 'PASS'}] no unexpected file-directed requests`, + JSON.stringify(unexpected), + ); + if (unexpected.length) failed++; +} finally { + await browser.close(); +} +process.exitCode = failed ? 1 : 0; diff --git a/scripts/qa-geo-import.mjs b/scripts/qa-geo-import.mjs new file mode 100644 index 000000000..074eb8fbd --- /dev/null +++ b/scripts/qa-geo-import.mjs @@ -0,0 +1,594 @@ +#!/usr/bin/env node +/** + * Rendered acceptance for DISPLAY ▸ Import: real files through the real file + * input and a real drop, parsed by Cesium in the page. + * + * Proves: GeoJSON, KML, KMZ (with a packed icon and ground overlay) and GPX + * each become one row and one data source; the imported shape actually paints + * on the globe in its row's color (a GeoJSON `stroke`/`fill` does not override + * it) and stops painting when hidden; no import makes the page request + * anything named in it (network link, remote icon, remote style, palette + * icon, balloon or extended-data HTML, maki marker) or anything outside the + * page for it; no entity keeps a description, label or properties; dated KML + * and GPX leave the app's clock alone and still draw now; a GPX track keeps + * its two segments apart; keyboard focus survives hide/show; a dropped file + * imports like a picked one; a file that is not geodata is refused with a + * reason and leaves nothing behind; Remove gives every data source back; no + * page errors. + * + * Usage: dev server up (`npm run dev`), then `node scripts/qa-geo-import.mjs`. + * QA_BASE_URL (default http://localhost:4173), QA_SHOT_DIR, QA_HEADFUL=1. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import puppeteer from 'puppeteer'; +import { hookRenderErrors, readRenderErrors } from './qa-browserEvidence.mjs'; + +const BASE_URL = process.env.QA_BASE_URL || 'http://localhost:4173'; +const SHOT_DIR = process.env.QA_SHOT_DIR || 'qa-shots/geo-import'; +const HEADFUL = process.env.QA_HEADFUL === '1'; +// Golden Gate Park: a big, flat, green rectangle the test polygon can cover. +const VIEW = { lon: -122.4862, lat: 37.7694, height: 4200, pitch: -88 }; +const TRAP_HOST = 'gev-qa-trap.invalid'; + +/* ── fixtures ──────────────────────────────────────────────────────────── */ + +const ring = [ + [-122.511, 37.7645], + [-122.4545, 37.7645], + [-122.4545, 37.7745], + [-122.511, 37.7745], + [-122.511, 37.7645], +]; +const geojson = JSON.stringify({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + // A file's own simplestyle and HTML must not survive the import. + properties: { + name: 'QA park', + stroke: '#ff0000', + fill: '#ff0000', + 'fill-opacity': 1, + description: ``, + }, + geometry: { type: 'Polygon', coordinates: [ring.map((p) => [...p, 0])] }, + }, + { + type: 'Feature', + properties: { name: 'QA pin', 'marker-symbol': 'rail' }, + geometry: { type: 'Point', coordinates: [-122.4862, 37.7694, 0] }, + }, + ], +}); +const kml = (iconHref) => ` + + + + + https://${TRAP_HOST}/link.kml + QA KML pin#s + ]]> + 2019-01-01T00:00:00Z2019-01-02T00:00:00Z + ]]> + -122.47,37.77,0 + QA palette pin#palette + -122.48,37.771,0 + QA moving + 2019-01-01T00:00:00Z2019-01-01T01:00:00Z + -122.49 37.767 0-122.475 37.772 0 + QA split tracks + -122.50 37.767 0-122.49 37.768 0 + -122.47 37.771 0-122.46 37.772 0 + + QA remote stylehttps://${TRAP_HOST}/styles.kml#x + -122.50,37.766 -122.46,37.772 + https://${TRAP_HOST}/overlay.png + 37.7737.76-122.46-122.47 + +`; +const gpx = ` + + QA waypoint + QA track<img src="https://${TRAP_HOST}/gpx.png"> + + + + +`; +// 1×1 PNG. +const PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==', + 'base64', +); +/** Stored (uncompressed) zip — enough for a KMZ fixture. */ +function storedZip(files) { + const parts = []; + const central = []; + let offset = 0; + for (const [name, data] of files) { + const n = Buffer.from(name); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt32LE(data.length, 18); + local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(n.length, 26); + const c = Buffer.alloc(46); + c.writeUInt32LE(0x02014b50, 0); + c.writeUInt32LE(data.length, 20); + c.writeUInt32LE(data.length, 24); + c.writeUInt16LE(n.length, 28); + c.writeUInt32LE(offset, 42); + parts.push(local, n, data); + central.push(c, n); + offset += 30 + n.length + data.length; + } + const dir = Buffer.concat(central); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(files.length, 8); + end.writeUInt16LE(files.length, 10); + end.writeUInt32LE(dir.length, 12); + end.writeUInt32LE(offset, 16); + return Buffer.concat([...parts, dir, end]); +} +const kmzDoc = kml('files/pin.png').replace( + `https://${TRAP_HOST}/overlay.png`, + 'files/overlay.png', +); + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-qa-geo-import-')); +const fixture = (name, data) => { + const file = path.join(dir, name); + fs.writeFileSync(file, data); + return file; +}; +const FILES = [ + fixture('park.geojson', geojson), + fixture('remote-bits.kml', kml(`https://${TRAP_HOST}/icon.png`)), + fixture( + 'packed.kmz', + storedZip([ + ['doc.kml', Buffer.from(kmzDoc)], + ['files/pin.png', PNG], + ['files/overlay.png', PNG], + ]), + ), + fixture('walk.gpx', gpx), +]; + +/* ── harness ───────────────────────────────────────────────────────────── */ + +const browser = await puppeteer.launch({ + headless: !HEADFUL, + args: [ + '--no-sandbox', + '--disable-background-timer-throttling', + '--disable-renderer-backgrounding', + ...(process.platform === 'darwin' + ? ['--use-angle=metal', '--enable-gpu'] + : ['--use-gl=angle', '--use-angle=swiftshader']), + ], +}); +const page = await browser.newPage(); +await hookRenderErrors(page); +const errors = []; +const trapped = []; +page.on('pageerror', (error) => errors.push(`pageerror: ${error.message}`)); +page.on('console', (message) => { + if (/Ignoring clampToGround/.test(message.text())) + errors.push(message.text()); +}); +// Anything a file names, and the icon assets Cesium would load on a file's +// behalf (KML palette icons from maps.google.com, GeoJSON maki markers). +const FILE_FETCH = [TRAP_HOST, 'maps.google.com/mapfiles', '/maki/']; +page.on('request', (request) => { + if (FILE_FETCH.some((needle) => request.url().includes(needle))) + trapped.push(request.url()); +}); +let failures = 0; +function check(name, passed, detail = '') { + console.log( + `[${passed ? 'PASS' : 'FAIL'}] ${name}${detail ? ` — ${detail}` : ''}`, + ); + if (!passed) failures++; +} +fs.mkdirSync(SHOT_DIR, { recursive: true }); +const shot = (name) => + page.screenshot({ + path: path.join(SHOT_DIR, `${name}.jpg`), + type: 'jpeg', + quality: 82, + }); +const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const CLIP = { x: 380, y: 240, width: 480, height: 320 }; + +/** Pixels in the clip close to the first import color (#39d0ff). */ +async function cyanPixels() { + await page.evaluate(() => window.__godsEyeView.viewer.scene.requestRender()); + await wait(600); + const base64 = await page.screenshot({ + clip: CLIP, + type: 'png', + encoding: 'base64', + }); + return page.evaluate(async (b64) => { + const bitmap = await createImageBitmap( + await (await fetch(`data:image/png;base64,${b64}`)).blob(), + ); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext('2d'); + context.drawImage(bitmap, 0, 0); + const { data } = context.getImageData(0, 0, bitmap.width, bitmap.height); + let n = 0; + for (let i = 0; i < data.length; i += 4) { + const [r, g, b] = [data[i], data[i + 1], data[i + 2]]; + if (b > 45 && g > 30 && r < 100 && b - r > 30) n += 1; + } + return n; + }, base64); +} +const state = () => + page.evaluate(() => ({ + files: window.__gevGeoImport.list(), + dataSources: window.__godsEyeView.viewer.dataSources.length, + rows: document.querySelectorAll('#geo-import-list .geo-import-row').length, + hint: document.getElementById('geo-import-hint')?.textContent || '', + })); + +try { + await page.setViewport({ width: 1280, height: 860 }); + await page.goto(`${BASE_URL}/?welcome=0`, { waitUntil: 'domcontentloaded' }); + await page.waitForFunction( + () => + window.__gevGeoImport && + window.__godsEyeView?.viewer && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 90_000 }, + ); + await page.evaluate(async (view) => { + const viewer = window.__godsEyeView.viewer; + for (const id of window.__godsEyeView.dataManager.layers.keys()) + await window.__godsEyeView.dataManager.setEnabled(id, false); + viewer.imageryLayers.removeAll(); + viewer.terrainProvider = new window.__CESIUM__.EllipsoidTerrainProvider(); + viewer.scene.globe.baseColor = window.__CESIUM__.Color.BLACK; + viewer.scene.globe.enableLighting = false; + viewer.camera.cancelFlight?.(); + viewer.camera.setView({ + destination: window.__CESIUM__.Cartesian3.fromDegrees( + view.lon, + view.lat, + view.height, + ), + orientation: { heading: 0, pitch: (view.pitch * Math.PI) / 180, roll: 0 }, + }); + viewer.scene.requestRender(); + }, VIEW); + await wait(4000); + await page.evaluate(() => { + const rail = document.getElementById('pp-toggles'); + if (rail?.classList.contains('collapsed')) + document + .querySelector('.panel-collapse-btn[data-collapse-target="pp-toggles"]') + ?.click(); + }); + await page.waitForFunction( + () => + document.getElementById('geo-import-button')?.getBoundingClientRect() + .width > 0, + { timeout: 10_000 }, + ); + const before = await state(); + check( + 'Import is on the DISPLAY rail with an empty, hidden list', + before.rows === 0 && + !(await page.evaluate(() => + document + .getElementById('geo-import-list') + .classList.contains('visible'), + )), + ); + const cyanBefore = await cyanPixels(); + await page.evaluate(() => { + window.__godsEyeView.viewer.clock.shouldAnimate = false; + }); + const clock = () => + page.evaluate(() => { + const c = window.__godsEyeView.viewer.clock; + const iso = (t) => window.__CESIUM__.JulianDate.toIso8601(t, 0); + return { + start: iso(c.startTime), + stop: iso(c.stopTime), + range: c.clockRange, + step: c.clockStep, + multiplier: c.multiplier, + animate: c.shouldAnimate, + // Wall-clock drift aside, the current time must not jump to 2018/2019. + currentTime: window.__CESIUM__.JulianDate.toIso8601(c.currentTime), + }; + }); + const clockBefore = await clock(); + const sourcesBefore = before.dataSources; + + // Picked through the real . + const input = await page.$('#geo-import-input'); + await input.uploadFile(...FILES); + const imported = await page + .waitForFunction(() => window.__gevGeoImport.list().length === 4, { + timeout: 60_000, + }) + .then( + () => true, + () => false, + ); + const after = await state(); + check( + 'four picked files become four rows and four data sources', + imported && + after.rows === 4 && + after.dataSources === before.dataSources + 4, + JSON.stringify( + after.files.map((f) => `${f.name}:${f.format}:${f.entityCount}`), + ), + ); + check( + 'formats are recognized', + after.files.map((f) => f.format).join() === 'geojson,kml,kmz,gpx', + ); + check( + 'the KML and KMZ had their outside references removed', + after.files[1].removed > 0 && after.files[2].removed > 0, + `removed ${after.files[1].removed}/${after.files[2].removed}`, + ); + check( + 'the hint says so', + /Added 4 files\..*left out/.test(after.hint), + after.hint, + ); + const clockAfter = await clock(); + check( + "dated KML and GPX leave the app's clock alone", + JSON.stringify(clockAfter) === JSON.stringify(clockBefore) && + (await page.evaluate( + () => + window.__godsEyeView.viewer.clockTrackedDataSource?.clock === + undefined, + )), + `${JSON.stringify(clockBefore)} → ${JSON.stringify(clockAfter)}`, + ); + const inside = await page.evaluate((from) => { + const viewer = window.__godsEyeView.viewer; + const now = viewer.clock.currentTime; + const sources = []; + for (let i = from; i < viewer.dataSources.length; i++) + sources.push(viewer.dataSources.get(i)); + const entities = sources.flatMap( + (ds) => ds.importedEntities || ds.entities.values, + ); + const named = (name) => entities.find((e) => e.name === name); + const gpx = sources[3].importedEntities || sources[3].entities.values; + const kmz = sources[2].importedEntities || sources[2].entities.values; + return { + sources: sources.length, + clocks: sources.filter((ds) => ds.clock !== undefined).length, + described: entities.filter((e) => e.description !== undefined).length, + labelled: entities.filter((e) => e.label !== undefined).length, + withProperties: entities.filter((e) => e.properties !== undefined).length, + withKml: entities.filter((e) => e.kml !== undefined).length, + hidden: entities.filter((e) => !e.isAvailable(now)).length, + datedPinNow: Boolean(named('QA KML pin')?.isAvailable(now)), + fallbackPins: ['QA KML pin', 'QA palette pin'].filter((name) => + named(name)?.billboard?.image?.getValue(now), + ).length, + movingIsLine: Boolean( + named('QA moving')?.polyline && !named('QA moving')?.position, + ), + movingVertices: + named('QA moving')?.polyline?.positions?.getValue(now)?.length, + splitTracks: (sources[1].importedEntities || sources[1].entities.values) + .filter((e) => e.parent?.name === 'QA split tracks' && e.polyline) + .map((e) => e.polyline.positions.getValue(now).length), + gpxLines: gpx.filter((e) => e.polyline).length, + gpxPoints: gpx.filter((e) => e.billboard || e.point).length, + kmzLocalImage: kmz.some((e) => + String(e.billboard?.image?.getValue(now) || '').startsWith('blob:'), + ), + kmzOverlay: kmz.some((e) => e.rectangle), + groundedLines: sources + .slice(1, 3) + .every((ds) => + (ds.importedEntities || ds.entities.values) + .filter((e) => e.polyline) + .every((e) => e.polyline.clampToGround?.getValue(now)), + ), + xyzPolygon: + (sources[0].importedEntities || sources[0].entities.values) + .find((e) => e.polygon) + ?.polygon.perPositionHeight?.getValue(now) !== true, + }; + }, sourcesBefore); + check( + 'no imported entity keeps a description, label, properties or KML data', + inside.sources === 4 && + inside.described + + inside.labelled + + inside.withProperties + + inside.withKml === + 0, + JSON.stringify(inside), + ); + check( + 'no imported source has a clock, and dated features draw now', + inside.clocks === 0 && + inside.hidden === 0 && + inside.datedPinNow && + inside.movingIsLine && + inside.movingVertices === 2, + JSON.stringify(inside), + ); + check( + 'blocked KML icon sources use drawable local fallback pins', + inside.fallbackPins === 2, + JSON.stringify({ fallbackPins: inside.fallbackPins }), + ); + check( + 'KML MultiTrack keeps its segments and original vertex count', + inside.splitTracks.join() === '2,2', + JSON.stringify(inside.splitTracks), + ); + check( + 'the GPX track keeps its two segments apart', + inside.gpxLines === 2 && inside.gpxPoints === 1, + JSON.stringify(inside), + ); + check( + 'KMZ retains its own packed image and overlay', + inside.kmzLocalImage && inside.kmzOverlay, + ); + check( + 'XYZ polygons and KML/KMZ lines use ground geometry', + inside.xyzPolygon && inside.groundedLines, + ); + await page.evaluate((from) => { + for (const entity of window.__godsEyeView.viewer.dataSources.get(from) + .entities.values) + if (entity.billboard) entity.show = false; + }, sourcesBefore); + await page.evaluate((view) => { + const viewer = window.__godsEyeView.viewer; + viewer.camera.cancelFlight?.(); + viewer.camera.setView({ + destination: window.__CESIUM__.Cartesian3.fromDegrees( + view.lon, + view.lat, + view.height, + ), + orientation: { heading: 0, pitch: (view.pitch * Math.PI) / 180, roll: 0 }, + }); + }, VIEW); + await wait(2500); + const cyanShown = await cyanPixels(); + await shot('imported'); + check( + 'the imported polygon paints on the globe', + cyanShown > cyanBefore * 4 && cyanShown > cyanBefore + 5000, + `${cyanBefore} → ${cyanShown}`, + ); + + const parkId = after.files[0].id; + await page.click( + `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`, + ); + const cyanHidden = await cyanPixels(); + check( + 'hiding it stops it painting', + cyanHidden < cyanShown / 3, + `${cyanShown} → ${cyanHidden}`, + ); + await page.click( + `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`, + ); + check( + 'showing it again paints it again', + (await cyanPixels()) > cyanHidden * 4, + ); + + // Keyboard: focus stays on the same control when the list is rebuilt. + const toggleSelector = `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`; + await page.focus(toggleSelector); + await page.keyboard.press('Enter'); + const focusAfterHide = await page.evaluate( + (sel) => document.activeElement === document.querySelector(sel), + toggleSelector, + ); + await page.keyboard.press('Enter'); + const focusAfterShow = await page.evaluate( + (sel) => + document.activeElement === document.querySelector(sel) && + document.activeElement.getAttribute('aria-pressed') === 'true', + toggleSelector, + ); + check( + 'keyboard focus stays on the toggle through hide and show', + focusAfterHide && focusAfterShow, + ); + + // Dropped onto the globe, as a person drags from their file manager. + await page.evaluate((text) => { + const container = window.__godsEyeView.viewer.container; + const transfer = new DataTransfer(); + transfer.items.add( + new File([text], 'dropped.geojson', { type: 'application/geo+json' }), + ); + for (const type of ['dragenter', 'dragover', 'drop']) + container.dispatchEvent( + new DragEvent(type, { + dataTransfer: transfer, + bubbles: true, + cancelable: true, + }), + ); + }, geojson); + const dropped = await page + .waitForFunction(() => window.__gevGeoImport.list().length === 5, { + timeout: 20_000, + }) + .then( + () => true, + () => false, + ); + check('a file dropped on the globe imports', dropped); + + // Not geodata: refused with a reason, nothing added. + await page.evaluate(() => { + const transfer = new DataTransfer(); + transfer.items.add( + new File(['just some notes'], 'notes.txt', { type: 'text/plain' }), + ); + window.__godsEyeView.viewer.container.dispatchEvent( + new DragEvent('drop', { + dataTransfer: transfer, + bubbles: true, + cancelable: true, + }), + ); + }); + await wait(800); + const refused = await state(); + check( + 'a file that is not geodata is refused with a reason', + refused.files.length === 5 && + /notes\.txt: Only GeoJSON, KML, KMZ and GPX/.test(refused.hint), + refused.hint, + ); + + // Remove every file through its row button. + for (const { id } of refused.files) + await page.click( + `#geo-import-list [data-import-id="${id}"] button[data-action="remove"]`, + ); + const cleared = await state(); + check( + 'Remove gives every data source back', + cleared.rows === 0 && cleared.dataSources === before.dataSources, + `${cleared.dataSources} vs ${before.dataSources}`, + ); + await wait(1500); + check( + 'nothing named inside a file, and no icon asset for one, was ever requested', + trapped.length === 0, + trapped.join(' '), + ); + errors.push(...(await readRenderErrors(page))); + check('no page or rendering errors', errors.length === 0, errors.join(' | ')); +} catch (error) { + check('harness ran to completion', false, error?.stack || String(error)); +} finally { + await browser.close(); + fs.rmSync(dir, { recursive: true, force: true }); +} +console.log(failures ? `\n${failures} check(s) failed` : '\nall checks passed'); +process.exit(failures ? 1 : 0); diff --git a/scripts/qa-gnss.mjs b/scripts/qa-gnss.mjs new file mode 100644 index 000000000..41933141e --- /dev/null +++ b/scripts/qa-gnss.mjs @@ -0,0 +1,988 @@ +#!/usr/bin/env node +/** + * qa-gnss.mjs — headless proof for the GNSS Integrity layer + * (id `gnss-integrity`): low navigation accuracy reported by ADS-B. + * + * Drives the REAL app in headless Chromium against a dev server (keyless). + * The live section reads adsb.lol through `/api/gnss-integrity` (60 s server + * cache, one upstream read per whole-degree anchor); the other sections + * stub that route in the page so they are deterministic. + * + * (i) LIVE — over Poland and the southern Baltic the layer settles with no error, + * one geometry per cell and legend band counts that sum to the cell + * count; cells are required only when the returned rows, binned + * with the layer's own rules, reach the per-cell minimum (live + * traffic varies by hour). Requires a non-stale response no older than 60 s. + * (ii) BANDS (synthetic) — a 10-aircraft cell with 5 degraded is high, + * a 30-aircraft cell with 2 degraded is medium, a 3-aircraft cell + * with 1 degraded is low (first bad discounted), a clean cell at sea + * is low, a 2-aircraft cell is withheld; `stale: true` reaches + * getStats. Every band is drawn and low is shown over land and sea. + * The row reads as low navigation accuracy, and stats carry the + * machine-readable provenance (GEV classifier, gpsjam bands, window). + * PICK verifies actual perimeter and empty-ground clicks under GNSS. + * CAMERA/GLOBE verify automatic sampling and the screen-centre anchor. + * (iii) FAILURE — the route stubbed to 502: stats.error is set and the + * cells already on the globe are kept. + * (iv) DISABLE — owned ground primitives are hidden. + * (v) KEYBOARD — the row's toggle button, focused, turns the layer on + * with Enter and off with Space, keeps focus and names its state. + * (vi) VIEWPORT — at a 390x844 phone viewport the open row and its legend + * fit the panel width with no horizontal overflow. + * (vii) AGE (synthetic) — a fresh page whose stubbed proxy replays the same + * stale snapshot observed 31 minutes ago draws nothing and reports no + * error; observed 29 minutes ago it draws its cells. The replay keeps + * its observation time instead of restarting the 30-minute window. + * + * Screenshots wait until every cell's ground geometry is built and show the + * open Data Layers row with its legend. Visual proof saved to + * qa-shots/gnss-*.png (gitignored). + * + * Run: node scripts/qa-gnss.mjs --url http://localhost:4173 + * (also against a built preview: npm run build && npm run preview, + * then pass the preview URL) + * Optional: --timeout-ms 180000 for a slow boot. + * Exits non-zero on any FAIL. Does not commit anything. + */ + +import puppeteer from 'puppeteer'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + accumulateGnssObservations, + binGnssCells, +} from '../src/layers/gnss/records.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, '..'); +const SHOTS_DIR = path.join(REPO_ROOT, 'qa-shots'); + +const argv = process.argv.slice(2); +const getOpt = (name, dflt) => { + const i = argv.indexOf(name); + return i >= 0 && argv[i + 1] ? argv[i + 1] : dflt; +}; +const APP_URL = getOpt('--url', 'http://localhost:4173'); +const HEADFUL = argv.includes('--headful'); +const TIMEOUT_MS = Number(getOpt('--timeout-ms', '60000')); +const LAYER_ID = 'gnss-integrity'; + +const results = []; +function record(name, ok, detail) { + results.push({ name, ok, detail }); + const tag = ok ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m'; + console.log(` [${tag}] ${name}${detail ? ` — ${detail}` : ''}`); +} +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +/** Boot the app, dismiss the intro and place the camera straight down. */ +async function boot(page, { lon, lat, height }) { + await page.goto(`${APP_URL}/?welcome=0`, { + waitUntil: 'domcontentloaded', + timeout: TIMEOUT_MS, + }); + await page.waitForFunction( + () => window.__godsEyeView?.viewer && window.__godsEyeView?.dataManager, + { timeout: TIMEOUT_MS }, + ); + await sleep(5000); + await page.keyboard.press('Escape'); + await page.evaluate( + (lo, la, h) => { + const gev = window.__godsEyeView; + const d2r = Math.PI / 180; + // The app's intro flyTo animation clobbers a setView issued mid-flight. + try { + gev.viewer.camera.cancelFlight(); + } catch { + /* no flight active */ + } + gev.viewer.camera.setView({ + destination: gev.viewer.scene.globe.ellipsoid.cartographicToCartesian({ + longitude: lo * d2r, + latitude: la * d2r, + height: h, + }), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + gev.viewer.scene.requestRender?.(); + }, + lon, + lat, + height, + ); +} + +/** Enable (or refresh) the layer and poll until it settles. */ +async function enableAndSettle(page, { refresh = false } = {}) { + return page.evaluate( + async (id, again) => { + const dm = window.__godsEyeView.dataManager; + const layer = dm.layers.get(id).module; + if (again) await dm.refreshLayer(id); + else await dm.setEnabled(id, true); + const stats = layer.getStats(); + const rowControls = layer.getRowControls().legend; + const legend = rowControls.map(({ label, count }) => ({ label, count })); + const blurb = rowControls[0]?.blurb ?? ''; + const rowName = + document.querySelector(`[data-layer-id="${id}"] .data-name`) + ?.textContent ?? ''; + const all = window.__godsEyeView.viewer.scene.groundPrimitives; + const ds = Array.from({ length: all.length }, (_, i) => all.get(i)).find( + (item) => item.id === id, + ); + return { + stats, + legend, + blurb, + rowName, + name: layer.name, + entities: ds?.get(0)?.geometryInstances.length ?? 0, + shown: ds?.show ?? null, + }; + }, + LAYER_ID, + refresh, + ); +} + +/** Open the Data Layers panel and bring the GNSS row into view. */ +async function showLayerRow(page) { + await page.evaluate((id) => { + const panel = document.getElementById('data-panel'); + if (panel.classList.contains('collapsed')) + panel.querySelector('[data-collapse-target="data-panel"]').click(); + document + .querySelector(`[data-layer-id="${id}"]`) + ?.scrollIntoView({ block: 'center' }); + }, LAYER_ID); + await sleep(800); +} + +/** Actual card pixels on the shared canvas, available in dev and production. */ +function paintedOverlayPixels() { + const canvas = document.getElementById('world-overlay-canvas'); + if (!canvas) return -1; + const pixels = canvas + .getContext('2d') + .getImageData(0, 0, canvas.width, canvas.height).data; + let count = 0; + for (let i = 3; i < pixels.length; i += 4) if (pixels[i] > 0) count++; + return count; +} + +async function shoot(page, name) { + await page + .waitForFunction( + () => window.__godsEyeView.viewer.scene.globe.tilesLoaded, + { timeout: TIMEOUT_MS }, + ) + .catch(() => {}); + await showLayerRow(page); + await page.screenshot({ path: path.join(SHOTS_DIR, `gnss-${name}.png`) }); +} + +/** Bounding-sphere state of every GNSS cell: DONE = 0, PENDING = 1, FAILED = 2. */ +function cellStates(id) { + const { viewer } = window.__godsEyeView; + const all = viewer.scene.groundPrimitives; + const ds = Array.from({ length: all.length }, (_, i) => all.get(i)).find( + (item) => item.id === id, + ); + if (!ds?.length) return []; + return [ds.get(0).ready ? 0 : 1]; +} + +/** Wait until every GNSS cell's ground rectangle has finished building. */ +async function waitForBuiltCells(page, timeoutMs = 60000) { + const built = await page + .waitForFunction( + `(${cellStates})(${JSON.stringify(LAYER_ID)}).every((state) => state === 0)`, + { timeout: timeoutMs, polling: 250 }, + ) + .then( + () => true, + () => false, + ); + const states = await page.evaluate( + `(${cellStates})(${JSON.stringify(LAYER_ID)})`, + ); + return { built, detail: `states=${JSON.stringify(states)}` }; +} + +const fixtureRow = (hex, lat, lon, degraded) => ({ + hex, + lat, + lon, + nic: degraded ? 0 : 8, + nacp: degraded ? 0 : 10, + gpsLost: false, + degraded, +}); + +/** + * Synthetic 0.5° cells: high at sea (5/10 bad), medium on land (2/30 bad), + * low on land (1/3 bad), low at sea (0/4 bad), withheld (2 aircraft). + */ +function fixturePayload() { + const rows = []; + const hex = (prefix, i) => `${prefix}${i.toString(16).padStart(2, '0')}`; + for (let i = 0; i < 10; i++) + rows.push(fixtureRow(hex('a000', i), 55.1 + i * 0.01, 19.1, i < 5)); + for (let i = 0; i < 30; i++) + rows.push(fixtureRow(hex('d000', i), 54.1 + i * 0.01, 19.6, i < 2)); + for (let i = 0; i < 3; i++) + rows.push(fixtureRow(hex('b000', i), 54.1 + i * 0.01, 18.1, i === 0)); + for (let i = 0; i < 4; i++) + rows.push(fixtureRow(hex('e000', i), 55.1 + i * 0.01, 18.1, false)); + for (let i = 0; i < 2; i++) + rows.push(fixtureRow(hex('c000', i), 53.1 + i * 0.01, 17.1, true)); + return { + fetchedAt: Date.now(), + anchor: { lat: 55, lon: 19 }, + radiusNm: 250, + rows, + stale: true, + }; +} + +/** Answer /api/gnss-integrity in-page from `fixturePayload()`. */ +async function stubRoute(page, { ageMs = 0, initialFailures = 0, stale } = {}) { + await page.evaluateOnNewDocument( + (payload, age, failures, staleOverride) => { + window.__gnssQaAgeMs = age; + window.__gnssQaRequests = []; + window.__gnssQaFailures = failures; + const realFetch = window.fetch.bind(window); + window.__gnssQaMode = 'fixture'; + window.fetch = (input, init) => { + const url = typeof input === 'string' ? input : input?.url || ''; + if (!url.includes('/api/gnss-integrity')) return realFetch(input, init); + window.__gnssQaRequests.push(url); + if (window.__gnssQaFailures > 0) { + window.__gnssQaFailures--; + return Promise.resolve( + Response.json( + { error: 'rate_limited' }, + { + status: 429, + headers: { 'Retry-After': '5' }, + }, + ), + ); + } + const ok = window.__gnssQaMode === 'fixture'; + const body = ok + ? { + ...payload, + fetchedAt: Date.now() - (window.__gnssQaAgeMs || 0), + ageMs: window.__gnssQaAgeMs || 0, + stale: staleOverride ?? payload.stale, + } + : { error: 'gnss_integrity_unavailable' }; + return Promise.resolve( + new Response(JSON.stringify(body), { + status: ok ? 200 : 502, + headers: { 'Content-Type': 'application/json' }, + }), + ); + }; + }, + fixturePayload(), + ageMs, + initialFailures, + stale, + ); +} + +/** Toggle button of the GNSS row, as the panel draws it. */ +function toggleState(id) { + const button = document.querySelector( + `[data-layer-id="${id}"] .data-toggle-btn`, + ); + const layer = window.__godsEyeView.dataManager.layers.get(id); + return { + focused: document.activeElement === button, + label: button?.getAttribute('aria-label') ?? '', + enabled: Boolean(layer?.enabled), + count: layer?.module.getStats().count ?? -1, + }; +} + +/** Press a key on the focused GNSS toggle and wait for the lifecycle to settle. */ +async function pressToggle(page, key, wantEnabled) { + await page.evaluate((id) => { + document.querySelector(`[data-layer-id="${id}"] .data-toggle-btn`)?.focus(); + }, LAYER_ID); + await page.keyboard.press(key); + await page + .waitForFunction( + (id, want) => { + const layer = window.__godsEyeView.dataManager.layers.get(id); + return ( + Boolean(layer?.enabled) === want && + (!want || layer.module.getStats().count > 0) + ); + }, + { timeout: 30000, polling: 250 }, + LAYER_ID, + wantEnabled, + ) + .catch(() => {}); + return page.evaluate(`(${toggleState})(${JSON.stringify(LAYER_ID)})`); +} + +async function main() { + console.log('\nGNSS Integrity proof (qa-gnss)'); + console.log(` App URL : ${APP_URL}\n`); + try { + const res = await fetch(APP_URL); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + } catch (e) { + console.error( + `\x1b[31mDev server not reachable at ${APP_URL} (${e.message}).\x1b[0m`, + ); + process.exit(2); + } + fs.mkdirSync(SHOTS_DIR, { recursive: true }); + const executablePath = + process.env.PUPPETEER_EXECUTABLE_PATH || + (await puppeteer.executablePath().catch(() => null)) || + undefined; + const browser = await puppeteer.launch({ + headless: HEADFUL ? false : 'new', + ...(executablePath && fs.existsSync(executablePath) + ? { executablePath } + : {}), + args: [ + '--use-gl=angle', + '--use-angle=swiftshader', + '--disable-dev-shm-usage', + '--disable-background-timer-throttling', + '--disable-renderer-backgrounding', + '--window-size=1440,900', + ], + }); + + let exitCode = 0; + const errors = []; + try { + // ── (i) LIVE ────────────────────────────────────────────────────────── + console.log( + '(i) LIVE — adsb.lol integrity snapshot over Poland and the southern Baltic...', + ); + const live = await browser.newPage(); + await live.setViewport({ width: 1440, height: 900 }); + live.on('pageerror', (e) => errors.push(e.message)); + const liveBodies = []; + live.on('response', (res) => { + if (res.url().includes('/api/gnss-integrity') && res.ok()) + liveBodies.push(res.json()); + }); + await boot(live, { lon: 19.6, lat: 52.6, height: 1_100_000 }); + const l = await enableAndSettle(live); + const legendTotal = l.legend.reduce((sum, { count }) => sum + count, 0); + // Bin what adsb.lol actually returned with the layer's own rules: live + // traffic varies by hour, so cells are required only when some cell + // really reached the minimum aircraft count. + const liveSnapshots = (await Promise.allSettled(liveBodies)) + .filter((r) => r.status === 'fulfilled') + .map((r) => r.value); + record( + 'LIVE: a successful, dated integrity response was received', + liveSnapshots.length > 0 && + liveSnapshots.every( + (body) => + Array.isArray(body.rows) && + Number.isFinite(body.ageMs) && + body.ageMs >= 0 && + body.ageMs <= 60000 && + body.stale !== true && + body.classifier === 'gev-nic-nacp-v2', + ), + `successful responses=${liveSnapshots.length}`, + ); + const liveRows = liveSnapshots.reduce( + (sum, body) => sum + (body.rows?.length || 0), + 0, + ); + const expectedStore = new Map(); + for (const body of liveSnapshots) { + const at = Date.now(); + accumulateGnssObservations(expectedStore, body.rows, at, { + observedAt: at - body.ageMs, + }); + } + const expectedCells = binGnssCells(expectedStore).length; + record( + 'LIVE: no error, one geometry per cell', + !l.stats.error && l.entities === l.stats.count, + `cells=${l.stats.count} entities=${l.entities} error=${JSON.stringify(l.stats.error)}`, + ); + if (expectedCells > 0) { + record( + 'LIVE: a snapshot with a qualifying cell yields cells', + l.stats.count > 0, + `rows=${liveRows} expected=${expectedCells} cells=${l.stats.count}`, + ); + } else { + console.log( + ` [info] ${liveRows} live rows, no cell reaches the minimum — cell count not asserted`, + ); + } + record( + 'LIVE: legend band counts sum to the cell count', + legendTotal === l.stats.count, + JSON.stringify(l.legend), + ); + const liveBuilt = await waitForBuiltCells(live); + record( + 'LIVE: every cell rectangle is built on a parked camera', + liveBuilt.built, + liveBuilt.detail, + ); + await shoot(live, 'live-poland-baltic'); + await live.close(); + + // ── (ii) BANDS ──────────────────────────────────────────────────────── + console.log( + '\n(ii) BANDS — synthetic integrity rows through the real layer...', + ); + const page = await browser.newPage(); + await page.setViewport({ width: 1440, height: 900 }); + page.on('pageerror', (e) => errors.push(e.message)); + // Stub the route in-page rather than with request interception: an + // intercepted page stalls Cesium's geometry workers, so ground rectangles + // never finish building and the screenshot would not show real behaviour. + await stubRoute(page); + await boot(page, { lon: 18.6, lat: 54.6, height: 450_000 }); + const f = await enableAndSettle(page); + const band = (prefix) => + f.legend.find(({ label }) => label.startsWith(prefix))?.count; + record( + 'BANDS: 4 cells shown, the 2-aircraft cell withheld', + f.stats.count === 4 && f.entities === 4, + `cells=${f.stats.count} entities=${f.entities}`, + ); + record( + 'BANDS: 5/10 is high, 2/30 is medium, 1/3 and 0/4 are low', + band('Low-accuracy score >') === 1 && + band('Low-accuracy score 2') === 1 && + band('Low-accuracy score <') === 2, + JSON.stringify(f.legend), + ); + const prov = f.stats.provenance; + record( + 'PROVENANCE: GEV classifier, gpsjam bands and visited-view window are machine-readable', + prov?.classifier?.id === 'gev-nic-nacp-v2' && + prov.classifier.definedBy === 'GEV' && + prov.classifier.validated === false && + prov.aggregation?.source === 'https://gpsjam.org/faq' && + JSON.stringify(prov.aggregation.bands) === '[2,10]' && + prov.window?.minutes === 30 && + prov.window.scope === 'visited-view' && + prov.interpretation?.validated === false, + JSON.stringify(prov?.classifier), + ); + record( + 'WORDING: the row leads with low navigation accuracy, interference only as the hedged reading', + f.name === 'GNSS Integrity' && + f.rowName.includes('GNSS Integrity') && + !/interference/i.test(f.rowName) && + f.legend.every(({ label }) => label.startsWith('Low-accuracy score')) && + /one aircraft is discounted/.test(f.blurb) && + /not a detection/.test(f.blurb), + `name=${JSON.stringify(f.name)}`, + ); + record( + 'BANDS: stale snapshot is reported', + f.stats.stale === true, + `stale=${f.stats.stale}`, + ); + const bandsBuilt = await waitForBuiltCells(page); + record( + 'BANDS: every cell rectangle is built on a parked camera', + bandsBuilt.built, + bandsBuilt.detail, + ); + await shoot(page, 'synthetic-bands'); + + // GNSS fills must not own map input, including over a real perimeter. + await page.evaluate(async () => { + const original = window.fetch.bind(window); + window.fetch = (input, init) => { + const url = typeof input === 'string' ? input : input.url; + if (url.includes('/api/fire-perimeters/inciweb')) + return Promise.resolve(Response.json({ rows: [] })); + if (url.includes('/api/fire-perimeters')) + return Promise.resolve( + Response.json({ + rows: [ + { + stableId: 'gnss-pick-qa', + name: 'GNSS PICK QA FIRE', + containedPct: 0, + acres: 100, + polygons: [ + [ + [ + [18.247, 54.247], + [18.253, 54.247], + [18.253, 54.253], + [18.247, 54.253], + [18.247, 54.247], + ], + ], + ], + }, + ], + }), + ); + return original(input, init); + }; + const { viewer, dataManager } = window.__godsEyeView; + viewer.camera.cancelFlight(); + viewer.camera.setView({ + destination: viewer.scene.globe.ellipsoid.cartographicToCartesian({ + longitude: (18.25 * Math.PI) / 180, + latitude: (54.25 * Math.PI) / 180, + height: 2500, + }), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + await dataManager.setEnabled('fire-perimeters', true); + viewer.scene.requestRender(); + }); + await page.waitForFunction( + () => { + const v = window.__godsEyeView.viewer, + p = v.scene.pick({ + x: v.scene.canvas.clientWidth / 2, + y: v.scene.canvas.clientHeight / 2, + }); + return String(p?.id?.id ?? p?.id ?? '').startsWith( + 'fire-perimeter:gnss-pick-qa:', + ); + }, + { timeout: TIMEOUT_MS, polling: 250 }, + ); + const pickProbe = await page.evaluate(() => { + const v = window.__godsEyeView.viewer, + c = v.scene.canvas; + const centre = { x: c.clientWidth / 2, y: c.clientHeight / 2 }; + const empty = { x: centre.x + 300, y: centre.y }; + const picked = v.scene.pick(empty), + box = c.getBoundingClientRect(); + return { + centre: { x: centre.x + box.left, y: centre.y + box.top }, + empty: { x: empty.x + box.left, y: empty.y + box.top }, + surface: !picked || picked.id === undefined, + height: v.camera.positionCartographic.height, + }; + }); + record('PICK: a perimeter remains selectable under GNSS', true); + record( + 'PICK: street ground remains a surface pick under GNSS', + pickProbe.surface, + ); + const beforeCardPixels = await page.evaluate(paintedOverlayPixels); + await page.mouse.click(pickProbe.centre.x, pickProbe.centre.y); + await page.waitForFunction( + `(${paintedOverlayPixels})() > ${beforeCardPixels + 100}`, + { timeout: 10000, polling: 250 }, + ); + const selectedCardPixels = await page.evaluate(paintedOverlayPixels); + record( + 'PICK: actual perimeter click paints its incident card', + selectedCardPixels > beforeCardPixels + 100, + `${beforeCardPixels} → ${selectedCardPixels} pixels`, + ); + await page.screenshot({ + path: path.join(SHOTS_DIR, 'gnss-perimeter-selected.png'), + }); + await page.mouse.click(pickProbe.empty.x, pickProbe.empty.y); + await page.waitForFunction( + `(${paintedOverlayPixels})() <= ${beforeCardPixels}`, + { timeout: 10000, polling: 250 }, + ); + const clearedCardPixels = await page.evaluate(paintedOverlayPixels); + record( + 'PICK: empty ground clears the perimeter card', + clearedCardPixels <= beforeCardPixels, + `${selectedCardPixels} → ${clearedCardPixels} pixels`, + ); + await page.mouse.click(pickProbe.empty.x, pickProbe.empty.y, { + clickCount: 2, + }); + await sleep(1000); + record( + 'PICK: empty double-click does not zoom to a GNSS cell', + await page.evaluate( + (height) => + Math.abs( + window.__godsEyeView.viewer.camera.positionCartographic.height - + height, + ) < 200, + pickProbe.height, + ), + ); + await page.evaluate(async () => { + await window.__godsEyeView.dataManager.setEnabled( + 'fire-perimeters', + false, + ); + }); + console.log('\nCAMERA — automatic regional refresh without toggling...'); + const beforeMove = await page.evaluate( + () => window.__gnssQaRequests.length, + ); + await page.evaluate(() => { + const { viewer } = window.__godsEyeView; + viewer.camera.setView({ + destination: viewer.scene.globe.ellipsoid.cartographicToCartesian({ + longitude: (139.7 * Math.PI) / 180, + latitude: (35.7 * Math.PI) / 180, + height: 1_100_000, + }), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + viewer.scene.requestRender(); + }); + await page.waitForFunction( + () => { + const url = window.__gnssQaRequests.at(-1); + return ( + url && + Math.abs( + Number(new URL(url, location.origin).searchParams.get('lon')) - + 139.7, + ) < 1 + ); + }, + { timeout: 15000 }, + ); + await sleep(3500); + const moved = await page.evaluate( + (id) => ({ + requests: window.__gnssQaRequests.length, + enabled: window.__godsEyeView.dataManager.isEnabled(id), + coverage: window.__godsEyeView.dataManager.layers + .get(id) + .module.getStats().coverage, + info: document.querySelector( + `[data-layer-id="${id}"] .data-toggle-controls-info`, + )?.textContent, + }), + LAYER_ID, + ); + record( + 'CAMERA: moving to Tokyo automatically refreshes the new region and updates the row', + moved.enabled && + moved.requests > beforeMove && + moved.requests <= beforeMove + 2 && + moved.coverage?.longitude === 140 && + /Regional sample.*250 nm radius/.test(moved.info), + JSON.stringify(moved), + ); + await shoot(page, 'camera-refresh'); + await page.evaluate(() => { + const { viewer } = window.__godsEyeView; + viewer.camera.setView({ + destination: viewer.scene.globe.ellipsoid.cartographicToCartesian({ + longitude: (12.4 * Math.PI) / 180, + latitude: (41.9 * Math.PI) / 180, + height: 16_000_000, + }), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + viewer.scene.requestRender(); + }); + await page.waitForFunction( + () => { + const url = window.__gnssQaRequests.at(-1); + const params = url && new URL(url, location.origin).searchParams; + return ( + params && + Math.abs(Number(params.get('lon')) - 12.4) < 1 && + Math.abs(Number(params.get('lat')) - 41.9) < 1 + ); + }, + { timeout: 15000 }, + ); + const world = await page.evaluate((id) => { + const { viewer, dataManager } = window.__godsEyeView; + const bounds = viewer.camera.computeViewRectangle( + viewer.scene.globe.ellipsoid, + ); + return { + enabled: dataManager.isEnabled(id), + height: viewer.camera.positionCartographic.height, + boundsMidLongitude: + bounds && ((bounds.west + bounds.east) * 90) / Math.PI, + coverage: dataManager.layers.get(id).module.getStats().coverage, + }; + }, LAYER_ID); + record( + 'GLOBE: zoomed-out sampling stays centred on Rome and exposes its regional radius', + world.enabled && + world.height > 15_000_000 && + world.coverage?.longitude === 12 && + world.coverage?.latitude === 42 && + world.coverage?.radiusNm === 250, + JSON.stringify(world), + ); + await shoot(page, 'globe-regional'); + await page.evaluate(() => { + const { viewer } = window.__godsEyeView; + viewer.camera.setView({ + destination: viewer.scene.globe.ellipsoid.cartographicToCartesian({ + longitude: (18.6 * Math.PI) / 180, + latitude: (54.6 * Math.PI) / 180, + height: 450_000, + }), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + viewer.scene.requestRender(); + }); + await page.waitForFunction( + () => { + const url = window.__gnssQaRequests.at(-1); + return ( + url && + Math.abs( + Number(new URL(url, location.origin).searchParams.get('lon')) - + 18.6, + ) < 1 + ); + }, + { timeout: 15000 }, + ); + await sleep(3500); + + // ── (iii) FAILURE ───────────────────────────────────────────────────── + console.log('\n(iii) FAILURE — /api/gnss-integrity stubbed to 502...'); + await page.evaluate(() => { + window.__gnssQaMode = 'error'; + }); + const e = await enableAndSettle(page, { refresh: true }); + record( + 'FAILURE: error surfaced, cells kept', + Boolean(e.stats.error) && e.stats.count === 4 && e.entities === 4, + `error=${JSON.stringify(e.stats.error)} cells=${e.stats.count}`, + ); + await shoot(page, 'failure'); + + // ── (iv) DISABLE ────────────────────────────────────────────────────── + console.log('\n(iv) DISABLE...'); + const shown = await page.evaluate(async (id) => { + await window.__godsEyeView.dataManager.setEnabled(id, false); + const all = window.__godsEyeView.viewer.scene.groundPrimitives; + return Array.from({ length: all.length }, (_, i) => all.get(i)).find( + (item) => item.id === id, + )?.show; + }, LAYER_ID); + record('DISABLE: data source hidden', shown === false, `show=${shown}`); + await shoot(page, 'disabled'); + + // ── (v) KEYBOARD ────────────────────────────────────────────────────── + console.log('\n(v) KEYBOARD — the row toggle, from the keyboard...'); + await page.evaluate(() => { + window.__gnssQaMode = 'fixture'; + }); + await showLayerRow(page); + const on = await pressToggle(page, 'Enter', true); + record( + 'KEYBOARD: Enter on the focused toggle enables the layer and keeps focus', + on.enabled && + on.focused && + on.count === 4 && + /^GNSS Integrity: /.test(on.label) && + !/OFF$/.test(on.label), + JSON.stringify(on), + ); + const off = await pressToggle(page, ' ', false); + record( + 'KEYBOARD: Space on the focused toggle disables it and the label says OFF', + !off.enabled && off.focused && /^GNSS Integrity: OFF$/.test(off.label), + JSON.stringify(off), + ); + + // ── (vi) VIEWPORT ───────────────────────────────────────────────────── + console.log( + '\n(vi) VIEWPORT — the open row on a 390x844 phone viewport...', + ); + await page.close(); + // A fresh page: switching an open page to isMobile reloads it, which + // would measure a half-booted app instead of the open row. + const phone = await browser.newPage(); + await phone.setViewport({ width: 390, height: 844, isMobile: true }); + phone.on('pageerror', (e) => errors.push(e.message)); + await stubRoute(phone); + await boot(phone, { lon: 18.6, lat: 54.6, height: 450_000 }); + await showLayerRow(phone); + await pressToggle(phone, 'Enter', true); + await showLayerRow(phone); + const fit = await phone.evaluate((id) => { + const row = document.querySelector(`[data-layer-id="${id}"]`); + const legend = [...row.querySelectorAll('.data-toggle-legend-item')]; + const rect = row.getBoundingClientRect(); + return { + width: window.innerWidth, + rowLeft: Math.round(rect.left), + rowRight: Math.round(rect.right), + overflow: row.scrollWidth - row.clientWidth, + legendItems: legend.length, + coverageInfo: row.querySelector('.data-toggle-controls-info') + ?.textContent, + legendInside: legend.every((item) => { + const r = item.getBoundingClientRect(); + return ( + r.width > 0 && r.left >= rect.left - 1 && r.right <= rect.right + 1 + ); + }), + }; + }, LAYER_ID); + record( + 'VIEWPORT: row and legend fit a 390 px viewport without horizontal overflow', + fit.rowLeft >= 0 && + fit.rowRight <= fit.width && + fit.overflow <= 0 && + fit.legendItems === 3 && + /Regional sample.*250 nm radius/.test(fit.coverageInfo) && + fit.legendInside, + JSON.stringify(fit), + ); + await phone.screenshot({ path: path.join(SHOTS_DIR, 'gnss-phone.png') }); + await phone.close(); + + console.log( + '\nSTARTUP — initial rate limit stays enabled and recovers automatically...', + ); + const retry = await browser.newPage(); + await retry.setViewport({ width: 1440, height: 900 }); + retry.on('pageerror', (e) => errors.push(e.message)); + // Pages share the disposable test context's storage. Start this one cold, + // so a previous fixture cannot auto-enable GNSS before the START test. + await retry.evaluateOnNewDocument(() => localStorage.clear()); + await stubRoute(retry, { initialFailures: 1, stale: false }); + await boot(retry, { lon: 18.6, lat: 54.6, height: 450_000 }); + await showLayerRow(retry); + await retry.evaluate((id) => { + window.__gnssQaFailedStarts = 0; + window.__godsEyeView.dataManager.subscribe((change) => { + if (change.layerId === id && change.type === 'visibility-failed') + window.__gnssQaFailedStarts++; + }); + }, LAYER_ID); + const initial = await enableAndSettle(retry); + const limited = await retry.evaluate((id) => { + const dm = window.__godsEyeView.dataManager; + const row = document.querySelector(`[data-layer-id="${id}"]`); + return { + enabled: dm.isEnabled(id), + failedStarts: window.__gnssQaFailedStarts, + label: row.querySelector('.data-toggle-btn').textContent, + info: row.querySelector('.data-toggle-controls-info').textContent, + }; + }, LAYER_ID); + record( + 'STARTUP: first 429 keeps the layer enabled and shows automatic retry timing', + limited.enabled && + limited.failedStarts === 0 && + limited.label === 'UNAVAILABLE' && + initial.stats.count === 0 && + initial.stats.retryInSec > 0 && + /rate limited.*Retrying automatically at/.test(limited.info), + JSON.stringify(limited), + ); + await retry.screenshot({ + path: path.join(SHOTS_DIR, 'gnss-startup-retrying.png'), + }); + await retry.waitForFunction( + (id) => { + const layer = window.__godsEyeView.dataManager.layers.get(id); + return ( + layer.enabled && + layer.module.getStats().error === null && + layer.module.getStats().count === 4 + ); + }, + { timeout: 20000 }, + LAYER_ID, + ); + const recovered = await retry.evaluate( + (id) => ({ + requests: window.__gnssQaRequests.length, + enabled: window.__godsEyeView.dataManager.isEnabled(id), + label: document.querySelector( + `[data-layer-id="${id}"] .data-toggle-btn`, + ).textContent, + info: document.querySelector( + `[data-layer-id="${id}"] .data-toggle-controls-info`, + ).textContent, + }), + LAYER_ID, + ); + record( + 'STARTUP: one automatic retry recovers to ON without toggling and clears the warning', + recovered.enabled && + recovered.requests === 2 && + recovered.label === 'ON' && + !/Retrying/.test(recovered.info), + JSON.stringify(recovered), + ); + await shoot(retry, 'startup-recovered'); + await retry.close(); + + // ── (vii) AGE ───────────────────────────────────────────────────────── + console.log( + '\n(vii) AGE — a replayed stale snapshot keeps its observation time...', + ); + const aged = await browser.newPage(); + await aged.setViewport({ width: 1440, height: 900 }); + aged.on('pageerror', (e) => errors.push(e.message)); + await stubRoute(aged, { ageMs: 31 * 60_000 }); + await boot(aged, { lon: 18.6, lat: 54.6, height: 450_000 }); + const old = await enableAndSettle(aged, {}); + record( + 'AGE: a snapshot observed 31 min ago adds no cells and is not an error', + old.stats.count === 0 && old.entities === 0 && !old.stats.error, + `cells=${old.stats.count} entities=${old.entities} error=${JSON.stringify(old.stats.error)}`, + ); + await aged.evaluate(() => { + window.__gnssQaAgeMs = 29 * 60_000; + }); + const recent = await enableAndSettle(aged, { refresh: true }); + record( + 'AGE: the same snapshot observed 29 min ago is still inside the window', + recent.stats.count === 4 && recent.entities === 4, + `cells=${recent.stats.count} entities=${recent.entities}`, + ); + await aged.close(); + record( + 'no uncaught browser errors', + errors.length === 0, + errors.slice(0, 3).join(' | '), + ); + } catch (err) { + console.error('\x1b[31mHarness error:\x1b[0m', err); + exitCode = 3; + } finally { + await browser.close(); + } + + const pass = results.filter((r) => r.ok).length; + const fail = results.length - pass; + console.log('\n' + '─'.repeat(60)); + console.log(` RESULT: ${pass} passed, ${fail} failed`); + console.log(` Shots : ${SHOTS_DIR}/gnss-*.png`); + console.log('─'.repeat(60) + '\n'); + process.exit(exitCode || (fail > 0 ? 1 : 0)); +} + +main().catch((e) => { + console.error(e); + process.exit(3); +}); diff --git a/scripts/qa-panel-resize.mjs b/scripts/qa-panel-resize.mjs index 026b6f2fb..d4c5b9287 100644 --- a/scripts/qa-panel-resize.mjs +++ b/scripts/qa-panel-resize.mjs @@ -381,11 +381,36 @@ async function main() { ); await nextFrames(page); }; - await drag(handlePoint(await readPanel(), 'se'), 0, 200); - const chosen = await readPanel(); + // Grow it from the corner once the restored window holds still, with the + // pointer really on the handle (a press beside it selects page text). + let chosen = await readPanel(); + const tried = []; + for ( + let attempt = 0; + attempt < 3 && chosen.height <= MIN_SIZE.height + 150; + attempt++ + ) { + await settle(); + const box = await readPanel(); + const corner = handlePoint(box, 'se'); + const hit = await page.evaluate(({ x, y }) => { + const node = document.elementFromPoint(x, y); + return { + dir: node?.dataset?.dir || null, + target: node + ? `${node.tagName.toLowerCase()}${node.id ? `#${node.id}` : ''}.${[...node.classList].join('.')}` + : 'nothing', + }; + }, corner); + tried.push( + `${Math.round(corner.x)},${Math.round(corner.y)} on ${hit.target} (box ${Math.round(box.left)},${Math.round(box.top)} ${Math.round(box.width)}x${Math.round(box.height)})`, + ); + if (hit.dir === 'se') await drag(corner, 0, 200); + chosen = await readPanel(); + } assert.ok( chosen.height > MIN_SIZE.height + 150, - `the window grew to ${chosen.height}px`, + `the window grew to ${chosen.height}px; tried ${tried.join('; ')}`, ); await setCollapsed(true); const strip = await readPanel(); diff --git a/scripts/qa-recent-imagery.mjs b/scripts/qa-recent-imagery.mjs index 1460ed3b5..89d60d96f 100644 --- a/scripts/qa-recent-imagery.mjs +++ b/scripts/qa-recent-imagery.mjs @@ -12,16 +12,22 @@ * Run: start the app (`./scripts/dev-fresh.sh`) and * node scripts/qa-recent-imagery.mjs * Env: QA_BASE_URL (default http://localhost:4173), QA_SHOT_DIR (default - * qa-shots/recent-imagery), QA_HEADFUL=1. Every wait is bounded. Without - * network access to NASA the run skips with exit code 0. + * qa-shots/recent-imagery), QA_HEADFUL=1. + * QA_RECENT_IMAGERY_SLOW_RENDER=1 permits loaded SwiftShader CI to use a + * 4 s ZOOM IN wait and 0.8 s height-stability tail (defaults: 2 s / 1.2 s). + * Every wait is bounded. Without NASA access the run skips with code 0. */ import fs from 'node:fs'; import path from 'node:path'; import puppeteer from 'puppeteer'; +import { chromeExecutablePath } from './qa-chrome.mjs'; const BASE_URL = process.env.QA_BASE_URL || 'http://localhost:4173'; const SHOT_DIR = process.env.QA_SHOT_DIR || 'qa-shots/recent-imagery'; const HEADFUL = process.env.QA_HEADFUL === '1'; +const SLOW_RENDER = process.env.QA_RECENT_IMAGERY_SLOW_RENDER === '1'; +const ZOOM_TIMEOUT_MS = SLOW_RENDER ? 4_000 : 2_000; +const HEIGHT_TAIL_SAMPLES = SLOW_RENDER ? 8 : 12; const STEP_TIMEOUT = 20_000; const AUSTIN = { lon: -97.74, lat: 30.27, height: 12_000 }; const PANEL = '#recent-imagery-panel'; @@ -49,8 +55,7 @@ const browser = await puppeteer.launch({ headless: !HEADFUL, ...(HEADFUL ? { - executablePath: - '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + executablePath: chromeExecutablePath(), } : {}), args: [ @@ -308,15 +313,21 @@ const CONTROLS = { }; const measure = () => page.evaluate((controls) => { + const panel = document.getElementById('recent-imagery-panel'); const body = document.getElementById('recent-imagery-panel-body'); + const panelRect = panel?.getBoundingClientRect(); const out = { bodyScroll: body?.scrollTop ?? null, rects: {} }; for (const [name, selector] of Object.entries(controls)) { const rect = document.querySelector(selector)?.getBoundingClientRect(); - out.rects[name] = rect - ? [rect.left, rect.top, rect.width, rect.height].map( - (v) => Math.round(v * 10) / 10, - ) - : null; + out.rects[name] = + rect && panelRect + ? [ + rect.left - panelRect.left, + rect.top - panelRect.top, + rect.width, + rect.height, + ].map((v) => Math.round(v * 10) / 10) + : null; } return out; }, CONTROLS); @@ -372,6 +383,9 @@ try { const initialStack = await page.evaluate(() => window.__riQa.activeStack()); const photoreal = initialStack === 'photoreal'; console.log(`initial map stack: ${initialStack}`); + console.log( + `stability: ${SLOW_RENDER ? 'slow-render' : 'default'} (zoom ${ZOOM_TIMEOUT_MS} ms, last ${HEIGHT_TAIL_SAMPLES} samples)`, + ); const onPhotorealOnly = (name, passed, detail) => photoreal ? check(name, passed, detail) @@ -1042,13 +1056,13 @@ try { ) <= target * 0.01, fitHeight, - 2_000, + ZOOM_TIMEOUT_MS, ); const zoomedHeight = await page.evaluate( () => window.__godsEyeView.viewer.camera.positionCartographic.height, ); check( - `ZOOM IN flies to ${Math.round(fitHeight / 1000)} km within 2 s`, + `ZOOM IN reaches target within ${ZOOM_TIMEOUT_MS / 1_000} s`, zoomed, `height=${Math.round(zoomedHeight)} target=${Math.round(fitHeight)}`, ); @@ -1258,7 +1272,8 @@ try { ); await wait(100); } - const tail = (key) => new Set(samples.slice(8).map((s) => s[key])); + const tail = (key) => + new Set(samples.slice(-HEIGHT_TAIL_SAMPLES).map((s) => s[key])); return { settled: tail('imagery').size === 1 && tail('cctv').size === 1, detail: `imagery=${samples.map((s) => s.imagery).join(',')} cctv=${samples.map((s) => s.cctv).join(',')}`, diff --git a/scripts/qa-street-level.mjs b/scripts/qa-street-level.mjs index 59b0584da..6f87762cf 100644 --- a/scripts/qa-street-level.mjs +++ b/scripts/qa-street-level.mjs @@ -3,8 +3,9 @@ * Browser QA for the Street Level layer against a running server. Run with * `npm run qa:street-level -- --url http://localhost:4173`. * - * `--fixtures` (the CI mode) answers every *.mapillary.com request from - * fixtures, so nothing reaches Mapillary. The server still needs a token + * `--fixtures` (the CI mode) answers every *.mapillary.com, KartaView and + * Panoramax request from fixtures, so nothing reaches those providers. The + * server still needs a token * because the bundle carries it; CI uses a dummy one. Fixture runs also stage * a rejected key and a keyless second server. * @@ -37,6 +38,26 @@ import { PHOTO_SEQUENCE_ID, photoImages, } from './fixtures/street-level/mapillaryGraph.mjs'; +import { + answerKartaViewRequest, + describeKartaViewCall, + KV_LINE, + KV_SEQUENCE_ID, + kartaViewLookups, + kartaViewPhotos, + resetKartaViewFixture, +} from './fixtures/street-level/kartaView.mjs'; +import { + answerPanoramaxRequest, + describePanoramaxCall, + PNX_CREATOR, + PNX_LINE, + PNX_PARK, + PNX_SEQUENCES, + panoramaxHdDownloads, + panoramaxPictures, + resetPanoramaxFixture, +} from './fixtures/street-level/panoramax.mjs'; const PANEL_ID = 'street-level-panel'; @@ -49,18 +70,32 @@ export const VIEWPORTS = Object.freeze([ ]); /** Provider chips the panel must show, in order. */ -export const EXPECTED_PROVIDERS = Object.freeze(['mapillary']); +export const EXPECTED_PROVIDERS = Object.freeze([ + 'mapillary', + 'google', + 'kartaview', + 'panoramax', +]); /** - * Skips no setup can avoid: CI forks have no Google key, and live runs cannot - * stage fixture-only cases. `no Mapillary key` is not accepted: CI's dummy - * token runs the keyed flow, so a keyless strict run means that wiring broke. + * Skips no setup can avoid: CI forks have no Google key (so neither Google 3D + * nor Street View), and live runs cannot stage fixture-only cases. `no + * Mapillary key` is not accepted: CI's dummy token runs the keyed flow, so a + * keyless strict run means that wiring broke. Nor is `Google key refused`: a + * key that is set must work. */ export const STRICT_ALLOWED_SKIPS = Object.freeze([ 'no Google 3D', + 'no Google key', 'fixtures only', ]); +/** A Sacramento street with Google Street View (10th St, by the Capitol). */ +export const STREET_VIEW_STREET = Object.freeze({ + lon: -121.49372, + lat: 38.57986, +}); + export function strictViolations(skips) { return skips.filter((skip) => !STRICT_ALLOWED_SKIPS.includes(skip.reason)); } @@ -220,6 +255,11 @@ export async function serveFixtures(page, fixture) { patterns: [ { urlPattern: '*://*.mapillary.com/*' }, { urlPattern: '*/api/mapillary/*' }, + { urlPattern: '*://api.kartaview.org/*' }, + { urlPattern: '*://cdn.kartaview.org/*' }, + { urlPattern: '*://*.openstreetcam.org/*' }, + { urlPattern: '*://api.panoramax.xyz/*' }, + { urlPattern: '*://panoramax.openstreetmap.fr/*' }, ], }); const fulfil = (requestId, { status, headers = {}, contentType, body }) => @@ -250,6 +290,19 @@ export async function serveFixtures(page, fixture) { }); fixture.calls.add(call); if (!answer.known) fixture.unknown.push(call); + } else if (/(^|\.)(kartaview|openstreetcam)\.org$/.test(address.hostname)) { + const call = describeKartaViewCall(method, address); + answer = answerKartaViewRequest({ method, url: request.url }); + fixture.calls.add(call); + if (!answer.known) fixture.unknown.push(call); + } else if ( + address.hostname === 'api.panoramax.xyz' || + address.hostname === 'panoramax.openstreetmap.fr' + ) { + const call = describePanoramaxCall(method, address); + answer = answerPanoramaxRequest({ method, url: request.url }); + fixture.calls.add(call); + if (!answer.known) fixture.unknown.push(call); } else if ( address.pathname === '/api/mapillary/status' && fixture.configured !== undefined @@ -434,10 +487,50 @@ async function assertKeylessGate(page) { assert.match(chip.title, /MAPILLARY_CLIENT_TOKEN/); } +/** + * With no Mapillary key the panel points at KartaView, which needs none: + * lit, it draws coverage and the layer is not key-gated. + */ +async function assertKeylessKartaView(page) { + assert.match( + await page.$eval('#sl-coverage-meta', (node) => node.textContent), + /light KARTAVIEW/, + 'KEY REQUIRED points at KartaView', + ); + for (let attempt = 0; attempt < 4; attempt++) { + await page.evaluate((at) => { + const v = window.__godsEyeView.viewer; + v.camera.cancelFlight?.(); + const C = v.camera.positionCartographic.constructor; + v.camera.setView({ + destination: v.scene.globe.ellipsoid.cartographicToCartesian( + C.fromDegrees(at.lon, at.lat, at.height), + ), + orientation: { heading: 0, pitch: at.pitch, roll: 0 }, + }); + }, PARK); + await sleep(1000); + } + await clickPanelControl( + page, + '#sl-provider-chips [data-chip-id="kartaview"]', + ); + await waitForCoverage(page); + const state = await page.evaluate(() => ({ + status: document.getElementById('sl-status').textContent, + keyRequired: window.__godsEyeView.dataManager.layers + .get('street-level') + .module.getUIState().keyRequired, + })); + assert.equal(state.keyRequired, false); + assert.doesNotMatch(state.status, /KEY/); +} + /** * Runs in the page. Find a point where a real click picks `prefix` + `id` (any * id when null), searched around `points` ([lon, lat]), the aimed billboards, * or else a grid mid-screen. Only points where the canvas is topmost count. + * Each call picks for at most `budgetMs`; the next call resumes the search. * @returns {{x: number, y: number, id: string}|null} client coordinates */ export function findPickPoint({ @@ -446,6 +539,7 @@ export function findPickPoint({ points, billboardIds, billboardPrefix = null, + budgetMs = 4000, }) { const viewer = window.__godsEyeView.viewer; const scene = viewer.scene; @@ -500,24 +594,54 @@ export function findPickPoint({ centres.push([at.x, at.y]); } } - for (const [cx, cy] of centres) - for (let r = 0; r <= 16; r += 2) + // A pick is a render of the scene (0.5–2 s each under SwiftShader): try + // the centres the canvas is on top of first, nearest the screen centre, + // so the first pick usually hits. + const uncovered = ([x, y]) => + document.elementFromPoint(rect.left + x, rect.top + y) === canvas; + const fromMiddle = ([x, y]) => + (x - rect.width / 2) ** 2 + (y - rect.height / 2) ** 2; + centres.sort( + (a, b) => + Number(uncovered(b)) - Number(uncovered(a)) || + fromMiddle(a) - fromMiddle(b), + ); + const candidates = []; + for (const [cx, cy] of centres) { + candidates.push([cx, cy]); + for (let r = 2; r <= 16; r += 2) for (const [dx, dy] of [ - [0, 0], [r, 0], [-r, 0], [0, r], [0, -r], - ]) { - const hit = tryAt(cx + dx, cy + dy); - if (hit) return hit; - } - if (points || billboardIds) return null; - for (let y = rect.height * 0.25; y < rect.height * 0.8; y += 18) - for (let x = rect.width * 0.25; x < rect.width * 0.7; x += 6) { - const hit = tryAt(x, y); - if (hit) return hit; + ]) + candidates.push([cx + dx, cy + dy]); + } + // Aimed at known places: never fall back to a scan of the whole screen. + if (!points && !billboardIds && billboardPrefix === null) + for (let y = rect.height * 0.25; y < rect.height * 0.8; y += 18) + for (let x = rect.width * 0.25; x < rect.width * 0.7; x += 6) + candidates.push([x, y]); + // One evaluation stays well inside the caller's wait: past the budget it + // returns null, and the next poll carries on from where this one stopped. + const key = JSON.stringify([prefix, id, billboardPrefix]); + const cursors = (window.__qaPickCursors ??= {}); + const deadline = performance.now() + budgetMs; + let at = cursors[key] ?? 0; + if (at >= candidates.length) at = 0; + for (; at < candidates.length; at++) { + const hit = tryAt(...candidates[at]); + if (hit) { + delete cursors[key]; + return hit; } + if (performance.now() > deadline) { + cursors[key] = at + 1; + return null; + } + } + delete cursors[key]; return null; } @@ -619,14 +743,21 @@ async function main() { } await page.setViewport(VIEWPORTS[0]); const expandStrip = async (target = page) => { - await target.click( + await clickPanelControl( + target, '.panel-collapse-btn[data-collapse-target="street-level-panel"]', ); - await settle( - target, - () => - getComputedStyle(document.getElementById('sl-body')).display === - 'flex', + assert.ok( + await settle( + target, + () => + !document + .getElementById('street-level-panel') + .classList.contains('collapsed') && + getComputedStyle(document.getElementById('sl-body')).display === + 'flex', + ), + 'Street Level panel did not expand before the timeout', ); }; await expandStrip(); @@ -643,14 +774,30 @@ async function main() { ), ].map((chip) => chip.dataset.chipId), ); - assert.deepEqual(chips, EXPECTED_PROVIDERS); - // One swatch per source plus "Selected"; each chip wears its colour. - assert.equal( - await page.evaluate( - () => document.querySelectorAll('#sl-legend li').length, - ), - EXPECTED_PROVIDERS.length + 1, + // Every registered provider usable on this map, in order (Street View + // only on Google 3D). + const listed = await page.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('street-level') + .module.getUIState() + .providers.map((provider) => provider.id), ); + assert.deepEqual(chips, listed); + assert.deepEqual( + EXPECTED_PROVIDERS.filter((id) => listed.includes(id)), + listed, + ); + assert.ok(listed.includes('mapillary')); + // One swatch per switched-on source plus "Selected" (Street View + // starts off); each chip wears its colour. + const legend = await page.evaluate(() => ({ + swatches: document.querySelectorAll('#sl-legend li').length, + on: window.__godsEyeView.dataManager.layers + .get('street-level') + .module.getUIState() + .providers.filter((provider) => provider.on).length, + })); + assert.equal(legend.swatches, legend.on + 1); assert.equal( await page.$eval('[data-chip-id="mapillary"]', (chip) => chip.style.getPropertyValue('--chip-color'), @@ -694,6 +841,762 @@ async function main() { `${count} steps: ${count - skips.length} ok, ${skips.length} skipped`, ); + /** + * KartaView: keyless coverage from its JSON tiles, a line click opening + * the nearest photo, cones along the drive, › and a 360° photo. Runs on a + * draped map so a line is where it is drawn; leaves Mapillary lit. + */ + async function kartaViewSteps() { + // The first lookup from here on is the rate-limited one. + resetKartaViewFixture(); + const ui = () => + page.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('street-level') + .module.getUIState(), + ); + const uiUntil = (test, arg = null, options) => + settle( + page, + `(${test})(window.__godsEyeView.dataManager.layers.get('street-level').module.getUIState(), ${JSON.stringify(arg)})`, + undefined, + options, + ); + const lit = (id) => + uiUntil( + (u, want) => + u.providers.find((p) => p.id === want)?.on === true && + u.providers.filter((p) => p.on).length === 1, + id, + ); + const stack = () => + page.evaluate(() => + window.__godsEyeView.mapStackController.getActiveId(), + ); + const setStack = async (id) => { + await page.evaluate( + (stackId) => + window.__godsEyeView.mapStackController.setStack(stackId), + id, + ); + await settle( + page, + (stackId) => + window.__godsEyeView.mapStackController.getActiveId() === stackId, + id, + ); + }; + const kvChip = '#sl-provider-chips [data-chip-id="kartaview"]'; + const original = await stack(); + const photos = kartaViewPhotos(); + await page.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('street-level') + .module.closeViewer(), + ); + await setStack('esri-imagery'); + // Over the fixture drive, looking down: its line is mid-screen. + for (let attempt = 0; attempt < 4; attempt++) { + await page.evaluate((at) => { + const v = window.__godsEyeView.viewer; + v.camera.cancelFlight?.(); + const C = v.camera.positionCartographic.constructor; + v.camera.setView({ + destination: v.scene.globe.ellipsoid.cartographicToCartesian( + C.fromDegrees(at.lon, at.lat, at.height), + ), + orientation: { heading: 0, pitch: at.pitch, roll: 0 }, + }); + }, PARK); + await sleep(1000); + } + await step( + 'KartaView: its chip draws keyless coverage, credits KartaView and offers Mapillary’s filters, SINCE noted as its upload date', + async () => { + await clickPanelControl(page, kvChip); + assert.ok(await lit('kartaview'), 'KartaView alone is lit'); + await waitForCoverage(page); + const u = await ui(); + assert.equal(u.keyRequired, false); + assert.deepEqual(u.filterControls, { pano: true, since: true }); + // KartaView's SINCE is its upload date: the panel says so. + assert.equal(u.sinceNote, 'KartaView: by upload date'); + assert.equal( + await page.$eval('#sl-since-note', (node) => + node.hidden ? null : node.textContent, + ), + 'KartaView: by upload date', + ); + assert.equal( + await page.$eval(kvChip, (chip) => + chip.style.getPropertyValue('--chip-color'), + ), + '#635bff', + ); + assert.match( + await page.$eval('#sl-coverage-meta', (node) => node.textContent), + /streets in view · click a line for its nearest photo/, + ); + await page.waitForFunction( + () => + document.body.innerHTML.includes('KartaView contributors'), + { timeout: 15_000 }, + ); + assert.ok( + await settle(page, () => + (new URLSearchParams(location.hash.slice(1)).get('lo') || '') + .split('_') + .some( + (entry) => + entry.startsWith('0.') && entry.includes('kartaview-1'), + ), + ), + `the share link lights KartaView: ${await page.evaluate(() => location.hash)}`, + ); + }, + ); + await step( + 'KartaView: 360° and SINCE thin its lines through KartaView’s filtered tiles, and ALL / ANY DATE bring them back', + async () => { + const before = (await ui()).coverage.count; + const setStop = (index) => + page.$eval( + '#sl-since', + (input, value) => { + input.value = String(value); + input.dispatchEvent(new Event('input', { bubbles: true })); + input.dispatchEvent(new Event('change', { bubbles: true })); + }, + index, + ); + // Filtered tiles are drawn by KartaView on demand: give them time. + const settled = async (test, arg, what) => + assert.ok( + await uiUntil(test, arg, { timeout: 60_000 }), + `${what}: ${JSON.stringify((await ui()).coverage)}`, + ); + await clickPanelControl(page, '[data-sl-pano="pano"]'); + await settled( + (u) => u.filter.pano === 'pano' && !u.coverage.loading, + null, + '360° drawn', + ); + const panos = (await ui()).coverage.count; + assert.ok(panos <= before, `${panos} ≤ ${before}`); + // Fixtures: the old street has only flat photos, the drive 360° too. + if (fixtures) + assert.ok(panos > 0 && panos < before, `0 < ${panos} < ${before}`); + await clickPanelControl(page, '[data-sl-pano="all"]'); + await settled( + (u, n) => + u.filter.pano === 'all' && + !u.coverage.loading && + u.coverage.count === n, + before, + 'every line again', + ); + await setStop(5); + await settled( + (u) => u.filter.sinceDays === 365 && !u.coverage.loading, + null, + 'LAST YEAR drawn', + ); + const lastYear = (await ui()).coverage.count; + assert.ok(lastYear <= before, `${lastYear} ≤ ${before}`); + // Fixtures: the drive is a month old, the old street from 2019. + if (fixtures) + assert.ok( + lastYear > 0 && lastYear < before, + `0 < ${lastYear} < ${before}`, + ); + await setStop(0); + await settled( + (u, n) => + u.filter.sinceDays === 0 && + !u.coverage.loading && + u.coverage.count === n, + before, + 'ANY DATE: every line again', + ); + assert.equal((await ui()).coverage.error, null); + }, + ); + await step( + 'KartaView: a click on a line opens the nearest photo where the camera is, cones follow its drive, and › steps along it, past an archived photo', + async ({ skip }) => { + const along = Array.from({ length: 9 }, (_, i) => [ + KV_LINE.lon, + KV_LINE.south + ((i + 2) / 13) * (KV_LINE.north - KV_LINE.south), + ]); + const handle = await page + .waitForFunction( + findPickPoint, + { timeout: 30_000, polling: 250 }, + { + prefix: 'kv:line:', + id: null, + points: fixtures ? along : null, + billboardIds: null, + }, + ) + .catch(() => null); + if (!handle && !fixtures) return skip('no KartaView line in view'); + assert.ok(handle, 'a KartaView line is pickable'); + const at = await handle.jsonValue(); + const camera = () => + page.evaluate(() => { + const c = window.__godsEyeView.viewer.camera.positionCartographic; + return { + lon: (c.longitude * 180) / Math.PI, + lat: (c.latitude * 180) / Math.PI, + height: c.height, + }; + }); + const parked = await camera(); + // The first lookup is rate-limited (fixtures): it waits and retries. + await page.mouse.click(at.x, at.y); + await page.waitForFunction( + () => { + const s = window.__godsEyeView.dataManager.layers + .get('street-level') + .module.getUIState().street; + return (s.imageId && !s.loading) || s.error; + }, + { timeout: 60_000 }, + ); + const street = (await ui()).street; + if (!fixtures && /No street-level imagery/.test(street.error || '')) + return skip('no KartaView photo near the line'); + if (fixtures) + assert.ok( + kartaViewLookups() >= 2, + `the rate-limited lookup was asked again (${kartaViewLookups()} lookups)`, + ); + assert.equal(street.error, null, `viewer error: ${street.error}`); + assert.equal(street.providerId, 'kartaview'); + assert.equal(street.renderModes, true, 'FIT / FILL for a flat photo'); + assert.ok( + await settle(page, () => { + const img = document.querySelector('.sl-photo-flat'); + return img && !img.hidden && img.naturalWidth > 0; + }), + 'the photo is on screen', + ); + // A line click picks a spot: the camera stays where it was. + await sleep(1500); + const now = await camera(); + assert.ok( + Math.abs(now.lon - parked.lon) < 1e-6 && + Math.abs(now.lat - parked.lat) < 1e-6 && + Math.abs(now.height - parked.height) < 0.5, + `the camera did not move: ${JSON.stringify({ parked, now })}`, + ); + if (fixtures) { + const opened = photos.find((photo) => photo.id === street.imageId); + assert.ok(opened, `${street.imageId} is a fixture photo`); + assert.equal( + opened.archived, + false, + 'the nearest photo that storage still has', + ); + assert.ok( + await uiUntil( + (u, want) => + u.sequence.selectedId === want.id && + !u.sequence.loading && + u.sequence.images === want.count, + { id: KV_SEQUENCE_ID, count: photos.length }, + { timeout: 30_000 }, + ), + `cones for the whole drive: ${JSON.stringify((await ui()).sequence)}`, + ); + // The drive runs north; the photos' compass says 200°. + assert.ok( + await uiUntil( + (u) => + Number.isFinite(u.street.bearing) && + Math.min(u.street.bearing, 360 - u.street.bearing) < 2, + ), + `the camera looks along the drive, not where the compass points: ${(await ui()).street.bearing}`, + ); + } + const before = street.imageId; + await clickPanelControl(page, '.sl-photo-step-next'); + assert.ok( + await uiUntil( + (u, was) => + u.street.imageId && u.street.imageId !== was && !u.street.error, + before, + { timeout: 30_000 }, + ), + '› opened the next photo', + ); + if (fixtures) { + const skipped = photos.find( + (photo) => photo.id === String(Number(before) + 1), + ); + assert.equal( + skipped?.archived, + true, + 'the photo after is archived', + ); + assert.equal( + (await ui()).street.imageId, + String(Number(before) + 2), + '› skipped the archived photo to the next one along the drive', + ); + } + }, + ); + if (fixtures) + await step( + 'KartaView: a 360° photo opens on the panorama without FIT / FILL, dragging turns it, and it survives a lost WebGL context', + async () => { + const pano = photos.find((photo) => photo.isPano); + await page.evaluate((id) => { + void window.__godsEyeView.dataManager.layers + .get('street-level') + .module.openImage('kartaview', id); + }, pano.id); + assert.ok( + await uiUntil( + (u, want) => + u.street.imageId === want && + !u.street.loading && + u.street.isPano, + pano.id, + { timeout: 60_000 }, + ), + `the panorama opened: ${JSON.stringify((await ui()).street)}`, + ); + const u = await ui(); + assert.equal(u.street.error, null); + assert.equal(u.street.renderModes, false, 'no frame to fit'); + assert.equal( + await page.$eval('.sl-photo-pano', (canvas) => canvas.hidden), + false, + ); + const box = await page.$eval('.sl-photo-pano', (canvas) => { + const r = canvas.getBoundingClientRect(); + return { x: r.x + r.width / 2, y: r.y + r.height / 2 }; + }); + const bearing = u.street.bearing; + await page.mouse.move(box.x, box.y); + await page.mouse.down(); + for (let i = 1; i <= 8; i++) + await page.mouse.move(box.x - i * 15, box.y); + await page.mouse.up(); + assert.ok( + await uiUntil((v, was) => v.street.bearing !== was, bearing), + 'dragging the panorama turned its bearing', + ); + // A GPU reset: the panorama comes back on a new canvas, and drags. + assert.ok( + await page.evaluate(() => { + const canvas = document.querySelector('.sl-photo-pano'); + window.__kvLostCanvas = canvas; + const lose = canvas + .getContext('webgl') + ?.getExtension('WEBGL_lose_context'); + lose?.loseContext(); + return Boolean(lose); + }), + 'the browser can lose a WebGL context on purpose', + ); + assert.ok( + await settle( + page, + () => { + const canvas = document.querySelector('.sl-photo-pano'); + return ( + canvas && + canvas !== window.__kvLostCanvas && + !canvas.hidden && + canvas.width > 1 + ); + }, + null, + { timeout: 30_000 }, + ), + 'the panorama is drawn again on a new canvas', + ); + const after = await ui(); + assert.equal(after.street.imageId, pano.id); + assert.equal(after.street.isPano, true); + assert.equal(after.street.error, null); + const turned = after.street.bearing; + await page.mouse.move(box.x, box.y); + await page.mouse.down(); + for (let i = 1; i <= 8; i++) + await page.mouse.move(box.x + i * 15, box.y); + await page.mouse.up(); + assert.ok( + await uiUntil((v, was) => v.street.bearing !== was, turned), + 'the new canvas drags', + ); + }, + ); + else + await skipStep( + 'fixtures only', + 'KartaView: a 360° photo opens on the panorama without FIT / FILL, dragging turns it, and it survives a lost WebGL context', + ); + await page.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('street-level') + .module.closeViewer(), + ); + await clickPanelControl( + page, + '#sl-provider-chips [data-chip-id="mapillary"]', + ); + assert.ok(await lit('mapillary'), 'Mapillary is lit again'); + await setStack(original); + } + + /** + * Panoramax: keyless orange coverage with both filters, a line click + * opening that sequence's picture nearest the click (author, licence, + * cones, ›), a 360° picture whose original is fetched once on EXPAND. + * Live runs park over Paris, where Panoramax has pictures. Runs on a + * draped map; leaves Mapillary lit. + */ + async function panoramaxSteps() { + resetPanoramaxFixture(); + const ui = () => + page.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('street-level') + .module.getUIState(), + ); + const uiUntil = (test, arg = null, options) => + settle( + page, + `(${test})(window.__godsEyeView.dataManager.layers.get('street-level').module.getUIState(), ${JSON.stringify(arg)})`, + undefined, + options, + ); + const lit = (id) => + uiUntil( + (u, want) => + u.providers.find((p) => p.id === want)?.on === true && + u.providers.filter((p) => p.on).length === 1, + id, + ); + const module = (call, ...args) => + page.evaluate( + (name, list) => + window.__godsEyeView.dataManager.layers + .get('street-level') + .module[name](...list), + call, + args, + ); + const stack = () => + page.evaluate(() => + window.__godsEyeView.mapStackController.getActiveId(), + ); + const setStack = async (id) => { + await page.evaluate( + (stackId) => + window.__godsEyeView.mapStackController.setStack(stackId), + id, + ); + await settle( + page, + (stackId) => + window.__godsEyeView.mapStackController.getActiveId() === stackId, + id, + ); + }; + const camera = () => + page.evaluate(() => { + const c = window.__godsEyeView.viewer.camera.positionCartographic; + return { + lon: (c.longitude * 180) / Math.PI, + lat: (c.latitude * 180) / Math.PI, + height: c.height, + }; + }); + const pnxChip = '#sl-provider-chips [data-chip-id="panoramax"]'; + const original = await stack(); + await module('closeViewer'); + await setStack('esri-imagery'); + for (let attempt = 0; attempt < 4; attempt++) { + await page.evaluate((at) => { + const v = window.__godsEyeView.viewer; + v.camera.cancelFlight?.(); + const C = v.camera.positionCartographic.constructor; + v.camera.setView({ + destination: v.scene.globe.ellipsoid.cartographicToCartesian( + C.fromDegrees(at.lon, at.lat, at.height), + ), + orientation: { heading: 0, pitch: at.pitch, roll: 0 }, + }); + }, PNX_PARK); + await sleep(1000); + } + try { + await step( + 'Panoramax: its chip draws keyless orange coverage, keeps both filters, credits Panoramax and lights in the share link', + async () => { + await clickPanelControl(page, pnxChip); + assert.ok(await lit('panoramax'), 'Panoramax alone is lit'); + await waitForCoverage(page); + const u = await ui(); + assert.equal(u.keyRequired, false); + assert.deepEqual(u.filterControls, { pano: true, since: true }); + assert.equal( + await page.$eval(pnxChip, (chip) => + chip.style.getPropertyValue('--chip-color'), + ), + '#ff6f00', + ); + assert.match( + await page.$eval('#sl-coverage-meta', (node) => node.textContent), + /sequences in view · click a line for its nearest photo/, + ); + await page.waitForFunction( + () => + document.body.innerHTML.includes('Panoramax contributors'), + { timeout: 15_000 }, + ); + assert.ok( + await settle(page, () => + (new URLSearchParams(location.hash.slice(1)).get('lo') || '') + .split('_') + .some( + (entry) => + entry.startsWith('0.') && entry.includes('panoramax-1'), + ), + ), + `the share link lights Panoramax: ${await page.evaluate(() => location.hash)}`, + ); + }, + ); + await step( + 'Panoramax: a click on a line opens that sequence’s picture nearest the click, camera still, with author and licence, cones and ›', + async ({ skip }) => { + // Points on the street: the fixture's, or live pictures there. + let points = null; + if (fixtures) + points = Array.from({ length: 9 }, (_, i) => [ + PNX_LINE.west + + ((i + 2) / 13) * (PNX_LINE.east - PNX_LINE.west), + PNX_LINE.lat, + ]); + else { + const near = await fetch( + `https://api.panoramax.xyz/api/search?place_position=${PNX_PARK.lon},${PNX_PARK.lat}&place_distance=0-150&place_fov_tolerance=180&limit=12`, + ) + .then((r) => r.json()) + .catch(() => null); + points = near?.features?.map((f) => f.geometry.coordinates); + if (!points?.length) + return skip('no Panoramax picture near the park'); + } + const handle = await page + .waitForFunction( + findPickPoint, + { timeout: 60_000, polling: 500 }, + { + prefix: 'pnx:seq:', + id: null, + points, + billboardIds: null, + }, + ) + .catch(() => null); + if (!handle && !fixtures) return skip('no Panoramax line in view'); + assert.ok(handle, 'a Panoramax line is pickable'); + let at = await handle.jsonValue(); + if (fixtures) { + // Click the top line where the other sequence's pictures are: + // the nearest picture is then the other's, and only the line + // click's own-sequence lookup opens the clicked line's. + const others = panoramaxPictures() + .filter((p) => p.sequenceId !== at.id) + .slice(40, 100) + .map((p) => [p.lon, p.lat]); + const aimed = await page + .waitForFunction( + findPickPoint, + { timeout: 60_000, polling: 500 }, + { + prefix: 'pnx:seq:', + id: at.id, + points: others, + billboardIds: null, + }, + ) + .catch(() => null); + assert.ok( + aimed, + 'the line is pickable over the other’s pictures', + ); + at = await aimed.jsonValue(); + } + const parked = await camera(); + await page.mouse.click(at.x, at.y); + await page.waitForFunction( + () => { + const s = window.__godsEyeView.dataManager.layers + .get('street-level') + .module.getUIState().street; + return (s.imageId && !s.loading) || s.error; + }, + { timeout: 60_000 }, + ); + const street = (await ui()).street; + assert.equal(street.error, null, `viewer error: ${street.error}`); + assert.equal(street.providerId, 'panoramax'); + assert.equal( + street.sequenceId, + at.id, + 'the picture is from the line clicked', + ); + assert.ok(street.license, 'the caption names the licence'); + assert.match( + street.externalUrl, + /^https:\/\/api\.panoramax\.xyz\/#pic=/, + ); + await sleep(1500); + const now = await camera(); + assert.ok( + Math.abs(now.lon - parked.lon) < 1e-6 && + Math.abs(now.lat - parked.lat) < 1e-6 && + Math.abs(now.height - parked.height) < 0.5, + `the camera did not move: ${JSON.stringify({ parked, now })}`, + ); + if (fixtures) { + const caption = await page.$eval( + '#sl-image-by', + (node) => node.textContent, + ); + if (street.sequenceId === PNX_SEQUENCES[0]) + assert.equal(caption, `Image by ${PNX_CREATOR} · CC BY-SA 4.0`); + else + assert.equal( + caption, + 'via panoramax.openstreetmap.fr · Licence Ouverte 2.0', + 'an account named by an email address is not shown; the instance is', + ); + const pictures = panoramaxPictures().filter( + (p) => p.sequenceId === street.sequenceId, + ); + assert.ok( + await uiUntil( + (u, want) => + u.sequence.selectedId === want && + !u.sequence.loading && + u.sequence.images > 0, + street.sequenceId, + { timeout: 30_000 }, + ), + `cones along the sequence: ${JSON.stringify((await ui()).sequence)}`, + ); + assert.ok((await ui()).sequence.images <= pictures.length); + } + const before = street.imageId; + await clickPanelControl(page, '.sl-photo-step-next'); + assert.ok( + await uiUntil( + (u, was) => + u.street.imageId && + u.street.imageId !== was && + !u.street.loading && + !u.street.error, + before, + { timeout: 30_000 }, + ), + '› opened the next picture', + ); + const after = (await ui()).street; + assert.equal(after.sequenceId, street.sequenceId); + if (fixtures) { + const all = panoramaxPictures(); + const from = all.find((p) => p.id === before); + assert.equal( + after.imageId, + all.find( + (p) => + p.sequenceId === from.sequenceId && + p.index === from.index + 1, + ).id, + '› followed the picture’s next link', + ); + } + }, + ); + if (fixtures) + await step( + 'Panoramax: a 360° picture opens on the panorama without FIT / FILL, drags, and EXPAND fetches its original once', + async () => { + const pano = panoramaxPictures().find((p) => p.isPano); + await module('openImage', 'panoramax', pano.id); + assert.ok( + await uiUntil( + (u, want) => + u.street.imageId === want && + !u.street.loading && + u.street.isPano, + pano.id, + { timeout: 60_000 }, + ), + `the panorama opened: ${JSON.stringify((await ui()).street)}`, + ); + const u = await ui(); + assert.equal(u.street.renderModes, false, 'no frame to fit'); + const box = await page.$eval('.sl-photo-pano', (canvas) => { + const r = canvas.getBoundingClientRect(); + return { x: r.x + r.width / 2, y: r.y + r.height / 2 }; + }); + const bearing = u.street.bearing; + await page.mouse.move(box.x, box.y); + await page.mouse.down(); + for (let i = 1; i <= 8; i++) + await page.mouse.move(box.x - i * 15, box.y); + await page.mouse.up(); + assert.ok( + await uiUntil((v, was) => v.street.bearing !== was, bearing), + 'dragging the panorama turned its bearing', + ); + assert.equal(panoramaxHdDownloads(), 0, 'docked: SD only'); + await clickPanelControl(page, '#sl-viewer-expand'); + assert.ok( + await settle(page, () => + Boolean(document.querySelector('.sl-viewer-wrap-expanded')), + ), + 'the viewer expanded', + ); + const deadline = Date.now() + 30_000; + while (panoramaxHdDownloads() === 0 && Date.now() < deadline) + await sleep(250); + await sleep(2000); + assert.equal( + panoramaxHdDownloads(), + 1, + 'the original, once, for the expanded viewer', + ); + await page.keyboard.press('Escape'); + }, + ); + else + await skipStep( + 'fixtures only', + 'Panoramax: a 360° picture opens on the panorama without FIT / FILL, drags, and EXPAND fetches its original once', + ); + } finally { + await module('closeViewer'); + await clickPanelControl( + page, + '#sl-provider-chips [data-chip-id="mapillary"]', + ); + assert.ok(await lit('mapillary'), 'Mapillary is lit again'); + await setStack(original); + } + } + async function keyedSteps() { // A late startup flight only shows itself by moving the camera, so park // and check it stays put rather than reading a state. @@ -1084,9 +1987,10 @@ async function main() { ((i + 2) / 13) * (PHOTO_LINE.north - PHOTO_LINE.south), ]); // Ground primitives are built over a few frames: poll for a pick. + // A loaded machine renders a frame in seconds: 60 s, not 30. const handle = await page.waitForFunction( findPickPoint, - { timeout: 30_000, polling: 250 }, + { timeout: 60_000, polling: 250 }, { prefix: 'mly:seq:', id: target, @@ -1111,22 +2015,28 @@ async function main() { ); if (fixtures) assert.equal(sequence.images, fixture.images.length, 'every photo'); - // The cones are on the globe: one of them picks as an image. + // The cones are on the globe: one of them picks as an image. Aim at + // where each cone billboard is drawn (its ground position at + // height 0 can sit a few pixels off, and every missed pick costs a + // render); the same 60 s as the line pick above. const cone = await page.waitForFunction( findPickPoint, - { timeout: 15_000, polling: 250 }, + { timeout: 60_000, polling: 250 }, { prefix: 'mly:img:', id: null, - points: fixtures - ? fixture.images.map((image) => [image.lon, image.lat]) - : null, + points: null, billboardIds: null, - // Live: the cones' positions are only known from the globe. - billboardPrefix: fixtures ? null : 'mly:img:', + billboardPrefix: 'mly:img:', }, ); - assert.ok(await cone.jsonValue(), 'an image cone is pickable'); + const picked = await cone.jsonValue(); + assert.ok(picked, 'an image cone is pickable'); + if (fixtures) + assert.ok( + fixture.images.some((image) => image.id === picked.id), + `the cone is a photo of the drive: ${picked.id}`, + ); await page.evaluate(() => window.__godsEyeView.dataManager.layers .get('street-level') @@ -1254,7 +2164,9 @@ async function main() { assert.ok(neighbour, 'a neighbour of the other camera type'); const handle = await page.waitForFunction( findPickPoint, - { timeout: 15_000, polling: 250 }, + // A pick is a render; under SwiftShader on a busy machine one + // evaluation can take seconds. + { timeout: 30_000, polling: 250 }, { prefix: 'mly:img:', id: neighbour.id, @@ -1715,6 +2627,295 @@ async function main() { ); }, ); + const googleChip = '#sl-provider-chips [data-chip-id="google"]'; + const googleKey = await page.evaluate(() => + Boolean(window.__GOOGLE_MAPS_API_KEY__), + ); + await step( + 'Street View starts off, appears only on the Google 3D map, and travels in the share link', + async () => { + const module = () => + window.__godsEyeView.dataManager.layers.get('street-level').module; + assert.equal( + await page.evaluate(`(${module})().getParams().google`), + false, + 'off until lit: panoramas are billed', + ); + const setStreetView = (on) => + page.evaluate( + (value) => + window.__godsEyeView.dataManager.setLayerParams( + 'street-level', + { google: value }, + { origin: 'user' }, + ), + on, + ); + const shown = () => + page.evaluate(() => ({ + chip: Boolean( + document.querySelector( + '#sl-provider-chips [data-chip-id="google"]', + ), + ), + legend: [...document.querySelectorAll('#sl-legend li')].some( + (item) => /Street View/.test(item.textContent), + ), + credit: document.body.innerHTML.includes('Street View imagery'), + button: !document.getElementById('sl-nearest-btn').hidden, + })); + const original = await stacks.active(); + // Off Google 3D (CI's only map) nothing of Street View shows, even + // switched on: Google's terms keep it off other maps. + if (original === 'photoreal') await stacks.set('esri-imagery'); + await setStreetView(true); + await uiUntil((u) => u.providers.every((p) => p.id !== 'google')); + assert.ok( + await settle( + page, + () => !document.body.innerHTML.includes('Street View imagery'), + ), + 'no Google credit on another map', + ); + assert.deepEqual(await shown(), { + chip: false, + legend: false, + credit: false, + button: false, + }); + // Street View rides the shared provider-switch field (r), beside + // the other providers' switches a chip click recorded. + const option = `${STREET_LEVEL_TOKEN}.r.google-1`; + assert.ok( + await settle( + page, + (prefix) => + (new URLSearchParams(location.hash.slice(1)).get('lo') || '') + .split('_') + .some( + (entry) => + entry.startsWith(prefix) && + entry + .slice(prefix.length) + .split('*') + .includes('google-1'), + ), + `${STREET_LEVEL_TOKEN}.r.`, + // The link is written once the view rests for 500 ms; under a + // software renderer with a photo open that can take a while. + { timeout: 30_000 }, + ), + `the share link keeps the choice: ${option}`, + ); + if (await stacks.photoreal()) { + await stacks.set('photoreal'); + await uiUntil((u) => u.providers.some((p) => p.id === 'google')); + const chipNow = await page.$eval(googleChip, (node) => ({ + className: node.className, + title: node.title, + color: node.style.getPropertyValue('--chip-color'), + })); + assert.equal(chipNow.color, '#4285f4'); + if (googleKey) assert.match(chipNow.className, /chip-active/); + else { + assert.match(chipNow.className, /chip-error/); + assert.match(chipNow.title, /GOOGLE_MAPS_API_KEY/); + } + assert.ok( + await settle(page, () => + document.body.innerHTML.includes('Street View imagery'), + ), + 'the Google credit shows on Google 3D', + ); + } + // Back to Mapillary (one source at a time), as the next steps expect. + await page.evaluate(() => + window.__godsEyeView.dataManager.setLayerParams( + 'street-level', + { mapillary: true, google: false }, + { origin: 'user' }, + ), + ); + await stacks.set(original); + await uiUntil( + (u) => u.providers.find((p) => p.id === 'google')?.on !== true, + ); + }, + ); + await step( + 'on Google 3D a click on a street opens Google Street View where it is, captioned and linked, and its view drives the pose', + async ({ skip }) => { + if (!googleKey) return skip('no Google key'); + // Google's terms: Street View only beside Google's own map. + if (!(await stacks.photoreal())) return skip('no Google 3D'); + const original = await stacks.active(); + const closePhoto = () => + page.evaluate(() => + window.__godsEyeView.dataManager.layers + .get('street-level') + .module.closeViewer(), + ); + const setOn = async (selector, id, on) => { + const now = (await ui()).providers.find((p) => p.id === id)?.on; + if (now === on) return; + await clickPanelControl(page, selector); + assert.ok( + await uiUntil( + (u, want) => + u.providers.find((p) => p.id === want.id)?.on === want.on, + { id, on }, + ), + `the ${id} chip switched ${on ? 'on' : 'off'}`, + ); + }; + await closePhoto(); + await stacks.set('photoreal'); + // A Mapillary line under the pointer wins the click: aim at bare + // ground with only Street View on (lit first: the last lit chip + // would switch the layer off). + await setOn(googleChip, 'google', true); + await setOn(chip, 'mapillary', false); + try { + const street = STREET_VIEW_STREET; + await page.evaluate((at) => { + const v = window.__godsEyeView.viewer; + v.camera.cancelFlight?.(); + const C = v.camera.positionCartographic.constructor; + v.camera.setView({ + destination: v.scene.globe.ellipsoid.cartographicToCartesian( + C.fromDegrees(at.lon, at.lat, 300), + ), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + }, street); + // At street zoom the hint invites the click. + assert.ok( + await uiUntil( + (u) => + u.providers.find((p) => p.id === 'google')?.hint === + 'click a street to open Street View', + ), + 'the click hint shows at street zoom', + ); + const camera = () => + page.evaluate(() => { + const c = window.__godsEyeView.viewer.camera.positionWC; + return [c.x, c.y, c.z]; + }); + const at = await page.evaluate((point) => { + const v = window.__godsEyeView.viewer; + const C = v.camera.positionCartographic.constructor; + const ground = v.scene.globe.ellipsoid.cartographicToCartesian( + C.fromDegrees(point.lon, point.lat, 0), + ); + const canvas = v.scene.cartesianToCanvasCoordinates(ground); + const box = v.scene.canvas.getBoundingClientRect(); + return ( + canvas && { x: box.left + canvas.x, y: box.top + canvas.y } + ); + }, street); + assert.ok(at, 'the street is on screen'); + const before = await camera(); + await page.mouse.click(at.x, at.y); + await uiUntil( + (u) => + Boolean(u.street.error) || + (u.street.providerId === 'google' && + Boolean(u.street.imageId) && + !u.street.loading), + null, + { timeout: 30_000 }, + ); + const opened = (await ui()).street; + if (/refused GOOGLE_MAPS_API_KEY/.test(opened.error || '')) + return skip('Google key refused'); + if (/did not answer/.test(opened.error || '')) + return skip('Google did not answer'); + assert.equal(opened.error, null); + assert.equal(opened.providerId, 'google'); + const after = await camera(); + assert.ok( + Math.hypot(...after.map((v, i) => v - before[i])) < 1, + 'the camera stays where the user clicked', + ); + const shown = await page.evaluate(() => ({ + by: document.getElementById('sl-image-by').textContent, + when: document.getElementById('sl-image-when').textContent, + label: document.getElementById('sl-image-link').textContent, + href: document.getElementById('sl-image-link').href, + fitFillHidden: + document.querySelector('[data-sl-render]').parentElement.hidden, + panorama: Boolean( + document.querySelector('#sl-viewer .sl-google-panorama'), + ), + mapillaryLeft: Boolean( + document.querySelector('#sl-viewer .mapillary-dom'), + ), + })); + assert.match(shown.by, /Image by /); + assert.match(shown.when, /\d{4}-\d{2}$/, 'dated by month'); + assert.equal(shown.label, 'GOOGLE STREET VIEW ↗'); + assert.match(shown.href, /map_action=pano&pano=/); + assert.equal(shown.fitFillHidden, true, 'no FIT/FILL for Google'); + assert.equal(shown.panorama, true); + assert.equal(shown.mapillaryLeft, false, 'no viewer left under it'); + // A real drag turns Google's view, and the pose follows it. + const box = await page.$eval( + '#sl-viewer .sl-google-panorama', + (node) => { + const r = node.getBoundingClientRect(); + return { x: r.left + r.width / 2, y: r.top + r.height / 2 }; + }, + ); + await page.mouse.move(box.x, box.y); + await page.mouse.down(); + await page.mouse.move(box.x - 120, box.y, { steps: 12 }); + await page.mouse.up(); + assert.ok( + await uiUntil( + (u, start) => + Number.isFinite(u.street.bearing) && + Math.abs(u.street.bearing - start) > 5, + opened.bearing ?? 0, + { timeout: 10_000 }, + ), + 'the pose bearing follows the drag', + ); + // The keyboard path: OPEN STREET VIEW at the view centre. + await closePhoto(); + await uiUntil((u) => u.street.open === false); + await clickPanelControl(page, '#sl-nearest-btn'); + assert.ok( + await uiUntil( + (u) => + u.street.providerId === 'google' && + Boolean(u.street.imageId) && + !u.street.loading, + null, + { timeout: 30_000 }, + ), + 'OPEN STREET VIEW opens the panorama at the view centre', + ); + // Leaving Google 3D closes the panorama (Google's terms). + await stacks.set('esri-imagery'); + assert.ok( + await uiUntil((u) => u.street.open === false), + 'the panorama closes off Google 3D', + ); + } finally { + await closePhoto(); + // Off Google 3D the Street View chip is hidden: return to it + // before switching the chip, or the cleanup throws and hides + // the step's own result. + await stacks.set('photoreal'); + await setOn(chip, 'mapillary', true); + await setOn(googleChip, 'google', false); + await stacks.set(original); + } + }, + ); + await kartaViewSteps(); + await panoramaxSteps(); await step( 'a reload with the layer on keeps a collapsed panel collapsed, and the share link carries its ui token', async () => { @@ -1773,7 +2974,7 @@ async function main() { ); if (fixtures) await step( - 'keyless install (a second page, no key on the server) gates the controls and reports KEY REQUIRED', + 'keyless install (a second page, no key on the server) gates the controls and reports KEY REQUIRED, and KartaView draws there', async () => { const keyless = await browser.newPage(); await keyless.setViewport(VIEWPORTS[0]); @@ -1782,6 +2983,7 @@ async function main() { await boot(keyless, url); await expandStrip(keyless); await assertKeylessGate(keyless); + await assertKeylessKartaView(keyless); renderErrorsSeen.push( ...(await readRenderErrors(keyless)).map( (text) => `[keyless] scene.renderError: ${text}`, @@ -1797,7 +2999,7 @@ async function main() { ); if (fixtures) await step( - 'every Mapillary request was answered from a fixture', + 'every Mapillary, KartaView and Panoramax request was answered from a fixture', () => { console.log( ` (answered: ${[...fixture.calls].sort().join(' | ')})`, diff --git a/scripts/qa-traffic-jamviz-ab.mjs b/scripts/qa-traffic-jamviz-ab.mjs index 73d28ca60..b236ab0b1 100644 --- a/scripts/qa-traffic-jamviz-ab.mjs +++ b/scripts/qa-traffic-jamviz-ab.mjs @@ -139,15 +139,19 @@ async function main() { try { const res = await fetch(APP_URL); + // Only the status matters; an unread body can hold the socket open. + await res.body?.cancel(); if (!res.ok) throw new Error(`HTTP ${res.status}`); } catch (e) { console.error(`Dev server not reachable at ${APP_URL} (${e.message}).`); - process.exit(2); + process.exitCode = 2; + return; } const status = await fetch(`${APP_URL}/api/tomtom/status`).then((r) => r.json()).catch(() => null); if (!status?.hasKey) { console.error('Server has no TomTom key — the A/B needs live flow.'); - process.exit(2); + process.exitCode = 2; + return; } fs.mkdirSync(OUT_DIR, { recursive: true }); diff --git a/scripts/qa-traffic-preset-ab.mjs b/scripts/qa-traffic-preset-ab.mjs index 44319af0d..d1b4950fd 100644 --- a/scripts/qa-traffic-preset-ab.mjs +++ b/scripts/qa-traffic-preset-ab.mjs @@ -145,15 +145,19 @@ async function main() { try { const res = await fetch(APP_URL); + // Only the status matters; an unread body can hold the socket open. + await res.body?.cancel(); if (!res.ok) throw new Error(`HTTP ${res.status}`); } catch (e) { console.error(`Dev server not reachable at ${APP_URL} (${e.message}).`); - process.exit(2); + process.exitCode = 2; + return; } const status = await fetch(`${APP_URL}/api/tomtom/status`).then((r) => r.json()).catch(() => null); if (!status?.hasKey) { console.error('Server has no TomTom key — the A/B needs live flow.'); - process.exit(2); + process.exitCode = 2; + return; } fs.mkdirSync(OUT_DIR, { recursive: true }); diff --git a/scripts/qa-traffic.mjs b/scripts/qa-traffic.mjs index 2a04d5595..424f3486e 100644 --- a/scripts/qa-traffic.mjs +++ b/scripts/qa-traffic.mjs @@ -113,17 +113,21 @@ async function main() { try { const res = await fetch(APP_URL); + // Only the status matters; an unread body can hold the socket open. + await res.body?.cancel(); if (!res.ok) throw new Error(`HTTP ${res.status}`); } catch (e) { console.error(`\x1b[31mDev server not reachable at ${APP_URL} (${e.message}).\x1b[0m`); - process.exit(2); + process.exitCode = 2; + return; } const statusBefore = FIXTURES ? {hasKey:true,dailyCount:0} : await fetch(`${APP_URL}/api/tomtom/status`).then((r) => r.json()).catch(() => null); if (FIXTURES) console.log('Source mode: synthetic roads and recorded flow tiles; live quota is not qualified.'); if (!statusBefore?.hasKey) { console.error('\x1b[31mServer has no TomTom key — run against the keyed dev server (:4410).\x1b[0m'); - process.exit(2); + process.exitCode = 2; + return; } fs.mkdirSync(SHOTS_DIR, { recursive: true }); diff --git a/scripts/qa-transit-heading.mjs b/scripts/qa-transit-heading.mjs index a16bab9bc..c229be96c 100644 --- a/scripts/qa-transit-heading.mjs +++ b/scripts/qa-transit-heading.mjs @@ -1,11 +1,11 @@ import puppeteer from 'puppeteer'; +import { chromeExecutablePath } from './qa-chrome.mjs'; /** Prove parked and moving headings in the real scene, without live feeds. */ export async function runTransitHeadingRegression(base, check) { const browser = await puppeteer.launch({ headless: true, - executablePath: - '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + executablePath: chromeExecutablePath(), args: ['--no-sandbox', '--disable-background-timer-throttling'], }); try { diff --git a/scripts/qa-transit-recovery.mjs b/scripts/qa-transit-recovery.mjs index e5fadda49..5f6404b41 100644 --- a/scripts/qa-transit-recovery.mjs +++ b/scripts/qa-transit-recovery.mjs @@ -1,5 +1,6 @@ /** Camera-jump regression against an existing server; never starts a server. */ import puppeteer from 'puppeteer'; +import { chromeExecutablePath } from './qa-chrome.mjs'; import { mkdir, writeFile } from 'node:fs/promises'; import { reduceTrailPixels } from '../src/layers/transit/qaMetrics.js'; @@ -8,8 +9,7 @@ const shots = process.env.QA_SHOTS || 'qa-shots/transit-recovery'; await mkdir(shots, { recursive: true }); const browser = await puppeteer.launch({ headless: false, - executablePath: - '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + executablePath: chromeExecutablePath(), args: [ '--disable-background-timer-throttling', '--disable-renderer-backgrounding', diff --git a/scripts/qa-transit.mjs b/scripts/qa-transit.mjs index aac1d8f60..7d83d77a4 100644 --- a/scripts/qa-transit.mjs +++ b/scripts/qa-transit.mjs @@ -36,6 +36,7 @@ import { */ import { mkdir, readFile, writeFile } from 'node:fs/promises'; import puppeteer from 'puppeteer'; +import { chromeExecutablePath } from './qa-chrome.mjs'; import { createTransitHistory } from '../server/providers/transitHistory.js'; import { DETECTION_THEME_MAP } from '../src/overlays/worldOverlayTokens.js'; import { transitModeTier } from '../src/data/transitPresetStyle.js'; @@ -262,8 +263,7 @@ await mkdir(SHOTS, { recursive: true }); const browser = await puppeteer.launch({ headless: false, protocolTimeout: 150000, - executablePath: - '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + executablePath: chromeExecutablePath(), args: [ '--no-sandbox', '--disable-background-timer-throttling', diff --git a/scripts/qa-voice-auth-focus.mjs b/scripts/qa-voice-auth-focus.mjs new file mode 100644 index 000000000..6d131a22c --- /dev/null +++ b/scripts/qa-voice-auth-focus.mjs @@ -0,0 +1,328 @@ +#!/usr/bin/env node +/** + * Check auth-mode keyboard focus with the real settings template and controls. + * Run: node --test --test-reporter=spec scripts/qa-voice-auth-focus.mjs + * Sign-in responses are fixtures. No credentials or live login are used. + */ +import assert from 'node:assert/strict'; +import { existsSync } from 'node:fs'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { after, before, test } from 'node:test'; +import puppeteer from 'puppeteer'; + +const root = new URL('../', import.meta.url); +const template = await readFile( + new URL('src/ui/templates/provider-settings.html', root), + 'utf8', +); +const assets = new Set([ + '/src/keySetup.js', + '/src/voice/cloudVoiceAuth.js', + '/src/ui/surfaceKeyboard.js', + '/src/ui/styles/provider-settings.css', +]); +const observations = []; +let server; +let browser; +let origin; + +before(async () => { + server = createServer(async (req, res) => { + const pathname = new URL(req.url, 'http://localhost').pathname; + if (pathname === '/') { + res.setHeader('Content-Type', 'text/html; charset=utf-8'); + res.end(` + Voice auth keyboard focus test + + ${template}`); + return; + } + if (!assets.has(pathname)) { + res.statusCode = 404; + res.end(); + return; + } + try { + res.setHeader( + 'Content-Type', + pathname.endsWith('.css') ? 'text/css' : 'text/javascript', + ); + res.end(await readFile(new URL(pathname.slice(1), root))); + } catch { + res.statusCode = 500; + res.end('Could not load the test asset'); + } + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + origin = `http://127.0.0.1:${server.address().port}`; + const chrome = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'; + browser = await puppeteer.launch({ + headless: true, + executablePath: + process.env.PUPPETEER_EXECUTABLE_PATH || + (existsSync(chrome) ? chrome : await puppeteer.executablePath()), + }); +}); + +after(async () => { + await browser?.close(); + if (server?.listening) await new Promise((resolve) => server.close(resolve)); + const output = new URL('.gev-logs/voice-auth-focus/', root); + await mkdir(output, { recursive: true }); + await writeFile( + new URL('results.json', output), + `${JSON.stringify(observations, null, 2)}\n`, + ); +}); + +const scenarios = [ + { + name: 'OAuth to API key', + initial: 'oauth', + expected: 'api-key', + flow: 'unused', + }, + { + name: 'API key to existing OAuth', + initial: 'api-key', + expected: 'oauth', + flow: 'existing', + }, + { + name: 'API key to OAuth ready at login launch', + initial: 'api-key', + expected: 'oauth', + flow: 'login-ready', + }, + { + name: 'API key to OAuth ready after polling', + initial: 'api-key', + expected: 'oauth', + flow: 'poll-ready', + }, +]; + +async function mountSettings(page, scenario) { + await page.goto(`${origin}/?setup=1`, { + waitUntil: 'domcontentloaded', + }); + await page.evaluate(async ({ initial, flow }) => { + localStorage.clear(); + localStorage.setItem('godsEyeView.voice.cloudAuth', initial); + window.__focusTestCalls = []; + let statusReads = 0; + const fetchImpl = async (url) => { + window.__focusTestCalls.push(url); + let payload; + let status = 200; + if (url === '/api/setup/status') { + payload = { + setCount: 0, + total: 1, + keys: [ + { + id: 'openai', + title: 'OPENAI', + tier: 'metered', + set: false, + unlocks: 'Voice control', + envVars: ['OPENAI_API_KEY'], + getUrl: 'https://example.invalid/', + }, + ], + }; + } else if (url === '/api/realtime/oauth-status') { + statusReads += 1; + if (flow === 'deferred') { + return await new Promise((resolve) => { + window.__resolveOAuthStatus = (available) => + resolve( + new Response( + JSON.stringify({ available, error: 'Fixture failure' }), + { + status: available ? 200 : 503, + }, + ), + ); + }); + } + payload = { + available: + flow === 'existing' || (flow === 'poll-ready' && statusReads > 1), + }; + } else if (url === '/api/realtime/oauth-login') { + const available = flow === 'login-ready'; + status = available ? 200 : 202; + payload = { available, pending: !available, started: !available }; + } else { + throw new Error(`Unexpected request: ${url}`); + } + return new Response(JSON.stringify(payload), { status }); + }; + const { initKeySetup } = await import('/src/keySetup.js'); + window.__focusTestSettings = await initKeySetup({ fetchImpl }); + }, scenario); + await page.waitForFunction( + () => + document.getElementById('key-setup').classList.contains('visible') && + document.activeElement.matches('input[data-env-var]'), + ); +} + +for (const scenario of scenarios) { + for (const key of ['Enter', 'Space', 'Mouse click']) { + test(`${scenario.name} keeps keyboard focus after ${key}`, async (t) => { + const page = await browser.newPage(); + try { + await mountSettings(page, scenario); + + if (key !== 'Mouse click') { + // Tab from the initial key field to the real auth button. + await page.keyboard.press('Tab'); + assert.equal( + await page.evaluate(() => + document.activeElement.matches('[data-cloud-voice-auth-toggle]'), + ), + true, + 'Keyboard navigation must reach the auth toggle before activation', + ); + } + await page.evaluate(() => { + const toggle = document.querySelector( + '[data-cloud-voice-auth-toggle]', + ); + window.__originalFocusToggle = toggle; + window.__focusAtActivation = null; + toggle.addEventListener( + 'click', + () => { + window.__focusAtActivation = document.activeElement === toggle; + }, + { capture: true, once: true }, + ); + }); + if (key === 'Mouse click') { + await page.waitForFunction( + () => + getComputedStyle(document.getElementById('key-setup')).opacity === + '1', + ); + await page.click('[data-cloud-voice-auth-toggle]'); + } else { + await page.keyboard.press(key); + } + await page.waitForFunction( + (expected) => + localStorage.getItem('godsEyeView.voice.cloudAuth') === expected && + !document.querySelector('[data-cloud-voice-auth-toggle]') + .disabled && + document + .querySelector('[data-cloud-voice-auth-toggle]') + .getAttribute('aria-disabled') !== 'true', + {}, + scenario.expected, + ); + const selection = await page.evaluate(() => ({ + focusOnToggleAtActivation: window.__focusAtActivation, + focusOnToggle: + document.activeElement === + document.querySelector('[data-cloud-voice-auth-toggle]'), + activeTag: document.activeElement.tagName, + originalToggleConnected: window.__originalFocusToggle.isConnected, + requests: window.__focusTestCalls, + })); + await page.keyboard.press('Tab'); + const nextTab = await page.evaluate(() => ({ + focusOnSave: document.activeElement.matches('[data-key-setup-apply]'), + activeLabel: + document.activeElement.getAttribute('aria-label') || + document.activeElement.textContent.trim(), + })); + const observed = { scenario: scenario.name, key, selection, nextTab }; + observations.push(observed); + t.diagnostic(JSON.stringify(observed)); + assert.equal( + selection.focusOnToggle, + true, + 'Auth selection must keep focus on its usable toggle', + ); + assert.equal( + nextTab.focusOnSave, + true, + 'Tab must continue to SAVE KEYS rather than restart at Close', + ); + } finally { + await page.close(); + } + }); + } +} + +for (const available of [true, false]) { + test(`pending OAuth ${available ? 'success' : 'failure'} blocks repeats without moving focus or losing a key draft`, async () => { + const page = await browser.newPage(); + try { + await mountSettings(page, { initial: 'api-key', flow: 'deferred' }); + await page.type('input[data-env-var]', 'test-draft-only'); + await page.keyboard.press('Tab'); + await page.keyboard.press('Enter'); + await page.waitForFunction( + () => typeof window.__resolveOAuthStatus === 'function', + ); + assert.equal( + await page.evaluate(() => + document.activeElement.matches('[data-cloud-voice-auth-toggle]'), + ), + true, + ); + await page.keyboard.press('Enter'); + await page.keyboard.press('Space'); + assert.equal( + await page.evaluate( + () => + window.__focusTestCalls.filter( + (url) => url === '/api/realtime/oauth-status', + ).length, + ), + 1, + ); + await page.keyboard.press('Tab'); + assert.equal( + await page.evaluate(() => + document.activeElement.matches('[data-key-setup-apply]'), + ), + true, + ); + await page.evaluate( + (available) => window.__resolveOAuthStatus(available), + available, + ); + await page.waitForFunction( + () => + document + .querySelector('[data-cloud-voice-auth-toggle]') + .getAttribute('aria-disabled') === 'false', + ); + const observed = await page.evaluate(() => ({ + mode: localStorage.getItem('godsEyeView.voice.cloudAuth'), + focusOnSave: document.activeElement.matches('[data-key-setup-apply]'), + draft: document.querySelector('input[data-env-var]').value, + requests: window.__focusTestCalls, + })); + observations.push({ scenario: 'pending OAuth', available, observed }); + assert.equal(observed.mode, available ? 'oauth' : 'api-key'); + assert.equal( + observed.focusOnSave, + true, + 'Completing a request must not take focus from another control', + ); + assert.equal(observed.draft, 'test-draft-only'); + } finally { + await page.close(); + } + }); +} diff --git a/scripts/qa-voice-auth.mjs b/scripts/qa-voice-auth.mjs new file mode 100644 index 000000000..800df9598 --- /dev/null +++ b/scripts/qa-voice-auth.mjs @@ -0,0 +1,291 @@ +#!/usr/bin/env node +/** + * Live Realtime auth acceptance with a prerecorded Chromium microphone. + * Run against the local dev server after configuring the selected credential: + * node scripts/qa-voice-auth.mjs http://127.0.0.1:4189 oauth microphone.wav + * Use a short spoken reply request with three seconds of leading silence. + * Saves only usage, transport metadata, and screenshots; never credentials. + */ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import puppeteer from 'puppeteer'; + +const root = fileURLToPath(new URL('../', import.meta.url)); +const url = new URL(process.argv[2] || 'http://127.0.0.1:4189'); +const auth = process.argv[3] || 'oauth'; +assert.ok(['oauth', 'api-key'].includes(auth), 'Choose oauth or api-key'); +assert.ok( + ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname), + 'Use a local server', +); +assert.ok(process.argv[4], 'Provide a prerecorded microphone WAV'); +const wav = path.resolve(process.argv[4]); +assert.ok(fs.existsSync(wav), 'Microphone WAV must exist'); +url.searchParams.set('welcome', '0'); +url.searchParams.set('setup', '1'); +const output = path.join(root, '.gev-logs', 'voice-auth'); +fs.mkdirSync(output, { recursive: true }); +const executablePath = + process.env.PUPPETEER_EXECUTABLE_PATH || + (fs.existsSync('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome') + ? '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' + : puppeteer.executablePath()); +const browser = await puppeteer.launch({ + headless: true, + executablePath, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--use-fake-device-for-media-stream', + `--use-file-for-fake-audio-capture=${wav}%noloop`, + '--autoplay-policy=no-user-gesture-required', + '--disable-background-timer-throttling', + '--window-size=1440,900', + ], +}); +const result = { + ok: false, + auth, + input: 'prerecorded microphone', + mints: [], + negotiations: [], +}; +let page; +const readSession = () => + page.evaluate(async () => { + const voice = window.__godsEyeView.voiceCommands; + const stats = voice.pc ? [...(await voice.pc.getStats()).values()] : []; + const input = stats.find( + (entry) => entry.type === 'outbound-rtp' && entry.kind === 'audio', + ); + const source = stats.find( + (entry) => entry.type === 'media-source' && entry.kind === 'audio', + ); + const audio = stats.find( + (entry) => entry.type === 'inbound-rtp' && entry.kind === 'audio', + ); + const meter = document.getElementById('gev-voice-cost-value'); + return { + ...window.__voiceAuthEvidence, + connection: voice.pc?.connectionState, + microphoneEnabled: voice.stream + ?.getAudioTracks() + .every((track) => track.enabled), + inputBytes: input?.bytesSent || 0, + inputAudioEnergy: source?.totalAudioEnergy ?? null, + outputBytes: audio?.bytesReceived || 0, + outputAudioEnergy: audio?.totalAudioEnergy || 0, + playbackTime: voice.audioEl?.currentTime || 0, + meter: { + text: meter?.textContent, + usage: meter?.title, + visible: Boolean(meter?.getClientRects().length), + }, + }; + }); +try { + await browser + .defaultBrowserContext() + .overridePermissions(url.origin, ['microphone']); + page = await browser.newPage(); + await page.setViewport({ width: 1440, height: 900 }); + page.on('response', (response) => { + const route = new URL(response.url()).pathname; + if (route === '/api/realtime/token') + result.mints.push({ + status: response.status(), + auth: response.headers()['x-gev-voice-auth'], + model: response.headers()['x-gev-voice-model'], + }); + if ( + route === '/v1/realtime/calls' && + response.request().method() === 'POST' + ) + result.negotiations.push({ status: response.status() }); + }); + // Observe the application's real transport. Do not replace fetch or replies. + await page.evaluateOnNewDocument(() => { + window.__voiceAuthEvidence = { + speechStopped: 0, + transcripts: [], + replies: [], + responses: [], + errors: [], + eventCounts: {}, + }; + const create = RTCPeerConnection.prototype.createDataChannel; + RTCPeerConnection.prototype.createDataChannel = function (...args) { + const channel = create.apply(this, args); + channel.addEventListener('message', ({ data }) => { + let event; + try { + event = JSON.parse(data); + } catch { + return; + } + const evidence = window.__voiceAuthEvidence; + evidence.eventCounts[event.type] = + (evidence.eventCounts[event.type] || 0) + 1; + if (event.type === 'input_audio_buffer.speech_stopped') + evidence.speechStopped += 1; + if ( + event.type === 'conversation.item.input_audio_transcription.completed' + ) + evidence.transcripts.push(event.transcript); + if (event.type === 'response.output_audio_transcript.done') + evidence.replies.push(event.transcript); + if (event.type === 'response.done') + evidence.responses.push({ + status: event.response?.status, + inputTokens: event.response?.usage?.input_tokens, + outputTokens: event.response?.usage?.output_tokens, + }); + if (event.type === 'error') + evidence.errors.push({ code: event.error?.code }); + }); + return channel; + }; + }); + await page.goto(url.href, { waitUntil: 'domcontentloaded', timeout: 30_000 }); + await page.waitForFunction( + () => + window.__godsEyeView?.voiceCommands && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 45_000 }, + ); + await page.waitForSelector('[data-cloud-voice-auth-toggle]', { + visible: true, + }); + await page.waitForFunction( + () => + getComputedStyle(document.getElementById('key-setup')).opacity === '1', + ); + const selected = await page.evaluate( + () => localStorage.getItem('godsEyeView.voice.cloudAuth') || 'api-key', + ); + if (selected !== auth) { + await page.click('[data-cloud-voice-auth-toggle]'); + await page.waitForFunction( + (mode) => localStorage.getItem('godsEyeView.voice.cloudAuth') === mode, + { timeout: 125_000 }, + auth, + ); + } + await page.click('[data-key-setup-close]'); + await page.click('#gev-voice-button'); + await page.waitForFunction( + () => + window.__godsEyeView.voiceCommands.dc?.readyState === 'open' || + window.__godsEyeView.voiceCommands.status === 'error', + { timeout: 35_000 }, + ); + assert.equal( + await page.evaluate( + () => window.__godsEyeView.voiceCommands.dc?.readyState, + ), + 'open', + 'Realtime channel must open', + ); + await page.waitForFunction( + () => + window.__voiceAuthEvidence.speechStopped > 0 && + window.__voiceAuthEvidence.responses.some( + (response) => + response.status === 'completed' && response.outputTokens > 0, + ), + { timeout: 45_000 }, + ); + await page.waitForFunction( + () => + window.__voiceAuthEvidence.eventCounts['output_audio_buffer.stopped'] > 0, + { timeout: 30_000 }, + ); + result.session = await readSession(); + assert.ok( + result.session.inputBytes > 0 && result.session.outputBytes > 0, + 'Audio must flow in both directions', + ); + assert.ok( + result.session.outputAudioEnergy > 0 && result.session.playbackTime > 0, + 'Reply audio must play', + ); + assert.equal( + result.session.meter.text === 'COST UNKNOWN', + auth === 'oauth', + 'Use the correct accounting display', + ); + await page.screenshot({ path: path.join(output, `${auth}-usage.png`) }); + await page.click('#key-setup-chip'); + await page.waitForSelector('[data-cloud-voice-auth-toggle]', { + visible: true, + }); + await page.waitForFunction( + () => + getComputedStyle(document.getElementById('key-setup')).opacity === '1', + ); + await page.screenshot({ path: path.join(output, `${auth}-settings.png`) }); + await page.click('[data-key-setup-close]'); + result.stopped = await page.evaluate(() => { + const voice = window.__godsEyeView.voiceCommands; + const pc = voice.pc; + const tracks = voice.stream?.getTracks() || []; + voice.session.stop(); + return ( + pc.connectionState === 'closed' && + tracks.every((track) => track.readyState === 'ended') + ); + }); + assert.ok(result.stopped, 'Stop must close the peer and microphone tracks'); + await page.click('#gev-voice-button'); + await page.waitForFunction( + () => + window.__godsEyeView.voiceCommands.dc?.readyState === 'open' || + window.__godsEyeView.voiceCommands.status === 'error', + { timeout: 35_000 }, + ); + assert.equal( + await page.evaluate( + () => window.__godsEyeView.voiceCommands.dc?.readyState, + ), + 'open', + 'Reconnect must work', + ); + assert.equal( + result.mints.length, + 2, + 'Reconnect must request a fresh client secret', + ); + assert.ok( + result.mints.every( + (mint) => + mint.status === 200 && + mint.auth === (auth === 'oauth' ? 'codex-oauth' : 'env'), + ), + ); + assert.ok(result.negotiations.every((entry) => entry.status === 201)); + result.ok = true; +} catch (error) { + result.error = error.message; + if (page) { + result.session = await readSession().catch(() => null); + result.detail = await page + .$eval('#gev-voice-detail', (element) => element.textContent) + .catch(() => null); + await page + .screenshot({ path: path.join(output, `${auth}-failure.png`) }) + .catch(() => {}); + } +} finally { + await page + ?.evaluate(() => window.__godsEyeView?.voiceCommands?.session?.stop()) + .catch(() => {}); + await browser.close(); + fs.writeFileSync( + path.join(output, `${auth}-result.json`), + `${JSON.stringify(result, null, 2)}\n`, + ); +} +console.log(JSON.stringify(result, null, 2)); +if (!result.ok) process.exitCode = 1; diff --git a/scripts/run-unit-tests.mjs b/scripts/run-unit-tests.mjs index 9bd7e70e0..6cc6aaa94 100644 --- a/scripts/run-unit-tests.mjs +++ b/scripts/run-unit-tests.mjs @@ -21,9 +21,20 @@ export function assertNode24AllocationRuntime(version = process.versions.node) { return version; } +/** + * Directories this runner searches for `*.test.mjs`, in stable order. + * + * `server/` belongs here because provider modules own tests of their own: a + * suite placed beside the module it covers must run, and CI's unit-test job runs + * `npm test`, so a directory missing from this list is a suite that never runs + * there. `unitTestRunner.test.mjs` asserts the list covers every `*.test.mjs` + * outside dependencies, build output and dot-directories, so adding a tree + * without adding it here fails. + */ +export const UNIT_TEST_ROOTS = Object.freeze(['src', 'server']); + /** Discover repository unit tests in stable path order. */ export function discoverUnitTestFiles(root = process.cwd()) { - const sourceRoot = path.join(root, 'src'); const files = []; const visit = (directory) => { for (const entry of readdirSync(directory, { withFileTypes: true })) { @@ -34,7 +45,7 @@ export function discoverUnitTestFiles(root = process.cwd()) { } } }; - visit(sourceRoot); + for (const testRoot of UNIT_TEST_ROOTS) visit(path.join(root, testRoot)); return files.sort(); } diff --git a/scripts/setup-doctor.mjs b/scripts/setup-doctor.mjs index 162cb173c..49c6722c2 100644 --- a/scripts/setup-doctor.mjs +++ b/scripts/setup-doctor.mjs @@ -176,7 +176,9 @@ export function buildCapabilitySummary( ? 'Google Photorealistic 3D Tiles through Cesium ion; Bing and world-terrain stacks available' : 'Esri World Imagery (keyless satellite basemap) with keyless terrain', flights, - voice: configured('OPENAI_API_KEY') ? 'available' : 'off until an OpenAI key is added', + voice: configured('OPENAI_API_KEY') + ? 'available' + : 'off until an OpenAI key is added, or a Codex ChatGPT sign-in is selected in Provider Settings (experimental)', vessels: configured('AISSTREAM_API_KEY') ? 'live AISStream feed' : 'off until an AISStream key is added', fires: configured('FIRMS_MAP_KEY') ? 'live NASA FIRMS feed' : 'off until a FIRMS key is added', traffic: configured('TOMTOM_API_KEY') ? 'live TomTom flow' : 'built-in traffic simulation', diff --git a/scripts/track-regression.mjs b/scripts/track-regression.mjs index 84aa2494e..756daccfc 100644 --- a/scripts/track-regression.mjs +++ b/scripts/track-regression.mjs @@ -443,7 +443,10 @@ async function main() { // for nearby names. Answer both with empty results: the entity-context // assertions are about the SELECTED contact, and a live lookup would // make the run non-hermetic (and bill the owner's Google quota). - if (url.hostname === 'maps.googleapis.com' && url.pathname.startsWith('/maps/api/geocode')) { + if ( + isAppRequest && + (url.pathname === '/api/google/reverse-geocode' || url.pathname === '/api/google/geocode') + ) { return Promise.resolve(jsonResponse({ status: 'ZERO_RESULTS', results: [] })); } if (isAppRequest && url.pathname === '/api/google/nearby-places') { diff --git a/server/providers/aircraft/adsb-lol.js b/server/providers/aircraft/adsb-lol.js index f67afed1d..8fc40170d 100644 --- a/server/providers/aircraft/adsb-lol.js +++ b/server/providers/aircraft/adsb-lol.js @@ -1,3 +1,5 @@ +import { readResponseTextCapped } from '../common/http.js'; + /** * Vite plugin: adsb.lol military aircraft proxy with 12 s response cache. * @@ -25,6 +27,10 @@ export function adsbLolProxy() { let _cooldownStatus = 0; /** Response cache TTL (ms). */ const CACHE_MS = 12000; + /** Upstream deadline (ms); the fetch previously had none and could hang a poll. */ + const UPSTREAM_TIMEOUT_MS = 12000; + /** /v2/mil measures ~67 KB; a relayed error page is bounded by the same cap. */ + const ADSBLOL_MIL_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; /** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */ const RATE_LIMIT_COOLDOWN_MS = 30000; /** Cooldown after a 5xx (ms). */ @@ -98,11 +104,17 @@ export function adsbLolProxy() { ); return; } + const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS); const upstream = await fetch('https://api.adsb.lol/v2/mil', { headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' }, + signal, }); if (upstream.ok) { - const body = await upstream.text(); + const body = await readResponseTextCapped( + upstream, + ADSBLOL_MIL_MAX_RESPONSE_BYTES, + signal, + ); _cache = body; _cacheAt = Date.now(); _cooldownUntil = 0; @@ -129,7 +141,11 @@ export function adsbLolProxy() { return; } } - const body = await upstream.text(); + const body = await readResponseTextCapped( + upstream, + ADSBLOL_MIL_MAX_RESPONSE_BYTES, + signal, + ); serve( res, upstream.status, diff --git a/server/providers/aircraft/enrichment.js b/server/providers/aircraft/enrichment.js index 400050151..9b7c1b1bb 100644 --- a/server/providers/aircraft/enrichment.js +++ b/server/providers/aircraft/enrichment.js @@ -1,5 +1,9 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; +import { readResponseJsonCapped } from '../common/http.js'; + +/** Route and aircraft lookups are a few hundred bytes; 1 MiB is a hard bound. */ +const ADSBDB_MAX_RESPONSE_BYTES = 1024 * 1024; /** * adsbdb.com enrichment proxy: callsign → route (airline + origin/destination * airports) and hex → aircraft type/registration. Free community API — cached @@ -80,12 +84,16 @@ export function adsbdbProxy() { kind === 'route' ? `https://api.adsbdb.com/v0/callsign/${encodeURIComponent(key)}` : `https://api.adsbdb.com/v0/aircraft/${encodeURIComponent(key)}`; - const res = await fetch(url, { signal: AbortSignal.timeout(8000) }); + const signal = AbortSignal.timeout(8000); + const res = await fetch(url, { signal }); if (res.ok) { + const payload = await readResponseJsonCapped( + res, + ADSBDB_MAX_RESPONSE_BYTES, + signal, + ); const data = - kind === 'route' - ? parseRoute(await res.json()) - : parseAircraft(await res.json()); + kind === 'route' ? parseRoute(payload) : parseAircraft(payload); store[key] = { at: Date.now(), data }; // data may be null — negative cache dirty = true; return data; diff --git a/server/providers/aircraft/opensky.js b/server/providers/aircraft/opensky.js index 187049aea..752b3cf64 100644 --- a/server/providers/aircraft/opensky.js +++ b/server/providers/aircraft/opensky.js @@ -2,6 +2,7 @@ import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback import { coalesceProxyRequest, readResponseJsonCapped, + readResponseTextCapped, } from '../common/http.js'; import { requiredFiniteQueryNumber } from '../common/query.js'; // --------------------------------------------------------------------------- @@ -29,11 +30,34 @@ const OPENSKY_CACHE_MS = 9000; const OPENSKY_ATTEMPT_TIMEOUT_MS = 10_000; const OPENSKY_STATES_URL = 'https://opensky-network.org/api/states/all?extended=1'; +/** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */ +const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; + +/** + * Credit-governor cooldown for a 429: OpenSky's retry-after bounded to + * 30 s … 30 min, or 2 min when the header is absent. + * @param {Response} upstream Rate-limited response. + * @returns {number} Cooldown (ms). + */ +function openSkyRateLimitCooldownMs(upstream) { + // headers.get() gives null when absent, and Number(null) is 0, not NaN. + const header = upstream.headers.get('x-rate-limit-retry-after-seconds'); + const retryAfterSec = + header == null || header.trim() === '' ? NaN : Number(header); + return Math.min( + Math.max( + Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000, + 30_000, + ), + 30 * 60_000, + ); +} /** * Fetch and read the global snapshot, giving each attempt * OPENSKY_ATTEMPT_TIMEOUT_MS and retrying once when an attempt times out or - * the connection fails. A second failure throws, which the proxy answers + * the connection fails. An oversized body is not retried: it would fail again + * and spend more credits. A second failure throws, which the proxy answers * from its stale cache or the regional fallback. * @param {Record} headers Request headers. * @returns {Promise<{upstream: Response, body: string}>} @@ -41,13 +65,27 @@ const OPENSKY_STATES_URL = async function fetchOpenSkyStates(headers) { for (let attempt = 1; ; attempt++) { try { - const upstream = await fetch(OPENSKY_STATES_URL, { - headers, - signal: AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS), - }); - return { upstream, body: await upstream.text() }; + const signal = AbortSignal.timeout(OPENSKY_ATTEMPT_TIMEOUT_MS); + const upstream = await fetch(OPENSKY_STATES_URL, { headers, signal }); + if (upstream.status === 429) { + // Start the cooldown from the headers and drop the rejected body + // unread, so a stalled or oversized 429 body cannot skip it. + _openskyCooldownUntil = + Date.now() + openSkyRateLimitCooldownMs(upstream); + void upstream.body?.cancel().catch(() => {}); + return { + upstream, + body: JSON.stringify({ error: 'OpenSky rate limited.' }), + }; + } + const body = await readResponseTextCapped( + upstream, + OPENSKY_STATES_MAX_RESPONSE_BYTES, + signal, + ); + return { upstream, body }; } catch (error) { - if (attempt >= 2) throw error; + if (attempt >= 2 || error?.code === 'RESPONSE_TOO_LARGE') throw error; console.warn( '[OpenSky] global snapshot attempt failed, retrying:', error?.message || error, @@ -544,19 +582,9 @@ export function openSkyProxy() { } if (upstream.status === 429) { reason = 'rate_limited'; - // Credit governor: honor OpenSky's retry-after (bounded 30 s … 30 min; - // 2 min when the header is absent) — no upstream attempts until then. - const retryAfterSec = Number( - upstream.headers.get('x-rate-limit-retry-after-seconds'), - ); - const cooldownMs = Math.min( - Math.max( - Number.isFinite(retryAfterSec) ? retryAfterSec * 1000 : 120_000, - 30_000, - ), - 30 * 60_000, - ); - _openskyCooldownUntil = now + cooldownMs; + // Credit governor: fetchOpenSkyStates already started the cooldown + // from OpenSky's retry-after — no upstream attempts until then. + const cooldownMs = openSkyRateLimitCooldownMs(upstream); // Serve the last-good body instead of the 429 when we have one — // the layer keeps rendering (STALE-cued) instead of dying. if (_openskyCacheBody && _openskyCacheStatus === 200) { diff --git a/server/providers/cctv.js b/server/providers/cctv.js index 9cf3b0c47..b24f38fa1 100644 --- a/server/providers/cctv.js +++ b/server/providers/cctv.js @@ -28,7 +28,7 @@ export { CCTV_FRAME_FETCH_TIMEOUT_MS, fetchCctvImageFromUpstream }; * GET /api/cctv/sources — list all registered camera sources * GET /api/cctv/health — per-camera health/status report * GET /api/cctv/stream/:id — stream info (feedType, URLs) for a camera - * GET /api/cctv/media/:id — proxy live video/image media from upstream + * GET /api/cctv/media/:id — proxy live video media from upstream * GET /api/cctv/frame/:id — single frame with fallback chain * * @returns {import('vite').Plugin} @@ -164,6 +164,8 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { sourceKind: source.sourceKind || (source.url ? 'configured' : 'fallback'), poseSource: source.poseSource, + model: source.model || '', + manufacturer: source.manufacturer || '', license: source.license, credit: source.credit || '', code: source.code || '', @@ -214,6 +216,21 @@ export function cctvProxy({ sourceRoot = process.cwd() } = {}) { const source = sourceById.get(cameraId); const mediaUrl = source?.url || ''; const feedType = normalizeFeedType(source?.feedType || 'image'); + // Live feeds only. A still camera is served by /frame, whose fetch + // keeps redirects on the registered origin, accepts only image/* and + // caps the body; relaying it here would bypass all three. + if (!isVideoFeedType(feedType) && feedType !== 'mjpeg') { + res.writeHead(404, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + }); + res.end( + JSON.stringify({ + error: 'No live media for this camera; use the frame endpoint', + }), + ); + return; + } const leaseId = url.searchParams.get('lease'); if (feedType === 'hls' && !/^[a-f0-9-]{36}$/i.test(leaseId || '')) { res.writeHead(400); diff --git a/server/providers/cctv/catalog.js b/server/providers/cctv/catalog.js index 9bc3b70ec..75e58d2a6 100644 --- a/server/providers/cctv/catalog.js +++ b/server/providers/cctv/catalog.js @@ -4,6 +4,7 @@ import { DEFAULT_CCTV_SOURCE_FILE, CCTV_SOURCE_CACHE_MS } from './constants.js'; import { allocateSourceCap, resolveCatalogCap } from './cap.js'; import { loadGroundHeights, joinGroundHeights } from './groundHeights.js'; import { cameraRegion, normalizeSourceItem } from './normalize.js'; +import { loadLithuaniaSources } from './lithuania.js'; import { loadAustinSourcesFromOpenData, loadCaltransSourcesFromOpenData, @@ -17,19 +18,30 @@ import { loadWarendorfSourcesFromCatalog, loadNswSourcesFromOpenData, loadCalgarySourcesFromOpenData, + loadCataloniaSourcesFromOpenData, loadDelDOTSourcesFromOpenData, + loadKingCountySourcesFromOpenData, + loadWsdotSourcesFromOpenData, loadVegvesenSourcesFromOpenData, + loadNycSourcesFromOpenData, + loadTorontoSourcesFromOpenData, } from './sources.js'; /** Env kill switch: unset or anything but "0" means enabled. */ const envEnabled = (name) => String(process.env[name] || '1').trim() !== '0'; +/** Env opt-in: only "1" enables. For packs whose feed access needs an + * operator agreement or whose terms are unconfirmed, so an unset flag never + * reaches the upstream. */ +const envOptIn = (name) => String(process.env[name] || '').trim() === '1'; /** * Live open-data packs, in merge order. Adding a region is one entry here - * plus its loader in sources.js; the catalog cap is shared across entries - * round-robin (cap.js), so a new pack never silently evicts an older one. + * plus its loader (in sources.js or its own module); the catalog cap is + * shared across entries round-robin (cap.js), so a new pack never silently + * evicts an older one. * Each pack fails independently (allSettled) and is gated by its own env - * kill switch. + * kill switch, or by an opt-in flag where access needs an agreement or the + * terms are unconfirmed. */ const LIVE_PACKS = [ { name: 'austin', enabled: () => true, load: loadAustinSourcesFromOpenData }, @@ -88,16 +100,50 @@ const LIVE_PACKS = [ enabled: () => envEnabled('CCTV_CALGARY_ENABLED'), load: loadCalgarySourcesFromOpenData, }, + { + name: 'catalonia', + enabled: () => envEnabled('CCTV_CATALONIA_ENABLED'), + load: loadCataloniaSourcesFromOpenData, + }, { name: 'deldot', enabled: () => envEnabled('CCTV_DELDOT_ENABLED'), load: loadDelDOTSourcesFromOpenData, }, + { + name: 'kingcounty', + enabled: () => envEnabled('CCTV_KINGCOUNTY_ENABLED'), + load: loadKingCountySourcesFromOpenData, + }, + { + name: 'wsdot', + enabled: () => envEnabled('CCTV_WSDOT_ENABLED'), + load: loadWsdotSourcesFromOpenData, + }, { name: 'vegvesen', enabled: () => envEnabled('CCTV_VEGVESEN_ENABLED'), load: loadVegvesenSourcesFromOpenData, }, + { + // NYC DOT requires a data-sharing agreement for this feed, so the pack is + // off unless the operator opts in (see DATA_SOURCES.md). + name: 'nyc', + enabled: () => envOptIn('CCTV_NYC_ENABLED'), + load: loadNycSourcesFromOpenData, + }, + { + name: 'toronto', + enabled: () => envEnabled('CCTV_TORONTO_ENABLED'), + load: loadTorontoSourcesFromOpenData, + }, + { + // Opt-in until the Via Lietuva open-data terms are confirmed to cover the + // camera photos (see DATA_SOURCES.md). + name: 'lithuania', + enabled: () => envOptIn('CCTV_LITHUANIA_ENABLED'), + load: loadLithuaniaSources, + }, ]; /** * Load CCTV sources from a local JSON file (CCTV_SOURCES_FILE env or default). diff --git a/server/providers/cctv/constants.js b/server/providers/cctv/constants.js index 6298e86a0..20b963770 100644 --- a/server/providers/cctv/constants.js +++ b/server/providers/cctv/constants.js @@ -8,8 +8,8 @@ export const DEFAULT_AUSTIN_MAX_SOURCES = 250; * Catalog-wide safety ceiling on served cameras. Each pack already caps * itself (nearest-to-anchor first); this bound only matters when the packs * together exceed it, and it is then filled round-robin across packs (see - * cap.js) so no region is silently dropped. Sized above the sum of the - * default per-pack caps so a default install never trims. + * cap.js) so no region is silently dropped. The default per-pack caps + * together can exceed it, so a default install may be thinned evenly here. */ export const DEFAULT_CCTV_MAX_SOURCES = 4000; /** Hard upper bound for CCTV_MAX_SOURCES; also sizes the health map. */ @@ -236,6 +236,56 @@ export const CALGARY_DOWNTOWN = { lat: 51.0461, lon: -114.0626 }; * body cannot be buffered without limit. */ export const CALGARY_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +/** + * Catalonia traffic cameras (Servei Català de Trànsit): one keyless WFS/GML + * XML list (verified 2026-09-15, hourly catalog refresh), fetched over https + * (verified 2026-09-27: same 200 body, no redirect). The feed carries a + * `font` field naming the actual publisher per camera — SCT's own highway + * cameras plus hotlinked Barcelona (IMI) and Terrassa municipal cameras are + * kept, each confirmed to return a real frame. Frame URLs are checked + * against a host allowlist rather than one pinned origin, since each + * publisher serves from its own host. + */ +export const CATALONIA_CAMERAS_URL = + 'https://www.gencat.cat/transit/opendata/cameres.xml'; +/** Byte ceiling for the Catalonia catalog read (~90 KB observed 2026-09-27), + * so a runaway body cannot be buffered without limit. */ +export const CATALONIA_MAX_CATALOG_BYTES = 1024 * 1024; +/** Allowed image hosts, one per kept `font` publisher in the feed, each + * mapped to the one directory its frame links live in (every link in the + * 2026-10 feed shares it); a link must name a single file directly inside. */ +export const CATALONIA_IMAGE_HOSTS = Object.freeze( + new Map([ + ['mct.gencat.cat', '/mct2bo/'], // SCT + ['www.bcn.cat', '/transit/imatges/'], // IMI (Ajuntament de Barcelona) + ['emap.terrassa.cat', '/it_terrassa/'], // Ajuntament de Terrassa + ]), +); +/** SCT `sctidcam` values: a bare image filename, which also keys the camera + * id (the feed's `fid` embeds an upstream server-start stamp). */ +export const CATALONIA_SCT_CAMERA_FILE_RE = /^[a-z0-9]+\.(gif|jpe?g)$/i; +/** Per-`font` partner credit, shown beside the provider in the CCTV panel + * (same pattern as DriveBC's partner-supplied cameras). SCT is the feed's + * own publisher and carries no separate credit. */ +export const CATALONIA_FONT_CREDIT = Object.freeze({ + IMI: 'Ajuntament de Barcelona', + Terrassa: 'Ajuntament de Terrassa', +}); +/** Ground-elevation priors in metres, by `font`: the feed carries no + * elevation, and this pack spans sea level (Barcelona) to Terrassa's plateau + * (~270 m). The client's ground snap corrects these on 3D-tile stacks. */ +export const CATALONIA_FONT_ELEVATION_M = Object.freeze({ + SCT: 80, + IMI: 20, + Terrassa: 270, +}); +export const DEFAULT_CATALONIA_MAX_SOURCES = 150; +/** Prioritization anchors: Barcelona and Terrassa. */ +export const CATALONIA_ANCHORS = [ + { lat: 41.3874, lon: 2.1686 }, // Barcelona + { lat: 41.5636, lon: 2.0111 }, // Terrassa +]; + /** Statens vegvesen (Norway) road cameras: the keyless OGC API Features view * of the official DATEX 3.1 CCTV site table, one GeoJSON request for the whole * country (~900 cameras). The raw DATEX node needs an account; this view does @@ -249,9 +299,9 @@ export const VEGVESEN_IMAGE_ORIGIN = * per-camera manifest path is registered. */ export const VEGVESEN_VIDEO_URL = (cameraId) => `https://kamera.vegvesen.no/public/${cameraId}/manifest.m3u8`; -/** Above the ~850 working cameras so a default install keeps them all, while - * staying close to them: the per-pack defaults are meant to sum to no more - * than DEFAULT_CCTV_MAX_SOURCES. */ +/** Above the ~850 working cameras so the pack keeps them all on its own. The + * catalog-wide DEFAULT_CCTV_MAX_SOURCES cap still applies across packs and is + * filled round-robin, so a full default install may thin this pack. */ export const DEFAULT_VEGVESEN_MAX_SOURCES = 900; /** Hard ceiling on the catalog body. The whole country is ~800 KB. */ export const VEGVESEN_MAX_CATALOG_BYTES = 8 * 1024 * 1024; @@ -267,6 +317,36 @@ export const NORWAY_ANCHORS = [ { lat: 69.6492, lon: 18.9553 }, // Tromsø ]; +/** NYC DOT Traffic Management Center cameras: one keyless JSON list for the + * five boroughs (~980 cameras). Frames are stills synthesized from each + * camera UUID on the city's webcam host — never taken from the payload. */ +export const NYC_CAMERAS_URL = 'https://webcams.nyctmc.org/api/cameras'; +/** The only origin NYC camera frames may come from. The catalog's `imageUrl` + * is ignored; the registered path is always this origin plus a validated + * UUID plus `/image`. */ +export const NYC_IMAGE_ORIGIN = 'https://webcams.nyctmc.org/api/cameras/'; +export const NYC_IMAGE_URL = (id) => `${NYC_IMAGE_ORIGIN}${id}/image`; +export const NYC_CAMERA_ID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +export const DEFAULT_NYC_MAX_SOURCES = 250; +/** Hard ceiling on the NYC catalog body. The live list is ~980 rows / ~250 KB; + * this only exists so an upstream that streams an unbounded body cannot be + * buffered without limit. */ +export const NYC_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +/** Ground-elevation prior in metres. Matches the New York CITY_POIS prior; + * the client's one-shot ground snap corrects it wherever 3D tiles load. */ +export const NYC_GROUND_ELEVATION_M = 10; +/** Prioritization anchors: one per borough so a cap keeps citywide coverage + * rather than Midtown alone. */ +export const NYC_ANCHORS = [ + { lat: 40.758, lon: -73.9855 }, // Manhattan (Times Square) + { lat: 40.7128, lon: -74.006 }, // Lower Manhattan + { lat: 40.6782, lon: -73.9442 }, // Brooklyn + { lat: 40.7282, lon: -73.7949 }, // Queens + { lat: 40.8448, lon: -73.8648 }, // Bronx + { lat: 40.5795, lon: -74.1502 }, // Staten Island +]; + /** DelDOT CCTV: one keyless statewide JSON catalog; live video via RTMP-over-HTTP (rtmpt:80). */ export const DELDOT_CCTV_URL = 'https://tmc.deldot.gov/json/videocamera.json'; export const DEFAULT_DELDOT_MAX_SOURCES = 300; @@ -275,6 +355,83 @@ export const DELDOT_ANCHORS = [ { lat: 39.1582, lon: -75.5244 }, // Dover (Kent) { lat: 38.6906, lon: -75.3877 }, // Georgetown (Sussex) ]; + +/** King County, WA road cameras: keyless public ArcGIS FeatureServer layer + * ("King County and WSDOT Traffic Cameras", King County DOT Road Services). + * GeoJSON with WGS84 point geometry. `outFields` is an explicit list on + * purpose: the layer also carries asset-administration columns (including + * staff names in `ModifiedBy`) that must never be fetched. Uniquely among the + * packs, the county publishes each camera's hardware `Manufacturer`/`Model` + * (Cohu 3950/3965, Bosch AUTODOME, Axis Q6135-LE, …), carried through as the + * source's `model`. */ +export const KING_COUNTY_CAMERAS_URL = + 'https://services.arcgis.com/Ej0PsM5Aw677QF1W/arcgis/rest/services/TRAFFICCAMERA_POINT_2029/FeatureServer/0/query?where=1%3D1&outFields=AssetID,Location,ImageURL,Live,Model,Manufacturer,Owner,CurrentStatus&f=geojson'; +/** Frames come only from the county's own image handler host. */ +export const KING_COUNTY_IMAGE_ORIGIN = 'https://info.kingcounty.gov/'; +/** The whole catalog is ~125 rows (~59 live county cameras); no thinning by + * default. */ +export const DEFAULT_KING_COUNTY_MAX_SOURCES = 125; +/** Prioritization anchor: Renton, roughly central to the county road network. */ +export const KING_COUNTY_ANCHORS = [{ lat: 47.4829, lon: -122.2171 }]; +/** Hard ceiling on the King County catalog body. The whole layer is ~125 rows + * and well under 100 KB; this only exists so an upstream that streams an + * unbounded body cannot be buffered without limit. */ +export const KING_COUNTY_MAX_CATALOG_BYTES = 1024 * 1024; +/** + * City of Toronto RESCU traffic cameras: the municipal network the province's + * 511 pack does not cover — downtown arterials plus the Gardiner, DVP, Allen + * and Lake Shore. One keyless list for the whole city (~336 cameras), from the + * City's open data portal. It is fetched on every catalog refresh, but the + * upstream file is not updated often and may lag the live camera network. + * + * The list is served as JSONP (`jsonTMCEarthCamerasCallback({...})`); the + * portal publishes the same data as .xml and .csv beside it. + */ +export const TORONTO_RESCU_LIST_URL = + 'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/Data/tmcearthcameras.json'; +/** + * The only origin Toronto camera frames may come from. The list carries no URL + * field: frame URLs are BUILT from this origin and a strictly-digits camera + * number (`loc####.jpg`, the convention the dataset documents), so no upstream + * field can steer the frame proxy off-host. + */ +export const TORONTO_RESCU_IMAGE_ORIGIN = + 'https://opendata.toronto.ca/transportation/tmc/rescucameraimages/CameraImages/'; +/** The whole city fits under this; the cap exists to stay tunable, not to trim. */ +export const DEFAULT_TORONTO_MAX_SOURCES = 336; +/** Yonge / Queen: the prioritization anchor. */ +export const TORONTO_DOWNTOWN = { lat: 43.6532, lon: -79.3832 }; +/** The city climbs from 76 m at the lake to 209 m at Steeles, so no single + * number fits: this is the mean across the 336 camera sites. The ground-height + * sidecar has no Toronto entries yet, so this prior seeds placement; a camera + * that gains a sidecar entry uses it instead, and the client's ground snap + * corrects the prior wherever 3D tiles are loaded. */ +export const TORONTO_GROUND_ELEVATION_M = 116; +/** Hard ceiling on the RESCU list body. The whole city is ~336 rows and under + * 60 KB; this only exists so an upstream that streams an unbounded body cannot + * be buffered without limit. */ +export const TORONTO_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +/** WSDOT Travel Center cameras (Washington State): keyless Esri JSON on the + * official open-data host — ~1,700 point features in EPSG:3857 (Web Mercator), + * ETag/Last-Modified served. WSDOT's metadata marks the feed "low volume" use; + * the catalog is fetched once per CCTV_SOURCE_CACHE_MS refresh. */ +export const WSDOT_CAMERAS_URL = + 'https://data.wsdot.wa.gov/travelcenter/Cameras.json'; +/** Only frames on WSDOT's own image host are registered; the catalog also + * lists ~70 partner cameras (Oregon DOT, City of Seattle, lodges, airports) + * whose own terms apply. */ +export const WSDOT_IMAGE_ORIGIN = 'https://images.wsdot.wa.gov/'; +export const DEFAULT_WSDOT_MAX_SOURCES = 250; +/** Hard ceiling on the WSDOT catalog body. The live layer is ~400 KB; this + * only exists so an upstream that streams an unbounded body cannot be + * buffered without limit. */ +export const WSDOT_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +/** Prioritization anchors: downtown Seattle and Spokane. */ +export const WSDOT_ANCHORS = [ + { lat: 47.6062, lon: -122.3321 }, // Seattle + { lat: 47.6588, lon: -117.426 }, // Spokane +]; + /** Camera CATALOGS change rarely; 15 min keeps multi-megabyte upstream list refetches (Austin rows.json + 4 Caltrans districts + TfL + Ontario 511) infrequent. Frames are fetched per-request and are unaffected. */ export const CCTV_SOURCE_CACHE_MS = 15 * 60 * 1000; /** Per-provider catalog-fetch timeout. Bounds the worst-case refresh so one diff --git a/server/providers/cctv/lithuania.js b/server/providers/cctv/lithuania.js new file mode 100644 index 000000000..840e68c6c --- /dev/null +++ b/server/providers/cctv/lithuania.js @@ -0,0 +1,154 @@ +import { CCTV_SOURCE_FETCH_TIMEOUT_MS } from './constants.js'; +import { + cameraDisplayCode, + fallbackHeadingFromId, + prioritizeSources, +} from './normalize.js'; +import { readResponseJsonCapped } from '../common/http.js'; + +const BASE = 'https://eismoinfo.lt/eismoinfo-backend'; +export const LITHUANIA_CAMERAS_URL = `${BASE}/camera-info-table`; +// Without lks=true this endpoint returns [latitude, longitude]. The camera +// table's x/y fields use LKS-94 metres and must never be treated as degrees. +export const LITHUANIA_LOCATIONS_URL = `${BASE}/layer-static-features/VKR`; +export const LITHUANIA_MAX_CATALOG_BYTES = 4 * 1024 * 1024; +const FRAME_PATH = '/eismoinfo-backend/image-provider/camera/last'; +const ANCHORS = [ + { lat: 54.6872, lon: 25.2797 }, // Vilnius + { lat: 54.8985, lon: 23.9036 }, // Kaunas + { lat: 55.7033, lon: 21.1443 }, // Klaipeda + { lat: 55.9349, lon: 23.3137 }, // Siauliai + { lat: 55.7348, lon: 24.3575 }, // Panevezys +]; + +/** Join public catalogs by ID; register only the official still endpoint. */ +export function lithuaniaCameraToSource(row, feature) { + const id = String(row?.id ?? ''); + if (!/^[1-9]\d{0,8}$/.test(id) || String(feature?.id) !== id) return null; + const point = feature?.points?.[0]?.point; + if (!Array.isArray(point) || point.length !== 2) return null; + const [lat, lon] = point; + if ( + !Number.isFinite(lat) || + !Number.isFinite(lon) || + lat < 53.8 || + lat > 56.5 || + lon < 20.8 || + lon > 26.9 + ) + return null; + try { + const url = new URL(row.image); + if ( + url.origin !== 'https://eismoinfo.lt' || + url.username || + url.password || + url.pathname !== FRAME_PATH || + url.searchParams.get('id') !== id + ) + return null; + } catch { + return null; + } + // Reconstruct so extra query parameters cannot alter the request. + const imageUrl = `https://eismoinfo.lt${FRAME_PATH}?id=${id}`; + const cameraId = `lithuania-${id}`; + const name = String(row.name || feature.name || `Lietuva ${id}`) + .trim() + .slice(0, 160); + return { + id: cameraId, + name, + city: 'Lithuania / Lietuva', + cityId: 'lithuania', + provider: 'Via Lietuva / Eismoinfo', + lat, + lon, + // No calibrated pose is published; use the app's low-confidence prior. + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'eismoinfo-public', + credit: + 'Via Lietuva / eismoinfo.lt — periodically updated road-camera stills', + code: cameraDisplayCode(name.toUpperCase()), + }; +} + +async function readCatalog(url, sibling) { + const signal = AbortSignal.any([ + AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + sibling.signal, + ]); + try { + const response = await fetch(url, { + headers: { Accept: 'application/json' }, + redirect: 'manual', + signal, + }); + if (!response.ok) { + await response.body?.cancel().catch(() => {}); + throw new Error( + response.status >= 300 && response.status < 400 + ? 'catalog redirected; redirects are not followed' + : `Catalog HTTP ${response.status}`, + ); + } + return await readResponseJsonCapped( + response, + LITHUANIA_MAX_CATALOG_BYTES, + signal, + ); + } catch (error) { + // The join needs both catalogs; release the other request at once + // rather than leaving its body open until the timeout. + sibling.abort(); + throw error; + } +} + +/** Two bounded requests per shared catalog refresh, no API key required. */ +export async function loadLithuaniaSources() { + try { + const sibling = new AbortController(); + const [rows, layers] = await Promise.all([ + readCatalog(LITHUANIA_CAMERAS_URL, sibling), + readCatalog(LITHUANIA_LOCATIONS_URL, sibling), + ]); + if (!Array.isArray(rows) || !Array.isArray(layers)) return []; + const features = layers.find((layer) => layer?.layer === 'VKR')?.features; + if (!Array.isArray(features)) return []; + const locations = new Map( + features.filter(Boolean).map((feature) => [String(feature.id), feature]), + ); + const cameras = new Map(); + for (const row of rows) { + const camera = lithuaniaCameraToSource( + row, + locations.get(String(row?.id)), + ); + if (camera && !cameras.has(camera.id)) cameras.set(camera.id, camera); + } + const rawCap = Number(process.env.CCTV_LITHUANIA_MAX_SOURCES || 400); + const cap = Number.isFinite(rawCap) + ? Math.max(1, Math.min(1000, Math.floor(rawCap))) + : 400; + const sources = prioritizeSources([...cameras.values()], cap, ANCHORS); + console.log( + `[CCTV] Loaded Lithuania camera sources: ${cameras.size} (using ${sources.length})`, + ); + return sources; + } catch (error) { + console.warn( + '[CCTV] Lithuania camera download error:', + error?.message || error, + ); + return []; + } +} diff --git a/server/providers/cctv/normalize.js b/server/providers/cctv/normalize.js index e62ba33f3..6c54622e9 100644 --- a/server/providers/cctv/normalize.js +++ b/server/providers/cctv/normalize.js @@ -367,6 +367,17 @@ export function isLikelyCalgaryCoordinate(lat, lon) { ); } +/** New York City's five boroughs, with slack for the harbor and city line. */ +export function isLikelyNycCoordinate(lat, lon) { + return ( + isPlausibleLatLon(lat, lon) && + lat >= 40.45 && + lat <= 40.95 && + lon >= -74.3 && + lon <= -73.65 + ); +} + /** Mainland Norway, with slack for the coast and the Finnmark border. */ export function isLikelyNorwayCoordinate(lat, lon) { return ( @@ -383,6 +394,44 @@ export function isLikelyFinlandCoordinate(lat, lon) { return lat >= 59.5 && lat <= 70.5 && lon >= 19 && lon <= 32; } +/** Catalonia bounding box (mainland is roughly 40.5..42.9 N, 0.15..3.4 E). */ +export function isLikelyCataloniaCoordinate(lat, lon) { + return ( + isPlausibleLatLon(lat, lon) && + lat >= 40.3 && + lat <= 42.9 && + lon >= 0.0 && + lon <= 3.4 + ); +} + +/** + * Decode numeric HTML character references, decimal ("í") and hex + * ("í") alike, the entity form the Catalonia feed uses for accented + * place names, plus the handful of named entities XML/HTML text commonly + * carries. A reference that names no valid character (NUL, a surrogate, or + * anything past U+10FFFF) is left as written rather than thrown on, so one + * bad field cannot fail a whole catalog. + * + * @param {string} text + * @returns {string} + */ +export function decodeNumericEntities(text) { + return String(text || '') + .replace(/&#(?:x([0-9a-f]+)|(\d+));/gi, (match, hex, dec) => { + const code = hex ? Number.parseInt(hex, 16) : Number(dec); + const valid = + code > 0 && code <= 0x10ffff && (code < 0xd800 || code > 0xdfff); + return valid ? String.fromCodePoint(code) : match; + }) + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/</g, '<') + .replace(/>/g, '>') + .trim(); +} + /** * Human label for one Fintraffic preset (camera view). * @@ -551,5 +600,30 @@ export function normalizeSourceItem(item) { // badge can distinguish them from raw automated priors (e.g. Austin Open // Data, which never sets this field). Passed through as-is to the client. poseSource: item.poseSource === 'curated' ? 'curated' : undefined, + // Optional hardware-model passthrough (additive, like poseSource): packs + // whose catalog publishes the camera's hardware model — King County's + // layer carries Manufacturer/Model per camera — pass it through so + // identified-hardware consumers can use it. Absent everywhere else. + model: + typeof item.model === 'string' && item.model.trim() + ? item.model.trim() + : undefined, + // Bare model strings collide across vendors (e.g. "3950"), so the + // publishing pack's manufacturer rides along for disambiguation. + manufacturer: + typeof item.manufacturer === 'string' && item.manufacturer.trim() + ? item.manufacturer.trim() + : undefined, }; } + +/** City of Toronto bounding box (Steeles to the lake, Etobicoke to Scarborough). */ +export function isLikelyTorontoCoordinate(lat, lon) { + return ( + isPlausibleLatLon(lat, lon) && + lat >= 43.55 && + lat <= 43.9 && + lon >= -79.7 && + lon <= -79.1 + ); +} diff --git a/server/providers/cctv/sources.js b/server/providers/cctv/sources.js index 7ecd3434d..c3d11e752 100644 --- a/server/providers/cctv/sources.js +++ b/server/providers/cctv/sources.js @@ -55,16 +55,47 @@ import { DEFAULT_CALGARY_MAX_SOURCES, CALGARY_DOWNTOWN, CALGARY_MAX_CATALOG_BYTES, + CATALONIA_CAMERAS_URL, + CATALONIA_MAX_CATALOG_BYTES, + CATALONIA_IMAGE_HOSTS, + CATALONIA_SCT_CAMERA_FILE_RE, + CATALONIA_FONT_CREDIT, + CATALONIA_FONT_ELEVATION_M, + DEFAULT_CATALONIA_MAX_SOURCES, + CATALONIA_ANCHORS, + TORONTO_RESCU_LIST_URL, + TORONTO_RESCU_IMAGE_ORIGIN, + DEFAULT_TORONTO_MAX_SOURCES, + TORONTO_DOWNTOWN, + TORONTO_GROUND_ELEVATION_M, + TORONTO_MAX_CATALOG_BYTES, CCTV_SOURCE_FETCH_TIMEOUT_MS, DELDOT_CCTV_URL, DEFAULT_DELDOT_MAX_SOURCES, DELDOT_ANCHORS, + KING_COUNTY_CAMERAS_URL, + KING_COUNTY_IMAGE_ORIGIN, + DEFAULT_KING_COUNTY_MAX_SOURCES, + KING_COUNTY_ANCHORS, + KING_COUNTY_MAX_CATALOG_BYTES, + WSDOT_CAMERAS_URL, + WSDOT_IMAGE_ORIGIN, + DEFAULT_WSDOT_MAX_SOURCES, + WSDOT_ANCHORS, + WSDOT_MAX_CATALOG_BYTES, DEFAULT_VEGVESEN_CCTV_URL, VEGVESEN_IMAGE_ORIGIN, VEGVESEN_VIDEO_URL, DEFAULT_VEGVESEN_MAX_SOURCES, VEGVESEN_MAX_CATALOG_BYTES, NORWAY_ANCHORS, + NYC_CAMERAS_URL, + NYC_IMAGE_URL, + NYC_CAMERA_ID_RE, + DEFAULT_NYC_MAX_SOURCES, + NYC_MAX_CATALOG_BYTES, + NYC_GROUND_ELEVATION_M, + NYC_ANCHORS, } from './constants.js'; import { toFiniteNumber, @@ -82,13 +113,21 @@ import { isLikelyTexasCoordinate, isLikelyNswCoordinate, isLikelyCalgaryCoordinate, + isLikelyCataloniaCoordinate, + decodeNumericEntities, + isLikelyNycCoordinate, isLikelyNorwayCoordinate, + isLikelyTorontoCoordinate, cameraDisplayCode, rowArrayToObject, prioritizeSources, } from './normalize.js'; import { directionToHeading } from '../../../src/data/directionText.js'; -import { readResponseJsonCapped } from '../common/http.js'; +import { + readResponseBytesCapped, + readResponseJsonCapped, + readResponseTextCapped, +} from '../common/http.js'; /** * Fetch and parse Austin traffic camera records from the city Open Data portal. * @@ -1605,6 +1644,261 @@ export async function loadCalgarySourcesFromOpenData() { } } +/** + * Parse the Catalonia (`cameres.xml`) WFS/GML feed into raw feature records. + * Regex-based rather than a full XML parser — the feed is small (~160 + * features) and this avoids a new dependency for one field set per feature. + * + * @param {string} xml + * @returns {Array<{fid:string, lat:number, lon:number, carretera:string, municipi:string, pk:string, link:string, font:string}>} + */ +export function parseCataloniaXml(xml) { + const out = []; + const blockRe = /([\s\S]*?)<\/gml:featureMember>/g; + const tag = (block, name) => { + const match = new RegExp(`([^<]*)<\\/cite:${name}>`).exec( + block, + ); + return match ? decodeNumericEntities(match[1]) : ''; + }; + let match; + while ((match = blockRe.exec(String(xml || ''))) !== null) { + const block = match[1]; + const fidMatch = /]*>([^<]+)<\/gml:coordinates>/.exec( + block, + ); + if (!fidMatch || !coordsMatch) continue; + const [lonRaw, latRaw] = coordsMatch[1].split(','); + out.push({ + fid: fidMatch[1], + lon: toFiniteNumber(lonRaw), + lat: toFiniteNumber(latRaw), + carretera: tag(block, 'carretera'), + municipi: tag(block, 'municipi'), + pk: tag(block, 'pk'), + link: tag(block, 'link'), + font: tag(block, 'font'), + }); + } + return out; +} + +/** + * Validate and normalize one Catalonia feed `link` against the publisher + * host allowlist (CATALONIA_IMAGE_HOSTS), upgrading to https. Rejects + * anything else, including a host that merely resembles an allowed one, any + * URL carrying userinfo (`https://user:pass@www.bcn.cat/...`) or an explicit + * port — the hostname check alone would pass either, letting catalog data + * inject credentials or steer the server-side frame request to another + * service on that host — and any path outside the host's frame directory. + * + * SCT's own `RenderService` frame link is a special case, for two stacked + * reasons (both verified 2026-09-15): + * 1. It always 302s to `TransitCamera`, on a Location the upstream + * hardcodes as plain http even when RenderService itself was requested + * over https. fetchWithinHost (media.js) only follows same-origin + * redirects — same scheme included, by design, to stop an upstream + * steering a host-pinned request into a plaintext downgrade — so that + * hop is refused and the camera would render as a placeholder. + * 2. mct.gencat.cat's TLS setup itself does not complete a handshake with + * Node's fetch/OpenSSL (`ERR_SSL_WRONG_SIGNATURE_TYPE`) even though it + * is reachable over https from other clients (e.g. curl) — an upstream + * server misconfiguration this proxy cannot route around. + * TransitCamera answers the frame directly over **http**, no redirect, and + * that is the only transport this host actually serves to this runtime — so + * this builds that URL itself from RenderService's `sctidcam` id instead of + * registering the (https, then-redirecting) RenderService link. The + * `sctidcam` id must be a bare image filename (CATALONIA_SCT_CAMERA_FILE_RE). + * + * @param {string} rawUrl + * @returns {string} Normalized URL, or '' if not accepted. + */ +export function normalizeCataloniaImageUrl(rawUrl) { + let parsed; + try { + parsed = new URL(String(rawUrl || '').trim()); + } catch { + return ''; + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') return ''; + if (parsed.username || parsed.password || parsed.port) return ''; + const dir = CATALONIA_IMAGE_HOSTS.get(parsed.hostname); + if (!dir || !parsed.pathname.startsWith(dir)) return ''; + const file = parsed.pathname.slice(dir.length); + if (!file || file.includes('/')) return ''; + + if (parsed.hostname === 'mct.gencat.cat') { + const camId = parsed.searchParams.get('sctidcam'); + if (file !== 'RenderService') return ''; + if (!camId || !CATALONIA_SCT_CAMERA_FILE_RE.test(camId)) return ''; + const direct = new URL('http://mct.gencat.cat/mct2bo/TransitCamera'); + direct.searchParams.set('nom', camId); + direct.searchParams.set('visualitzacio', 'imatge'); + return direct.toString(); + } + + parsed.protocol = 'https:'; + return parsed.toString(); +} + +/** + * Stable camera id for one normalized Catalonia frame URL (the output of + * normalizeCataloniaImageUrl). The feed's `fid` is not used: it embeds an + * upstream server-start stamp and changes on every upstream restart. SCT + * cameras key on the `sctidcam` filename stem (`cat-sct-nc87`), partner + * cameras on the frame host and path (`cat-www-bcn-cat-transit-imatges-…`). + * + * @param {string} imageUrl - A URL normalizeCataloniaImageUrl accepted. + * @returns {string} + */ +export function cataloniaCameraId(imageUrl) { + const parsed = new URL(imageUrl); + const slug = (text) => + text + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + if (parsed.hostname === 'mct.gencat.cat') { + const file = parsed.searchParams.get('nom') || ''; + return `cat-sct-${slug(file.replace(/\.[^.]*$/, ''))}`; + } + return `cat-${slug(parsed.hostname)}-${slug(parsed.pathname)}`; +} + +/** + * Human label for one Catalonia camera: " ()", + * skipping the parenthetical when the municipality is already named in the + * road/place text or when one half is missing. + * + * @param {string} carretera + * @param {string} municipi + * @returns {string} + */ +export function cataloniaCameraName(carretera, municipi) { + const road = String(carretera || '').trim(); + const city = String(municipi || '').trim(); + if (road && city) { + return road.toLowerCase().includes(city.toLowerCase()) + ? road + : `${road} (${city})`; + } + return road || city || 'Càmera de trànsit'; +} + +/** + * Fetch Catalonia traffic cameras (Servei Català de Trànsit), keyless: one + * WFS/GML XML list. The feed carries a `font` field naming the real + * publisher per camera; only SCT's own highway cameras and the hotlinked + * Barcelona (IMI) and Terrassa municipal cameras are kept — the publishers + * whose frame links were confirmed to serve images. Frame URLs are checked + * against a host allowlist (CATALONIA_IMAGE_HOSTS), so a row from any other + * publisher is dropped. The feed carries no camera status, so a camera its + * publisher has taken offline still registers and serves that publisher's + * own "image unavailable" still. + * Each non-SCT camera carries its publisher as a `credit` alongside the + * shared `provider`. The feed carries no heading, so every camera takes the + * id-hash fallback, low-confidence pose (same as TfL and Fintraffic). + * + * Ids come from the validated frame URL (cataloniaCameraId), not the feed's + * restart-dependent `fid`, and rows sharing one are collapsed before the cap + * (the live feed repeats a few SCT cameras). + * + * The list decides ids, coordinates, labels and which allowlisted frame path + * is proxied, so it is fetched over https, redirects are refused (the list + * host cannot be steered or downgraded) and the body read is capped at + * CATALONIA_MAX_CATALOG_BYTES. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadCataloniaSourcesFromOpenData() { + try { + const resp = await fetch(CATALONIA_CAMERAS_URL, { + headers: { Accept: 'application/xml,text/xml,*/*' }, + redirect: 'manual', + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + // A response this loader will not read still owns its transport until the + // body is released, so every rejection path cancels before returning. + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn( + '[CCTV] Catalonia catalog redirected; redirects are not followed', + ); + return discard(); + } + if (!resp.ok) { + console.warn('[CCTV] Catalonia camera download failed:', resp.status); + return discard(); + } + const xml = await readResponseTextCapped(resp, CATALONIA_MAX_CATALOG_BYTES); + const rows = parseCataloniaXml(xml); + if (!rows.length) return []; + + const cameras = []; + for (const row of rows) { + if (!isLikelyCataloniaCoordinate(row.lat, row.lon)) continue; + const imageUrl = normalizeCataloniaImageUrl(row.link); + if (!imageUrl) continue; + + const font = row.font.trim(); + const cameraId = cataloniaCameraId(imageUrl); + + cameras.push({ + id: cameraId, + name: cataloniaCameraName(row.carretera, row.municipi), + city: row.municipi.trim() || 'Catalunya', + cityId: 'catalunya', + provider: 'Servei Català de Trànsit', + lat: row.lat, + lon: row.lon, + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + groundElevationM: CATALONIA_FONT_ELEVATION_M[font] ?? 80, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'catalonia-open-data', + license: + "Generalitat de Catalunya — Llicència oberta d'ús d'informació", + credit: CATALONIA_FONT_CREDIT[font] || '', + }); + } + + const unique = Array.from( + new Map(cameras.map((camera) => [camera.id, camera])).values(), + ); + const maxRaw = Number( + process.env.CCTV_CATALONIA_MAX_SOURCES || DEFAULT_CATALONIA_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(600, Math.floor(maxRaw))) + : DEFAULT_CATALONIA_MAX_SOURCES; + const prioritized = prioritizeSources(unique, maxCount, CATALONIA_ANCHORS); + console.log( + `[CCTV] Loaded Catalonia camera sources: ${unique.length} (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn( + '[CCTV] Catalonia camera download error:', + error?.message || error, + ); + return []; + } +} + export async function loadDelDOTSourcesFromOpenData() { try { const resp = await fetch(DELDOT_CCTV_URL, { @@ -1709,6 +2003,319 @@ export async function loadDelDOTSourcesFromOpenData() { } } +/** Rough King County, WA bounding box. */ +function isLikelyKingCountyCoordinate(lat, lon) { + return ( + Number.isFinite(lat) && + Number.isFinite(lon) && + lat >= 47.0 && + lat <= 48.0 && + lon >= -123.1 && + lon <= -121.0 + ); +} + +/** + * A King County `Model` value, whitespace-collapsed, or '' when it is a + * vendor-generic string ("PTZ", "PTZ IP", "BOSCH/PTZ") that names no model. + */ +function kingCountyModel(raw) { + const model = String(raw || '') + .replace(/\s+/g, ' ') + .trim(); + return /^(?:[a-z]+\/)?ptz(?: ip)?$/i.test(model) ? '' : model; +} + +/** + * Fetch King County road cameras (Washington State). Keyless: one public + * ArcGIS FeatureServer GeoJSON query (King County DOT Road Services), with an + * explicit `outFields` list so the layer's asset-administration columns are + * never fetched. Only cameras that are live and active (`Live`, + * `CurrentStatus`) with a positive integer id, finite in-county coordinates, + * and a frame on the county's own image handler are kept. WSDOT-owned rows in + * the layer are skipped because their frames are WSDOT-hosted, outside the + * pinned image origin, while partner-city cameras hosted on the county image + * handler (e.g. Newcastle) name their owner as the per-camera credit, + * DriveBC-style. The layer publishes no facing, so every heading is the id + * hash at low confidence. + * + * Uniquely among the packs, the county publishes each camera's hardware + * `Manufacturer`/`Model`; the model string (whitespace-collapsed — the feed + * carries stray newlines) rides along as the source's `model` for + * identified-hardware consumers. Vendor-generic strings ("PTZ", "PTZ IP", + * "BOSCH/PTZ") name no model and are treated as absent. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadKingCountySourcesFromOpenData() { + try { + const resp = await fetch(KING_COUNTY_CAMERAS_URL, { + headers: { Accept: 'application/json' }, + redirect: 'manual', + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + // A response this loader will not read still owns its transport until the + // body is released, so every rejection path cancels before returning. + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn( + '[CCTV] King County catalog redirected; redirects are not followed', + ); + return discard(); + } + if (!resp.ok) { + console.warn('[CCTV] King County camera download failed:', resp.status); + return discard(); + } + const payload = await readResponseJsonCapped( + resp, + KING_COUNTY_MAX_CATALOG_BYTES, + ); + const rows = Array.isArray(payload?.features) ? payload.features : []; + + const cameras = []; + const seen = new Set(); + for (const row of rows) { + const attrs = row?.properties; + const rawId = attrs?.AssetID; + if (!Number.isSafeInteger(rawId) || rawId <= 0) continue; + if (attrs.Live !== 1) continue; + if (String(attrs.CurrentStatus || '').trim() !== 'Active') continue; + const owner = String(attrs.Owner || '').trim(); + // WSDOT-owned rows serve WSDOT-hosted frames, outside the pinned + // county image origin, so they are skipped. + if (owner === 'WSDOT') continue; + // The handler is served over HTTPS; the feed still writes http://. + const imageUrl = String(attrs.ImageURL || '') + .trim() + .replace(/^http:\/\/info\.kingcounty\.gov\//, KING_COUNTY_IMAGE_ORIGIN); + if (!imageUrl.startsWith(KING_COUNTY_IMAGE_ORIGIN)) continue; + const [lon, lat] = Array.isArray(row?.geometry?.coordinates) + ? row.geometry.coordinates + : []; + if (!isLikelyKingCountyCoordinate(lat, lon)) continue; + + const cameraId = `kingcounty-${rawId}`; + if (seen.has(cameraId)) continue; + seen.add(cameraId); + const model = kingCountyModel(attrs.Model); + const manufacturer = String(attrs.Manufacturer || '') + .replace(/\s+/g, ' ') + .trim(); + const camera = { + id: cameraId, + name: + String(attrs.Location || '').trim() || `King County camera ${rawId}`, + city: 'King County', + cityId: 'king-county', + provider: 'King County Road Services', + lat, + lon, + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + // The low-confidence pose personality shared by the other packs. + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'kingcounty-open-data', + license: 'King County GIS open data (KCGIS Center terms of use)', + credit: owner && owner !== 'King County' ? owner : '', + }; + if (model) camera.model = model; + if (model && manufacturer) camera.manufacturer = manufacturer; + cameras.push(camera); + } + + const maxRaw = Number( + process.env.CCTV_KINGCOUNTY_MAX_SOURCES || + DEFAULT_KING_COUNTY_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(1200, Math.floor(maxRaw))) + : DEFAULT_KING_COUNTY_MAX_SOURCES; + const prioritized = prioritizeSources( + cameras, + maxCount, + KING_COUNTY_ANCHORS, + ); + console.log( + `[CCTV] Loaded King County camera sources: ${cameras.length} published (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn( + '[CCTV] King County camera download error:', + error?.message || error, + ); + return []; + } +} + +/** WSDOT `CompassDirection` codes as headings in degrees. The live feed sends + * single cardinals (N/E/S/W), `B` (both directions — most rows), `O`, and + * null; the diagonals are mapped in case they appear. `B`, `O`, null, and any + * other code carry no single facing and fall through to the id hash. */ +const WSDOT_COMPASS_HEADINGS = Object.freeze({ + N: 0, + NE: 45, + E: 90, + SE: 135, + S: 180, + SW: 225, + W: 270, + NW: 315, +}); + +/** + * Converts an EPSG:3857 (Web Mercator) coordinate pair to WGS84 degrees. + * The WSDOT camera layer publishes point geometry in wkid 102100. + * @param {number} x - Web Mercator metres east. + * @param {number} y - Web Mercator metres north. + * @returns {{lat: number, lon: number}} WGS84 degrees. + */ +export function webMercatorToWgs84(x, y) { + const R = 6378137; + const lon = (x / R) * (180 / Math.PI); + const lat = (2 * Math.atan(Math.exp(y / R)) - Math.PI / 2) * (180 / Math.PI); + return { lat, lon }; +} + +/** Rough Washington State bounding box (with the Columbia River border cams). */ +function isLikelyWashingtonCoordinate(lat, lon) { + return ( + Number.isFinite(lat) && + Number.isFinite(lon) && + lat >= 45.3 && + lat <= 49.3 && + lon >= -125 && + lon <= -116.5 + ); +} + +/** + * Fetch WSDOT highway cameras (Washington State). Keyless: one Esri JSON + * layer on WSDOT's official open-data host (the same layer the WSDOT Travel + * Center map consumes; a keyed Traveler Information API serves the identical + * population for higher-volume use). Only cameras with a positive integer id, + * finite in-state coordinates, and a frame on WSDOT's own image host are + * kept — the catalog's ~70 partner cameras (Oregon DOT, City of Seattle, + * lodges, airports) are skipped rather than proxied under someone else's + * terms. A single-facing `CompassDirection` (a cardinal) gives a + * high-confidence heading; `B` (both directions), `O`, and null fall back to + * the id hash. + * The feed carries Windows-1252 bytes (en-dashes in titles) with no charset + * declaration, so the body is decoded explicitly before parsing. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadWsdotSourcesFromOpenData() { + try { + const resp = await fetch(WSDOT_CAMERAS_URL, { + headers: { Accept: 'application/json' }, + redirect: 'manual', + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn( + '[CCTV] WSDOT catalog redirected; redirects are not followed', + ); + return discard(); + } + if (!resp.ok) { + console.warn('[CCTV] WSDOT camera download failed:', resp.status); + return discard(); + } + const payload = JSON.parse( + new TextDecoder('windows-1252').decode( + await readResponseBytesCapped(resp, WSDOT_MAX_CATALOG_BYTES), + ), + ); + const rows = Array.isArray(payload?.features) ? payload.features : []; + + const cameras = []; + for (const row of rows) { + const attrs = row?.attributes; + const rawId = attrs?.CameraID; + if (!Number.isSafeInteger(rawId) || rawId <= 0) continue; + const imageUrl = String(attrs.ImageURL || '').trim(); + if (!imageUrl.startsWith(WSDOT_IMAGE_ORIGIN)) continue; + const x = row?.geometry?.x; + const y = row?.geometry?.y; + if (!Number.isFinite(x) || !Number.isFinite(y)) continue; + const { lat, lon } = webMercatorToWgs84(x, y); + if (!isLikelyWashingtonCoordinate(lat, lon)) continue; + + const cameraId = `wsdot-${rawId}`; + const heading = + WSDOT_COMPASS_HEADINGS[ + String(attrs.CompassDirection || '') + .trim() + .toUpperCase() + ]; + const hasHeading = Number.isFinite(heading); + cameras.push({ + id: cameraId, + name: String(attrs.CameraTitle || '').trim() || `WSDOT camera ${rawId}`, + city: 'Washington', + cityId: 'washington-state', + provider: 'WSDOT', + lat, + lon, + headingDeg: hasHeading ? heading : fallbackHeadingFromId(cameraId), + headingConfidence: hasHeading ? 'high' : 'low', + // Same two pose personalities as the other packs: raw priors that the + // client's ground snap and manual calibration refine. + pitchDeg: hasHeading ? -24 : -18, + fovDeg: hasHeading ? 56 : 44, + rangeM: hasHeading ? 210 : 145, + mountHeightM: hasHeading ? 10 : 8, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'wsdot-open-data', + license: 'WSDOT Travel Center camera data (public low-volume feed)', + }); + } + + const maxRaw = Number( + process.env.CCTV_WSDOT_MAX_SOURCES || DEFAULT_WSDOT_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(1200, Math.floor(maxRaw))) + : DEFAULT_WSDOT_MAX_SOURCES; + const prioritized = prioritizeSources(cameras, maxCount, WSDOT_ANCHORS); + console.log( + `[CCTV] Loaded WSDOT camera sources: ${cameras.length} published (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn( + '[CCTV] WSDOT camera download error:', + error?.message || error, + ); + return []; + } +} + /** * One Statens vegvesen `CctvSimple` feature -> one catalog source, or null. * @@ -1863,3 +2470,317 @@ export async function loadVegvesenSourcesFromOpenData() { return []; } } + +const NYC_BOROUGHS = new Set([ + 'Manhattan', + 'Brooklyn', + 'Queens', + 'Bronx', + 'Staten Island', +]); + +/** NYC TMC publishes `isOnline` as the strings "true"/"false". */ +export function nycCameraIsOnline(value) { + return ( + value === true || + String(value ?? '') + .trim() + .toLowerCase() === 'true' + ); +} + +/** + * Label for one NYC camera: the intersection an operator recognises + * ("Central Park West @ 86 St"). The catalog's `area` is the borough, not + * a facing, and is not part of the name. + * + * @param {object} record - Raw TMC camera row. + * @param {string} cameraId - Derived stable id. + * @returns {string} + */ +export function nycCameraName(record, cameraId) { + const name = String(record?.name ?? '').trim(); + if (name) return name; + const short = String(cameraId || '') + .replace(/^nyc-/, '') + .slice(0, 8); + return short ? `NYC Camera ${short}` : 'NYC Camera'; +} + +/** + * One NYC DOT TMC camera row -> one catalog source, or null. + * + * Frame URLs are synthesized from a validated UUID on the pinned TMC image + * origin. The payload's `imageUrl` is ignored so a catalog edit cannot steer + * the frame proxy off-host. There is no heading in the feed; headings use + * the shared id-hash fallback at low confidence, like Calgary and Fintraffic. + * + * @param {object} record - Raw TMC camera row. + * @returns {?object} + */ +export function nycCameraToSource(record) { + if (!record || typeof record !== 'object') return null; + if (!nycCameraIsOnline(record.isOnline)) return null; + + const lat = toFiniteNumber(record.latitude); + const lon = toFiniteNumber(record.longitude); + if (!isLikelyNycCoordinate(lat, lon)) return null; + + const uuid = String(record.id ?? '').trim(); + if (!NYC_CAMERA_ID_RE.test(uuid)) return null; + const cameraId = `nyc-${uuid.toLowerCase()}`; + const imageUrl = NYC_IMAGE_URL(uuid); + const name = nycCameraName(record, cameraId); + const area = String(record.area ?? '').trim(); + const city = NYC_BOROUGHS.has(area) ? area : 'New York'; + + return { + id: cameraId, + name, + city, + cityId: 'nyc', + provider: 'NYC DOT Traffic Management Center', + lat, + lon, + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + groundElevationM: NYC_GROUND_ELEVATION_M, + feedType: 'image', + url: imageUrl, + snapshotUrl: imageUrl, + sourceKind: 'nyc-dot-tmc', + license: 'Public NYC DOT Traffic Management Center traffic camera', + code: cameraDisplayCode(name.toUpperCase()), + }; +} + +/** + * Fetch NYC DOT Traffic Management Center cameras from the keyless + * `webcams.nyctmc.org` catalog. Frames are stills on that same host, + * registered from a validated UUID rather than from the payload URL. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadNycSourcesFromOpenData() { + try { + const endpoint = process.env.CCTV_NYC_CAMERAS_URL || NYC_CAMERAS_URL; + const resp = await fetch(endpoint, { + headers: { Accept: 'application/json' }, + redirect: 'manual', + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn('[CCTV] NYC catalog redirected; redirects are not followed'); + return discard(); + } + if (!resp.ok) { + console.warn('[CCTV] NYC camera download failed:', resp.status); + return discard(); + } + const rows = await readResponseJsonCapped(resp, NYC_MAX_CATALOG_BYTES); + if (!Array.isArray(rows)) return []; + const cameras = []; + const seen = new Set(); + for (const record of rows) { + const camera = nycCameraToSource(record); + if (!camera || seen.has(camera.id)) continue; + seen.add(camera.id); + cameras.push(camera); + } + const maxRaw = Number( + process.env.CCTV_NYC_MAX_SOURCES || DEFAULT_NYC_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(400, Math.floor(maxRaw))) + : DEFAULT_NYC_MAX_SOURCES; + const prioritized = prioritizeSources(cameras, maxCount, NYC_ANCHORS); + console.log( + `[CCTV] Loaded NYC camera sources: ${cameras.length} (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn('[CCTV] NYC camera download error:', error?.message || error); + return []; + } +} + +/** + * The JSONP envelope the City of Toronto wraps its camera list in. Anchored at + * both ends, so only that exact callback is unwrapped — a payload that merely + * contains the name somewhere is not treated as the list. + */ +const RESCU_JSONP = /^jsonTMCEarthCamerasCallback\(([\s\S]*)\)\s*;?$/; + +/** + * Unwrap the RESCU camera list. The portal serves the list as JSONP; should it + * ever drop the wrapper, the bare JSON document still parses. Anything else — + * a foreign callback, a truncated body, an error page — yields no cameras + * rather than a thrown loader. + * + * @param {string} text - Raw response body. + * @returns {Array} The `Data` rows, or [] when the body is unusable. + */ +export function parseRescuCameraList(text) { + const body = String(text ?? '').trim(); + if (!body) return []; + const unwrapped = RESCU_JSONP.exec(body)?.[1] ?? body; + try { + const parsed = JSON.parse(unwrapped); + const rows = parsed?.Data ?? parsed?.data; + return Array.isArray(rows) ? rows : []; + } catch { + return []; + } +} + +/** + * Convert one RESCU list row into a camera source. + * + * The row carries no URL: the frame URL is built from the documented + * `loc####.jpg` convention and a camera number validated as plain digits, so + * the frame proxy is pinned to the city origin by construction and no upstream + * field — present or planted — can steer it. + * + * No compass heading exists in this dataset. `D1`-`D4` are the directions of + * the *comparison* images (static reference stills captured in 2016, served + * from a different path); the live frame is one pan-tilt-zoom camera pointing + * wherever the operator left it. Treating those letters as a facing would put + * a confident, wrong bearing on every camera in the city, so every row takes + * the id-hash fallback and the low-confidence pose, as headingless Austin, + * TfL, Fintraffic and Calgary cameras do. + * + * @param {object} row - One entry from the RESCU `Data` array. + * @returns {object|null} Normalized camera source, or null when unusable. + */ +export function torontoCameraToSource(row) { + if (!row || typeof row !== 'object') return null; + const number = String(row.Number ?? row.number ?? '').trim(); + // Plain digits only: a number is about to become a URL path segment. + if (!/^\d{1,6}$/.test(number)) return null; + + const lat = toFiniteNumber(row.Latitude ?? row.latitude); + const lon = toFiniteNumber(row.Longitude ?? row.longitude); + if (!isLikelyTorontoCoordinate(lat, lon)) return null; + + const cameraId = `toronto-${number}`; + // The feed's own label, verbatim. It is upper-case ("MCCOWAN RD", "ST + // HELEN'S AVE"); title-casing it would mangle more names than it fixed. + const name = + String(row.Name ?? row.name ?? '') + .replace(/\s+/g, ' ') + .trim() || `Toronto Camera ${number}`; + + return { + id: cameraId, + name, + city: 'Toronto', + cityId: 'toronto', + provider: 'City of Toronto', + lat, + lon, + headingDeg: fallbackHeadingFromId(cameraId), + headingConfidence: 'low', + pitchDeg: -18, + fovDeg: 44, + rangeM: 145, + mountHeightM: 8, + groundElevationM: TORONTO_GROUND_ELEVATION_M, + feedType: 'image', + url: `${TORONTO_RESCU_IMAGE_ORIGIN}loc${number}.jpg`, + snapshotUrl: `${TORONTO_RESCU_IMAGE_ORIGIN}loc${number}.jpg`, + sourceKind: 'toronto-rescu-open-data', + license: + 'Contains information licensed under the Open Government Licence – Toronto', + // Unselected-label code: the intersection, so a camera at rest reads as a + // place rather than as its id. + code: cameraDisplayCode(name.toUpperCase()), + }; +} + +/** + * Fetch City of Toronto RESCU traffic cameras from the City's open data + * portal, keyless. One JSONP list for the whole city; frames are stills on the + * same portal origin. + * + * The list fetch refuses redirects (`redirect: 'manual'`) so the list host + * cannot be steered, and caps the body it will buffer. + * + * Attribution: Open Government Licence – Toronto, registered in + * src/data/dataCredits.js. + * + * @returns {Promise>} Normalized camera source objects. + */ +export async function loadTorontoSourcesFromOpenData() { + try { + const endpoint = + process.env.CCTV_TORONTO_LIST_URL || TORONTO_RESCU_LIST_URL; + const resp = await fetch(endpoint, { + headers: { Accept: 'application/json, text/javascript' }, + redirect: 'manual', + signal: AbortSignal.timeout(CCTV_SOURCE_FETCH_TIMEOUT_MS), + }); + // A response this loader will not read still owns its transport until the + // body is released, so every rejection path cancels before returning. + const discard = async () => { + try { + await resp.body?.cancel(); + } catch { + /* no-op */ + } + return []; + }; + if (resp.status >= 300 && resp.status < 400) { + console.warn( + '[CCTV] Toronto camera list redirected; redirects are not followed', + ); + return discard(); + } + if (!resp.ok) { + console.warn('[CCTV] Toronto camera download failed:', resp.status); + return discard(); + } + const text = await readResponseTextCapped(resp, TORONTO_MAX_CATALOG_BYTES); + const rows = parseRescuCameraList(text); + if (!rows.length) return []; + + const cameras = []; + const seen = new Set(); + for (const record of rows) { + const camera = torontoCameraToSource(record); + if (!camera || seen.has(camera.id)) continue; + seen.add(camera.id); + cameras.push(camera); + } + const maxRaw = Number( + process.env.CCTV_TORONTO_MAX_SOURCES || DEFAULT_TORONTO_MAX_SOURCES, + ); + const maxCount = Number.isFinite(maxRaw) + ? Math.max(8, Math.min(500, Math.floor(maxRaw))) + : DEFAULT_TORONTO_MAX_SOURCES; + const prioritized = prioritizeSources(cameras, maxCount, [ + TORONTO_DOWNTOWN, + ]); + console.log( + `[CCTV] Loaded Toronto camera sources: ${cameras.length} (using nearest ${prioritized.length})`, + ); + return prioritized; + } catch (error) { + console.warn( + '[CCTV] Toronto camera download error:', + error?.message || error, + ); + return []; + } +} diff --git a/server/providers/firms.js b/server/providers/firms.js index 636968348..c73ca10ce 100644 --- a/server/providers/firms.js +++ b/server/providers/firms.js @@ -2,6 +2,7 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js'; +import { readResponseTextCapped } from './common/http.js'; /** * NASA FIRMS live active-fire proxy with a memory + disk cache. @@ -35,6 +36,12 @@ export function firmsProxy() { 'VIIRS_SNPP_NRT', 'MODIS_NRT', ]; + /** + * Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about + * 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a + * bound, not a budget. + */ + const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024; const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); const CACHE_PATH = path.join(CACHE_DIR, 'firms.json'); @@ -83,9 +90,12 @@ export function firmsProxy() { */ async function fetchSource(key, source) { const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`; - const res = await fetch(url, { signal: AbortSignal.timeout(60_000) }); + const signal = AbortSignal.timeout(60_000); + const res = await fetch(url, { signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); - const records = parseFirmsCsv(await res.text()); + const records = parseFirmsCsv( + await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal), + ); if (records === null) throw new Error('non-CSV upstream response'); return records; } diff --git a/server/providers/gbfs.js b/server/providers/gbfs.js index 9741bb4e6..990642c8b 100644 --- a/server/providers/gbfs.js +++ b/server/providers/gbfs.js @@ -3,6 +3,7 @@ import { isAllowedGbfsHost, isAllowedGbfsPath, gbfsCacheControl, + gbfsUpstreamHeaders, } from '../../src/data/gbfsSource.js'; // --------------------------------------------------------------------------- @@ -11,6 +12,9 @@ import { /** Upstream fetch timeout for GBFS requests (ms). */ const GBFS_PROXY_TIMEOUT_MS = 12000; +/** Value of the Client-Identifier header sent with every upstream request. */ +export const GBFS_CLIENT_IDENTIFIER = 'gods-eye-view'; + export const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB function gbfsRedirectHost(location, requestUrl) { @@ -60,6 +64,10 @@ export async function fetchGbfsUpstream( headers: { Accept: 'application/json', 'User-Agent': 'gods-eye-view-gbfs-proxy/1.0', + // Urban Sharing (the Norwegian city-bike feeds) asks every client to + // name itself in this header; other operators ignore it. + 'Client-Identifier': GBFS_CLIENT_IDENTIFIER, + ...gbfsUpstreamHeaders(new URL(url).hostname), }, redirect: 'manual', signal: controller.signal, @@ -181,7 +189,7 @@ export function gbfsProxy() { return; } - if (!isAllowedGbfsPath(upstreamUrl.pathname)) { + if (!isAllowedGbfsPath(upstreamUrl.pathname, upstreamUrl.hostname)) { res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', diff --git a/server/providers/gnssIntegrity.js b/server/providers/gnssIntegrity.js new file mode 100644 index 000000000..adf28cea5 --- /dev/null +++ b/server/providers/gnssIntegrity.js @@ -0,0 +1,238 @@ +import { + GNSS_CLASSIFIER, + GNSS_WINDOW_MS, + normalizeGnssAircraft, +} from '../../src/layers/gnss/records.js'; +import { adsbLolFallbackAnchor } from './aircraft/opensky.js'; +import { coalesceProxyRequest, readResponseJsonCapped } from './common/http.js'; +import { makeRateLimiter, clientKey } from './common/rate-limit.js'; + +// adsb.lol regional snapshot (keyless, ODbL). Only the integrity fields the +// GNSS layer needs leave this proxy, so the browser never sees the full feed. +// This is a separate read from the flights fallback on purpose: that point +// cache keeps only the fields the flights layer renders (no nic / nac_p), is +// keyed on 0.25° anchors and lives for 12 s, while integrity cells need +// whole-degree anchors held for a minute. +const RADIUS_NM = 250; +const CACHE_MS = 60_000; +const CACHE_MAX = 64; +/** Default 429 cooldown when upstream sends no usable Retry-After. */ +const COOLDOWN_MS = 60_000; +/** Bound sane upstream deadlines to one hour; never shorten a deadline within that range. */ +const COOLDOWN_MIN_MS = 5_000; +const COOLDOWN_MAX_MS = 60 * 60_000; +const FAILURE_COOLDOWN_MS = 15_000; +const MAX_IN_FLIGHT = 8; +const MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +const TIMEOUT_MS = 10_000; + +/** Snap a view anchor to a whole degree so nearby views share one upstream read. */ +export function gnssIntegrityAnchor(url) { + const anchor = adsbLolFallbackAnchor({ url }); + if (!anchor) return null; + return { + lat: Math.round(anchor.latitude), + lon: Math.round(anchor.longitude), + }; +} + +/** Cooldown (ms) an upstream 429 earns, honouring Retry-After when sane. */ +export function gnssRetryCooldownMs(raw, nowMs) { + if (raw) { + const seconds = /^\d+(?:\.\d+)?$/.test(raw.trim()) ? Number(raw) : NaN; + const ms = seconds * 1000; + if (Number.isFinite(seconds) && seconds > 0) + return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, ms)); + const at = + Number.isNaN(seconds) && /[a-z]/i.test(raw) ? Date.parse(raw) : NaN; + if (Number.isFinite(at) && at > nowMs) + return Math.min(COOLDOWN_MAX_MS, Math.max(COOLDOWN_MIN_MS, at - nowMs)); + } + return COOLDOWN_MS; +} + +/** Fixed-origin, bounded adsb.lol integrity route for dev and preview. */ +export function gnssIntegrityProxy({ + fetchImpl = (...args) => globalThis.fetch(...args), + now = () => Date.now(), + monotonicNow = () => performance.now(), + warn = (...args) => console.warn(...args), +} = {}) { + const cache = new Map(); + const inFlight = new Map(); + const allow = makeRateLimiter({ windowMs: 60_000, max: 30, globalMax: 600 }); + let cooldownUntil = 0; + let cooldownStatus = 429; + + async function load({ lat, lon }) { + if (monotonicNow() < cooldownUntil) + throw Object.assign(new Error('cooling_down'), { + status: cooldownStatus, + }); + const signal = AbortSignal.timeout(TIMEOUT_MS); + try { + const response = await fetchImpl( + `https://api.adsb.lol/v2/lat/${lat}/lon/${lon}/dist/${RADIUS_NM}`, + { + signal, + redirect: 'error', + headers: { + Accept: 'application/json', + 'User-Agent': 'gods-eye-view-gnss-integrity/1.0', + }, + }, + ); + if (!response.ok) { + await response.body?.cancel(); + if (response.status === 429) { + const at = monotonicNow(); + const raw = response.headers.get('retry-after'); + const delay = gnssRetryCooldownMs(raw, now()); + if (delay === COOLDOWN_MAX_MS) + warn('[gnss-integrity] upstream cooldown capped at one hour'); + cooldownStatus = 429; + cooldownUntil = Math.max(cooldownUntil, at + delay); + throw Object.assign(new Error('upstream_rate_limited'), { + status: 429, + }); + } + throw new Error('upstream_unavailable'); + } + const payload = await readResponseJsonCapped( + response, + MAX_RESPONSE_BYTES, + signal, + ); + const rows = normalizeGnssAircraft(payload); + if (!rows) throw new Error('invalid_snapshot'); + const sourceNow = payload.now; + if (!Number.isFinite(sourceNow) || sourceNow <= 0) + throw new Error('undated_snapshot'); + const fetchedAt = Math.min( + now(), + sourceNow > 10_000_000_000 ? sourceNow : sourceNow * 1000, + ); + if (now() - fetchedAt > GNSS_WINDOW_MS) + throw new Error('expired_snapshot'); + return { + fetchedAt, + anchor: { lat, lon }, + radiusNm: RADIUS_NM, + // `degraded` on each row is this classifier's verdict, not gpsjam's. + classifier: GNSS_CLASSIFIER.id, + rows, + }; + } catch (error) { + if (error.status !== 429) { + const until = monotonicNow() + FAILURE_COOLDOWN_MS; + if (until > cooldownUntil) { + cooldownUntil = until; + cooldownStatus = 502; + } + } + throw error; + } + } + + function age(entry) { + if (!entry) return Infinity; + const elapsed = monotonicNow() - entry.savedAt; + // A backwards injected clock cannot rejuvenate a snapshot. + return elapsed < 0 ? Infinity : entry.ageAtReceipt + elapsed; + } + + async function acquire(anchor, peer) { + const key = `${anchor.lat},${anchor.lon}`; + const previous = cache.get(key); + if (age(previous) < CACHE_MS) + return { value: previous.value, ageMs: age(previous), stale: false }; + try { + if (!inFlight.has(key) && inFlight.size >= MAX_IN_FLIGHT) + throw Object.assign(new Error('busy'), { status: 429, local: true }); + const { promise } = coalesceProxyRequest(inFlight, key, async () => { + if (monotonicNow() >= cooldownUntil && !allow(peer)) + throw Object.assign(new Error('rate_limited'), { + status: 429, + local: true, + }); + const value = await load(anchor); + const entry = { + value, + savedAt: monotonicNow(), + ageAtReceipt: Math.max(0, now() - value.fetchedAt), + }; + cache.delete(key); + cache.set(key, entry); + while (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value); + return entry; + }); + const entry = await promise; + const ageMs = age(entry); + return { value: entry.value, ageMs, stale: ageMs >= CACHE_MS }; + } catch (error) { + if (age(previous) <= GNSS_WINDOW_MS) + return { value: previous.value, ageMs: age(previous), stale: true }; + throw error; + } + } + + async function handler(req, res) { + const json = (status, value, stale = false, retryAfterS = 60) => { + if (res.destroyed) return; + res.writeHead(status, { + 'Content-Type': 'application/json', + 'Cache-Control': 'no-store', + ...(status === 405 ? { Allow: 'GET' } : {}), + ...([429, 502].includes(status) + ? { 'Retry-After': String(retryAfterS) } + : {}), + ...(stale ? { 'X-Data-Stale': 'true' } : {}), + }); + res.end(JSON.stringify(value)); + }; + if (req.method !== 'GET') return json(405, { error: 'method_not_allowed' }); + const path = (req.url || '/').split('?')[0]; + if (path !== '/' && path !== '') + return json(404, { error: 'unknown_route' }); + const anchor = gnssIntegrityAnchor(req.url); + if (!anchor) return json(400, { error: 'invalid_anchor' }); + try { + const { value, stale, ageMs } = await acquire(anchor, clientKey(req)); + // ageMs is measured on this server's clock, so the browser can date + // the rows without comparing its clock to ours. + json(200, { ...value, ageMs, ...(stale ? { stale: true } : {}) }, stale); + } catch (error) { + if (error.status === 429) { + // Tell the browser how long the shared upstream cooldown has left. + const retryAfterS = Math.max( + 1, + error.local ? 60 : Math.ceil((cooldownUntil - monotonicNow()) / 1000), + ); + return json( + 429, + { + error: error.local ? error.message : 'gnss_integrity_rate_limited', + }, + false, + retryAfterS, + ); + } + json( + 502, + { error: 'gnss_integrity_unavailable' }, + false, + Math.max(1, Math.ceil((cooldownUntil - monotonicNow()) / 1000)), + ); + } + } + + return { + name: 'gnss-integrity', + configureServer({ middlewares }) { + middlewares.use('/api/gnss-integrity', handler); + }, + configurePreviewServer({ middlewares }) { + middlewares.use('/api/gnss-integrity', handler); + }, + }; +} diff --git a/server/providers/local.js b/server/providers/local.js index 9acbade31..854b77f33 100644 --- a/server/providers/local.js +++ b/server/providers/local.js @@ -24,6 +24,7 @@ import { mapillaryProxy } from './mapillary.js'; import { keySetupEndpoint } from '../standalone/key-setup.js'; import { weatherProxy } from './weather.js'; import { firePerimetersProxy } from './firePerimeters.js'; +import { gnssIntegrityProxy } from './gnssIntegrity.js'; import { cycloneProxy } from './cyclones.js'; import { windProxy } from './wind.js'; @@ -60,6 +61,7 @@ function localProviderPlugins({ realtime } = {}) { cycloneProxy(), mapillaryProxy(), firePerimetersProxy(), + gnssIntegrityProxy(), keySetupEndpoint(), ]; } @@ -81,6 +83,9 @@ export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js'; export { googlePlacesContextProxy } from './places.js'; export { googleServerApiKey } from './places.js'; export { keylessGooglePlacesResponse } from './places.js'; +export { keylessGoogleGeocodeResponse } from './places.js'; +export { validateGeocodeAddress } from './places.js'; +export { geocodeBounds } from './places.js'; export { adsbLolFallbackAnchor } from './aircraft/opensky.js'; export { readResponseTextCapped } from './common/http.js'; export { readResponseJsonCapped } from './common/http.js'; diff --git a/server/providers/military-installations.js b/server/providers/military-installations.js index 08ad87d9f..7d252b8ed 100644 --- a/server/providers/military-installations.js +++ b/server/providers/military-installations.js @@ -9,7 +9,10 @@ import { fetchOverpassPayload, overpassNotConfigured, } from './overpass/transport.js'; -import { resolveOverpassUpstreams } from './overpass/constants.js'; +import { + resolveOverpassUpstreams, + resolveOverpassUpstreamTag, +} from './overpass/constants.js'; import { _militaryInstallationCache, trimMilitaryInstallationCache, @@ -74,7 +77,13 @@ function militaryInstallationsProxy() { retrievedAt: new Date().toISOString(), status: 'ready', }; - const entry = { payload, cachedAt: Date.now() }; + // Tagged with the configured upstream set so the disk layer does not serve + // this answer back after the operator repoints (see overpass/cache.js). + const entry = { + payload, + cachedAt: Date.now(), + upstreams: resolveOverpassUpstreamTag(), + }; _militaryInstallationCache.set(key, entry); trimMilitaryInstallationCache(); writeMilitaryInstallationDisk(key, entry); diff --git a/server/providers/military-installations/cache.js b/server/providers/military-installations/cache.js index c77627e29..4d77981e6 100644 --- a/server/providers/military-installations/cache.js +++ b/server/providers/military-installations/cache.js @@ -5,6 +5,7 @@ import { MILITARY_INSTALLATION_DISK_DIR, MILITARY_INSTALLATION_MAX_CACHE, } from './constants.js'; +import { cachedByConfiguredUpstream } from '../overpass/constants.js'; import path from 'node:path'; import { createHash } from 'node:crypto'; import { promises as fsp } from 'node:fs'; @@ -114,6 +115,9 @@ async function readMilitaryInstallationDisk( const entry = JSON.parse( await fsp.readFile(militaryInstallationDiskPath(cacheKey, dir), 'utf8'), ); + // Same reason as the Overpass disk cache: the key is the bbox alone, so an + // entry written against a different OVERPASS_UPSTREAMS is otherwise a hit. + if (!cachedByConfiguredUpstream(entry)) return null; if (!militaryInstallationDiskFresh(entry, maxAgeMs)) return null; return migrateMilitaryInstallationEntry(entry); } catch { diff --git a/server/providers/openai.js b/server/providers/openai.js index e5d84e0a9..ad2161d6e 100644 --- a/server/providers/openai.js +++ b/server/providers/openai.js @@ -1,20 +1,26 @@ import { defaultSourceRoot } from './common/source-root.js'; import { handleHudSummary } from './openai/hud-summary.js'; import { createDebugLogHandler } from './openai/debug-log.js'; -import { createRealtimeTokenHandler } from './openai/realtime.js'; +import { + createRealtimeOAuthLoginHandler, + createRealtimeOAuthStatusHandler, + createRealtimeTokenHandler, +} from './openai/realtime.js'; import { sameSiteGated } from './common/same-site.js'; +import { createCodexOAuthLogin } from './openai/codex-auth.js'; /** * Vite plugin: OpenAI Realtime ephemeral client secret. * - * Keeps OPENAI_API_KEY server-side while the browser connects to the - * Realtime API over WebRTC with a short-lived secret. + * Keeps the selected cloud credential server-side while the browser connects + * to the Realtime API over WebRTC with a short-lived secret. */ function openAiRealtimeProxy({ sourceRoot = defaultSourceRoot, annotationGuidance, realtime = {}, } = {}) { + const oauthLogin = createCodexOAuthLogin(realtime); function install(middlewares) { // Cost-bearing and log endpoints refuse cross-site browser requests // (see server/providers/common/same-site.js and SECURITY.md). @@ -31,16 +37,29 @@ function openAiRealtimeProxy({ createRealtimeTokenHandler({ ...realtime, annotationGuidance }), ), ); + + middlewares.use( + '/api/realtime/oauth-status', + sameSiteGated(createRealtimeOAuthStatusHandler({ oauthLogin })), + ); + + middlewares.use( + '/api/realtime/oauth-login', + sameSiteGated(createRealtimeOAuthLoginHandler({ oauthLogin })), + ); } return { name: 'openai-realtime-proxy', configureServer(server) { install(server.middlewares); + server.httpServer?.once('close', oauthLogin.dispose); }, configurePreviewServer(server) { install(server.middlewares); + server.httpServer?.once('close', oauthLogin.dispose); }, + closeBundle: oauthLogin.dispose, }; } diff --git a/server/providers/openai/codex-auth.js b/server/providers/openai/codex-auth.js new file mode 100644 index 000000000..5f16ee824 --- /dev/null +++ b/server/providers/openai/codex-auth.js @@ -0,0 +1,360 @@ +import { spawn } from 'node:child_process'; +import { + accessSync, + constants as fsConstants, + readFileSync, + statSync, +} from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export function codexAuthPath({ + environment = process.env, + home = os.homedir(), +} = {}) { + const explicit = String(environment.CODEX_AUTH_JSON || '').trim(); + if (explicit) return path.resolve(explicit); + const codexHome = String(environment.CODEX_HOME || '').trim(); + return path.join( + codexHome ? path.resolve(codexHome) : path.join(home, '.codex'), + 'auth.json', + ); +} + +/** An optional operator lock that refuses the metered API-key voice lane. */ +export function preferCodexOAuth(environment = process.env) { + if (!Object.hasOwn(environment, 'GEV_PREFER_CODEX_OAUTH')) return false; + const value = String(environment.GEV_PREFER_CODEX_OAUTH || '') + .trim() + .toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(value)) return true; + if (['0', 'false', 'no', 'off'].includes(value)) return false; + throw new Error( + 'GEV_PREFER_CODEX_OAUTH must be true or false. Refusing to choose a voice auth mode.', + ); +} + +function codexOAuthError(message, code = 'CODEX_OAUTH_UNAVAILABLE') { + const error = new Error(message); + error.code = code; + return error; +} + +export function readCodexOAuthAccessToken({ + authPath = codexAuthPath(), + readFile = readFileSync, + statFile = statSync, + nowMs = Date.now(), +} = {}) { + const unavailable = + 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.'; + let parsed; + try { + parsed = JSON.parse(readFile(authPath, 'utf8')); + } catch { + throw codexOAuthError(unavailable); + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw codexOAuthError(unavailable); + } + const mode = String(parsed.auth_mode || '') + .trim() + .toLowerCase(); + const chatGptMode = mode + ? mode === 'chatgpt' || mode === 'chatgptauthtokens' + : typeof parsed.OPENAI_API_KEY !== 'string'; + const token = parsed.tokens?.access_token; + if (!chatGptMode || typeof token !== 'string' || !token.trim()) { + throw codexOAuthError(unavailable); + } + + // This only checks token shape and expiry. OpenAI verifies the signature when + // minting the short-lived Realtime credential; we never modify auth.json. + const parts = token.split('.'); + if ( + parts.length !== 3 || + parts.some((part) => !/^[A-Za-z0-9_-]+$/.test(part)) + ) { + throw codexOAuthError(unavailable); + } + let payload; + try { + payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')); + } catch { + throw codexOAuthError(unavailable); + } + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + throw codexOAuthError(unavailable); + } + let expiresAtMs; + if (Object.hasOwn(payload, 'exp')) { + if (typeof payload.exp !== 'number' || !Number.isFinite(payload.exp)) { + throw codexOAuthError(unavailable); + } + expiresAtMs = payload.exp * 1000; + } else { + try { + expiresAtMs = statFile(authPath).mtimeMs + 60 * 60 * 1000; + } catch { + throw codexOAuthError(unavailable); + } + } + if (!Number.isFinite(expiresAtMs) || expiresAtMs <= nowMs + 60_000) { + throw codexOAuthError( + 'ChatGPT sign-in has expired. Re-authenticate in Provider Settings or run codex login and try again.', + 'CODEX_OAUTH_REAUTH_REQUIRED', + ); + } + return token; +} + +export function resolveCodexExecutable({ + environment = process.env, + home = os.homedir(), + access = accessSync, +} = {}) { + const explicit = String(environment.CODEX_BIN || '').trim(); + if (explicit) return explicit; + + for (const candidate of [ + path.join(home, '.local', 'bin', 'codex'), + path.join( + home, + '.codex', + 'packages', + 'standalone', + 'current', + 'bin', + 'codex', + ), + ]) { + try { + access(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Fall through to the next known location, then PATH. + } + } + return 'codex'; +} + +// What `codex login` needs to find its home, open a browser and reach OpenAI. +// The server's own environment also holds every provider secret, so the +// child receives only these names. +const CODEX_LOGIN_ENV = + /^(?:PATH|PATHEXT|HOME|USER|USERNAME|LOGNAME|SHELL|LANG|LANGUAGE|LC_[A-Z_]+|TERM|TZ|TMPDIR|TEMP|TMP|XDG_[A-Z_]+|DISPLAY|WAYLAND_DISPLAY|DBUS_SESSION_BUS_ADDRESS|BROWSER|WSL_[A-Z_]+|WSLENV|(?:HTTPS?|ALL|NO)_PROXY|SSL_CERT_(?:FILE|DIR)|NODE_EXTRA_CA_CERTS|SYSTEMROOT|WINDIR|COMSPEC|USERPROFILE|APPDATA|LOCALAPPDATA|PROGRAMDATA|HOMEDRIVE|HOMEPATH|CODEX_[A-Z_]+)$/i; + +/** The environment `codex login` runs with: the allowlist, then `CODEX_HOME`. */ +export function codexLoginEnvironment(environment, codexHome) { + const env = {}; + for (const [name, value] of Object.entries(environment || {})) + if (CODEX_LOGIN_ENV.test(name) && value !== undefined) env[name] = value; + env.CODEX_HOME = codexHome; + return env; +} + +export function startCodexChatGptLogin({ + spawnImpl = spawn, + environment = process.env, + home = os.homedir(), + executable = resolveCodexExecutable({ environment, home }), +} = {}) { + // Codex owns its credential storage. An explicit read path must agree with + // the CLI's write path; never silently sign in to another credential store. + const configuredHome = String(environment.CODEX_HOME || '').trim(); + const loginHome = configuredHome + ? path.resolve(configuredHome) + : path.join(home, '.codex'); + if ( + codexAuthPath({ environment, home }) !== path.join(loginHome, 'auth.json') + ) { + return Promise.reject( + codexOAuthError( + 'CODEX_AUTH_JSON does not match the Codex login storage path. Set CODEX_HOME to the matching directory or sign in to Codex separately.', + 'CODEX_OAUTH_STORAGE_MISMATCH', + ), + ); + } + + return new Promise((resolve, reject) => { + let child; + let spawned = false; + let finish; + const completion = new Promise((done) => { + finish = done; + }); + const fail = () => { + reject( + codexOAuthError( + 'Could not start ChatGPT sign-in. Check that Codex is installed, or set CODEX_BIN to its executable.', + 'CODEX_OAUTH_LOGIN_FAILED', + ), + ); + finish({ exitCode: null }); + }; + try { + child = spawnImpl(executable, ['login'], { + cwd: home, + detached: true, + stdio: 'ignore', + env: codexLoginEnvironment(environment, loginHome), + }); + } catch { + fail(); + return; + } + + child.once('error', () => { + if (!spawned) fail(); + else finish({ exitCode: null }); + }); + child.once('exit', (exitCode) => finish({ exitCode })); + child.once('spawn', () => { + spawned = true; + child.unref?.(); + resolve({ + completion, + cancel() { + try { + child.kill?.('SIGTERM'); + } catch { + /* Already stopped. */ + } + }, + }); + }); + }); +} + +/** One local login owner shared by the launch and status endpoints. */ +export function createCodexOAuthLogin({ + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), + startOAuthLogin = () => startCodexChatGptLogin(), + loginTimeoutMs = 120_000, + setTimer = setTimeout, + clearTimer = clearTimeout, +} = {}) { + let active = null; + let failure = null; + let disposed = false; + + function credentials() { + try { + if (resolveOAuthAccessToken()) return { available: true }; + } catch (error) { + const code = error?.code || 'CODEX_OAUTH_UNAVAILABLE'; + return { + available: false, + error: error?.message || 'ChatGPT sign-in is unavailable', + code, + reauthRequired: code === 'CODEX_OAUTH_REAUTH_REQUIRED', + }; + } + return { + available: false, + error: 'ChatGPT sign-in is unavailable', + code: 'CODEX_OAUTH_UNAVAILABLE', + reauthRequired: false, + }; + } + + function status() { + const current = credentials(); + if (current.available) return current; + if (failure) return { available: false, ...failure, loginFailed: true }; + return active ? { ...current, pending: true } : current; + } + + function finish(attempt, error) { + if (active !== attempt) return; + clearTimer(attempt.timer); + active = null; + failure = error; + } + + async function start() { + if (disposed) + throw codexOAuthError( + 'ChatGPT sign-in is unavailable because the server has stopped.', + 'CODEX_OAUTH_LOGIN_FAILED', + ); + if (credentials().available) + return { available: true, started: false, pending: false }; + if (active) return { available: false, started: false, pending: true }; + + // Reserve the attempt before awaiting spawn so concurrent POSTs share it. + const attempt = { process: null, timer: null }; + active = attempt; + failure = null; + attempt.timer = setTimer(() => { + finish(attempt, { + code: 'CODEX_OAUTH_LOGIN_TIMEOUT', + error: + 'ChatGPT sign-in timed out. Click USE CHATGPT OAUTH to try again.', + }); + attempt.process?.cancel(); + }, loginTimeoutMs); + attempt.timer?.unref?.(); + + try { + const login = await startOAuthLogin(); + if (!login?.completion || typeof login.cancel !== 'function') + throw new Error('Invalid login process'); + attempt.process = login; + if (active !== attempt) { + login.cancel(); + return { ...status(), started: false, pending: false }; + } + Promise.resolve(login.completion).then( + (result) => { + if (active !== attempt) return; + if (result?.exitCode !== 0) { + finish(attempt, { + code: 'CODEX_OAUTH_LOGIN_FAILED', + error: + 'ChatGPT sign-in did not complete. Try again in Provider Settings, or run codex login separately.', + }); + } else if (!credentials().available) { + finish(attempt, { + code: 'CODEX_OAUTH_STORAGE_UNAVAILABLE', + error: + 'Codex sign-in finished, but its configured auth.json has no usable ChatGPT token. Check CODEX_HOME and Codex credential storage. GEV can read file storage only.', + }); + } else finish(attempt, null); + }, + () => + finish(attempt, { + code: 'CODEX_OAUTH_LOGIN_FAILED', + error: + 'ChatGPT sign-in did not complete. Try again in Provider Settings.', + }), + ); + return { available: false, started: true, pending: true }; + } catch (error) { + const code = + error?.code === 'CODEX_OAUTH_STORAGE_MISMATCH' + ? error.code + : 'CODEX_OAUTH_LOGIN_FAILED'; + finish(attempt, { + code, + error: + code === 'CODEX_OAUTH_STORAGE_MISMATCH' + ? error.message + : 'Could not start ChatGPT sign-in. Check that Codex is installed, or set CODEX_BIN to its executable.', + }); + return { ...status(), started: false, pending: false }; + } + } + + return { + status, + start, + dispose() { + disposed = true; + if (!active) return; + const attempt = active; + finish(attempt, null); + attempt.process?.cancel(); + }, + }; +} diff --git a/server/providers/openai/realtime.js b/server/providers/openai/realtime.js index b99f6c51f..6823c918d 100644 --- a/server/providers/openai/realtime.js +++ b/server/providers/openai/realtime.js @@ -14,12 +14,126 @@ import { } from './constants.js'; import { realtimeInstructions } from './instructions.js'; import { GEV_REALTIME_TOOLS } from './tools.js'; +import { + preferCodexOAuth, + readCodexOAuthAccessToken, + createCodexOAuthLogin, +} from './codex-auth.js'; + +const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']); + +function isLoopbackRequest(req) { + return LOOPBACK_ADDRESSES.has(String(req.socket?.remoteAddress || '')); +} + +function isSameOriginRequest(req) { + const origin = String(req.headers?.origin || '').trim(); + if (!origin) return true; + const host = String(req.headers?.host || '').trim(); + if (!host) return false; + const protocol = req.socket?.encrypted ? 'https:' : 'http:'; + try { + return new URL(origin).origin === new URL(`${protocol}//${host}`).origin; + } catch { + return false; + } +} + +function requestQuery(req) { + try { + return new URL(req.url || '', 'http://localhost').searchParams; + } catch { + return new URLSearchParams(); + } +} + +function resolveRealtimeAuthMode(query) { + const requested = String(query.get('auth') || 'api-key').toLowerCase(); + if (requested === 'api-key' || requested === 'oauth') return requested; + return null; +} + +function createRealtimeOAuthStatusHandler({ + oauthLogin = createCodexOAuthLogin(), +} = {}) { + return (req, res) => { + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Content-Type', 'application/json'); + if (req.method !== 'GET') { + res.statusCode = 405; + res.end(JSON.stringify({ error: 'Method not allowed' })); + return; + } + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'ChatGPT OAuth is available only from this machine', + }), + ); + return; + } + res.statusCode = 200; + res.end(JSON.stringify(oauthLogin.status())); + }; +} + +function createRealtimeOAuthLoginHandler({ + oauthLogin = createCodexOAuthLogin(), +} = {}) { + return async (req, res) => { + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Content-Type', 'application/json'); + if (req.method !== 'POST') { + res.statusCode = 405; + res.end(JSON.stringify({ error: 'Method not allowed' })); + return; + } + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'ChatGPT OAuth sign-in is available only from this machine', + }), + ); + return; + } + if (!isSameOriginRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'Cross-origin ChatGPT OAuth sign-in is not allowed', + }), + ); + return; + } + + try { + const result = await oauthLogin.start(); + res.statusCode = result.available ? 200 : result.loginFailed ? 503 : 202; + res.end(JSON.stringify(result)); + } catch { + res.statusCode = 503; + res.end( + JSON.stringify({ + available: false, + error: 'Could not start ChatGPT sign-in', + }), + ); + } + }; +} function createRealtimeTokenHandler({ annotationGuidance, endpoint = 'https://api.openai.com/v1/realtime/client_secrets', fetchImpl = (...args) => fetch(...args), resolveApiKey = () => process.env.OPENAI_API_KEY, + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), + environment = process.env, models = {}, tools = GEV_REALTIME_TOOLS, } = {}) { @@ -32,15 +146,84 @@ function createRealtimeTokenHandler({ return; } + const query = requestQuery(req); + const authMode = resolveRealtimeAuthMode(query); + if (!authMode) { + res.statusCode = 400; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'Unsupported cloud voice auth mode' })); + return; + } + + let oauthPinned; + try { + oauthPinned = preferCodexOAuth(environment); + } catch (error) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: error.message })); + return; + } + if (oauthPinned && authMode !== 'oauth') { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: + 'ChatGPT OAuth is required by GEV_PREFER_CODEX_OAUTH. Select USE CHATGPT OAUTH in Provider Settings.', + code: 'CODEX_OAUTH_REQUIRED', + }), + ); + return; + } + // Per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). On by default; 0 disables. if (!enforceRateLimit(openAiRateLimiter(), req, res)) return; - const apiKey = resolveApiKey(); - if (!apiKey) { - res.statusCode = 503; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' })); - return; + let bearer; + if (authMode === 'oauth') { + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: 'ChatGPT OAuth voice is available only from this machine', + }), + ); + return; + } + try { + bearer = resolveOAuthAccessToken(); + } catch (error) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: error?.message || 'ChatGPT sign-in is unavailable', + code: error?.code || 'CODEX_OAUTH_UNAVAILABLE', + }), + ); + return; + } + if (!bearer) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: 'ChatGPT sign-in is unavailable', + code: 'CODEX_OAUTH_UNAVAILABLE', + }), + ); + return; + } + } else { + bearer = resolveApiKey(); + if (!bearer) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' })); + return; + } } // Voice model tier, requested by the client as ?tier=standard|mini. @@ -49,15 +232,7 @@ function createRealtimeTokenHandler({ // bad querystring degrades to a normal session rather than a dead mic. // The env overrides stay authoritative per tier (see .env.example) — // a wrong upstream model id is then a config fix, not a code change. - const requestedTier = (() => { - try { - return new URL(req.url || '', 'http://localhost').searchParams.get( - 'tier', - ); - } catch { - return null; - } - })(); + const requestedTier = query.get('tier'); const tier = resolveVoiceModel(requestedTier).tier; const model = tier === 'mini' @@ -135,7 +310,7 @@ function createRealtimeTokenHandler({ redirect: 'error', signal: AbortSignal.timeout(30_000), headers: { - Authorization: `Bearer ${apiKey}`, + Authorization: `Bearer ${bearer}`, 'Content-Type': 'application/json', 'OpenAI-Safety-Identifier': 'gev-local-dev', }, @@ -149,6 +324,10 @@ function createRealtimeTokenHandler({ // case where a bogus ?tier= was silently downgraded to standard. res.setHeader('X-GEV-Voice-Tier', tier); res.setHeader('X-GEV-Voice-Model', model); + res.setHeader( + 'X-GEV-Voice-Auth', + authMode === 'oauth' ? 'codex-oauth' : 'env', + ); // Captions are billed separately; the client meters them with this id. res.setHeader( 'X-GEV-Voice-Transcribe-Model', @@ -183,4 +362,8 @@ function createRealtimeTokenHandler({ }; } -export { createRealtimeTokenHandler }; +export { + createRealtimeOAuthLoginHandler, + createRealtimeOAuthStatusHandler, + createRealtimeTokenHandler, +}; diff --git a/server/providers/openai/toolDescriptions.js b/server/providers/openai/toolDescriptions.js index 450c9da81..18de877db 100644 --- a/server/providers/openai/toolDescriptions.js +++ b/server/providers/openai/toolDescriptions.js @@ -179,7 +179,7 @@ export const ACTION_DESCRIPTIONS = { }, get_current_view_state: { description: - 'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action.', + 'Read the current camera, style, Context, Cockpit, HUD, detection, map stack, post-processing, scene-playback, tracked-entity, and layer state before choosing another action. Tracked aircraft emergency status is a crowd-sourced transponder broadcast, not a confirmed incident: quote emergencyBroadcast and never infer a hijack or an incident ending from a code change.', $position: 1, }, set_hud: { @@ -543,7 +543,7 @@ export const ACTION_DESCRIPTIONS = { }, analyst_query: { description: - 'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values).', + 'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values). Aircraft emergency fields describe broadcasts, not confirmed incidents: narrate emergencyBroadcast with emergencySource. emergencyKind values are general, unlawful, nordo, downed, minfuel and lifeguard; GEV treats the last two as priorities (emergency:false). lastContactAtMs says when the transponder was last heard, not when the status arrived.', $position: 1, parameters: { properties: { diff --git a/server/providers/overpass/cache.js b/server/providers/overpass/cache.js index b4e53ab21..442f599a6 100644 --- a/server/providers/overpass/cache.js +++ b/server/providers/overpass/cache.js @@ -5,6 +5,7 @@ import { OVERPASS_DISK_DIR, OVERPASS_CACHE_MS, OVERPASS_CACHE_MAX_ENTRIES, + cachedByConfiguredUpstream, } from './constants.js'; import path from 'node:path'; import { createHash } from 'node:crypto'; @@ -47,6 +48,11 @@ async function readOverpassDisk(cacheKey, maxAgeMs) { // Older versions persisted 4xx refusals with normal data TTLs. Ignore // them on both fresh and stale reads so an upgrade can recover immediately. if (!overpassPayloadIsData(payload)) return null; + // The key is the query alone, so a file written by a different upstream — + // including the public instances that were the default before #742 — is a + // hit for the same query. Refuse it: the operator's configured instance + // must be what answers, on the stale path too. + if (!cachedByConfiguredUpstream(payload)) return null; if (Date.now() - payload.cachedAt > maxAgeMs) return null; return payload; } catch { diff --git a/server/providers/overpass/constants.js b/server/providers/overpass/constants.js index 40f244d11..6cfd7257a 100644 --- a/server/providers/overpass/constants.js +++ b/server/providers/overpass/constants.js @@ -1,4 +1,5 @@ import path from 'node:path'; +import { createHash } from 'node:crypto'; // --------------------------------------------------------------------------- // Overpass API proxy constants and cache state @@ -27,16 +28,68 @@ function parseOverpassUpstreams(raw) { return endpoints.slice(0, 8); } -let upstreamMemo = { raw: null, endpoints: [] }; +let upstreamMemo = { raw: null, endpoints: [], tag: 'none' }; + +/** + * Short, stable identifier for the configured upstream set. Cached responses + * carry it so an answer produced by a different instance is not served after + * the operator repoints `OVERPASS_UPSTREAMS` — the disk cache outlives the + * configuration change by 7 days, a month for boundary queries, and forever on + * the serve-stale path. It is a hash, never the URLs, and it is only ever + * compared on read: it is not sent to clients and does not appear in errors. + */ +function upstreamTag(endpoints) { + if (!endpoints.length) return 'none'; + // Sorted, so reordering the same instances for failover keeps the cache. + return createHash('sha256') + .update([...endpoints].sort().join('\n')) + .digest('hex'); +} /** Resolve after environment loading. Public Overpass instances are not used by default. */ function resolveOverpassUpstreams() { - const raw = process.env.OVERPASS_UPSTREAMS || ''; - if (upstreamMemo.raw !== raw) - upstreamMemo = { raw, endpoints: parseOverpassUpstreams(raw) }; + readUpstreamMemo(); return [...upstreamMemo.endpoints]; } +/** The configured-upstream tag stored alongside, and checked against, cache entries. */ +function resolveOverpassUpstreamTag() { + return readUpstreamMemo().tag; +} + +let reportedForeignCache = false; + +/** + * Was this cached entry produced by the upstream set configured now? + * + * With nothing configured there is no instance to contradict, and serving + * last-good data at any age is the point of the cache (the proxy's + * not-configured path depends on it), so any entry is accepted. Once an + * operator does configure an instance, only its own answers are served: an + * entry with no tag, or one from a different instance, is refused rather than + * matched on the query alone. Shared by both caches fed by these upstreams. + */ +function cachedByConfiguredUpstream(entry) { + const tag = resolveOverpassUpstreamTag(); + if (tag === 'none' || entry?.upstreams === tag) return true; + if (!reportedForeignCache) { + reportedForeignCache = true; + console.warn( + '[Overpass] ignoring cached responses produced by a different OVERPASS_UPSTREAMS; they will be refetched from the configured instance.', + ); + } + return false; +} + +function readUpstreamMemo() { + const raw = process.env.OVERPASS_UPSTREAMS || ''; + if (upstreamMemo.raw !== raw) { + const endpoints = parseOverpassUpstreams(raw); + upstreamMemo = { raw, endpoints, tag: upstreamTag(endpoints) }; + } + return upstreamMemo; +} + /** * TTL for FRESH cached Overpass responses (ms). Road geometry is static for * months — the original 45 s TTL forced a public-mirror round-trip on nearly @@ -153,6 +206,8 @@ export { OVERPASS_MAX_RESPONSE_BYTES, parseOverpassUpstreams, resolveOverpassUpstreams, + resolveOverpassUpstreamTag, + cachedByConfiguredUpstream, OVERPASS_USER_AGENT, OVERPASS_TIMEOUT_MS, }; diff --git a/server/providers/overpass/transport.js b/server/providers/overpass/transport.js index 86b9eae15..38c31bb08 100644 --- a/server/providers/overpass/transport.js +++ b/server/providers/overpass/transport.js @@ -1,6 +1,7 @@ import { OVERPASS_MAX_RESPONSE_BYTES, resolveOverpassUpstreams, + resolveOverpassUpstreamTag, OVERPASS_USER_AGENT, OVERPASS_TIMEOUT_MS, } from './constants.js'; @@ -187,6 +188,9 @@ async function fetchOverpassPayload( contentType: 'application/json', // Never retain a secret-bearing endpoint in cache or response metadata. endpoint: 'configured', + // Which configured instance produced this, as a hash rather than a URL, + // so a cached answer is not reused after the operator repoints. + upstreams: resolveOverpassUpstreamTag(), rateLimited: false, }; } catch { diff --git a/server/providers/places.js b/server/providers/places.js index 12f0688c6..9c9e88645 100644 --- a/server/providers/places.js +++ b/server/providers/places.js @@ -1,8 +1,13 @@ /** Place-search and route middleware for Node development servers. */ -export { googlePlacesContextProxy } from './places/google.js'; +export { + googlePlacesContextProxy, + validateGeocodeAddress, + geocodeBounds, +} from './places/google.js'; export { googleServerApiKey, keylessGooglePlacesResponse, + keylessGoogleGeocodeResponse, } from './places/google-key.js'; export { installRouteMiddleware } from './places/routes.js'; export { diff --git a/server/providers/places/google-key.js b/server/providers/places/google-key.js index 353bc9e24..ff5fe6356 100644 --- a/server/providers/places/google-key.js +++ b/server/providers/places/google-key.js @@ -14,12 +14,34 @@ export function keylessGooglePlacesResponse(apiKey) { } /** - * Google API key for the SERVER-SIDE calls (Places nearby/text search, the - * CCTV Street View fallback). These never reach the browser, so this key can - * be restricted by server IP and scoped to Places API + Street View Static - * API — while GOOGLE_MAPS_API_KEY stays referrer-restricted to Map Tiles + - * Geocoding for the browser (#33). Splitting them is opt-in: unset, this - * falls back to the shared browser key and nothing changes. + * Geocoding speaks Google's own response shape rather than the `places: []` + * contract its two siblings use, because the browser adapters read `status` + * and `results` directly. The keyless capability reads the same way: a status + * that is not `OK` and no results, which the adapters already treat as "this + * provider did not answer" and pass to the next geocoder in the chain. + */ +export function keylessGoogleGeocodeResponse(apiKey) { + if (String(apiKey ?? '').trim()) return null; + return { + statusCode: 200, + payload: { + configured: false, + status: 'REQUEST_DENIED', + results: [], + error: null, + }, + }; +} + +/** + * Google API key for the SERVER-SIDE calls (Places nearby/text search, + * geocoding, the CCTV Street View fallback). These never reach the browser, so + * this key can be restricted by server IP and scoped to Places API + Geocoding + * API + Street View Static API — while GOOGLE_MAPS_API_KEY stays + * referrer-restricted to Map Tiles + Maps JavaScript API for the browser + * (#33, #363). Splitting them is opt-in: unset, this falls back to the shared + * browser key, which then has to carry every API both sides use and cannot + * take a referrer restriction. */ export function googleServerApiKey() { return resolveGoogleServerKey(process.env); diff --git a/server/providers/places/google.js b/server/providers/places/google.js index a2225c606..1ddaa9593 100644 --- a/server/providers/places/google.js +++ b/server/providers/places/google.js @@ -1,9 +1,11 @@ import { googleServerApiKey, keylessGooglePlacesResponse, + keylessGoogleGeocodeResponse, } from './google-key.js'; import { makeCostRateLimiter, clientKey } from '../common/rate-limit.js'; import { admitSameSite } from '../common/same-site.js'; +import { readResponseJsonCapped } from '../common/http.js'; import { projectNearbyPlaces, projectTextSearchPlaces, @@ -25,7 +27,7 @@ let _googleRateLimiter; */ export const GOOGLE_DEFAULT_PER_MIN = 120; -/** Google cost endpoints (nearby-places + text-search). Null only when set to 0. */ +/** Google cost endpoints (nearby-places, text-search, geocoding). Null only when set to 0. */ function googleRateLimiter() { if (_googleRateLimiter === undefined) _googleRateLimiter = makeCostRateLimiter( @@ -56,6 +58,51 @@ export function validatePlacesCoordinates(searchParams) { return { ok: true, latitude, longitude }; } +/** Longest accepted geocoder query. Real place names are far shorter. */ +const GEOCODE_MAX_QUERY = 200; + +/** + * Upstream deadline and body ceiling for the geocode routes. The deadline stays + * armed through body consumption, so a stalled upstream cannot hold a handler + * open, and the cap bounds a chunked or length-omitted reply. A geocode answer + * is a handful of results; 256 KiB is far above any honest one. + */ +const GEOCODE_UPSTREAM_TIMEOUT_MS = 10_000; + +const GEOCODE_MAX_RESPONSE_BYTES = 256 * 1024; + +/** Validate a free-text geocoder query before consuming request quota. */ +export function validateGeocodeAddress(searchParams) { + const address = String(searchParams.get('address') || '').trim(); + if (!address) return { ok: false, error: 'address is required' }; + if (address.length > GEOCODE_MAX_QUERY) + return { ok: false, error: 'address is too long' }; + return { ok: true, address }; +} + +/** + * Validate the optional viewport bias, which Google reads as + * `sw_lat,sw_lng|ne_lat,ne_lng`. An unparsed value is dropped rather than + * refused: the bias only ranks results, so a malformed one costs relevance, + * not an answer. + */ +export function geocodeBounds(searchParams) { + const raw = String(searchParams.get('bounds') || '').trim(); + if (!raw) return null; + const corners = raw.split('|'); + if (corners.length !== 2) return null; + const parts = corners.flatMap((corner) => corner.split(',')); + // A blank component must be refused before Number() turns it into 0 and a + // half-written corner like `30.4,` reads as the equator. + if (parts.length !== 4 || parts.some((part) => !part.trim())) return null; + const numbers = parts.map(Number); + if (!numbers.every(Number.isFinite)) return null; + const [swLat, swLon, neLat, neLon] = numbers; + if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null; + if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null; + return raw; +} + /** Nearby place labels and view-biased text search, with request-time key resolution. */ export function googlePlacesContextProxy({ resolveApiKey = googleServerApiKey, @@ -293,6 +340,149 @@ export function googlePlacesContextProxy({ ); } }); + + /** + * Geocoding belongs on the server, not in the page. Google's Geocoding web + * service refuses referrer-restricted keys, so calling it from the browser + * forces GOOGLE_MAPS_API_KEY — which ships in the bundle by design — to be + * left unrestricted (#363). Here the request carries the server key, and + * the browser key no longer needs the Geocoding API at all. + * + * Unlike the two Places routes above, these answer in Google's own shape + * (`status` + `results`) rather than the `places: []` contract: the browser + * normalizers read that shape directly, and keeping it means the proxy adds + * a hop without adding a translation nobody asked for. + */ + function installGeocodeRoute(path, readQuery) { + middlewares.use(path, async (req, res) => { + const refuse = (statusCode, error) => { + res.statusCode = statusCode; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ status: 'REQUEST_DENIED', results: [], error }), + ); + }; + + // Gate first, like the Places routes above: a cross-site caller learns + // nothing about this endpoint's method surface. + if (admitSameSite(req, res)) return; + if (req.method !== 'GET') { + refuse(405, 'Method not allowed'); + return; + } + + // A keyless geocode has no provider cost, so it resolves before the + // paid-endpoint limiter can consume or exhaust quota (mirrors the + // Places routes above). + const apiKey = resolveApiKey(); + const keyless = keylessGoogleGeocodeResponse(apiKey); + if (keyless) { + res.statusCode = keyless.statusCode; + res.setHeader('Content-Type', 'application/json'); + res.setHeader('Cache-Control', 'no-store'); + res.end(JSON.stringify(keyless.payload)); + return; + } + + const requestUrl = new URL(req.url || '', 'http://localhost'); + const query = readQuery(requestUrl.searchParams); + if (!query.ok) { + refuse(400, query.error); + return; + } + + // Per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN), shared with the + // Places routes because both spend the same Google budget. + const _grl = googleRateLimiter(); + if (_grl && !_grl(clientKey(req))) { + res.statusCode = 429; + res.setHeader('Content-Type', 'application/json'); + res.setHeader('Retry-After', '5'); + res.end( + JSON.stringify({ + status: 'OVER_QUERY_LIMIT', + results: [], + error: 'Rate limit exceeded', + }), + ); + return; + } + + try { + const upstream = new URL( + endpoints.geocode || + 'https://maps.googleapis.com/maps/api/geocode/json', + ); + for (const [name, value] of Object.entries(query.params)) + upstream.searchParams.set(name, value); + upstream.searchParams.set('key', apiKey); + const deadline = AbortSignal.timeout(GEOCODE_UPSTREAM_TIMEOUT_MS); + const response = await fetchImpl(upstream.toString(), { + redirect: 'error', + signal: deadline, + }); + // An unreadable, oversized or timed-out body is not an answer: it + // falls through to the same shaped failure as a refusal below. + const data = await readResponseJsonCapped( + response, + GEOCODE_MAX_RESPONSE_BYTES, + deadline, + ).catch(() => null); + + // A body we could not read is an upstream failure, not an answer with + // a status of its own, so it does not borrow the upstream's code. + if (!data) { + res.statusCode = 502; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end( + JSON.stringify({ + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }), + ); + return; + } + + res.statusCode = response.ok ? 200 : response.status; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.setHeader('Cache-Control', 'private, max-age=300'); + res.end(JSON.stringify(data)); + } catch { + // Fixed public text: a transport, URL or runtime message would + // describe the server to the browser (SECURITY.md, sanitized errors). + res.statusCode = 502; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end( + JSON.stringify({ + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }), + ); + } + }); + } + + installGeocodeRoute('/api/google/geocode', (searchParams) => { + const address = validateGeocodeAddress(searchParams); + if (!address.ok) return address; + const bounds = geocodeBounds(searchParams); + return { + ok: true, + params: { + address: address.address, + ...(bounds ? { bounds } : {}), + }, + }; + }); + + installGeocodeRoute('/api/google/reverse-geocode', (searchParams) => { + const coordinates = validatePlacesCoordinates(searchParams); + if (!coordinates.ok) return coordinates; + const { latitude, longitude } = coordinates; + return { ok: true, params: { latlng: `${latitude},${longitude}` } }; + }); } return { diff --git a/server/providers/radio/transport.js b/server/providers/radio/transport.js index 7bbe33bcd..c4b7f2b5b 100644 --- a/server/providers/radio/transport.js +++ b/server/providers/radio/transport.js @@ -1,6 +1,9 @@ import https from 'node:https'; import { Readable } from 'node:stream'; import { isNonGlobalIpv4 } from './stations.js'; + +const NULL_BODY_STATUSES = new Set([204, 205, 304]); + export function radioMirrorOrigin(value) { const hostname = String(value ?? '') .toLowerCase() @@ -124,19 +127,33 @@ export function fetchPinnedRadioResponse(url, options, addresses) { }, }, (response) => { - const headers = new Headers(); - for (const [name, value] of Object.entries(response.headers)) { - if (Array.isArray(value)) - value.forEach((item) => headers.append(name, item)); - else if (value !== undefined) headers.set(name, String(value)); + // Nothing thrown here may escape: this callback runs outside the + // promise, so an exception would be uncaught and crash the server. + try { + const status = response.statusCode || 500; + // A `Response` only represents statuses up to 599; Node never hands + // this callback a 1xx. + if (status > 599) + throw new Error( + `Radio Browser returned unsupported status ${status}`, + ); + const headers = new Headers(); + for (const [name, value] of Object.entries(response.headers)) { + if (Array.isArray(value)) + value.forEach((item) => headers.append(name, item)); + else if (value !== undefined) headers.set(name, String(value)); + } + const nullBody = NULL_BODY_STATUSES.has(status); + const upstream = new Response( + nullBody ? null : Readable.toWeb(response), + { status, statusText: response.statusMessage || '', headers }, + ); + if (nullBody) response.resume(); + resolve(upstream); + } catch (error) { + response.destroy(); + reject(error); } - resolve( - new Response(Readable.toWeb(response), { - status: response.statusCode || 500, - statusText: response.statusMessage || '', - headers, - }), - ); }, ); request.on('error', reject); diff --git a/server/providers/space/celestrak.js b/server/providers/space/celestrak.js index a76cc590d..8baf59ba7 100644 --- a/server/providers/space/celestrak.js +++ b/server/providers/space/celestrak.js @@ -1,6 +1,7 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js'; +import { readResponseTextCapped } from '../common/http.js'; /** * Vite plugin: CelesTrak TLE proxy. @@ -48,10 +49,14 @@ export function celestrakProxy() { } } + /** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */ + const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024; + async function fetchUpstream(group) { const url = celestrakTleUrl(group); + const signal = AbortSignal.timeout(20000); const res = await fetch(url.toString(), { - signal: AbortSignal.timeout(20000), + signal, // CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a // descriptive User-Agent with a contact point. headers: { @@ -60,7 +65,11 @@ export function celestrakProxy() { }, }); if (!res.ok) throw new Error(`HTTP ${res.status}`); - const body = await res.text(); + const body = await readResponseTextCapped( + res, + CELESTRAK_MAX_RESPONSE_BYTES, + signal, + ); // An upstream error page parses to zero TLEs — treat as failure, keep cache. if (!/^1 /m.test(body)) throw new Error('no TLE lines in response'); return { at: Date.now(), body }; diff --git a/server/providers/traffic.js b/server/providers/traffic.js index 2a3c2e60a..d53db6530 100644 --- a/server/providers/traffic.js +++ b/server/providers/traffic.js @@ -1,6 +1,7 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; +import { readResponseBytesCapped } from './common/http.js'; import { isValidTileCoord as isValidTomTomTile, utcDayKey as tomtomUtcDayKey, @@ -48,6 +49,8 @@ export function tomtomProxy() { const DEFAULT_DAILY_BUDGET = 6000; const MEM_MAX_ENTRIES = 256; const UPSTREAM_TIMEOUT_MS = 15000; + /** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */ + const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; /** @type {Map} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */ const mem = new Map(); @@ -143,11 +146,15 @@ export function tomtomProxy() { 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' + `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`; recordUpstreamFetch(); // attempts count — upstream bills the request either way - const res = await fetch(url, { - signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), - }); + const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS); + const res = await fetch(url, { signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); - const buf = Buffer.from(await res.arrayBuffer()); + const bytes = await readResponseBytesCapped( + res, + TOMTOM_TILE_MAX_RESPONSE_BYTES, + signal, + ); + const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); if (buf.length === 0) throw new Error('empty tile body'); return buf; } diff --git a/server/providers/transitHistory.test.mjs b/server/providers/transitHistory.test.mjs index 6ea9209f1..cb06c182a 100644 --- a/server/providers/transitHistory.test.mjs +++ b/server/providers/transitHistory.test.mjs @@ -142,9 +142,11 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde const rows = Array.from({ length: 15000 }, (_, i) => row(now, `vehicle-${i}`), ); - const start = performance.now(); + // Main-thread CPU: excludes scheduler waits and parallel GC helper threads. + const start = process.threadCpuUsage(); h.ingest(feed, rows, now); - times.push(performance.now() - start); + const usage = process.threadCpuUsage(start); + times.push((usage.user + usage.system) / 1000); } assert.equal(h.diagnostics().keys, 15000); for (let i = 0; i < 15000; i++) { @@ -154,12 +156,12 @@ test('three identical-order 15000-vehicle snapshots retain paths within a bounde } assert.ok(h.diagnostics().allocatedBytes < 16 * 1024 * 1024); t.diagnostic( - `15000-vehicle ingestion ms: ${times.map((n) => n.toFixed(1)).join('/')}`, + `15000-vehicle ingestion CPU ms: ${times.map((n) => n.toFixed(1)).join('/')}`, ); - // A ceiling that says "not quadratic", not a benchmark: 15,000 rows in - // well under two seconds on a loaded laptop or a slow CI runner. The - // per-run figure above is the diagnostic to read. - assert.ok(Math.max(...times) < 2000, `ingestion ms: ${times}`); + // CPU time measures ingestion work without counting scheduler delays on + // loaded runners. Keep the two-second ceiling as a guard against quadratic + // work, not a benchmark; the per-poll figures above show any slowdown. + assert.ok(Math.max(...times) < 2000, `ingestion CPU ms: ${times}`); }); test('snapshot capacity pressure preserves members not yet visited and discloses recreated history', (t) => { diff --git a/src/annotations/annotationEngine.test.mjs b/src/annotations/annotationEngine.test.mjs index 813163d8c..5f39a077a 100644 --- a/src/annotations/annotationEngine.test.mjs +++ b/src/annotations/annotationEngine.test.mjs @@ -313,7 +313,9 @@ test('retry: a plain HTTP 500 still exhausts the existing 8s/25s transient ladde t.mock.timers.enable({ apis: ['setTimeout', 'Date'], now: 100_000 }); globalThis.window = { setTimeout: globalThis.setTimeout, clearTimeout: globalThis.clearTimeout }; const requestTimes = []; - globalThis.fetch = async () => { + globalThis.fetch = async (url) => { + if (String(url).endsWith('/status')) + return Response.json({ configured: true }); requestTimes.push(Date.now()); return httpFailure(500); }; @@ -402,7 +404,7 @@ test('outline upgrade updates the rendered element in place without remove/add', }; let overpassCall = 0; globalThis.fetch = async (url) => { - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => ({ status: 'OK', results: [{ diff --git a/src/annotations/annotationResolver.test.mjs b/src/annotations/annotationResolver.test.mjs index 452d6997c..2c51213c1 100644 --- a/src/annotations/annotationResolver.test.mjs +++ b/src/annotations/annotationResolver.test.mjs @@ -200,7 +200,7 @@ test('ask-side admin bypass: "the Texas Capitol" recovers near-view despite a fa const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -235,7 +235,7 @@ test('ask-side admin bypass: explicit "state of Texas" skips recovery and proxim const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - assert.match(String(url), /^https:\/\/maps\.googleapis\.com\/maps\/api\/geocode/); + assert.match(String(url), /^\/api\/google\/geocode\?/); return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -273,7 +273,7 @@ for (const fixture of [ const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: fixture.lat, lon: fixture.lon, @@ -299,7 +299,7 @@ test('ask-side admin bypass: bare "Texas" remains on the guarded recovery path', const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -328,7 +328,7 @@ test('ask-side admin bypass: admin level 2/3 result types never grant a township const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { const query = new URL(String(url)).searchParams.get('address'); return { json: async () => geocodePayload({ lat: 39.7817, @@ -380,7 +380,7 @@ function installCapitolMocks(t, elements, components = [ { long_name: 'Capitol Hill', types: ['neighborhood', 'political'] }, ]) { installGoogleMocks(t, async (url) => { - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => ({ status: 'OK', results: [{ formatted_address: 'Washington, DC 20004, USA', types: ['establishment', 'landmark', 'point_of_interest', 'tourist_attraction'], @@ -458,7 +458,7 @@ test('keyless: a key that geocodes to nothing still anchors the annotation', asy const requests = []; installGoogleMocks(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (String(url).startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; return { ok: true, json: async () => ({ @@ -495,7 +495,7 @@ test('keyless: a key that geocodes to nothing still anchors the annotation', asy high: { latitude: 30.28, longitude: -97.68 }, }); // Google is asked first and exactly once; one Photon call answers it. - assert.equal(requests.filter((url) => url.includes('maps.googleapis.com')).length, 1); + assert.equal(requests.filter((url) => url.includes('/api/google/geocode')).length, 1); assert.equal(requests.filter((url) => url.includes('photon.komoot.io')).length, 1); }); @@ -512,7 +512,7 @@ test('keyless: OSM matching uses the feature\'s canonical name, not the user\'s installGoogleMocks(t, async (url, init) => { const href = String(url); - if (href.startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (href.startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; if (href.startsWith('/api/google/text-search')) return { ok: true, json: async () => ({ places: [] }) }; if (href.startsWith('https://photon.komoot.io/')) { return { @@ -575,7 +575,7 @@ test('keyless: a Photon outage is not remembered as "no such place"', async (t) const requests = []; installGoogleMocks(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (String(url).startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; if (!photonReachable) throw new Error('network down'); return photonUp; }); diff --git a/src/annotations/regionRing.test.mjs b/src/annotations/regionRing.test.mjs index 09814247b..cd8ad6447 100644 --- a/src/annotations/regionRing.test.mjs +++ b/src/annotations/regionRing.test.mjs @@ -51,9 +51,11 @@ test('region ring: Natural Earth names resolve without a geocoder', async () => test('region ring: a slow admin-boundary lookup returns region-timeout within the budget', async () => { let adminSignal; let release; + let lookupStarted; const { resolveRegionRingForQuery } = createAnnotationResolver({ featureSource: featureSource({ getAdministrativeAreas(_point, { signal }) { + lookupStarted = Date.now(); adminSignal = signal; return new Promise((resolve) => { release = resolve; @@ -61,7 +63,6 @@ test('region ring: a slow admin-boundary lookup returns region-timeout within th }, }), }); - const started = Date.now(); const region = await resolveRegionRingForQuery( 'Atlantis', undefined, @@ -73,8 +74,10 @@ test('region ring: a slow admin-boundary lookup returns region-timeout within th ring: null, error: 'region-timeout', }); + // Timed from the boundary lookup, so the bundled pack scans before it do not + // count. Under the 3 s default budget, so an ignored budgetMs still fails. assert.ok( - Date.now() - started < 1000, + Date.now() - lookupStarted < 2_000, 'returns at the budget, not the lookup', ); // The lookup is left running so it can fill the boundary cache. diff --git a/src/app/catalog.test.mjs b/src/app/catalog.test.mjs index fc51c7fcd..ff6d73db5 100644 --- a/src/app/catalog.test.mjs +++ b/src/app/catalog.test.mjs @@ -65,13 +65,28 @@ test('data setup seals the caller catalog before controls can restore and drains }, }, }, - catalog: createLayerCatalog(layers, metadata(layers)), + catalog: { + ...createLayerCatalog(layers, metadata(layers)), + getSourceAvailability: (id) => + id === 'one' + ? { + available: false, + reason: 'One: data source not configured', + } + : undefined, + }, defer: (release) => releases.push(release), onData: (manager) => { observed = manager; }, }); assert.equal(observed, dataManager); + assert.equal(await dataManager.setEnabled('one', true), false); + assert.equal(dataManager.layers.get('one').initialized, false); + assert.equal( + dataManager.getAll().find(({ id }) => id === 'one').stats.error, + 'One: data source not configured', + ); assert.deepEqual(attached, [['one', 'two']]); assert.deepEqual([...dataManager.layers.keys()], ['one', 'two']); assert.throws(() => dataManager.register({ id: 'late' }), /finalized/); diff --git a/src/app/constructCatalog.js b/src/app/constructCatalog.js index 0a8ef4fe8..832c174bc 100644 --- a/src/app/constructCatalog.js +++ b/src/app/constructCatalog.js @@ -1,9 +1,9 @@ +import { resolveCatalogSources } from './sourceComposition.js'; import { createWeatherClock } from '../layers/weather/clock.js'; import { createWeatherLayer } from '../layers/weather/index.js'; import { createCyclonesLayer } from '../layers/cyclones/index.js'; import { createWindLayer } from '../layers/wind/index.js'; import { createLayerCatalog } from './catalog.js'; -import { MAPILLARY_SOURCE_METHODS } from '../layers/streetLevel/providers/mapillary/policy.js'; import { LAYER_STATE_REGISTRY } from '../data/layerState.js'; import { createMilitaryRegistry } from '../layers/aircraft/classification.js'; import { createApplicationFlights } from './layers/flights.js'; @@ -24,7 +24,9 @@ import { createApplicationLocalAdsb } from './layers/localAdsb.js'; import { createApplicationAwareness } from './layers/militaryAwareness.js'; import { createApplicationFirms } from './layers/firms.js'; import { createApplicationEarthquakes } from './layers/earthquakes.js'; +import { createApplicationAirQuality } from './layers/airQuality.js'; import { createApplicationFirePerimeters } from './layers/perimeters.js'; +import { createApplicationGnssIntegrity } from './layers/gnss.js'; import { createApplicationCables } from './layers/submarineCables.js'; import { createApplicationStreetLevel } from './layers/streetLevel.js'; import { createInfrastructureLayers } from '../data/infrastructure.js'; @@ -32,34 +34,6 @@ import { localGeoJsonServices } from './localGeojsonServices.js'; import { createBhoteKoshiEventLayer } from '../data/bhoteKoshiEvent.js'; import { createBhoteKoshiLocatorLayer } from '../data/bhoteKoshiLocator.js'; -const SOURCE_METHODS = Object.freeze({ - flights: ['getSnapshot'], - military: ['getSnapshot'], - vessels: ['getSnapshot'], - cctv: ['getCatalog', 'getHealth', 'getFrameUrl', 'getMediaUrl'], - radio: ['getDirectory', 'recordClick'], - traffic: [ - 'requestRoads', - 'getStatus', - 'fetchFlowForBounds', - 'getFlowSessionStats', - 'resetFlowTileCache', - ], - bikeshare: ['getStations'], - installations: ['getMappedSites', 'searchNearby'], - satellites: ['readGroup'], - launches: ['getLaunches', 'getActiveTle'], - alpr: ['fetch'], - firms: ['getSnapshot'], - wind: ['getSnapshot'], - weather: ['getSnapshot'], - cyclones: ['getSnapshot'], - earthquakes: ['getSnapshot'], - 'fire-perimeters': ['getSnapshot'], - cables: ['fetch'], - mapillary: MAPILLARY_SOURCE_METHODS, -}); - /** * Hardware-local layers are registered like any other but never enter share * links or stored layer state: another browser cannot have this receiver. @@ -86,6 +60,7 @@ export function createApplicationCatalog({ vesselOptions, resolveAsset, nepalBoundaryResolver, + streetLevelProviders, }) { if (!signal?.addEventListener) throw new TypeError('An application lifetime signal is required'); @@ -93,12 +68,8 @@ export function createApplicationCatalog({ if (!surface?.groundFloor || !surface?.terrain) throw new TypeError('Application surface services are required'); - for (const [name, methods] of Object.entries(SOURCE_METHODS)) { - if ( - methods.some((method) => typeof sources?.[name]?.[method] !== 'function') - ) - throw new TypeError(`Invalid catalog source: ${name}`); - } + const sourceComposition = resolveCatalogSources(sources); + sources = sourceComposition.sources; const militaryRegistry = createMilitaryRegistry(); const weatherClock = createWeatherClock(); const dispose = () => { @@ -108,7 +79,8 @@ export function createApplicationCatalog({ }; signal.addEventListener('abort', dispose, { once: true }); try { - militaryRegistry.configureSource(sources.military, { signal }); + if (sourceComposition.isConfigured('military')) + militaryRegistry.configureSource(sources.military, { signal }); const flights = createApplicationFlights({ surface, source: sources.flights, @@ -147,13 +119,18 @@ export function createApplicationCatalog({ ...(resolveAsset ? { resolveAsset } : {}), }), createApplicationEarthquakes({ source: sources.earthquakes }), + createApplicationAirQuality({ source: sources['air-quality'] }), createApplicationFirePerimeters({ source: sources['fire-perimeters'], }), + createApplicationGnssIntegrity({ + source: sources['gnss-integrity'], + }), createApplicationAlpr({ surface, source: sources.alpr }), createApplicationStreetLevel({ surface, sources: { mapillary: sources.mapillary }, + providers: streetLevelProviders, }), satellites, createApplicationLaunches({ source: sources.launches, satellites }), @@ -162,8 +139,8 @@ export function createApplicationCatalog({ createApplicationRadio({ surface, source: sources.radio }), createApplicationTransit({ surface, source: sources.transit }), createApplicationBikeshare({ source: sources.bikeshare }), - createApplicationDirections(), - createApplicationRecentImagery(), + createApplicationDirections({ source: sources.directions }), + createApplicationRecentImagery({ source: sources['recent-imagery'] }), vessels, installations, createApplicationAwareness({ @@ -204,6 +181,9 @@ export function createApplicationCatalog({ ); return Object.freeze({ ...catalog, + getSourceAvailability: sourceComposition.createAvailabilityLookup( + catalog.layers, + ), militaryRegistry, surface, weatherClock, diff --git a/src/app/constructCatalog.test.mjs b/src/app/constructCatalog.test.mjs index 3a802edbe..9e1059c33 100644 --- a/src/app/constructCatalog.test.mjs +++ b/src/app/constructCatalog.test.mjs @@ -5,6 +5,7 @@ import { createApplicationCatalog } from './constructCatalog.js'; import { createStandaloneLayerSources } from '../standalone/layerSources.js'; import { catalogControlServices } from './catalog.js'; import { LayerLifecycle } from '../data/lifecycle.js'; +import { fakeStreetLevelProvider } from '../testSupport/streetLevelFakes.mjs'; function fixtureSources(ids, calls) { const sources = createStandaloneLayerSources(); @@ -39,7 +40,7 @@ test('catalogs construct distinct layers and classification from their supplied signal: b.signal, surface: fixtureSurface(b.signal), }); - assert.equal(first.layers.length, 30); + assert.equal(first.layers.length, 32); assert.ok(first.get('local-adsb'), 'Local ADS-B is registered'); assert.deepEqual( first.metadata.find(({ id }) => id === 'local-adsb'), @@ -59,6 +60,7 @@ test('catalogs construct distinct layers and classification from their supplied '2026-09-21T12:00:00.000Z', ); assert.ok(first.get('fire-perimeters')); + assert.ok(first.get('gnss-integrity')); assert.ok(first.get('transit')); const order = first.layers.map(({ id }) => id); assert.deepEqual( @@ -116,7 +118,7 @@ test('invalid or already cancelled construction fails before classification can assert.throws( () => createApplicationCatalog({ - sources: {}, + sources: { flights: {} }, signal: lifetime.signal, surface: fixtureSurface(lifetime.signal), }), @@ -140,3 +142,53 @@ function fixtureSurface(signal) { eventTarget: null, }); } + +test('Mapillary can be omitted or replaced without changing catalog membership', (t) => { + const lifetime = new AbortController(); + t.after(() => lifetime.abort()); + const sources = fixtureSources([], []); + delete sources.mapillary; + const options = { + sources, + signal: lifetime.signal, + surface: fixtureSurface(lifetime.signal), + }; + const catalog = createApplicationCatalog(options); + assert.equal(catalog.layers.length, 32); + assert.deepEqual(catalog.get('street-level').providerIds, []); + assert.equal( + catalogControlServices(catalog).streetLevelLayer, + catalog.get('street-level'), + ); + assert.match( + catalog.get('street-level').getUIState().coverage.hint, + /No street-level imagery providers/, + ); + const provider = fakeStreetLevelProvider({ + id: 'example', + pickPrefix: 'example:', + }); + const replacement = createApplicationCatalog({ + ...options, + streetLevelProviders: [provider], + }); + assert.deepEqual(replacement.get('street-level').providerIds, ['example']); + assert.deepEqual(replacement.metadata, catalog.metadata); + assert.equal(provider.calls.activate, 0, 'construction must not acquire'); + assert.deepEqual( + createApplicationCatalog({ + ...options, + sources: fixtureSources([], []), + streetLevelProviders: [], + }).get('street-level').providerIds, + [], + ); + assert.throws( + () => + createApplicationCatalog({ + ...options, + sources: { ...sources, mapillary: {} }, + }), + /Mapillary source/, + ); +}); diff --git a/src/app/data.js b/src/app/data.js index 85a3183fc..ce3ad0731 100644 --- a/src/app/data.js +++ b/src/app/data.js @@ -13,6 +13,7 @@ export function createApplicationData({ // Initialize data layer manager const dataManager = new LayerLifecycle(viewer, { allowQaRegistration, + getSourceAvailability: catalog?.getSourceAvailability, }); defer(async () => { await dataManager.destroyAll(); diff --git a/src/app/layers/airQuality.js b/src/app/layers/airQuality.js new file mode 100644 index 000000000..1f25c06d9 --- /dev/null +++ b/src/app/layers/airQuality.js @@ -0,0 +1,6 @@ +import { createAirQualityLayer } from '../../layers/airQuality/index.js'; +import { overlayHost } from './overlayHost.js'; +/** Wire air-quality readings to the application overlay host. */ +export function createApplicationAirQuality(options) { + return createAirQualityLayer({ overlayHost, ...options }); +} diff --git a/src/app/layers/directions.js b/src/app/layers/directions.js index 1e535fc88..8b16c5cff 100644 --- a/src/app/layers/directions.js +++ b/src/app/layers/directions.js @@ -1,12 +1,6 @@ import { createApplicationDirectionsLayer } from '../../data/directions.js'; -/** - * Construct one Directions layer using the application scene owners. - * - * It takes no data source: a route is fetched from `/api/route` when the - * operator places both endpoints, so there is nothing to supply up front. - * @returns {object} A fresh layer instance for this catalog. - */ -export function createApplicationDirections() { - return createApplicationDirectionsLayer(); +/** Construct Directions with application scene owners and an explicit route source. */ +export function createApplicationDirections({ source }) { + return createApplicationDirectionsLayer({ source }); } diff --git a/src/app/layers/gnss.js b/src/app/layers/gnss.js new file mode 100644 index 000000000..46983953b --- /dev/null +++ b/src/app/layers/gnss.js @@ -0,0 +1,6 @@ +import { createGnssIntegrityLayer } from '../../layers/gnss/index.js'; + +/** Wire the adsb.lol GNSS navigation-integrity layer into the application catalog. */ +export function createApplicationGnssIntegrity(options) { + return createGnssIntegrityLayer(options); +} diff --git a/src/app/layers/recentImagery.js b/src/app/layers/recentImagery.js index 1de4b22d4..cc4659727 100644 --- a/src/app/layers/recentImagery.js +++ b/src/app/layers/recentImagery.js @@ -3,17 +3,17 @@ import { createRecentImageryRenderer } from '../../layers/recentImagery/renderin import { createThumbnailLoader } from '../../layers/recentImagery/thumbnails.js'; import { governorRequestRender } from '../../renderGovernor.js'; -/** - * Construct the Recent Imagery layer with the real GIBS renderer and the - * browser thumbnail loader. It takes no data source: the catalog is NASA - * CMR and the tiles are GIBS, both browser-direct and keyless. - * @returns {object} A fresh layer instance for this catalog. - */ -export function createApplicationRecentImagery() { +/** Construct imagery presentation over one explicitly supplied acquisition source. */ +export function createApplicationRecentImagery({ source }) { return createRecentImageryLayer({ + catalog: { searchHls: (...args) => source.searchHls(...args) }, renderer: createRecentImageryRenderer({ requestRender: governorRequestRender, + tileTemplate: (...args) => source.getTileTemplate(...args), + credit: source.credit || source.label || 'Imagery', + }), + thumbnails: createThumbnailLoader({ + requestThumbnail: (...args) => source.getThumbnail(...args), }), - thumbnails: createThumbnailLoader(), }); } diff --git a/src/app/layers/streetLevel.js b/src/app/layers/streetLevel.js index 56ba06590..b441d12d7 100644 --- a/src/app/layers/streetLevel.js +++ b/src/app/layers/streetLevel.js @@ -1,18 +1,48 @@ import { createStreetLevelLayer } from '../../layers/streetLevel/index.js'; import { createMapillaryProvider } from '../../layers/streetLevel/providers/mapillary/index.js'; +import { createGoogleProvider } from '../../layers/streetLevel/providers/google/index.js'; +import { createKartaViewProvider } from '../../layers/streetLevel/providers/kartaview/index.js'; +import { createPanoramaxProvider } from '../../layers/streetLevel/providers/panoramax/index.js'; import * as sprites from '../../data/spriteOrder.js'; import * as picking from '../../data/pickRegistry.js'; import * as input from '../../data/inputOwnership.js'; +import * as scenePick from '../../data/scenePick.js'; import * as render from '../../renderGovernor.js'; -/** A new provider registers here; its chip, credit and share bit follow. */ -export function createApplicationStreetLevel({ surface, sources }) { +/** The browser Maps key the scene was started with (src/app/scene.js). */ +const googleApiKey = () => globalThis.window?.__GOOGLE_MAPS_API_KEY__ || null; + +/** + * Mapillary, Google Street View, KartaView and Panoramax, once the Mapillary + * source is supplied; else none. + */ +function defaultProviders(sources) { + if (sources.mapillary == null) return []; + return [ + createMapillaryProvider({ source: sources.mapillary }), + createGoogleProvider({ getApiKey: googleApiKey }), + createKartaViewProvider(), + createPanoramaxProvider(), + ]; +} + +/** + * Inject providers, or use the defaults. An explicit provider list replaces + * them; an unused sources.mapillary is neither constructed nor validated, + * including when the list is empty. + */ +export function createApplicationStreetLevel({ + surface, + sources = {}, + providers = defaultProviders(sources), +}) { return createStreetLevelLayer({ - providers: [createMapillaryProvider({ source: sources.mapillary })], + providers, services: { sprites, picking, input, + scenePick, render, ground: surface?.groundFloor ?? null, meshFloor: surface?.meshFloor ?? null, diff --git a/src/app/layers/streetLevel.test.mjs b/src/app/layers/streetLevel.test.mjs new file mode 100644 index 000000000..e82e762e5 --- /dev/null +++ b/src/app/layers/streetLevel.test.mjs @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createApplicationStreetLevel } from './streetLevel.js'; +import { fakeStreetLevelProvider } from '../../testSupport/streetLevelFakes.mjs'; + +test('explicit providers replace the default source and an empty list removes it', () => { + const sources = { mapillary: {} }; + const provider = fakeStreetLevelProvider({ + id: 'example', + pickPrefix: 'example:', + }); + const layer = createApplicationStreetLevel({ + sources, + providers: [provider], + }); + assert.deepEqual(layer.providerIds, ['example']); + assert.deepEqual( + createApplicationStreetLevel({ sources, providers: [] }).providerIds, + [], + ); + assert.throws( + () => createApplicationStreetLevel({ sources }), + /Mapillary source/, + ); + assert.throws( + () => createApplicationStreetLevel({ providers: null }), + /array/, + ); +}); + +test('the defaults are Mapillary (given its source), Street View, KartaView and Panoramax, which start off and travel as provider switches', async () => { + const { createDefaultLayerState } = await import('../../data/layerState.js'); + const mapillary = { + getStatus() {}, + getTile() {}, + getSequenceImages() {}, + nearestImages() {}, + }; + const layer = createApplicationStreetLevel({ sources: { mapillary } }); + assert.deepEqual(layer.providerIds, [ + 'mapillary', + 'google', + 'kartaview', + 'panoramax', + ]); + // A fresh page and a restored default agree: Street View is billed, so off. + layer.setParams(createDefaultLayerState().options['street-level']); + assert.equal(layer.getParams().google, false); + assert.equal(layer.getParams().mapillary, true); + // A link that lit Street View carries it in the shared switch field. + layer.setParams({ providerSwitches: 'google-1' }); + assert.equal(layer.getParams().google, true); + assert.equal( + layer.getParams().providerSwitches, + 'google-1*kartaview-0*panoramax-0', + ); + // KartaView is keyless and free, but one source shows at a time. + layer.setParams({ providerSwitches: 'kartaview-1' }); + assert.equal(layer.getParams().kartaview, true); + assert.equal(layer.getParams().google, false); + assert.equal(layer.getParams().mapillary, false); + // As is Panoramax: lighting it puts KartaView out. + layer.setParams({ providerSwitches: 'panoramax-1' }); + assert.equal(layer.getParams().panoramax, true); + assert.equal(layer.getParams().kartaview, false); + assert.equal(layer.getParams().mapillary, false); + // The form the app writes (the README and CHANGELOG quote it). + assert.equal( + layer.getParams().providerSwitches, + 'google-0*kartaview-0*panoramax-1', + ); + // Without the application's Street Level source there are none. + assert.deepEqual(createApplicationStreetLevel({}).providerIds, []); +}); diff --git a/src/app/sourceComposition.js b/src/app/sourceComposition.js new file mode 100644 index 000000000..0fc619142 --- /dev/null +++ b/src/app/sourceComposition.js @@ -0,0 +1,115 @@ +const contract = (layers, methods) => + Object.freeze({ + layers: Object.freeze(layers), + methods: Object.freeze(methods), + }); + +/** Acquisition contracts and their consumers; records remain family-specific. */ +export const CATALOG_SOURCE_CONTRACTS = Object.freeze({ + directions: contract(['directions'], ['getRoute']), + 'recent-imagery': contract( + ['recent-imagery'], + ['searchHls', 'getThumbnail', 'getTileTemplate'], + ), + flights: contract(['flights'], ['getSnapshot']), + military: contract(['military'], ['getSnapshot']), + vessels: contract(['ais-live-vessels'], ['getSnapshot']), + cctv: contract( + ['cctv'], + ['getCatalog', 'getHealth', 'getFrameUrl', 'getMediaUrl'], + ), + radio: contract(['radio'], ['getDirectory', 'recordClick']), + traffic: contract( + ['traffic'], + [ + 'requestRoads', + 'getStatus', + 'fetchFlowForBounds', + 'getFlowSessionStats', + 'resetFlowTileCache', + ], + ), + bikeshare: contract(['bikeshare'], ['getStations']), + installations: contract( + ['military-installations'], + ['getMappedSites', 'searchNearby'], + ), + satellites: contract(['satellites'], ['readGroup']), + launches: contract(['rocket-launches'], ['getLaunches', 'getActiveTle']), + alpr: contract(['alpr-cameras'], ['fetch']), + firms: contract(['local-firms'], ['getSnapshot']), + wind: contract(['wind'], ['getSnapshot']), + weather: contract( + ['weather-radar', 'weather-satellite', 'weather-lightning'], + ['getSnapshot'], + ), + cyclones: contract(['weather-cyclones'], ['getSnapshot']), + earthquakes: contract(['earthquakes'], ['getSnapshot']), + 'fire-perimeters': contract(['fire-perimeters'], ['getSnapshot']), + 'air-quality': contract(['air-quality'], ['getSnapshot']), + 'gnss-integrity': contract(['gnss-integrity'], ['getSnapshot']), + cables: contract(['telegeography-submarine-cables'], ['fetch']), + transit: contract(['transit'], ['requestSnapshot', 'getHistory']), +}); + +/** An omitted provider is unavailable, never an authoritative empty dataset. */ +export class SourceUnavailableError extends Error { + constructor(name) { + super(`Data source not configured: ${name}`); + this.name = 'SourceUnavailableError'; + this.code = 'unavailable'; + this.unavailable = true; + this.retryable = false; + } +} + +/** Validate supplied contracts and keep omitted sources inert during construction. */ +export function resolveCatalogSources(supplied = {}) { + if (!supplied || typeof supplied !== 'object' || Array.isArray(supplied)) + throw new TypeError('Catalog sources must be an object'); + const sources = { ...supplied }; + const missing = new Set(); + for (const [name, { methods }] of Object.entries(CATALOG_SOURCE_CONTRACTS)) { + const source = sources[name]; + if (source == null) { + missing.add(name); + const unavailable = () => { + throw new SourceUnavailableError(name); + }; + sources[name] = Object.freeze({ + label: name, + ...Object.fromEntries(methods.map((method) => [method, unavailable])), + }); + } else if (methods.some((method) => typeof source[method] !== 'function')) { + throw new TypeError(`Invalid catalog source: ${name}`); + } + } + return { + sources, + isConfigured: (name) => !missing.has(name), + createAvailabilityLookup(layers) { + const byId = new Map(layers.map((layer) => [layer.id, layer])); + const availability = new Map(); + for (const [name, { layers: ids }] of Object.entries( + CATALOG_SOURCE_CONTRACTS, + )) { + for (const id of ids) { + const layer = byId.get(id); + if (!layer) throw new Error(`Missing source consumer: ${id}`); + availability.set( + id, + Object.freeze({ + available: !missing.has(name), + source: name, + reason: missing.has(name) + ? `${layer.name || id}: data source not configured` + : null, + }), + ); + } + } + return (id) => + availability.get(id) ?? byId.get(id)?.getSourceAvailability?.(); + }, + }; +} diff --git a/src/app/sourceComposition.test.mjs b/src/app/sourceComposition.test.mjs new file mode 100644 index 000000000..422308f1e --- /dev/null +++ b/src/app/sourceComposition.test.mjs @@ -0,0 +1,156 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + resolveCatalogSources, + CATALOG_SOURCE_CONTRACTS, +} from './sourceComposition.js'; +import { createApplicationCatalog } from './constructCatalog.js'; +import { createSurfaceServices } from './surfaceServices.js'; +import { createStandaloneLayerSources } from '../standalone/layerSources.js'; +import { LayerLifecycle } from '../data/lifecycle.js'; + +function catalog(sources, t) { + const lifetime = new AbortController(); + t.after(() => lifetime.abort()); + return createApplicationCatalog({ + sources, + signal: lifetime.signal, + surface: createSurfaceServices({ + terrainSource: { getHeights: async () => [] }, + signal: lifetime.signal, + eventTarget: null, + }), + }); +} + +test('every acquisition source can be omitted without losing catalog membership', (t) => { + const baseline = catalog(createStandaloneLayerSources(), t); + for (const name of Object.keys(CATALOG_SOURCE_CONTRACTS)) { + const sources = createStandaloneLayerSources({ [name]: null }); + const actual = catalog(sources, t); + assert.deepEqual(actual.metadata, baseline.metadata, name); + for (const id of CATALOG_SOURCE_CONTRACTS[name].layers) { + assert.equal(actual.getSourceAvailability(id).available, false, id); + } + } +}); + +test('all acquisition sources may be absent, and enable fails before initializing or fetching', async (t) => { + const actual = catalog({}, t); + assert.equal(actual.layers.length, 32); + const manager = new LayerLifecycle( + {}, + { getSourceAvailability: actual.getSourceAvailability }, + ); + for (const layer of actual.layers) manager.register(layer); + for (const { layers: ids } of Object.values(CATALOG_SOURCE_CONTRACTS)) { + for (const id of ids) { + assert.equal(await manager.setEnabled(id, true), false, id); + assert.equal(manager.layers.get(id).initialized, false, id); + const row = manager.getAll().find((entry) => entry.id === id); + assert.equal(row.enabled, false); + assert.equal(row.stats.status, 'unavailable'); + assert.match(row.stats.error, /data source not configured/); + } + } +}); + +test('configured sources retain identity; invalid implementations never become unavailable silently', () => { + const source = { getSnapshot: async () => [] }; + assert.equal( + resolveCatalogSources({ earthquakes: source }).sources.earthquakes, + source, + ); + for (const name of Object.keys(CATALOG_SOURCE_CONTRACTS)) { + assert.throws( + () => resolveCatalogSources({ [name]: {} }), + /Invalid catalog source/, + ); + } + const missing = resolveCatalogSources({}).sources.earthquakes; + assert.throws(() => missing.getSnapshot(), { code: 'unavailable' }); +}); + +test('standalone defaults allow exact replacements and explicit removal, and reject typos', () => { + const source = { getSnapshot: async () => [] }; + const sources = createStandaloneLayerSources({ + flights: source, + military: null, + }); + assert.equal(sources.flights, source); + assert.equal(sources.military, null); + assert.equal(typeof sources.cctv.getCatalog, 'function'); + assert.throws( + () => createStandaloneLayerSources({ fligts: source }), + /Unknown source/, + ); +}); + +test('empty Street Level uses the shared unavailable lifecycle without initialization', async (t) => { + const actual = catalog({}, t); + const manager = new LayerLifecycle( + {}, + { getSourceAvailability: actual.getSourceAvailability }, + ); + manager.register(actual.get('street-level')); + assert.equal(await manager.setEnabled('street-level', true), false); + assert.equal(manager.layers.get('street-level').initialized, false); + const row = manager.getAll()[0]; + assert.equal(row.stats.status, 'unavailable'); + assert.equal(row.stats.error, 'No street-level imagery providers configured'); + assert.equal(await manager.setEnabled('street-level', false), true); + const configured = catalog(createStandaloneLayerSources(), t); + assert.equal( + configured.get('street-level').getSourceAvailability().available, + true, + ); +}); + +test('availability lookup accepts frozen layers and uses their display names', () => { + const composition = resolveCatalogSources({}); + const layers = Object.values(CATALOG_SOURCE_CONTRACTS) + .flatMap(({ layers }) => layers) + .map((id) => + Object.freeze({ + id, + name: id === 'fire-perimeters' ? 'Fire perimeters' : id, + }), + ); + const lookup = composition.createAvailabilityLookup(layers); + assert.equal( + lookup('fire-perimeters').reason, + 'Fire perimeters: data source not configured', + ); + assert.equal(Object.hasOwn(layers[0], 'getSourceAvailability'), false); + const manager = new LayerLifecycle({}, { getSourceAvailability: lookup }); + manager.register(layers.find(({ id }) => id === 'fire-perimeters')); + assert.equal( + manager.getAll()[0].stats.error, + 'Fire perimeters: data source not configured', + ); +}); + +test('absent military acquisition does not poll, suppress flights, or erase supplied classification', async (t) => { + let fetches = 0; + t.mock.method(globalThis, 'fetch', async () => { + fetches++; + throw new Error('Unexpected fallback acquisition'); + }); + const actual = catalog(createStandaloneLayerSources({ military: null }), t); + const registry = actual.militaryRegistry; + assert.equal(actual.getSourceAvailability('flights').available, true); + assert.equal(registry.isMilitaryLayerActive(), false); + assert.equal(registry.isMilitaryIcao('abc123'), false); + registry.registerMilitaryIcaos([' ABC123 ']); + await registry.refreshMilitaryRegistryIfStale(); + assert.equal(registry.isMilitaryIcao('abc123'), true); + assert.equal(registry.isMilitaryIcao('def456'), false); + const manager = new LayerLifecycle( + {}, + { getSourceAvailability: actual.getSourceAvailability }, + ); + manager.register(actual.get('military')); + assert.equal(await manager.setEnabled('military', true), false); + assert.equal(registry.isMilitaryLayerActive(), false); + assert.equal(fetches, 0); +}); diff --git a/src/app/tools.js b/src/app/tools.js index 25d763e2d..7b3ea83c9 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -1,6 +1,7 @@ import { SceneDirector } from '../scenes/director.js'; import { initAnnotations } from '../annotations/index.js'; import { initDrawTool } from '../annotations/drawTool.js'; +import { initGeoFileImport } from '../ui/geoFileImport.js'; import { initImageryBoxTool } from '../ui/imageryBoxTool.js'; import { createRecentImageryPanel } from '../ui/recentImagery.js'; import { initGevVoiceCommands } from '../voice/gevRealtime.js'; @@ -56,6 +57,16 @@ export function createApplicationTools({ // lifetime rather than to whoever last pressed the button. const drawTool = initDrawTool({ viewer, annotations }); defer(() => drawTool?.destroy()); + // DISPLAY ▸ Import: the person's own GeoJSON/KML/KMZ/GPX files, each its + // own data source. They live for the page session, so the application + // lifetime removes them and the drop listeners on the viewer container. + const geoFileImport = initGeoFileImport({ + viewer, + navigation: styleManager, + ground: operations.surface?.groundFloor, + subscribeMapSource: (listener) => mapStackController?.subscribe?.(listener), + }); + defer(() => geoFileImport?.destroy()); // DATA ▸ Recent Imagery: the box tool claims the pointer like Draw and the // panel lives on the right rail, so both belong to the application // lifetime. The tileset lets the layer drape while the globe is hidden. diff --git a/src/cockpitMath.js b/src/cockpitMath.js index a0bd83676..74e6d1451 100644 --- a/src/cockpitMath.js +++ b/src/cockpitMath.js @@ -83,6 +83,26 @@ export function cockpitAltitudeDisplayFt(altitudeM, onGround) { return Number.isFinite(altitudeM) ? altitudeM * 3.28084 : null; } +/** + * HUD wording for a tracked aircraft's transponder emergency, or null. + * States what was broadcast, never a conclusion about the flight: the Mode A + * code when it names the status, else the ADS-B field it came from (an + * unrelated squawk is dropped). + * @param {{label?: string, squawk?: string|null, source?: string, + * severity?: string}|null|undefined} emergency + * @returns {string|null} + */ +export function formatCockpitEmergency(emergency) { + const label = String(emergency?.label || '') + .trim() + .toUpperCase(); + if (!label) return null; + if (emergency.source === 'squawk' && emergency.squawk) + return `SQUAWK ${emergency.squawk} · ${label}`; + const field = emergency.severity === 'priority' ? 'PRIORITY' : 'EMERGENCY'; + return `ADS-B ${field} · ${label}`; +} + /** Format the cockpit Context scope without overstating installation coverage. */ export function formatCockpitContextScope( subjectLabel, diff --git a/src/cockpitMath.test.mjs b/src/cockpitMath.test.mjs index ef1e8382b..2189ca35e 100644 --- a/src/cockpitMath.test.mjs +++ b/src/cockpitMath.test.mjs @@ -13,6 +13,7 @@ import { compassDivisions, formatAltitudeRulerTick, formatCockpitContextScope, + formatCockpitEmergency, formatCompassDivision, formatSpeedRulerTick, normalizeHeading, @@ -364,3 +365,21 @@ test('tracked-identity resolution falls back to layer precedence', () => { assert.equal(resolveTrackedAircraftInfo({ trackedId: 'flights:aaa077' }), null); assert.equal(resolveTrackedAircraftInfo(), null); }); + +test('cockpit emergency wording states the broadcast code, not a conclusion', () => { + assert.equal( + formatCockpitEmergency({ label: 'General emergency', squawk: '7700', source: 'squawk', severity: 'emergency' }), + 'SQUAWK 7700 · GENERAL EMERGENCY', + ); + assert.equal( + formatCockpitEmergency({ label: 'Downed aircraft', squawk: '1200', source: 'ads-b', severity: 'emergency' }), + 'ADS-B EMERGENCY · DOWNED AIRCRAFT', + 'an unrelated squawk is not quoted beside an ADS-B status', + ); + assert.equal( + formatCockpitEmergency({ label: 'Minimum fuel', squawk: null, source: 'ads-b', severity: 'priority' }), + 'ADS-B PRIORITY · MINIMUM FUEL', + ); + assert.equal(formatCockpitEmergency(null), null); + assert.equal(formatCockpitEmergency({ label: '' }), null); +}); diff --git a/src/codexOauthRealtime.test.mjs b/src/codexOauthRealtime.test.mjs new file mode 100644 index 000000000..4a577ac70 --- /dev/null +++ b/src/codexOauthRealtime.test.mjs @@ -0,0 +1,276 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import test from 'node:test'; +import path from 'node:path'; +import { + codexAuthPath, + preferCodexOAuth, + readCodexOAuthAccessToken, + resolveCodexExecutable, + startCodexChatGptLogin, +} from '../server/providers/openai/codex-auth.js'; + +const nowMs = Date.UTC(2026, 8, 27); +const jwt = (payload) => + `${Buffer.from('{}').toString('base64url')}.${Buffer.from(JSON.stringify(payload)).toString('base64url')}.signature`; +const auth = (token, extras = {}) => + JSON.stringify({ + auth_mode: 'chatgpt', + tokens: { access_token: token, refresh_token: 'refresh-fixture' }, + ...extras, + }); + +test('optional Codex OAuth pin fails closed on ambiguous values', () => { + assert.equal(preferCodexOAuth({}), false); + assert.equal(preferCodexOAuth({ GEV_PREFER_CODEX_OAUTH: 'true' }), true); + assert.equal(preferCodexOAuth({ GEV_PREFER_CODEX_OAUTH: 'off' }), false); + assert.throws( + () => preferCodexOAuth({ GEV_PREFER_CODEX_OAUTH: 'maybe' }), + /must be true or false/, + ); +}); + +test('Codex auth path follows explicit auth file, CODEX_HOME, then home directory', () => { + assert.equal( + codexAuthPath({ + environment: { CODEX_AUTH_JSON: '/tmp/custom-auth.json' }, + home: '/home/fixture', + }), + path.resolve('/tmp/custom-auth.json'), + ); + assert.equal( + codexAuthPath({ + environment: { CODEX_HOME: '/tmp/codex-home' }, + home: '/home/fixture', + }), + path.join(path.resolve('/tmp/codex-home'), 'auth.json'), + ); + assert.equal( + codexAuthPath({ environment: {}, home: '/home/fixture' }), + path.join('/home/fixture', '.codex', 'auth.json'), + ); +}); + +test('Codex OAuth reader accepts a current ChatGPT token and rejects unusable credentials', () => { + const current = jwt({ exp: nowMs / 1000 + 3600 }); + assert.equal( + readCodexOAuthAccessToken({ + authPath: '/unused', + nowMs, + readFile: () => auth(current), + }), + current, + ); + assert.equal( + readCodexOAuthAccessToken({ + authPath: '/unused', + nowMs, + readFile: () => + JSON.stringify({ + auth_mode: 'chatgpt', + tokens: { access_token: current }, + }), + }), + current, + 'GEV does not require or consume the vendor refresh token', + ); + for (const readFile of [ + () => '{bad json', + () => JSON.stringify({ tokens: {} }), + () => auth(current, { auth_mode: 'apikey' }), + () => + JSON.stringify({ + OPENAI_API_KEY: 'sk-fixture', + tokens: { access_token: current, refresh_token: 'refresh-fixture' }, + }), + () => auth('opaque-token'), + () => auth('header.not-json.signature'), + () => auth(jwt({ exp: 'tomorrow' })), + () => { + throw new Error('missing'); + }, + ]) { + assert.throws( + () => readCodexOAuthAccessToken({ authPath: '/unused', readFile, nowMs }), + /ChatGPT sign-in is unavailable/, + ); + } +}); + +test('Codex OAuth reader enforces expiry margin and bounds tokens without exp by file age', () => { + for (const exp of [nowMs / 1000 - 1, nowMs / 1000 + 59]) { + assert.throws( + () => { + readCodexOAuthAccessToken({ + authPath: '/unused', + nowMs, + readFile: () => auth(jwt({ exp })), + }); + }, + (error) => { + assert.match(error.message, /sign-in has expired/); + assert.equal(error.code, 'CODEX_OAUTH_REAUTH_REQUIRED'); + return true; + }, + ); + } + const noExp = jwt({ sub: 'fixture' }); + const base = { authPath: '/unused', nowMs, readFile: () => auth(noExp) }; + assert.equal( + readCodexOAuthAccessToken({ + ...base, + statFile: () => ({ mtimeMs: nowMs }), + }), + noExp, + ); + assert.throws( + () => + readCodexOAuthAccessToken({ + ...base, + statFile: () => ({ mtimeMs: nowMs - 3600_000 }), + }), + /sign-in has expired/, + ); + assert.throws( + () => + readCodexOAuthAccessToken({ + ...base, + statFile: () => { + throw new Error('missing'); + }, + }), + /sign-in is unavailable/, + ); +}); + +test('Codex login resolves an installed executable before PATH and starts browser auth detached', async () => { + const checked = []; + // Built with path.join so the stub matches the resolver on Windows too, + // where the separators are backslashes. + const installed = path.join('/home/fixture', '.local', 'bin', 'codex'); + assert.equal( + resolveCodexExecutable({ + environment: {}, + home: '/home/fixture', + access(candidate) { + checked.push(candidate); + if (candidate === installed) return; + throw new Error('missing'); + }, + }), + installed, + ); + assert.equal(checked.length, 1); + + const child = new EventEmitter(); + let unref = 0; + child.unref = () => { + unref += 1; + }; + const calls = []; + const pending = startCodexChatGptLogin({ + executable: '/fixture/codex', + home: '/home/fixture', + environment: { + PATH: '/fixture', + https_proxy: 'http://proxy.fixture:8080', + OPENAI_API_KEY: 'fixture-api-key', + GOOGLE_MAPS_SERVER_API_KEY: 'fixture-google-key', + OPENSKY_CLIENT_SECRET: 'fixture-opensky-secret', + }, + spawnImpl(command, args, options) { + calls.push({ command, args, options }); + queueMicrotask(() => child.emit('spawn')); + return child; + }, + }); + const login = await pending; + assert.equal(unref, 1); + assert.equal(calls[0].command, '/fixture/codex'); + assert.deepEqual(calls[0].args, ['login']); + assert.equal(calls[0].options.detached, true); + assert.equal(calls[0].options.stdio, 'ignore'); + assert.deepEqual( + calls[0].options.env, + { + PATH: '/fixture', + https_proxy: 'http://proxy.fixture:8080', + CODEX_HOME: path.join('/home/fixture', '.codex'), + }, + 'provider secrets stay out of the login process', + ); + child.emit('exit', 1); + assert.deepEqual(await login.completion, { exitCode: 1 }); +}); + +test('Codex login refuses a read/write storage mismatch before spawning', async () => { + await assert.rejects( + startCodexChatGptLogin({ + environment: { CODEX_AUTH_JSON: '/tmp/another-auth.json' }, + home: '/home/fixture', + executable: 'codex', + spawnImpl: () => assert.fail('mismatched storage must not launch login'), + }), + { code: 'CODEX_OAUTH_STORAGE_MISMATCH' }, + ); +}); + +test('Codex login resolves relative CODEX_HOME before changing the child directory', async () => { + const child = new EventEmitter(); + const environment = { CODEX_HOME: './fixture-codex' }; + const login = await startCodexChatGptLogin({ + environment, + executable: 'codex', + spawnImpl(_command, _args, options) { + assert.equal( + options.env.CODEX_HOME, + path.resolve(environment.CODEX_HOME), + ); + queueMicrotask(() => child.emit('spawn')); + return child; + }, + }); + assert.equal( + environment.CODEX_HOME, + './fixture-codex', + 'caller environment is unchanged', + ); + child.emit('exit', 0); + assert.deepEqual(await login.completion, { exitCode: 0 }); +}); + +test('Codex login hides spawn errors and can cancel only its owned child', async () => { + const child = new EventEmitter(); + await assert.rejects( + startCodexChatGptLogin({ + executable: 'codex', + spawnImpl() { + queueMicrotask(() => + child.emit('error', new Error('private-token-fixture')), + ); + return child; + }, + }), + (error) => + error.code === 'CODEX_OAUTH_LOGIN_FAILED' && + !error.message.includes('private-token-fixture'), + ); + + const running = new EventEmitter(); + let cancelled = 0; + running.kill = (signal) => { + assert.equal(signal, 'SIGTERM'); + cancelled += 1; + running.emit('exit', null); + }; + const login = await startCodexChatGptLogin({ + executable: 'codex', + spawnImpl() { + queueMicrotask(() => running.emit('spawn')); + return running; + }, + }); + login.cancel(); + assert.equal(cancelled, 1); + assert.deepEqual(await login.completion, { exitCode: null }); +}); diff --git a/src/data/adsbLolFallback.js b/src/data/adsbLolFallback.js index 7cd887dc6..40dd2bbc0 100644 --- a/src/data/adsbLolFallback.js +++ b/src/data/adsbLolFallback.js @@ -43,11 +43,18 @@ export function normalizeAdsbLolAircraftState(aircraft, nowSeconds) { .toLowerCase(); const latitude = finiteNumber(aircraft?.lat); const longitude = finiteNumber(aircraft?.lon); - if (!hex || latitude === null || longitude === null) return null; + if ( + !hex || + latitude === null || + longitude === null || + Math.abs(latitude) > 90 || + Math.abs(longitude) > 180 + ) + return null; const seenPosition = Math.max( 0, - finiteNumber(aircraft?.seen_pos) ?? finiteNumber(aircraft?.seen) ?? 0, + finiteNumber(aircraft?.seen_pos) ?? finiteNumber(aircraft?.seen) ?? NaN, ); const seen = Math.max(0, finiteNumber(aircraft?.seen) ?? seenPosition); const onGround = aircraft?.alt_baro === 'ground'; @@ -62,8 +69,10 @@ export function normalizeAdsbLolAircraftState(aircraft, nowSeconds) { hex, String(aircraft?.flight || aircraft?.r || '').trim() || null, null, - Math.max(0, nowSeconds - seenPosition), - Math.max(0, nowSeconds - seen), + Number.isFinite(seenPosition) + ? Math.max(0, nowSeconds - seenPosition) + : null, + Number.isFinite(seen) ? Math.max(0, nowSeconds - seen) : null, longitude, latitude, barometricFeet === null ? null : barometricFeet * FOOT_TO_M, @@ -94,8 +103,28 @@ export function normalizeAdsbLolPointResponse(payload) { : Math.floor( responseNow > 10_000_000_000 ? responseNow / 1000 : responseNow, ); - const states = (Array.isArray(payload?.ac) ? payload.ac : []) - .map((aircraft) => normalizeAdsbLolAircraftState(aircraft, nowSeconds)) - .filter(Boolean); - return { time: nowSeconds, states }; + const aircraft = Array.isArray(payload?.ac) ? payload.ac : []; + const states = []; + // Keep the OpenSky vectors unchanged. The regional fallback's extra readsb + // status travels alongside them, keyed only to admitted aircraft. + const admitted = new Set(); + const statuses = []; + for (const row of aircraft) { + const state = normalizeAdsbLolAircraftState(row, nowSeconds); + if (!state) continue; + states.push(state); + // The live contract admits the first valid vector for each ICAO. Its + // status must come from that same row, never an invalid or later duplicate. + const id = state[0]; + if (admitted.has(id)) continue; + admitted.add(id); + if (typeof row.emergency === 'string') + statuses.push([id, row.emergency.trim().toLowerCase()]); + } + const emergencyByIcao = Object.fromEntries(statuses); + return { + time: nowSeconds, + states, + ...(Object.keys(emergencyByIcao).length ? { emergencyByIcao } : {}), + }; } diff --git a/src/data/aircraftEmergency.js b/src/data/aircraftEmergency.js new file mode 100644 index 000000000..78bd4308f --- /dev/null +++ b/src/data/aircraftEmergency.js @@ -0,0 +1,93 @@ +/** + * Transponder emergency status for one aircraft, from the Mode A squawk and + * the ADS-B emergency/priority field. + * + * Emergency squawks are ICAO-standard worldwide: 7500 unlawful interference, + * 7600 radio failure, 7700 general emergency. readsb (adsb.lol) also reports + * `emergency`, "a superset of the 7x00 squawks": none, general, lifeguard, + * minfuel, nordo, unlawful, downed, reserved. OpenSky state vectors carry the + * squawk only. + * + * The status is what the transponder broadcast in THIS poll, never a finding: + * crews pass through 7x00 codes while dialling. Highlighting follows the latest + * broadcast, which does not confirm an incident or its resolution. + */ + +/** @constant {string} Tint for aircraft broadcasting an emergency (not a priority). */ +export const EMERGENCY_TINT_CSS = '#FF4444'; + +const SQUAWK_KINDS = { 7500: 'unlawful', 7600: 'nordo', 7700: 'general' }; + +const KINDS = { + general: { label: 'General emergency', severity: 'emergency' }, + unlawful: { label: 'Unlawful interference', severity: 'emergency' }, + nordo: { label: 'Radio failure', severity: 'emergency' }, + downed: { label: 'Downed aircraft', severity: 'emergency' }, + minfuel: { label: 'Minimum fuel', severity: 'priority' }, + lifeguard: { label: 'Lifeguard (medical priority)', severity: 'priority' }, +}; + +/** + * A Mode A code as its four octal digits, else null. + * @param {unknown} value + * @returns {string|null} + */ +export function normalizeSquawk(value) { + const code = String(value ?? '').trim(); + return /^[0-7]{4}$/.test(code) ? code : null; +} + +/** + * Emergency status from one observation. Higher severity wins; an ADS-B + * emergency wins a tie. A + * 7x00 squawk counts even when the ADS-B field says `none`. + * @param {{squawk?: unknown, emergency?: unknown}} [observation] + * @returns {{kind: string, label: string, severity: 'emergency'|'priority', + * squawk: string|null, source: 'ads-b'|'squawk'}|null} `source` is + * 'squawk' when the Mode A code itself names the status (so it can be + * quoted), else 'ads-b'. + */ +export function aircraftEmergency({ squawk, emergency } = {}) { + const code = normalizeSquawk(squawk); + const reported = String(emergency ?? '') + .trim() + .toLowerCase(); + const fromAdsb = Object.hasOwn(KINDS, reported); + const squawkKind = SQUAWK_KINDS[code] ?? null; + const kind = + fromAdsb && (KINDS[reported].severity === 'emergency' || !squawkKind) + ? reported + : squawkKind; + if (!kind) return null; + return { + kind, + ...KINDS[kind], + squawk: code, + source: SQUAWK_KINDS[code] === kind ? 'squawk' : 'ads-b', + }; +} + +/** + * How a summary may phrase the status without flattening it into an incident: + * quote the code when the Mode A squawk itself names the status, otherwise + * attribute it to the ADS-B emergency field. + * @param {{kind?: string, squawk?: string|null, source?: string}|null|undefined} status + * @returns {string|null} e.g. 'broadcasting squawk 7700 (general emergency)' + * or 'ADS-B reports general emergency'; null when there is no status. + */ +export function emergencyBroadcastPhrase(status) { + if (!status || !Object.hasOwn(KINDS, status.kind)) return null; + const label = KINDS[status.kind].label.toLowerCase(); + return status.source === 'squawk' && status.squawk + ? `broadcasting squawk ${status.squawk} (${label})` + : `ADS-B reports ${label}`; +} + +/** + * True when the aircraft is broadcasting an emergency (not a priority status). + * @param {{severity?: string}|null|undefined} status + * @returns {boolean} + */ +export function isEmergency(status) { + return status?.severity === 'emergency'; +} diff --git a/src/data/aircraftEmergency.test.mjs b/src/data/aircraftEmergency.test.mjs new file mode 100644 index 000000000..6646aa6bd --- /dev/null +++ b/src/data/aircraftEmergency.test.mjs @@ -0,0 +1,95 @@ +// src/data/aircraftEmergency.test.mjs +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + aircraftEmergency, + emergencyBroadcastPhrase, + isEmergency, + normalizeSquawk, +} from './aircraftEmergency.js'; + +test('the three ICAO emergency squawks map to their kinds', () => { + assert.equal(aircraftEmergency({ squawk: '7500' }).kind, 'unlawful'); + assert.equal(aircraftEmergency({ squawk: '7600' }).kind, 'nordo'); + const general = aircraftEmergency({ squawk: ' 7700 ' }); + assert.deepEqual(general, { + kind: 'general', + label: 'General emergency', + severity: 'emergency', + squawk: '7700', + source: 'squawk', + }); +}); + +test('ordinary codes, blanks and non-octal values are not emergencies', () => { + for (const squawk of ['1200', '7000', '2000', '', null, undefined, '7800']) + assert.equal(aircraftEmergency({ squawk }), null, String(squawk)); + assert.equal(aircraftEmergency(), null); + assert.equal(normalizeSquawk('7800'), null); + assert.equal(normalizeSquawk(7700), '7700'); + assert.equal(normalizeSquawk('770'), null); +}); + +test('the ADS-B emergency field wins, and `none` defers to the squawk', () => { + const minfuel = aircraftEmergency({ squawk: '4521', emergency: 'MinFuel' }); + assert.equal(minfuel.kind, 'minfuel'); + assert.equal(minfuel.severity, 'priority'); + assert.equal(minfuel.source, 'ads-b'); + assert.equal(minfuel.squawk, '4521'); + assert.equal( + aircraftEmergency({ squawk: '7700', emergency: 'none' }).source, + 'squawk', + ); + assert.equal( + aircraftEmergency({ squawk: '7700', emergency: 'general' }).source, + 'squawk', + 'a code that names the status stays quotable', + ); + assert.equal( + aircraftEmergency({ squawk: '1200', emergency: 'general' }).source, + 'ads-b', + ); + assert.equal(aircraftEmergency({ emergency: 'downed' }).kind, 'downed'); + assert.equal(aircraftEmergency({ emergency: 'none' }), null); + assert.equal(aircraftEmergency({ emergency: 'reserved' }), null); +}); + +test('priority statuses are reported but are not emergencies', () => { + assert.equal(isEmergency(aircraftEmergency({ squawk: '7700' })), true); + assert.equal( + isEmergency(aircraftEmergency({ emergency: 'lifeguard' })), + false, + ); + assert.equal(isEmergency(aircraftEmergency({ emergency: 'minfuel' })), false); + assert.equal(isEmergency(null), false); +}); + +test('the broadcast phrase quotes a naming code, else attributes ADS-B', () => { + assert.equal( + emergencyBroadcastPhrase(aircraftEmergency({ squawk: '7700' })), + 'broadcasting squawk 7700 (general emergency)', + ); + assert.equal( + emergencyBroadcastPhrase( + aircraftEmergency({ squawk: '1200', emergency: 'general' }), + ), + 'ADS-B reports general emergency', + ); + assert.equal( + emergencyBroadcastPhrase(aircraftEmergency({ emergency: 'lifeguard' })), + 'ADS-B reports lifeguard (medical priority)', + ); + assert.equal(emergencyBroadcastPhrase(null), null); + assert.equal(emergencyBroadcastPhrase({ kind: 'reserved' }), null); +}); + +test('an emergency squawk outranks an ADS-B priority', () => { + assert.equal( + aircraftEmergency({ squawk: '7700', emergency: 'lifeguard' }).kind, + 'general', + ); + assert.equal( + aircraftEmergency({ squawk: '7500', emergency: 'minfuel' }).kind, + 'unlawful', + ); +}); diff --git a/src/data/aircraftEmergencyAnalyst.test.mjs b/src/data/aircraftEmergencyAnalyst.test.mjs new file mode 100644 index 000000000..27deb2b46 --- /dev/null +++ b/src/data/aircraftEmergencyAnalyst.test.mjs @@ -0,0 +1,524 @@ +import * as Cesium from 'cesium'; +import { TRACKED_ICON_COLOR } from '../layers/military/policy.js'; +import { createRendering as civilRendering } from '../layers/flights/rendering.js'; +import { createRendering as militaryRendering } from '../layers/military/rendering.js'; +import { createMilitarySnapshotRenderer } from '../layers/military/snapshotRenderer.js'; +import { aircraftEmergency } from './aircraftEmergency.js'; +// Source-to-voice provenance, status retention/clearing, contact-time contracts, +// and civil/military billboard, Cockpit and model tint regressions. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { openSkySnapshot, readsbSnapshot } from '../sources/live/index.js'; +import { FlightRecords } from '../layers/flights/records.js'; +import { MilitaryFlightRecords } from '../layers/military/records.js'; +import { mapAnalystRecord as mapFlight } from './flights.js'; +import { mapAnalystRecord as mapMilitary } from './militaryFlights.js'; +import { createAnalystEngine } from './analystEngine.js'; +import { normalizeAdsbLolPointResponse } from './adsbLolFallback.js'; +import { createGevActionRunner } from '../voice/gevActions.js'; + +const NOW_MS = 1800000000000; + +test('fallback status belongs to the first valid admitted aircraft row', () => { + const valid = { hex: 'a1b2c3', lat: 30, lon: -97, squawk: '1200' }; + for (const status of [undefined, 'general', 'none']) { + const payload = normalizeAdsbLolPointResponse({ + now: NOW_MS, + ac: [ + { ...valid, lat: 100, emergency: 'general' }, + { ...valid, emergency: status }, + { ...valid, lat: null, emergency: 'general' }, + { ...valid, emergency: status === 'general' ? 'none' : 'general' }, + ], + }); + const snapshot = openSkySnapshot(payload, { now: NOW_MS }); + assert.equal(snapshot.records.length, 1); + assert.equal(snapshot.records[0].emergency, status); + assert.equal(snapshot.records[0].latitude, 30); + } +}); + +function civilStore() { + return new FlightRecords({ + geoidHeight: () => 0, + cachedGroundFloor: () => null, + floorAltitudeM: (value) => value, + approxDistanceKm: () => 0, + }); +} +const receiveContext = { + viewerLatDeg: null, + viewerLonDeg: null, + trackedId: null, + floorWarmPoints: [], +}; + +test('real civil fallback preserves ADS-B emergency and priority, then clears', () => { + const store = civilStore(); + const aircraft = { + hex: 'a1b2c3', + flight: 'TEST1', + lat: 30, + lon: -97, + alt_baro: 10000, + gs: 150, + track: 90, + squawk: '1200', + seen: 3, + seen_pos: 1, + }; + for (const [status, expectedKind, flagged] of [ + ['general', 'general', true], + ['minfuel', 'minfuel', false], + ['none', null, false], + ]) { + const payload = normalizeAdsbLolPointResponse({ + now: NOW_MS, + ac: [{ ...aircraft, emergency: status }], + }); + assert.equal( + payload.states[0].length, + 18, + 'fallback vectors retain the OpenSky shape', + ); + const [observation] = openSkySnapshot(payload, { now: NOW_MS }).records; + const result = mapFlight( + aircraft.hex, + store.receive(observation, receiveContext).meta, + ); + assert.equal(result.emergency, flagged); + assert.equal(result.emergencyKind, expectedKind); + assert.equal(result.emergencySource, expectedKind ? 'ads-b' : null); + assert.equal(result.lastContactAtMs, NOW_MS - 3000); + } +}); + +test('changed status without contact time retains general contact, including voice projection', async () => { + const store = civilStore(); + const [observation] = readsbSnapshot( + { + ac: [ + { + hex: 'a1b2c3', + flight: 'TEST1', + lat: 30, + lon: -97, + alt_baro: 10000, + gs: 150, + track: 90, + squawk: '1200', + emergency: 'general', + seen: 3, + seen_pos: 1, + }, + ], + }, + { observedAtMs: NOW_MS, now: NOW_MS }, + ).records; + store.receive(observation, receiveContext); + const meta = store.receive( + { ...observation, contactTimeMs: null, squawk: '7600', emergency: 'none' }, + receiveContext, + ).meta; + const record = mapFlight(observation.id, meta); + assert.equal(record.emergencyKind, 'nordo'); + assert.equal(record.lastContactAtMs, NOW_MS - 3000); + assert.equal(Object.hasOwn(record, 'observedAtMs'), false); + globalThis.window = globalThis.window || { + clearTimeout, + setTimeout, + requestIdleCallback: null, + }; + const flights = { + id: 'flights', + source: 'fixture', + getStats: () => ({ count: 1 }), + getAnalystRecords: () => [record], + }; + const runner = createGevActionRunner({ + viewer: { + clock: { onTick: { addEventListener: () => () => {} } }, + scene: { canvas: { addEventListener() {}, removeEventListener() {} } }, + camera: { + moveEnd: { addEventListener() {} }, + positionCartographic: { + height: 300000, + latitude: 0.52, + longitude: -1.71, + }, + }, + }, + styleManager: { + getContextModeState: () => ({ mode: null, active: false }), + getCockpitState: () => ({ active: false }), + }, + dataManager: { + layers: new Map([['flights', { module: flights }]]), + isEnabled: (id) => id === 'flights', + getAll: () => [ + { id: 'flights', name: 'Flights', enabled: true, stats: { count: 1 } }, + ], + }, + }); + try { + const result = await runner('analyst_query', { + layers: ['flights'], + scope: { kind: 'anywhere' }, + filters: [{ field: 'emergency', op: 'eq', value: true }], + }); + assert.equal(result.ok, true); + assert.equal(result.items[0].emergency, true); + assert.equal(result.items[0].lastContactAtMs, NOW_MS - 3000); + assert.equal(result.items[0].emergencySource, 'squawk'); + assert.equal( + result.items[0].emergencyBroadcast, + 'broadcasting squawk 7600 (radio failure)', + ); + assert.equal( + Object.hasOwn(result.items[0], 'time'), + false, + 'general contact is not a generic incident timestamp', + ); + } finally { + runner.dispose?.(); + } +}); + +// OpenSky /states/all: [icao24, callsign, origin_country, time_position, +// last_contact, lon, lat, baro_alt, on_ground, velocity, true_track, +// vertical_rate, sensors, geo_alt, squawk, spi, position_source, category]. +const OPENSKY = { + time: NOW_MS / 1000, + states: [ + [ + 'a1b2c3', + 'SWA696 ', + 'United States', + NOW_MS / 1000 - 6, + NOW_MS / 1000 - 3, + -97.67, + 30.19, + 3000, + false, + 150, + 180, + -5, + null, + 3040, + '7700', + false, + 0, + 4, + ], + ], +}; + +// adsb.lol /v2/mil: readsb aircraft with `seen`/`seen_pos` ages in seconds. +const ADSB_LOL = { + ac: [ + { + hex: 'ae01ce', + flight: 'RCH123 ', + t: 'C17', + lat: 31.1, + lon: -97.4, + alt_baro: 18000, + gs: 300, + track: 90, + squawk: '1200', + emergency: 'general', + seen: 1.5, + seen_pos: 2, + }, + ], +}; + +function civilAnalystRecord() { + const [row] = openSkySnapshot(OPENSKY, { now: NOW_MS }).records; + const store = new FlightRecords({ + geoidHeight: () => 0, + cachedGroundFloor: () => null, + floorAltitudeM: (altitude) => altitude, + approxDistanceKm: () => 0, + }); + const { meta } = store.receive(row, { + viewerLatDeg: null, + viewerLonDeg: null, + trackedId: null, + floorWarmPoints: [], + }); + return mapFlight(row.id, meta); +} + +function militaryAnalystRecord() { + const [row] = readsbSnapshot(ADSB_LOL, { + observedAtMs: NOW_MS, + now: NOW_MS, + }).records; + const store = new MilitaryFlightRecords({ + geoidHeight: () => 0, + cachedGroundFloor: () => null, + floorAltitudeM: (altitude) => altitude, + }); + const { meta } = store.receive(row, { + observedAtMs: NOW_MS, + floorWarmPoints: [], + modelOwnsVisual: false, + }); + return mapMilitary(row.id, meta); +} + +test('OpenSky 7700 and adsb.lol emergency=general differ only in provenance', () => { + const civil = civilAnalystRecord(); + const military = militaryAnalystRecord(); + + assert.equal(civil.emergency, true); + assert.equal(civil.emergencyKind, 'general'); + assert.equal(civil.squawk, '7700'); + assert.equal(civil.emergencySource, 'squawk'); + assert.equal( + civil.emergencyBroadcast, + 'broadcasting squawk 7700 (general emergency)', + ); + assert.equal(civil.lastContactAtMs, NOW_MS - 3000, 'OpenSky last_contact'); + + assert.equal(military.emergency, true); + assert.equal(military.emergencyKind, 'general'); + assert.equal(military.squawk, '1200', 'the code is not an emergency code'); + assert.equal(military.emergencySource, 'ads-b'); + assert.equal(military.emergencyBroadcast, 'ADS-B reports general emergency'); + assert.equal(military.lastContactAtMs, NOW_MS - 1500, 'snapshot time - seen'); +}); + +test('the analyst can filter emergencies by source', async () => { + const records = { + flights: [civilAnalystRecord()], + military: [militaryAnalystRecord()], + }; + const engine = createAnalystEngine({ + getRecords: (key) => records[key] || [], + resolveRegionRing: async () => null, + getViewContext: () => ({ lat: 30.5, lon: -97.5, viewRadiusKm: 500 }), + }); + const query = (value) => + engine.query({ + layers: ['flights', 'military'], + scope: { kind: 'anywhere' }, + filters: [ + { field: 'emergency', op: 'eq', value: true }, + { field: 'emergencySource', op: 'eq', value }, + ], + }); + const bySquawk = await query('squawk'); + assert.deepEqual( + bySquawk.items.map((item) => [item.icao24, item.emergencyBroadcast]), + [['a1b2c3', 'broadcasting squawk 7700 (general emergency)']], + ); + const byAdsb = await query('ads-b'); + assert.deepEqual( + byAdsb.items.map((item) => [item.icao24, item.emergencyBroadcast]), + [['ae01ce', 'ADS-B reports general emergency']], + ); +}); + +test('military status changes retain only the latest known general contact', () => { + const store = new MilitaryFlightRecords({ + geoidHeight: () => 0, + cachedGroundFloor: () => null, + floorAltitudeM: (v) => v, + }); + const [observation] = readsbSnapshot(ADSB_LOL, { + observedAtMs: NOW_MS, + now: NOW_MS, + }).records; + const context = { + observedAtMs: NOW_MS, + floorWarmPoints: [], + modelOwnsVisual: false, + }; + const unknown = mapMilitary( + observation.id, + store.receive({ ...observation, contactTimeMs: null }, context).meta, + ); + assert.equal(unknown.lastContactAtMs, null); + store.receive(observation, context); + const changed = mapMilitary( + observation.id, + store.receive( + { + ...observation, + contactTimeMs: null, + squawk: '7600', + emergency: 'none', + }, + context, + ).meta, + ); + assert.equal(changed.emergencyKind, 'nordo'); + assert.equal(changed.lastContactAtMs, NOW_MS - 1500); +}); + +test('fallback missing contact age retains known contact through the real converter', () => { + const store = civilStore(); + const row = { + hex: 'a1b2c3', + lat: 30, + lon: -97, + squawk: '1200', + emergency: 'general', + seen: 3, + seen_pos: 1, + }; + const receive = (row) => + store.receive( + openSkySnapshot( + normalizeAdsbLolPointResponse({ now: NOW_MS, ac: [row] }), + { now: NOW_MS }, + ).records[0], + receiveContext, + ).meta; + const first = receive(row); + const second = receive({ ...row, seen: undefined, seen_pos: undefined }); + assert.equal(second.lastContactEpochMs, first.lastContactEpochMs); +}); + +test('one omitted ADS-B status poll is retained; explicit none and changed squawk clear immediately', () => { + const store = civilStore(); + const row = { + hex: 'a1b2c3', + lat: 30, + lon: -97, + squawk: '1200', + emergency: 'general', + seen: 3, + }; + const receive = (row) => + store.receive( + openSkySnapshot( + normalizeAdsbLolPointResponse({ now: NOW_MS, ac: [row] }), + { now: NOW_MS }, + ).records[0], + receiveContext, + ).meta; + assert.equal(receive(row).emergency.kind, 'general'); + assert.equal( + receive({ ...row, emergency: undefined }).emergency?.kind, + 'general', + ); + assert.equal(receive({ ...row, emergency: undefined }).emergency, null); + receive(row); + assert.equal(receive({ ...row, emergency: 'none' }).emergency, null); + receive(row); + assert.equal( + receive({ ...row, emergency: undefined, squawk: '2000' }).emergency, + null, + ); +}); + +function renderHarness(factory) { + const flightState = { records: { data: new Map() }, _trackedIcao: null }; + const rendering = factory({ + flightState, + services: { + militaryRegistry: { isMilitaryIcao: (id) => id === 'mil' }, + aircraftPresentation: {}, + camera: {}, + focus: {}, + recession: {}, + }, + parts: {}, + }); + return { flightState, rendering }; +} + +test('tint priorities cover normal, priority, emergency, tracking and Cockpit dots', () => { + for (const factory of [civilRendering, militaryRendering]) { + const { flightState, rendering } = renderHarness(factory); + const normal = rendering._fleetBillboardColor('civil', {}); + assert.equal( + rendering._fleetBillboardColor('civil', { + emergency: aircraftEmergency({ emergency: 'minfuel' }), + }), + normal, + ); + const red = rendering._fleetBillboardColor('civil', { + emergency: aircraftEmergency({ squawk: '7700' }), + }); + assert.ok(red.red > 0.9 && red.green < 0.5); + flightState.records.data.set('civil', { + emergency: aircraftEmergency({ squawk: '7700' }), + }); + assert.equal(rendering._modelColor('civil'), red); + if (rendering._cockpitContactColor) + assert.equal(rendering._cockpitContactColor('civil'), red); + flightState._trackedIcao = 'civil'; + assert.equal( + rendering._modelColor('civil'), + factory === civilRendering ? Cesium.Color.CYAN : TRACKED_ICON_COLOR, + ); + flightState._trackedIcao = null; + flightState.records.data.set('civil', { emergency: null }); + assert.equal(rendering._modelColor('civil'), normal); + } +}); + +test('GERMAN: an existing untracked military billboard repaints each poll without alpha loss', () => { + const { flightState, rendering } = renderHarness(militaryRendering); + const bb = { + color: rendering._fleetBillboardColor('abc123').withAlpha(0.65), + }; + Object.assign(flightState, { + _billboards: new Map([['abc123', bb]]), + _models: new Map(), + _positionHistory: new Map(), + _cullPositions: new Map(), + _groundSnap: { forget() {} }, + _cockpitContactMode: false, + }); + const records = new MilitaryFlightRecords({ + geoidHeight: () => 0, + cachedGroundFloor: () => null, + floorAltitudeM: (value) => value, + }); + flightState.records = records; + const apply = createMilitarySnapshotRenderer({ + flightState, + records, + groundFloor: { warmGroundFloor() {}, cachedGroundFloor: () => null }, + meshFloor: { sampleMeshFloorCells() {} }, + militaryRegistry: { registerMilitaryIcaos() {} }, + rendering: { + ...rendering, + _syncModelToClass() {}, + _applyFleetBillboardPresentation() {}, + }, + motion: {}, + tracking: {}, + queries: {}, + }); + for (const [emergency, red] of [ + ['general', true], + ['none', false], + ['general', true], + ]) { + const snapshot = readsbSnapshot( + { + ac: [ + { + hex: 'abc123', + lat: 30, + lon: -97, + alt_baro: 10000, + squawk: '1200', + emergency, + seen_pos: 1, + }, + ], + }, + { observedAtMs: NOW_MS }, + ); + apply(snapshot, {}); + assert.equal(bb.color.green < 0.5, red); + assert.equal(bb.color.alpha, 0.65); + } +}); diff --git a/src/data/aircraftRecession.js b/src/data/aircraftRecession.js index b260c5f1b..04c41e604 100644 --- a/src/data/aircraftRecession.js +++ b/src/data/aircraftRecession.js @@ -242,7 +242,7 @@ export function applyAircraftBillboardTreatment({ } /** - * Apply the already-composed aircraft alpha to an ambient glTF model without + * Apply layer-owned hue and composed aircraft alpha to an ambient glTF model without * disturbing Cesium's existing MIX/colorBlendAmount presentation semantics. * @param {object} input * @param {object} input.model @@ -261,7 +261,10 @@ export function applyAircraftModelTreatment({ if (!model || !baseColor || !Number.isFinite(alpha)) return 0; if ( Number.isFinite(model.color?.alpha) && - Math.abs(model.color.alpha - alpha) <= tuning.writeEpsilon + Math.abs(model.color.alpha - alpha) <= tuning.writeEpsilon && + model.color.red === baseColor.red && + model.color.green === baseColor.green && + model.color.blue === baseColor.blue ) return 0; model.color = baseColor.withAlpha(alpha); diff --git a/src/data/aircraftRecession.test.mjs b/src/data/aircraftRecession.test.mjs index 29bce05c5..23b8ceffb 100644 --- a/src/data/aircraftRecession.test.mjs +++ b/src/data/aircraftRecession.test.mjs @@ -142,3 +142,15 @@ test('ambient model presentation receives composed alpha without changing blend assert.equal(model.colorBlendAmount, 0.9); assert.equal(applyAircraftModelTreatment({ model, baseColor: color, alpha: 0.2, params }), 0); }); + + +test('model hue changes at constant alpha, then clears without redundant writes', () => { + const color = (red,green,blue) => ({red,green,blue,withAlpha(alpha){return {red,green,blue,alpha};}}); + const model = {color:color(1,1,1).withAlpha(0.65)}; + assert.equal(applyAircraftModelTreatment({model,baseColor:color(1,0.2,0.2),alpha:0.65}),1); + assert.equal(model.color.green,0.2); + assert.equal(model.color.alpha,0.65); + assert.equal(applyAircraftModelTreatment({model,baseColor:color(1,0.2,0.2),alpha:0.65}),0); + assert.equal(applyAircraftModelTreatment({model,baseColor:color(1,1,1),alpha:0.65}),1); + assert.equal(model.color.green,1); +}); diff --git a/src/data/ambiguous-camera-direction.test.mjs b/src/data/ambiguous-camera-direction.test.mjs new file mode 100644 index 000000000..b5996d7a7 --- /dev/null +++ b/src/data/ambiguous-camera-direction.test.mjs @@ -0,0 +1,30 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { directionToHeading } from './directionText.js'; +test('conflicting facing directions cannot manufacture one heading', () => { + for (const bare of [false, true]) { + for (const value of ['NB / SB', 'EASTBOUND WESTBOUND', 'NE SW']) + assert.ok(Number.isNaN(directionToHeading(value, bare)), value); + assert.equal(directionToHeading('NB northbound', bare), 0); + assert.equal(directionToHeading('IH-35 SOUTHBOUND AT 15TH', bare), 180); + } + assert.ok(Number.isNaN(directionToHeading('North / East', true))); + assert.equal(directionToHeading('West / SR-24', true), 270); +}); + +test('travel tokens retain precedence over quadrant streets and bare words', () => { + for (const [text, heading] of [ + ['I-5 NB at NE 45th St', 0], + ['SR 520 EB at NE 148th', 90], + ['US 13 SB @ SW Front St', 180], + ]) { + for (const bare of [false, true]) + assert.equal(directionToHeading(text, bare), heading); + } + assert.equal( + directionToHeading('Hwy 401 east of Keele St - Westbound', true), + 270, + ); + assert.equal(directionToHeading('NE near West St', true), 45); + assert.ok(Number.isNaN(directionToHeading('NB SB near NE St', true))); +}); diff --git a/src/data/cameraFraming.js b/src/data/cameraFraming.js new file mode 100644 index 000000000..a2d38b5ec --- /dev/null +++ b/src/data/cameraFraming.js @@ -0,0 +1,15 @@ +import * as Cesium from 'cesium'; + +/** Camera range with breathing room in both viewport dimensions. */ +export function rangeForBoundingSphere(viewer, radius) { + const frustum = viewer.camera?.frustum; + const verticalFov = Number(frustum?.fov) || Cesium.Math.toRadians(60); + const aspectRatio = Math.max(0.5, Number(frustum?.aspectRatio) || 1); + const horizontalFov = 2 * Math.atan(Math.tan(verticalFov / 2) * aspectRatio); + const limitingFov = Math.min(verticalFov, horizontalFov); + // Target 57% sphere occupancy so perspective still leaves at least 40% + // measured roof-to-base breathing room in ordinary oblique building views. + const occupiedViewportFraction = 0.57; + const desiredAngularRadius = (limitingFov * occupiedViewportFraction) / 2; + return (radius / Math.sin(desiredAngularRadius)) * 1.05; +} diff --git a/src/data/cctvCataloniaSource.test.mjs b/src/data/cctvCataloniaSource.test.mjs new file mode 100644 index 000000000..be472784f --- /dev/null +++ b/src/data/cctvCataloniaSource.test.mjs @@ -0,0 +1,683 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; +import { + CATALONIA_CAMERAS_URL, + CATALONIA_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/constants.js'; +import { decodeNumericEntities } from '../../server/providers/cctv/normalize.js'; +import { + parseCataloniaXml, + normalizeCataloniaImageUrl, + cataloniaCameraName, + loadCataloniaSourcesFromOpenData, +} from '../../server/providers/cctv/sources.js'; + +/** Set (or, for `undefined`, delete) environment variables for one test. */ +function withEnv(t, env) { + for (const [name, value] of Object.entries(env)) { + const previous = process.env[name]; + t.after(() => { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + }); + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +} + +/** Silence the loaders' progress and failure logging. */ +function quiet(t) { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +} + +/** + * A response whose body is a live stream, plus a flag that flips when the + * stream is cancelled. A rejection path that returns without cancelling holds + * the transport open, so the flag is what the refusal tests actually assert. + */ +const streamingResponse = (init = {}) => { + const state = { cancelled: false }; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('` block matching the real feed's shape. */ +function cataloniaFeature({ + fid, + lon, + lat, + carretera = '', + municipi = '', + pk = '', + link = '', + font = '', +}) { + const pkTag = pk ? `${pk}` : ''; + return ( + `` + + `` + + `${lon},${lat}` + + `` + + `${carretera}` + + `${municipi}${pkTag}` + + `${link}` + + `${font}` + + `` + ); +} + +function cataloniaFeed(features) { + return ( + '' + + features.join('') + + '' + ); +} + +test('parseCataloniaXml extracts every field from a feature block', () => { + const xml = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-52bf6eca_1a0a06bea1c_386f', + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + municipi: 'Nus Trinitat', + pk: '0.50', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]); + const rows = parseCataloniaXml(xml); + assert.equal(rows.length, 1); + assert.deepEqual(rows[0], { + fid: 'cameres.fid-52bf6eca_1a0a06bea1c_386f', + lon: 2.1849528, + lat: 41.45989301, + carretera: 'C-58', + municipi: 'Nus Trinitat', + pk: '0.50', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }); +}); + +test('normalizeCataloniaImageUrl rewrites SCT RenderService links to the direct http TransitCamera URL', () => { + // RenderService always 302s to a hardcoded http:// Location (which + // fetchWithinHost refuses as a scheme downgrade), and mct.gencat.cat's TLS + // setup fails the handshake with Node's fetch regardless — TransitCamera + // over plain http is the only transport that actually works here, so we + // build that URL from RenderService's sctidcam id instead of registering + // the redirector. + assert.equal( + normalizeCataloniaImageUrl( + 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + ), + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=nc87.gif&visualitzacio=imatge', + ); + assert.equal( + normalizeCataloniaImageUrl( + 'https://mct.gencat.cat/mct2bo/RenderService?sctidcam=sc64.gif', + ), + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=sc64.gif&visualitzacio=imatge', + 'an already-https RenderService link is rewritten the same way', + ); + assert.equal( + normalizeCataloniaImageUrl('https://mct.gencat.cat/mct2bo/RenderService'), + '', + 'a RenderService link with no sctidcam id is rejected', + ); +}); + +test('normalizeCataloniaImageUrl accepts the other registered hosts, upgraded to https', () => { + assert.equal( + normalizeCataloniaImageUrl('http://www.bcn.cat/transit/imatges/a.gif'), + 'https://www.bcn.cat/transit/imatges/a.gif', + ); + assert.equal( + normalizeCataloniaImageUrl( + 'https://emap.terrassa.cat/it_terrassa/cam02.jpeg', + ), + 'https://emap.terrassa.cat/it_terrassa/cam02.jpeg', + ); + assert.equal( + normalizeCataloniaImageUrl('https://evil.example.com/mct.gencat.cat.gif'), + '', + 'a look-alike host is rejected', + ); + assert.equal( + normalizeCataloniaImageUrl('https://unlisted-publisher.example/frame.gif'), + '', + 'any host outside the allowlist is rejected', + ); + assert.equal(normalizeCataloniaImageUrl(''), ''); + assert.equal(normalizeCataloniaImageUrl('not a url'), ''); +}); + +test('normalizeCataloniaImageUrl rejects credential-bearing URLs on every allowed host', () => { + // The hostname matches the allowlist, so only an explicit userinfo check + // stops catalog data injecting credentials into the frame request. + for (const link of [ + 'https://user:pass@www.bcn.cat/transit/imatges/a.gif', + 'http://user@www.bcn.cat/transit/imatges/a.gif', + 'https://:pass@emap.terrassa.cat/it_terrassa/cam02.jpeg', + 'http://user:pass@mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + ]) { + assert.equal(normalizeCataloniaImageUrl(link), '', link); + } +}); + +test("normalizeCataloniaImageUrl rejects explicit ports and paths outside each host's frame directory", () => { + // An allowlisted hostname alone would let catalog data aim the frame + // request at another service or path on that host. + for (const link of [ + 'http://www.bcn.cat:8080/transit/imatges/a.gif', + 'https://emap.terrassa.cat:8443/it_terrassa/cam02.jpeg', + 'http://mct.gencat.cat:8080/mct2bo/RenderService?sctidcam=nc87.gif', + 'http://www.bcn.cat/admin/a.gif', + 'http://www.bcn.cat/transit/imatges/', + 'http://www.bcn.cat/transit/imatges/sub/a.gif', + 'http://www.bcn.cat/transit/imatges/../../admin/a.gif', + 'https://emap.terrassa.cat/other/cam02.jpeg', + 'http://mct.gencat.cat/mct2bo/OtherService?sctidcam=nc87.gif', + ]) { + assert.equal(normalizeCataloniaImageUrl(link), '', link); + } +}); + +test('normalizeCataloniaImageUrl accepts only a bare image filename as the SCT sctidcam', () => { + for (const camId of [ + '../../etc/passwd', + '..%2F..%2Fetc%2Fpasswd', + 'nc87.png', + 'nc87', + 'nc87.gif/../x.gif', + 'nc87.gif&nom=other.gif', + 'nc-87.gif', + ]) { + const link = `http://mct.gencat.cat/mct2bo/RenderService?sctidcam=${encodeURIComponent(camId)}`; + assert.equal(normalizeCataloniaImageUrl(link), '', camId); + } + assert.equal( + normalizeCataloniaImageUrl( + 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=CB10.JPEG', + ), + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=CB10.JPEG&visualitzacio=imatge', + ); +}); + +test('decodeNumericEntities decodes decimal and hex references and leaves invalid ones as written', () => { + assert.equal(decodeNumericEntities('Lluís'), 'Lluís'); + assert.equal(decodeNumericEntities('Lluís 🚦'), 'Lluís 🚦'); + assert.equal(decodeNumericEntities('Lluís'), 'Lluís'); + for (const raw of [ + 'a�b', + 'a�b', + 'a�b', + 'a�b', + 'a�b', + 'a�b', + 'a�b', + ]) { + assert.equal(decodeNumericEntities(raw), raw, raw); + } +}); + +test('cataloniaCameraName joins road and municipality, skipping a redundant municipality', () => { + assert.equal( + cataloniaCameraName('C-58', 'Nus Trinitat'), + 'C-58 (Nus Trinitat)', + ); + assert.equal( + cataloniaCameraName('Plaça Urquinaona', 'Barcelona'), + 'Plaça Urquinaona (Barcelona)', + ); + assert.equal( + cataloniaCameraName('Barcelona Ronda Litoral', 'Barcelona'), + 'Barcelona Ronda Litoral', + ); + assert.equal(cataloniaCameraName('', ''), 'Càmera de trànsit'); + assert.equal(cataloniaCameraName('C-58', ''), 'C-58'); +}); + +test('Catalonia loader keeps SCT, Barcelona and Terrassa cameras with per-publisher credit', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url, init) => { + requested.push([String(url), init.redirect]); + const xml = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-1', + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + municipi: 'Nus Trinitat', + pk: '0.50', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: 'cameres.fid-2', + lon: '2.17228212', + lat: '41.38890733', + carretera: 'Plaça Urquinaona', + municipi: 'Barcelona', + link: 'http://www.bcn.cat/transit/imatges/PlUrquinaona.gif?a=1', + font: 'IMI', + }), + cataloniaFeature({ + fid: 'cameres.fid-3', + lon: '2.0111', + lat: '41.5636', + carretera: 'Cam02', + municipi: 'Terrassa', + link: 'https://emap.terrassa.cat/it_terrassa/cam02.jpeg?a=1', + font: 'Terrassa', + }), + // Rejected: link host not in the allowlist. + cataloniaFeature({ + fid: 'cameres.fid-4', + lon: '2.0', + lat: '41.5', + link: 'https://example.com/fake.gif', + font: 'SCT', + }), + // Rejected: outside the Catalonia bounding box. + cataloniaFeature({ + fid: 'cameres.fid-5', + lon: '-3.7', + lat: '40.4', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=x.gif', + font: 'SCT', + }), + ]); + return new Response(xml, { + status: 200, + headers: { 'Content-Type': 'application/xml' }, + }); + }); + + const cameras = await loadCataloniaSourcesFromOpenData(); + + assert.deepEqual(requested, [[CATALONIA_CAMERAS_URL, 'manual']]); + assert.ok( + CATALONIA_CAMERAS_URL.startsWith('https://'), + 'the catalog is fetched over https', + ); + assert.deepEqual( + cameras.map((camera) => camera.id).sort(), + [ + 'cat-emap-terrassa-cat-it-terrassa-cam02-jpeg', + 'cat-sct-nc87', + 'cat-www-bcn-cat-transit-imatges-plurquinaona-gif', + ], + 'the off-allowlist link (fid-4) and the out-of-bbox row (fid-5) are excluded', + ); + + const sct = cameras.find((c) => c.id === 'cat-sct-nc87'); + assert.equal(sct.name, 'C-58 (Nus Trinitat)'); + assert.equal(sct.provider, 'Servei Català de Trànsit'); + assert.equal(sct.credit, '', 'SCT is the publisher, no separate credit'); + assert.equal( + sct.url, + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=nc87.gif&visualitzacio=imatge', + 'registers the direct, non-redirecting http frame URL, not the RenderService redirector', + ); + assert.equal(sct.groundElevationM, 80); + assert.equal(sct.headingConfidence, 'low', 'the feed carries no heading'); + + const bcn = cameras.find( + (c) => c.id === 'cat-www-bcn-cat-transit-imatges-plurquinaona-gif', + ); + assert.equal(bcn.credit, 'Ajuntament de Barcelona'); + assert.equal(bcn.city, 'Barcelona'); + assert.equal(bcn.groundElevationM, 20); + + const terrassa = cameras.find( + (c) => c.id === 'cat-emap-terrassa-cat-it-terrassa-cam02-jpeg', + ); + assert.equal(terrassa.credit, 'Ajuntament de Terrassa'); + assert.equal(terrassa.groundElevationM, 270); +}); + +test('Catalonia loader fails soft on HTTP errors, empty payloads and network errors', async (t) => { + quiet(t); + let respond; + t.mock.method(globalThis, 'fetch', (...args) => respond(...args)); + for (respond of [ + async () => new Response('unavailable', { status: 503 }), + async () => new Response(''), + async () => { + throw new TypeError('fetch failed'); + }, + ]) { + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + } +}); + +test('the Catalonia catalog fetch refuses redirects and oversized bodies', async (t) => { + t.mock.method(console, 'warn', () => {}); + // A redirect is never followed: the list host cannot be steered or + // downgraded. Its body is a live stream, so the test also proves the + // refusal releases the transport. + const seen = []; + const redirected = streamingResponse({ + status: 302, + headers: { location: 'http://www.gencat.cat/transit/opendata/cameres.xml' }, + }); + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init.redirect]); + return redirected.response; + }); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + assert.deepEqual(seen, [[CATALONIA_CAMERAS_URL, 'manual']]); + assert.equal( + redirected.state.cancelled, + true, + 'the redirect body is cancelled', + ); + + // An HTTP error body is released too. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + const failed = streamingResponse({ status: 503 }); + t.mock.method(globalThis, 'fetch', async () => failed.response); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + assert.equal(failed.state.cancelled, true, 'the error body is cancelled'); + + const validFeed = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-1', + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]); + + // A body that declares a size over the cap is refused rather than buffered, + // even though its actual content would parse. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + return new Response(validFeed, { + headers: { + 'Content-Type': 'application/xml', + 'content-length': String(CATALONIA_MAX_CATALOG_BYTES + 1), + }, + }); + }); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); + + // So is a body with no declared size that only turns out too long while + // streaming — a valid feed padded past the cap. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + const oversized = + validFeed + ' '.repeat(CATALONIA_MAX_CATALOG_BYTES + 1024); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(oversized)); + controller.close(); + }, + }), + { headers: { 'Content-Type': 'application/xml' } }, + ); + }); + assert.deepEqual(await loadCataloniaSourcesFromOpenData(), []); +}); + +test('CCTV catalog merges Catalonia cameras and CCTV_CATALONIA_ENABLED=0 skips the request', async (t) => { + quiet(t); + const sourceRoot = fs.mkdtempSync( + path.join(os.tmpdir(), 'gev-cctv-catalog-catalonia-'), + ); + t.after(() => fs.rmSync(sourceRoot, { recursive: true, force: true })); + withEnv(t, { + CCTV_SOURCES_FILE: undefined, + CCTV_SOURCES_JSON: undefined, + CCTV_FORCE_AUSTIN: undefined, + CCTV_PREFER_AUSTIN: undefined, + CCTV_MAX_SOURCES: undefined, + CCTV_CALTRANS_DISTRICTS: undefined, + CCTV_TFL_ENABLED: undefined, + CCTV_CATALONIA_ENABLED: undefined, + CCTV_CATALONIA_MAX_SOURCES: undefined, + }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url, init) => { + requested.push([String(url), init.redirect]); + if (String(url) !== CATALONIA_CAMERAS_URL) { + return new Response('unavailable', { status: 503 }); + } + const xml = cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-1', + lon: '2.1849528', + lat: '41.45989301', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]); + return new Response(xml, { status: 200 }); + }); + + const sources = await createCctvCatalog({ sourceRoot })(); + assert.deepEqual( + sources.map((source) => source.id), + ['cat-sct-nc87'], + ); + + process.env.CCTV_CATALONIA_ENABLED = '0'; + requested.length = 0; + assert.deepEqual(await createCctvCatalog({ sourceRoot })(), []); + assert.ok(requested.length > 0, 'the other live packs still load'); + assert.equal( + requested.some(([url]) => url === CATALONIA_CAMERAS_URL), + false, + 'a disabled pack makes no request', + ); +}); + +/** The two SCT, one Barcelona and one Terrassa rows of a real feed snapshot, + * under one upstream run's `fid` prefix. */ +function snapshotFeed(fidPrefix) { + return cataloniaFeed([ + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_386f`, + lon: '2.1849528', + lat: '41.45989301', + carretera: 'C-58', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_b48`, + lon: '2.18', + lat: '41.46', + carretera: 'C-31', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc85.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_1c2`, + lon: '2.17228212', + lat: '41.38890733', + carretera: 'Plaça Urquinaona', + municipi: 'Barcelona', + link: 'http://www.bcn.cat/transit/imatges/PlUrquinaona.gif?a=1', + font: 'IMI', + }), + cataloniaFeature({ + fid: `cameres.fid-${fidPrefix}_1c3`, + lon: '2.0111', + lat: '41.5636', + municipi: 'Terrassa', + link: 'https://emap.terrassa.cat/it_terrassa/cam01.jpeg?a=1 ', + font: 'Terrassa', + }), + ]); +} + +async function loadFromFeed(t, xml) { + t.mock.method(globalThis, 'fetch', async () => { + return new Response(xml, { + status: 200, + headers: { 'Content-Type': 'application/xml' }, + }); + }); + try { + return await loadCataloniaSourcesFromOpenData(); + } finally { + t.mock.restoreAll(); + } +} + +test('Catalonia camera ids survive an upstream restart that changes every fid', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + // The fid embeds the upstream server-start stamp: the PR fixture's + // 1a0a06bea1c run and a later 1a140311953 run name the same cameras. + const before = await loadFromFeed(t, snapshotFeed('52bf6eca_1a0a06bea1c')); + const after = await loadFromFeed(t, snapshotFeed('52bf6eca_1a140311953')); + const ids = (cameras) => cameras.map((camera) => camera.id).sort(); + assert.deepEqual(ids(before), [ + 'cat-emap-terrassa-cat-it-terrassa-cam01-jpeg', + 'cat-sct-nc85', + 'cat-sct-nc87', + 'cat-www-bcn-cat-transit-imatges-plurquinaona-gif', + ]); + assert.deepEqual(ids(after), ids(before)); +}); + +test('Catalonia loader collapses rows the feed repeats for one camera', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + // The live feed lists nc85, cb5, cb10, cb25 and cd17 twice each, under + // different fids. + const row = (fid, camId) => + cataloniaFeature({ + fid, + lon: '2.18', + lat: '41.46', + carretera: 'C-31', + link: `http://mct.gencat.cat/mct2bo/RenderService?sctidcam=${camId}`, + font: 'SCT', + }); + const cameras = await loadFromFeed( + t, + cataloniaFeed([ + row('cameres.fid-x_1', 'nc85.gif'), + row('cameres.fid-x_2', 'cb5.gif'), + row('cameres.fid-x_3', 'nc85.gif'), + row('cameres.fid-x_4', 'cb5.gif'), + row('cameres.fid-x_5', 'cb10.gif'), + ]), + ); + assert.deepEqual(cameras.map((camera) => camera.id).sort(), [ + 'cat-sct-cb10', + 'cat-sct-cb5', + 'cat-sct-nc85', + ]); +}); + +test('Catalonia loader drops ported and bad-sctidcam links but keeps the rest', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + const cameras = await loadFromFeed( + t, + cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-x_1', + lon: '2.17', + lat: '41.39', + link: 'http://www.bcn.cat:8080/transit/imatges/PlUrquinaona.gif?a=1', + font: 'IMI', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_2', + lon: '2.0111', + lat: '41.5636', + link: 'https://emap.terrassa.cat:8443/it_terrassa/cam02.jpeg?a=1', + font: 'Terrassa', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_3', + lon: '2.18', + lat: '41.46', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=../../etc/passwd', + font: 'SCT', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_4', + lon: '2.18', + lat: '41.46', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]), + ); + assert.deepEqual( + cameras.map((camera) => [camera.id, camera.url]), + [ + [ + 'cat-sct-nc87', + 'http://mct.gencat.cat/mct2bo/TransitCamera?nom=nc87.gif&visualitzacio=imatge', + ], + ], + ); +}); + +test('a malformed numeric entity in one row neither throws nor empties the Catalonia pack', async (t) => { + quiet(t); + withEnv(t, { CCTV_CATALONIA_MAX_SOURCES: undefined }); + const cameras = await loadFromFeed( + t, + cataloniaFeed([ + cataloniaFeature({ + fid: 'cameres.fid-x_1', + lon: '2.18', + lat: '41.46', + carretera: 'C-31 �', + municipi: 'Bad�', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc85.gif', + font: 'SCT', + }), + cataloniaFeature({ + fid: 'cameres.fid-x_2', + lon: '2.18', + lat: '41.46', + carretera: 'Plaça Catalunya', + municipi: 'Llinars del Vallès', + link: 'http://mct.gencat.cat/mct2bo/RenderService?sctidcam=nc87.gif', + font: 'SCT', + }), + ]), + ); + const byId = new Map(cameras.map((camera) => [camera.id, camera])); + assert.equal(cameras.length, 2); + assert.equal( + byId.get('cat-sct-nc85').name, + 'C-31 � (Bad�)', + 'an invalid reference is kept as written', + ); + assert.equal( + byId.get('cat-sct-nc87').name, + 'Plaça Catalunya (Llinars del Vallès)', + ); +}); diff --git a/src/data/cctvDriveBcSource.test.mjs b/src/data/cctvDriveBcSource.test.mjs index 617bb1350..63988de5e 100644 --- a/src/data/cctvDriveBcSource.test.mjs +++ b/src/data/cctvDriveBcSource.test.mjs @@ -17,6 +17,7 @@ import { DEFAULT_TALLINN_MAX_SOURCES, DEFAULT_TARKTEE_MAX_SOURCES, DEFAULT_NSW_MAX_SOURCES, + DEFAULT_CATALONIA_MAX_SOURCES, DRIVEBC_WEBCAMS_URL, } from '../../server/providers/cctv/constants.js'; import { @@ -221,7 +222,7 @@ test('CCTV catalog merges DriveBC cameras and CCTV_DRIVEBC_ENABLED=0 skips the r ); }); -test('default per-pack camera caps fit inside the default catalog cap', () => { +test('default caps of the original packs fit inside the default catalog cap', () => { // Every default pack at its own cap (Warendorf ships 1 curated camera). const packs = DEFAULT_AUSTIN_MAX_SOURCES + @@ -234,6 +235,7 @@ test('default per-pack camera caps fit inside the default catalog cap', () => { DEFAULT_TALLINN_MAX_SOURCES + DEFAULT_TARKTEE_MAX_SOURCES + DEFAULT_NSW_MAX_SOURCES + + DEFAULT_CATALONIA_MAX_SOURCES + 1; assert.ok( packs <= DEFAULT_CCTV_MAX_SOURCES, diff --git a/src/data/cctvKingCountySource.test.mjs b/src/data/cctvKingCountySource.test.mjs new file mode 100644 index 000000000..5da0ae708 --- /dev/null +++ b/src/data/cctvKingCountySource.test.mjs @@ -0,0 +1,367 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { + DEFAULT_KING_COUNTY_MAX_SOURCES, + KING_COUNTY_CAMERAS_URL, + KING_COUNTY_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/constants.js'; +import { loadKingCountySourcesFromOpenData } from '../../server/providers/cctv/sources.js'; +import { normalizeSourceItem } from '../../server/providers/cctv/normalize.js'; +import { cctvProxy } from '../../server/providers/cctv.js'; +import { createCatalog } from '../layers/cctv/catalog.js'; + +/** Set (or, for `undefined`, delete) environment variables for one test. */ +function withEnv(t, env) { + for (const [name, value] of Object.entries(env)) { + const previous = process.env[name]; + t.after(() => { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + }); + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +} + +/** Silence the loaders' progress and failure logging. */ +function quiet(t) { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +} + +/** A FeatureServer GeoJSON feature carrying the fields the loader reads. */ +function kingCountyFeature(id, lat, lon, overrides = {}) { + const { properties = {}, geometry } = overrides; + return { + type: 'Feature', + geometry: + geometry === undefined + ? { type: 'Point', coordinates: [lon, lat] } + : geometry, + properties: { + AssetID: id, + Location: `Road ${id} at Test Ave`, + ImageURL: `http://info.kingcounty.gov/transportation/kcdot/Roads/TrafficCameras/ImageHandler/Handler.ashx?id=cam${id}.jpg`, + Live: 1, + Model: '3950', + Manufacturer: 'Cohu', + Owner: 'King County', + CurrentStatus: 'Active', + ...properties, + }, + }; +} + +test('King County loader keeps live county cameras and upgrades frame URLs to HTTPS', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: undefined }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + return Response.json({ + type: 'FeatureCollection', + features: [ + kingCountyFeature(56, 47.70059, -122.02443, { + properties: { + Location: 'NE Novelty Hill Road at Trilogy Parkway NE', + // The live layer carries stray whitespace in model strings. + Model: 'Q6135-LE\n', + Manufacturer: 'Axis', + }, + }), + // City-owned camera the county hosts: kept, owner becomes the credit. + kingCountyFeature(90, 47.674, -122.119, { + properties: { Owner: 'City of Newcastle', Model: null }, + }), + // WSDOT-owned row: its frames are WSDOT-hosted, so it is skipped. + kingCountyFeature(70, 47.6, -122.3, { properties: { Owner: 'WSDOT' } }), + // Off-host frame, not live, retired, bad id, bad geometry, out of county. + kingCountyFeature(71, 47.6, -122.3, { + properties: { ImageURL: 'https://example.com/cam71.jpg' }, + }), + kingCountyFeature(72, 47.6, -122.3, { properties: { Live: 0 } }), + kingCountyFeature(73, 47.6, -122.3, { + properties: { CurrentStatus: 'Retired' }, + }), + kingCountyFeature('74', 47.6, -122.3), + kingCountyFeature(75, 47.6, -122.3, { geometry: null }), + kingCountyFeature(76, 34.05, -118.24), + ], + }); + }); + + const cameras = await loadKingCountySourcesFromOpenData(); + + assert.deepEqual(requested, [KING_COUNTY_CAMERAS_URL]); + assert.deepEqual(cameras.map((camera) => camera.id).sort(), [ + 'kingcounty-56', + 'kingcounty-90', + ]); + + const novelty = cameras.find((camera) => camera.id === 'kingcounty-56'); + assert.equal(novelty.name, 'NE Novelty Hill Road at Trilogy Parkway NE'); + assert.equal(novelty.provider, 'King County Road Services'); + assert.equal( + novelty.url, + 'https://info.kingcounty.gov/transportation/kcdot/Roads/TrafficCameras/ImageHandler/Handler.ashx?id=cam56.jpg', + ); + assert.equal(novelty.snapshotUrl, novelty.url); + assert.equal(novelty.feedType, 'image'); + // Published hardware model rides along, whitespace-collapsed. + assert.equal(novelty.model, 'Q6135-LE'); + assert.equal(novelty.manufacturer, 'Axis'); + assert.equal(novelty.credit, ''); + // No facing in the layer: id-hash fallback at low confidence. + assert.equal(novelty.headingConfidence, 'low'); + assert.equal(novelty.fovDeg, 44); + assert.ok(novelty.headingDeg >= 0 && novelty.headingDeg < 360); + + const newcastle = cameras.find((camera) => camera.id === 'kingcounty-90'); + assert.equal(newcastle.credit, 'City of Newcastle'); + assert.equal(newcastle.model, undefined); + // A manufacturer without a model identifies nothing, so it is dropped too. + assert.equal(newcastle.manufacturer, undefined); +}); + +test('normalizeSourceItem passes a hardware model through and omits it otherwise', () => { + const withModel = normalizeSourceItem({ + id: 'kingcounty-56', + lat: 47.7, + lon: -122.0, + model: ' Q6135-LE ', + manufacturer: ' Axis ', + }); + assert.equal(withModel.model, 'Q6135-LE'); + assert.equal(withModel.manufacturer, 'Axis'); + + const withoutModel = normalizeSourceItem({ + id: 'austin-1', + lat: 30.3, + lon: -97.7, + }); + assert.equal(withoutModel.model, undefined); + assert.equal(withoutModel.manufacturer, undefined); + assert.equal(normalizeSourceItem({ id: 'x', model: ' ' }).model, undefined); +}); + +/** Drive the real `/api/cctv/sources` middleware and return its parsed body. */ +async function requestSources(plugin) { + let handler; + plugin.configureServer({ + middlewares: { + use(_route, fn) { + handler = fn; + }, + }, + }); + const res = { + writeHead(status, headers) { + Object.assign(this, { status, headers }); + }, + end(body) { + this.body = body; + }, + }; + await handler({ url: '/sources', method: 'GET' }, res); + assert.equal(res.status, 200); + return JSON.parse(res.body); +} + +/** The browser catalog with the pose math stubbed out: only field + * passthrough is under test here. */ +function browserCatalog() { + const model = { + safeNumber: (value, fallback = NaN) => + Number.isFinite(Number(value)) ? Number(value) : fallback, + normalizeHeading: (deg) => ((deg % 360) + 360) % 360, + clamp: (value, lo, hi) => Math.min(hi, Math.max(lo, value)), + headingFromId: () => 0, + normalizeFeedType: (type) => type, + ensureCameraPose: () => {}, + }; + return createCatalog({ + state: {}, + services: { locations: { CITY_POIS: {} } }, + parts: { model }, + source: {}, + }); +} + +test('King County model and manufacturer survive the /api/cctv/sources boundary into the browser catalog', async (t) => { + quiet(t); + const root = mkdtempSync(path.join(tmpdir(), 'gev-kingcounty-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + // No file/env catalog, so the live packs load; every pack but King County + // sees an upstream failure and contributes nothing. + withEnv(t, { + CCTV_SOURCES_FILE: undefined, + CCTV_SOURCES_JSON: undefined, + CCTV_FORCE_AUSTIN: undefined, + CCTV_PREFER_AUSTIN: undefined, + CCTV_KINGCOUNTY_ENABLED: undefined, + CCTV_KINGCOUNTY_MAX_SOURCES: undefined, + CCTV_MAX_SOURCES: undefined, + }); + t.mock.method(globalThis, 'fetch', async (url) => { + if (String(url) !== KING_COUNTY_CAMERAS_URL) + return new Response('', { status: 503 }); + return Response.json({ + type: 'FeatureCollection', + features: [ + kingCountyFeature(56, 47.70059, -122.02443, { + properties: { Model: 'Q6135-LE\n', Manufacturer: 'Axis' }, + }), + kingCountyFeature(90, 47.674, -122.119, { + properties: { Model: null, Manufacturer: 'Cohu' }, + }), + ], + }); + }); + + const body = await requestSources(cctvProxy({ sourceRoot: root })); + const served = new Map(body.sources.map((source) => [source.id, source])); + assert.deepEqual([...served.keys()].sort(), [ + 'kingcounty-56', + 'kingcounty-90', + ]); + assert.equal(served.get('kingcounty-56').model, 'Q6135-LE'); + assert.equal(served.get('kingcounty-56').manufacturer, 'Axis'); + // Absent hardware is an explicit empty string on the wire, like + // headingConfidence, so consumers never have to probe for the key. + assert.equal(served.get('kingcounty-90').model, ''); + assert.equal(served.get('kingcounty-90').manufacturer, ''); + + const cameras = browserCatalog().buildCatalogFromSources(body.sources); + const byId = new Map(cameras.map((camera) => [camera.id, camera])); + assert.equal(byId.get('kingcounty-56').model, 'Q6135-LE'); + assert.equal(byId.get('kingcounty-56').manufacturer, 'Axis'); + assert.equal(byId.get('kingcounty-90').model, ''); + assert.equal(byId.get('kingcounty-90').manufacturer, ''); +}); + +test('King County loader honors the max-sources cap', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: '8' }); + const features = []; + for (let i = 0; i < 24; i += 1) { + features.push( + kingCountyFeature(100 + i, 47.2 + i * 0.02, -122.4 + i * 0.02), + ); + } + t.mock.method(globalThis, 'fetch', async () => + Response.json({ type: 'FeatureCollection', features }), + ); + + const cameras = await loadKingCountySourcesFromOpenData(); + assert.equal(cameras.length, 8); +}); + +test('King County loader defaults its cap and survives upstream failure', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: undefined }); + assert.equal(DEFAULT_KING_COUNTY_MAX_SOURCES, 125); + + t.mock.method( + globalThis, + 'fetch', + async () => new Response('', { status: 503 }), + ); + assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); + + t.mock.method(globalThis, 'fetch', async () => { + throw new Error('network down'); + }); + assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); +}); + +test('King County loader does not follow a redirected catalog', async (t) => { + quiet(t); + let init; + t.mock.method(globalThis, 'fetch', async (_url, options) => { + init = options; + return new Response( + JSON.stringify({ + type: 'FeatureCollection', + features: [kingCountyFeature(56, 47.70059, -122.02443)], + }), + { status: 302, headers: { Location: 'https://example.com/cams' } }, + ); + }); + + assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); + assert.equal(init.redirect, 'manual'); +}); + +test('King County loader rejects an oversized catalog', async (t) => { + quiet(t); + const features = [kingCountyFeature(56, 47.70059, -122.02443)]; + const body = JSON.stringify({ + type: 'FeatureCollection', + features, + padding: 'x'.repeat(KING_COUNTY_MAX_CATALOG_BYTES), + }); + t.mock.method(globalThis, 'fetch', async () => new Response(body)); + + assert.deepEqual(await loadKingCountySourcesFromOpenData(), []); +}); + +test('King County loader drops duplicate ids before the per-pack cap', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: '8' }); + // Repeats of the camera nearest the anchor would fill an 8-camera cap + // before the second, farther camera if duplicates reached prioritization. + const features = []; + for (let i = 0; i < 10; i += 1) { + features.push(kingCountyFeature(56, 47.4829, -122.2171)); + } + features.push(kingCountyFeature(57, 47.70059, -122.02443)); + t.mock.method(globalThis, 'fetch', async () => + Response.json({ type: 'FeatureCollection', features }), + ); + + const cameras = await loadKingCountySourcesFromOpenData(); + assert.deepEqual(cameras.map((camera) => camera.id).sort(), [ + 'kingcounty-56', + 'kingcounty-57', + ]); +}); + +test('King County loader omits vendor-generic model strings', async (t) => { + quiet(t); + withEnv(t, { CCTV_KINGCOUNTY_MAX_SOURCES: undefined }); + t.mock.method(globalThis, 'fetch', async () => + Response.json({ + type: 'FeatureCollection', + features: [ + kingCountyFeature(61, 47.5, -122.2, { + properties: { Model: 'PTZ', Manufacturer: 'Bosch' }, + }), + kingCountyFeature(62, 47.51, -122.2, { + properties: { Model: 'PTZ IP', Manufacturer: 'Cohu' }, + }), + kingCountyFeature(63, 47.52, -122.2, { + properties: { Model: 'BOSCH/PTZ', Manufacturer: 'Bosch' }, + }), + kingCountyFeature(64, 47.53, -122.2, { + properties: { Model: 'AUTODOME IP 5000', Manufacturer: 'Bosch' }, + }), + ], + }), + ); + + const byId = new Map( + (await loadKingCountySourcesFromOpenData()).map((camera) => [ + camera.id, + camera, + ]), + ); + for (const id of ['kingcounty-61', 'kingcounty-62', 'kingcounty-63']) { + assert.equal(byId.get(id).model, undefined, id); + assert.equal(byId.get(id).manufacturer, undefined, id); + } + assert.equal(byId.get('kingcounty-64').model, 'AUTODOME IP 5000'); + assert.equal(byId.get('kingcounty-64').manufacturer, 'Bosch'); +}); diff --git a/src/data/cctvLithuania.test.mjs b/src/data/cctvLithuania.test.mjs new file mode 100644 index 000000000..af28a51f6 --- /dev/null +++ b/src/data/cctvLithuania.test.mjs @@ -0,0 +1,198 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + lithuaniaCameraToSource, + loadLithuaniaSources, + LITHUANIA_CAMERAS_URL, + LITHUANIA_LOCATIONS_URL, + LITHUANIA_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/lithuania.js'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; + +const row = { + id: 72, + name: 'Vilnius A1 10,04', + x: 576154, + y: 6056867, + image: + 'https://eismoinfo.lt/eismoinfo-backend/image-provider/camera/last?id=72', +}; +const feature = { + id: '72', + points: [{ point: [54.642550940833566, 25.17981587877114] }], +}; +const layers = [{ layer: 'VKR', features: [feature] }]; + +test('Lithuania joins by ID and uses WGS84 latitude/longitude, never LKS-94 metres', () => { + const camera = lithuaniaCameraToSource(row, feature); + assert.equal(camera.id, 'lithuania-72'); + assert.equal(camera.lat, 54.642550940833566); + assert.equal(camera.lon, 25.17981587877114); + assert.equal(camera.name, 'Vilnius A1 10,04'); + assert.equal(camera.feedType, 'image'); + assert.equal(camera.headingConfidence, 'low'); + assert.equal(camera.url, row.image); + assert.match(camera.credit, /Via Lietuva/); +}); + +test('invalid joins, coordinates and image destinations cannot enter the proxy registry', () => { + for (const invalid of [ + null, + { ...feature, id: '8' }, + { ...feature, points: [] }, + { ...feature, points: [{ point: [25.18, 54.64] }] }, + { ...feature, points: [{ point: [576154, 6056867] }] }, + { ...feature, points: [{ point: [null, 25] }] }, + ]) { + assert.equal(lithuaniaCameraToSource(row, invalid), null); + } + for (const image of [ + 'http://eismoinfo.lt/eismoinfo-backend/image-provider/camera/last?id=72', + row.image.replace('eismoinfo.lt', 'eismoinfo.lt.evil.test'), + row.image.replace('https://', 'https://user:pass@'), + row.image.replace('id=72', 'id=8'), + 'http://127.0.0.1/private', + '', + null, + ]) { + assert.equal(lithuaniaCameraToSource({ ...row, image }, feature), null); + } + assert.equal( + lithuaniaCameraToSource( + { ...row, image: row.image + '&extra=ignored' }, + feature, + ).url, + row.image, + ); + assert.equal(lithuaniaCameraToSource(null, feature), null); +}); + +function mockCatalogs(t) { + const requested = []; + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async (url, options) => { + requested.push(String(url)); + if (url === LITHUANIA_CAMERAS_URL) { + assert.equal(options.redirect, 'manual'); + return Response.json([row, row, { ...row, id: 999 }]); + } + if (url === LITHUANIA_LOCATIONS_URL) return Response.json(layers); + return Response.json([]); + }); + return requested; +} + +test('loader deduplicates and omits cameras without matching published coordinates', async (t) => { + const requested = mockCatalogs(t); + assert.deepEqual( + (await loadLithuaniaSources()).map((c) => c.id), + ['lithuania-72'], + ); + assert.deepEqual( + requested.sort(), + [LITHUANIA_CAMERAS_URL, LITHUANIA_LOCATIONS_URL].sort(), + ); +}); + +/** Run one catalog refresh with CCTV_LITHUANIA_ENABLED set to `flag` + * (undefined leaves it unset), restoring the environment afterwards. */ +async function runCatalogWithLithuaniaFlag(t, flag) { + const requested = mockCatalogs(t); + const saved = process.env.CCTV_LITHUANIA_ENABLED; + try { + if (flag === undefined) delete process.env.CCTV_LITHUANIA_ENABLED; + else process.env.CCTV_LITHUANIA_ENABLED = flag; + const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); + return { requested, sources }; + } finally { + if (saved === undefined) delete process.env.CCTV_LITHUANIA_ENABLED; + else process.env.CCTV_LITHUANIA_ENABLED = saved; + } +} + +test('CCTV_LITHUANIA_ENABLED=1 wires the Lithuania pack into the catalog', async (t) => { + const { sources } = await runCatalogWithLithuaniaFlag(t, '1'); + assert.equal( + sources.find((c) => c.id === 'lithuania-72')?.cityId, + 'lithuania', + ); +}); + +// The camera endpoints are the eismoinfo.lt website backend, not a service the +// open-data statement lists, so an install that has not opted in must never +// contact them. +for (const [label, flag] of [ + ['an unset CCTV_LITHUANIA_ENABLED', undefined], + ['an empty CCTV_LITHUANIA_ENABLED', ''], + ['CCTV_LITHUANIA_ENABLED=0', '0'], + ['CCTV_LITHUANIA_ENABLED=true', 'true'], +]) { + test(`${label} makes no Lithuania request`, async (t) => { + const { requested, sources } = await runCatalogWithLithuaniaFlag(t, flag); + assert.ok(requested.length > 0, 'the other packs still refreshed'); + assert.equal( + requested.some((href) => href.includes('eismoinfo.lt')), + false, + 'the pack never reaches its upstream', + ); + assert.ok(!sources.some((c) => c.cityId === 'lithuania')); + }); +} + +test('a failed catalog cancels the sibling request instead of waiting for the timeout', async (t) => { + t.mock.method(console, 'warn', () => {}); + let siblingSignal; + t.mock.method(globalThis, 'fetch', async (url, options) => { + if (url === LITHUANIA_CAMERAS_URL) { + return new Response('', { status: 503 }); + } + siblingSignal = options.signal; + // Never settles on its own; only an abort ends it. + return new Promise((_, reject) => { + options.signal.addEventListener('abort', () => + reject(options.signal.reason), + ); + }); + }); + assert.deepEqual(await loadLithuaniaSources(), []); + assert.equal(siblingSignal?.aborted, true); + assert.notEqual(siblingSignal.reason?.name, 'TimeoutError'); +}); + +test('a redirected catalog is reported as not followed', async (t) => { + const warnings = []; + t.mock.method(console, 'warn', (...args) => warnings.push(args.join(' '))); + t.mock.method( + globalThis, + 'fetch', + async () => + new Response('', { + status: 302, + headers: { Location: 'http://127.0.0.1/' }, + }), + ); + assert.deepEqual(await loadLithuaniaSources(), []); + assert.ok(warnings.some((line) => /redirects are not followed/.test(line))); +}); + +test('failed, redirected, malformed and oversized catalogs degrade to an empty pack', async (t) => { + t.mock.method(console, 'warn', () => {}); + for (const makeResponse of [ + () => new Response('', { status: 503 }), + () => + new Response('', { + status: 302, + headers: { Location: 'http://127.0.0.1/' }, + }), + () => new Response('invalid json'), + () => Response.json({ unexpected: true }), + () => + new Response('[]', { + headers: { 'Content-Length': String(LITHUANIA_MAX_CATALOG_BYTES + 1) }, + }), + ]) { + t.mock.method(globalThis, 'fetch', async () => makeResponse()); + assert.deepEqual(await loadLithuaniaSources(), []); + } +}); diff --git a/src/data/cctvLod.js b/src/data/cctvLod.js index 65ddc2e12..44b2e8ba3 100644 --- a/src/data/cctvLod.js +++ b/src/data/cctvLod.js @@ -38,6 +38,10 @@ const PROVIDER_STATIC_REFRESH_MS = Object.freeze({ // collectionInterval, 600 s for every station sampled; polling faster only // re-fetches the same JPEG. fintraffic: 10 * 60 * 1000, + // Measured empirically (2026-09-15): polling the same SCT/Terrassa frame + // every 20s showed no change inside 64s but a new frame by ~90-150s later — + // matches the other highway packs' 3-minute cadence. + 'servei català de trànsit': 3 * 60 * 1000, }); /** diff --git a/src/data/cctvLod.test.mjs b/src/data/cctvLod.test.mjs index c6f6e4923..4f5982496 100644 --- a/src/data/cctvLod.test.mjs +++ b/src/data/cctvLod.test.mjs @@ -641,6 +641,10 @@ test('staticFrameRefreshMs follows known pack cadences and bounds explicit value assert.equal(staticFrameRefreshMs({ provider: 'TxDOT' }), 180_000); // Digitraffic publishes one weathercam frame per 600 s collection interval. assert.equal(staticFrameRefreshMs({ provider: 'Fintraffic' }), 600_000); + assert.equal( + staticFrameRefreshMs({ provider: 'Servei Català de Trànsit' }), + 180_000, + ); assert.equal(staticFrameRefreshMs({ frameRefreshMs: 5_000 }), 60_000); assert.equal(staticFrameRefreshMs({ frameRefreshMs: 3_000_000 }), 1_200_000); assert.equal(staticFrameRefreshMs({ provider: 'Unknown Provider' }), 300_000); diff --git a/src/data/cctvMediaFeedType.test.mjs b/src/data/cctvMediaFeedType.test.mjs new file mode 100644 index 000000000..5791e433b --- /dev/null +++ b/src/data/cctvMediaFeedType.test.mjs @@ -0,0 +1,270 @@ +// /api/cctv/media is the live-feed relay. A still camera used to be relayed +// through it as well, by an ordinary redirect-following fetch with no content +// type check and no body cap, which side-stepped everything /api/cctv/frame +// enforces for the same URL. These pin that a still is refused there and only +// reachable through /frame, while live feeds keep their relay. +// +// Run with: npm test (node --test) +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { Readable } from 'node:stream'; +import { cctvProxy } from '../../server/providers/cctv.js'; + +const STILL = { + id: 'still-fixture', + name: 'Still fixture', + lat: 30.27, + lon: -97.74, + feedType: 'image', + url: 'https://cams.upstream.invalid/still.jpg', +}; +// 'jpg' normalizes to 'image'; the alias must not reopen the relay. +const STILL_ALIAS = { + ...STILL, + id: 'still-alias', + feedType: 'jpg', + url: 'https://cams.upstream.invalid/alias.jpg', +}; +const VIDEO = { + id: 'video-fixture', + name: 'Video fixture', + lat: 30.27, + lon: -97.74, + feedType: 'video', + url: 'https://cams.upstream.invalid/live.mp4', +}; +const MJPEG = { + id: 'mjpeg-fixture', + name: 'MJPEG fixture', + lat: 30.27, + lon: -97.74, + feedType: 'mjpeg', + url: 'https://cams.upstream.invalid/live.mjpg', +}; + +/** A minimal upstream Response with a streaming body. */ +function upstreamResponse({ status = 200, headers = {}, body = 'payload' }) { + const store = new Map( + Object.entries(headers).map(([key, value]) => [key.toLowerCase(), value]), + ); + return { + ok: status >= 200 && status < 300, + status, + headers: { get: (name) => store.get(String(name).toLowerCase()) ?? null }, + body: Readable.toWeb(Readable.from([Buffer.from(String(body))])), + }; +} + +/** Collects what a handler wrote, the way a Node ServerResponse would. */ +function recordingResponse() { + const chunks = []; + let finished; + const done = new Promise((resolve) => { + finished = resolve; + }); + const res = { + statusCode: 0, + headers: {}, + writableEnded: false, + body: '', + writeHead(status, headers) { + res.statusCode = status; + res.headers = headers || {}; + }, + write(chunk) { + chunks.push(Buffer.from(chunk)); + return true; + }, + end(chunk) { + if (chunk) chunks.push(Buffer.from(chunk)); + res.writableEnded = true; + res.body = Buffer.concat(chunks).toString('utf8'); + finished(); + }, + on() {}, + once() {}, + emit() {}, + destroy() {}, + }; + return { res, done }; +} + +/** + * Mount the real provider over the fixture cameras and answer every upstream + * request from the supplied function. + */ +function mount(t, respond) { + const before = { + json: process.env.CCTV_SOURCES_JSON, + file: process.env.CCTV_SOURCES_FILE, + austin: process.env.CCTV_FORCE_AUSTIN, + serverKey: process.env.GOOGLE_MAPS_SERVER_API_KEY, + browserKey: process.env.GOOGLE_MAPS_API_KEY, + }; + process.env.CCTV_SOURCES_JSON = JSON.stringify([ + STILL, + STILL_ALIAS, + VIDEO, + MJPEG, + ]); + process.env.CCTV_SOURCES_FILE = 'absent-source-file.json'; + process.env.CCTV_FORCE_AUSTIN = '0'; + // No Street View key: a /frame miss lands on the synthetic billboard rather + // than a network call this file does not model. + delete process.env.GOOGLE_MAPS_SERVER_API_KEY; + delete process.env.GOOGLE_MAPS_API_KEY; + const nativeFetch = globalThis.fetch; + const requests = []; + globalThis.fetch = async (url, init = {}) => { + requests.push({ url: String(url), redirect: init.redirect }); + return respond({ url: String(url), init }); + }; + t.after(() => { + globalThis.fetch = nativeFetch; + for (const [name, value] of [ + ['CCTV_SOURCES_JSON', before.json], + ['CCTV_SOURCES_FILE', before.file], + ['CCTV_FORCE_AUSTIN', before.austin], + ['GOOGLE_MAPS_SERVER_API_KEY', before.serverKey], + ['GOOGLE_MAPS_API_KEY', before.browserKey], + ]) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + }); + + let handler = null; + cctvProxy().configureServer({ + middlewares: { + use: (_route, fn) => { + handler = fn; + }, + }, + }); + assert.ok(handler, 'the CCTV provider must mount a middleware'); + + const call = async (path) => { + const { res, done } = recordingResponse(); + await handler({ url: path, headers: {}, method: 'GET', on() {} }, res); + await done; + return res; + }; + const health = async (id) => { + const res = await call('/health'); + return JSON.parse(res.body).cameras.find((entry) => entry.id === id); + }; + return { call, health, requests }; +} + +test('a still camera is refused by /media without any upstream request', async (t) => { + // Everything the old relay would have done with this camera: redirect to a + // private address, then answer with HTML. + const app = mount(t, ({ url }) => + url.includes('169.254.169.254') + ? upstreamResponse({ + headers: { 'content-type': 'text/html' }, + body: 'metadata', + }) + : upstreamResponse({ + status: 302, + headers: { location: 'http://169.254.169.254/latest/meta-data/' }, + body: '', + }), + ); + for (const camera of [STILL, STILL_ALIAS]) { + const res = await app.call(`/media/${camera.id}`); + assert.equal(res.statusCode, 404, camera.id); + assert.equal(res.headers['Content-Type'], 'application/json'); + assert.deepEqual(JSON.parse(res.body), { + error: 'No live media for this camera; use the frame endpoint', + }); + assert.doesNotMatch(res.body, /metadata/); + // Being asked on the wrong route is not a camera fault. + assert.equal(await app.health(camera.id), undefined); + } + assert.deepEqual(app.requests, [], 'a still camera reached its upstream'); +}); + +test('an unregistered camera id is refused by /media the same way', async (t) => { + const app = mount(t, () => { + throw new Error('an unknown camera must not fetch'); + }); + const res = await app.call('/media/not-a-camera'); + assert.equal(res.statusCode, 404); + assert.equal(app.requests.length, 0); +}); + +test('a video camera is still relayed by /media', async (t) => { + const app = mount(t, () => + upstreamResponse({ + headers: { 'content-type': 'video/mp4', 'content-length': '5' }, + body: 'video', + }), + ); + const res = await app.call(`/media/${VIDEO.id}`); + assert.equal(res.statusCode, 200); + assert.equal(res.headers['Content-Type'], 'video/mp4'); + assert.equal(res.headers['X-CCTV-Source'], 'live-media'); + assert.equal(res.body, 'video'); + assert.deepEqual( + app.requests.map(({ url }) => url), + [VIDEO.url], + ); + assert.equal((await app.health(VIDEO.id)).status, 'ok'); +}); + +test('an MJPEG camera is still relayed by /media', async (t) => { + const app = mount(t, () => + upstreamResponse({ + headers: { 'content-type': 'multipart/x-mixed-replace; boundary=frame' }, + body: '--frame', + }), + ); + const res = await app.call(`/media/${MJPEG.id}`); + assert.equal(res.statusCode, 200); + assert.match(res.headers['Content-Type'], /^multipart\/x-mixed-replace/); + assert.equal(res.body, '--frame'); + assert.deepEqual( + app.requests.map(({ url }) => url), + [MJPEG.url], + ); +}); + +test('a still camera is still served through /frame with its bounded fetch', async (t) => { + const app = mount(t, () => + upstreamResponse({ + headers: { 'content-type': 'image/jpeg' }, + body: 'jpeg-bytes', + }), + ); + const res = await app.call(`/frame/${STILL.id}`); + assert.equal(res.statusCode, 200); + assert.equal(res.headers['Content-Type'], 'image/jpeg'); + assert.equal(res.headers['X-CCTV-Source'], 'upstream-image'); + assert.equal(res.body, 'jpeg-bytes'); + assert.deepEqual(app.requests, [{ url: STILL.url, redirect: 'manual' }]); +}); + +test('/frame does not follow a still camera off its origin', async (t) => { + // The protection /media used to bypass, shown on the route that keeps it. + const app = mount(t, ({ url }) => + url === STILL.url + ? upstreamResponse({ + status: 302, + headers: { location: 'http://169.254.169.254/latest/meta-data/' }, + body: '', + }) + : upstreamResponse({ + headers: { 'content-type': 'image/jpeg' }, + body: 'private', + }), + ); + const res = await app.call(`/frame/${STILL.id}`); + assert.equal(res.statusCode, 200); + assert.equal(res.headers['X-CCTV-Source'], 'synthetic'); + assert.doesNotMatch(res.body, /private/); + assert.deepEqual( + app.requests.map(({ url }) => url), + [STILL.url], + ); +}); diff --git a/src/data/cctvNyc.test.mjs b/src/data/cctvNyc.test.mjs new file mode 100644 index 000000000..e96755f5a --- /dev/null +++ b/src/data/cctvNyc.test.mjs @@ -0,0 +1,391 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + loadNycSourcesFromOpenData, + nycCameraIsOnline, + nycCameraName, + nycCameraToSource, +} from '../../server/providers/cctv/sources.js'; +import { + DEFAULT_CCTV_MAX_SOURCES, + DEFAULT_NYC_MAX_SOURCES, + NYC_CAMERAS_URL, + NYC_GROUND_ELEVATION_M, + NYC_IMAGE_ORIGIN, + NYC_IMAGE_URL, + NYC_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/constants.js'; +import { + CAMERA_CODE_MAX_CHARS, + isLikelyNycCoordinate, +} from '../../server/providers/cctv/normalize.js'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; + +/** + * A response whose body is a live stream, plus a flag that flips when the + * stream is cancelled. A rejection path that returns without cancelling holds + * the transport open, so the flag is what the refusal tests actually assert. + */ +const streamingResponse = (init = {}) => { + const state = { cancelled: false }; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('[')); + }, + cancel() { + state.cancelled = true; + }, + }); + return { response: new Response(body, init), state }; +}; + +const MIDTOWN_ID = '8a6bc417-4877-4ebe-8052-88c1b261baf1'; +const BROOKLYN_ID = '11111111-2222-3333-4444-555555555555'; + +/** One TMC camera row, shaped like the live `webcams.nyctmc.org` payload. */ +const row = (overrides = {}) => ({ + id: MIDTOWN_ID, + name: 'Central Park West @ 86 St', + latitude: 40.785302, + longitude: -73.969353, + area: 'Manhattan', + isOnline: 'true', + imageUrl: NYC_IMAGE_URL(MIDTOWN_ID), + ...overrides, +}); + +test('a NYC row maps to a source on the pinned HTTPS frame host', () => { + const source = nycCameraToSource(row()); + assert.equal(source.id, `nyc-${MIDTOWN_ID}`); + assert.equal(source.name, 'Central Park West @ 86 St'); + assert.equal(source.city, 'Manhattan'); + assert.equal(source.cityId, 'nyc'); + assert.equal(source.provider, 'NYC DOT Traffic Management Center'); + assert.equal(source.lat, 40.785302); + assert.equal(source.lon, -73.969353); + assert.equal(source.feedType, 'image'); + assert.equal(source.sourceKind, 'nyc-dot-tmc'); + assert.equal( + source.license, + 'Public NYC DOT Traffic Management Center traffic camera', + ); + assert.equal(source.groundElevationM, NYC_GROUND_ELEVATION_M); + assert.equal(source.url, NYC_IMAGE_URL(MIDTOWN_ID)); + assert.equal(source.url, source.snapshotUrl); + assert.ok(source.url.startsWith(NYC_IMAGE_ORIGIN)); + assert.equal(source.headingConfidence, 'low'); + assert.ok(Number.isFinite(source.headingDeg)); +}); + +test('frame URLs are synthesized from the UUID; the payload URL is ignored', () => { + const source = nycCameraToSource( + row({ imageUrl: 'https://evil.example/cameras/x/image' }), + ); + assert.equal(source.url, NYC_IMAGE_URL(MIDTOWN_ID)); + assert.equal(source.snapshotUrl, source.url); +}); + +test('offline cameras, off-city geometry and unusable ids are dropped', () => { + assert.equal(nycCameraToSource(row({ isOnline: 'false' })), null); + assert.equal(nycCameraToSource(row({ isOnline: false })), null); + assert.equal( + nycCameraToSource(row({ isOnline: 'TRUE' })).id, + `nyc-${MIDTOWN_ID}`, + ); + assert.equal( + nycCameraToSource(row({ isOnline: true })).id, + `nyc-${MIDTOWN_ID}`, + ); + // Austin: a plausible lat/lon, but not a New York camera. + assert.equal( + nycCameraToSource(row({ latitude: 30.2672, longitude: -97.7431 })), + null, + ); + assert.equal(nycCameraToSource(row({ latitude: 0, longitude: 0 })), null); + assert.equal(nycCameraToSource(row({ id: '../etc/passwd' })), null); + assert.equal(nycCameraToSource(row({ id: 'not-a-uuid' })), null); + assert.equal(nycCameraToSource(row({ id: `${MIDTOWN_ID}/../x` })), null); + assert.equal(nycCameraToSource(null), null); +}); + +test('ids lower-case the UUID and unknown boroughs fall back to New York', () => { + const mixed = nycCameraToSource( + row({ id: MIDTOWN_ID.toUpperCase(), area: 'Long Island' }), + ); + assert.equal(mixed.id, `nyc-${MIDTOWN_ID}`); + assert.equal(mixed.city, 'New York'); + assert.equal( + mixed.url, + NYC_IMAGE_URL(MIDTOWN_ID.toUpperCase()), + 'the registered frame path keeps the catalog UUID spelling', + ); +}); + +test('a nameless row still gets a label', () => { + assert.equal( + nycCameraName({ name: ' FDR Drive @ 23 St ' }, 'nyc-x'), + 'FDR Drive @ 23 St', + ); + assert.equal(nycCameraName({}, `nyc-${MIDTOWN_ID}`), 'NYC Camera 8a6bc417'); + assert.equal(nycCameraIsOnline('true'), true); + assert.equal(nycCameraIsOnline('false'), false); +}); + +test('the unselected label is the intersection, trimmed to the code width', () => { + assert.equal(nycCameraToSource(row()).code, 'CENTRAL PARK WEST @ 86 ST'); + const long = nycCameraToSource( + row({ + name: 'Brooklyn Queens Expressway @ Atlantic Avenue / Flatbush Avenue Extension', + }), + ); + assert.ok(long.code.length <= CAMERA_CODE_MAX_CHARS); + assert.ok(long.code.endsWith('…')); + assert.ok(long.code.startsWith('BROOKLYN')); +}); + +test('the loader reads the keyless catalog and collapses duplicate ids', async (t) => { + t.mock.method(console, 'log', () => {}); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url, init) => { + requested.push([String(url), init.redirect]); + return Response.json([ + row({ latitude: 40.758, longitude: -73.9855 }), + row({ + name: 'Central Park West @ 86 St (duplicate)', + latitude: 40.758, + longitude: -73.9855, + }), + row({ + id: BROOKLYN_ID, + name: 'Atlantic Ave @ Flatbush Ave', + latitude: 40.65, + longitude: -73.96, + area: 'Brooklyn', + imageUrl: NYC_IMAGE_URL(BROOKLYN_ID), + }), + row({ isOnline: 'false', name: 'Offline camera' }), + row({ id: 'https://evil.example/x' }), + ]); + }); + const cameras = await loadNycSourcesFromOpenData(); + assert.deepEqual(requested, [[NYC_CAMERAS_URL, 'manual']]); + assert.deepEqual( + cameras.map((camera) => camera.id), + // Nearest to a borough anchor first: the Times Square camera before Brooklyn. + [`nyc-${MIDTOWN_ID}`, `nyc-${BROOKLYN_ID}`], + ); + assert.equal(cameras[0].name, 'Central Park West @ 86 St'); + assert.equal(cameras[1].city, 'Brooklyn'); +}); + +test('an upstream failure yields an empty pack and releases the response', async (t) => { + t.mock.method(console, 'warn', () => {}); + const failed = streamingResponse({ status: 503 }); + t.mock.method(globalThis, 'fetch', async () => failed.response); + assert.deepEqual(await loadNycSourcesFromOpenData(), []); + assert.equal(failed.state.cancelled, true, 'the failed body is cancelled'); + + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + throw new Error('network down'); + }); + assert.deepEqual(await loadNycSourcesFromOpenData(), []); +}); + +test('the catalog fetch refuses redirects and oversized bodies', async (t) => { + t.mock.method(console, 'warn', () => {}); + const seen = []; + const redirected = streamingResponse({ + status: 302, + headers: { location: 'https://evil.example/cameras.json' }, + }); + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init.redirect]); + return redirected.response; + }); + assert.deepEqual(await loadNycSourcesFromOpenData(), []); + assert.deepEqual(seen, [[NYC_CAMERAS_URL, 'manual']]); + assert.equal( + redirected.state.cancelled, + true, + 'the redirect body is cancelled', + ); + + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + const body = JSON.stringify([row()]); + return new Response(body, { + headers: { + 'Content-Type': 'application/json', + 'content-length': String(NYC_MAX_CATALOG_BYTES + 1), + }, + }); + }); + assert.deepEqual(await loadNycSourcesFromOpenData(), []); + + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + const oversized = 'x'.repeat(NYC_MAX_CATALOG_BYTES + 1024); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(oversized)); + controller.close(); + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ); + }); + assert.deepEqual(await loadNycSourcesFromOpenData(), []); +}); + +test('a reduced pack cap keeps cameras nearest the borough anchors', async (t) => { + t.mock.method(console, 'log', () => {}); + const saved = process.env.CCTV_NYC_MAX_SOURCES; + t.after(() => { + if (saved === undefined) delete process.env.CCTV_NYC_MAX_SOURCES; + else process.env.CCTV_NYC_MAX_SOURCES = saved; + }); + process.env.CCTV_NYC_MAX_SOURCES = '8'; + const rows = [ + // Far southern Staten Island, listed first so only distance can keep them. + ...[40.5, 40.505, 40.51, 40.515, 40.52, 40.525].map((lat, i) => + row({ + id: `aaaaaaaa-bbbb-cccc-dddd-${String(i).padStart(12, '0')}`, + latitude: lat, + longitude: -74.24, + area: 'Staten Island', + name: `SI ${i}`, + }), + ), + ...[40.758, 40.757, 40.756, 40.755, 40.754, 40.753].map((lat, i) => + row({ + id: `bbbbbbbb-cccc-dddd-eeee-${String(i).padStart(12, '0')}`, + latitude: lat, + longitude: -73.9855, + area: 'Manhattan', + name: `Midtown ${i}`, + }), + ), + ]; + t.mock.method(globalThis, 'fetch', async () => Response.json(rows)); + const cameras = await loadNycSourcesFromOpenData(); + assert.equal(cameras.length, 8); + assert.deepEqual( + cameras.map((camera) => camera.city), + [ + 'Manhattan', + 'Manhattan', + 'Manhattan', + 'Manhattan', + 'Manhattan', + 'Manhattan', + 'Staten Island', + 'Staten Island', + ], + 'Midtown fills first; the Staten Island anchor still keeps a share', + ); +}); + +/** + * Serve the NYC catalog to the NYC endpoint and an empty payload to every + * other pack, so one catalog refresh exercises the registration without + * reaching the network. + */ +const runCatalogWithMockedUpstreams = async (t) => { + const requested = []; + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async (url) => { + const href = String(url); + requested.push(href); + if (href === NYC_CAMERAS_URL) { + return Response.json([ + row({ latitude: 40.758, longitude: -73.9855 }), + row({ + id: BROOKLYN_ID, + name: 'Atlantic Ave @ Flatbush Ave', + latitude: 40.65, + longitude: -73.96, + area: 'Brooklyn', + }), + ]); + } + return Response.json([]); + }); + const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); + return { requested, sources }; +}; + +/** Run one catalog refresh with CCTV_NYC_ENABLED set to `flag` (undefined + * leaves it unset), restoring the environment afterwards. */ +const runCatalogWithNycFlag = async (t, flag) => { + const saved = { ...process.env }; + try { + delete process.env.CCTV_SOURCES_FILE; + delete process.env.CCTV_SOURCES_JSON; + if (flag === undefined) delete process.env.CCTV_NYC_ENABLED; + else process.env.CCTV_NYC_ENABLED = flag; + return await runCatalogWithMockedUpstreams(t); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + } +}; + +test('CCTV_NYC_ENABLED=1 wires the NYC lane into the catalog', async (t) => { + const { requested, sources } = await runCatalogWithNycFlag(t, '1'); + assert.ok( + requested.includes(NYC_CAMERAS_URL), + 'the catalog refresh invokes the NYC loader', + ); + assert.deepEqual( + sources.filter((s) => s.cityId === 'nyc').map((s) => s.id), + [`nyc-${MIDTOWN_ID}`, `nyc-${BROOKLYN_ID}`], + 'NYC cameras reach the served catalog through the registered lane', + ); +}); + +// NYC DOT requires a data-sharing agreement for this feed, so an install that +// has not opted in must never contact it. +for (const [label, flag] of [ + ['an unset CCTV_NYC_ENABLED', undefined], + ['an empty CCTV_NYC_ENABLED', ''], + ['CCTV_NYC_ENABLED=0', '0'], + ['CCTV_NYC_ENABLED=true', 'true'], +]) { + test(`${label} makes no NYC request`, async (t) => { + const { requested, sources } = await runCatalogWithNycFlag(t, flag); + assert.ok(requested.length > 0, 'the other packs still refreshed'); + assert.equal( + requested.some((href) => href.includes('nyctmc.org')), + false, + 'the lane never reaches its upstream', + ); + assert.deepEqual( + sources.filter((s) => s.cityId === 'nyc'), + [], + ); + }); +} + +test('the shipped catalog ceiling is not raised to make room for this pack', () => { + assert.equal(DEFAULT_CCTV_MAX_SOURCES, 4000); + assert.equal(DEFAULT_NYC_MAX_SOURCES, 250); +}); + +test('isLikelyNycCoordinate spans the five boroughs and rejects the rest', () => { + assert.equal(isLikelyNycCoordinate(40.758, -73.9855), true); // Times Square + assert.equal(isLikelyNycCoordinate(40.5795, -74.1502), true); // Staten Island + assert.equal(isLikelyNycCoordinate(40.8448, -73.8648), true); // Bronx + assert.equal(isLikelyNycCoordinate(40.5077, -74.2304), true); // southern SI slack + assert.equal(isLikelyNycCoordinate(30.2672, -97.7431), false); // Austin + assert.equal(isLikelyNycCoordinate(0, 0), false); + assert.equal(isLikelyNycCoordinate(NaN, -73.98), false); +}); diff --git a/src/data/cctvToronto.test.mjs b/src/data/cctvToronto.test.mjs new file mode 100644 index 000000000..c2e3c0e1d --- /dev/null +++ b/src/data/cctvToronto.test.mjs @@ -0,0 +1,423 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + parseRescuCameraList, + torontoCameraToSource, + loadTorontoSourcesFromOpenData, +} from '../../server/providers/cctv/sources.js'; +import { + TORONTO_RESCU_LIST_URL, + TORONTO_RESCU_IMAGE_ORIGIN, + TORONTO_MAX_CATALOG_BYTES, + DEFAULT_CCTV_MAX_SOURCES, +} from '../../server/providers/cctv/constants.js'; +import { CAMERA_CODE_MAX_CHARS } from '../../server/providers/cctv/normalize.js'; +import { allocateSourceCap } from '../../server/providers/cctv/cap.js'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; + +/** + * A response whose body is a live stream, plus a flag that flips when the + * stream is cancelled. A rejection path that returns without cancelling holds + * the transport open, so the flag is what the refusal tests actually assert. + */ +const streamingResponse = (init = {}) => { + const state = { cancelled: false }; + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('j')); + }, + cancel() { + state.cancelled = true; + }, + }); + return { response: new Response(body, init), state }; +}; + +/** One RESCU row, shaped like the live `tmcearthcameras.json` payload. */ +const row = (overrides = {}) => ({ + Number: '8048', + Name: 'BATHURST ST & BLOOR ST W', + Latitude: '43.665249', + Longitude: '-79.41114', + D1: 'n', + D2: 'e', + D3: 's', + D4: 'w', + Group: 'Arterial', + ...overrides, +}); + +/** The city wraps the list in a JSONP callback; this is that envelope. */ +const jsonp = (rows) => + `jsonTMCEarthCamerasCallback(${JSON.stringify({ Data: rows })});`; + +test('a RESCU row maps to a source on the constructed frame URL', () => { + const source = torontoCameraToSource(row()); + assert.equal(source.id, 'toronto-8048'); + assert.equal(source.name, 'BATHURST ST & BLOOR ST W'); + assert.equal(source.city, 'Toronto'); + assert.equal(source.cityId, 'toronto'); + assert.equal(source.provider, 'City of Toronto'); + assert.equal(source.lat, 43.665249); + assert.equal(source.lon, -79.41114); + assert.equal(source.feedType, 'image'); + assert.equal(source.sourceKind, 'toronto-rescu-open-data'); + assert.equal( + source.license, + 'Contains information licensed under the Open Government Licence – Toronto', + ); + assert.equal( + source.url, + `${TORONTO_RESCU_IMAGE_ORIGIN}loc8048.jpg`, + 'the documented loc####.jpg convention', + ); + assert.equal(source.url, source.snapshotUrl); + assert.ok(source.url.startsWith(TORONTO_RESCU_IMAGE_ORIGIN)); +}); + +test('the frame URL is built from the camera number, never read from the payload', () => { + // The list carries no URL field at all, so an added one must stay inert: the + // frame proxy is pinned to the city origin by construction. + const planted = torontoCameraToSource( + row({ + url: 'https://evil.example/x.jpg', + snapshotUrl: 'https://evil.example/x.jpg', + Url: 'https://evil.example/x.jpg', + }), + ); + assert.equal(planted.url, `${TORONTO_RESCU_IMAGE_ORIGIN}loc8048.jpg`); + assert.equal(planted.snapshotUrl, planted.url); + + // A number that is not plain digits cannot reach the URL builder: path + // traversal and absolute URLs are refused outright rather than escaped. + for (const Number_ of [ + '../../etc/passwd', + '8048/../../x', + 'https://evil.example/1', + '80 48', + '8048.jpg', + '-8048', + '', + ' ', + null, + ]) { + assert.equal( + torontoCameraToSource(row({ Number: Number_ })), + null, + `refused: ${String(Number_)}`, + ); + } +}); + +test('the comparison directions never become a camera heading', () => { + // D1-D4 are the city's *comparison image* directions (static 2016 reference + // stills served from ComparisonImages/), not a facing for the live frame: + // these cameras pan. The letters read like a compass and map trivially onto + // one (n -> 0, w -> 270), so the temptation to wire them up is real — and a + // heading derived from them would be confidently wrong for every camera in + // the city. Nobody may "fix" this. + const base = torontoCameraToSource(row()); + for (const dirs of [ + { D1: 'n', D2: 'e', D3: 's', D4: 'w' }, + { D1: '', D2: 'e', D3: '', D4: 'w' }, + { D1: '', D2: '', D3: '', D4: '' }, + { D1: 's', D2: '', D3: '', D4: '' }, + ]) { + const source = torontoCameraToSource(row(dirs)); + assert.equal(source.headingConfidence, 'low'); + assert.ok(Number.isFinite(source.headingDeg)); + // Same id, same fallback heading, whatever the directions say. + assert.equal(source.headingDeg, base.headingDeg); + } +}); + +test('the two cameras at one intersection are kept apart', () => { + // 8047 and 8048 are both "BATHURST ST & BLOOR ST W", five metres apart. They + // are distinct units, so both survive, and the id-hash fallback fans their + // gizmos apart instead of stacking them on one bearing. + const a = torontoCameraToSource( + row({ Number: '8047', Latitude: '43.665223', Longitude: '-79.411083' }), + ); + const b = torontoCameraToSource(row()); + assert.equal(a.id, 'toronto-8047'); + assert.equal(b.id, 'toronto-8048'); + assert.notEqual(a.url, b.url); + assert.notEqual(a.headingDeg, b.headingDeg); +}); + +test('rows outside Toronto and unusable geometry are dropped', () => { + // Calgary: a plausible lat/lon, but not a Toronto camera. + assert.equal( + torontoCameraToSource(row({ Latitude: '51.0453', Longitude: '-114.1413' })), + null, + ); + // Null island, and the empty strings that would become it. + assert.equal( + torontoCameraToSource(row({ Latitude: '0', Longitude: '0' })), + null, + ); + assert.equal( + torontoCameraToSource(row({ Latitude: '', Longitude: '' })), + null, + ); + assert.equal( + torontoCameraToSource(row({ Latitude: 'north', Longitude: 'west' })), + null, + ); + assert.equal(torontoCameraToSource({}), null); + assert.equal(torontoCameraToSource(null), null); +}); + +test('a nameless row still gets a label', () => { + const source = torontoCameraToSource(row({ Name: ' ' })); + assert.equal(source.name, 'Toronto Camera 8048'); +}); + +test('the JSONP envelope is unwrapped, and only that envelope', () => { + assert.deepEqual(parseRescuCameraList(jsonp([row()])), [row()]); + // Whitespace and a missing trailing semicolon are both tolerated. + assert.equal( + parseRescuCameraList(`\n ${jsonp([row()]).replace(/;$/, '')} \n`).length, + 1, + ); + // Should the city ever drop the wrapper, the bare document still parses. + assert.equal( + parseRescuCameraList(JSON.stringify({ Data: [row()] })).length, + 1, + ); + + // Anything else yields no cameras rather than a thrown loader. + for (const text of [ + 'otherCallback({"Data":[]})', + 'jsonTMCEarthCamerasCallback(', + 'jsonTMCEarthCamerasCallback({"Data":', + '{"Data":"not-an-array"}', + '{"nope":[]}', + '503', + '', + null, + ]) { + assert.deepEqual(parseRescuCameraList(text), [], `rejected: ${text}`); + } +}); + +test('the loader reads the keyless list and collapses duplicate numbers', async (t) => { + t.mock.method(console, 'log', () => {}); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + return new Response( + jsonp([ + row(), + // Same camera number twice: one camera, not two. + row({ Name: 'BATHURST ST & BLOOR ST W (duplicate)' }), + row({ + Number: '8001', + Name: 'YORK ST & BREMNER BLVD / RAPTORS WAY', + Latitude: '43.643120', + Longitude: '-79.381386', + }), + row({ Number: '../evil' }), + ]), + ); + }); + const cameras = await loadTorontoSourcesFromOpenData(); + assert.deepEqual(requested, [TORONTO_RESCU_LIST_URL]); + assert.deepEqual( + cameras.map((camera) => camera.id), + // Nearest downtown first: York & Bremner is at the core, Bathurst & Bloor + // is three kilometres out. + ['toronto-8001', 'toronto-8048'], + ); + assert.equal(cameras[1].name, 'BATHURST ST & BLOOR ST W'); +}); + +test('an upstream failure yields an empty pack and releases the response', async (t) => { + t.mock.method(console, 'warn', () => {}); + // A 503 can still arrive with a streaming body; returning without cancelling + // it would hold the connection until the socket times out. + const failed = streamingResponse({ status: 503 }); + t.mock.method(globalThis, 'fetch', async () => failed.response); + assert.deepEqual(await loadTorontoSourcesFromOpenData(), []); + assert.equal(failed.state.cancelled, true, 'the failed body is cancelled'); + + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + throw new Error('network down'); + }); + assert.deepEqual(await loadTorontoSourcesFromOpenData(), []); +}); + +test('the list fetch refuses redirects and oversized bodies', async (t) => { + t.mock.method(console, 'warn', () => {}); + // A redirect is never followed: the list host cannot be steered. Its body is + // a live stream, so the test also proves the refusal releases the transport. + const seen = []; + const redirected = streamingResponse({ + status: 302, + headers: { location: 'https://evil.example/cameras.json' }, + }); + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init.redirect]); + return redirected.response; + }); + assert.deepEqual(await loadTorontoSourcesFromOpenData(), []); + assert.deepEqual(seen, [[TORONTO_RESCU_LIST_URL, 'manual']]); + assert.equal( + redirected.state.cancelled, + true, + 'the redirect body is cancelled', + ); + + // A body over the cap is refused rather than buffered. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + return new Response(jsonp([row()]), { + headers: { + 'content-length': String(TORONTO_MAX_CATALOG_BYTES + 1), + }, + }); + }); + assert.deepEqual(await loadTorontoSourcesFromOpenData(), []); + + // So is a body that only declares its size once it is already too long. + t.mock.restoreAll(); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async () => { + const oversized = 'x'.repeat(TORONTO_MAX_CATALOG_BYTES + 1024); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(oversized)); + controller.close(); + }, + }), + ); + }); + assert.deepEqual(await loadTorontoSourcesFromOpenData(), []); +}); + +test('the unselected label is the intersection, trimmed to the code width', () => { + assert.equal(torontoCameraToSource(row()).code, 'BATHURST ST & BLOOR ST W'); + const long = torontoCameraToSource( + row({ + Name: 'BAYVIEW AVE & POTTERY RD / PRIVATE ACCESS @ 620m SOUTH OF - 550 BAYVIEW AVE (EVERGREEN)', + }), + ); + assert.ok(long.code.length <= CAMERA_CODE_MAX_CHARS); + assert.ok(long.code.endsWith('…')); + assert.ok(long.code.startsWith('BAYVIEW AVE & POTTERY')); +}); + +test('a reduced catalog cap thins every pack instead of dropping Toronto', () => { + // Toronto merges last in LIVE_PACKS, so a positional slice would delete it + // outright. The round-robin allocation must give it its share. + const lane = (name, count) => ({ + name, + sources: Array.from({ length: count }, (_, i) => ({ id: `${name}-${i}` })), + }); + const { sources, packs } = allocateSourceCap( + [lane('austin', 250), lane('calgary', 215), lane('toronto', 336)], + 30, + ); + assert.equal(sources.length, 30); + assert.deepEqual( + packs.map((p) => [p.name, p.kept]), + [ + ['austin', 10], + ['calgary', 10], + ['toronto', 10], + ], + ); + // Toronto contributes its own highest-priority cameras, in its own order. + assert.deepEqual( + sources.filter((s) => s.id.startsWith('toronto-')).map((s) => s.id), + Array.from({ length: 10 }, (_, i) => `toronto-${i}`), + ); +}); + +/** + * Serve the RESCU list to the Toronto endpoint and an empty payload to every + * other pack, so one catalog refresh exercises the registration without + * reaching the network. Returns the URLs that were requested. + */ +const runCatalogWithMockedUpstreams = async (t) => { + const requested = []; + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); + t.mock.method(globalThis, 'fetch', async (url) => { + const href = String(url); + requested.push(href); + if (href === TORONTO_RESCU_LIST_URL) { + return new Response( + jsonp([ + row({ + Number: '8001', + Name: 'YORK ST & BREMNER BLVD / RAPTORS WAY', + Latitude: '43.643120', + Longitude: '-79.381386', + }), + row(), + ]), + ); + } + return Response.json([]); + }); + const sources = await createCctvCatalog({ sourceRoot: '/nonexistent' })(); + return { requested, sources }; +}; + +test('the pack is registered, so its cameras reach the served catalog', async (t) => { + const saved = { ...process.env }; + try { + delete process.env.CCTV_SOURCES_FILE; + delete process.env.CCTV_SOURCES_JSON; + delete process.env.CCTV_TORONTO_ENABLED; + const { requested, sources } = await runCatalogWithMockedUpstreams(t); + assert.ok( + requested.includes(TORONTO_RESCU_LIST_URL), + 'the registered lane reaches its upstream', + ); + assert.deepEqual( + sources.filter((s) => s.cityId === 'toronto').map((s) => s.id), + ['toronto-8001', 'toronto-8048'], + 'Toronto cameras reach the served catalog through the registered lane', + ); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + } +}); + +test('CCTV_TORONTO_ENABLED=0 keeps the lane from being loaded at all', async (t) => { + const saved = { ...process.env }; + try { + delete process.env.CCTV_SOURCES_FILE; + delete process.env.CCTV_SOURCES_JSON; + process.env.CCTV_TORONTO_ENABLED = '0'; + const { requested, sources } = await runCatalogWithMockedUpstreams(t); + assert.equal( + requested.includes(TORONTO_RESCU_LIST_URL), + false, + 'the disabled lane never reaches its upstream', + ); + assert.deepEqual( + sources.filter((s) => s.cityId === 'toronto'), + [], + ); + } finally { + for (const key of Object.keys(process.env)) { + if (!(key in saved)) delete process.env[key]; + } + Object.assign(process.env, saved); + } +}); + +test('the shipped catalog ceiling is not raised to make room for this pack', () => { + assert.equal(DEFAULT_CCTV_MAX_SOURCES, 4000); +}); diff --git a/src/data/cctvWsdotSource.test.mjs b/src/data/cctvWsdotSource.test.mjs new file mode 100644 index 000000000..6b2ad3398 --- /dev/null +++ b/src/data/cctvWsdotSource.test.mjs @@ -0,0 +1,302 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createCctvCatalog } from '../../server/providers/cctv/catalog.js'; +import { + DEFAULT_WSDOT_MAX_SOURCES, + WSDOT_CAMERAS_URL, + WSDOT_MAX_CATALOG_BYTES, +} from '../../server/providers/cctv/constants.js'; +import { + loadWsdotSourcesFromOpenData, + webMercatorToWgs84, +} from '../../server/providers/cctv/sources.js'; + +/** Set (or, for `undefined`, delete) environment variables for one test. */ +function withEnv(t, env) { + for (const [name, value] of Object.entries(env)) { + const previous = process.env[name]; + t.after(() => { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; + }); + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } +} + +/** Silence the loaders' progress and failure logging. */ +function quiet(t) { + t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'warn', () => {}); +} + +/** Web Mercator metres for a WGS84 coordinate (inverse of the loader's math). */ +function mercator(lat, lon) { + const R = 6378137; + return { + x: (lon * Math.PI * R) / 180, + y: R * Math.log(Math.tan(Math.PI / 4 + (lat * Math.PI) / 360)), + }; +} + +/** A `Cameras.json` point feature carrying the fields the loader reads. */ +function wsdotFeature(id, lat, lon, overrides = {}) { + const { attributes = {}, geometry } = overrides; + return { + attributes: { + CameraID: id, + CameraTitle: `Camera ${id}`, + CompassDirection: 'N', + ImageURL: `https://images.wsdot.wa.gov/nw/${id}.jpg`, + ...attributes, + }, + geometry: geometry === undefined ? mercator(lat, lon) : geometry, + }; +} + +/** Serialize a payload for the mocked fetch as raw Windows-1252 bytes. */ +function cp1252Response(payload) { + const text = JSON.stringify(payload); + const bytes = new Uint8Array(text.length); + for (let i = 0; i < text.length; i += 1) { + const code = text.charCodeAt(i); + // U+2013 EN DASH is 0x96 in Windows-1252 — the byte the live feed ships. + bytes[i] = code === 0x2013 ? 0x96 : code; + } + return new Response(bytes, { + headers: { 'Content-Type': 'application/json' }, + }); +} + +test('webMercatorToWgs84 recovers WGS84 degrees from layer geometry', () => { + // Real feature from the live layer: I-5 at the Interstate Bridge. + const { lat, lon } = webMercatorToWgs84(-13656007.102, 5719738.545); + assert.ok(Math.abs(lat - 45.62) < 0.05, `lat ${lat}`); + assert.ok(Math.abs(lon - -122.675) < 0.05, `lon ${lon}`); +}); + +test('WSDOT loader keeps official-host cameras and maps compass directions', async (t) => { + quiet(t); + withEnv(t, { CCTV_WSDOT_MAX_SOURCES: undefined }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + return cp1252Response({ + features: [ + wsdotFeature(1001, 47.6097, -122.3331, { + attributes: { + // The 0x2013 below is serialized as the feed's 0x96 byte. + CameraTitle: 'I-5 – Seattle, looking south', + CompassDirection: 'S', + }, + }), + wsdotFeature(9059, 47.66, -117.43, { + attributes: { CompassDirection: 'B' }, + }), + // Partner-hosted frame (Oregon DOT): skipped, never proxied. + wsdotFeature(1002, 45.62, -122.675, { + attributes: { + ImageURL: 'https://www.tripcheck.com/RoadCams/cams/bridge.jpg', + }, + }), + // Bad rows: non-integer id, missing geometry, out-of-state point. + wsdotFeature('1003', 47.6, -122.3), + wsdotFeature(1004, 47.6, -122.3, { geometry: null }), + wsdotFeature(1005, 34.05, -118.24), + ], + }); + }); + + const cameras = await loadWsdotSourcesFromOpenData(); + + assert.deepEqual(requested, [WSDOT_CAMERAS_URL]); + // Exactly the two valid official-host cameras survive; the partner-hosted + // frame and the malformed rows are dropped. + assert.deepEqual(cameras.map((camera) => camera.id).sort(), [ + 'wsdot-1001', + 'wsdot-9059', + ]); + + const seattle = cameras.find((camera) => camera.id === 'wsdot-1001'); + assert.equal(seattle.name, 'I-5 – Seattle, looking south'); + assert.equal(seattle.provider, 'WSDOT'); + assert.equal(seattle.url, 'https://images.wsdot.wa.gov/nw/1001.jpg'); + assert.equal(seattle.snapshotUrl, seattle.url); + assert.equal(seattle.feedType, 'image'); + assert.equal(seattle.headingDeg, 180); + assert.equal(seattle.headingConfidence, 'high'); + assert.equal(seattle.fovDeg, 56); + assert.ok(Math.abs(seattle.lat - 47.6097) < 0.001); + assert.ok(Math.abs(seattle.lon - -122.3331) < 0.001); + + // `B` (both directions) carries no single facing: id-hash fallback at low + // confidence, with the low-confidence pose personality. + const spokane = cameras.find((camera) => camera.id === 'wsdot-9059'); + assert.equal(spokane.headingConfidence, 'low'); + assert.equal(spokane.fovDeg, 44); + assert.ok(spokane.headingDeg >= 0 && spokane.headingDeg < 360); +}); + +test('WSDOT loader maps every CompassDirection code', async (t) => { + quiet(t); + withEnv(t, { CCTV_WSDOT_MAX_SOURCES: undefined }); + const codes = { + N: 0, + NE: 45, + E: 90, + SE: 135, + S: 180, + SW: 225, + W: 270, + NW: 315, + // Lower-case and padded codes are the same code. + ' nw ': 315, + }; + // The live feed's no-facing codes, plus an unknown one. + const noFacing = ['B', 'O', null, 'BW']; + const entries = [...Object.keys(codes), ...noFacing]; + t.mock.method(globalThis, 'fetch', async () => + cp1252Response({ + features: entries.map((code, i) => + wsdotFeature(3000 + i, 47.0 + i * 0.05, -122.9 + i * 0.05, { + attributes: { CompassDirection: code }, + }), + ), + }), + ); + + const cameras = await loadWsdotSourcesFromOpenData(); + const byId = new Map(cameras.map((camera) => [camera.id, camera])); + entries.forEach((code, i) => { + const camera = byId.get(`wsdot-${3000 + i}`); + if (code in codes) { + assert.equal(camera.headingDeg, codes[code], `heading for ${code}`); + assert.equal(camera.headingConfidence, 'high', `confidence for ${code}`); + assert.equal(camera.fovDeg, 56, `pose personality for ${code}`); + } else { + assert.equal(camera.headingConfidence, 'low', `confidence for ${code}`); + assert.equal(camera.fovDeg, 44, `pose personality for ${code}`); + } + }); +}); + +test('WSDOT loader honors the max-sources cap', async (t) => { + quiet(t); + withEnv(t, { CCTV_WSDOT_MAX_SOURCES: '8' }); + const features = []; + for (let i = 0; i < 24; i += 1) { + features.push(wsdotFeature(2000 + i, 47.0 + i * 0.05, -122.9 + i * 0.05)); + } + t.mock.method(globalThis, 'fetch', async () => cp1252Response({ features })); + + const cameras = await loadWsdotSourcesFromOpenData(); + assert.equal(cameras.length, 8); +}); + +test('WSDOT loader defaults its cap and survives upstream failure', async (t) => { + quiet(t); + withEnv(t, { CCTV_WSDOT_MAX_SOURCES: undefined }); + assert.equal(DEFAULT_WSDOT_MAX_SOURCES, 250); + + t.mock.method( + globalThis, + 'fetch', + async () => new Response('', { status: 503 }), + ); + assert.deepEqual(await loadWsdotSourcesFromOpenData(), []); + + t.mock.method(globalThis, 'fetch', async () => { + throw new Error('network down'); + }); + assert.deepEqual(await loadWsdotSourcesFromOpenData(), []); +}); + +test('the catalog fetch refuses a redirect and cancels its body', async (t) => { + quiet(t); + // A redirect is never followed, and its still-open body is cancelled. + const seen = []; + const state = { cancelled: false }; + t.mock.method(globalThis, 'fetch', async (url, init) => { + seen.push([String(url), init.redirect]); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{')); + }, + cancel() { + state.cancelled = true; + }, + }); + return new Response(body, { + status: 302, + headers: { location: 'https://evil.example/Cameras.json' }, + }); + }); + assert.deepEqual(await loadWsdotSourcesFromOpenData(), []); + assert.deepEqual(seen, [[WSDOT_CAMERAS_URL, 'manual']]); + assert.equal(state.cancelled, true, 'the redirect body is cancelled'); +}); + +test('the catalog fetch refuses a body over the byte cap', async (t) => { + quiet(t); + // A valid catalog padded past the cap is refused rather than buffered. + t.mock.method(globalThis, 'fetch', async () => { + const text = JSON.stringify({ + features: [wsdotFeature(1001, 47.6097, -122.3331)], + }); + const padded = text + ' '.repeat(WSDOT_MAX_CATALOG_BYTES); + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(padded)); + controller.close(); + }, + }), + { headers: { 'Content-Type': 'application/json' } }, + ); + }); + assert.deepEqual(await loadWsdotSourcesFromOpenData(), []); +}); + +test('CCTV_WSDOT_ENABLED=0 skips the WSDOT request', async (t) => { + quiet(t); + const sourceRoot = fs.mkdtempSync( + path.join(os.tmpdir(), 'gev-cctv-catalog-'), + ); + t.after(() => fs.rmSync(sourceRoot, { recursive: true, force: true })); + withEnv(t, { + CCTV_SOURCES_FILE: undefined, + CCTV_SOURCES_JSON: undefined, + CCTV_FORCE_AUSTIN: undefined, + CCTV_PREFER_AUSTIN: undefined, + CCTV_MAX_SOURCES: undefined, + CCTV_WSDOT_ENABLED: undefined, + CCTV_WSDOT_MAX_SOURCES: undefined, + }); + const requested = []; + t.mock.method(globalThis, 'fetch', async (url) => { + requested.push(String(url)); + // Every other live pack is down; only WSDOT answers. + return String(url) === WSDOT_CAMERAS_URL + ? cp1252Response({ features: [wsdotFeature(1001, 47.6097, -122.3331)] }) + : new Response('unavailable', { status: 503 }); + }); + + const sources = await createCctvCatalog({ sourceRoot })(); + assert.deepEqual( + sources.map((source) => source.id), + ['wsdot-1001'], + ); + + process.env.CCTV_WSDOT_ENABLED = '0'; + requested.length = 0; + assert.deepEqual(await createCctvCatalog({ sourceRoot })(), []); + assert.ok(requested.length > 0, 'the other live packs still load'); + assert.equal( + requested.includes(WSDOT_CAMERAS_URL), + false, + 'a disabled pack makes no request', + ); +}); diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 2967f0a2b..956671757 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -91,6 +91,13 @@ export function hideOsmCredit(viewer, owner) { return true; } +/** NYC DOT asks every application that uses its data to carry this + * disclaimer verbatim (nyc.gov/html/dot/html/about/datafeeds.shtml). */ +export const NYC_DOT_DISCLAIMER = + 'This application may contain and/or utilize information which was originally compiled by the New York City Department of Transportation (NYC DOT) for governmental purposes; the information may subsequently been modified by entity/entities other than NYC DOT. ' + + 'NYC DOT and the City of New York make no representation as to the accuracy or usefulness of the information provided by this application or the information's suitability for any purpose and disclaim any liability for omissions or errors that may be contained therein. ' + + 'The public is advised to observe posted signage for compliance with applicable laws and regulations.'; + export const DATA_CREDITS = [ // ── Live sources ──────────────────────────────────────────────── { @@ -104,7 +111,7 @@ export const DATA_CREDITS = [ { key: 'adsblol', html: - 'Military flights, aircraft traces & bounded regional flight fallback: ' + + 'Military flights, aircraft traces, bounded regional flight fallback & GNSS navigation integrity: ' + 'adsb.lol ' + '(ODbL 1.0)', }, @@ -162,6 +169,13 @@ export const DATA_CREDITS = [ }, OSM_CREDIT, OPENMAPTILES_CREDIT, + { + key: 'eccc-aqhi', + html: + 'Air quality (AQHI): Environment and Climate Change Canada — MSC GeoMet ' + + '(eccc-msc.github.io/open-data) ' + + '· ECCC Data Servers End-use Licence', + }, { key: 'overture-military-names', html: 'Military area names: Overture Maps Foundation (ODbL)', @@ -234,12 +248,38 @@ export const DATA_CREDITS = [ 'digitraffic.fi, ' + 'license CC BY 4.0', }, + { + key: 'lithuania-cctv', + html: + // Opt-in: the camera endpoints are the eismoinfo.lt website backend, not + // among the services Via Lietuva's open-data statement lists (its + // conditions: attribution, reuse on the same terms). + 'Road-camera stills (Lithuania, when enabled): ' + + 'Via Lietuva / Eismoinfo; periodically updated images', + }, { key: 'calgary-cctv', html: 'Traffic cameras (Calgary): contains information licensed under the ' + 'Open Government Licence – City of Calgary', }, + { + key: 'catalonia-cctv', + html: + 'CCTV cameras & frames (Catalonia): ' + + 'Servei Català de Trànsit ' + + '(Llicència oberta d’ús d’informació - Catalunya); some cameras are supplied by partners ' + + '(Ajuntament de Barcelona, Ajuntament de Terrassa), each named in the CCTV panel', + }, + { + key: 'kingcounty-cctv', + html: + 'CCTV cameras & frames (King County, WA): King County Road Services — ' + + 'kingcounty.gov, ' + + 'per the ' + + 'KCGIS Center terms. ' + + 'Partner-city cameras hosted on the county image handler (e.g. Newcastle) name their owner in the CCTV panel.', + }, { key: 'vegvesen-cctv', html: @@ -247,9 +287,44 @@ export const DATA_CREDITS = [ 'Norwegian licence for Open Government data (NLOD) ' + 'distributed by Statens vegvesen', }, + { + key: 'nyc-cctv', + html: + 'CCTV cameras & frames (New York City, when enabled): ' + + 'NYC DOT Traffic Management Center. ' + + NYC_DOT_DISCLAIMER, + }, + { + key: 'toronto-cctv', + html: + 'Traffic cameras (Toronto): contains information licensed under the ' + + 'Open Government Licence – Toronto', + }, { key: 'gbfs', - html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)', + html: + 'Bikeshare availability: GBFS operator feeds ' + + '(e.g. Austin BCycle; Bike Share Toronto, operated by the Toronto Parking Authority)', + }, + { + key: 'urbansharing-gbfs', + html: + 'City bikes (Norway): contains data under the ' + + 'Norwegian licence for Open Government data (NLOD) ' + + 'made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel', + }, + { + key: 'entur-mobility-gbfs', + html: + 'City bikes (Stavanger, Skien/Porsgrunn): contains data under the ' + + 'Norwegian licence for Open Government data (NLOD) ' + + 'made available by Entur (Kolumbus Bysykkel, Farte Bysykkel)', + }, + { + key: 'jcdecaux-gbfs', + html: + 'City bikes (Lillestrøm): Bysykkel Lillestrøm, JCDecaux — ' + + 'Licence Ouverte', }, { key: 'osrm-routing', @@ -328,6 +403,16 @@ export const DATA_CREDITS = [ 'Open Government Licence – British Columbia. ' + 'Some cameras are supplied by partners (TransLink, the City of Vancouver, the City of Surrey, Parks Canada and others); each names its provider in the CCTV panel.', }, + { + key: 'wsdot-cctv', + html: + 'CCTV cameras & frames (Washington State): ' + + 'Washington State Department of Transportation ' + + '(Travel Center), ' + + 'used per the ' + + 'WSDOT travel information disclaimer ' + + '(low-volume public feed; catalog cached, frames proxied on demand)', + }, { key: 'tallinn-cctv', html: diff --git a/src/data/dataCredits.test.mjs b/src/data/dataCredits.test.mjs index 46de48972..30a19b8fe 100644 --- a/src/data/dataCredits.test.mjs +++ b/src/data/dataCredits.test.mjs @@ -1,6 +1,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { DATA_CREDITS } from './dataCredits.js'; +import { DATA_CREDITS, NYC_DOT_DISCLAIMER } from './dataCredits.js'; test('every credit carries a unique key and some markup to render', () => { const keys = DATA_CREDITS.map((entry) => entry.key); @@ -120,3 +120,11 @@ test('tile attribution follows source changes and viewers have independent owner assert.equal(a.credits.size, 0); assert.equal(b.credits.size, 1); }); + +test('the NYC camera credit carries NYC DOT’s requested disclaimer', () => { + const credit = DATA_CREDITS.find((entry) => entry.key === 'nyc-cctv'); + assert.ok(credit, 'NYC DOT camera frames must be credited'); + assert.ok(credit.html.includes(NYC_DOT_DISCLAIMER)); + assert.match(credit.html, /make no representation as to the accuracy/); + assert.match(credit.html, /observe posted signage/); +}); diff --git a/src/data/directionText.js b/src/data/directionText.js index 78f1be0df..eb78065d2 100644 --- a/src/data/directionText.js +++ b/src/data/directionText.js @@ -28,22 +28,34 @@ export function directionToHeading(value, allowBare = false) { .trim() .toUpperCase(); if (!text) return NaN; - // Explicit travel/intercardinal forms — safe on free-form text (a street - // name almost never contains "NORTHBOUND" or a lone "NB" token). - if (/\bNORTHBOUND\b|\bNB\b/.test(text)) return 0; - if (/\bSOUTHBOUND\b|\bSB\b/.test(text)) return 180; - if (/\bEASTBOUND\b|\bEB\b/.test(text)) return 90; - if (/\bWESTBOUND\b|\bWB\b/.test(text)) return 270; - if (/\bNORTHEAST\b|\bNE\b/.test(text)) return 45; - if (/\bNORTHWEST\b|\bNW\b/.test(text)) return 315; - if (/\bSOUTHEAST\b|\bSE\b/.test(text)) return 135; - if (/\bSOUTHWEST\b|\bSW\b/.test(text)) return 225; - // Bare cardinals — dedicated direction fields only. - if (allowBare) { - if (/\bNORTH\b/.test(text)) return 0; - if (/\bSOUTH\b/.test(text)) return 180; - if (/\bEAST\b/.test(text)) return 90; - if (/\bWEST\b/.test(text)) return 270; + const tiers = [ + [ + [/\bNORTHBOUND\b|\bNB\b/, 0], + [/\bSOUTHBOUND\b|\bSB\b/, 180], + [/\bEASTBOUND\b|\bEB\b/, 90], + [/\bWESTBOUND\b|\bWB\b/, 270], + ], + [ + [/\bNORTHEAST\b|\bNE\b/, 45], + [/\bNORTHWEST\b|\bNW\b/, 315], + [/\bSOUTHEAST\b|\bSE\b/, 135], + [/\bSOUTHWEST\b|\bSW\b/, 225], + ], + ]; + if (allowBare) + tiers.push([ + [/\bNORTH\b/, 0], + [/\bSOUTH\b/, 180], + [/\bEAST\b/, 90], + [/\bWEST\b/, 270], + ]); + for (const forms of tiers) { + const headings = new Set(); + for (const [pattern, heading] of forms) { + if (pattern.test(text)) headings.add(heading); + } + if (headings.size) + return headings.size === 1 ? headings.values().next().value : NaN; } return NaN; } diff --git a/src/data/directions.js b/src/data/directions.js index ddb1109f7..ef5565488 100644 --- a/src/data/directions.js +++ b/src/data/directions.js @@ -1,3 +1,4 @@ +import { createDirectionsSource } from '../layers/directions/source.js'; /** * @module directions * @description Service bundle for the Directions layer. @@ -37,8 +38,10 @@ export const directionsServices = Object.freeze({ }); /** Construct one Directions layer over the application scene owners. */ -export function createApplicationDirectionsLayer() { - return createDirectionsLayer({ services: directionsServices }); +export function createApplicationDirectionsLayer({ + source = createDirectionsSource(), +} = {}) { + return createDirectionsLayer({ services: directionsServices, source }); } export { diff --git a/src/data/firms-calendar-validity.test.mjs b/src/data/firms-calendar-validity.test.mjs new file mode 100644 index 000000000..1ccaf7435 --- /dev/null +++ b/src/data/firms-calendar-validity.test.mjs @@ -0,0 +1,23 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { acquisitionMsUtc } from './firmsCsv.js'; +test('FIRMS acquisition dates cannot roll into a different calendar day', () => { + for (const [date, time] of [ + ['2026-02-30', '1200'], + ['2026-04-31', '1200'], + ['2026-13-01', '1200'], + ['2026-07-16junk', '1200'], + ['2026-07-16', '2400'], + ['2026-07-16', '1260'], + ['2026-07-16', null], + ['2026-07-16', ''], + ]) { + assert.ok(Number.isNaN(acquisitionMsUtc(date, time)), `${date}:${time}`); + } + for (const [date, time, expected] of [ + ['2024-02-29', '45', Date.UTC(2024, 1, 29, 0, 45)], + ['2026-07-16', '0', Date.UTC(2026, 6, 16)], + ]) { + assert.equal(acquisitionMsUtc(date, time), expected); + } +}); diff --git a/src/data/firms-coordinate-evidence.test.mjs b/src/data/firms-coordinate-evidence.test.mjs new file mode 100644 index 000000000..d60a7ee37 --- /dev/null +++ b/src/data/firms-coordinate-evidence.test.mjs @@ -0,0 +1,69 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { parseFirmsCsv } from './firmsCsv.js'; +import { adaptFirmsRecords } from './firmsAdapt.js'; +test('missing positions do not become fire detections at zero coordinates', () => { + const csv = + 'latitude,longitude,acq_date,acq_time,confidence,frp\n,1,2026-07-16,1200,n,1\n1, ,2026-07-16,1200,n,1\n95,1,2026-07-16,1200,n,1\n1,181,2026-07-16,1200,n,1\n0,0,2026-07-16,1200,n,1\n-90,-180,2026-07-16,1200,n,1'; + assert.deepEqual( + parseFirmsCsv(csv).map((r) => [r.lat, r.lon]), + [ + [0, 0], + [-90, -180], + ], + ); + const records = [ + { lat: null, lon: 1 }, + { lat: '', lon: 1 }, + { lat: false, lon: 1 }, + { lat: 95, lon: 1 }, + { lat: 1, lon: 181 }, + { lat: '0', lon: 0 }, + { lat: -90, lon: -180 }, + ]; + assert.deepEqual( + adaptFirmsRecords(records).map((r) => [r.index, r.lat, r.lon]), + [ + [0, 0, 0], + [1, -90, -180], + ], + ); +}); + +test('both axes keep their exact bounds and drop values just past them', () => { + const row = (lat, lon) => `${lat},${lon},2026-07-16,1200,n,1`; + const csv = [ + 'latitude,longitude,acq_date,acq_time,confidence,frp', + row(90, 180), + row(-90, -180), + row(90.000001, 0), + row(-90.000001, 0), + row(0, 180.000001), + row(0, -180.000001), + row(' ', 5), + row(5, '\t'), + ].join('\n'); + assert.deepEqual( + parseFirmsCsv(csv).map((r) => [r.lat, r.lon]), + [ + [90, 180], + [-90, -180], + ], + ); + const kept = adaptFirmsRecords([ + { lat: 90, lon: 180 }, + { lat: '-90', lon: '-180' }, + { lat: -90.000001, lon: 0 }, + { lat: 0, lon: -180.000001 }, + { lat: 5, lon: ' ' }, + { lat: 5, lon: undefined }, + { lat: [5], lon: 5 }, + ]); + assert.deepEqual( + kept.map((r) => [r.lat, r.lon]), + [ + [90, 180], + [-90, -180], + ], + ); +}); diff --git a/src/data/firmsAdapt.js b/src/data/firmsAdapt.js index 2fc909bce..312838e80 100644 --- a/src/data/firmsAdapt.js +++ b/src/data/firmsAdapt.js @@ -7,8 +7,8 @@ */ /** - * Map proxy fire records into internal fire records. Records with - * non-finite coordinates are skipped; `index` is the post-skip position + * Map proxy fire records into internal fire records. Records with missing, + * non-numeric or out-of-range (WGS84) coordinates are skipped; `index` is the post-skip position * (it keys pick ids and context-store ids, so it must stay sequential). * `contextEntity`/`position` start null and are lazily filled by the layer. * @param {?Array} records - /api/firms `fires` array. @@ -21,7 +21,15 @@ export function adaptFirmsRecords(records) { for (const record of records) { const lat = Number(record?.lat); const lon = Number(record?.lon); - if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; + if ( + !hasCoordinateValue(record?.lat) || + !hasCoordinateValue(record?.lon) || + !Number.isFinite(lat) || + !Number.isFinite(lon) || + Math.abs(lat) > 90 || + Math.abs(lon) > 180 + ) + continue; fires.push({ index: fires.length, lat, @@ -110,3 +118,11 @@ function finiteNumber(value) { const number = Number(value); return Number.isFinite(number) ? number : 0; } + +// Same rule as parseFirmsCsv: a blank or non-numeric value is not a zero. +function hasCoordinateValue(value) { + return ( + typeof value === 'number' || + (typeof value === 'string' && value.trim() !== '') + ); +} diff --git a/src/data/firmsCsv.js b/src/data/firmsCsv.js index 7f9019e18..368ef4f73 100644 --- a/src/data/firmsCsv.js +++ b/src/data/firmsCsv.js @@ -105,7 +105,17 @@ export function parseFirmsCsv(text) { if (parts.length < header.length) continue; // malformed row — skip const lat = Number(parts[iLat]); const lon = Number(parts[iLon]); - if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; + // Blank or out-of-range positions are dropped, never read as 0 (see + // hasCoordinateValue in firmsAdapt.js for the proxy-record side). + if ( + !parts[iLat].trim() || + !parts[iLon].trim() || + !Number.isFinite(lat) || + !Number.isFinite(lon) || + Math.abs(lat) > 90 || + Math.abs(lon) > 180 + ) + continue; records.push({ lat, @@ -153,7 +163,13 @@ export function acquisitionMsUtc(acqDate, acqTime) { minutes > 59 ) return NaN; - return Date.UTC(year, month - 1, day, hours, minutes); + const ms = Date.UTC(year, month - 1, day, hours, minutes); + const parsed = new Date(ms); + return parsed.getUTCFullYear() === year && + parsed.getUTCMonth() === month - 1 && + parsed.getUTCDate() === day + ? ms + : NaN; } /** diff --git a/src/data/flights.test.mjs b/src/data/flights.test.mjs index b09dc7014..c325ea744 100644 --- a/src/data/flights.test.mjs +++ b/src/data/flights.test.mjs @@ -79,6 +79,12 @@ test('flights analyst record: full record maps every contract field', () => { aircraftClass: 'airliner', originCountry: 'United States', operator: 'Southwest Airlines', + squawk: null, + emergency: false, + emergencyKind: null, + emergencySource: null, + emergencyBroadcast: null, + lastContactAtMs: null, routeOrigin: 'AUS', routeDestination: 'LAX', }); diff --git a/src/data/gbfsProxy.test.mjs b/src/data/gbfsProxy.test.mjs index 191cb57ba..c95256137 100644 --- a/src/data/gbfsProxy.test.mjs +++ b/src/data/gbfsProxy.test.mjs @@ -1,6 +1,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { + GBFS_CLIENT_IDENTIFIER, GBFS_MAX_BODY_BYTES, fetchGbfsUpstream, } from '../../server/providers/gbfs.js'; @@ -56,6 +57,41 @@ test('GBFS upstream fetch passes a direct 200 through unchanged', async () => { }); }); +test('GBFS upstream fetch names the app in the Client-Identifier header', async () => { + let observedHeaders = null; + await fetchGbfsUpstream( + 'https://gbfs.urbansharing.com/oslobysykkel.no/station_status.json', + { + fetchImpl: async (_url, options) => { + observedHeaders = options.headers; + return new Response(STATION_BODY, { status: 200 }); + }, + }, + ); + assert.equal(GBFS_CLIENT_IDENTIFIER, 'gods-eye-view'); + assert.equal(observedHeaders['Client-Identifier'], GBFS_CLIENT_IDENTIFIER); +}); + +test('GBFS upstream fetch adds ET-Client-Name for Entur and only for Entur', async () => { + const headersFor = async (url) => { + let observed = null; + await fetchGbfsUpstream(url, { + fetchImpl: async (_url, options) => { + observed = options.headers; + return new Response(STATION_BODY, { status: 200 }); + }, + }); + return observed; + }; + const entur = await headersFor( + 'https://api.entur.io/mobility/v2/gbfs/v2/kolumbusbysykkel/station_status', + ); + assert.equal(entur['ET-Client-Name'], 'gods-eye-view-bikeshare'); + assert.equal(entur['Client-Identifier'], GBFS_CLIENT_IDENTIFIER); + const lyft = await headersFor(STATION_URL); + assert.equal(lyft['ET-Client-Name'], undefined); +}); + test('GBFS upstream fetch forwards a non-redirect error status for the middleware to relay', async () => { const result = await fetchGbfsUpstream(STATION_URL, { fetchImpl: async () => new Response(null, { status: 404 }), diff --git a/src/data/gbfsSource.js b/src/data/gbfsSource.js index a15a806d9..57228554e 100644 --- a/src/data/gbfsSource.js +++ b/src/data/gbfsSource.js @@ -4,10 +4,35 @@ const GBFS_ALLOWED_HOSTS = new Set([ 'gbfs.bluebikes.com', 'gbfs.bcycle.com', 'gbfs.biketownpdx.com', - 'gbfs.cogobikeshare.com', + 'reddybikeshare.socialbicycles.com', 'austin.publicbikesystem.net', 'hon.publicbikesystem.net', 'chat.publicbikesystem.net', + 'gbfs.urbansharing.com', + 'api.entur.io', + 'api.cyclocity.fr', +]); + +/** + * Hosts that serve more than GBFS keep the proxy to their GBFS tree: Entur's + * api.entur.io fronts every Entur API, and api.cyclocity.fr every JCDecaux + * contract endpoint. + */ +const GBFS_HOST_PATH_PREFIXES = new Map([ + ['api.entur.io', '/mobility/v2/gbfs/'], + ['api.cyclocity.fr', '/contracts/'], +]); + +/** + * Identifying headers an operator asks every GBFS client to send. Entur rate + * limits or blocks consumers that do not send ET-Client-Name, in the form + * "company-application". + */ +const GBFS_HOST_HEADERS = new Map([ + [ + 'api.entur.io', + Object.freeze({ 'ET-Client-Name': 'gods-eye-view-bikeshare' }), + ], ]); /** @@ -28,13 +53,40 @@ export function isAllowedGbfsHost(hostname) { } /** - * Only allow station_information.json and station_status.json endpoints. + * Only allow station_information and station_status endpoints. The `.json` + * suffix is optional because Entur serves its feeds without one. When a + * hostname is given and that host serves more than GBFS, the path must also + * sit under the host's GBFS prefix. * * @param {string} pathname + * @param {string} [hostname] * @returns {boolean} */ -export function isAllowedGbfsPath(pathname) { - return /\/station_(information|status)\.json$/i.test(String(pathname || '')); +export function isAllowedGbfsPath(pathname, hostname = '') { + const path = String(pathname || ''); + if (!/\/station_(information|status)(\.json)?$/i.test(path)) return false; + const prefix = GBFS_HOST_PATH_PREFIXES.get( + String(hostname || '') + .trim() + .toLowerCase(), + ); + return !prefix || path.startsWith(prefix); +} + +/** + * Extra request headers an upstream GBFS host asks clients to send. + * + * @param {string} hostname + * @returns {Record} + */ +export function gbfsUpstreamHeaders(hostname) { + return ( + GBFS_HOST_HEADERS.get( + String(hostname || '') + .trim() + .toLowerCase(), + ) || {} + ); } /** @@ -47,7 +99,7 @@ export function isAllowedGbfsPath(pathname) { * @returns {string} Cache-Control header value. */ export function gbfsCacheControl(pathname) { - if (/\/station_information\.json$/i.test(String(pathname || ''))) { + if (/\/station_information(\.json)?$/i.test(String(pathname || ''))) { return 'public, max-age=300'; } return 'no-store'; diff --git a/src/data/geoFileImport.js b/src/data/geoFileImport.js new file mode 100644 index 000000000..0ceaf9ef5 --- /dev/null +++ b/src/data/geoFileImport.js @@ -0,0 +1,1390 @@ +/** Session-local geodata loading with bounded geometry and local-only file references. */ +import * as Cesium from 'cesium'; +import { createImportImageBudget } from './geoImportImages.js'; +import { createImportedPins } from './geoImportPins.js'; + +/** Largest file accepted, in bytes. */ +export const MAX_IMPORT_BYTES = 50 * 1024 * 1024; +/** Largest total a KMZ may inflate to — a zip bomb stops here. */ +export const MAX_KMZ_INFLATED_BYTES = 120 * 1024 * 1024; +/** Most files on the globe at once. */ +export const MAX_IMPORTED_FILES = 12; +/** Most entities one file may create before it is refused. */ +export const MAX_IMPORT_ENTITIES = 20_000; +/** Most vertices (points of every line, ring and marker) in one file. */ +export const MAX_IMPORT_POSITIONS = 250_000; +/** Most normalized style definitions and local image links in one KML/KMZ. */ +export const MAX_KML_LINKS = 2_000; +/** Longest plain-text name kept from a file's features. */ +export const MAX_IMPORT_NAME_LENGTH = 120; +/** What the file picker offers. */ +export const GEO_FILE_ACCEPT = '.geojson,.json,.kml,.kmz,.gpx'; + +const GEOJSON_TYPES = new Set([ + 'FeatureCollection', + 'Feature', + 'Point', + 'MultiPoint', + 'LineString', + 'MultiLineString', + 'Polygon', + 'MultiPolygon', + 'GeometryCollection', +]); +const KMZ_IMAGE_TYPES = Object.freeze({ + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + bmp: 'image/bmp', +}); +// External content and metadata the session-local import does not use. +const KML_REMOVED_ELEMENTS = new Set([ + 'NetworkLink', + 'NetworkLinkControl', + 'ScreenOverlay', + 'PhotoOverlay', + 'Model', + 'Tour', + 'description', + 'BalloonStyle', + 'Snippet', + 'ExtendedData', + 'TimeSpan', + 'TimeStamp', + 'LookAt', + 'Camera', + 'visibility', + 'viewRefreshMode', + 'viewFormat', + 'httpQuery', +]); + +/** An error whose message is meant for the person who chose the file. */ +export class GeoImportError extends Error { + constructor(message) { + super(message); + this.name = 'GeoImportError'; + } +} + +/** + * The format of a file, from its name first and its first bytes second. + * @param {string} name File name. + * @param {Uint8Array} [head] The first bytes of the file. + * @returns {'geojson'|'kml'|'kmz'|'gpx'|null} + */ +export function detectGeoFileFormat(name, head) { + const ext = /\.([a-z0-9]+)$/i.exec(String(name || ''))?.[1]?.toLowerCase(); + if (ext === 'geojson') return 'geojson'; + if (ext === 'kml' || ext === 'kmz' || ext === 'gpx') return ext; + if (!head?.length) return ext === 'json' ? 'geojson' : null; + if ( + head[0] === 0x50 && + head[1] === 0x4b && + head[2] === 0x03 && + head[3] === 0x04 + ) + return 'kmz'; + const text = new TextDecoder() + .decode(head.subarray(0, 1024)) + .replace(/^/, '') + .trimStart(); + if (text.startsWith('{')) return 'geojson'; + if (text.startsWith('<')) { + if (/]/i.test(text)) return 'kml'; + if (/]/i.test(text)) return 'gpx'; + } + return ext === 'json' ? 'geojson' : null; +} + +/** + * A short, printable name for the list: the file name without control + * characters, cut to 60 characters. + * @param {string} name + * @returns {string} + */ +export function importDisplayName(name) { + const clean = String(name || '') + .replace(/[\u0000-\u001f\u007f]/g, '') + .trim(); + if (!clean) return 'Imported file'; + return clean.length > 60 ? `${clean.slice(0, 57)}…` : clean; +} + +/** + * Parse and check GeoJSON text. Throws `GeoImportError` for anything that is + * not a GeoJSON object. + * @param {string} text + * @returns {object} + */ +export function parseGeoJsonText(text) { + let value; + try { + value = JSON.parse(String(text).replace(/^/, '')); + } catch { + throw new GeoImportError('That file is not valid JSON.'); + } + if (!value || typeof value !== 'object' || !GEOJSON_TYPES.has(value.type)) + throw new GeoImportError('That JSON file is not GeoJSON.'); + if (value.type === 'FeatureCollection' && !Array.isArray(value.features)) + throw new GeoImportError('That FeatureCollection has no features list.'); + return value; +} + +/** + * Count features and vertices in a parsed GeoJSON object without recursion + * limits or array spreading, stopping as soon as a limit is passed. + * @param {object} geojson + * @param {{maxFeatures?: number, maxPositions?: number, maxEntities?: number}} [limits] + * @returns {{features: number, positions: number, entities: number, + * over: ''|'features'|'positions'|'entities'}} + */ +export function countGeoJsonGeometry( + geojson, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + maxEntities = MAX_IMPORT_ENTITIES, + } = {}, +) { + let features = ['Feature', 'FeatureCollection'].includes(geojson?.type) + ? 0 + : 1; + let positions = 0; + let entities = 0; + const result = (over = '') => ({ features, positions, entities, over }); + const stack = [geojson]; + while (stack.length) { + const node = stack.pop(); + if (!node || typeof node !== 'object') continue; + if (node.type === 'FeatureCollection') { + const list = Array.isArray(node.features) ? node.features : []; + for (let i = 0; i < list.length; i++) { + if (list[i]?.type !== 'Feature') + throw new GeoImportError( + 'Every GeoJSON feature must have type Feature.', + ); + stack.push(list[i]); + } + continue; + } + if (node.type === 'Feature') { + features += 1; + if (features > maxFeatures) return result('features'); + if (node.geometry == null) { + entities += 1; + if (entities > maxEntities) return result('entities'); + } + stack.push(node.geometry); + continue; + } + if (node.type === 'GeometryCollection') { + const list = node.geometries; + if (!Array.isArray(list)) + throw new GeoImportError('That GeoJSON has invalid geometries.'); + for (let i = 0; i < list.length; i++) { + if (!list[i] || ['Feature', 'FeatureCollection'].includes(list[i].type)) + throw new GeoImportError( + 'That GeoJSON contains an unsupported geometry.', + ); + stack.push(list[i]); + } + continue; + } + const depth = { + Point: 0, + MultiPoint: 1, + LineString: 1, + MultiLineString: 2, + Polygon: 2, + MultiPolygon: 3, + }[node.type]; + if (depth !== undefined) { + // Multipart geometries create one Cesium entity per component, even + // when the document wraps them in just one Feature. + entities += node.type.startsWith('Multi') + ? Array.isArray(node.coordinates) + ? node.coordinates.length + : 0 + : 1; + if (entities > maxEntities) return result('entities'); + // Walk the coordinate arrays iteratively: a position is an array whose + // first item is a number. + const coords = [[node.coordinates, depth]]; + while (coords.length) { + const [c, remaining] = coords.pop(); + if (!Array.isArray(c)) + throw new GeoImportError('That GeoJSON has invalid coordinates.'); + if (remaining === 0) { + if ( + c.length < 2 || + c.length > 3 || + !c.every(Number.isFinite) || + Math.abs(c[0]) > 180 || + Math.abs(c[1]) > 90 || + (c.length === 3 && Math.abs(c[2]) > 1_000_000) + ) + throw new GeoImportError('That GeoJSON has invalid coordinates.'); + positions += 1; + if (positions > maxPositions) return result('positions'); + } else + for (let i = 0; i < c.length; i++) coords.push([c[i], remaining - 1]); + } + } else + throw new GeoImportError( + 'That GeoJSON contains an unsupported geometry.', + ); + } + return result(); +} + +/** The refusal for a file over a size limit, in the person's terms. */ +function overLimitError({ over, features }) { + if (over === 'entities') + return new GeoImportError( + `That file creates more than ${MAX_IMPORT_ENTITIES.toLocaleString()} map objects (the limit).`, + ); + if (over === 'links') + return new GeoImportError( + `That file has more than ${MAX_KML_LINKS.toLocaleString()} style definitions and image links after cleanup (the limit).`, + ); + if (over === 'features') + return new GeoImportError( + `That file has more than ${MAX_IMPORT_ENTITIES.toLocaleString()} features (the limit).`, + ); + return new GeoImportError( + `That file has more than ${MAX_IMPORT_POSITIONS.toLocaleString()} points (the limit)${features ? ` across ${features.toLocaleString()} features` : ''}.`, + ); +} + +// The import uses WGS84 ground footprints and a short name only. Remove +// loader-sensitive metadata before Cesium allocates properties or entity IDs. +function prepareGeoJson(geojson) { + const stack = [geojson]; + while (stack.length) { + const node = stack.pop(); + if (!node) continue; + delete node.crs; + delete node.id; + if (node.type === 'FeatureCollection') { + for (const feature of node.features) stack.push(feature); + } else if (node.type === 'Feature') { + const name = importFeatureName(node.properties?.name); + node.properties = name ? { name } : {}; + stack.push(node.geometry); + } else if (node.type === 'GeometryCollection') { + for (const geometry of node.geometries) stack.push(geometry); + } else { + const coords = [node.coordinates]; + while (coords.length) { + const value = coords.pop(); + if (typeof value[0] === 'number') value.length = 2; + else for (const child of value) coords.push(child); + } + } + } +} + +/** Inflate raw-deflate bytes with the platform's stream, stopping at `maxBytes`. */ +async function inflateRaw(bytes, maxBytes) { + const stream = new Blob([bytes]) + .stream() + .pipeThrough(new DecompressionStream('deflate-raw')); + const reader = stream.getReader(); + const chunks = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.length; + if (total > maxBytes) + throw new GeoImportError( + 'That KMZ unpacks to more than the import limit.', + ); + chunks.push(value); + } + } catch (error) { + await reader.cancel().catch(() => {}); + if (error instanceof GeoImportError) throw error; + throw new GeoImportError('That KMZ is damaged and could not be unpacked.'); + } + const out = new Uint8Array(total); + let at = 0; + for (const chunk of chunks) { + out.set(chunk, at); + at += chunk.length; + } + return out; +} + +/** + * Read the entries of a zip archive (stored or deflated; no encryption, no + * ZIP64). Directory entries are skipped. + * @param {Uint8Array} bytes The whole archive. + * @param {{maxInflatedBytes?: number}} [options] + * @returns {Promise>} Entry path → contents. + */ +export async function readZipEntries( + bytes, + { maxInflatedBytes = MAX_KMZ_INFLATED_BYTES } = {}, +) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const fail = (message = 'That KMZ is not a readable zip archive.') => { + throw new GeoImportError(message); + }; + // End of central directory: 22 bytes plus a comment of up to 64 KiB. + let eocd = -1; + for ( + let i = bytes.length - 22; + i >= Math.max(0, bytes.length - 22 - 0xffff); + i-- + ) { + if (view.getUint32(i, true) === 0x06054b50) { + eocd = i; + break; + } + } + if (eocd < 0) fail(); + const count = view.getUint16(eocd + 10, true); + const dirSize = view.getUint32(eocd + 12, true); + let at = view.getUint32(eocd + 16, true); + if (count === 0xffff || at === 0xffffffff || at + dirSize > eocd) + fail('That KMZ uses a zip format this import does not read.'); + const entries = new Map(); + const names = new TextDecoder(); + let inflated = 0; + for (let n = 0; n < count; n++) { + if (at + 46 > bytes.length || view.getUint32(at, true) !== 0x02014b50) + fail(); + const flags = view.getUint16(at + 8, true); + const method = view.getUint16(at + 10, true); + const compressed = view.getUint32(at + 20, true); + const size = view.getUint32(at + 24, true); + const nameLength = view.getUint16(at + 28, true); + const extraLength = view.getUint16(at + 30, true); + const commentLength = view.getUint16(at + 32, true); + const local = view.getUint32(at + 42, true); + const name = names.decode(bytes.subarray(at + 46, at + 46 + nameLength)); + at += 46 + nameLength + extraLength + commentLength; + if (name.endsWith('/')) continue; + if (flags & 0x1) fail('That KMZ is encrypted.'); + if (compressed === 0xffffffff || size === 0xffffffff) + fail('That KMZ uses a zip format this import does not read.'); + if (local + 30 > bytes.length || view.getUint32(local, true) !== 0x04034b50) + fail(); + const start = + local + + 30 + + view.getUint16(local + 26, true) + + view.getUint16(local + 28, true); + if (start + compressed > bytes.length) fail(); + const data = bytes.subarray(start, start + compressed); + let contents; + if (method === 0) contents = data; + else if (method === 8) + contents = await inflateRaw(data, maxInflatedBytes - inflated); + else fail('That KMZ uses a compression method this import does not read.'); + inflated += contents.length; + if (inflated > maxInflatedBytes) + fail('That KMZ unpacks to more than the import limit.'); + entries.set(name, contents); + } + return entries; +} + +/** + * The KML document inside a KMZ: `doc.kml` when present, else the first + * `.kml` entry nearest the archive root. + * @param {Map} entries + * @returns {string|null} Entry path. + */ +export function kmzMainDocument(entries) { + if (entries.has('doc.kml')) return 'doc.kml'; + const kml = [...entries.keys()] + .filter((name) => /\.kml$/i.test(name)) + .sort( + (a, b) => a.split('/').length - b.split('/').length || a.localeCompare(b), + ); + return kml[0] ?? null; +} + +/** + * Resolve a KMZ-relative href to an entry path, or null. + * @param {string} href As written in the KML. + * @param {string} base The KML entry's path. + * @param {Map} entries + * @returns {string|null} + */ +export function resolveKmzPath(href, base, entries) { + const raw = String(href || '').trim(); + if (!raw || /^[a-z][a-z0-9+.-]*:/i.test(raw) || raw.startsWith('//')) + return null; + let decoded; + try { + decoded = decodeURIComponent(raw.split(/[?#]/)[0]); + } catch { + return null; + } + const parts = base.split('/').slice(0, -1); + for (const part of decoded.replace(/\\/g, '/').split('/')) { + if (!part || part === '.') continue; + if (part === '..') parts.pop(); + else parts.push(part); + } + const path = parts.join('/'); + return entries.has(path) ? path : null; +} + +/** MIME type for a KMZ image entry, or null when it is not an image we serve. */ +export function kmzImageType(path) { + const ext = /\.([a-z0-9]+)$/i.exec(path)?.[1]?.toLowerCase(); + return KMZ_IMAGE_TYPES[ext] ?? null; +} + +const elementsNamed = (doc, localName) => + [...doc.getElementsByTagName('*')].filter((el) => el.localName === localName); + +/** + * A `resolveLocal` for `sanitizeKmlDocument`: the blob URL of an image packed + * in the KMZ, or null. Each entry gets one blob and one URL however many + * hrefs name it, so repeated icons don't copy (and decode) the image again. + * @param {Map} entries + * @param {string} main Path of the KMZ's KML document. + * @param {(url: string) => void} onUrl Called once per new URL (to revoke later). + * @param {(blob: Blob) => string} [createUrl] + * @returns {(href: string) => string|null} + */ +export function kmzImageResolver( + entries, + main, + onUrl, + createUrl = (blob) => URL.createObjectURL(blob), +) { + const urls = new Map(); + return (href) => { + const path = resolveKmzPath(href, main, entries); + const type = path && kmzImageType(path); + if (!type) return null; + let url = urls.get(path); + if (!url) { + url = createUrl(new Blob([entries.get(path)], { type })); + urls.set(path, url); + onUrl(url); + } + return url; + }; +} + +/** + * Strip everything from a parsed KML document that would make the page reach + * outside the file. `resolveLocal(href)` returns a URL for an image packed in + * the same KMZ (or null); every other href is removed, so icons fall back to + * Cesium's default pin and a ground overlay without a local image is dropped. + * @param {Document} doc + * @param {(href: string) => string|null} [resolveLocal] + * @returns {{removed: number}} How many elements were taken out. + */ +export function sanitizeKmlDocument(doc, resolveLocal = () => null) { + let removed = 0; + const drop = (el) => { + if (el.parentNode) { + el.parentNode.removeChild(el); + removed += 1; + } + }; + // One snapshot and pass, rather than rescanning the whole XML tree for + // every discarded element type. Strip unused metadata before the loader + // constructs descriptions, availability intervals and camera objects. + for (const el of [...doc.getElementsByTagName('*')]) + if ( + KML_REMOVED_ELEMENTS.has(el.localName) || + el.localName in Object.prototype + ) + drop(el); + else if (!['Style', 'StyleMap'].includes(el.localName)) + el.removeAttribute('id'); + for (const href of elementsNamed(doc, 'href')) { + const value = (href.textContent || '').trim(); + if (!value) continue; + const local = resolveLocal(value); + if (local) href.textContent = local; + else if (/^data:image\/(png|jpeg|gif|webp|bmp);/i.test(value)) continue; + else { + const icon = href.parentNode; + const overlay = icon?.parentNode; + if (overlay?.localName === 'GroundOverlay') { + drop(overlay); + continue; + } + // An empty Icon explicitly hides the billboard in Cesium. Remove the + // blocked icon definition so the loader supplies its local default pin. + // Authored empty Icons (with no blocked href) still mean no marker. + drop( + icon?.localName === 'Icon' && icon.parentNode?.localName === 'IconStyle' + ? icon + : href, + ); + } + } + for (const styleUrl of elementsNamed(doc, 'styleUrl')) { + const value = (styleUrl.textContent || '').trim(); + if (!value.startsWith('#')) drop(styleUrl); + else styleUrl.textContent = value; + } + for (const pair of elementsNamed(doc, 'Pair')) + if ( + !childText(pair, 'key') || + (!childrenNamed(pair, 'styleUrl').length && + !childrenNamed(pair, 'Style').length) + ) + drop(pair); + // Draw recorded KML paths as their original static vertices. Letting the + // loader build time-dynamic tracks would expand each into thousands of + // interpolated points later and exceed the checked geometry budget. + for (const track of elementsNamed(doc, 'Track')) { + const line = doc.createElementNS( + 'http://www.opengis.net/kml/2.2', + 'LineString', + ); + const coordinates = doc.createElementNS( + 'http://www.opengis.net/kml/2.2', + 'coordinates', + ); + const recordedCoordinates = childrenNamed(track, 'coord').map((coord) => + coord.textContent.trim().split(/\s+/).join(','), + ); + coordinates.textContent = recordedCoordinates.join(' '); + line.appendChild(coordinates); + for (const mode of childrenNamed(track, 'altitudeMode')) + line.appendChild(mode.cloneNode(true)); + track.parentNode?.replaceChild(line, track); + } + for (const tracks of elementsNamed(doc, 'MultiTrack')) { + const geometry = doc.createElementNS( + 'http://www.opengis.net/kml/2.2', + 'MultiGeometry', + ); + for (const line of childrenNamed(tracks, 'LineString')) + geometry.appendChild(line); + tracks.parentNode?.replaceChild(geometry, tracks); + } + for (const line of [ + ...elementsNamed(doc, 'LineString'), + ...elementsNamed(doc, 'LinearRing'), + ]) { + const mode = childText(line, 'altitudeMode'); + if ( + !mode || + mode === 'clampToGround' || + mode === 'clampToSeaFloor' || + !childrenNamed(line, 'tessellate').length + ) { + let tessellate = childrenNamed(line, 'tessellate')[0]; + if (!tessellate) { + tessellate = doc.createElementNS( + 'http://www.opengis.net/kml/2.2', + 'tessellate', + ); + line.appendChild(tessellate); + } + tessellate.textContent = '1'; + } + } + return { removed }; +} + +/** Child elements of `el` with this local name (any namespace). */ +const childrenNamed = (el, localName) => { + const out = []; + const kids = el?.childNodes || []; + for (let i = 0; i < kids.length; i++) + if (kids[i].nodeType === 1 && kids[i].localName === localName) + out.push(kids[i]); + return out; +}; +const childText = (el, localName) => + (childrenNamed(el, localName)[0]?.textContent || '').trim(); + +/** + * Share identical styles before the loader builds their graphics. Inline + * overrides stay inline StyleMaps, so an existing styleUrl still supplies + * properties the override leaves unset. Named references are redirected. + * Random-color styles remain unchanged; duplicate IDs + * keep the loader's original precedence instead of being rewritten. + * @param {Document} doc Sanitized KML. + * @param {{serializeStyle?: Function, maxStyles?: number}} [options] + * @returns {number} Number of original style blocks replaced. + */ +export function deduplicateKmlStyles( + doc, + { serializeStyle, maxStyles = MAX_KML_LINKS } = {}, +) { + const serializer = serializeStyle ? null : new XMLSerializer(); + const serialize = + serializeStyle || ((el) => serializer.serializeToString(el)); + const groups = new Map(); + const usedIds = new Set(); + const duplicateIds = new Set(); + for (const el of doc.getElementsByTagName('*')) { + const id = el.getAttribute('id'); + if (id !== null) { + if (usedIds.has(id)) duplicateIds.add(id); + usedIds.add(id); + } + } + for (const style of elementsNamed(doc, 'Style')) { + if ( + elementsNamed(style, 'colorMode').some( + (el) => el.textContent.trim() === 'random', + ) + ) + continue; + const template = style.cloneNode(true); + template.removeAttribute('id'); + const key = serialize(template); + let group = groups.get(key); + if (!group) { + group = { template, nodes: [] }; + groups.set(key, group); + if (groups.size > maxStyles) throw overLimitError({ over: 'links' }); + } + group.nodes.push(style); + } + const redirects = new Map(); + let replaced = 0; + let serial = 0; + let container; + for (const { template, nodes } of groups.values()) { + if ( + nodes.length < 2 || + nodes.some((el) => duplicateIds.has(el.getAttribute('id'))) + ) + continue; + const ns = template.namespaceURI || 'http://www.opengis.net/kml/2.2'; + if (!container) { + const root = doc.documentElement; + container = + root.localName === 'Document' + ? root + : childrenNamed(root, 'Document')[0]; + if (!container) { + container = doc.createElementNS(ns, 'Document'); + while (root.firstChild) container.appendChild(root.firstChild); + root.appendChild(container); + } + } + let id; + do { + id = `gev-import-style-${++serial}`; + } while (usedIds.has(id)); + usedIds.add(id); + template.setAttribute('id', id); + container.appendChild(template); + for (const style of nodes) { + const originalId = style.getAttribute('id'); + if (originalId !== null) redirects.set(originalId, id); + let reference; + if (style.parentNode.localName === 'Pair') { + reference = doc.createElementNS(ns, 'styleUrl'); + reference.textContent = `#${id}`; + } else { + reference = doc.createElementNS(ns, 'StyleMap'); + for (const state of ['normal', 'highlight']) { + const pair = doc.createElementNS(ns, 'Pair'); + const key = doc.createElementNS(ns, 'key'); + key.textContent = state; + const url = doc.createElementNS(ns, 'styleUrl'); + url.textContent = `#${id}`; + pair.appendChild(key); + pair.appendChild(url); + reference.appendChild(pair); + } + } + style.parentNode.replaceChild(reference, style); + replaced += 1; + } + } + for (const url of elementsNamed(doc, 'styleUrl')) { + const value = url.textContent.trim(); + const id = value.startsWith('#') && redirects.get(value.slice(1)); + if (id) url.textContent = `#${id}`; + } + return replaced; +} + +/** + * Count placemarks and vertices in a parsed KML document before Cesium reads + * it: every `` tuple and every `gx:coord`, plus normalized + * style definitions and distinct image `` values (`links`). + * Inline StyleMaps are references, not additional shared style definitions. + * @param {Document} doc + * @returns {{features: number, entities: number, positions: number, links: number, + * over: ''|'features'|'entities'|'positions'|'links'}} + */ +export function countKmlGeometry( + doc, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxEntities = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + maxLinks = MAX_KML_LINKS, + } = {}, +) { + let features = 0; + let entities = 0; + let positions = 0; + let links = 0; + const imageLinks = new Set(); + const result = (over) => ({ features, entities, positions, links, over }); + const geometryNames = new Set([ + 'Point', + 'LineString', + 'LinearRing', + 'Polygon', + 'Track', + 'MultiTrack', + 'MultiGeometry', + 'Model', + ]); + for (const el of doc.getElementsByTagName('*')) { + const name = el.localName; + if ( + ['north', 'south', 'east', 'west', 'rotation', 'altitude'].includes(name) + ) { + const value = Number(el.textContent); + const limit = ['north', 'south'].includes(name) + ? 90 + : name === 'altitude' + ? 1_000_000 + : 360; + if ( + !el.textContent.trim() || + !Number.isFinite(value) || + Math.abs(value) > limit + ) + throw new GeoImportError('That KML has invalid geometry bounds.'); + } + // Cesium creates entities for folders, a parent for every placemark, and + // a child for each geometry in a MultiGeometry, including nested containers. + if ( + name === 'Placemark' || + name === 'Folder' || + name === 'GroundOverlay' || + (geometryNames.has(name) && + ['MultiGeometry', 'MultiTrack'].includes(el.parentNode?.localName)) + ) { + entities += 1; + if (entities > maxEntities) return result('entities'); + } + if ( + name === 'Style' || + (name === 'StyleMap' && el.getAttribute('id') !== null) + ) { + links += 1; + if (links > maxLinks) return result('links'); + } else if (name === 'href') { + const href = el.textContent.trim(); + if (!imageLinks.has(href)) { + imageLinks.add(href); + links += 1; + if (links > maxLinks) return result('links'); + } + } else if (name === 'Placemark') { + features += 1; + if (features > maxFeatures) return result('features'); + } else if (name === 'coordinates') { + const text = el.textContent || ''; + for (const match of text.matchAll(/\S+/g)) { + checkKmlPosition(match[0].split(',')); + positions += 1; + if (positions > maxPositions) return result('positions'); + } + } else if (name === 'coord') { + checkKmlPosition(el.textContent.trim().split(/\s+/)); + positions += 1; + if (positions > maxPositions) return result('positions'); + } + } + return result(''); +} + +function checkKmlPosition(tuple) { + const numbers = tuple.map(Number); + if ( + tuple.length < 2 || + tuple.length > 3 || + tuple.some((v) => !v.trim()) || + !numbers.every(Number.isFinite) || + Math.abs(numbers[0]) > 180 || + Math.abs(numbers[1]) > 90 || + (numbers.length === 3 && Math.abs(numbers[2]) > 1_000_000) + ) + throw new GeoImportError('That KML has invalid coordinates.'); +} + +/** + * Read a parsed GPX document into GeoJSON: waypoints become points, each + * route a line and each track one MultiLineString with a line per `` + * (segment breaks are gaps in the recording and stay gaps). Only names are + * kept, and no times, so a dated track draws without a clock. Throws + * `GeoImportError` past the size limits. + * @param {Document} doc + * @returns {object} A FeatureCollection. + */ +export function gpxToGeoJson( + doc, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + } = {}, +) { + const root = doc.documentElement; + if (root?.localName !== 'gpx') + throw new GeoImportError('That GPX file has no document.'); + const features = []; + let positions = 0; + const point = (el) => { + const rawLat = el.getAttribute('lat'); + const rawLon = el.getAttribute('lon'); + if (rawLat == null || rawLon == null || !rawLat.trim() || !rawLon.trim()) + return null; + const lat = Number(rawLat); + const lon = Number(rawLon); + if ( + !Number.isFinite(lat) || + !Number.isFinite(lon) || + Math.abs(lat) > 90 || + Math.abs(lon) > 180 + ) + return null; + positions += 1; + if (positions > maxPositions) + throw overLimitError({ + over: 'positions', + features: features.length, + positions, + }); + return [lon, lat]; + }; + const add = (name, geometry) => { + if (features.length >= maxFeatures) + throw overLimitError({ + over: 'features', + features: features.length + 1, + positions, + }); + features.push({ + type: 'Feature', + properties: name ? { name } : {}, + geometry, + }); + }; + for (const wpt of childrenNamed(root, 'wpt')) { + const at = point(wpt); + if (at) add(childText(wpt, 'name'), { type: 'Point', coordinates: at }); + } + for (const rte of childrenNamed(root, 'rte')) { + const line = []; + for (const pt of childrenNamed(rte, 'rtept')) { + const at = point(pt); + if (at) line.push(at); + } + if (line.length >= 2) + add(childText(rte, 'name'), { type: 'LineString', coordinates: line }); + } + for (const trk of childrenNamed(root, 'trk')) { + const lines = []; + for (const seg of childrenNamed(trk, 'trkseg')) { + const line = []; + for (const pt of childrenNamed(seg, 'trkpt')) { + const at = point(pt); + if (at) line.push(at); + } + if (line.length >= 2) lines.push(line); + } + if (lines.length) + add(childText(trk, 'name'), { + type: 'MultiLineString', + coordinates: lines, + }); + } + return { type: 'FeatureCollection', features }; +} + +/** A feature name as short plain text (no control characters), or undefined. */ +export function importFeatureName(value) { + const text = String(value ?? '') + .replace(/[\u0000-\u001f\u007f]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + if (!text) return undefined; + return text.length > MAX_IMPORT_NAME_LENGTH + ? `${text.slice(0, MAX_IMPORT_NAME_LENGTH - 1)}…` + : text; +} + +/** + * Make every entity of an imported data source inert: no description, + * balloon, extended data or properties, no native label, and a name that is + * short plain text. Generated pin canvases share a data URL so Cesium uploads + * each image once, rather than allocating a texture entry for every entity. + * Returns how many entities were touched. + * @param {{entities: {values: object[]}}} dataSource + * @returns {number} + */ +export function scrubImportedEntities(dataSource) { + const collection = dataSource?.entities; + const list = collection?.values || []; + const images = new Map(); + collection?.suspendEvents?.(); + try { + for (let i = 0; i < list.length; i++) { + const entity = list[i]; + let name; + try { + name = importFeatureName( + typeof entity.name === 'string' ? entity.name : undefined, + ); + } catch { + name = undefined; + } + entity.name = name; + entity.description = undefined; + entity.label = undefined; + entity.properties = undefined; + if (entity.kml && typeof entity.kml === 'object') entity.kml = undefined; + if ( + entity.billboard?.heightReference?.getValue?.() === + Cesium.HeightReference.CLAMP_TO_GROUND + ) + entity.billboard.disableDepthTestDistance = Number.POSITIVE_INFINITY; + const property = entity.billboard?.image; + if (property?.isConstant === false) continue; + const image = property?.getValue?.(); + if (image?.tagName === 'CANVAS') { + let url = images.get(image); + if (!url) { + url = image.toDataURL(); + images.set(image, url); + } + entity.billboard.image = url; + } + } + } finally { + collection?.resumeEvents?.(); + } + return list.length; +} + +/** + * Take time out of an imported data source so adding it cannot move the app's + * shared clock: the data source reports no clock (the viewer adopts the clock + * of every added source that has one), every entity is available for all + * time. KML tracks have already become their original static vertices in + * `sanitizeKmlDocument`, so no sampled-position fallback is needed here. + * @param {object} dataSource + * @returns {object} The same data source. + */ +export function detachImportedTime(dataSource) { + Object.defineProperty(dataSource, 'clock', { + configurable: true, + enumerable: false, + get: () => undefined, + set: () => {}, + }); + const list = dataSource?.entities?.values || []; + for (let i = 0; i < list.length; i++) { + const entity = list[i]; + entity.availability = undefined; + } + return dataSource; +} + +function parseXml(text, what) { + // Match the namespace repairs Cesium applies to ordinary exported KML. + if (what !== 'GPX') + text = String(text).replace(/]*>/i, (root) => { + const seen = new Set(); + root = root.replace( + /\s+xmlns:([\w.-]+)\s*=\s*(["'])[^"']*\2/g, + (declaration, prefix) => { + if (seen.has(prefix)) return ''; + seen.add(prefix); + return declaration; + }, + ); + if (!seen.has('xsi') && /[<\s]xsi:/.test(text)) + root = root.replace( + />$/, + ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">', + ); + return root; + }); + const doc = new DOMParser().parseFromString( + String(text).replace(/^/, ''), + 'application/xml', + ); + if (doc.getElementsByTagName('parsererror').length) + throw new GeoImportError(`That ${what} file is not valid XML.`); + return doc; +} + +function decodeXml(bytes) { + const encoding = + bytes[0] === 0xff && bytes[1] === 0xfe + ? 'utf-16le' + : bytes[0] === 0xfe && bytes[1] === 0xff + ? 'utf-16be' + : 'utf-8'; + return new TextDecoder(encoding).decode(bytes); +} + +function checkLocalImages(doc, entries = new Map(), main = '') { + const charge = createImportImageBudget(); + const checked = new Set(); + for (const href of elementsNamed(doc, 'href')) { + const value = href.textContent.trim(); + if (checked.has(value)) continue; + checked.add(value); + const path = resolveKmzPath(value, main, entries); + try { + if (path && kmzImageType(path)) charge(path, entries.get(path)); + else if (/^data:image\/(png|jpeg|gif|webp|bmp);/i.test(value)) { + const comma = value.indexOf(','); + const header = value.slice(0, comma); + const body = value.slice(comma + 1); + const raw = /;base64$/i.test(header) + ? atob(body) + : decodeURIComponent(body); + charge( + value, + Uint8Array.from(raw, (c) => c.charCodeAt(0)), + ); + } + } catch { + throw new GeoImportError( + 'That file has an unreadable image or exceeds the 16-megapixel image limit.', + ); + } + } +} + +// Cesium also processes KML in deferred tasks; an exception there does not +// reject its load promise. Keep a failed file from blocking the serial queue. +async function loadKml(doc, viewer, signal) { + let removeError; + try { + return await boundedImport( + Promise.race([ + Cesium.KmlDataSource.load(doc, { + camera: viewer.scene.camera, + canvas: viewer.scene.canvas, + clampToGround: true, + }), + new Promise((_, reject) => { + const failed = () => { + reject(new GeoImportError('That KML could not be read.')); + }; + globalThis.addEventListener?.('error', failed); + removeError = () => globalThis.removeEventListener?.('error', failed); + }), + ]), + signal, + ); + } finally { + removeError?.(); + } +} + +// Resolve a static file's ground pins in batches before they are visualized. +// Per-billboard CLAMP_TO_GROUND subscriptions scale poorly on large imports. +const groundPins = new WeakMap(); +async function settleGroundPins(dataSource, viewer, signal) { + let pins = groundPins.get(dataSource); + if (!pins) { + pins = []; + for (const entity of dataSource.entities.values) { + const reference = entity.billboard?.heightReference?.getValue?.(); + if ( + reference === Cesium.HeightReference.CLAMP_TO_GROUND || + reference === Cesium.HeightReference.RELATIVE_TO_GROUND + ) { + const position = entity.position?.getValue?.(Cesium.JulianDate.now()); + if (!position) continue; + const cartographic = Cesium.Cartographic.fromCartesian(position); + if (!cartographic) + throw new GeoImportError('That file has invalid point coordinates.'); + pins.push({ + entity, + original: position, + cartographic, + offset: + reference === Cesium.HeightReference.RELATIVE_TO_GROUND + ? cartographic.height + : 0, + }); + entity.billboard.heightReference = Cesium.HeightReference.NONE; + entity.billboard.disableDepthTestDistance = Number.POSITIVE_INFINITY; + } + } + groundPins.set(dataSource, pins); + } + for (let at = 0; at < pins.length; at += 256) { + signal?.throwIfAborted(); + const batch = pins.slice(at, at + 256); + const positions = batch.map( + ({ cartographic: c }) => + new Cesium.Cartographic(c.longitude, c.latitude, 0), + ); + const mesh = + viewer.scene?.globe?.show === false && + viewer.scene?.clampToHeightSupported; + // A coarse rendered ellipsoid mesh can report a height kilometres below + // zero. Its provider's exact height is zero even while a 3D map hides it; + // a buried starting point can make the downward mesh ray miss the surface. + const ellipsoid = + viewer.terrainProvider instanceof Cesium.EllipsoidTerrainProvider; + for (let i = 0; i < positions.length; i++) { + const sampled = ellipsoid + ? 0 + : viewer.scene?.globe?.getHeight?.(positions[i]); + positions[i].height = mesh + ? (batch[i].groundHeight ?? + Math.max(0, Number.isFinite(sampled) ? sampled : 0)) + : Number.isFinite(sampled) + ? sampled + : batch[i].groundHeight || 0; + } + const points = positions.map((c) => + Cesium.Cartesian3.fromRadians(c.longitude, c.latitude, c.height), + ); + const height = viewer.camera?.positionCartographic?.height; + if (mesh && Number.isFinite(height) && height < 250_000) { + const rectangle = viewer.camera.computeViewRectangle?.(); + const indices = []; + for (let i = 0; i < positions.length; i++) + if (rectangle && Cesium.Rectangle.contains(rectangle, positions[i])) + indices.push(i); + if (indices.length) { + const resolved = await boundedImport( + viewer.scene.clampToHeightMostDetailed( + indices.map((i) => { + const c = positions[i]; + return Cesium.Cartesian3.fromRadians( + c.longitude, + c.latitude, + Math.max(0, c.height), + ); + }), + ), + signal, + ); + for (let i = 0; i < indices.length; i++) + if (resolved[i]) points[indices[i]] = resolved[i]; + } + } + signal?.throwIfAborted(); + for (let i = 0; i < batch.length; i++) { + const item = batch[i]; + const c = points[i] && Cesium.Cartographic.fromCartesian(points[i]); + if (!c) + throw new GeoImportError( + 'Ground heights are not available for that file. Try again when the map has loaded.', + ); + item.groundHeight = c.height; + c.height += item.offset; + item.entity.position = + Math.abs(c.height - item.cartographic.height) < 1e-6 + ? item.original + : Cesium.Cartesian3.fromRadians(c.longitude, c.latitude, c.height); + } + if (at + batch.length < pins.length) + await new Promise((resolve) => setTimeout(resolve, 0)); + } +} + +async function boundedImport(promise, signal) { + let timer, abort; + try { + signal?.throwIfAborted(); + return await Promise.race([ + promise, + new Promise((_, reject) => { + abort = () => reject(signal.reason); + signal?.addEventListener('abort', abort, { once: true }); + timer = setTimeout( + () => reject(new GeoImportError('That file took too long to read.')), + 60_000, + ); + }), + ]); + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + } +} + +/** + * Load a ground footprint, retaining explicit KML altitude modes. Call mount after + * adding the source; revoke owns pin and blob cleanup. + * @param {File|Blob & {name?: string}} file + * @param {{viewer: Cesium.Viewer, color?: string}} options `color` is a CSS color. + * @returns {Promise<{dataSource: Cesium.DataSource, name: string, + * format: string, entityCount: number, removed: number, mount: Function, revoke: Function}>} + */ +export async function loadGeoFile( + file, + { viewer, color = '#39d0ff', signal } = {}, +) { + signal = signal + ? AbortSignal.any([signal, AbortSignal.timeout(60_000)]) + : AbortSignal.timeout(60_000); + signal?.throwIfAborted(); + if (!file || !Number.isFinite(file.size)) + throw new GeoImportError('Nothing to import.'); + if (file.size > MAX_IMPORT_BYTES) + throw new GeoImportError( + `That file is larger than ${MAX_IMPORT_BYTES / 1024 / 1024} MiB.`, + ); + let bytes = new Uint8Array(await boundedImport(file.arrayBuffer(), signal)); + const format = detectGeoFileFormat(file.name, bytes.subarray(0, 1024)); + if (!format) + throw new GeoImportError( + 'Only GeoJSON, KML, KMZ and GPX files can be imported.', + ); + const stroke = Cesium.Color.fromCssColorString(color); + const urls = []; + let pins; + const revoke = () => { + pins?.destroy(); + urls.splice(0).forEach((url) => URL.revokeObjectURL(url)); + }; + const name = importDisplayName(file.name); + let dataSource; + let removed = 0; + try { + if (format === 'geojson' || format === 'gpx') { + let geojson; + if (format === 'gpx') { + geojson = gpxToGeoJson(parseXml(decodeXml(bytes), 'GPX')); + } else { + geojson = parseGeoJsonText(new TextDecoder().decode(bytes)); + } + const size = countGeoJsonGeometry(geojson); + if (size.over) throw overLimitError(size); + if (!size.positions) + throw new GeoImportError('That file has nothing to draw.'); + prepareGeoJson(geojson); + bytes = null; + dataSource = await boundedImport( + Cesium.GeoJsonDataSource.load(geojson, { + clampToGround: true, + stroke, + fill: stroke.withAlpha(0.25), + strokeWidth: 3, + markerColor: stroke, + // No HTML description table is built from the file's properties. + describe: () => undefined, + }), + signal, + ); + } else { + let text; + let entries; + let main; + let resolveLocal = () => null; + if (format === 'kmz') { + entries = await readZipEntries(bytes); + main = kmzMainDocument(entries); + if (!main) throw new GeoImportError('That KMZ holds no KML document.'); + text = decodeXml(entries.get(main)); + resolveLocal = kmzImageResolver(entries, main, (url) => urls.push(url)); + } else { + text = decodeXml(bytes); + } + const doc = parseXml(text, format.toUpperCase()); + text = null; + bytes = null; + // Bound geometry before transformations, then count the styles and + // image links that survive sanitizing and sharing repeated styles. + const size = countKmlGeometry(doc, { maxLinks: Infinity }); + if (size.over) throw overLimitError(size); + checkLocalImages(doc, entries, main); + removed = sanitizeKmlDocument(doc, resolveLocal).removed; + entries?.clear(); + deduplicateKmlStyles(doc); + const normalizedSize = countKmlGeometry(doc); + if (normalizedSize.over) throw overLimitError(normalizedSize); + dataSource = await loadKml(doc, viewer, signal); + } + } catch (error) { + revoke(); + if (error instanceof GeoImportError) throw error; + throw new GeoImportError( + `That ${format.toUpperCase()} file could not be read.`, + ); + } + const entityCount = dataSource.entities.values.length; + if (entityCount > MAX_IMPORT_ENTITIES) { + revoke(); + throw new GeoImportError( + `That file has ${entityCount.toLocaleString()} map objects; the limit is ${MAX_IMPORT_ENTITIES.toLocaleString()}.`, + ); + } + const now = Cesium.JulianDate.now(); + const drawable = dataSource.entities.values.some( + (e) => + e.billboard?.image?.getValue(now) || + e.point || + e.rectangle || + e.polyline?.positions?.getValue(now)?.length >= 2 || + e.wall?.positions?.getValue(now)?.length >= 2 || + e.polygon?.hierarchy?.getValue(now)?.positions?.length >= 3, + ); + if (!drawable) { + revoke(); + throw new GeoImportError('That file has nothing to draw.'); + } + scrubImportedEntities(dataSource); + try { + await settleGroundPins(dataSource, viewer, signal); + } catch (error) { + revoke(); + throw error instanceof GeoImportError + ? error + : new GeoImportError( + 'Ground heights could not be read. Try that file again.', + ); + } + // Keep the clock invariant explicit even though unused KML dates have + // already been stripped and all supported geometries are static. + detachImportedTime(dataSource); + try { + pins = createImportedPins(dataSource, viewer); + } catch { + revoke(); + throw new GeoImportError('That file has invalid point graphics.'); + } + // The display name travels beside the data source (the file name, not + // anything the document says about itself). + return { + dataSource, + name, + format, + entityCount, + removed, + revoke, + mount: () => pins.mount(), + refreshGround: (refreshSignal) => + settleGroundPins( + dataSource, + viewer, + refreshSignal + ? AbortSignal.any([refreshSignal, AbortSignal.timeout(15_000)]) + : AbortSignal.timeout(15_000), + ).then(() => pins.sync()), + }; +} diff --git a/src/data/geoFileImport.test.mjs b/src/data/geoFileImport.test.mjs new file mode 100644 index 000000000..af02ec222 --- /dev/null +++ b/src/data/geoFileImport.test.mjs @@ -0,0 +1,1173 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { deflateRawSync } from 'node:zlib'; +import * as Cesium from 'cesium'; +import { + GeoImportError, + loadGeoFile, + MAX_IMPORT_POSITIONS, + MAX_IMPORT_ENTITIES, + countGeoJsonGeometry, + countKmlGeometry, + deduplicateKmlStyles, + detachImportedTime, + gpxToGeoJson, + importFeatureName, + scrubImportedEntities, + sanitizeKmlDocument, + detectGeoFileFormat, + importDisplayName, + kmzImageResolver, + kmzImageType, + kmzMainDocument, + parseGeoJsonText, + readZipEntries, + resolveKmzPath, +} from './geoFileImport.js'; +import { createImportImageBudget, importImageSize } from './geoImportImages.js'; + +test('real loader grounds XYZ GeoJSON and strips loader-sensitive metadata before allocation', async () => { + const previous = Cesium.PinBuilder.prototype.fromColor; + Cesium.PinBuilder.prototype.fromColor = () => ({ + toDataURL: () => 'data:image/png;base64,local', + tagName: 'CANVAS', + }); + try { + const object = { + type: 'FeatureCollection', + crs: { type: 'name', properties: { name: 'unsupported' } }, + features: [ + { + type: 'Feature', + id: 'kartaview-owned', + properties: { name: 'Test', addProperty: 1, 'marker-symbol': 'rail' }, + geometry: { type: 'Point', coordinates: [1, 2, 999] }, + }, + { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [1, 2, 0], + [2, 2, 0], + [2, 3, 0], + [1, 2, 0], + ], + ], + }, + }, + ], + }; + const loaded = await loadGeoFile( + new File([JSON.stringify(object)], 'xyz.geojson'), + { + viewer: { + scene: { + frameState: { mode: Cesium.SceneMode.SCENE3D }, + primitives: new Cesium.PrimitiveCollection(), + globe: { getHeight: () => 1234 }, + }, + }, + }, + ); + const [point, polygon] = + loaded.dataSource.importedEntities || loaded.dataSource.entities.values; + assert.notEqual(point.id, 'kartaview-owned'); + assert.equal(point.name, 'Test'); + assert.equal(point.properties, undefined); + assert.equal( + point.billboard.heightReference.getValue(), + Cesium.HeightReference.NONE, + ); + assert.equal(point.billboard.disableDepthTestDistance.getValue(), Infinity); + assert.ok( + Math.abs( + Cesium.Cartographic.fromCartesian( + point.position.getValue(Cesium.JulianDate.now()), + ).height - 1234, + ) < 1e-6, + ); + assert.notEqual(polygon.polygon.perPositionHeight?.getValue(), true); + assert.equal(loaded.dataSource.clock, undefined); + assert.equal( + loaded.dataSource.entities.values.length, + 1, + 'only polygon geometry remains in the per-frame entity collection', + ); + assert.equal(loaded.dataSource.importedBillboards.length, 1); + const update = Cesium.BillboardCollection.prototype.update; + let passes = 0; + Cesium.BillboardCollection.prototype.update = () => passes++; + try { + const collection = loaded.dataSource.importedBillboards; + collection.update({ passes: { pick: true, offscreen: true } }); + assert.equal(passes, 0, 'height picks ignore imported markers'); + collection.update({ passes: { pick: true, offscreen: false } }); + collection.update({ passes: { render: true, offscreen: true } }); + assert.equal(passes, 2, 'ordinary picks and screenshots include markers'); + } finally { + Cesium.BillboardCollection.prototype.update = update; + } + + loaded.mount(); + loaded.dataSource.show = false; + assert.equal(loaded.dataSource.importedBillboards.show, false); + loaded.dataSource.show = true; + await loaded.refreshGround(); + assert.equal( + point.billboard.heightReference.getValue(), + Cesium.HeightReference.NONE, + ); + loaded.revoke(); + assert.ok(loaded.dataSource.importedBillboards.isDestroyed()); + } finally { + Cesium.PinBuilder.prototype.fromColor = previous; + } +}); + +test('real loader refuses empty files and malformed nested geometry before allocating entities', async () => { + for (const object of [ + { type: 'Feature', geometry: null }, + { + type: 'FeatureCollection', + features: [{ geometry: { type: 'Point', coordinates: [1, 2] } }], + }, + { type: 'Feature', geometry: { type: 'Topology', arcs: [] } }, + ...[ + [1, '2'], + [Infinity, 0], + [0, 91], + [0, 0, 1e100], + ].map((coordinates) => ({ type: 'Point', coordinates })), + ]) + await assert.rejects( + loadGeoFile(new File([JSON.stringify(object)], 'bad.geojson'), { + viewer: {}, + }), + GeoImportError, + ); +}); + +test('KML tracks retain their explicit altitude and ground lines are tessellated', () => { + const track = xmlEl('Track', {}, [ + xmlEl('altitudeMode', {}, 'absolute'), + xmlEl('coord', {}, '1 2 10000'), + xmlEl('coord', {}, '2 3 10000'), + ]); + const line = xmlEl('LineString', {}, [xmlEl('coordinates', {}, '1,2 2,3')]); + const ground = xmlEl('Track', {}, [ + xmlEl('coord', {}, '1 2 0'), + xmlEl('coord', {}, '2 3 0'), + ]); + const root = xmlEl('kml', {}, [track, line, ground]); + sanitizeKmlDocument(xmlDoc(root)); + assert.equal( + root.childNodes[0].childNodes.find((n) => n.localName === 'altitudeMode') + .textContent, + 'absolute', + ); + assert.equal( + root.childNodes[1].childNodes.find((n) => n.localName === 'tessellate') + .textContent, + '1', + ); + assert.equal( + root.childNodes[2].childNodes.find((n) => n.localName === 'tessellate') + .textContent, + '1', + ); +}); + +test('sanitizer normalizes styles, removes incomplete pairs and keeps colored overlays', () => { + const local = xmlEl('styleUrl', {}, ' #s '); + const pair = xmlEl('Pair', {}, [ + xmlEl('key', {}, 'normal'), + xmlEl('styleUrl', {}, 'https://example.invalid/s'), + ]); + const overlay = xmlEl('GroundOverlay', {}, [xmlEl('color', {}, 'ff00ff00')]); + const hidden = xmlEl('visibility', {}, '0'); + const id = xmlEl('Placemark', { id: 'kartaview-123' }); + const root = xmlEl('kml', {}, [ + local, + xmlEl('StyleMap', { id: 's' }, [pair]), + overlay, + hidden, + id, + xmlEl('toString'), + ]); + sanitizeKmlDocument(xmlDoc(root)); + assert.equal(local.textContent, '#s'); + assert.equal(pair.parentNode, null); + assert.ok(root.childNodes.includes(overlay)); + assert.equal(hidden.parentNode, null); + assert.equal(id.getAttribute('id'), null); + assert.ok(!root.childNodes.some((n) => n.localName === 'toString')); +}); + +test('image budget rejects large decoded rasters before decoding and shares repeated images', () => { + const png = (w, h) => { + const b = new Uint8Array(24); + const v = new DataView(b.buffer); + v.setUint32(0, 0x89504e47); + b.set([73, 72, 68, 82], 12); + v.setUint32(16, w); + v.setUint32(20, h); + return b; + }; + assert.deepEqual(importImageSize(png(4096, 4096)), [4096, 4096]); + const charge = createImportImageBudget(); + charge('shared', png(4096, 4096)); + charge('shared', png(4096, 4096)); + assert.throws(() => charge('other', png(1, 1)), /limit/); + assert.throws( + () => createImportImageBudget()('huge', png(16384, 16384)), + /limit/, + ); +}); + +test('pending file reads can be cancelled without waiting for the loader timeout', async () => { + const controller = new AbortController(); + const file = { + name: 'waiting.geojson', + size: 1, + arrayBuffer: () => new Promise(() => {}), + }; + const promise = loadGeoFile(file, { viewer: {}, signal: controller.signal }); + controller.abort(new Error('cancelled by teardown')); + await assert.rejects(promise, /cancelled by teardown/); +}); + +test('a hidden coarse globe never seeds a mesh-clamp ray below the surface', async () => { + const previous = Cesium.PinBuilder.prototype.fromColor; + Cesium.PinBuilder.prototype.fromColor = () => ({ + tagName: 'CANVAS', + toDataURL: () => 'data:image/png;base64,local', + }); + const starts = []; + try { + for (const terrainProvider of [new Cesium.EllipsoidTerrainProvider(), {}]) { + const viewer = { + terrainProvider, + camera: { + positionCartographic: { height: 100000 }, + computeViewRectangle: () => Cesium.Rectangle.MAX_VALUE, + }, + scene: { + frameState: { mode: Cesium.SceneMode.SCENE3D }, + globe: { show: false, getHeight: () => -12311 }, + clampToHeightSupported: true, + clampToHeightMostDetailed: async (points) => + points.map((p) => { + const c = Cesium.Cartographic.fromCartesian(p); + starts.push(c.height); + return c.height < -9000 + ? undefined + : Cesium.Cartesian3.fromRadians(c.longitude, c.latitude, 2000); + }), + }, + }; + const loaded = await loadGeoFile( + new File( + [JSON.stringify({ type: 'Point', coordinates: [1, 2] })], + 'surface.geojson', + ), + { viewer }, + ); + assert.ok( + Math.abs(starts.at(-1)) < 1e-6, + `ray started at ${starts.at(-1)}`, + ); + assert.ok( + Math.abs( + Cesium.Cartographic.fromCartesian( + loaded.dataSource.importedBillboards.get(0).position, + ).height - 2000, + ) < 1e-6, + ); + const sampled = starts.length; + viewer.camera.computeViewRectangle = () => undefined; + await loaded.refreshGround(); + assert.equal( + starts.length, + sampled, + 'distant points do not start a detailed mesh sweep', + ); + assert.ok( + Math.abs( + Cesium.Cartographic.fromCartesian( + loaded.dataSource.importedBillboards.get(0).position, + ).height - 2000, + ) < 1e-6, + 'the last valid mesh height survives outside the camera footprint', + ); + const point = loaded.dataSource.importedEntities[0]; + const other = new Cesium.Entity({ id: 'unrelated-aircraft' }); + viewer.trackedEntity = starts.length === 1 ? other : point; + viewer.selectedEntity = point; + loaded.revoke(); + assert.equal(viewer.selectedEntity, undefined); + assert.equal( + viewer.trackedEntity, + starts.length === 1 ? other : undefined, + ); + } + } finally { + Cesium.PinBuilder.prototype.fromColor = previous; + } +}); + +const bytes = (text) => new TextEncoder().encode(text); + +/** A minimal zip writer: stored (0) or deflated (8) entries, optional flags. */ +function zip(files) { + const locals = []; + const centrals = []; + let offset = 0; + for (const { name, data, method = 8, flags = 0, size } of files) { + const nameBytes = bytes(name); + const body = method === 8 ? new Uint8Array(deflateRawSync(data)) : data; + const local = new Uint8Array(30 + nameBytes.length); + const lv = new DataView(local.buffer); + lv.setUint32(0, 0x04034b50, true); + lv.setUint16(6, flags, true); + lv.setUint16(8, method, true); + lv.setUint32(18, body.length, true); + lv.setUint32(22, size ?? data.length, true); + lv.setUint16(26, nameBytes.length, true); + local.set(nameBytes, 30); + const central = new Uint8Array(46 + nameBytes.length); + const cv = new DataView(central.buffer); + cv.setUint32(0, 0x02014b50, true); + cv.setUint16(8, flags, true); + cv.setUint16(10, method, true); + cv.setUint32(20, body.length, true); + cv.setUint32(24, size ?? data.length, true); + cv.setUint16(28, nameBytes.length, true); + cv.setUint32(42, offset, true); + central.set(nameBytes, 46); + locals.push(local, body); + centrals.push(central); + offset += local.length + body.length; + } + const dirSize = centrals.reduce((n, c) => n + c.length, 0); + const eocd = new Uint8Array(22); + const ev = new DataView(eocd.buffer); + ev.setUint32(0, 0x06054b50, true); + ev.setUint16(8, files.length, true); + ev.setUint16(10, files.length, true); + ev.setUint32(12, dirSize, true); + ev.setUint32(16, offset, true); + const parts = [...locals, ...centrals, eocd]; + const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0)); + let at = 0; + for (const p of parts) { + out.set(p, at); + at += p.length; + } + return out; +} + +test('format comes from the extension, then from the first bytes', () => { + assert.equal(detectGeoFileFormat('trail.GPX'), 'gpx'); + assert.equal(detectGeoFileFormat('farm.kml'), 'kml'); + assert.equal(detectGeoFileFormat('farm.kmz'), 'kmz'); + assert.equal(detectGeoFileFormat('parcels.geojson'), 'geojson'); + assert.equal(detectGeoFileFormat('parcels.json'), 'geojson'); + assert.equal( + detectGeoFileFormat('export', bytes(' {"type":"Feature"}')), + 'geojson', + ); + assert.equal( + detectGeoFileFormat( + 'export', + bytes(''), + ), + 'kml', + ); + assert.equal( + detectGeoFileFormat( + 'export', + bytes('\n'), + ), + 'gpx', + ); + assert.equal( + detectGeoFileFormat('export', new Uint8Array([0x50, 0x4b, 0x03, 0x04, 0])), + 'kmz', + ); + assert.equal(detectGeoFileFormat('notes.txt', bytes('hello')), null); + assert.equal( + detectGeoFileFormat('image.png', new Uint8Array([0x89, 0x50, 0x4e, 0x47])), + null, + ); +}); + +test('display names are printable and bounded', () => { + assert.equal(importDisplayName(' hike\u0000.gpx '), 'hike.gpx'); + assert.equal(importDisplayName(''), 'Imported file'); + const long = importDisplayName(`${'x'.repeat(80)}.kml`); + assert.equal(long.length, 58); + assert.ok(long.endsWith('…')); +}); + +test('GeoJSON text must be a GeoJSON object', () => { + assert.equal( + parseGeoJsonText('{"type":"FeatureCollection","features":[]}').type, + 'FeatureCollection', + ); + assert.equal( + parseGeoJsonText('{"type":"Point","coordinates":[0,0]}').type, + 'Point', + ); + assert.throws(() => parseGeoJsonText('{nope'), GeoImportError); + assert.throws(() => parseGeoJsonText('{"name":"not geo"}'), /not GeoJSON/); + assert.throws(() => parseGeoJsonText('[1,2]'), /not GeoJSON/); + assert.throws( + () => parseGeoJsonText('{"type":"FeatureCollection"}'), + /no features list/, + ); +}); + +test('a KMZ is read with stored and deflated entries, directories skipped', async () => { + const kml = bytes(''); + const png = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 1, 2, 3]); + const archive = zip([ + { name: 'files/', data: new Uint8Array(0), method: 0 }, + { name: 'doc.kml', data: kml }, + { name: 'files/icon.png', data: png, method: 0 }, + ]); + const entries = await readZipEntries(archive); + assert.deepEqual([...entries.keys()], ['doc.kml', 'files/icon.png']); + assert.deepEqual(entries.get('doc.kml'), kml); + assert.deepEqual(entries.get('files/icon.png'), png); +}); + +test('a KMZ that inflates past the limit is refused', async () => { + const archive = zip([{ name: 'doc.kml', data: new Uint8Array(200_000) }]); + await assert.rejects( + readZipEntries(archive, { maxInflatedBytes: 100_000 }), + /more than the import limit/, + ); +}); + +test('encrypted, damaged and non-zip archives are refused with a reason', async () => { + await assert.rejects( + readZipEntries( + zip([{ name: 'doc.kml', data: bytes(''), flags: 1, method: 0 }]), + ), + /encrypted/, + ); + await assert.rejects( + readZipEntries(bytes('not a zip at all, just text')), + /not a readable zip/, + ); + const truncated = zip([ + { name: 'doc.kml', data: bytes(''), method: 0 }, + ]); + const view = new DataView(truncated.buffer); + view.setUint32(truncated.length - 6, 9999, true); // directory offset past the end + await assert.rejects(readZipEntries(truncated), GeoImportError); + await assert.rejects( + readZipEntries( + zip([{ name: 'doc.kml', data: bytes(''), method: 12 }]), + ), + /compression method/, + ); +}); + +test('the main KML is doc.kml, else the shallowest .kml', () => { + const e = (...names) => new Map(names.map((n) => [n, new Uint8Array(0)])); + assert.equal(kmzMainDocument(e('a/b.kml', 'doc.kml')), 'doc.kml'); + assert.equal( + kmzMainDocument(e('deep/x/z.kml', 'top.kml', 'images/a.png')), + 'top.kml', + ); + assert.equal(kmzMainDocument(e('images/a.png')), null); +}); + +test('KMZ hrefs resolve only to entries inside the archive', () => { + const entries = new Map([ + ['doc.kml', new Uint8Array(0)], + ['files/icon one.png', new Uint8Array(0)], + ['overlay.jpg', new Uint8Array(0)], + ]); + assert.equal( + resolveKmzPath('files/icon%20one.png', 'doc.kml', entries), + 'files/icon one.png', + ); + assert.equal( + resolveKmzPath('./overlay.jpg', 'doc.kml', entries), + 'overlay.jpg', + ); + assert.equal( + resolveKmzPath('../overlay.jpg', 'files/sub.kml', entries), + 'overlay.jpg', + ); + assert.equal( + resolveKmzPath('https://example.com/overlay.jpg', 'doc.kml', entries), + null, + ); + assert.equal( + resolveKmzPath('//example.com/overlay.jpg', 'doc.kml', entries), + null, + ); + assert.equal(resolveKmzPath('file:///etc/passwd', 'doc.kml', entries), null); + assert.equal(resolveKmzPath('missing.png', 'doc.kml', entries), null); + assert.equal(resolveKmzPath('%E0%A4%A', 'doc.kml', entries), null); + assert.equal(kmzImageType('files/a.PNG'), 'image/png'); + assert.equal(kmzImageType('files/a.svg'), null); +}); + +/* ── a tiny element tree with the DOM surface the readers use ─────────── */ +function xmlEl(localName, attrs = {}, kids = []) { + const el = { + nodeType: 1, + localName, + namespaceURI: 'http://www.opengis.net/kml/2.2', + childNodes: [], + getAttribute: (n) => (n in attrs ? String(attrs[n]) : null), + get attributes() { + return Object.entries(attrs); + }, + setAttribute: (n, value) => { + attrs[n] = value; + }, + removeAttribute: (n) => { + delete attrs[n]; + }, + cloneNode() { + return xmlEl( + localName, + { ...attrs }, + text ?? el.childNodes.map((child) => child.cloneNode(true)), + ); + }, + getElementsByTagName(name) { + return xmlDoc(this).getElementsByTagName(name).slice(1); + }, + appendChild(child) { + child.parentNode?.removeChild(child); + child.parentNode = this; + this.childNodes.push(child); + }, + replaceChild(next, previous) { + const index = this.childNodes.indexOf(previous); + assert.ok(index >= 0); + next.parentNode?.removeChild(next); + this.childNodes[index] = next; + next.parentNode = this; + previous.parentNode = null; + }, + removeChild(child) { + this.childNodes.splice(this.childNodes.indexOf(child), 1); + child.parentNode = null; + }, + }; + for (const k of typeof kids === 'string' ? [] : kids) { + k.parentNode = el; + el.childNodes.push(k); + } + let text = typeof kids === 'string' ? kids : null; + Object.defineProperty(el, 'firstChild', { + get: () => el.childNodes[0] || null, + }); + Object.defineProperty(el, 'textContent', { + get: () => text ?? el.childNodes.map((c) => c.textContent).join(''), + set: (value) => { + text = value; + el.childNodes = []; + }, + }); + return el; +} +const xmlDoc = (root) => ({ + documentElement: root, + createElementNS: (_ns, name) => xmlEl(name), + getElementsByTagName(name) { + assert.equal(name, '*'); + const out = []; + const walk = (el) => { + out.push(el); + el.childNodes.forEach(walk); + }; + walk(root); + return out; + }, +}); +const serializeStyle = (el) => + JSON.stringify({ + name: el.localName, + attributes: el.attributes, + text: el.childNodes.length ? null : el.textContent, + children: el.childNodes.map(serializeStyle), + }); + +test('repeated inline styles are shared without replacing their inherited styleUrl', () => { + const style = () => + xmlEl('Style', {}, [xmlEl('IconStyle', {}, [xmlEl('scale', {}, '0.6')])]); + const placemarks = Array.from({ length: 2100 }, () => + xmlEl('Placemark', {}, [xmlEl('styleUrl', {}, '#base'), style()]), + ); + const doc = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('Document', {}, [ + xmlEl('Style', { id: 'gev-import-style-1' }, [xmlEl('LineStyle')]), + ...placemarks, + ]), + ]), + ); + assert.equal(countKmlGeometry(doc).over, 'links'); + assert.equal(deduplicateKmlStyles(doc, { serializeStyle }), 2100); + assert.equal(countKmlGeometry(doc).over, ''); + const styles = doc + .getElementsByTagName('*') + .filter((el) => el.localName === 'Style'); + assert.equal(styles.length, 2); + assert.equal(styles[1].getAttribute('id'), 'gev-import-style-2'); + for (const placemark of placemarks) { + assert.equal(placemark.childNodes[0].textContent, '#base'); + assert.equal(placemark.childNodes[1].localName, 'StyleMap'); + assert.deepEqual( + placemark.childNodes[1].childNodes.map((pair) => + pair.childNodes.map((el) => el.textContent), + ), + [ + ['normal', '#gev-import-style-2'], + ['highlight', '#gev-import-style-2'], + ], + ); + } +}); + +test('identical named styles redirect existing references and Pair styles remain valid', () => { + const style = (id) => + xmlEl('Style', id ? { id } : {}, [ + xmlEl('LineStyle', {}, [xmlEl('width', {}, '3')]), + ]); + const reference = xmlEl('styleUrl', {}, '#b'); + const pair = xmlEl('Pair', {}, [xmlEl('key', {}, 'normal'), style()]); + const doc = xmlDoc( + xmlEl('kml', {}, [ + style('a'), + style('b'), + xmlEl('StyleMap', { id: 'map' }, [pair]), + xmlEl('Placemark', {}, [reference]), + ]), + ); + assert.equal(deduplicateKmlStyles(doc, { serializeStyle }), 3); + assert.equal(doc.documentElement.firstChild.localName, 'Document'); + assert.equal(reference.textContent, '#gev-import-style-1'); + assert.equal(pair.childNodes[1].localName, 'styleUrl'); + assert.equal(pair.childNodes[1].textContent, '#gev-import-style-1'); + assert.equal( + doc.getElementsByTagName('*').filter((el) => el.localName === 'Style') + .length, + 1, + ); +}); + +test('random colors and ambiguous style IDs retain independent evaluations and precedence', () => { + const random = () => + xmlEl('Style', {}, [ + xmlEl('IconStyle', {}, [xmlEl('colorMode', {}, 'random')]), + ]); + const named = (width) => + xmlEl('Style', { id: 'collision' }, [ + xmlEl('LineStyle', {}, [xmlEl('width', {}, width)]), + ]); + const original = [random(), random(), named('1'), named('1'), named('2')]; + const doc = xmlDoc(xmlEl('kml', {}, original)); + assert.equal(deduplicateKmlStyles(doc, { serializeStyle }), 0); + assert.deepEqual(doc.documentElement.childNodes, original); +}); + +test('distinct style definitions and named StyleMaps still have a pre-loader budget', () => { + const doc = xmlDoc( + xmlEl( + 'kml', + {}, + ['1', '2', '3'].map((width) => + xmlEl('Style', {}, [ + xmlEl('LineStyle', {}, [xmlEl('width', {}, width)]), + ]), + ), + ), + ); + assert.throws( + () => deduplicateKmlStyles(doc, { serializeStyle, maxStyles: 2 }), + GeoImportError, + ); + const maps = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('StyleMap', { id: 'a' }), + xmlEl('StyleMap', { id: 'b' }), + ]), + ); + assert.equal(countKmlGeometry(maps, { maxLinks: 1 }).over, 'links'); +}); + +test('KML extended metadata is removed before the document reaches the loader', () => { + const point = xmlEl('Point', {}, [xmlEl('coordinates', {}, '1,2,0')]); + const metadata = xmlEl('ExtendedData', {}, [ + xmlEl('Data', { name: 'note' }, [xmlEl('value', {}, 'ordinary metadata')]), + ]); + const placemark = xmlEl('Placemark', {}, [metadata, point]); + const doc = xmlDoc(xmlEl('kml', {}, [placemark])); + assert.equal(sanitizeKmlDocument(doc).removed, 1); + assert.deepEqual(placemark.childNodes, [point]); + assert.equal(metadata.parentNode, null); +}); +test('blocked icon sources fall back to local pins without changing intentional empty icons', () => { + const blocked = xmlEl('Icon', {}, [ + xmlEl('href', {}, 'https://example.invalid/icon.png'), + ]); + const blank = xmlEl('Icon'); + const embedded = xmlEl('Icon', {}, [ + xmlEl('href', {}, 'data:image/png;base64,AAAA'), + ]); + const packed = xmlEl('Icon', {}, [xmlEl('href', {}, 'packed.png')]); + const iconStyle = xmlEl('IconStyle', {}, [ + xmlEl('scale', {}, '0.3'), + blocked, + ]); + const doc = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('Style', {}, [iconStyle]), + ...[blank, embedded, packed].map((icon) => + xmlEl('Style', {}, [xmlEl('IconStyle', {}, [icon])]), + ), + ]), + ); + assert.equal( + sanitizeKmlDocument(doc, (href) => + href === 'packed.png' ? 'blob:local-image' : null, + ).removed, + 1, + ); + assert.equal(blocked.parentNode, null); + assert.equal(iconStyle.childNodes[0].textContent, '0.3'); + assert.equal(blank.parentNode.localName, 'IconStyle'); + assert.equal( + embedded.childNodes[0].textContent, + 'data:image/png;base64,AAAA', + ); + assert.equal(packed.childNodes[0].textContent, 'blob:local-image'); +}); + +const trkpt = (lat, lon, time) => + xmlEl('trkpt', { lat, lon }, time ? [xmlEl('time', {}, time)] : []); + +test('GPX keeps each track segment as its own line, names as text, no times', () => { + const doc = xmlDoc( + xmlEl('gpx', {}, [ + xmlEl('wpt', { lat: 37.77, lon: -122.49 }, [xmlEl('name', {}, 'Camp')]), + xmlEl('wpt', { lat: 99, lon: 0 }), // out of range: skipped + xmlEl('rte', {}, [ + xmlEl('name', {}, 'Route'), + xmlEl('rtept', { lat: 1, lon: 2 }), + xmlEl('rtept', { lat: 3, lon: 4 }), + ]), + xmlEl('trk', {}, [ + xmlEl('name', {}, 'Morning walk'), + xmlEl('trkseg', {}, [ + trkpt(10, 20, '2024-05-01T10:00:00Z'), + trkpt(10.1, 20.1, '2024-05-01T10:01:00Z'), + ]), + // A pause in the recording: a new segment, not a straight jump. + xmlEl('trkseg', {}, [ + trkpt(11, 21, '2024-05-01T11:00:00Z'), + trkpt(11.1, 21.1, '2024-05-01T11:01:00Z'), + trkpt(11.2, 21.2, '2024-05-01T11:02:00Z'), + ]), + xmlEl('trkseg', {}, [trkpt(12, 22)]), // one point: not a line + ]), + ]), + ); + const fc = gpxToGeoJson(doc); + assert.deepEqual( + fc.features.map((f) => [f.properties.name, f.geometry.type]), + [ + ['Camp', 'Point'], + ['Route', 'LineString'], + ['Morning walk', 'MultiLineString'], + ], + ); + assert.deepEqual(fc.features[2].geometry.coordinates, [ + [ + [20, 10], + [20.1, 10.1], + ], + [ + [21, 11], + [21.1, 11.1], + [21.2, 11.2], + ], + ]); + assert.doesNotMatch(JSON.stringify(fc), /2024|time/); + assert.throws(() => gpxToGeoJson(xmlDoc(xmlEl('kml'))), GeoImportError); +}); + +test('GPX past the point limit is refused while reading, before Cesium', () => { + const seg = xmlEl( + 'trkseg', + {}, + Array.from({ length: 30 }, (_, i) => trkpt(1 + i / 100, 2)), + ); + const doc = xmlDoc(xmlEl('gpx', {}, [xmlEl('trk', {}, [seg])])); + assert.throws( + () => gpxToGeoJson(doc, { maxPositions: 20 }), + (e) => e instanceof GeoImportError && /more than .* points/.test(e.message), + ); + assert.throws( + () => + gpxToGeoJson( + xmlDoc( + xmlEl('gpx', {}, [ + xmlEl('wpt', { lat: 1, lon: 1 }), + xmlEl('wpt', { lat: 2, lon: 2 }), + ]), + ), + { maxFeatures: 1 }, + ), + /more than .* features/, + ); +}); + +test('GeoJSON features and vertices are counted before loading, without recursion', () => { + const line = Array.from({ length: 50 }, (_, i) => [i / 10, 1]); + const fc = { + type: 'FeatureCollection', + features: [ + { type: 'Feature', geometry: { type: 'LineString', coordinates: line } }, + { + type: 'Feature', + geometry: { + type: 'GeometryCollection', + geometries: [ + { type: 'Point', coordinates: [1, 2] }, + { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 0], + ], + ], + }, + ], + }, + }, + { type: 'Feature', geometry: null }, + ], + }; + assert.deepEqual(countGeoJsonGeometry(fc), { + features: 3, + positions: 55, + entities: 4, + over: '', + }); + assert.equal( + countGeoJsonGeometry(fc, { maxPositions: 54 }).over, + 'positions', + ); + assert.equal(countGeoJsonGeometry(fc, { maxFeatures: 2 }).over, 'features'); + assert.deepEqual( + countGeoJsonGeometry({ type: 'Point', coordinates: [1, 2] }), + { + features: 1, + positions: 1, + entities: 1, + over: '', + }, + ); + // A very long line is counted iteratively and stops at the limit. + const huge = { + type: 'LineString', + coordinates: Array.from({ length: MAX_IMPORT_POSITIONS + 5 }, () => [0, 0]), + }; + assert.equal(countGeoJsonGeometry(huge).over, 'positions'); +}); + +test('multipart GeoJSON is refused before loading when it expands past the entity cap', () => { + const parts = Array.from({ length: MAX_IMPORT_ENTITIES + 1 }, () => [1, 2]); + for (const geometry of [ + { type: 'MultiPoint', coordinates: parts }, + { + type: 'GeometryCollection', + geometries: parts.map((coordinates) => ({ type: 'Point', coordinates })), + }, + { type: 'MultiLineString', coordinates: parts.map((p) => [p, [2, 3]]) }, + ]) { + const size = countGeoJsonGeometry({ type: 'Feature', geometry }); + assert.equal(size.over, 'entities'); + } +}); + +test('GPX points with missing or blank coordinates are skipped rather than placed at zero', () => { + const doc = xmlDoc( + xmlEl('gpx', {}, [ + xmlEl('wpt', { lon: 2 }), + xmlEl('wpt', { lat: 1 }), + xmlEl('wpt', { lat: '', lon: 2 }), + xmlEl('wpt', { lat: 1, lon: ' ' }), + xmlEl('wpt', { lat: 0, lon: 0 }), + ]), + ); + assert.equal(gpxToGeoJson(doc).features.length, 1); + assert.deepEqual(gpxToGeoJson(doc).features[0].geometry.coordinates, [0, 0]); +}); + +test('KML placemarks and coordinate tuples are counted before loading', () => { + const coords = (n) => + xmlEl( + 'coordinates', + {}, + Array.from({ length: n }, (_, i) => `${i},1,0`).join('\n '), + ); + const doc = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('Placemark', {}, [xmlEl('LineString', {}, [coords(40)])]), + xmlEl('Placemark', {}, [ + xmlEl('Track', {}, [ + xmlEl('coord', {}, '1 2 0'), + xmlEl('coord', {}, '1 3 0'), + ]), + ]), + ]), + ); + assert.deepEqual(countKmlGeometry(doc), { + features: 2, + entities: 2, + positions: 42, + links: 0, + over: '', + }); + assert.equal(countKmlGeometry(doc, { maxPositions: 41 }).over, 'positions'); + assert.equal(countKmlGeometry(doc, { maxFeatures: 1 }).over, 'features'); +}); + +test('KML vertex counting recognizes the Unicode whitespace Cesium accepts', () => { + for (const separator of ['\u00a0', '\u2003', '\ufeff']) { + const doc = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('Placemark', {}, [ + xmlEl('LineString', {}, [ + xmlEl('coordinates', {}, ['1,2', '3,4', '5,6'].join(separator)), + ]), + ]), + ]), + ); + assert.equal(countKmlGeometry(doc).positions, 3); + assert.equal(countKmlGeometry(doc, { maxPositions: 2 }).over, 'positions'); + } +}); + +test('KML multipart geometries count every Cesium entity before loading', () => { + const doc = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('Placemark', {}, [ + xmlEl('MultiGeometry', {}, [ + xmlEl('Point'), + xmlEl('MultiGeometry', {}, [xmlEl('Point'), xmlEl('LineString')]), + xmlEl('MultiTrack', {}, [xmlEl('Track'), xmlEl('Track')]), + ]), + ]), + xmlEl('GroundOverlay'), + ]), + ); + assert.equal(countKmlGeometry(doc).entities, 9); + assert.equal(countKmlGeometry(doc, { maxEntities: 8 }).over, 'entities'); + const oversized = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('Placemark', {}, [ + xmlEl( + 'MultiGeometry', + {}, + Array.from({ length: MAX_IMPORT_ENTITIES }, () => xmlEl('Point')), + ), + ]), + ]), + ); + assert.equal(countKmlGeometry(oversized).over, 'entities'); +}); + +test('KML folders contribute to the pre-loader entity budget', () => { + const doc = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('Folder', {}, [xmlEl('Folder', {}, [xmlEl('Placemark')])]), + ]), + ); + assert.equal(countKmlGeometry(doc).entities, 3); + assert.equal(countKmlGeometry(doc, { maxEntities: 2 }).over, 'entities'); +}); + +test('KML style definitions and distinct icon links have their own limit', () => { + const style = () => + xmlEl('Style', {}, [ + xmlEl('IconStyle', {}, [xmlEl('Icon', {}, [xmlEl('href', {}, 'a.png')])]), + ]); + const doc = xmlDoc(xmlEl('kml', {}, [style(), style(), style()])); + assert.equal(countKmlGeometry(doc).links, 4); + assert.equal(countKmlGeometry(doc, { maxLinks: 3 }).over, 'links'); + const distinct = xmlDoc( + xmlEl( + 'kml', + {}, + ['a.png', 'b.png', 'c.png'].map((href) => xmlEl('href', {}, href)), + ), + ); + assert.equal(countKmlGeometry(distinct, { maxLinks: 2 }).over, 'links'); +}); + +test('a KMZ image gets one blob URL however many hrefs name it', () => { + const entries = new Map([ + ['doc.kml', new Uint8Array(0)], + ['a.png', new Uint8Array(8)], + ['b.png', new Uint8Array(8)], + ]); + const made = []; + const kept = []; + const resolve = kmzImageResolver( + entries, + 'doc.kml', + (url) => kept.push(url), + (blob) => { + made.push(blob); + return `blob:test/${made.length}`; + }, + ); + for (let i = 0; i < 1000; i++) assert.equal(resolve('a.png'), 'blob:test/1'); + assert.equal(resolve('./b.png'), 'blob:test/2'); + assert.equal(resolve('a.png'), 'blob:test/1'); + assert.equal(resolve('https://example.com/a.png'), null); + assert.equal(resolve('doc.kml'), null); + assert.equal(made.length, 2); + assert.deepEqual(kept, ['blob:test/1', 'blob:test/2']); +}); + +test('imported entities keep only a short plain-text name: no description, label or properties', () => { + const ds = new Cesium.CustomDataSource('t'); + const e = ds.entities.add({ + name: 'Trail\u0007 head\n' + 'x'.repeat(300), + description: '', + label: { text: 'Trail head' }, + properties: { note: '' }, + position: Cesium.Cartesian3.fromDegrees(1, 2), + }); + e.kml = { extendedData: { note: { value: '