From c5b1d41cbade33efd598286a104e49b9cd1d2952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Saggese?= Date: Mon, 21 Sep 2026 11:30:49 +0200 Subject: [PATCH 001/202] fix(search): geocode through the server, not the browser - Google's Geocoding web service refuses referrer-restricted keys, so the browser call forced GOOGLE_MAPS_API_KEY to ship unrestricted - add /api/google/geocode and /api/google/reverse-geocode, using the server key like Places and Street View already do - the browser key now needs the Map Tiles API only --- CHANGELOG.md | 9 ++ SECURITY.md | 6 +- docs/CURRENT-STATE.md | 19 ++- server/providers/local.js | 3 + server/providers/places.js | 7 +- server/providers/places/google-key.js | 33 +++- server/providers/places/google.js | 155 ++++++++++++++++++ src/annotations/annotationEngine.test.mjs | 2 +- src/annotations/annotationResolver.test.mjs | 20 +-- src/googleGeocodeProxy.test.mjs | 167 ++++++++++++++++++++ src/googleServerKey.test.mjs | 16 +- src/search/defaults.js | 23 +-- src/search/http.js | 15 +- src/search/placeSearch.test.mjs | 2 +- src/voice/gevActions.test.mjs | 6 +- 15 files changed, 436 insertions(+), 47 deletions(-) create mode 100644 src/googleGeocodeProxy.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 744f4be0a..526a89bc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +- Move Google geocoding behind the local server. Forward and reverse lookups now + go through `/api/google/geocode` and `/api/google/reverse-geocode` with the + server key, joining Places and Street View. Google's Geocoding web service + refuses referrer-restricted keys, so the previous browser call left + `SECURITY.md`'s advice and working search mutually exclusive: the bundled + browser key had to be unrestricted for search to work at all. That key now + needs the Map Tiles API only. Keyless sessions and the Photon/Nominatim + fallback order are unchanged. + - Enable responsive trackpad pinch zoom on the globe. Browser pixel-mode `Ctrl+wheel` pinch gestures now reach Cesium with bounded amplification, while ordinary wheel, line-mode and touch-pinch inputs retain their existing diff --git a/SECURITY.md b/SECURITY.md index a21ea99a4..661e8764f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -20,18 +20,18 @@ The golden rule: **secret-bearing API keys stay on the server side.** The dev/pr | `OPENAI_API_KEY` | Server only | Browser fetches a short-lived **ephemeral** Realtime session token from `/api/realtime/token`; the real key never ships | | `AISSTREAM_API_KEY` | Server only | Server holds the AISStream websocket; browser polls the same-origin `/api/ais-live` cache | | OpenSky OAuth (`OPENSKY_CLIENT_ID/SECRET`) | Server only | Server mints + refreshes the token behind `/api/opensky` | -| `GOOGLE_MAPS_SERVER_API_KEY` (optional, #33) | Server only | Server calls Places (`/api/google/nearby-places`, `/api/google/text-search`) and the Street View fallback with this key; falls back to `GOOGLE_MAPS_API_KEY` when unset | +| `GOOGLE_MAPS_SERVER_API_KEY` (optional, #33) | Server only | Server calls Places (`/api/google/nearby-places`, `/api/google/text-search`), Geocoding (`/api/google/geocode`, `/api/google/reverse-geocode`) and the Street View fallback with this key; falls back to `GOOGLE_MAPS_API_KEY` when unset | ### Two deliberately client-side keys — restrict them These are designed to be used directly in the browser (like a Mapbox public token). They are injected into the client bundle via Vite's `define`, so they **will** be visible in browser devtools. Scope and restrict them rather than trying to hide them: -1. **Google Maps API key** — loads Photorealistic 3D Tiles directly and powers GEV place search. **Restrict it** (HTTP referrer + API restriction to the required Google APIs) in the Google Cloud Console. An unrestricted key in a public deployment can be abused and billed to you. +1. **Google Maps API key** — loads Photorealistic 3D Tiles directly. **Restrict it** (HTTP referrer + API restriction to the required Google APIs) in the Google Cloud Console. An unrestricted key in a public deployment can be abused and billed to you. Restrict it to the **Map Tiles API only**: a referrer restriction applies to the Maps JavaScript API and the client SDKs, not to the web services, so a key carrying Geocoding under one is refused with `API keys with referer restrictions cannot be used with this API`. Search does not need it — geocoding goes through the server proxies above (#363). 2. **Cesium ion token** (`CESIUM_ION_TOKEN`, optional — for ion-hosted Google Photorealistic 3D Tiles, Bing world imagery, and world terrain) — used as `Cesium.Ion.defaultAccessToken` client-side. Use a public **`assets:read`** token with **URL restrictions** for any hosted deployment. The Community plan has eligibility and usage limits; a public token is not a secret, but it can still consume the account's quota. > The explicit browser `define` block in `build/vite.js` controls exactly what reaches the client: only these two keys. Everything else stays server-side. -**Places and Street View never needed to be on that list** (#33): they're called from the server-side proxies in the table above, which use `GOOGLE_MAPS_SERVER_API_KEY` when it's set. Splitting it from the browser-exposed key lets each key's Google Cloud restriction actually match what it does — the browser key referrer-restricted to the APIs the client loads, the server key IP-restricted (never a referrer, since it never leaves your server) to Places + Street View Static — instead of one key that has to be either over-permissioned or broken for one of its two jobs. A single shared `GOOGLE_MAPS_API_KEY` still works if you don't split them; it just has to cover every API both sides use. +**Places, Geocoding and Street View never needed to be on that list** (#33, #363): they're called from the server-side proxies in the table above, which use `GOOGLE_MAPS_SERVER_API_KEY` when it's set. Splitting it from the browser-exposed key lets each key's Google Cloud restriction actually match what it does — the browser key referrer-restricted to Map Tiles, the server key IP-restricted (never a referrer, since it never leaves your server) to Places + Geocoding + Street View Static — instead of one key that has to be either over-permissioned or broken for one of its jobs. A single shared `GOOGLE_MAPS_API_KEY` still works if you don't split them, but then it cannot carry a referrer restriction at all: the web services refuse one, and search would be the half that breaks. Never commit real keys. `.env` is gitignored; only `.env.example` (placeholder names) is tracked. On macOS `dev-fresh.sh` can read keys from the Keychain; plain Vite uses env vars or a local `.env`, and Pinokio uses its ignored app `ENVIRONMENT` file. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9c381ac0c..e038bfe20 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3445,7 +3445,24 @@ Providers normalize coordinates, canonical name, label, place types and optional bounds. Camera framing, nearby landmark recovery and footprint selection remain in their consumers, including the Capitol identity/containment safeguards. Radio keeps localized country names in labels rather than station filters. -Reverse geocoding and nearby/text-search endpoints retain their existing behavior. +Nearby/text-search endpoints retain their existing behavior. + +**Google geocoding is a server route, not a browser call** (#363). Forward and +reverse lookups go to `/api/google/geocode` and `/api/google/reverse-geocode`, +which carry `GOOGLE_MAPS_SERVER_API_KEY` (falling back to `GOOGLE_MAPS_API_KEY`) +and share the Places routes' opt-in `GEV_RATELIMIT_GOOGLE_PER_MIN` throttle. +Google's Geocoding **web service** refuses referrer-restricted keys, so the +previous browser call forced the bundled browser key — visible in devtools by +design — to be left unrestricted; it now needs the Map Tiles API only. Both +routes answer in Google's own `status`/`results` shape rather than the Places +routes' `places: []` contract, because the browser adapters normalize that shape +directly. The browser key still gates the provider client-side, so a keyless +session spends no request to learn it has no Google: the route's own keyless +answer reads as an unanswered provider, which would stop the chain caching the +negative Photon or Nominatim then produces. Its query is capped at 200 +characters, its reverse coordinates are validated against WGS84 bounds before +the request is paid for, and a malformed viewport bias is dropped rather than +refused, since a bias only ranks results. Only valid answers and definitive misses enter bounded caches; malformed replies, HTTP refusals and outages remain retryable. Searches share a 12-second total diff --git a/server/providers/local.js b/server/providers/local.js index 68f61a3cb..07e1a5bb8 100644 --- a/server/providers/local.js +++ b/server/providers/local.js @@ -66,6 +66,9 @@ export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js'; export { googlePlacesContextProxy } from './places.js'; export { googleServerApiKey } from './places.js'; export { keylessGooglePlacesResponse } from './places.js'; +export { keylessGoogleGeocodeResponse } from './places.js'; +export { validateGeocodeAddress } from './places.js'; +export { geocodeBounds } from './places.js'; export { adsbLolFallbackAnchor } from './aircraft/opensky.js'; export { readResponseTextCapped } from './common/http.js'; export { readResponseJsonCapped } from './common/http.js'; diff --git a/server/providers/places.js b/server/providers/places.js index 648be2aa8..6c36627e0 100644 --- a/server/providers/places.js +++ b/server/providers/places.js @@ -1,8 +1,13 @@ /** Place-search and route middleware for Node development servers. */ -export { googlePlacesContextProxy } from './places/google.js'; +export { + googlePlacesContextProxy, + validateGeocodeAddress, + geocodeBounds, +} from './places/google.js'; export { googleServerApiKey, keylessGooglePlacesResponse, + keylessGoogleGeocodeResponse, } from './places/google-key.js'; export { installRouteMiddleware } from './places/routes.js'; export { diff --git a/server/providers/places/google-key.js b/server/providers/places/google-key.js index 353bc9e24..a5c54b302 100644 --- a/server/providers/places/google-key.js +++ b/server/providers/places/google-key.js @@ -14,12 +14,33 @@ export function keylessGooglePlacesResponse(apiKey) { } /** - * Google API key for the SERVER-SIDE calls (Places nearby/text search, the - * CCTV Street View fallback). These never reach the browser, so this key can - * be restricted by server IP and scoped to Places API + Street View Static - * API — while GOOGLE_MAPS_API_KEY stays referrer-restricted to Map Tiles + - * Geocoding for the browser (#33). Splitting them is opt-in: unset, this - * falls back to the shared browser key and nothing changes. + * Geocoding speaks Google's own response shape rather than the `places: []` + * contract its two siblings use, because the browser adapters read `status` + * and `results` directly. The keyless capability reads the same way: a status + * that is not `OK` and no results, which the adapters already treat as "this + * provider did not answer" and pass to the next geocoder in the chain. + */ +export function keylessGoogleGeocodeResponse(apiKey) { + if (String(apiKey ?? '').trim()) return null; + return { + statusCode: 200, + payload: { + configured: false, + status: 'REQUEST_DENIED', + results: [], + error: null, + }, + }; +} + +/** + * Google API key for the SERVER-SIDE calls (Places nearby/text search, + * geocoding, the CCTV Street View fallback). These never reach the browser, so + * this key can be restricted by server IP and scoped to Places API + Geocoding + * API + Street View Static API — while GOOGLE_MAPS_API_KEY stays + * referrer-restricted to Map Tiles for the browser (#33, #363). Splitting them + * is opt-in: unset, this falls back to the shared browser key, which then has + * to carry every API both sides use and cannot take a referrer restriction. */ export function googleServerApiKey() { return resolveGoogleServerKey(process.env); diff --git a/server/providers/places/google.js b/server/providers/places/google.js index 1ee83a892..8895bc19d 100644 --- a/server/providers/places/google.js +++ b/server/providers/places/google.js @@ -1,6 +1,7 @@ import { googleServerApiKey, keylessGooglePlacesResponse, + keylessGoogleGeocodeResponse, } from './google-key.js'; import { makeOptInRateLimiter, clientKey } from '../common/rate-limit.js'; import { @@ -42,6 +43,37 @@ export function validatePlacesCoordinates(searchParams) { return { ok: true, latitude, longitude }; } +/** Longest accepted geocoder query. Real place names are far shorter. */ +const GEOCODE_MAX_QUERY = 200; + +/** Validate a free-text geocoder query before consuming request quota. */ +export function validateGeocodeAddress(searchParams) { + const address = String(searchParams.get('address') || '').trim(); + if (!address) return { ok: false, error: 'address is required' }; + if (address.length > GEOCODE_MAX_QUERY) + return { ok: false, error: 'address is too long' }; + return { ok: true, address }; +} + +/** + * Validate the optional viewport bias, which Google reads as + * `sw_lat,sw_lng|ne_lat,ne_lng`. An unparsed value is dropped rather than + * refused: the bias only ranks results, so a malformed one costs relevance, + * not an answer. + */ +export function geocodeBounds(searchParams) { + const raw = String(searchParams.get('bounds') || '').trim(); + if (!raw) return null; + const corners = raw.split('|'); + if (corners.length !== 2) return null; + const numbers = corners.flatMap((corner) => corner.split(',').map(Number)); + if (numbers.length !== 4 || !numbers.every(Number.isFinite)) return null; + const [swLat, swLon, neLat, neLon] = numbers; + if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null; + if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null; + return raw; +} + /** Nearby place labels and view-biased text search, with request-time key resolution. */ export function googlePlacesContextProxy({ resolveApiKey = googleServerApiKey, @@ -274,6 +306,129 @@ export function googlePlacesContextProxy({ ); } }); + + /** + * Geocoding belongs on the server, not in the page. Google's Geocoding web + * service refuses referrer-restricted keys, so calling it from the browser + * forces GOOGLE_MAPS_API_KEY — which ships in the bundle by design — to be + * left unrestricted (#363). Here the request carries the server key, and + * the browser key no longer needs the Geocoding API at all. + * + * Unlike the two Places routes above, these answer in Google's own shape + * (`status` + `results`) rather than the `places: []` contract: the browser + * normalizers read that shape directly, and keeping it means the proxy adds + * a hop without adding a translation nobody asked for. + */ + function installGeocodeRoute(path, readQuery) { + middlewares.use(path, async (req, res) => { + const refuse = (statusCode, error) => { + res.statusCode = statusCode; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ status: 'REQUEST_DENIED', results: [], error }), + ); + }; + + if (req.method !== 'GET') { + refuse(405, 'Method not allowed'); + return; + } + + // A keyless geocode has no provider cost, so it resolves before the + // paid-endpoint limiter can consume or exhaust quota (mirrors the + // Places routes above). + const apiKey = resolveApiKey(); + const keyless = keylessGoogleGeocodeResponse(apiKey); + if (keyless) { + res.statusCode = keyless.statusCode; + res.setHeader('Content-Type', 'application/json'); + res.setHeader('Cache-Control', 'no-store'); + res.end(JSON.stringify(keyless.payload)); + return; + } + + const requestUrl = new URL(req.url || '', 'http://localhost'); + const query = readQuery(requestUrl.searchParams); + if (!query.ok) { + refuse(400, query.error); + return; + } + + // Opt-in per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN), shared with + // the Places routes because both spend the same Google budget. + const _grl = googleRateLimiter(); + if (_grl && !_grl(clientKey(req))) { + res.statusCode = 429; + res.setHeader('Content-Type', 'application/json'); + res.setHeader('Retry-After', '5'); + res.end( + JSON.stringify({ + status: 'OVER_QUERY_LIMIT', + results: [], + error: 'Rate limit exceeded', + }), + ); + return; + } + + try { + const upstream = new URL( + endpoints.geocode || + 'https://maps.googleapis.com/maps/api/geocode/json', + ); + for (const [name, value] of Object.entries(query.params)) + upstream.searchParams.set(name, value); + upstream.searchParams.set('key', apiKey); + const response = await fetchImpl(upstream.toString(), { + redirect: 'error', + }); + const data = await response.json().catch(() => null); + + res.statusCode = response.ok ? 200 : response.status; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.setHeader('Cache-Control', 'private, max-age=300'); + res.end( + JSON.stringify( + data ?? { + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }, + ), + ); + } catch (error) { + res.statusCode = 502; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end( + JSON.stringify({ + status: 'UNKNOWN_ERROR', + results: [], + error: error?.message || 'Google Geocoding request failed', + }), + ); + } + }); + } + + installGeocodeRoute('/api/google/geocode', (searchParams) => { + const address = validateGeocodeAddress(searchParams); + if (!address.ok) return address; + const bounds = geocodeBounds(searchParams); + return { + ok: true, + params: { + address: address.address, + ...(bounds ? { bounds } : {}), + }, + }; + }); + + installGeocodeRoute('/api/google/reverse-geocode', (searchParams) => { + const coordinates = validatePlacesCoordinates(searchParams); + if (!coordinates.ok) return coordinates; + const { latitude, longitude } = coordinates; + return { ok: true, params: { latlng: `${latitude},${longitude}` } }; + }); } return { diff --git a/src/annotations/annotationEngine.test.mjs b/src/annotations/annotationEngine.test.mjs index 0d26974ae..cc43b612a 100644 --- a/src/annotations/annotationEngine.test.mjs +++ b/src/annotations/annotationEngine.test.mjs @@ -400,7 +400,7 @@ test('outline upgrade updates the rendered element in place without remove/add', }; let overpassCall = 0; globalThis.fetch = async (url) => { - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => ({ status: 'OK', results: [{ diff --git a/src/annotations/annotationResolver.test.mjs b/src/annotations/annotationResolver.test.mjs index 452d6997c..2c51213c1 100644 --- a/src/annotations/annotationResolver.test.mjs +++ b/src/annotations/annotationResolver.test.mjs @@ -200,7 +200,7 @@ test('ask-side admin bypass: "the Texas Capitol" recovers near-view despite a fa const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -235,7 +235,7 @@ test('ask-side admin bypass: explicit "state of Texas" skips recovery and proxim const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - assert.match(String(url), /^https:\/\/maps\.googleapis\.com\/maps\/api\/geocode/); + assert.match(String(url), /^\/api\/google\/geocode\?/); return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -273,7 +273,7 @@ for (const fixture of [ const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: fixture.lat, lon: fixture.lon, @@ -299,7 +299,7 @@ test('ask-side admin bypass: bare "Texas" remains on the guarded recovery path', const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -328,7 +328,7 @@ test('ask-side admin bypass: admin level 2/3 result types never grant a township const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { const query = new URL(String(url)).searchParams.get('address'); return { json: async () => geocodePayload({ lat: 39.7817, @@ -380,7 +380,7 @@ function installCapitolMocks(t, elements, components = [ { long_name: 'Capitol Hill', types: ['neighborhood', 'political'] }, ]) { installGoogleMocks(t, async (url) => { - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => ({ status: 'OK', results: [{ formatted_address: 'Washington, DC 20004, USA', types: ['establishment', 'landmark', 'point_of_interest', 'tourist_attraction'], @@ -458,7 +458,7 @@ test('keyless: a key that geocodes to nothing still anchors the annotation', asy const requests = []; installGoogleMocks(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (String(url).startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; return { ok: true, json: async () => ({ @@ -495,7 +495,7 @@ test('keyless: a key that geocodes to nothing still anchors the annotation', asy high: { latitude: 30.28, longitude: -97.68 }, }); // Google is asked first and exactly once; one Photon call answers it. - assert.equal(requests.filter((url) => url.includes('maps.googleapis.com')).length, 1); + assert.equal(requests.filter((url) => url.includes('/api/google/geocode')).length, 1); assert.equal(requests.filter((url) => url.includes('photon.komoot.io')).length, 1); }); @@ -512,7 +512,7 @@ test('keyless: OSM matching uses the feature\'s canonical name, not the user\'s installGoogleMocks(t, async (url, init) => { const href = String(url); - if (href.startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (href.startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; if (href.startsWith('/api/google/text-search')) return { ok: true, json: async () => ({ places: [] }) }; if (href.startsWith('https://photon.komoot.io/')) { return { @@ -575,7 +575,7 @@ test('keyless: a Photon outage is not remembered as "no such place"', async (t) const requests = []; installGoogleMocks(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (String(url).startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; if (!photonReachable) throw new Error('network down'); return photonUp; }); diff --git a/src/googleGeocodeProxy.test.mjs b/src/googleGeocodeProxy.test.mjs new file mode 100644 index 000000000..2518d02cb --- /dev/null +++ b/src/googleGeocodeProxy.test.mjs @@ -0,0 +1,167 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + googlePlacesContextProxy, + validateGeocodeAddress, + geocodeBounds, +} from '../vite.config.js'; + +const UPSTREAM = 'https://geocode.test/json'; + +function installRoutes({ apiKey = 'server-secret', fetchImpl } = {}) { + const routes = new Map(); + googlePlacesContextProxy({ + resolveApiKey: () => apiKey, + fetchImpl, + endpoints: { geocode: UPSTREAM }, + }).configureServer({ + middlewares: { use: (name, handler) => routes.set(name, handler) }, + }); + return routes; +} + +function invokeRoute(handler, { method = 'GET', url = '/' } = {}) { + return new Promise((resolve, reject) => { + const headers = new Map(); + const res = { + statusCode: 200, + setHeader(name, value) { + headers.set(String(name).toLowerCase(), String(value)); + }, + end(body = '') { + resolve({ + statusCode: this.statusCode, + headers: Object.fromEntries(headers), + body: body ? JSON.parse(String(body)) : null, + }); + }, + }; + Promise.resolve( + handler( + { method, url, headers: {}, socket: { remoteAddress: '127.0.0.1' } }, + res, + ), + ).catch(reject); + }); +} + +test('geocoder query and viewport bias are validated before Google is paid', () => { + assert.deepEqual(validateGeocodeAddress(new URLSearchParams()), { + ok: false, + error: 'address is required', + }); + assert.deepEqual(validateGeocodeAddress(new URLSearchParams({ address: ' ' })), { + ok: false, + error: 'address is required', + }); + assert.deepEqual( + validateGeocodeAddress(new URLSearchParams({ address: 'x'.repeat(201) })), + { ok: false, error: 'address is too long' }, + ); + assert.deepEqual( + validateGeocodeAddress(new URLSearchParams({ address: ' Austin ' })), + { ok: true, address: 'Austin' }, + ); + + // A bias only ranks results, so an unusable one is dropped, not refused. + const bias = '30.1,-97.9|30.4,-97.6'; + assert.equal(geocodeBounds(new URLSearchParams({ bounds: bias })), bias); + for (const bounds of [ + '', + '30.1,-97.9', + '30.1,-97.9|30.4', + 'north,-97.9|30.4,-97.6', + '91,-97.9|30.4,-97.6', + '30.1,-181|30.4,-97.6', + ]) + assert.equal(geocodeBounds(new URLSearchParams({ bounds })), null); +}); + +test('the geocode routes keep the key server-side and answer in Google shape', async () => { + const upstream = []; + const routes = installRoutes({ + fetchImpl: async (url) => { + upstream.push(new URL(String(url))); + return { + ok: true, + json: async () => ({ status: 'OK', results: [{ place_id: 'fixture' }] }), + }; + }, + }); + const forward = routes.get('/api/google/geocode'); + const reverse = routes.get('/api/google/reverse-geocode'); + assert.equal(typeof forward, 'function'); + assert.equal(typeof reverse, 'function'); + + const forwarded = await invokeRoute(forward, { + url: '/?address=Austin&bounds=30.1,-97.9|30.4,-97.6', + }); + assert.equal(forwarded.statusCode, 200); + assert.deepEqual(forwarded.body, { + status: 'OK', + results: [{ place_id: 'fixture' }], + }); + assert.equal(upstream.at(-1).searchParams.get('address'), 'Austin'); + assert.equal(upstream.at(-1).searchParams.get('bounds'), '30.1,-97.9|30.4,-97.6'); + assert.equal(upstream.at(-1).searchParams.get('key'), 'server-secret'); + + const reversed = await invokeRoute(reverse, { url: '/?lat=30.27&lon=-97.74' }); + assert.equal(reversed.statusCode, 200); + assert.equal(upstream.at(-1).searchParams.get('latlng'), '30.27,-97.74'); + assert.equal(upstream.at(-1).searchParams.get('key'), 'server-secret'); + + // The key travels to Google, never back to the page that asked. + assert.ok(!JSON.stringify(forwarded.body).includes('server-secret')); + assert.ok(!JSON.stringify(reversed.body).includes('server-secret')); + + // A malformed bias is dropped rather than forwarded. + await invokeRoute(forward, { url: '/?address=Austin&bounds=north' }); + assert.equal(upstream.at(-1).searchParams.has('bounds'), false); +}); + +test('a bad or keyless geocode request never reaches Google', async () => { + const upstream = []; + const fetchImpl = async (url) => { + upstream.push(String(url)); + return { ok: true, json: async () => ({ status: 'OK', results: [] }) }; + }; + + const routes = installRoutes({ fetchImpl }); + for (const [path, url] of [ + ['/api/google/geocode', '/?address='], + ['/api/google/reverse-geocode', '/?lat=91&lon=-97.74'], + ]) { + const response = await invokeRoute(routes.get(path), { url }); + assert.equal(response.statusCode, 400); + assert.deepEqual(response.body.results, []); + assert.equal(response.body.status, 'REQUEST_DENIED'); + } + + for (const path of ['/api/google/geocode', '/api/google/reverse-geocode']) { + const rejected = await invokeRoute(routes.get(path), { + method: 'POST', + url: '/?address=Austin&lat=30&lon=-97', + }); + assert.equal(rejected.statusCode, 405); + } + + // Keyless is an empty capability, not an outage: a 200 the browser adapters + // read as "this provider did not answer", so the chain moves on. It resolves + // ahead of validation, like the Places routes: with nothing configured there + // is no request to validate and no quota to protect. + const keyless = installRoutes({ apiKey: ' ', fetchImpl }); + for (const path of ['/api/google/geocode', '/api/google/reverse-geocode']) + for (const url of ['/?address=Austin&lat=30&lon=-97', '/?address=&lat=91']) { + const response = await invokeRoute(keyless.get(path), { url }); + assert.equal(response.statusCode, 200); + assert.equal(response.headers['cache-control'], 'no-store'); + assert.deepEqual(response.body, { + configured: false, + status: 'REQUEST_DENIED', + results: [], + error: null, + }); + } + + assert.deepEqual(upstream, []); +}); diff --git a/src/googleServerKey.test.mjs b/src/googleServerKey.test.mjs index 0851afcd5..2789219f5 100644 --- a/src/googleServerKey.test.mjs +++ b/src/googleServerKey.test.mjs @@ -67,11 +67,19 @@ test('the Street View tool resolves per-variable overrides before preferring the } }); -test('both Places routes select the intended key and keep it out of responses', async (t) => { +test('every Google route selects the intended key and keeps it out of responses', async (t) => { const original = { server: process.env.GOOGLE_MAPS_SERVER_API_KEY, browser: process.env.GOOGLE_MAPS_API_KEY, limit: process.env.GEV_RATELIMIT_GOOGLE_PER_MIN }; const calls = []; t.mock.method(globalThis, 'fetch', async (url, options) => { - calls.push({ url, key: options.headers['X-Goog-Api-Key'] }); + // Places sends the key as a header; Geocoding's web service takes it as a + // query parameter. Either way it must be the server key, and never the + // browser's copy of it. + calls.push({ + url, + key: + options?.headers?.['X-Goog-Api-Key'] ?? + new URL(String(url)).searchParams.get('key'), + }); return Response.json({ places: [] }); }); try { @@ -88,12 +96,12 @@ test('both Places routes select the intended key and keep it out of responses', for (const install of ['configureServer', 'configurePreviewServer']) { const routes = new Map(); googlePlacesContextProxy()[install]({ middlewares: { use: (name, handler) => routes.set(name, handler) } }); - assert.equal(routes.size, 2); + assert.equal(routes.size, 4); for (const handler of routes.values()) { const before = calls.length; let body; const res = { setHeader() {}, end(value) { body = value; } }; - await handler({ method: 'GET', url: '/?lat=30&lon=-97&q=capitol', headers: {}, socket: { remoteAddress: '127.0.0.1' } }, res); + await handler({ method: 'GET', url: '/?lat=30&lon=-97&q=capitol&address=capitol', headers: {}, socket: { remoteAddress: '127.0.0.1' } }, res); assert.equal(res.statusCode, 200); assert.ok(!body.includes('server-secret')); if (expected) { diff --git a/src/search/defaults.js b/src/search/defaults.js index 2b0428025..247512405 100644 --- a/src/search/defaults.js +++ b/src/search/defaults.js @@ -40,18 +40,23 @@ export function createDefaultPlaceSearch({ ...(selected ? [selected] : [ + // Google answers through the local proxy, which holds the server + // key: its Geocoding web service refuses referrer-restricted keys, + // so calling it from here would force the bundled browser key to be + // left unrestricted (#363). That key still gates the provider, + // because a keyless session must not spend a request to learn it + // has no Google: the proxy's "not configured" reads as an + // unanswered provider, which stops the chain caching the negative + // Photon or Nominatim then produces. createGoogleGeocoder({ request(query, { bias, signal }) { - const key = resolveApiKey?.(); - if (!key) return null; - const url = new URL( - endpoints.geocode || - 'https://maps.googleapis.com/maps/api/geocode/json', + if (!resolveApiKey?.()) return null; + const params = new URLSearchParams({ address: query }); + if (bias) params.set('bounds', bias); + return fetchImpl( + `${endpoints.geocode || '/api/google/geocode'}?${params}`, + { signal }, ); - url.searchParams.set('address', query); - url.searchParams.set('key', key); - if (bias) url.searchParams.set('bounds', bias); - return fetchImpl(url.toString(), { signal }); }, }), createPhotonGeocoder({ fetchImpl, endpoint: endpoints.photon }), diff --git a/src/search/http.js b/src/search/http.js index 5b1642c67..5afa35b75 100644 --- a/src/search/http.js +++ b/src/search/http.js @@ -53,7 +53,9 @@ export function createHttpGeospatialProvider({ endpoints = {}, } = {}) { const urls = { - reverse: 'https://maps.googleapis.com/maps/api/geocode/json', + // Same-origin like its siblings below: Google's Geocoding web service + // refuses referrer-restricted keys, so the key stays on the server (#363). + reverse: '/api/google/reverse-geocode', textSearch: '/api/google/text-search', nearby: '/api/google/nearby-places', route: '/api/route', @@ -88,14 +90,11 @@ export function createHttpGeospatialProvider({ }, routeProfiles: ['foot', 'car', 'bike'], async reverseGeocode(latitude, longitude, options) { - const key = resolveApiKey?.(); - if (resolveApiKey && !key) return null; + // The browser key gates the provider without travelling with the + // request: the proxy supplies the key Google actually sees. + if (resolveApiKey && !resolveApiKey()) return null; return normalizeGoogleReverse( - await json( - urls.reverse, - { latlng: `${latitude},${longitude}`, ...(key ? { key } : {}) }, - options, - ), + await json(urls.reverse, { lat: latitude, lon: longitude }, options), ); }, async textSearch(query, { latitude, longitude, radiusM = 6000 }, options) { diff --git a/src/search/placeSearch.test.mjs b/src/search/placeSearch.test.mjs index e609f9da9..05e1aaea0 100644 --- a/src/search/placeSearch.test.mjs +++ b/src/search/placeSearch.test.mjs @@ -20,7 +20,7 @@ test('standalone geocoding falls back after Google connection, JSON and refusal for (const fail of [() => { throw new Error('offline'); }, () => new Response('invalid json'), () => Response.json({ status: 'REQUEST_DENIED' })]) { const urls = []; const service = createStandalonePlaceSearch({ resolveApiKey: () => 'fixture', fetchImpl: async (url) => { - urls.push(url); return url.includes('maps.googleapis.com') ? fail() : hit(); + urls.push(url); return url.includes('/api/google/geocode') ? fail() : hit(); } }); const result = await service.geocode('Hanoi'); assert.equal(result.place.name, 'Hà Nội'); diff --git a/src/voice/gevActions.test.mjs b/src/voice/gevActions.test.mjs index ba0ed9cf7..1d8d28c97 100644 --- a/src/voice/gevActions.test.mjs +++ b/src/voice/gevActions.test.mjs @@ -3082,7 +3082,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl const requests = []; installKeyedFetch(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) }; } assert.match(String(url), /^https:\/\/photon\.komoot\.io\/api\/\?/); @@ -3107,7 +3107,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl assert.ok(Math.abs(calls[0].criteria.anchor.lat - 20.9101) < 1e-9); assert.ok(Math.abs(calls[0].criteria.anchor.lon - 107.1839) < 1e-9); // Google is asked first and exactly once; Photon answers unbiased, in one call. - assert.equal(requests.filter((url) => url.includes('maps.googleapis.com')).length, 1); + assert.equal(requests.filter((url) => url.includes('/api/google/geocode')).length, 1); assert.equal(requests.filter((url) => url.includes('photon.komoot.io')).length, 1); assert.match(requests.at(-1), /[?&]q=H%E1%BA%A1\+Long\+Bay/); assert.doesNotMatch(requests.at(-1), /[?&](lat|lon|bbox)=/, 'a named radio location is not viewport-biased'); @@ -3120,7 +3120,7 @@ test('voice Radio: the keyless path applies no country filter the keyed path wou // "No Radio station matched" for exactly the places it just resolved, while a // keyed install placed a station. The label may carry it; the filter may not. const { calls, dataManager } = radioSelectionHarness(); - installKeyedFetch(t, async (url) => (String(url).startsWith('https://maps.googleapis.com/') + installKeyedFetch(t, async (url) => (String(url).startsWith('/api/google/geocode') ? { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) } : { ok: true, From 771c9e5632ae6f0177adc09bc4ab4c34b4698765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Saggese?= Date: Mon, 21 Sep 2026 11:56:33 +0200 Subject: [PATCH 002/202] fix(search): bound and sanitize the geocode proxy - reject blank viewport-bias components before Number() reads them as 0 - give the upstream call a 10s deadline and a 256 KiB capped body read - return fixed public text on failure instead of the upstream message --- server/providers/places/google.js | 60 +++++++++++++++++++++++-------- src/googleGeocodeProxy.test.mjs | 47 +++++++++++++++++++++--- 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/server/providers/places/google.js b/server/providers/places/google.js index 8895bc19d..a4aa48a78 100644 --- a/server/providers/places/google.js +++ b/server/providers/places/google.js @@ -4,6 +4,7 @@ import { keylessGoogleGeocodeResponse, } from './google-key.js'; import { makeOptInRateLimiter, clientKey } from '../common/rate-limit.js'; +import { readResponseJsonCapped } from '../common/http.js'; import { projectNearbyPlaces, projectTextSearchPlaces, @@ -46,6 +47,16 @@ export function validatePlacesCoordinates(searchParams) { /** Longest accepted geocoder query. Real place names are far shorter. */ const GEOCODE_MAX_QUERY = 200; +/** + * Upstream deadline and body ceiling for the geocode routes. The deadline stays + * armed through body consumption, so a stalled upstream cannot hold a handler + * open, and the cap bounds a chunked or length-omitted reply. A geocode answer + * is a handful of results; 256 KiB is far above any honest one. + */ +const GEOCODE_UPSTREAM_TIMEOUT_MS = 10_000; + +const GEOCODE_MAX_RESPONSE_BYTES = 256 * 1024; + /** Validate a free-text geocoder query before consuming request quota. */ export function validateGeocodeAddress(searchParams) { const address = String(searchParams.get('address') || '').trim(); @@ -66,8 +77,12 @@ export function geocodeBounds(searchParams) { if (!raw) return null; const corners = raw.split('|'); if (corners.length !== 2) return null; - const numbers = corners.flatMap((corner) => corner.split(',').map(Number)); - if (numbers.length !== 4 || !numbers.every(Number.isFinite)) return null; + const parts = corners.flatMap((corner) => corner.split(',')); + // A blank component must be refused before Number() turns it into 0 and a + // half-written corner like `30.4,` reads as the equator. + if (parts.length !== 4 || parts.some((part) => !part.trim())) return null; + const numbers = parts.map(Number); + if (!numbers.every(Number.isFinite)) return null; const [swLat, swLon, neLat, neLon] = numbers; if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null; if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null; @@ -379,31 +394,48 @@ export function googlePlacesContextProxy({ for (const [name, value] of Object.entries(query.params)) upstream.searchParams.set(name, value); upstream.searchParams.set('key', apiKey); + const deadline = AbortSignal.timeout(GEOCODE_UPSTREAM_TIMEOUT_MS); const response = await fetchImpl(upstream.toString(), { redirect: 'error', + signal: deadline, }); - const data = await response.json().catch(() => null); + // An unreadable, oversized or timed-out body is not an answer: it + // falls through to the same shaped failure as a refusal below. + const data = await readResponseJsonCapped( + response, + GEOCODE_MAX_RESPONSE_BYTES, + deadline, + ).catch(() => null); + + // A body we could not read is an upstream failure, not an answer with + // a status of its own, so it does not borrow the upstream's code. + if (!data) { + res.statusCode = 502; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end( + JSON.stringify({ + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }), + ); + return; + } res.statusCode = response.ok ? 200 : response.status; res.setHeader('Content-Type', 'application/json; charset=utf-8'); res.setHeader('Cache-Control', 'private, max-age=300'); - res.end( - JSON.stringify( - data ?? { - status: 'UNKNOWN_ERROR', - results: [], - error: 'Google Geocoding request failed', - }, - ), - ); - } catch (error) { + res.end(JSON.stringify(data)); + } catch { + // Fixed public text: a transport, URL or runtime message would + // describe the server to the browser (SECURITY.md, sanitized errors). res.statusCode = 502; res.setHeader('Content-Type', 'application/json; charset=utf-8'); res.end( JSON.stringify({ status: 'UNKNOWN_ERROR', results: [], - error: error?.message || 'Google Geocoding request failed', + error: 'Google Geocoding request failed', }), ); } diff --git a/src/googleGeocodeProxy.test.mjs b/src/googleGeocodeProxy.test.mjs index 2518d02cb..98ffd876e 100644 --- a/src/googleGeocodeProxy.test.mjs +++ b/src/googleGeocodeProxy.test.mjs @@ -73,6 +73,10 @@ test('geocoder query and viewport bias are validated before Google is paid', () 'north,-97.9|30.4,-97.6', '91,-97.9|30.4,-97.6', '30.1,-181|30.4,-97.6', + // A blank component must not read as 0 — that is the equator, not a corner. + '30.1,-97.9|30.4,', + '30.1,|30.4,-97.6', + '30.1,-97.9| ,-97.6', ]) assert.equal(geocodeBounds(new URLSearchParams({ bounds })), null); }); @@ -82,10 +86,7 @@ test('the geocode routes keep the key server-side and answer in Google shape', a const routes = installRoutes({ fetchImpl: async (url) => { upstream.push(new URL(String(url))); - return { - ok: true, - json: async () => ({ status: 'OK', results: [{ place_id: 'fixture' }] }), - }; + return Response.json({ status: 'OK', results: [{ place_id: 'fixture' }] }); }, }); const forward = routes.get('/api/google/geocode'); @@ -123,7 +124,7 @@ test('a bad or keyless geocode request never reaches Google', async () => { const upstream = []; const fetchImpl = async (url) => { upstream.push(String(url)); - return { ok: true, json: async () => ({ status: 'OK', results: [] }) }; + return Response.json({ status: 'OK', results: [] }); }; const routes = installRoutes({ fetchImpl }); @@ -165,3 +166,39 @@ test('a bad or keyless geocode request never reaches Google', async () => { assert.deepEqual(upstream, []); }); + +test('an upstream failure reaches the browser as fixed public text', async () => { + const routes = installRoutes({ + fetchImpl: async () => { + throw new Error('connect ECONNREFUSED 10.0.0.7:443'); + }, + }); + for (const path of ['/api/google/geocode', '/api/google/reverse-geocode']) { + const response = await invokeRoute(routes.get(path), { + url: '/?address=Austin&lat=30&lon=-97', + }); + assert.equal(response.statusCode, 502); + assert.deepEqual(response.body, { + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }); + } +}); + +test('an oversized upstream body is dropped rather than buffered', async () => { + const routes = installRoutes({ + // Two hundred times the 256 KiB ceiling, declared honestly. + fetchImpl: async () => + Response.json({ status: 'OK', results: [{ pad: 'x'.repeat(52_428_800) }] }), + }); + const response = await invokeRoute(routes.get('/api/google/geocode'), { + url: '/?address=Austin', + }); + assert.equal(response.statusCode, 502); + assert.deepEqual(response.body, { + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }); +}); From 0da4d390eaf5b54cac0efeb269879d92e6a562bd Mon Sep 17 00:00:00 2001 From: daikaginza Date: Sat, 26 Sep 2026 17:36:59 +0700 Subject: [PATCH 003/202] docs: add the realtime backend to the voice ownership table The ownership table lists every owner except the backend, though realtimeBackend.js owns the two requests, their separate transports, the abort/timeout scope around both, and client-secret expiry. The method signatures are already documented with application construction, so link there rather than repeat them, and state only the ownership facts written nowhere: the connection is the backend's only consumer in the application, the resolved credential is opaque to it and is handed to negotiate whole, and client-secret expiry is refused inside the backend. --- docs/VOICE-OWNERSHIP.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/VOICE-OWNERSHIP.md b/docs/VOICE-OWNERSHIP.md index d5553df00..be7a6c26b 100644 --- a/docs/VOICE-OWNERSHIP.md +++ b/docs/VOICE-OWNERSHIP.md @@ -8,6 +8,7 @@ The session adapter, action runner and backend interfaces remain unchanged. | Owner | State and responsibility | | --- | --- | | `realtimeConnection.js` | Start generation, request cancellation, peer/data channel, acquired microphone stream and playback element | +| `realtimeBackend.js` | Token and SDP requests, their separate transports, the abort/timeout scope around both, and client-secret expiry | | `realtimeTurns.js` | Active and superseded responses, pending confirmations, deduplicated calls and action cancellation | | `realtimeRadio.js` | Speaker ducking, prepared playback, stronger-action reservations and playback observers | | `realtimeInput.js` | Physical Space gesture, microphone mute, input/output meters and their frame/listener lifetime | @@ -15,6 +16,15 @@ The session adapter, action runner and backend interfaces remain unchanged. | `realtimeViewport.js` | One retained image, bounded deletion identities and capture generation | | `realtimeDiagnostics.js` | Bounded error history and sanitized optional diagnostics | +The backend is replaceable per deployment: `realtimeController.js` accepts it as a +`backend` option and falls back to `createRealtimeBackend()`. Its two method +signatures live with [application construction](APPLICATION.md); what belongs here +is the ownership. The connection is the backend's only consumer in the application. +The resolved credential is opaque to the connection and is handed to `negotiate` +whole. Client-secret expiry is refused inside the backend, not by the connection. +`realtimeOwners.test.mjs` drives whole sessions through a substitute that +implements only those two methods. + Pure input policy, preferences and protocol-response policy have separate modules. Owners receive named readers, operations or focused collaborators. Cross-owner cancellation uses operations; response handling does not own Radio's reservations From 5f8a275487338d3fc08d8e3b91dbc8d856148676 Mon Sep 17 00:00:00 2001 From: Michael Johnston Date: Tue, 15 Sep 2026 22:08:53 -0400 Subject: [PATCH 004/202] fix(providers): bound the remaining upstream body reads Route the six proxied fetches that still buffered upstream bodies without a limit through the shared capped readers in src/sources/httpBody.js: FIRMS CSV, CelesTrak TLE groups, adsb.lol /v2/mil (both the OK and relayed-status paths), TomTom flow tiles, OpenSky states/all and adsbdb enrichment lookups. Each cap is a bound sized well above the measured payload, not a budget. readResponseBytesCapped gains the same optional abort signal as the text reader, with the same cancel-on-abort plumbing and reader release, so the TomTom tile path can share the route deadline instead of reading arrayBuffer() unbounded. Where a route already had a deadline, the same signal now covers the body read as well as the headers. Two polls had no deadline at all: the adsb.lol military poll gains a 12-second one and the OpenSky states/all exchange, including its Basic-auth retry, a 20-second one; both miss into their existing STALE paths. FIRMS map-key status, the OpenSky token exchange and CCTV media are left as they are. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 + docs/CURRENT-STATE.md | 20 ++++ scripts/package-boundaries.json | 14 ++- server/providers/aircraft/adsb-lol.js | 20 +++- server/providers/aircraft/enrichment.js | 16 ++- server/providers/aircraft/opensky.js | 21 +++- server/providers/firms.js | 14 ++- server/providers/space/celestrak.js | 13 ++- server/providers/traffic.js | 15 ++- src/sources/httpBody.js | 44 +++++--- src/sources/httpBody.test.mjs | 133 ++++++++++++++++++++++++ 11 files changed, 278 insertions(+), 34 deletions(-) create mode 100644 src/sources/httpBody.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 29dbedfaa..f5433b87f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +- Bound the remaining upstream body reads (FIRMS, CelesTrak, adsb.lol, TomTom, OpenSky, adsbdb) with streaming size caps and shared deadlines; the adsb.lol military and OpenSky `states/all` polls gain deadlines of their own. + - CCTV cameras whose bearing is a guess now say so. Packs mark bearings derived from a hash of the camera id as `headingConfidence: 'low'`, but nothing read the flag, so roughly 70% of a default catalog rendered like surveyed facings. The HUD diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fdb42fc0c..7853d590e 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -827,6 +827,26 @@ feed address is read from the path, and a query-string form is refused with a 40 streaming. The 12-second deadline includes reading the body, and rejected or stalled downloads are cancelled. Development and preview use the same handler. +## Upstream body bounds + +Every proxied fetch outside CCTV media now reads its body through the shared +streaming readers in `src/sources/httpBody.js`, so a declared or observed body +over the cap is refused with `RESPONSE_TOO_LARGE` and cancelled rather than +buffered. `readResponseBytesCapped` accepts the same optional abort signal as +the text reader and cancels a body still streaming when it fires; TomTom flow +tiles use it alongside GTFS-Realtime. Caps are bounds, not budgets, sized well +above measured payloads: FIRMS 64 MiB per satellite source, CelesTrak 16 MiB +per group, OpenSky `states/all` 8 MiB, adsb.lol `/v2/mil` 8 MiB, TomTom tiles +8 MiB and adsbdb lookups 1 MiB. Where a route already had a deadline (FIRMS +60 s, CelesTrak 20 s, TomTom 15 s, adsbdb 8 s) that deadline now covers the +body read as well as the headers. Two polls that had no deadline gain one: the +adsb.lol military poll waits at most 12 seconds, and the OpenSky `states/all` +exchange, including its Basic-auth retry, at most 20. A rejected or timed-out +body is an upstream miss and follows each route's existing error and +cached-data paths. FIRMS map-key status, the OpenSky token exchange and CCTV +media keep their current handling. + + ## Remaining local service modules Overpass query validation, geometry simplification, disk caching and upstream diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 189192f10..e47e5bb16 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -113,7 +113,12 @@ "traffic-provider": { "runtime": "node", "exports": ["./server/providers/traffic"], - "modules": ["server/providers/traffic.js", "src/data/tomtomTiles.js"], + "modules": [ + "server/providers/traffic.js", + "server/providers/common/http.js", + "src/sources/httpBody.js", + "src/data/tomtomTiles.js" + ], "external": [] }, "traffic-source": { @@ -129,7 +134,12 @@ "firms-provider": { "runtime": "node", "exports": ["./server/providers/firms"], - "modules": ["server/providers/firms.js", "src/data/firmsCsv.js"], + "modules": [ + "server/providers/firms.js", + "server/providers/common/http.js", + "src/sources/httpBody.js", + "src/data/firmsCsv.js" + ], "external": [] }, "gbfs-provider": { diff --git a/server/providers/aircraft/adsb-lol.js b/server/providers/aircraft/adsb-lol.js index fc5ab0bd3..606b598cd 100644 --- a/server/providers/aircraft/adsb-lol.js +++ b/server/providers/aircraft/adsb-lol.js @@ -1,3 +1,5 @@ +import { readResponseTextCapped } from '../common/http.js'; + /** * Vite plugin: adsb.lol military aircraft proxy with 12 s response cache. * @@ -25,6 +27,10 @@ export function adsbLolProxy() { let _cooldownStatus = 0; /** Response cache TTL (ms). */ const CACHE_MS = 12000; + /** Upstream deadline (ms); the fetch previously had none and could hang a poll. */ + const UPSTREAM_TIMEOUT_MS = 12000; + /** /v2/mil measures ~67 KB; a relayed error page is bounded by the same cap. */ + const ADSBLOL_MIL_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; /** Cooldown after a 429 when upstream sends no usable Retry-After (ms). */ const RATE_LIMIT_COOLDOWN_MS = 30000; /** Cooldown after a 5xx (ms). */ @@ -94,11 +100,17 @@ export function adsbLolProxy() { ); return; } + const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS); const upstream = await fetch('https://api.adsb.lol/v2/mil', { headers: { 'User-Agent': 'gods-eye-view-adsblol-proxy/1.0' }, + signal, }); if (upstream.ok) { - const body = await upstream.text(); + const body = await readResponseTextCapped( + upstream, + ADSBLOL_MIL_MAX_RESPONSE_BYTES, + signal, + ); _cache = body; _cacheAt = Date.now(); _cooldownUntil = 0; @@ -125,7 +137,11 @@ export function adsbLolProxy() { return; } } - const body = await upstream.text(); + const body = await readResponseTextCapped( + upstream, + ADSBLOL_MIL_MAX_RESPONSE_BYTES, + signal, + ); serve( res, upstream.status, diff --git a/server/providers/aircraft/enrichment.js b/server/providers/aircraft/enrichment.js index 400050151..9b7c1b1bb 100644 --- a/server/providers/aircraft/enrichment.js +++ b/server/providers/aircraft/enrichment.js @@ -1,5 +1,9 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; +import { readResponseJsonCapped } from '../common/http.js'; + +/** Route and aircraft lookups are a few hundred bytes; 1 MiB is a hard bound. */ +const ADSBDB_MAX_RESPONSE_BYTES = 1024 * 1024; /** * adsbdb.com enrichment proxy: callsign → route (airline + origin/destination * airports) and hex → aircraft type/registration. Free community API — cached @@ -80,12 +84,16 @@ export function adsbdbProxy() { kind === 'route' ? `https://api.adsbdb.com/v0/callsign/${encodeURIComponent(key)}` : `https://api.adsbdb.com/v0/aircraft/${encodeURIComponent(key)}`; - const res = await fetch(url, { signal: AbortSignal.timeout(8000) }); + const signal = AbortSignal.timeout(8000); + const res = await fetch(url, { signal }); if (res.ok) { + const payload = await readResponseJsonCapped( + res, + ADSBDB_MAX_RESPONSE_BYTES, + signal, + ); const data = - kind === 'route' - ? parseRoute(await res.json()) - : parseAircraft(await res.json()); + kind === 'route' ? parseRoute(payload) : parseAircraft(payload); store[key] = { at: Date.now(), data }; // data may be null — negative cache dirty = true; return data; diff --git a/server/providers/aircraft/opensky.js b/server/providers/aircraft/opensky.js index 612d01e11..f1ded6401 100644 --- a/server/providers/aircraft/opensky.js +++ b/server/providers/aircraft/opensky.js @@ -2,6 +2,7 @@ import { normalizeAdsbLolPointResponse } from '../../../src/data/adsbLolFallback import { coalesceProxyRequest, readResponseJsonCapped, + readResponseTextCapped, } from '../common/http.js'; import { requiredFiniteQueryNumber } from '../common/query.js'; // --------------------------------------------------------------------------- @@ -72,6 +73,15 @@ const ADSBLOL_POINT_CACHE_MS = 12000; const ADSBLOL_POINT_CACHE_MAX = 80; const ADSBLOL_POINT_RADIUS_NM = 250; const ADSBLOL_POINT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +/** states/all?extended=1 measures ~0.8 MB anonymously; bounded, not budgeted. */ +const OPENSKY_STATES_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +/** + * Deadline for the whole states/all exchange: headers, the Basic-auth retry + * and the body. The fetch previously had none, so a hung upstream connection + * stalled the worldwide poll indefinitely; a miss here serves STALE like any + * other upstream failure. + */ +const OPENSKY_STATES_TIMEOUT_MS = 20000; // A 200 response can still contain an old OpenSky snapshot. Past this point // the viewport-scoped adsb.lol source is more honest and keeps local motion // current instead of coasting a stale worldwide frame indefinitely. @@ -470,9 +480,10 @@ export function openSkyProxy() { } } + const signal = AbortSignal.timeout(OPENSKY_STATES_TIMEOUT_MS); let upstream = await fetch( 'https://opensky-network.org/api/states/all?extended=1', - { headers }, + { headers, signal }, ); // Auto-mode fallback: if OAuth was rejected, retry with Basic credentials if ( @@ -487,13 +498,17 @@ export function openSkyProxy() { }; upstream = await fetch( 'https://opensky-network.org/api/states/all?extended=1', - { headers: retryHeaders }, + { headers: retryHeaders, signal }, ); usedMode = 'basic'; reason = 'oauth_rejected_fallback_basic'; } - let body = await upstream.text(); + let body = await readResponseTextCapped( + upstream, + OPENSKY_STATES_MAX_RESPONSE_BYTES, + signal, + ); const sourceEpochMs = upstream.ok ? openSkySourceEpochMs(body) : null; if ( upstream.ok && diff --git a/server/providers/firms.js b/server/providers/firms.js index 636968348..c73ca10ce 100644 --- a/server/providers/firms.js +++ b/server/providers/firms.js @@ -2,6 +2,7 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; import { filterTrailing24h, parseFirmsCsv } from '../../src/data/firmsCsv.js'; +import { readResponseTextCapped } from './common/http.js'; /** * NASA FIRMS live active-fire proxy with a memory + disk cache. @@ -35,6 +36,12 @@ export function firmsProxy() { 'VIIRS_SNPP_NRT', 'MODIS_NRT', ]; + /** + * Per-source body cap. A VIIRS world/2 pull is ~130k rows x ~100 B, about + * 13 MB per satellite in heavy fire season (MODIS runs smaller), so this is a + * bound, not a budget. + */ + const FIRMS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024; const CACHE_DIR = path.join(process.cwd(), '.gev-cache'); const CACHE_PATH = path.join(CACHE_DIR, 'firms.json'); @@ -83,9 +90,12 @@ export function firmsProxy() { */ async function fetchSource(key, source) { const url = `https://firms.modaps.eosdis.nasa.gov/api/area/csv/${encodeURIComponent(key)}/${source}/world/2`; - const res = await fetch(url, { signal: AbortSignal.timeout(60_000) }); + const signal = AbortSignal.timeout(60_000); + const res = await fetch(url, { signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); - const records = parseFirmsCsv(await res.text()); + const records = parseFirmsCsv( + await readResponseTextCapped(res, FIRMS_MAX_RESPONSE_BYTES, signal), + ); if (records === null) throw new Error('non-CSV upstream response'); return records; } diff --git a/server/providers/space/celestrak.js b/server/providers/space/celestrak.js index a76cc590d..8baf59ba7 100644 --- a/server/providers/space/celestrak.js +++ b/server/providers/space/celestrak.js @@ -1,6 +1,7 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; import { celestrakTleUrl } from '../../../src/data/spaceProviderRequests.js'; +import { readResponseTextCapped } from '../common/http.js'; /** * Vite plugin: CelesTrak TLE proxy. @@ -48,10 +49,14 @@ export function celestrakProxy() { } } + /** The largest group (`active`) measures ~2.7 MB; this bounds a runaway body. */ + const CELESTRAK_MAX_RESPONSE_BYTES = 16 * 1024 * 1024; + async function fetchUpstream(group) { const url = celestrakTleUrl(group); + const signal = AbortSignal.timeout(20000); const res = await fetch(url.toString(), { - signal: AbortSignal.timeout(20000), + signal, // CelesTrak 403s bulk groups (e.g. `active`) unless the request carries a // descriptive User-Agent with a contact point. headers: { @@ -60,7 +65,11 @@ export function celestrakProxy() { }, }); if (!res.ok) throw new Error(`HTTP ${res.status}`); - const body = await res.text(); + const body = await readResponseTextCapped( + res, + CELESTRAK_MAX_RESPONSE_BYTES, + signal, + ); // An upstream error page parses to zero TLEs — treat as failure, keep cache. if (!/^1 /m.test(body)) throw new Error('no TLE lines in response'); return { at: Date.now(), body }; diff --git a/server/providers/traffic.js b/server/providers/traffic.js index 2a3c2e60a..d53db6530 100644 --- a/server/providers/traffic.js +++ b/server/providers/traffic.js @@ -1,6 +1,7 @@ import path from 'node:path'; import { promises as fsp } from 'node:fs'; +import { readResponseBytesCapped } from './common/http.js'; import { isValidTileCoord as isValidTomTomTile, utcDayKey as tomtomUtcDayKey, @@ -48,6 +49,8 @@ export function tomtomProxy() { const DEFAULT_DAILY_BUDGET = 6000; const MEM_MAX_ENTRIES = 256; const UPSTREAM_TIMEOUT_MS = 15000; + /** Flow tiles are well under 1 MB; this bounds a runaway body, not a budget. */ + const TOMTOM_TILE_MAX_RESPONSE_BYTES = 8 * 1024 * 1024; /** @type {Map} tile key `z/x/y` -> cached tile (kept past TTL for serve-stale). */ const mem = new Map(); @@ -143,11 +146,15 @@ export function tomtomProxy() { 'https://api.tomtom.com/traffic/map/4/tile/flow/relative/' + `${z}/${x}/${y}.pbf?key=${encodeURIComponent(process.env.TOMTOM_API_KEY)}`; recordUpstreamFetch(); // attempts count — upstream bills the request either way - const res = await fetch(url, { - signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), - }); + const signal = AbortSignal.timeout(UPSTREAM_TIMEOUT_MS); + const res = await fetch(url, { signal }); if (!res.ok) throw new Error(`HTTP ${res.status}`); - const buf = Buffer.from(await res.arrayBuffer()); + const bytes = await readResponseBytesCapped( + res, + TOMTOM_TILE_MAX_RESPONSE_BYTES, + signal, + ); + const buf = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); if (buf.length === 0) throw new Error('empty tile body'); return buf; } diff --git a/src/sources/httpBody.js b/src/sources/httpBody.js index 6f176ca52..481dd5b7f 100644 --- a/src/sources/httpBody.js +++ b/src/sources/httpBody.js @@ -55,38 +55,52 @@ export async function readResponseJsonCapped(response, maxBytes, signal) { /** * Read a fetch() Response body as bytes with the same hard cap as - * readResponseTextCapped — for protobuf upstreams (GTFS-Realtime). + * readResponseTextCapped — for protobuf and tile upstreams (GTFS-Realtime, + * TomTom flow). Same early Content-Length rejection, running cap while + * streaming and cancellation on abort, so callers can treat the two alike. * Throws { code:'RESPONSE_TOO_LARGE' }. */ -export async function readResponseBytesCapped(response, maxBytes) { +export async function readResponseBytesCapped(response, maxBytes, signal) { const tooLarge = () => { const err = new Error('Upstream response too large'); err.code = 'RESPONSE_TOO_LARGE'; return err; }; const declared = Number(response.headers.get('content-length')); - if (Number.isFinite(declared) && declared > maxBytes) throw tooLarge(); + if (Number.isFinite(declared) && declared > maxBytes) { + void response.body?.cancel().catch(() => {}); + throw tooLarge(); + } + signal?.throwIfAborted(); const reader = response.body?.getReader?.(); if (!reader) { const bytes = new Uint8Array(await response.arrayBuffer()); + signal?.throwIfAborted(); if (bytes.byteLength > maxBytes) throw tooLarge(); return bytes; } + const cancel = () => { + void reader.cancel().catch(() => {}); + }; + signal?.addEventListener('abort', cancel, { once: true }); const chunks = []; let total = 0; - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - total += value.byteLength; - if (total > maxBytes) { - try { - await reader.cancel(); - } catch { - /* no-op */ - } - throw tooLarge(); + try { + for (;;) { + signal?.throwIfAborted(); + const { done, value } = await reader.read(); + signal?.throwIfAborted(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) throw tooLarge(); + chunks.push(value); } - chunks.push(value); + } catch (error) { + cancel(); + throw error; + } finally { + signal?.removeEventListener('abort', cancel); + reader.releaseLock(); } const out = new Uint8Array(total); let offset = 0; diff --git a/src/sources/httpBody.test.mjs b/src/sources/httpBody.test.mjs new file mode 100644 index 000000000..aa8e77d1a --- /dev/null +++ b/src/sources/httpBody.test.mjs @@ -0,0 +1,133 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readResponseBytesCapped } from './httpBody.js'; + +/** + * A Response that streams `chunk` forever until the reader cancels it. The + * zero high-water mark keeps the stream from pulling eagerly at construction, + * so `pulls()` counts only what a reader actually asked for. + */ +function endlessResponse(chunk, onCancel, headers = {}) { + let pulls = 0; + const stream = new ReadableStream( + { + pull(controller) { + pulls += 1; + controller.enqueue(chunk); + }, + cancel: onCancel, + }, + { highWaterMark: 0 }, + ); + return { response: new Response(stream, { headers }), pulls: () => pulls }; +} + +function chunkedResponse(chunks, headers = {}) { + const stream = new ReadableStream( + { + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + controller.close(); + }, + }, + { highWaterMark: 0 }, + ); + return new Response(stream, { headers }); +} + +const bytes = (n, fill) => new Uint8Array(n).fill(fill); + +test('returns the exact bytes of a body that fits, untouched by text decoding', async () => { + // 0xFF and 0xFE are invalid UTF-8 lead bytes; a text path would mangle them. + const payload = Uint8Array.from([0xff, 0xfe, 0x00, 0x7f, 0x80, 0x01]); + const buf = await readResponseBytesCapped(chunkedResponse([payload]), 1024); + assert.ok(buf instanceof Uint8Array); + assert.deepEqual([...buf], [...payload]); +}); + +test('joins chunks in order and reports the total length', async () => { + const buf = await readResponseBytesCapped( + chunkedResponse([bytes(3, 1), bytes(2, 2), bytes(1, 3)]), + 1024, + ); + assert.deepEqual([...buf], [1, 1, 1, 2, 2, 3]); +}); + +test('refuses on a declared Content-Length without pulling the body', async () => { + let cancelled = false; + const { response, pulls } = endlessResponse( + bytes(512, 0), + () => { + cancelled = true; + }, + { 'content-length': '4096' }, + ); + await assert.rejects(readResponseBytesCapped(response, 1024), { + code: 'RESPONSE_TOO_LARGE', + }); + assert.equal(pulls(), 0); + assert.ok(cancelled); +}); + +test('refuses a chunked body the moment it passes the cap and stops pulling', async () => { + let cancelled = false; + const { response, pulls } = endlessResponse(bytes(512, 0), () => { + cancelled = true; + }); + await assert.rejects(readResponseBytesCapped(response, 1024), { + code: 'RESPONSE_TOO_LARGE', + }); + // Three 512-byte chunks is the first total over 1024; an uncapped read of + // this never-ending stream would not terminate at all. + assert.ok(pulls() <= 3, `pulled ${pulls()} chunks`); + assert.ok(cancelled); +}); + +test('accepts a body exactly at the cap', async () => { + const buf = await readResponseBytesCapped( + chunkedResponse([bytes(512, 7), bytes(512, 7)]), + 1024, + ); + assert.equal(buf.byteLength, 1024); +}); + +test('cancels a body still streaming when the signal aborts, and never returns a partial one', async () => { + const controller = new AbortController(); + let cancelled = false; + let pulls = 0; + const stream = new ReadableStream( + { + pull(ctrl) { + pulls += 1; + ctrl.enqueue(bytes(64, 0)); + if (pulls === 2) controller.abort(); + }, + cancel() { + cancelled = true; + }, + }, + { highWaterMark: 0 }, + ); + await assert.rejects( + readResponseBytesCapped(new Response(stream), 1 << 20, controller.signal), + ); + assert.ok(cancelled); + assert.ok(pulls <= 3, `pulled ${pulls} chunks after abort`); +}); + +test('refuses immediately when the signal is already aborted', async () => { + const controller = new AbortController(); + controller.abort(); + const { response, pulls } = endlessResponse(bytes(8, 0), () => {}); + await assert.rejects( + readResponseBytesCapped(response, 1024, controller.signal), + ); + assert.equal(pulls(), 0); +}); + +test('falls back to arrayBuffer() for a body with no stream reader', async () => { + const response = new Response(bytes(16, 9)); + const buf = await readResponseBytesCapped(response, 1024); + assert.equal(buf.byteLength, 16); + assert.equal(buf[0], 9); +}); From d585d25b19c0f41746268e07ea840f11365bd0e3 Mon Sep 17 00:00:00 2001 From: Michael Johnston Date: Tue, 15 Sep 2026 22:08:54 -0400 Subject: [PATCH 005/202] chore(format): adopt the httpBody reader test Co-Authored-By: Claude Opus 5 --- scripts/format-scope.json | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index eee546dd5..d661d7a6b 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -699,6 +699,7 @@ "src/search/nominatim.js", "src/search/nominatim.test.mjs", "src/sources/httpBody.js", + "src/sources/httpBody.test.mjs", "src/sources/overpass.js", "src/sources/nominatim.js", "server/providers/transit.js", From 881e24485cf92da68af79ef91c3372ad7ba598d4 Mon Sep 17 00:00:00 2001 From: devv-shayan Date: Wed, 16 Sep 2026 17:45:27 +0500 Subject: [PATCH 006/202] feat(voice): add ChatGPT OAuth auth option --- .env.example | 5 +- README.md | 8 +- docs/CURRENT-STATE.md | 6 +- scripts/dev-fresh.sh | 2 +- scripts/package-boundaries.json | 3 + scripts/setup-doctor.mjs | 4 +- server/providers/openai.js | 20 ++- server/providers/openai/codex-auth.js | 100 +++++++++++ server/providers/openai/realtime.js | 232 ++++++++++++++++++++++++-- src/codexOauthRealtime.test.mjs | 93 +++++++++++ src/keySetup.js | 144 ++++++++++++++++ src/keySetup.test.mjs | 23 +++ src/keySetupCore.mjs | 2 +- src/setupDoctor.test.mjs | 7 + src/tooling/localServices.test.mjs | 144 +++++++++++++++- src/ui/styles/provider-settings.css | 37 ++++ src/voice/cloudVoiceAuth.js | 41 +++++ src/voice/cloudVoiceAuth.test.mjs | 29 ++++ src/voice/gevRealtime.test.mjs | 14 ++ src/voice/realtimeBackend.js | 12 +- src/voice/realtimeBackend.test.mjs | 16 ++ src/voice/realtimeConnection.js | 2 + src/voice/realtimeCost.js | 8 + 23 files changed, 918 insertions(+), 34 deletions(-) create mode 100644 server/providers/openai/codex-auth.js create mode 100644 src/codexOauthRealtime.test.mjs create mode 100644 src/voice/cloudVoiceAuth.js create mode 100644 src/voice/cloudVoiceAuth.test.mjs diff --git a/.env.example b/.env.example index e16e85a27..4c7edb372 100644 --- a/.env.example +++ b/.env.example @@ -52,7 +52,10 @@ GOOGLE_MAPS_SERVER_API_KEY= # CLIENT-EXPOSED — use a public assets:read token with URL restrictions. CESIUM_ION_TOKEN= -# Optional: OpenAI Realtime voice control. Do not prefix with VITE_. +# Optional: OpenAI Realtime voice control. API key is the default cloud auth. +# Provider Settings can instead opt voice into the signed-in local +# ChatGPT/Codex OAuth session; the browser still receives only a short-lived +# Realtime client secret. Do not prefix with VITE_. OPENAI_API_KEY= OPENAI_REALTIME_MODEL=gpt-realtime-2 # Cheaper voice tier, used when the GEV MIC panel's MINI toggle is on. diff --git a/README.md b/README.md index 88690d52f..9676ab76f 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ _Why cockpit mode exists: you're riding a real aircraft over real terrain — an ## 🎙️ Talk to It -> Voice needs an **OpenAI key**. Without one the entire app still runs — the mic button just reports voice is unavailable. The same key drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. +> Voice can use an **OpenAI API key** or, on the same machine, an opt-in signed-in **ChatGPT/Codex OAuth** session. The API-key path remains the default. Selecting ChatGPT OAuth in Provider Settings reuses an existing local sign-in or starts the Codex browser sign-in flow and selects OAuth when it completes. The OpenAI key still drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. Click **GEV MIC**, grant the microphone, and just talk. This is more than a voice-controlled remote: @@ -239,7 +239,7 @@ Click **GEV MIC**, grant the microphone, and just talk. This is more than a voic - **🎯 Entity Q&A.** Click any plane, ship, or datacenter and ask _"what's this?"_ It answers using the object's live telemetry. - **👁️ Visual grounding.** At street level, it reads a viewport screenshot to identify legible signage and building names, and is instructed never to hallucinate labels. - **🎬 Cinematic framing.** _"Show me the planes overhead"_ pulls the camera back, angles it, and frames the live traffic like a director. -- **🔒 Honest and secure.** The agent only confirms actions that succeeded. Your `OPENAI_API_KEY` never touches the browser; the client only gets a short-lived session token. +- **🔒 Honest and secure.** The agent only confirms actions that succeeded. Your `OPENAI_API_KEY` never touches the browser; the client only gets a short-lived session token. If you choose **USE CHATGPT OAUTH** in Provider Settings, the local server uses that sign-in only to mint the same kind of short-lived Realtime credential, and refuses the OAuth path for non-loopback clients. Twenty-nine tools, four jobs — the commands below come straight from the product's voice test suite and tool playbook: @@ -412,7 +412,7 @@ Six keys. Four have a free tier, and the two 🔴 ones are metered: | --- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 🟡 | **Cesium ion** | 🗺️ Google Photorealistic 3D, world terrain, and additional ion-hosted imagery stacks. The free Community plan is for eligible individual, personal/non-commercial use and has quotas | [cesium.com/ion](https://cesium.com/ion) — use a public `assets:read` token and check current [pricing/eligibility](https://cesium.com/platform/cesium-ion/pricing/) | | 🔴 | **Google Maps** | Direct Google Photorealistic 3D + Google place search ([Map Tiles API](https://developers.google.com/maps/documentation/tile)) | [Google Cloud Console](https://console.cloud.google.com/) — URL-restrict it | -| 🔴 | **OpenAI** | 🎙️ The voice experience + AI HUD summary. The mini model works; the standard model is noticeably smarter. Want Gemini or another provider behind the mic? PRs welcome | [platform.openai.com](https://platform.openai.com) — metered, see costs below | +| 🔴 | **OpenAI** | 🎙️ Voice + AI HUD summary through an API key. Provider Settings also offers **USE CHATGPT OAUTH** for cloud voice without removing the API-key option | [platform.openai.com](https://platform.openai.com) — API-key usage is metered, see costs below | | 🟡 | **AISStream** | 🚢 Live global ships | [aisstream.io](https://aisstream.io) — free signup | | 🟡 | **NASA FIRMS** | 🔥 Live active fires | [firms.modaps.eosdis.nasa.gov](https://firms.modaps.eosdis.nasa.gov/api/map_key/) — free | | 🟡 | **TomTom** | 🚦 Live flow speeds and congestion colors for the simulated traffic layer | [developer.tomtom.com](https://developer.tomtom.com) — free tier available | @@ -490,7 +490,7 @@ Honest numbers, roughly, as of mid-2026 — always check the provider pricing pa | **🟢 Most layers** | **$0, no signup.** OpenSky anon, USGS, CelesTrak, adsb.lol, city CCTV, Radio Browser, GBFS, Launch Library 2, transit feeds, OSRM routing, NOAA and ECMWF weather, OpenStreetMap ALPR mapping, bundled datasets. | | **🟡 The free-key tier** | **$0 with a signup.** AISStream, FIRMS, TomTom, OpenSky, plus Cesium ion for eligible personal/non-commercial use. Provider quotas and eligibility still apply. | | **🗺️ Google 3D tiles** | **Free through an eligible Cesium ion Community account within its quota; metered through a direct Google key.** Use the direct route for Google place search or commercial deployment, verify current provider terms, and set budget alerts where billing is enabled. | -| **🔴 OpenAI voice** | **The one that costs real money — so the app meters it for you.** Realtime audio runs a few cents per active minute; an evening of heavy use is single-digit dollars. A live session-spend readout sits next to the mic, with an STD/MINI model toggle, a $2 warning, and a **$5 hard cap that ends the session**. The voice context window is kept deliberately short too. | +| **🔴 OpenAI voice** | API-key sessions keep the live spend readout, STD/MINI model toggle, $2 warning, and **$5 hard cap**. ChatGPT OAuth is an explicit choice in Provider Settings. Its usage is tracked without an API-dollar estimate or spend cap. The voice context window stays deliberately short. | Google's direct 3D route is surprisingly generous: the first 1,000 Photorealistic 3D Tiles sessions each month are currently free, and one root request supports diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fdb42fc0c..410e648aa 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3607,7 +3607,7 @@ silently demoting every later lookup for the session. `GEV MIC` button (bottom UI) starts an OpenAI Realtime session over WebRTC: -- **Token flow**: browser fetches a short-lived client secret from `/api/realtime/token`; the Vite middleware holds `OPENAI_API_KEY` and posts the full session config (instructions, tool schemas, VAD, truncation) to `api.openai.com/v1/realtime/client_secrets`. SDP exchange goes directly to `api.openai.com/v1/realtime/calls` with the ephemeral token. +- **Token flow**: browser fetches a short-lived client secret from `/api/realtime/token`; the Vite middleware uses `OPENAI_API_KEY` by default, or the signed-in local ChatGPT/Codex OAuth access token when OAuth was explicitly selected in Provider Settings, and posts the full session config (instructions, tool schemas, VAD, truncation) to `api.openai.com/v1/realtime/client_secrets`. Both long-lived credentials stay server-side; OAuth minting is loopback-only. SDP exchange goes directly to `api.openai.com/v1/realtime/calls` with the ephemeral token. - **Session defaults** (env-tunable): model `gpt-realtime-2` (or `gpt-realtime-2.1-mini` when the MINI tier is selected — see the model-tier entry below), voice `marin`, reasoning effort `low`, semantic VAD with low eagerness, no response interruption, context window truncated to ~3,000 post-instruction tokens with 0.5 retention ratio — the conversational window stays short because map state is fetched live per turn. - **Twenty-nine tools** (argument schemas in `src/voice/actionSchemas.js`, served with their descriptions by `server/providers/openai/tools.js`, executed client-side in `src/voice/gevActions.js`): `fly_to_location`, `select_nearest_aircraft`, `adjust_camera_zoom`, `zoom_to_globe`, `set_layer_visibility`, `show_data_layers_menu`, `set_panel_open`, `set_visual_style`, `get_entity_context`, `get_current_view_state`, `set_hud`, `set_detection`, `set_map_stack`, `set_post_processing`, `control_scene`, `control_cctv`, `set_context_mode`, `control_cockpit`, `control_radio`, `track_entity`, `stop_tracking`, `frame_overhead`, `annotate_map`, `clear_annotations`, `move_camera`, `fly_route`, `analyst_query`, `next_iss_pass`, and `next_satellite_pass`. @@ -3653,7 +3653,7 @@ silently demoting every later lookup for the session. 2. **Contacts OFF** → "nearby" means **in view**; "near \" means a radius around that place. A radius query with Contacts active and no explicit centre is centred on the **active contact**, not the camera. 3. **Every count names its scope in words** — "42 in your window", "8 in view", "about 30 within 250 km of Austin" — never a bare number. `analyst_query` returns `scopeLabel` so this is mechanical. Two different numbers with named scopes are not a contradiction. 4. **The loaded-data caveat is stated once when relevant**: counts cover loaded data, and the flights layer loads where you look (appended to `coverage.note` for radius/view scopes over viewport-loaded layers). -- **Degradation**: without `OPENAI_API_KEY`, `/api/realtime/token` returns 503 and the mic button surfaces the error; the rest of the app is unaffected. +- **Degradation**: API-key mode without `OPENAI_API_KEY` returns 503 as before. Selecting OAuth without a usable local ChatGPT/Codex sign-in starts the local `codex login` browser flow, polls for the resulting local token, and selects OAuth when sign-in completes. If the Codex login flow cannot start or does not complete, the rest of the app is unaffected. ### AI HUD Summary (June 2026) @@ -3835,7 +3835,7 @@ are omitted rather than framing the wrong part of the globe. - GBFS proxy refuses upstream redirects (`redirect: 'manual'`; any 3xx becomes a 502 and the redirect target is logged server-side only) and enforces its 5 MB response cap while the body streams, cancelling the upstream read past the cap; CCTV health map is bounded. - Proxy error payloads are sanitized (no internal error details returned to clients). - That holds for the OpenAI and CCTV media paths too: `/api/openai/hud-summary` never relays OpenAI's own `error.message`, `/api/realtime/token` passes successful ephemeral-token responses through but answers with a fixed error when minting fails or upstream rejects the request, and a failed CCTV media fetch stores a fixed camera health `message` — `GET /api/cctv/health` serializes that field and the CCTV panel renders it as a status label, so it is a client surface as much as the response body is. -- `OPENAI_API_KEY` is server-side only; the browser receives ephemeral Realtime client secrets from `/api/realtime/token`. +- `OPENAI_API_KEY` remains server-side only and remains the default voice credential. The optional OAuth mode reads the local ChatGPT/Codex access token server-side on loopback only; the browser receives only ephemeral Realtime client secrets from `/api/realtime/token`. - `AISSTREAM_API_KEY` is server-side only; the browser reads the same-origin `/api/ais-live` cache. - `/api/google/nearby-places` keeps the Google key out of Places requests issued for voice scene context. - `/api/google/text-search` keeps the Google key server-side for view-biased Places recovery used by annotation resolution. diff --git a/scripts/dev-fresh.sh b/scripts/dev-fresh.sh index d3e175050..8c134c745 100755 --- a/scripts/dev-fresh.sh +++ b/scripts/dev-fresh.sh @@ -347,7 +347,7 @@ case "${OPENSKY_AUTH_MODE}" in echo "OpenSky auth: disabled (anonymous mode)" ;; esac -[[ -n "${OPENAI_API_KEY}" ]] && echo "OpenAI key (voice + HUD summary): configured" || echo "OpenAI key (voice + HUD summary): not set — GEV MIC disabled" +[[ -n "${OPENAI_API_KEY}" ]] && echo "OpenAI key (voice + HUD summary): configured" || echo "OpenAI key: not set — GEV MIC can use ChatGPT OAuth from Provider Settings" [[ -n "${AISSTREAM_API_KEY}" ]] && echo "AISStream key (live vessels): configured" || echo "AISStream key (live vessels): not set — ships layer empty" if [[ -n "${GOOGLE_MAPS_API_KEY}" ]]; then echo "Startup map: Google Photorealistic 3D Tiles (direct)" diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 189192f10..12d03ef35 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -262,6 +262,7 @@ "server/providers/common/request.js", "server/providers/common/source-root.js", "server/providers/openai.js", + "server/providers/openai/codex-auth.js", "server/providers/openai/constants.js", "server/providers/openai/debug-log.js", "server/providers/openai/hud-summary.js", @@ -1220,6 +1221,7 @@ "modules": [ "src/voice/realtimeController.js", "src/voice/realtimeBackend.js", + "src/voice/cloudVoiceAuth.js", "src/voice/control.js", "src/voice/layerSummary.js", "src/voice/voiceCost.js", @@ -1795,6 +1797,7 @@ "src/voice/gevActions.js", "src/voice/gevRealtime.js", "src/voice/layerSummary.js", + "src/voice/cloudVoiceAuth.js", "src/voice/realtimeBackend.js", "src/voice/realtimeController.js", "src/voice/voiceCost.js", diff --git a/scripts/setup-doctor.mjs b/scripts/setup-doctor.mjs index 954369030..813548dbc 100644 --- a/scripts/setup-doctor.mjs +++ b/scripts/setup-doctor.mjs @@ -175,7 +175,9 @@ export function buildCapabilitySummary( ? 'Google Photorealistic 3D Tiles through Cesium ion; Bing and world-terrain stacks available' : 'Esri World Imagery (keyless satellite basemap) with keyless terrain', flights, - voice: configured('OPENAI_API_KEY') ? 'available' : 'off until an OpenAI key is added', + voice: configured('OPENAI_API_KEY') + ? 'available' + : 'ChatGPT/Codex OAuth can be selected at runtime', vessels: configured('AISSTREAM_API_KEY') ? 'live AISStream feed' : 'off until an AISStream key is added', fires: configured('FIRMS_MAP_KEY') ? 'live NASA FIRMS feed' : 'off until a FIRMS key is added', traffic: configured('TOMTOM_API_KEY') ? 'live TomTom flow' : 'built-in traffic simulation', diff --git a/server/providers/openai.js b/server/providers/openai.js index 5dd03db5c..ac1fcd1a7 100644 --- a/server/providers/openai.js +++ b/server/providers/openai.js @@ -1,13 +1,17 @@ import { defaultSourceRoot } from './common/source-root.js'; import { handleHudSummary } from './openai/hud-summary.js'; import { createDebugLogHandler } from './openai/debug-log.js'; -import { createRealtimeTokenHandler } from './openai/realtime.js'; +import { + createRealtimeOAuthLoginHandler, + createRealtimeOAuthStatusHandler, + createRealtimeTokenHandler, +} from './openai/realtime.js'; /** * Vite plugin: OpenAI Realtime ephemeral client secret. * - * Keeps OPENAI_API_KEY server-side while the browser connects to the - * Realtime API over WebRTC with a short-lived secret. + * Keeps the selected cloud credential server-side while the browser connects + * to the Realtime API over WebRTC with a short-lived secret. */ function openAiRealtimeProxy({ sourceRoot = defaultSourceRoot, @@ -26,6 +30,16 @@ function openAiRealtimeProxy({ '/api/realtime/token', createRealtimeTokenHandler({ ...realtime, annotationGuidance }), ); + + middlewares.use( + '/api/realtime/oauth-status', + createRealtimeOAuthStatusHandler(realtime), + ); + + middlewares.use( + '/api/realtime/oauth-login', + createRealtimeOAuthLoginHandler(realtime), + ); } return { diff --git a/server/providers/openai/codex-auth.js b/server/providers/openai/codex-auth.js new file mode 100644 index 000000000..6e882accc --- /dev/null +++ b/server/providers/openai/codex-auth.js @@ -0,0 +1,100 @@ +import { spawn } from 'node:child_process'; +import { accessSync, constants as fsConstants, readFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export function codexAuthPath({ + environment = process.env, + home = os.homedir(), +} = {}) { + const explicit = String(environment.CODEX_AUTH_JSON || '').trim(); + if (explicit) return path.resolve(explicit); + const codexHome = String(environment.CODEX_HOME || '').trim(); + return path.join( + codexHome ? path.resolve(codexHome) : path.join(home, '.codex'), + 'auth.json', + ); +} + +export function readCodexOAuthAccessToken({ + authPath = codexAuthPath(), + readFile = readFileSync, +} = {}) { + let parsed; + try { + parsed = JSON.parse(readFile(authPath, 'utf8')); + } catch { + throw new Error( + 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.', + ); + } + const token = String(parsed?.tokens?.access_token || '').trim(); + if (!token) { + throw new Error( + 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.', + ); + } + return token; +} + +export function resolveCodexExecutable({ + environment = process.env, + home = os.homedir(), + access = accessSync, +} = {}) { + const explicit = String(environment.CODEX_BIN || '').trim(); + if (explicit) return explicit; + + for (const candidate of [ + path.join(home, '.local', 'bin', 'codex'), + path.join( + home, + '.codex', + 'packages', + 'standalone', + 'current', + 'bin', + 'codex', + ), + ]) { + try { + access(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Fall through to the next known location, then PATH. + } + } + return 'codex'; +} + +export function startCodexChatGptLogin({ + spawnImpl = spawn, + environment = process.env, + home = os.homedir(), + executable = resolveCodexExecutable({ environment, home }), +} = {}) { + return new Promise((resolve, reject) => { + let child; + try { + child = spawnImpl(executable, ['login'], { + cwd: home, + detached: true, + stdio: 'ignore', + env: environment, + }); + } catch (error) { + reject(error); + return; + } + + child.once('error', reject); + child.once('spawn', () => { + child.unref?.(); + resolve({ executable }); + }); + }).catch((error) => { + throw new Error( + `Could not start ChatGPT sign-in with Codex: ${error?.message || error}`, + ); + }); +} diff --git a/server/providers/openai/realtime.js b/server/providers/openai/realtime.js index b4dfb4b5a..98fea5318 100644 --- a/server/providers/openai/realtime.js +++ b/server/providers/openai/realtime.js @@ -13,12 +13,167 @@ import { } from './constants.js'; import { realtimeInstructions } from './instructions.js'; import { GEV_REALTIME_TOOLS } from './tools.js'; +import { + readCodexOAuthAccessToken, + startCodexChatGptLogin, +} from './codex-auth.js'; + +const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']); + +function isLoopbackRequest(req) { + return LOOPBACK_ADDRESSES.has(String(req.socket?.remoteAddress || '')); +} + +function isSameOriginRequest(req) { + const origin = String(req.headers?.origin || '').trim(); + if (!origin) return true; + const host = String(req.headers?.host || '').trim(); + if (!host) return false; + const protocol = req.socket?.encrypted ? 'https:' : 'http:'; + try { + return new URL(origin).origin === new URL(`${protocol}//${host}`).origin; + } catch { + return false; + } +} + +function requestQuery(req) { + try { + return new URL(req.url || '', 'http://localhost').searchParams; + } catch { + return new URLSearchParams(); + } +} + +function resolveRealtimeAuthMode(query) { + const requested = String(query.get('auth') || 'api-key').toLowerCase(); + if (requested === 'api-key' || requested === 'oauth') return requested; + return null; +} + +function createRealtimeOAuthStatusHandler({ + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), +} = {}) { + return (req, res) => { + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Content-Type', 'application/json'); + if (req.method !== 'GET') { + res.statusCode = 405; + res.end(JSON.stringify({ error: 'Method not allowed' })); + return; + } + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'ChatGPT OAuth is available only from this machine', + }), + ); + return; + } + try { + const token = resolveOAuthAccessToken(); + if (!token) throw new Error('ChatGPT sign-in is unavailable'); + res.statusCode = 200; + res.end(JSON.stringify({ available: true })); + } catch (error) { + res.statusCode = 200; + res.end( + JSON.stringify({ + available: false, + error: error?.message || 'ChatGPT sign-in is unavailable', + }), + ); + } + }; +} + +function createRealtimeOAuthLoginHandler({ + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), + startOAuthLogin = () => startCodexChatGptLogin(), + now = () => Date.now(), + restartCooldownMs = 30_000, +} = {}) { + let lastStartedAt = 0; + return async (req, res) => { + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Content-Type', 'application/json'); + if (req.method !== 'POST') { + res.statusCode = 405; + res.end(JSON.stringify({ error: 'Method not allowed' })); + return; + } + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'ChatGPT OAuth sign-in is available only from this machine', + }), + ); + return; + } + if (!isSameOriginRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'Cross-origin ChatGPT OAuth sign-in is not allowed', + }), + ); + return; + } + + try { + if (resolveOAuthAccessToken()) { + res.statusCode = 200; + res.end( + JSON.stringify({ available: true, started: false, pending: false }), + ); + return; + } + } catch { + // No usable local token yet; launch the supported Codex sign-in flow. + } + + const currentTime = now(); + if ( + lastStartedAt > 0 && + currentTime - lastStartedAt < Math.max(0, restartCooldownMs) + ) { + res.statusCode = 202; + res.end( + JSON.stringify({ available: false, started: false, pending: true }), + ); + return; + } + + try { + await startOAuthLogin(); + lastStartedAt = currentTime; + res.statusCode = 202; + res.end( + JSON.stringify({ available: false, started: true, pending: true }), + ); + } catch (error) { + res.statusCode = 503; + res.end( + JSON.stringify({ + available: false, + error: error?.message || 'Could not start ChatGPT sign-in', + }), + ); + } + }; +} function createRealtimeTokenHandler({ annotationGuidance, endpoint = 'https://api.openai.com/v1/realtime/client_secrets', fetchImpl = (...args) => fetch(...args), resolveApiKey = () => process.env.OPENAI_API_KEY, + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), models = {}, } = {}) { return async (req, res) => { @@ -30,15 +185,62 @@ function createRealtimeTokenHandler({ return; } + const query = requestQuery(req); + const authMode = resolveRealtimeAuthMode(query); + if (!authMode) { + res.statusCode = 400; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'Unsupported cloud voice auth mode' })); + return; + } + // Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). No-op when unset. if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return; - const apiKey = resolveApiKey(); - if (!apiKey) { - res.statusCode = 503; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' })); - return; + let bearer; + if (authMode === 'oauth') { + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: 'ChatGPT OAuth voice is available only from this machine', + }), + ); + return; + } + try { + bearer = resolveOAuthAccessToken(); + } catch (error) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: error?.message || 'ChatGPT sign-in is unavailable', + code: 'CODEX_OAUTH_UNAVAILABLE', + }), + ); + return; + } + if (!bearer) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: 'ChatGPT sign-in is unavailable', + code: 'CODEX_OAUTH_UNAVAILABLE', + }), + ); + return; + } + } else { + bearer = resolveApiKey(); + if (!bearer) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' })); + return; + } } // Voice model tier, requested by the client as ?tier=standard|mini. @@ -47,15 +249,7 @@ function createRealtimeTokenHandler({ // bad querystring degrades to a normal session rather than a dead mic. // The env overrides stay authoritative per tier (see .env.example) — // a wrong upstream model id is then a config fix, not a code change. - const requestedTier = (() => { - try { - return new URL(req.url || '', 'http://localhost').searchParams.get( - 'tier', - ); - } catch { - return null; - } - })(); + const requestedTier = query.get('tier'); const tier = resolveVoiceModel(requestedTier).tier; const model = tier === 'mini' @@ -124,7 +318,7 @@ function createRealtimeTokenHandler({ redirect: 'error', signal: AbortSignal.timeout(30_000), headers: { - Authorization: `Bearer ${apiKey}`, + Authorization: `Bearer ${bearer}`, 'Content-Type': 'application/json', 'OpenAI-Safety-Identifier': 'gev-local-dev', }, @@ -167,4 +361,8 @@ function createRealtimeTokenHandler({ }; } -export { createRealtimeTokenHandler }; +export { + createRealtimeOAuthLoginHandler, + createRealtimeOAuthStatusHandler, + createRealtimeTokenHandler, +}; diff --git a/src/codexOauthRealtime.test.mjs b/src/codexOauthRealtime.test.mjs new file mode 100644 index 000000000..4afadb0e6 --- /dev/null +++ b/src/codexOauthRealtime.test.mjs @@ -0,0 +1,93 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import test from 'node:test'; +import path from 'node:path'; +import { + codexAuthPath, + readCodexOAuthAccessToken, + resolveCodexExecutable, + startCodexChatGptLogin, +} from '../server/providers/openai/codex-auth.js'; + +test('Codex auth path follows explicit auth file, CODEX_HOME, then home directory', () => { + assert.equal( + codexAuthPath({ + environment: { CODEX_AUTH_JSON: '/tmp/custom-auth.json' }, + home: '/home/fixture', + }), + path.resolve('/tmp/custom-auth.json'), + ); + assert.equal( + codexAuthPath({ + environment: { CODEX_HOME: '/tmp/codex-home' }, + home: '/home/fixture', + }), + path.join(path.resolve('/tmp/codex-home'), 'auth.json'), + ); + assert.equal( + codexAuthPath({ environment: {}, home: '/home/fixture' }), + path.join('/home/fixture', '.codex', 'auth.json'), + ); +}); + +test('Codex OAuth token reader returns only the access token and fails closed', () => { + assert.equal( + readCodexOAuthAccessToken({ + authPath: '/unused', + readFile: () => JSON.stringify({ tokens: { access_token: ' oauth-fixture ' } }), + }), + 'oauth-fixture', + ); + for (const readFile of [ + () => '{bad json', + () => JSON.stringify({ tokens: {} }), + () => { + throw new Error('missing'); + }, + ]) { + assert.throws( + () => readCodexOAuthAccessToken({ authPath: '/unused', readFile }), + /ChatGPT sign-in is unavailable/, + ); + } +}); + +test('Codex login resolves an installed executable before PATH and starts browser auth detached', async () => { + const checked = []; + assert.equal( + resolveCodexExecutable({ + environment: {}, + home: '/home/fixture', + access(candidate) { + checked.push(candidate); + if (candidate.endsWith('/.local/bin/codex')) return; + throw new Error('missing'); + }, + }), + path.join('/home/fixture', '.local', 'bin', 'codex'), + ); + assert.equal(checked.length, 1); + + const child = new EventEmitter(); + let unref = 0; + child.unref = () => { + unref += 1; + }; + const calls = []; + const pending = startCodexChatGptLogin({ + executable: '/fixture/codex', + home: '/home/fixture', + environment: { PATH: '/fixture' }, + spawnImpl(command, args, options) { + calls.push({ command, args, options }); + queueMicrotask(() => child.emit('spawn')); + return child; + }, + }); + assert.deepEqual(await pending, { executable: '/fixture/codex' }); + assert.equal(unref, 1); + assert.equal(calls[0].command, '/fixture/codex'); + assert.deepEqual(calls[0].args, ['login']); + assert.equal(calls[0].options.detached, true); + assert.equal(calls[0].options.stdio, 'ignore'); +}); diff --git a/src/keySetup.js b/src/keySetup.js index ee1d3b2dd..e137ff4d8 100644 --- a/src/keySetup.js +++ b/src/keySetup.js @@ -1,4 +1,8 @@ import { createSurfaceKeyboard } from './ui/surfaceKeyboard.js'; +import { + readStoredCloudVoiceAuthMode, + writeStoredCloudVoiceAuthMode, +} from './voice/cloudVoiceAuth.js'; /** * The POWER UP surface — paste a key, get a power. @@ -38,6 +42,46 @@ export function collectKeyUpdates(fields) { return updates; } +function sleepWithSignal(ms, signal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason || new DOMException('Aborted', 'AbortError')); + return; + } + const onAbort = () => { + clearTimeout(timer); + reject(signal.reason || new DOMException('Aborted', 'AbortError')); + }; + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + +export async function waitForChatGptOAuth({ + fetchImpl, + signal, + timeoutMs = 120_000, + pollMs = 1_000, + now = () => Date.now(), + sleep = sleepWithSignal, +} = {}) { + const deadline = now() + Math.max(0, timeoutMs); + while (!signal?.aborted && now() <= deadline) { + const response = await fetchImpl('/api/realtime/oauth-status', { + cache: 'no-store', + signal, + }); + const payload = await response.json().catch(() => ({})); + if (response.ok && payload.available) return true; + if (now() >= deadline) break; + await sleep(Math.max(0, pollMs), signal); + } + return false; +} + /** * After the FIRST Google key lands, the restart's reload should boot the * photoreal default — not faithfully restore the auto-selected keyless OSM @@ -144,6 +188,26 @@ function buildRow(documentRef, key) { } row.append(fields); } + if (key.id === 'openai') { + const auth = documentRef.createElement('div'); + auth.className = 'key-setup-cloud-auth'; + const mode = readStoredCloudVoiceAuthMode(); + const state = documentRef.createElement('span'); + state.className = 'key-setup-cloud-auth-state'; + state.textContent = + mode === 'oauth' ? 'VOICE AUTH · CHATGPT OAUTH' : 'VOICE AUTH · API KEY'; + const toggle = documentRef.createElement('button'); + toggle.type = 'button'; + toggle.className = 'key-setup-cloud-auth-toggle'; + toggle.dataset.cloudVoiceAuthToggle = 'true'; + toggle.textContent = mode === 'oauth' ? 'USE API KEY' : 'USE CHATGPT OAUTH'; + toggle.title = + mode === 'oauth' + ? 'Use OPENAI_API_KEY for the next cloud voice session' + : 'Use the signed-in local ChatGPT/Codex OAuth session for the next cloud voice session'; + auth.append(state, toggle); + row.append(auth); + } return row; } @@ -363,6 +427,86 @@ export async function initKeySetup({ applyButton?.addEventListener('click', onApply); // Remove buttons are rendered per row; delegate so re-renders stay wired. rowsHost?.addEventListener('click', (event) => { + const authButton = event.target?.closest?.( + '[data-cloud-voice-auth-toggle]', + ); + if (authButton && !disposed) { + void (async () => { + const current = readStoredCloudVoiceAuthMode(); + if (current === 'oauth') { + writeStoredCloudVoiceAuthMode('api-key'); + render(status); + say('Cloud voice will use OPENAI_API_KEY on the next session.'); + return; + } + authButton.disabled = true; + say('Checking local ChatGPT OAuth sign-in…'); + try { + const response = await doFetch('/api/realtime/oauth-status', { + cache: 'no-store', + signal: lifetime.signal, + }); + const payload = await response.json().catch(() => ({})); + if (response.ok && payload.available) { + writeStoredCloudVoiceAuthMode('oauth'); + render(status); + say( + 'ChatGPT OAuth selected for cloud voice. Your API key stays saved and available.', + ); + return; + } + + say('Opening ChatGPT sign-in in your browser…'); + const loginResponse = await doFetch('/api/realtime/oauth-login', { + method: 'POST', + cache: 'no-store', + signal: lifetime.signal, + }); + const loginPayload = await loginResponse.json().catch(() => ({})); + if (!loginResponse.ok && loginResponse.status !== 202) { + say( + loginPayload.error || + payload.error || + 'Could not start ChatGPT sign-in on this machine.', + ); + return; + } + if (loginPayload.available) { + writeStoredCloudVoiceAuthMode('oauth'); + render(status); + say( + 'ChatGPT OAuth selected for cloud voice. Your API key stays saved and available.', + ); + return; + } + + say( + 'Finish ChatGPT sign-in in the browser. Waiting for it to complete…', + ); + const available = await waitForChatGptOAuth({ + fetchImpl: doFetch, + signal: lifetime.signal, + }); + if (!available) { + say( + 'ChatGPT sign-in is still pending. Finish it in the browser, then click USE CHATGPT OAUTH again.', + ); + return; + } + writeStoredCloudVoiceAuthMode('oauth'); + render(status); + say( + 'ChatGPT sign-in complete. OAuth will be used for the next cloud voice session.', + ); + } catch (error) { + if (lifetime.signal.aborted) return; + say(`OAuth check failed: ${error?.message || error}`); + } finally { + authButton.disabled = false; + } + })(); + return; + } const button = event.target?.closest?.('[data-key-setup-remove]'); if (disposed || !button || busy) return; let envVars = []; diff --git a/src/keySetup.test.mjs b/src/keySetup.test.mjs index c0d9520c0..a03f2d556 100644 --- a/src/keySetup.test.mjs +++ b/src/keySetup.test.mjs @@ -5,6 +5,7 @@ import { keySetupChipLabel, stripKeylessBasemapFromHash, bindKeySetupPlacement, + waitForChatGptOAuth, } from './keySetup.js'; test('setup placement moves the same button only in Cyber and disconnects on teardown', () => { @@ -61,6 +62,28 @@ test('collectKeyUpdates keeps only non-empty trimmed values', () => { assert.deepEqual(collectKeyUpdates(null), {}); }); +test('OAuth login polling stops as soon as local ChatGPT auth becomes available', async () => { + let checks = 0; + let clock = 0; + const available = await waitForChatGptOAuth({ + fetchImpl: async () => { + checks += 1; + return { + ok: true, + json: async () => ({ available: checks >= 3 }), + }; + }, + timeoutMs: 10_000, + pollMs: 100, + now: () => clock, + sleep: async (ms) => { + clock += ms; + }, + }); + assert.equal(available, true); + assert.equal(checks, 3); +}); + test('the first Google key strips ONLY the keyless OSM basemap from the share hash', () => { const stripped = stripKeylessBasemapFromHash('lat=30.2&lon=-97.7&map=osm&style=normal'); assert.ok(stripped !== null); diff --git a/src/keySetupCore.mjs b/src/keySetupCore.mjs index e8bfef92c..a23c9825c 100644 --- a/src/keySetupCore.mjs +++ b/src/keySetupCore.mjs @@ -52,7 +52,7 @@ export const KEY_SETUP_KEYS = Object.freeze([ Object.freeze({ id: 'openai', title: 'OPENAI', - unlocks: 'Voice control — talk to the planet', + unlocks: 'Voice control — API key or ChatGPT OAuth', getUrl: 'https://platform.openai.com/api-keys', envVars: Object.freeze(['OPENAI_API_KEY']), tier: 'metered', diff --git a/src/setupDoctor.test.mjs b/src/setupDoctor.test.mjs index c66d9e686..aa825f126 100644 --- a/src/setupDoctor.test.mjs +++ b/src/setupDoctor.test.mjs @@ -224,6 +224,13 @@ test('doctor describes the credential ladder without exposing values', () => { assert.match(capabilities.map, /Google Photorealistic 3D Tiles through Cesium ion/); assert.match(capabilities.map, /Bing and world-terrain stacks/); assert.equal(capabilities.voice, 'available'); + assert.equal( + buildCapabilitySummary({ + ...credentials, + OPENAI_API_KEY: { configured: false }, + }).voice, + 'ChatGPT/Codex OAuth can be selected at runtime', + ); assert.match(capabilities.missions, /token allowance/); assert.equal(capabilities.flights, 'OpenSky keyless anonymous access (rate-limited)'); diff --git a/src/tooling/localServices.test.mjs b/src/tooling/localServices.test.mjs index 26eebfa08..7014659b7 100644 --- a/src/tooling/localServices.test.mjs +++ b/src/tooling/localServices.test.mjs @@ -37,6 +37,7 @@ function request( url = '/', body = '', origin = 'http://localhost:4173', + remoteAddress = '127.0.0.1', } = {}, ) { return new Promise((resolve, reject) => { @@ -49,7 +50,7 @@ function request( origin, 'content-type': 'application/json', }, - socket: { remoteAddress: '127.0.0.1' }, + socket: { remoteAddress }, }); const headers = {}; const res = { @@ -195,6 +196,147 @@ test('Realtime handler preserves tools and default instructions, isolates suppli assert.equal(sent[0].session.instructions, sent[2].session.instructions); }); +test('Realtime handler keeps API key as default and uses OAuth only when requested', async (t) => { + env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined); + let oauthReads = 0; + const authorizations = []; + const routes = install( + openAiRealtimeProxy({ + realtime: { + resolveApiKey: () => 'fixture-api-key', + resolveOAuthAccessToken: () => { + oauthReads += 1; + return 'fixture-oauth-token'; + }, + fetchImpl: async (_url, options) => { + authorizations.push(options.headers.Authorization); + return Response.json({ value: 'fixture-ephemeral' }); + }, + }, + }), + ); + const token = routes.get('/api/realtime/token'); + + assert.equal((await request(token)).status, 200); + assert.equal(oauthReads, 0, 'default cloud voice does not read OAuth'); + assert.equal(authorizations.at(-1), 'Bearer fixture-api-key'); + + assert.equal((await request(token, { url: '/?auth=oauth' })).status, 200); + assert.equal(oauthReads, 1); + assert.equal(authorizations.at(-1), 'Bearer fixture-oauth-token'); +}); + +test('OAuth status and token minting never expose local OAuth to remote clients', async () => { + let oauthReads = 0; + const routes = install( + openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => { + oauthReads += 1; + return 'must-not-be-read-remotely'; + }, + fetchImpl: () => + assert.fail('remote OAuth request must not reach OpenAI'), + }, + }), + ); + const remote = { remoteAddress: '192.168.1.20' }; + assert.equal( + (await request(routes.get('/api/realtime/oauth-status'), remote)).status, + 403, + ); + assert.equal( + ( + await request(routes.get('/api/realtime/token'), { + ...remote, + url: '/?auth=oauth', + }) + ).status, + 403, + ); + assert.equal(oauthReads, 0); +}); + +test('OAuth status reports local sign-in availability without returning the token', async () => { + const routes = install( + openAiRealtimeProxy({ + realtime: { resolveOAuthAccessToken: () => 'fixture-oauth-secret' }, + }), + ); + const response = await request(routes.get('/api/realtime/oauth-status')); + assert.equal(response.status, 200); + assert.deepEqual(response.json(), { available: true }); + assert.equal(response.body.includes('fixture-oauth-secret'), false); +}); + +test('OAuth login endpoint launches local Codex sign-in once and never exposes credentials', async () => { + let starts = 0; + let clock = 1_000; + const routes = install( + openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => { + throw new Error('not signed in'); + }, + startOAuthLogin: async () => { + starts += 1; + }, + now: () => clock, + }, + }), + ); + const handler = routes.get('/api/realtime/oauth-login'); + + const started = await request(handler, { method: 'POST' }); + assert.equal(started.status, 202); + assert.deepEqual(started.json(), { + available: false, + started: true, + pending: true, + }); + assert.equal(starts, 1); + + clock += 1_000; + const duplicate = await request(handler, { method: 'POST' }); + assert.equal(duplicate.status, 202); + assert.equal(duplicate.json().pending, true); + assert.equal(starts, 1, 'rapid retries must not spawn another login'); + + const remote = await request(handler, { + method: 'POST', + remoteAddress: '192.168.1.20', + }); + assert.equal(remote.status, 403); + assert.equal(starts, 1); + + const crossOrigin = await request(handler, { + method: 'POST', + origin: 'https://example.com', + }); + assert.equal(crossOrigin.status, 403); + assert.equal(starts, 1); +}); + +test('OAuth login endpoint selects an existing local sign-in without launching Codex', async () => { + const handler = install( + openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => 'fixture-existing-secret', + startOAuthLogin: () => + assert.fail('existing auth must not launch login'), + }, + }), + ).get('/api/realtime/oauth-login'); + const response = await request(handler, { method: 'POST' }); + assert.equal(response.status, 200); + assert.deepEqual(response.json(), { + available: true, + started: false, + pending: false, + }); + assert.equal(response.body.includes('fixture-existing-secret'), false); +}); + test('debug logging resolves each supplied application directory independently', async (t) => { const first = root(t), second = root(t); diff --git a/src/ui/styles/provider-settings.css b/src/ui/styles/provider-settings.css index b68593990..043e046d6 100644 --- a/src/ui/styles/provider-settings.css +++ b/src/ui/styles/provider-settings.css @@ -300,6 +300,43 @@ color: rgba(148, 178, 188, 0.42); } +.key-setup-cloud-auth { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.key-setup-cloud-auth-state { + color: rgba(190, 214, 222, 0.7); + font-family: var(--font-mono); + font-size: 0.56rem; + letter-spacing: 0.05rem; +} + +.key-setup-cloud-auth-toggle { + margin-left: auto; + padding: 0.28rem 0.55rem; + color: rgba(54, 220, 255, 0.9); + font-family: var(--font-mono); + font-size: 0.55rem; + letter-spacing: 0.06rem; + background: rgba(54, 220, 255, 0.08); + border: 1px solid rgba(54, 220, 255, 0.28); + border-radius: 0.32rem; + cursor: pointer; +} + +.key-setup-cloud-auth-toggle:hover, +.key-setup-cloud-auth-toggle:focus-visible { + border-color: rgba(54, 220, 255, 0.65); + outline: none; +} + +.key-setup-cloud-auth-toggle:disabled { + cursor: wait; + opacity: 0.5; +} + .key-setup-footer { margin-top: 0.85rem; } diff --git a/src/voice/cloudVoiceAuth.js b/src/voice/cloudVoiceAuth.js new file mode 100644 index 000000000..77feae8aa --- /dev/null +++ b/src/voice/cloudVoiceAuth.js @@ -0,0 +1,41 @@ +export const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']); +export const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key'; +export const CLOUD_VOICE_AUTH_STORAGE_KEY = 'godsEyeView.voice.cloudAuth'; + +function storageRef(storage) { + if (storage) return storage; + try { + return typeof localStorage === 'undefined' ? null : localStorage; + } catch { + return null; + } +} + +export function normalizeCloudVoiceAuthMode(mode) { + const normalized = String(mode || '') + .trim() + .toLowerCase(); + return CLOUD_VOICE_AUTH_MODES.includes(normalized) + ? normalized + : DEFAULT_CLOUD_VOICE_AUTH_MODE; +} + +export function readStoredCloudVoiceAuthMode(storage) { + try { + return normalizeCloudVoiceAuthMode( + storageRef(storage)?.getItem(CLOUD_VOICE_AUTH_STORAGE_KEY), + ); + } catch { + return DEFAULT_CLOUD_VOICE_AUTH_MODE; + } +} + +export function writeStoredCloudVoiceAuthMode(mode, storage) { + const resolved = normalizeCloudVoiceAuthMode(mode); + try { + storageRef(storage)?.setItem(CLOUD_VOICE_AUTH_STORAGE_KEY, resolved); + } catch { + /* best effort */ + } + return resolved; +} diff --git a/src/voice/cloudVoiceAuth.test.mjs b/src/voice/cloudVoiceAuth.test.mjs new file mode 100644 index 000000000..c7a4c2e4d --- /dev/null +++ b/src/voice/cloudVoiceAuth.test.mjs @@ -0,0 +1,29 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + CLOUD_VOICE_AUTH_STORAGE_KEY, + readStoredCloudVoiceAuthMode, + writeStoredCloudVoiceAuthMode, +} from './cloudVoiceAuth.js'; + +function memoryStorage(seed = {}) { + const values = new Map(Object.entries(seed)); + return { + getItem: (key) => values.get(key) ?? null, + setItem: (key, value) => values.set(key, value), + }; +} + +test('cloud voice auth defaults to API key and persists OAuth independently', () => { + const storage = memoryStorage(); + assert.equal(readStoredCloudVoiceAuthMode(storage), 'api-key'); + assert.equal(writeStoredCloudVoiceAuthMode('oauth', storage), 'oauth'); + assert.equal(readStoredCloudVoiceAuthMode(storage), 'oauth'); + assert.equal(writeStoredCloudVoiceAuthMode('unknown', storage), 'api-key'); + assert.equal( + readStoredCloudVoiceAuthMode( + memoryStorage({ [CLOUD_VOICE_AUTH_STORAGE_KEY]: 'oauth' }), + ), + 'oauth', + ); +}); diff --git a/src/voice/gevRealtime.test.mjs b/src/voice/gevRealtime.test.mjs index d7056a0e3..66e1b1367 100644 --- a/src/voice/gevRealtime.test.mjs +++ b/src/voice/gevRealtime.test.mjs @@ -3194,6 +3194,20 @@ test('F1: the toggle still records the next-session preference while live', () = assert.match(ui.tierButton.title, /this session stays on/i); }); +test('OAuth cloud sessions hide API spend and do not trip the API spend guard', () => { + const { controller, ui } = costControllerHarness(); + controller.status = 'listening'; + controller._cost.cloudVoiceAuth = 'oauth'; + controller._cost.sessionCloudVoiceAuth = 'oauth'; + controller.syncCostUi(); + const before = controller.costTracker.state(); + assert.equal(ui.costValue.hidden, true); + assert.equal(controller.recordUsage(usdUsage(100)), null); + const after = controller.costTracker.state(); + assert.equal(after.totalUsd, before.totalUsd); + assert.equal(controller.costCapStopped, false); +}); + test('F1: when idle, toggling does re-price the preview meter', () => { const { controller } = costControllerHarness(); controller.status = 'idle'; diff --git a/src/voice/realtimeBackend.js b/src/voice/realtimeBackend.js index 2a8af2d0d..b429e17c5 100644 --- a/src/voice/realtimeBackend.js +++ b/src/voice/realtimeBackend.js @@ -15,11 +15,19 @@ export function createRealtimeBackend({ ); return Object.freeze({ protocol: 'openai-realtime', - async requestToken({ tier = DEFAULT_VOICE_TIER, signal } = {}) { + async requestToken({ + tier = DEFAULT_VOICE_TIER, + authMode = 'api-key', + signal, + } = {}) { signal = scoped(signal); signal.throwIfAborted(); const separator = tokenEndpoint.includes('?') ? '&' : '?'; - const url = `${tokenEndpoint}${separator}tier=${encodeURIComponent(resolveVoiceModel(tier).tier)}`; + const query = new URLSearchParams({ + tier: resolveVoiceModel(tier).tier, + }); + if (authMode === 'oauth') query.set('auth', 'oauth'); + const url = `${tokenEndpoint}${separator}${query}`; const response = await tokenTransport(url, { signal, cache: 'no-store', diff --git a/src/voice/realtimeBackend.test.mjs b/src/voice/realtimeBackend.test.mjs index 1fc21da9d..d179f5b6b 100644 --- a/src/voice/realtimeBackend.test.mjs +++ b/src/voice/realtimeBackend.test.mjs @@ -41,6 +41,22 @@ test('separate configured token and SDP transports preserve model metadata and c ]); }); +test('OAuth is an additive token query while API key stays the default', async () => { + const calls = []; + const backend = createRealtimeBackend({ + tokenTransport: async (input) => { + calls.push(input); + return tokenReply(); + }, + }); + await backend.requestToken(); + await backend.requestToken({ authMode: 'oauth' }); + assert.deepEqual(calls, [ + '/api/realtime/token?tier=standard', + '/api/realtime/token?tier=standard&auth=oauth', + ]); +}); + test('denied tokens never negotiate or retry; reconnect requests a fresh token', async () => { let requests = 0; const backend = createRealtimeBackend({ diff --git a/src/voice/realtimeConnection.js b/src/voice/realtimeConnection.js index b20afb67d..b0409eb92 100644 --- a/src/voice/realtimeConnection.js +++ b/src/voice/realtimeConnection.js @@ -82,6 +82,7 @@ export class RealtimeConnection { this.debugLog('session.starting', { epoch, tier: this.cost.voiceTier, + authMode: this.cost.sessionCloudVoiceAuth, connection: this.connectionDiagnostics(), }); let localStream = null; @@ -89,6 +90,7 @@ export class RealtimeConnection { try { const minted = await this.backend.requestToken({ tier: this.cost.voiceTier, + authMode: this.cost.sessionCloudVoiceAuth, signal, }); const token = minted.token; diff --git a/src/voice/realtimeCost.js b/src/voice/realtimeCost.js index 9c1169490..83d19785e 100644 --- a/src/voice/realtimeCost.js +++ b/src/voice/realtimeCost.js @@ -4,6 +4,7 @@ import { writeStoredVoiceTier, writeStoredVoiceLimits, } from './realtimePreferences.js'; +import { readStoredCloudVoiceAuthMode } from './cloudVoiceAuth.js'; import { createVoiceCostTracker, resolveVoiceModel, @@ -16,6 +17,8 @@ export class RealtimeCost { Object.assign(this, { readUi, readStatus }, operations); this.voiceTier = readStoredVoiceTier(); this.voiceLimits = readStoredVoiceLimits(); + this.cloudVoiceAuth = readStoredCloudVoiceAuthMode(); + this.sessionCloudVoiceAuth = null; this.costTracker = createVoiceCostTracker({ tier: this.voiceTier, limits: this.voiceLimits, @@ -56,6 +59,8 @@ export class RealtimeCost { }; applies next session`; } if (this.ui?.costValue) { + const liveAuth = this.sessionCloudVoiceAuth || this.cloudVoiceAuth; + this.ui.costValue.hidden = liveAuth === 'oauth'; this.ui.costValue.textContent = state.display; this.ui.costValue.dataset.level = state.level; this.ui.costValue.title = @@ -136,6 +141,7 @@ export class RealtimeCost { */ recordUsage(usage) { if (!usage) return null; + if (this.sessionCloudVoiceAuth === 'oauth') return null; const state = this.costTracker.record(usage); this.syncCostUi(); if (state.warnCrossed) { @@ -190,6 +196,8 @@ export class RealtimeCost { prepareSession() { this.voiceTier = readStoredVoiceTier(); this.voiceLimits = readStoredVoiceLimits(); + this.cloudVoiceAuth = readStoredCloudVoiceAuthMode(); + this.sessionCloudVoiceAuth = this.cloudVoiceAuth; this.costCapStopped = false; // Provisional meter (tier-priced) so the readout shows $0.00 while // connecting. It is REPLACED below with one bound to the model the server From a398a259341537dc45497acbad2f1f17ce82eac4 Mon Sep 17 00:00:00 2001 From: devv-shayan Date: Sun, 27 Sep 2026 21:09:26 +0500 Subject: [PATCH 007/202] fix(voice): validate Codex OAuth and report unknown usage --- .env.example | 3 + server/providers/openai/codex-auth.js | 85 +++++++++++++++++++++++--- server/providers/openai/realtime.js | 28 +++++++++ src/codexOauthRealtime.test.mjs | 86 +++++++++++++++++++++++++-- src/tooling/localServices.test.mjs | 33 +++++++++- src/voice/gevRealtime.test.mjs | 17 +++++- src/voice/realtimeController.js | 5 +- src/voice/realtimeCost.js | 27 ++++++++- 8 files changed, 265 insertions(+), 19 deletions(-) diff --git a/.env.example b/.env.example index 4c7edb372..552962ba1 100644 --- a/.env.example +++ b/.env.example @@ -57,6 +57,9 @@ CESIUM_ION_TOKEN= # ChatGPT/Codex OAuth session; the browser still receives only a short-lived # Realtime client secret. Do not prefix with VITE_. OPENAI_API_KEY= +# Optional: forbid API-key voice sessions. Select ChatGPT OAuth in Provider +# Settings; missing or expired Codex sign-in then fails closed. +# GEV_PREFER_CODEX_OAUTH=true OPENAI_REALTIME_MODEL=gpt-realtime-2 # Cheaper voice tier, used when the GEV MIC panel's MINI toggle is on. # Model ids and prices are external facts that drift — if OpenAI moves this id, diff --git a/server/providers/openai/codex-auth.js b/server/providers/openai/codex-auth.js index 6e882accc..68c20e929 100644 --- a/server/providers/openai/codex-auth.js +++ b/server/providers/openai/codex-auth.js @@ -1,5 +1,10 @@ import { spawn } from 'node:child_process'; -import { accessSync, constants as fsConstants, readFileSync } from 'node:fs'; +import { + accessSync, + constants as fsConstants, + readFileSync, + statSync, +} from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -16,22 +21,88 @@ export function codexAuthPath({ ); } +/** An optional operator lock that refuses the metered API-key voice lane. */ +export function preferCodexOAuth(environment = process.env) { + if (!Object.hasOwn(environment, 'GEV_PREFER_CODEX_OAUTH')) return false; + const value = String(environment.GEV_PREFER_CODEX_OAUTH || '') + .trim() + .toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(value)) return true; + if (['0', 'false', 'no', 'off'].includes(value)) return false; + throw new Error( + 'GEV_PREFER_CODEX_OAUTH must be true or false. Refusing to choose a voice auth mode.', + ); +} + export function readCodexOAuthAccessToken({ authPath = codexAuthPath(), readFile = readFileSync, + statFile = statSync, + nowMs = Date.now(), } = {}) { + const unavailable = + 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.'; let parsed; try { parsed = JSON.parse(readFile(authPath, 'utf8')); } catch { - throw new Error( - 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.', - ); + throw new Error(unavailable); } - const token = String(parsed?.tokens?.access_token || '').trim(); - if (!token) { + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(unavailable); + } + const mode = String(parsed.auth_mode || '') + .trim() + .toLowerCase(); + const chatGptMode = mode + ? mode === 'chatgpt' || mode === 'chatgptauthtokens' + : typeof parsed.OPENAI_API_KEY !== 'string'; + const token = parsed.tokens?.access_token; + const refresh = parsed.tokens?.refresh_token; + if ( + !chatGptMode || + typeof token !== 'string' || + !token.trim() || + typeof refresh !== 'string' || + !refresh.trim() + ) { + throw new Error(unavailable); + } + + // This only checks token shape and expiry. OpenAI verifies the signature when + // minting the short-lived Realtime credential; we never modify auth.json. + const parts = token.split('.'); + if ( + parts.length !== 3 || + parts.some((part) => !/^[A-Za-z0-9_-]+$/.test(part)) + ) { + throw new Error(unavailable); + } + let payload; + try { + payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')); + } catch { + throw new Error(unavailable); + } + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + throw new Error(unavailable); + } + let expiresAtMs; + if (Object.hasOwn(payload, 'exp')) { + if (typeof payload.exp !== 'number' || !Number.isFinite(payload.exp)) { + throw new Error(unavailable); + } + expiresAtMs = payload.exp * 1000; + } else { + try { + expiresAtMs = statFile(authPath).mtimeMs + 60 * 60 * 1000; + } catch { + throw new Error(unavailable); + } + } + if (!Number.isFinite(expiresAtMs) || expiresAtMs <= nowMs + 60_000) { throw new Error( - 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.', + 'ChatGPT sign-in has expired. Run codex login and try again.', ); } return token; diff --git a/server/providers/openai/realtime.js b/server/providers/openai/realtime.js index 98fea5318..1869ebd24 100644 --- a/server/providers/openai/realtime.js +++ b/server/providers/openai/realtime.js @@ -14,6 +14,7 @@ import { import { realtimeInstructions } from './instructions.js'; import { GEV_REALTIME_TOOLS } from './tools.js'; import { + preferCodexOAuth, readCodexOAuthAccessToken, startCodexChatGptLogin, } from './codex-auth.js'; @@ -174,6 +175,7 @@ function createRealtimeTokenHandler({ fetchImpl = (...args) => fetch(...args), resolveApiKey = () => process.env.OPENAI_API_KEY, resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), + environment = process.env, models = {}, } = {}) { return async (req, res) => { @@ -194,6 +196,28 @@ function createRealtimeTokenHandler({ return; } + let oauthPinned; + try { + oauthPinned = preferCodexOAuth(environment); + } catch (error) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: error.message })); + return; + } + if (oauthPinned && authMode !== 'oauth') { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: + 'ChatGPT OAuth is required by GEV_PREFER_CODEX_OAUTH. Select USE CHATGPT OAUTH in Provider Settings.', + code: 'CODEX_OAUTH_REQUIRED', + }), + ); + return; + } + // Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). No-op when unset. if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return; @@ -332,6 +356,10 @@ function createRealtimeTokenHandler({ // case where a bogus ?tier= was silently downgraded to standard. res.setHeader('X-GEV-Voice-Tier', tier); res.setHeader('X-GEV-Voice-Model', model); + res.setHeader( + 'X-GEV-Voice-Auth', + authMode === 'oauth' ? 'codex-oauth' : 'env', + ); if (requestedTier && !isKnownVoiceTier(requestedTier)) { res.setHeader('X-GEV-Voice-Tier-Fallback', '1'); } diff --git a/src/codexOauthRealtime.test.mjs b/src/codexOauthRealtime.test.mjs index 4afadb0e6..ffd2b169f 100644 --- a/src/codexOauthRealtime.test.mjs +++ b/src/codexOauthRealtime.test.mjs @@ -4,11 +4,32 @@ import test from 'node:test'; import path from 'node:path'; import { codexAuthPath, + preferCodexOAuth, readCodexOAuthAccessToken, resolveCodexExecutable, startCodexChatGptLogin, } from '../server/providers/openai/codex-auth.js'; +const nowMs = Date.UTC(2026, 8, 27); +const jwt = (payload) => + `${Buffer.from('{}').toString('base64url')}.${Buffer.from(JSON.stringify(payload)).toString('base64url')}.signature`; +const auth = (token, extras = {}) => + JSON.stringify({ + auth_mode: 'chatgpt', + tokens: { access_token: token, refresh_token: 'refresh-fixture' }, + ...extras, + }); + +test('optional Codex OAuth pin fails closed on ambiguous values', () => { + assert.equal(preferCodexOAuth({}), false); + assert.equal(preferCodexOAuth({ GEV_PREFER_CODEX_OAUTH: 'true' }), true); + assert.equal(preferCodexOAuth({ GEV_PREFER_CODEX_OAUTH: 'off' }), false); + assert.throws( + () => preferCodexOAuth({ GEV_PREFER_CODEX_OAUTH: 'maybe' }), + /must be true or false/, + ); +}); + test('Codex auth path follows explicit auth file, CODEX_HOME, then home directory', () => { assert.equal( codexAuthPath({ @@ -30,28 +51,85 @@ test('Codex auth path follows explicit auth file, CODEX_HOME, then home director ); }); -test('Codex OAuth token reader returns only the access token and fails closed', () => { +test('Codex OAuth reader accepts a current ChatGPT token and rejects unusable credentials', () => { + const current = jwt({ exp: nowMs / 1000 + 3600 }); assert.equal( readCodexOAuthAccessToken({ authPath: '/unused', - readFile: () => JSON.stringify({ tokens: { access_token: ' oauth-fixture ' } }), + nowMs, + readFile: () => auth(current), }), - 'oauth-fixture', + current, ); for (const readFile of [ () => '{bad json', () => JSON.stringify({ tokens: {} }), + () => auth(current, { auth_mode: 'apikey' }), + () => + JSON.stringify({ + OPENAI_API_KEY: 'sk-fixture', + tokens: { access_token: current, refresh_token: 'refresh-fixture' }, + }), + () => + JSON.stringify({ + auth_mode: 'chatgpt', + tokens: { access_token: current }, + }), + () => auth('opaque-token'), + () => auth('header.not-json.signature'), + () => auth(jwt({ exp: 'tomorrow' })), () => { throw new Error('missing'); }, ]) { assert.throws( - () => readCodexOAuthAccessToken({ authPath: '/unused', readFile }), + () => readCodexOAuthAccessToken({ authPath: '/unused', readFile, nowMs }), /ChatGPT sign-in is unavailable/, ); } }); +test('Codex OAuth reader enforces expiry margin and bounds tokens without exp by file age', () => { + for (const exp of [nowMs / 1000 - 1, nowMs / 1000 + 59]) { + assert.throws( + () => + readCodexOAuthAccessToken({ + authPath: '/unused', + nowMs, + readFile: () => auth(jwt({ exp })), + }), + /sign-in has expired/, + ); + } + const noExp = jwt({ sub: 'fixture' }); + const base = { authPath: '/unused', nowMs, readFile: () => auth(noExp) }; + assert.equal( + readCodexOAuthAccessToken({ + ...base, + statFile: () => ({ mtimeMs: nowMs }), + }), + noExp, + ); + assert.throws( + () => + readCodexOAuthAccessToken({ + ...base, + statFile: () => ({ mtimeMs: nowMs - 3600_000 }), + }), + /sign-in has expired/, + ); + assert.throws( + () => + readCodexOAuthAccessToken({ + ...base, + statFile: () => { + throw new Error('missing'); + }, + }), + /sign-in is unavailable/, + ); +}); + test('Codex login resolves an installed executable before PATH and starts browser auth detached', async () => { const checked = []; assert.equal( diff --git a/src/tooling/localServices.test.mjs b/src/tooling/localServices.test.mjs index 7014659b7..408dbbbc3 100644 --- a/src/tooling/localServices.test.mjs +++ b/src/tooling/localServices.test.mjs @@ -217,15 +217,44 @@ test('Realtime handler keeps API key as default and uses OAuth only when request ); const token = routes.get('/api/realtime/token'); - assert.equal((await request(token)).status, 200); + const apiResponse = await request(token); + assert.equal(apiResponse.status, 200); + assert.equal(apiResponse.headers['x-gev-voice-auth'], 'env'); assert.equal(oauthReads, 0, 'default cloud voice does not read OAuth'); assert.equal(authorizations.at(-1), 'Bearer fixture-api-key'); - assert.equal((await request(token, { url: '/?auth=oauth' })).status, 200); + const oauthResponse = await request(token, { url: '/?auth=oauth' }); + assert.equal(oauthResponse.status, 200); + assert.equal(oauthResponse.headers['x-gev-voice-auth'], 'codex-oauth'); assert.equal(oauthReads, 1); assert.equal(authorizations.at(-1), 'Bearer fixture-oauth-token'); }); +test('a Codex OAuth pin refuses API-key minting without silently switching modes', async (t) => { + env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined); + const authorizations = []; + const routes = install( + openAiRealtimeProxy({ + realtime: { + environment: { GEV_PREFER_CODEX_OAUTH: 'true' }, + resolveApiKey: () => assert.fail('pinned OAuth cannot read an API key'), + resolveOAuthAccessToken: () => 'fixture-oauth-token', + fetchImpl: async (_url, options) => { + authorizations.push(options.headers.Authorization); + return Response.json({ value: 'fixture-ephemeral' }); + }, + }, + }), + ); + const token = routes.get('/api/realtime/token'); + const refused = await request(token); + assert.equal(refused.status, 503); + assert.equal(refused.json().code, 'CODEX_OAUTH_REQUIRED'); + assert.equal(authorizations.length, 0); + assert.equal((await request(token, { url: '/?auth=oauth' })).status, 200); + assert.deepEqual(authorizations, ['Bearer fixture-oauth-token']); +}); + test('OAuth status and token minting never expose local OAuth to remote clients', async () => { let oauthReads = 0; const routes = install( diff --git a/src/voice/gevRealtime.test.mjs b/src/voice/gevRealtime.test.mjs index 66e1b1367..f16e02a91 100644 --- a/src/voice/gevRealtime.test.mjs +++ b/src/voice/gevRealtime.test.mjs @@ -3194,18 +3194,29 @@ test('F1: the toggle still records the next-session preference while live', () = assert.match(ui.tierButton.title, /this session stays on/i); }); -test('OAuth cloud sessions hide API spend and do not trip the API spend guard', () => { +test('OAuth cloud sessions report unknown cost and usage without tripping the API spend guard', () => { const { controller, ui } = costControllerHarness(); controller.status = 'listening'; controller._cost.cloudVoiceAuth = 'oauth'; controller._cost.sessionCloudVoiceAuth = 'oauth'; controller.syncCostUi(); const before = controller.costTracker.state(); - assert.equal(ui.costValue.hidden, true); - assert.equal(controller.recordUsage(usdUsage(100)), null); + assert.equal(ui.costValue.hidden, false); + assert.equal(ui.costValue.textContent, 'COST UNKNOWN'); + assert.deepEqual(controller.recordUsage(usdUsage(100)), { + responses: 1, + input: 0, + output: 1562500, + costKnown: false, + }); const after = controller.costTracker.state(); assert.equal(after.totalUsd, before.totalUsd); assert.equal(controller.costCapStopped, false); + assert.match(ui.costValue.title, /1 response\(s\).*1562500 output tokens/); + controller.dc = {}; + controller.responseActive = true; + controller.stop(); + assert.match(ui.costValue.title, /Usage is incomplete/); }); test('F1: when idle, toggling does re-price the preview meter', () => { diff --git a/src/voice/realtimeController.js b/src/voice/realtimeController.js index d0c9f1bae..999297c27 100644 --- a/src/voice/realtimeController.js +++ b/src/voice/realtimeController.js @@ -239,7 +239,10 @@ export class GevRealtimeController extends RealtimeFacade { status: this.status, connection: this.connectionDiagnostics(), }); - if (this.dc && this.responseActive) this.costTracker.markIncomplete(); + if (this.dc && this.responseActive) { + this.costTracker.markIncomplete(); + this.syncCostUi(); + } this._connection.closeTransport(); this.stopVoiceVisualizer(); this._connection.releaseMedia(); diff --git a/src/voice/realtimeCost.js b/src/voice/realtimeCost.js index 83d19785e..0de89065c 100644 --- a/src/voice/realtimeCost.js +++ b/src/voice/realtimeCost.js @@ -19,6 +19,7 @@ export class RealtimeCost { this.voiceLimits = readStoredVoiceLimits(); this.cloudVoiceAuth = readStoredCloudVoiceAuthMode(); this.sessionCloudVoiceAuth = null; + this.oauthUsage = { responses: 0, input: 0, output: 0 }; this.costTracker = createVoiceCostTracker({ tier: this.voiceTier, limits: this.voiceLimits, @@ -60,7 +61,20 @@ export class RealtimeCost { } if (this.ui?.costValue) { const liveAuth = this.sessionCloudVoiceAuth || this.cloudVoiceAuth; - this.ui.costValue.hidden = liveAuth === 'oauth'; + if (liveAuth === 'oauth') { + const { responses, input, output } = this.oauthUsage; + this.ui.costValue.hidden = false; + this.ui.costValue.textContent = 'COST UNKNOWN'; + this.ui.costValue.dataset.level = 'unknown'; + this.ui.costValue.title = + `ChatGPT OAuth session: ${responses} response(s), ${input} input and ${output} output tokens reported. ` + + 'USD cost and the API spend cap are unavailable for this auth mode.' + + (state.incomplete + ? ' Usage is incomplete because a response was still in flight when the session ended.' + : ''); + return; + } + this.ui.costValue.hidden = false; this.ui.costValue.textContent = state.display; this.ui.costValue.dataset.level = state.level; this.ui.costValue.title = @@ -141,7 +155,15 @@ export class RealtimeCost { */ recordUsage(usage) { if (!usage) return null; - if (this.sessionCloudVoiceAuth === 'oauth') return null; + if (this.sessionCloudVoiceAuth === 'oauth') { + const count = (value) => + Number.isFinite(value) && value > 0 ? value : 0; + this.oauthUsage.responses += 1; + this.oauthUsage.input += count(usage.input_tokens); + this.oauthUsage.output += count(usage.output_tokens); + this.syncCostUi(); + return { ...this.oauthUsage, costKnown: false }; + } const state = this.costTracker.record(usage); this.syncCostUi(); if (state.warnCrossed) { @@ -198,6 +220,7 @@ export class RealtimeCost { this.voiceLimits = readStoredVoiceLimits(); this.cloudVoiceAuth = readStoredCloudVoiceAuthMode(); this.sessionCloudVoiceAuth = this.cloudVoiceAuth; + this.oauthUsage = { responses: 0, input: 0, output: 0 }; this.costCapStopped = false; // Provisional meter (tier-priced) so the readout shows $0.00 while // connecting. It is REPLACED below with one bound to the model the server From 993b1bcdb1d7871a1f39b7cabc7c16387d93dd5d Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:29:19 -0500 Subject: [PATCH 008/202] docs: add Pinokio browser performance workaround --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 88690d52f..0ff6c32e1 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,14 @@ locked dependencies, finds a free local port, and opens the app. Version 8.2 fixes the launcher installation issue; [details from the Pinokio maintainer](https://pinokio.co/posts/01m1m4p9xxm3qw7dnnpj2wr93g). +**Pinokio opens but the globe is very choppy?** Before treating that as a GPU +compatibility or minimum-hardware problem, copy the local server URL shown by +Pinokio and open the same URL in your normal hardware-accelerated browser. A +Pinokio 8.2.0 Linux report reproduced severe stutter in Pinokio's embedded view +while the same running server was smooth in an external browser ([#365](https://github.com/bilawalsidhu/gods-eye-view/issues/365)). +That isolates the browser/compositor path from the app server and your machine; +it does not establish a minimum GPU requirement. + ### Path 2 — Terminal / coding agent Use **Node.js 24.x (24.14.0 or later) or 26.x**. The setup doctor warns about From 4ab530b8a83c6f56b34612c1ea5c9bc07c34e3ea Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:29:21 -0500 Subject: [PATCH 009/202] docs: distinguish embedded browser performance --- docs/PERFORMANCE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 34b207f79..873ef8768 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -128,6 +128,12 @@ Use the same controls before attributing a difference to the application: 6. Record live object counts before attributing a difference to the client. 7. Treat a live-source outage as missing coverage, not as evidence of low client rendering cost. +8. Record whether the app is running in Pinokio's embedded view or in an + external browser. The same Pinokio 8.2.0 Linux server was reported as + severely stuttery in the embedded view and smooth when its local URL was + opened directly in a normal browser ([#365](https://github.com/bilawalsidhu/gods-eye-view/issues/365)). + Treat that as a browser/compositor-path difference until the GPU renderer + proves otherwise. ## What is not established yet From c87f007e34a627bc9554c4bfe8070267f9c5e0b5 Mon Sep 17 00:00:00 2001 From: daikaginza Date: Mon, 28 Sep 2026 11:39:19 +0700 Subject: [PATCH 010/202] fix(build): discover unit tests under server/, not only src/ npm test is the only suite CI runs, and its discovery walked src/ alone, so server/providers/transitHistory.test.mjs has never executed since it was added. Its 8 tests pass, so this was silent coverage loss rather than a hidden failure: the file reads like covered code in review and never ran anywhere. Add server/ to the searched roots, and pin the property with a test that walks the repository itself and compares the two sets, so a tree added later without updating the roots fails loudly instead of going quiet. Suite goes from 5090 to 5099 tests, 0 failures: the 8 tests in that file plus the regression test added here. --- scripts/run-unit-tests.mjs | 14 ++++++++-- src/unitTestRunner.test.mjs | 51 ++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/scripts/run-unit-tests.mjs b/scripts/run-unit-tests.mjs index 9bd7e70e0..7024ea38c 100644 --- a/scripts/run-unit-tests.mjs +++ b/scripts/run-unit-tests.mjs @@ -21,9 +21,19 @@ export function assertNode24AllocationRuntime(version = process.versions.node) { return version; } +/** + * Directories this runner searches for `*.test.mjs`, in stable order. + * + * `server/` belongs here because provider modules own tests of their own: a + * suite placed beside the module it covers must run, and `npm test` is the only + * thing CI invokes, so a directory missing from this list is a suite that never + * runs anywhere. `unitTestRunner.test.mjs` asserts the list covers every test + * file in the repository, so adding a tree without adding it here fails. + */ +export const UNIT_TEST_ROOTS = Object.freeze(['src', 'server']); + /** Discover repository unit tests in stable path order. */ export function discoverUnitTestFiles(root = process.cwd()) { - const sourceRoot = path.join(root, 'src'); const files = []; const visit = (directory) => { for (const entry of readdirSync(directory, { withFileTypes: true })) { @@ -34,7 +44,7 @@ export function discoverUnitTestFiles(root = process.cwd()) { } } }; - visit(sourceRoot); + for (const testRoot of UNIT_TEST_ROOTS) visit(path.join(root, testRoot)); return files.sort(); } diff --git a/src/unitTestRunner.test.mjs b/src/unitTestRunner.test.mjs index 002dcb796..201f6ed8e 100644 --- a/src/unitTestRunner.test.mjs +++ b/src/unitTestRunner.test.mjs @@ -1,14 +1,40 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { readFileSync, readdirSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { ALLOCATION_TEST_FILES, + UNIT_TEST_ROOTS, allocationTestArgs, assertNode24AllocationRuntime, buildUnitTestPlan, + discoverUnitTestFiles, isCalibratedAllocationRuntime, } from '../scripts/run-unit-tests.mjs'; +const repositoryRoot = fileURLToPath(new URL('..', import.meta.url)); + +/** Every `*.test.mjs` a contributor could add, found without consulting the runner. */ +function testFilesOnDisk(root) { + // Build outputs and dependencies are not places a contributor writes a test, + // and walking them would be slow enough to matter. Everything else counts. + const skip = new Set(['node_modules', 'dist', 'build', 'public']); + const found = []; + const visit = (directory) => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (entry.name.startsWith('.') || skip.has(entry.name)) continue; + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) visit(absolute); + else if (entry.isFile() && entry.name.endsWith('.test.mjs')) { + found.push(path.relative(root, absolute).split(path.sep).join('/')); + } + } + }; + visit(root); + return found.sort(); +} + test('unit runner serializes only GC-bracketed allocation microbenchmarks', () => { const ordinary = [ 'src/data/manager.test.mjs', @@ -37,6 +63,29 @@ test('unit runner serializes only GC-bracketed allocation microbenchmarks', () = ); }); +test('the runner discovers every test file in the repository', () => { + // `npm test` is the only suite CI runs, and it runs exactly what this + // function returns. A `*.test.mjs` outside the searched roots therefore never + // executes anywhere, while still reading — in review and to its author — like + // covered code. Compare the two sets rather than their sizes, so the failure + // names the files instead of only disagreeing about a count. + const discovered = discoverUnitTestFiles(repositoryRoot); + const onDisk = testFilesOnDisk(repositoryRoot); + const undiscovered = onDisk.filter((file) => !discovered.includes(file)); + assert.deepEqual( + undiscovered, + [], + `test files the runner never executes: ${undiscovered.join(', ')}`, + ); + assert.deepEqual(discovered, onDisk); + for (const file of discovered) { + assert.ok( + UNIT_TEST_ROOTS.some((root) => file.startsWith(`${root}/`)), + `${file} should sit under one of ${UNIT_TEST_ROOTS.join(', ')}`, + ); + } +}); + test('allocation runtime calibration is explicit and pinned to Node 24', () => { assert.equal(assertNode24AllocationRuntime('24.19.0'), '24.19.0'); assert.throws( From 8887ca47ebea3ca593a5c8d1956652f1739c8b8a Mon Sep 17 00:00:00 2001 From: rileygramlich Date: Wed, 23 Sep 2026 22:40:03 -0600 Subject: [PATCH 011/202] feat(air-quality): add the Air quality layer to the Weather group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reworks #568 to the shape asked for in review. Generalized from "AQHI" to a provider-backed Air quality layer, with ECCC's AQHI as the first registered provider. Readings are carried as { provider, scale, value, band } rather than a bare number, because air-quality indices are not interchangeable: AQHI runs 1-10+ on health risk, the US AQI runs 0-500 on concentration breakpoints, and Europe's EAQI is a five-step category. Naming the index a value belongs to is what lets another network be added without inventing a shared numeric meaning. Coverage is stated as "Canada only" on the provider, the layer row and the card. Adds the WEATHER panel readout (readout: true) carrying the band legend and the worst readings currently in view, with getRowControls/setRowControlsListener following the wind and cyclone cards. An unresolvable view rectangle degrades to the worst readings overall rather than to a blank card, and a rectangle crossing the antimeridian is handled. update() now takes the earthquakes layer's abort-and-guard shape: it aborts any in-flight request, passes the signal through to the source, and refuses to touch the scene when the request was superseded, aborted, or the layer was disabled while it was outstanding. disable() and destroy() abort too. Tests cover the late-response, superseded-refresh and disabled-update paths. Rebased on main: the source is registered in src/sources/reference.js, and the allocation profile was re-measured against the current worker — median 3,900 B/frame, max 4,114, 85.7 B/candidate, inside the shared 154 ceiling, budgeted at 5,200 for 26% headroom. The earlier figures no longer held. Freshness is now judged on the observation time rather than on when the last response arrived. lastUpdate only says the feed answered; a station that keeps returning a reading from four hours ago would otherwise be narrated as live by the HUD and the voice provenance envelope. getStats() reports observedAt and stale, so layerFeedState() calls the layer stale past two missed hourly publications, and degraded when a refresh fails over readings that are still good. The share-link token is 0, which is what layer-token:next allocates. Taking it moves the ledger fixtures that had hard-coded 0 as the free digit and 3 as the one after; those shift to 3 and 4. The pinned equality between the reservation ledger and LEGACY_LAYER_STATE_TOKENS also had to become "the closed legacy set plus air-quality", since that set is documented as never growing. Happy to re-pin this differently if you would rather the fixtures derive the provisional digits from nextLayerStateToken() so the next layer does not move them again. The layer and its source are registered as ./layers/air-quality and ./layers/air-quality/source, each with its own boundary group, following the perimeters pair. check:boundaries requires the package exports and the group exports to be the same set, so a layer that ships a source needs both. --- CHANGELOG.md | 10 + DATA_SOURCES.md | 9 + README.md | 1 + docs/CURRENT-STATE.md | 1 + package.json | 2 + scripts/format-scope.json | 6 + scripts/package-boundaries.json | 33 +- scripts/qa-air-quality.mjs | 171 ++++++++ src/app/constructCatalog.js | 3 + src/app/constructCatalog.test.mjs | 2 +- src/app/layers/airQuality.js | 6 + src/data/dataCredits.js | 7 + src/data/layerState.js | 5 + src/data/layerState.test.mjs | 12 +- src/data/layerStateTokenLedger.test.mjs | 63 +-- src/data/layerStateTokenReservations.json | 1 + src/layers/airQuality/index.js | 308 +++++++++++++++ src/layers/airQuality/model.js | 374 ++++++++++++++++++ src/layers/airQuality/model.test.mjs | 250 ++++++++++++ src/layers/airQuality/ownership.test.mjs | 326 +++++++++++++++ src/layers/airQuality/source.js | 78 ++++ src/overlays/worldOverlayAllocation.test.mjs | 19 + .../worldOverlayAllocation.worker.mjs | 78 +++- src/sources/reference.js | 2 + src/sources/reference.test.mjs | 3 + src/ui/layerPanel.js | 1 + 26 files changed, 1730 insertions(+), 41 deletions(-) create mode 100644 scripts/qa-air-quality.mjs create mode 100644 src/app/layers/airQuality.js create mode 100644 src/layers/airQuality/index.js create mode 100644 src/layers/airQuality/model.js create mode 100644 src/layers/airQuality/model.test.mjs create mode 100644 src/layers/airQuality/ownership.test.mjs create mode 100644 src/layers/airQuality/source.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 58103d04d..10a525546 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -600,6 +600,16 @@ of current runtime behavior, see [`docs/CURRENT-STATE.md`](docs/CURRENT-STATE.md ## [Unreleased] +- Add a keyless Air quality layer to the Weather group, with Environment and + Climate Change Canada's AQHI as the first registered provider. Readings carry + `{ provider, scale, value, band }` so another network can be added without + inventing a shared numeric meaning across indices that do not share one; the + row and readout both state that coverage is Canada only. The WEATHER card + shows the band legend and the highest readings currently in view. AQHI is + reported as an integer from 1 with an open-ended "10+", so readings round to + that published form rather than the decimal precision the API returns. + Readings older than six hours are dropped, and the layer reports `stale` + through `getStats()` so feed provenance cannot describe an aged feed as live. - Add ECMWF IFS model selection to Wind (#464, thanks @beneduzi), with model-scoped forecast-step caches, cancellation of replaced requests, and separate issue/valid timestamps. diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index ab059cebb..4e498cb6b 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -31,6 +31,7 @@ How to read this: | **The Space Devs — Launch Library 2 v2.3** | Recent launch, payload, stage, and recovery metadata for Space Missions (30d) | [The Space Devs terms of use](https://github.com/TheSpaceDevs/Tutorials/blob/main/faqs/faq_TSD.md#terms-of-use): data may be used and shared in any form; avoid forwarding it without added value; attribution is encouraged (not mandatory). [Official API limits](https://ll.thespacedevs.com/docs/): 15 unauthenticated calls/hour; optional token | "Launch Library 2 — The Space Devs" (courtesy attribution) | | **Esri World Imagery** (ArcGIS Online tile service) | The keyless satellite basemap — the default landing when no Google/ion credential is configured, and the "Esri Satellite" map stack | [Esri Master Agreement](https://www.esri.com/en-us/legal/terms/full-master-agreement): the public World Imagery service is usable in public-facing apps with attribution; no key is required for this classic endpoint, but Esri governs and can change access — an app at scale should review current ArcGIS Location Platform terms | "Powered by Esri — Source: Esri, Maxar, Earthstar Geographics, and the GIS User Community" (provider carries the service's own credit line) | | **USGS** | Earthquakes | U.S. public domain | "Data courtesy of the U.S. Geological Survey" | +| **Environment and Climate Change Canada** (MSC GeoMet OGC API) | Air quality (AQHI) — Canadian station observations | [ECCC Data Servers End-use Licence](https://eccc-msc.github.io/open-data/licence/readme_en/) (open, attribution required, no endorsement implied) | "Contains information licensed under the Environment and Climate Change Canada Data Servers End-use Licence" | | **NASA GIBS** (WMTS tiles + Worldview Snapshots) and **NASA CMR** (granule catalog) | Recent Imagery layer: HLS S30/L30 (Sentinel-2, Landsat 8/9) and VIIRS true-colour tiles draped on the map, per-day thumbnails and PNG exports; the HLS granule catalog (dates, cloud, footprints) for a selected box | U.S. public domain (NASA data); keyless, browser-direct (CORS `*`); no bulk use — one 30-day catalog query per box, tiles bounded to the box, six tile requests in flight | "We acknowledge the use of imagery provided by services from NASA's Global Imagery Browse Services (GIBS), part of NASA's Earth Science Data and Information System (ESDIS)." — [gibs.earthdata.nasa.gov](https://gibs.earthdata.nasa.gov), [HLS](https://lpdaac.usgs.gov/products/hlss30v002/) | | **NIFC WFIGS** (Wildland Fire Interagency Geospatial Services, ArcGIS feature service) | Fire Perimeters layer: current interagency wildfire incident perimeters, containment, and incident facts | U.S. public domain (interagency wildland-fire data published through the [NIFC Open Data portal](https://data-nifc.opendata.arcgis.com/)); fetched keyless through `/api/fire-perimeters` (5-minute server cache) with `exceededTransferLimit` paging | "Wildfire perimeters: National Interagency Fire Center (WFIGS)" | | **InciWeb** (inciweb.wildfire.gov) | Per-incident "InciWeb ↗" card links to official incident pages (narratives, evacuation/closure notices) | U.S. government public incident information. Links are matched best-effort by incident name and state, with incident page origin and update times checked on selection. Same-origin `/api/fire-perimeters/inciweb` routes cache the catalog for 1 hour and publication metadata for 30 minutes; linked pages remain InciWeb content | "Incident information: InciWeb (inciweb.wildfire.gov)" | @@ -72,6 +73,14 @@ The optional Local ADS-B layer and the broadcast-FM receiver use radio you recei ### Notes on the live sources +- **Air quality (ECCC AQHI).** Keyless. Read from the MSC GeoMet OGC API (`api.weather.gc.ca`): the `aqhi-stations` catalog plus `aqhi-observations-realtime`. **Coverage is Canada only** — ECCC is the single registered provider, and the layer row and readout card both say so. Roughly 134 stations across ECCC's five administrative zones (Atlantic, Quebec, Ontario, Prairies & North, Pacific & Yukon). AQHI publishes hourly, so readings poll every 20 minutes and the station catalog, which changes a few times a year, is refetched only every 12 hours. Observations are requested with `latest=true`; without it the collection returns the full history, roughly 2,000 rows nationally. Readings older than six hours are dropped rather than displayed as current. + + Readings are stored as `{ provider, scale, value, band }` rather than a bare number, because air-quality indices are not interchangeable: AQHI runs 1-10+ on health risk, the US AQI runs 0-500 on concentration breakpoints, and Europe's EAQI is a five-step category. Naming the index a value belongs to is what lets another network be added later without inventing a shared numeric meaning that does not exist. + + AQHI is reported by ECCC as an integer from 1 to 10 with an open-ended "10+" above that. The API returns the underlying decimal (1.08, 2.32); the layer rounds to the published form rather than displaying a precision the index does not claim. + + **Quebec is thinly represented**, and that is upstream rather than a filter here: the province runs its own Info-Smog program, so only a handful of Quebec stations appear in ECCC's national feed. The reporting station count also varies hour to hour as stations report or fall silent. + - **Google Maps Platform.** You supply your own API key and are bound by [Google's ToS](https://cloud.google.com/maps-platform/terms). Google Maps Content (tiles, geocodes, places) **may not be cached, stored, rehosted, or committed** — this app only ever uses it live, which is the compliant pattern. The "Google" attribution is displayed on the globe and must stay visible. Restrict your key (see [SECURITY.md](SECURITY.md)). - **OpenSky Network.** Its license is **non-commercial**, and operational use of the REST API in a live product can require a prior written agreement with OpenSky — even for non-profit/government use. If you deploy this commercially, contact OpenSky for your own terms. The flights layer is a toggle and runs anonymously by default. - **adsb.lol flight fallback.** When OpenSky is unavailable and no last-good OpenSky response exists, the server requests a cached, capped 250 nm adsb.lol point snapshot around the current camera subpoint. This is regional observed context, not worldwide completeness; provenance is exposed in the Flights stats/context row. Military ICAOs remain reconciled through the existing dedicated military registry rather than duplicated. diff --git a/README.md b/README.md index f6a3ae886..9f69fdbd6 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,7 @@ Nineteen layers and map sources. **Seventeen have a keyless path.** Some offer a | 🚢 **Live Vessels** | Thousands of ships worldwide | AISStream | 🟡 | | 🛰️ **Satellites** | 838-object catalog, color-coded by class with a live legend — the **DENSE** chip drops in the whole Starlink shell | CelesTrak | 🟢 | | 🌍 **Earthquakes** | Global seismic activity, last 24h | USGS | 🟢 | +| ◍ **Air quality** | Live AQHI on the published 1–10+ health-risk scale, with a Weather-panel card showing the band legend and the highest readings in view. **Canada only** — ECCC is the first registered provider | ECCC MSC GeoMet | 🟢 | | 🚗 **Traffic** | Simulated vehicles on OSM roads. With TomTom, live flow speeds drive the simulation and congestion colors below ~8 km; individual vehicle positions are not live observations | TomTom + OSM | 🟢 simulation · 🟡 live flow speeds | | 📹 **CCTV Mesh** | ~3,600 public cameras projected _into_ the 3D space — Austin · Texas (TxDOT) · California (Caltrans) · London (TfL) · Ontario (511) · Finland (Fintraffic) · British Columbia (DriveBC) · Estonia (Tallinn, Tarktee) · Delaware (DelDOT live video) · New South Wales (Live Traffic NSW) · Calgary. Positions are published; poses are estimated priors **you calibrate by dragging a gizmo on the camera itself** | City APIs | 🟢 | | 📷 **Mapped ALPR Cameras** | License-plate-reader camera locations tagged by OpenStreetMap contributors, loaded one city-sized view at a time, with **SHOW NEAREST**. Locations and tags only: no plate data, no video | OpenStreetMap (incl. DeFlock mapping) | 🟢 | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 22ce1eb3b..cdaa687b8 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -2848,6 +2848,7 @@ its criteria cannot be silently ignored. | Live AIS Vessels 🚢 | AISStream websocket | `src/data/aisLiveVessels.js` | `/api/ais-live` | 60s (+800ms visibility pass) | | Mapped Installations ⌖ | OpenFreeMap military areas; optional configured Overpass names and Google Places search | `src/data/militaryInstallations.js` | OpenFreeMap tiles, `/api/military-installations`, `/api/google/text-search` | viewport-driven; transient errors back off, missing capability switches to tiles | | Earthquakes | USGS | `src/data/earthquakes.js` | — | 60s | +| Air quality | ECCC MSC GeoMet OGC API (AQHI, Canada only) | `src/layers/airQuality/` | — | 20 min readings, 12 h station catalog | | Satellites | CelesTrak | `src/data/satellites.js` | `/api/celestrak` | 120s | | Space Missions (30d) | Launch Library 2 + CelesTrak | `src/data/rocketLaunches.js` | `/api/launches` + `/api/celestrak/active` | 5 min | | Traffic | Selectable TomTom / OpenStreetMap / Hybrid roads; optional TomTom flow (BYOK) | `src/data/traffic.js` | browser-direct OpenFreeMap tiles; `/api/tomtom` for flow | viewport-driven; capped tile cache | diff --git a/package.json b/package.json index e48881c6d..eca8aa8b7 100644 --- a/package.json +++ b/package.json @@ -168,6 +168,7 @@ "./layers/bikeshare": "./src/layers/bikeshare/index.js", "./layers/directions": "./src/layers/directions/index.js", "./layers/cctv": "./src/layers/cctv/index.js", + "./layers/air-quality": "./src/layers/airQuality/index.js", "./layers/alpr": "./src/layers/alpr/index.js", "./layers/radio": "./src/layers/radio/index.js", "./layers/submarine-cables": "./src/layers/submarineCables/index.js", @@ -228,6 +229,7 @@ "./layers/alpr/source": "./src/layers/alpr/source.js", "./layers/bikeshare/source": "./src/layers/bikeshare/source.js", "./layers/cctv/source": "./src/layers/cctv/source.js", + "./layers/air-quality/source": "./src/layers/airQuality/source.js", "./layers/earthquakes/source": "./src/layers/earthquakes/source.js", "./layers/perimeters/source": "./src/layers/perimeters/source.js", "./layers/firms/source": "./src/layers/firms/source.js", diff --git a/scripts/format-scope.json b/scripts/format-scope.json index cd836a5cb..2e27bf753 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -346,6 +346,12 @@ "src/layers/vessels/tracking.js", "src/layers/vessels/ownership.test.mjs", "src/data/earthquakes.js", + "src/app/layers/airQuality.js", + "src/layers/airQuality/index.js", + "src/layers/airQuality/model.js", + "src/layers/airQuality/model.test.mjs", + "src/layers/airQuality/ownership.test.mjs", + "src/layers/airQuality/source.js", "src/layers/earthquakes/index.js", "src/layers/earthquakes/model.js", "src/layers/earthquakes/source.js", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 276b9e0db..67f304f3f 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -2027,7 +2027,11 @@ "src/sources/requestDeadline.js", "src/data/local_data/osm_military_names/names.json", "src/layers/installations/namedMarkers.js", - "src/data/infrastructureOverlayEntry.js" + "src/data/infrastructureOverlayEntry.js", + "src/app/layers/airQuality.js", + "src/layers/airQuality/index.js", + "src/layers/airQuality/model.js", + "src/layers/airQuality/source.js" ], "external": [ "hls.js", @@ -2511,7 +2515,11 @@ "src/sources/requestDeadline.js", "src/data/local_data/osm_military_names/names.json", "src/layers/installations/namedMarkers.js", - "src/data/infrastructureOverlayEntry.js" + "src/data/infrastructureOverlayEntry.js", + "src/app/layers/airQuality.js", + "src/layers/airQuality/index.js", + "src/layers/airQuality/model.js", + "src/layers/airQuality/source.js" ], "external": [ "hls.js", @@ -2853,7 +2861,9 @@ "src/sources/httpBody.js", "src/layers/perimeters/source.js", "src/layers/perimeters/records.js", - "src/layers/submarineCables/bundledSource.js" + "src/layers/submarineCables/bundledSource.js", + "src/layers/airQuality/model.js", + "src/layers/airQuality/source.js" ], "external": [] }, @@ -3072,5 +3082,22 @@ "src/layers/perimeters/source.js" ], "external": [] + }, + "air-quality": { + "exports": ["./layers/air-quality"], + "modules": [ + "src/layers/airQuality/index.js", + "src/layers/airQuality/model.js", + "src/layers/airQuality/source.js" + ], + "external": ["cesium"] + }, + "air-quality-source": { + "exports": ["./layers/air-quality/source"], + "modules": [ + "src/layers/airQuality/model.js", + "src/layers/airQuality/source.js" + ], + "external": [] } } diff --git a/scripts/qa-air-quality.mjs b/scripts/qa-air-quality.mjs new file mode 100644 index 000000000..68e6bd505 --- /dev/null +++ b/scripts/qa-air-quality.mjs @@ -0,0 +1,171 @@ +/** + * qa-air-quality.mjs — headless proof for the Air quality layer. + * + * Checks what unit tests cannot: that the live ECCC feed answers, that readings + * reach the app in the provider-shaped form other networks will share, and that + * the Weather readout descriptor carries a legend and an in-view list. + * + * AQHI is reported by ECCC as an integer from 1 with "10+" above ten. The API + * returns the underlying decimal, so a reading arriving as 1.08 — or a station + * with no reading surfacing as a confident 1 — is a real defect this catches. + * + * Run: node scripts/qa-air-quality.mjs --url http://localhost:4173 + */ + +import puppeteer from 'puppeteer'; + +const argv = process.argv.slice(2); +const getOpt = (name, dflt) => { + const i = argv.indexOf(name); + return i >= 0 && argv[i + 1] ? argv[i + 1] : dflt; +}; +const APP_URL = getOpt('--url', 'http://localhost:4173'); + +const results = []; +function record(name, ok, detail) { + results.push({ name, ok, detail }); + const tag = ok ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m'; + console.log(` [${tag}] ${name}${detail ? ` — ${detail}` : ''}`); +} +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +async function main() { + let exitCode = 0; + const consoleErrors = []; + const browser = await puppeteer.launch({ + headless: 'new', + args: [ + '--use-gl=swiftshader', + '--enable-unsafe-swiftshader', + '--no-sandbox', + '--disable-dev-shm-usage', + ], + defaultViewport: { width: 1400, height: 900 }, + }); + + try { + const page = await browser.newPage(); + page.on('pageerror', (e) => consoleErrors.push(String(e).slice(0, 160))); + page.on('console', (m) => { + if (m.type() === 'error') consoleErrors.push(m.text().slice(0, 160)); + }); + + await page.goto(APP_URL, { waitUntil: 'domcontentloaded', timeout: 90000 }); + await page.waitForFunction( + () => window.__godsEyeView?.viewer && window.__godsEyeView?.dataManager, + { timeout: 90000 }, + ); + await sleep(1500); + + const stats = await page.evaluate(async () => { + const dm = window.__godsEyeView.dataManager; + await dm.setEnabled('air-quality', true); + const mod = dm.layers.get('air-quality')?.module; + if (!mod) return { missing: true }; + let s = null; + for (let i = 0; i < 45; i++) { + await new Promise((r) => setTimeout(r, 1000)); + s = mod.getStats(); + if (s.count > 0 || s.error) break; + } + return s; + }); + + const loaded = !stats?.missing && !stats?.error && stats?.count > 0; + record('stations report a current index', loaded, + `count=${stats?.count} error=${stats?.error}`); + if (!loaded) exitCode = 1; + + const records = await page.evaluate( + () => + window.__godsEyeView.dataManager + .layers.get('air-quality') + ?.module.getAnalystRecords(500) || [], + ); + + // The provider-neutral contract other networks will share. + const shaped = records.every( + (r) => r.provider && r.scale && Number.isFinite(r.value) && r.band, + ); + record('every reading carries provider, scale, value and band', shaped, + records.length + ? `sample={provider:${records[0].provider}, scale:${records[0].scale}, value:${records[0].value}, band:${records[0].band}}` + : 'no records'); + if (!shaped) exitCode = 1; + + const published = records.every((r) => Number.isInteger(r.value) && r.value >= 1); + record('every AQHI value is in the published integer form, floored at 1', published, + `values=${[...new Set(records.map((r) => r.value))].sort((a, b) => a - b).join(',')}`); + if (!published) exitCode = 1; + + const zones = [...new Set(records.map((r) => r.zone).filter(Boolean))].sort(); + record('coverage spans multiple ECCC administrative zones', zones.length >= 3, + `zones=${zones.join(',') || 'none'} stations=${records.length}`); + if (zones.length < 3) exitCode = 1; + + // The card's whole point is "highest readings in view", so put the camera + // over Canada before asserting on it — the default view is over Austin, + // where an empty in-view list is correct and proves nothing. + await page.evaluate(() => { + const gev = window.__godsEyeView; + const ellipsoid = gev.viewer.scene.globe.ellipsoid; + const d2r = Math.PI / 180; + try { + gev.viewer.camera.cancelFlight(); + } catch { + /* no flight in progress */ + } + gev.viewer.camera.setView({ + destination: ellipsoid.cartographicToCartesian({ + longitude: -100 * d2r, + latitude: 55 * d2r, + height: 4_000_000, + }), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + gev.viewer.scene.requestRender?.(); + }); + await sleep(2500); + + const controls = await page.evaluate( + () => + window.__godsEyeView.dataManager + .layers.get('air-quality') + ?.module.getRowControls?.() || null, + ); + const readoutOk = + controls?.readout === true && + Array.isArray(controls?.legend?.colors) && + controls.legend.colors.length === controls.legend.labels.length && + Array.isArray(controls?.list?.items) && + controls.list.items.length > 0 && + controls.list.items.every((item) => item.id && item.label && item.value) && + /Canada only/.test(controls?.summary?.coverage || ''); + record('the Weather readout lists the worst readings in view, with a legend', readoutOk, + controls + ? `bands=${controls.legend?.labels?.join('/')} inView=${controls.list?.items?.length} coverage="${controls.summary?.coverage}"` + : 'no descriptor'); + if (!readoutOk) exitCode = 1; + + record('no console errors', consoleErrors.length === 0, + consoleErrors.length ? consoleErrors.slice(0, 2).join(' | ') : 'clean'); + if (consoleErrors.length) exitCode = 1; + } catch (e) { + console.error('\x1b[31mHarness error:\x1b[0m', e); + exitCode = 3; + } finally { + await browser.close(); + } + + const pass = results.filter((r) => r.ok === true).length; + const fail = results.filter((r) => r.ok === false).length; + console.log('\n' + '─'.repeat(60)); + console.log(` RESULT: ${pass} passed, ${fail} failed`); + console.log('─'.repeat(60) + '\n'); + process.exit(exitCode || (fail > 0 ? 1 : 0)); +} + +main().catch((e) => { + console.error(e); + process.exit(3); +}); diff --git a/src/app/constructCatalog.js b/src/app/constructCatalog.js index 7ed03d0d2..a9728f317 100644 --- a/src/app/constructCatalog.js +++ b/src/app/constructCatalog.js @@ -23,6 +23,7 @@ import { createApplicationLocalAdsb } from './layers/localAdsb.js'; import { createApplicationAwareness } from './layers/militaryAwareness.js'; import { createApplicationFirms } from './layers/firms.js'; import { createApplicationEarthquakes } from './layers/earthquakes.js'; +import { createApplicationAirQuality } from './layers/airQuality.js'; import { createApplicationFirePerimeters } from './layers/perimeters.js'; import { createApplicationCables } from './layers/submarineCables.js'; import { createInfrastructureLayers } from '../data/infrastructure.js'; @@ -53,6 +54,7 @@ const SOURCE_METHODS = Object.freeze({ weather: ['getSnapshot'], cyclones: ['getSnapshot'], earthquakes: ['getSnapshot'], + 'air-quality': ['getSnapshot'], 'fire-perimeters': ['getSnapshot'], cables: ['fetch'], }); @@ -144,6 +146,7 @@ export function createApplicationCatalog({ ...(resolveAsset ? { resolveAsset } : {}), }), createApplicationEarthquakes({ source: sources.earthquakes }), + createApplicationAirQuality({ source: sources['air-quality'] }), createApplicationFirePerimeters({ source: sources['fire-perimeters'], }), diff --git a/src/app/constructCatalog.test.mjs b/src/app/constructCatalog.test.mjs index 6043b1262..3a802edbe 100644 --- a/src/app/constructCatalog.test.mjs +++ b/src/app/constructCatalog.test.mjs @@ -39,7 +39,7 @@ test('catalogs construct distinct layers and classification from their supplied signal: b.signal, surface: fixtureSurface(b.signal), }); - assert.equal(first.layers.length, 29); + assert.equal(first.layers.length, 30); assert.ok(first.get('local-adsb'), 'Local ADS-B is registered'); assert.deepEqual( first.metadata.find(({ id }) => id === 'local-adsb'), diff --git a/src/app/layers/airQuality.js b/src/app/layers/airQuality.js new file mode 100644 index 000000000..1f25c06d9 --- /dev/null +++ b/src/app/layers/airQuality.js @@ -0,0 +1,6 @@ +import { createAirQualityLayer } from '../../layers/airQuality/index.js'; +import { overlayHost } from './overlayHost.js'; +/** Wire air-quality readings to the application overlay host. */ +export function createApplicationAirQuality(options) { + return createAirQualityLayer({ overlayHost, ...options }); +} diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index f9fc23b70..f47c454a2 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -162,6 +162,13 @@ export const DATA_CREDITS = [ }, OSM_CREDIT, OPENMAPTILES_CREDIT, + { + key: 'eccc-aqhi', + html: + 'Air quality (AQHI): Environment and Climate Change Canada — MSC GeoMet ' + + '(eccc-msc.github.io/open-data) ' + + '· ECCC Data Servers End-use Licence', + }, { key: 'overture-military-names', html: 'Military area names: Overture Maps Foundation (ODbL)', diff --git a/src/data/layerState.js b/src/data/layerState.js index eccc555f6..ea43cfd15 100644 --- a/src/data/layerState.js +++ b/src/data/layerState.js @@ -553,6 +553,11 @@ export const LAYER_STATE_TOKEN_RESERVATIONS = * owns stable URL ordering. */ export const LAYER_STATE_REGISTRY = Object.freeze([ + Object.freeze({ + id: 'air-quality', + token: '0', + disposition: 'enabled-only', + }), Object.freeze({ id: 'ais-live-vessels', token: 'a', diff --git a/src/data/layerState.test.mjs b/src/data/layerState.test.mjs index 1b504a439..3bebbc7b3 100644 --- a/src/data/layerState.test.mjs +++ b/src/data/layerState.test.mjs @@ -199,8 +199,8 @@ function encode(state) { test('production registry is exact, canonical, and rejects incomplete contracts', async () => { assert.equal(validateLayerStateRegistry(), true); - assert.equal(REGISTERED_LAYER_IDS.length, 28); - assert.equal(new Set(REGISTERED_LAYER_IDS).size, 28); + assert.equal(REGISTERED_LAYER_IDS.length, 29); + assert.equal(new Set(REGISTERED_LAYER_IDS).size, 29); assert.ok(REGISTERED_LAYER_IDS.includes('transit')); assert.deepEqual(REGISTERED_LAYER_IDS, [...REGISTERED_LAYER_IDS].sort()); assert.deepEqual(LEGACY_LAYER_STATE_TOKENS, { @@ -239,7 +239,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts' for (const [id, token] of Object.entries(LEGACY_LAYER_STATE_TOKENS)) { assert.equal(LAYER_STATE_TOKEN_RESERVATIONS[id], token); } - assert.equal(nextLayerStateToken(), '0'); + assert.equal(nextLayerStateToken(), '3'); assert.equal( nextLayerStateToken({ ...LAYER_STATE_TOKEN_RESERVATIONS, alpha: '0', bravo: '3' }), '4', @@ -247,7 +247,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts' const digitsExhausted = { ...LAYER_STATE_TOKEN_RESERVATIONS, ...Object.fromEntries( - [...'03456789'].map((digit) => [`prior-${digit}`, digit]), + [...'3456789'].map((digit) => [`prior-${digit}`, digit]), ), }; assert.equal(nextLayerStateToken(digitsExhausted), '00'); @@ -298,7 +298,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts' assert.equal( validateLayerStateAllocations( LAYER_STATE_TOKEN_RESERVATIONS, - { ...LAYER_STATE_TOKEN_RESERVATIONS, future: '0', next: '3' }, + { ...LAYER_STATE_TOKEN_RESERVATIONS, future: '3', next: '4' }, ), true, ); @@ -307,7 +307,7 @@ test('production registry is exact, canonical, and rejects incomplete contracts' ...LAYER_STATE_TOKEN_RESERVATIONS, future: '00', }), - /next free token 0/, + /next free token 3/, ); const beforeLastDigit = { ...digitsExhausted }; delete beforeLastDigit['prior-9']; diff --git a/src/data/layerStateTokenLedger.test.mjs b/src/data/layerStateTokenLedger.test.mjs index 7b0a9263c..28223762e 100644 --- a/src/data/layerStateTokenLedger.test.mjs +++ b/src/data/layerStateTokenLedger.test.mjs @@ -122,13 +122,20 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed { ...parseLayerStateTokenReservations(reservationRows) }, { ...LAYER_STATE_TOKEN_RESERVATIONS }, ); - assert.equal(Object.keys(LAYER_STATE_TOKEN_RESERVATIONS).length, 28); + assert.equal(Object.keys(LAYER_STATE_TOKEN_RESERVATIONS).length, 29); + // The legacy assignments are a closed compatibility set that never grows, so + // the ledger is exactly that set plus every token reserved since. assert.deepEqual( { ...LAYER_STATE_TOKEN_RESERVATIONS }, - { ...LEGACY_LAYER_STATE_TOKENS }, + { ...LEGACY_LAYER_STATE_TOKENS, 'air-quality': '0' }, ); assert.throws( - () => parseLayerStateTokenReservations(reservationRows.slice(1)), + () => + parseLayerStateTokenReservations( + // Dropping any legacy row must fail. This used to slice off row 0, + // which is no longer a legacy reservation. + reservationRows.filter(([id]) => id !== 'ais-live-vessels'), + ), /Missing or changed legacy/, ); assert.throws( @@ -145,9 +152,9 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed /Duplicate layer-state token reservation/, ); assert.equal( - parseLayerStateTokenReservations([...reservationRows, ['future', '0']]) + parseLayerStateTokenReservations([...reservationRows, ['future', '3']]) .future, - '0', + '3', ); assert.throws( () => @@ -165,14 +172,14 @@ test('reservation ledger is complete, pinned, and rejects duplicate or malformed test('published retired digits and competing PRs advance the merge-time allocation', () => { withPublishedBase( - { rows: [...reservationRows, ['retired-layer', '0']] }, + { rows: [...reservationRows, ['retired-layer', '3']] }, (cwd) => { const published = readPublishedLayerStateReservations('HEAD', cwd); - assert.equal(nextLayerStateToken(published), '3'); + assert.equal(nextLayerStateToken(published), '4'); assert.equal( validateLayerStateAllocations(published, { ...published, - newcomer: '3', + newcomer: '4', }), true, ); @@ -182,7 +189,7 @@ test('published retired digits and competing PRs advance the merge-time allocati ...published, competing: '0', }), - /next free token 3/, + /next free token 4/, ); assert.throws( () => @@ -196,8 +203,8 @@ test('published retired digits and competing PRs advance the merge-time allocati ); }); -test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => { - withPublishedBase({ rows: [...reservationRows, ['pr-a', '0']] }, (cwd) => { +test('PR B manually replaces provisional 3 with 4 after PR A publishes 3', () => { + withPublishedBase({ rows: [...reservationRows, ['pr-a', '3']] }, (cwd) => { mkdirSync(path.join(cwd, 'scripts')); const fixtureChecker = path.join( cwd, @@ -218,8 +225,8 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => writeCodecFixture( cwd, - [...reservationRows, ['pr-b', '0']], - [{ id: 'pr-b', token: '0' }], + [...reservationRows, ['pr-b', '3']], + [{ id: 'pr-b', token: '3' }], ); const beforeRebase = runCheck(); assert.equal(beforeRebase.status, 1); @@ -227,10 +234,10 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => writeCodecFixture( cwd, - [...reservationRows, ['pr-a', '0'], ['pr-b', '0']], + [...reservationRows, ['pr-a', '3'], ['pr-b', '3']], [ - { id: 'pr-a', token: '0' }, - { id: 'pr-b', token: '0' }, + { id: 'pr-a', token: '3' }, + { id: 'pr-b', token: '3' }, ], ); const staleRebase = runCheck(); @@ -238,16 +245,16 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => assert.match(staleRebase.stderr, /Duplicate layer-state token reservation/); const published = readPublishedLayerStateReservations('HEAD', cwd); - assert.equal(nextLayerStateToken(published), '3'); - const validBaselineRows = [...reservationRows, ['pr-a', '0']]; - writeCodecFixture(cwd, validBaselineRows, [{ id: 'pr-a', token: '0' }]); + assert.equal(nextLayerStateToken(published), '4'); + const validBaselineRows = [...reservationRows, ['pr-a', '3']]; + writeCodecFixture(cwd, validBaselineRows, [{ id: 'pr-a', token: '3' }]); const helper = spawnSync( process.execPath, [realpathSync(fixtureNext), 'pr-b'], { cwd, encoding: 'utf8' }, ); assert.equal(helper.status, 0, helper.stderr); - assert.match(helper.stdout, /pr-b: 3/); + assert.match(helper.stdout, /pr-b: 4/); assert.deepEqual( JSON.parse(readFileSync(path.join(cwd, LEDGER_PATH), 'utf8')), validBaselineRows, @@ -255,20 +262,20 @@ test('PR B manually replaces provisional 0 with 3 after PR A publishes 0', () => ); writeCodecFixture( cwd, - [...reservationRows, ['pr-a', '0'], ['pr-b', '3']], + [...reservationRows, ['pr-a', '3'], ['pr-b', '4']], [ - { id: 'pr-a', token: '0' }, - { id: 'pr-b', token: '3' }, + { id: 'pr-a', token: '3' }, + { id: 'pr-b', token: '4' }, ], ); const corrected = runCheck(); assert.equal(corrected.status, 0, corrected.stderr); - assert.match(corrected.stdout, /29 published, 1 new/); + assert.match(corrected.stdout, /30 published, 1 new/); }); }); test('allocation batches cross the last digit and base-36 pair boundaries in order', () => { - const digitRows = [...'03456789'].map((digit) => [`retired-${digit}`, digit]); + const digitRows = [...'3456789'].map((digit) => [`retired-${digit}`, digit]); const allDigits = parseLayerStateTokenReservations([ ...reservationRows, ...digitRows, @@ -320,7 +327,7 @@ test('allocation batches cross the last digit and base-36 pair boundaries in ord test('an isolated valid two-character fixture round-trips an l field beyond the old 64-character cap', async () => { const cwd = mkdtempSync(path.join(tmpdir(), 'gev-layer-token-width-')); try { - const priorDigits = [...'03456789'].map((digit) => [ + const priorDigits = [...'3456789'].map((digit) => [ `qa-prior-digit-${digit}`, digit, ]); @@ -357,7 +364,7 @@ test('an isolated valid two-character fixture round-trips an l field beyond the const params = new URLSearchParams([['v', '2']]); codec.encodeLayerStateParams(params, state); assert.ok(params.get('l').length > 64, 'fixture must cross the old cap'); - assert.equal(params.get('l').length, 67); + assert.equal(params.get('l').length, 69); const restored = codec.decodeLayerStateParams(params); assert.deepEqual(restored?.enabledLayerIds, expectedLayerIds); assert.deepEqual(restored?.options, state.options); @@ -375,7 +382,7 @@ test('checker reads a complete future base ledger and rejects retired-token reus }, (cwd) => { const published = readPublishedLayerStateReservations('HEAD', cwd); - assert.equal(Object.keys(published).length, 29); + assert.equal(Object.keys(published).length, 30); assert.equal(published['retired-layer'], '00'); assert.throws( () => diff --git a/src/data/layerStateTokenReservations.json b/src/data/layerStateTokenReservations.json index 600358c7c..7329ad59f 100644 --- a/src/data/layerStateTokenReservations.json +++ b/src/data/layerStateTokenReservations.json @@ -1,4 +1,5 @@ [ + ["air-quality", "0"], ["ais-live-vessels", "a"], ["alpr-cameras", "p"], ["bhote-koshi-2026", "h"], diff --git a/src/layers/airQuality/index.js b/src/layers/airQuality/index.js new file mode 100644 index 000000000..43a76f857 --- /dev/null +++ b/src/layers/airQuality/index.js @@ -0,0 +1,308 @@ +import * as Cesium from 'cesium'; +import { + AIR_QUALITY_BANDS, + AIR_QUALITY_OVERLAY_SOURCE_ID, + AIR_QUALITY_OVERLAY_COHORT_LIMIT, + AIR_QUALITY_OVERLAY_COLLISION_CAPACITY, + AIR_QUALITY_PROVIDERS, + createAirQualityOverlayEntry, + mapAnalystRecord, + newestObservation, + readingsAreStale, + readingsInView, + selectAirQualityOverlayCohort, +} from './model.js'; +export * from './model.js'; +export { createEcccAirQualitySource } from './source.js'; + +/** Station markers are a point reading, not an extent. */ +const STATION_MARKER_RADIUS_M = 6_000; +/** Readings listed in the readout card. */ +const READOUT_LIMIT = 5; + +/** Own one air-quality display, its refresh lifecycle, and its readout card. */ +export function createAirQualityLayer({ source, overlayHost } = {}) { + if (typeof source?.getSnapshot !== 'function') + throw new TypeError('Air quality requires a snapshot source'); + if (!overlayHost) throw new TypeError('Air quality requires an overlay host'); + let _viewer = null; + let _request = null; + let _dataSource = null; + let _readings = []; + let _count = 0; + let _lastUpdate = null; + let _lastError = null; + let _enabled = false; + let rowControlsListener = null; + + const notify = () => rowControlsListener?.(); + + /** The camera's lon/lat rectangle in degrees, or null when unresolvable. */ + const viewRectangle = () => { + const rectangle = _viewer?.camera?.computeViewRectangle?.(); + if (!rectangle) return null; + const toDeg = Cesium.Math.toDegrees; + return { + west: toDeg(rectangle.west), + south: toDeg(rectangle.south), + east: toDeg(rectangle.east), + north: toDeg(rectangle.north), + }; + }; + + const layer = { + id: 'air-quality', + name: 'Air quality', + icon: '◍', + source: 'ECCC MSC GeoMet', + updateInterval: 1_200_000, + + init(viewer) { + if (_viewer) throw new Error('Air quality layer is already initialized'); + _viewer = viewer; + _dataSource = new Cesium.CustomDataSource('air-quality'); + _dataSource.show = false; + viewer.dataSources.add(_dataSource); + _readings = []; + _count = 0; + _lastUpdate = null; + _lastError = null; + _enabled = false; + overlayHost.setVisible(AIR_QUALITY_OVERLAY_SOURCE_ID, false); + }, + + enable() { + _enabled = true; + if (_dataSource) _dataSource.show = true; + overlayHost.setVisible(AIR_QUALITY_OVERLAY_SOURCE_ID, true); + notify(); + }, + + disable() { + // Abort in flight: a response arriving after disable must not redraw. + _request?.abort(); + _request = null; + _enabled = false; + if (_dataSource) _dataSource.show = false; + overlayHost.clearSource(AIR_QUALITY_OVERLAY_SOURCE_ID); + overlayHost.setVisible(AIR_QUALITY_OVERLAY_SOURCE_ID, false); + notify(); + }, + + async update() { + if (!_enabled || !_dataSource) return false; + _request?.abort(); + const request = new AbortController(); + _request = request; + try { + const readings = await source.getSnapshot({ signal: request.signal }); + // A superseded or aborted request must not touch the scene, and neither + // must one that lands after the layer was switched off. + if (request.signal.aborted || _request !== request || !_enabled) + return false; + _readings = readings; + this._render(); + _count = _readings.length; + _lastUpdate = Date.now(); + _lastError = null; + notify(); + return true; + } catch (error) { + if (request.signal.aborted || _request !== request) return false; + _lastError = error?.message || 'Air quality request failed'; + console.warn('[Data:AirQuality] Refresh failed:', _lastError); + notify(); + return false; + } finally { + if (_request === request) _request = null; + } + }, + + /** Rebuild entities and overlay labels from the current readings. */ + _render() { + if (!_dataSource) return; + _dataSource.entities.removeAll(); + const overlayEntries = []; + for (const reading of _readings) { + const position = Cesium.Cartesian3.fromDegrees( + reading.lon, + reading.lat, + ); + const color = Cesium.Color.fromCssColorString(reading.color); + _dataSource.entities.add( + new Cesium.Entity({ + id: `air-quality:${reading.id}`, + position, + ellipse: { + // Static axes, as in the earthquake layer: a per-frame axis + // re-tessellates the clamped ground primitive every frame. + semiMajorAxis: STATION_MARKER_RADIUS_M, + semiMinorAxis: STATION_MARKER_RADIUS_M, + material: new Cesium.ColorMaterialProperty(color.withAlpha(0.35)), + outline: true, + outlineColor: color.withAlpha(0.9), + outlineWidth: 2, + heightReference: Cesium.HeightReference.CLAMP_TO_GROUND, + }, + properties: { + provider: reading.provider, + scale: reading.scale, + value: reading.value, + band: reading.band, + name: reading.name, + zone: reading.zone, + risk: reading.risk, + observedMs: reading.observedMs, + note: reading.note, + lat: reading.lat, + lon: reading.lon, + }, + }), + ); + overlayEntries.push( + createAirQualityOverlayEntry({ + id: reading.id, + position, + title: `${reading.name} ${reading.label}`, + accent: reading.color, + value: reading.value, + }), + ); + } + if (_enabled) { + overlayHost.setEntries( + AIR_QUALITY_OVERLAY_SOURCE_ID, + selectAirQualityOverlayCohort(overlayEntries), + { + cohortLimit: AIR_QUALITY_OVERLAY_COHORT_LIMIT, + collisionCapacity: AIR_QUALITY_OVERLAY_COLLISION_CAPACITY, + moving: false, + }, + ); + } + }, + + destroy(viewer = _viewer) { + _request?.abort(); + _request = null; + _enabled = false; + overlayHost.clearSource(AIR_QUALITY_OVERLAY_SOURCE_ID); + overlayHost.setVisible(AIR_QUALITY_OVERLAY_SOURCE_ID, false); + if (_dataSource) { + viewer?.dataSources?.remove(_dataSource, true); + _dataSource = null; + } + _viewer = null; + _readings = []; + _count = 0; + _lastUpdate = null; + _lastError = null; + rowControlsListener = null; + }, + + /** + * Readout descriptor for the Weather panel: the band legend, and the worst + * readings currently in view. + * + * `readout: true` renders the toggle and source line only; the card owns + * the rest. Coverage is stated plainly because this layer is not global. + */ + getRowControls() { + const provider = AIR_QUALITY_PROVIDERS.eccc; + const inView = readingsInView(_readings, viewRectangle(), READOUT_LIMIT); + const newest = _readings.reduce( + (max, reading) => Math.max(max, reading.observedMs || 0), + 0, + ); + return { + readout: true, + summary: { + label: provider.label, + coverage: provider.coverage, + validTime: newest ? new Date(newest).toISOString() : undefined, + detail: `${provider.network} · ${provider.scale}`, + status: _lastError + ? _lastError + : _count === 0 + ? 'No readings available' + : `${_count} stations reporting`, + units: provider.scale, + }, + legend: { + colors: AIR_QUALITY_BANDS.map((band) => band.color), + labels: AIR_QUALITY_BANDS.map((band) => band.label), + units: provider.scale, + categorical: true, + }, + list: { + ariaLabel: 'Highest air-quality readings in view', + items: inView.map((reading) => ({ + id: reading.id, + label: reading.name, + value: `${reading.scale} ${reading.label}`, + accent: reading.color, + detail: reading.risk, + })), + }, + }; + }, + + setRowControlsListener(listener) { + rowControlsListener = typeof listener === 'function' ? listener : null; + }, + + /** + * The current reading nearest a point — what "the air quality here" means. + * @param {number} lat + * @param {number} lon + * @returns {object|null} Reading with distanceKm, or null when none loaded. + */ + nearestReading(lat, lon) { + if (!Number.isFinite(lat) || !Number.isFinite(lon) || !_readings.length) + return null; + const cosLat = Math.cos((lat * Math.PI) / 180); + let best = null; + let bestDistance = Infinity; + for (const reading of _readings) { + const dLat = (reading.lat - lat) * 111.32; + const dLon = (reading.lon - lon) * 111.32 * cosLat; + const distance = Math.hypot(dLat, dLon); + if (distance < bestDistance) { + bestDistance = distance; + best = reading; + } + } + return best ? { ...best, distanceKm: bestDistance } : null; + }, + + /** + * Snapshot readings as JSON-safe records for the analyst query engine. + * @param {number} [maxCount=500] + * @returns {Array} + */ + getAnalystRecords(maxCount = 500) { + if (!_enabled || !_readings.length) return []; + const limit = Number.isFinite(maxCount) + ? Math.max(1, Math.floor(maxCount)) + : 500; + return _readings + .slice(0, limit) + .map((reading, index) => mapAnalystRecord(reading, index)); + }, + + getStats() { + // `lastUpdate` is when a response was received; `observedAt` is when the + // network actually measured. Freshness is judged on the latter, so a feed + // that keeps answering with hours-old observations reports STALE rather + // than being narrated as live by the HUD and voice provenance envelope. + return { + count: _count, + lastUpdate: _lastUpdate, + error: _lastError, + observedAt: newestObservation(_readings), + stale: readingsAreStale(_readings), + }; + }, + }; + return layer; +} diff --git a/src/layers/airQuality/model.js b/src/layers/airQuality/model.js new file mode 100644 index 000000000..93f13404f --- /dev/null +++ b/src/layers/airQuality/model.js @@ -0,0 +1,374 @@ +/** + * @module layers/airQuality/model + * + * Air-quality readings, normalized across networks. Pure — no Cesium, no + * network. + * + * Air-quality indices are not interchangeable numbers. Canada's AQHI runs 1 to + * 10+ on health risk; the US AQI runs 0 to 500 on concentration breakpoints; + * Europe's EAQI is a five-step category. A reading is therefore carried as + * `{ provider, scale, value, band }` — who measured it, which index it is, the + * value on that index's own terms, and the band it falls in — so a second + * network can be added without retrofitting a shared numeric meaning that does + * not exist. + * + * Only ECCC's AQHI is implemented here; `AIR_QUALITY_PROVIDERS` is the seam a + * later provider registers against. + */ + +/** Band identifiers shared across providers, in ascending severity. */ +export const AIR_QUALITY_BANDS = Object.freeze([ + Object.freeze({ id: 'low', label: 'Low', color: '#4cc9f0' }), + Object.freeze({ id: 'moderate', label: 'Moderate', color: '#ffd43b' }), + Object.freeze({ id: 'high', label: 'High', color: '#ff6b35' }), + Object.freeze({ id: 'very-high', label: 'Very High', color: '#c1121f' }), +]); + +const BAND_BY_ID = new Map(AIR_QUALITY_BANDS.map((band) => [band.id, band])); + +/** Colour used when a station reports no usable reading. */ +export const AIR_QUALITY_UNKNOWN_COLOR = '#6b7785'; + +export const AIR_QUALITY_OVERLAY_SOURCE_ID = 'air-quality'; +export const AIR_QUALITY_OVERLAY_COHORT_LIMIT = 48; +export const AIR_QUALITY_OVERLAY_COLLISION_CAPACITY = 32; + +/** + * A reading this old is stale enough that displaying it would misinform. + * AQHI publishes hourly, so six hours is five missed publications. + */ +export const OBSERVATION_MAX_AGE_MS = 6 * 60 * 60 * 1000; + +/** + * How old the newest observation may be before the layer reports itself stale. + * + * This is deliberately separate from the age at which a reading is dropped. + * `lastUpdate` records when we last *received* a response, which stays current + * even when the network publishes nothing new; freshness has to be judged on + * the source's own observation time instead. AQHI publishes hourly, so a newest + * reading beyond two hours means at least one publication was missed and the + * feed should not be described as live. + */ +export const STALE_AFTER_MS = 2 * 60 * 60 * 1000; + +/** + * The newest observation time across a set of readings, or null when empty. + * @param {Array} readings + * @returns {number|null} Epoch ms. + */ +export function newestObservation(readings) { + if (!Array.isArray(readings) || readings.length === 0) return null; + let newest = null; + for (const reading of readings) { + const at = Number(reading?.observedMs); + if (Number.isFinite(at) && (newest === null || at > newest)) newest = at; + } + return newest; +} + +/** + * Is a set of readings stale, judged on source observation time? + * @param {Array} readings + * @param {number} [now=Date.now()] + * @returns {boolean} + */ +export function readingsAreStale(readings, now = Date.now()) { + const newest = newestObservation(readings); + if (newest === null) return false; // nothing to be stale about + return now - newest > STALE_AFTER_MS; +} + +/** + * Registered provider descriptors. `coverage` is stated plainly because the + * layer is not global and the row must say so. + */ +export const AIR_QUALITY_PROVIDERS = Object.freeze({ + eccc: Object.freeze({ + id: 'eccc', + scale: 'AQHI', + label: 'Air quality', + network: 'Environment and Climate Change Canada', + coverage: 'Canada only · ECCC AQHI network', + /** ECCC's published band thresholds, evaluated in order. */ + bands: Object.freeze([ + Object.freeze({ id: 'low', max: 3 }), + Object.freeze({ id: 'moderate', max: 6 }), + Object.freeze({ id: 'high', max: 10 }), + Object.freeze({ id: 'very-high', max: Infinity }), + ]), + }), +}); + +/** Band descriptor for an id, or null. */ +export function airQualityBandById(bandId) { + return BAND_BY_ID.get(String(bandId ?? '')) || null; +} + +/** Accent colour for a band id, falling back to neutral grey. */ +export function airQualityColor(bandId) { + return airQualityBandById(bandId)?.color || AIR_QUALITY_UNKNOWN_COLOR; +} + +/** + * The published AQHI value for a raw ECCC reading: an integer from 1 up. + * + * ECCC reports whole numbers with a floor of 1 — there is no "AQHI 0" and no + * "AQHI 1.08". A raw value below 1 is a legitimate low reading, not an error, + * and is published as 1. + * + * @param {number|string|null|undefined} raw - Raw `aqhi` value from the API. + * @returns {number|null} Published integer AQHI, or null when unusable. + */ +export function aqhiDisplayValue(raw) { + // Guard the empty forms BEFORE coercing: Number(null) and Number('') are both + // 0, which would otherwise floor to a confident "AQHI 1" for a station that + // reported nothing at all. + if (raw === null || raw === undefined || raw === '') return null; + if (typeof raw !== 'number' && typeof raw !== 'string') return null; + if (typeof raw === 'string' && raw.trim() === '') return null; + const value = Number(raw); + if (!Number.isFinite(value) || value < 0) return null; + return Math.max(1, Math.round(value)); +} + +/** + * The band id for a value on a provider's scale. + * @param {string} providerId + * @param {number|null} value - Already in the provider's published form. + * @returns {string|null} + */ +export function bandForValue(providerId, value) { + const provider = AIR_QUALITY_PROVIDERS[String(providerId ?? '')]; + if (!provider || !Number.isFinite(value)) return null; + return provider.bands.find((band) => value <= band.max)?.id ?? null; +} + +/** + * Display text for a value on a provider's scale. + * AQHI collapses everything above ten into the open-ended "10+". + * @param {string} providerId + * @param {number|null} value + * @returns {string} + */ +export function valueLabel(providerId, value) { + if (!Number.isFinite(value)) return '--'; + if (providerId === 'eccc') return value > 10 ? '10+' : String(value); + return String(value); +} + +/** Plain-language risk phrase for a band id. */ +export function bandRiskText(bandId) { + const band = airQualityBandById(bandId); + return band ? `${band.label} health risk` : null; +} + +/** + * Validate a station FeatureCollection into an id-keyed catalog. + * + * Atomic: a structurally broken response returns null so the caller keeps its + * last good catalog. A station missing coordinates is skipped, since one + * unusable station is not evidence the catalog is broken. + * + * @param {object} geojson - GeoJSON FeatureCollection from `aqhi-stations`. + * @returns {Map|null} Stations by location_id, or null. + */ +export function normalizeAqhiStations(geojson) { + if (!Array.isArray(geojson?.features)) return null; + const stations = new Map(); + for (const feature of geojson.features) { + const properties = feature?.properties; + if ( + !properties || + typeof properties !== 'object' || + Array.isArray(properties) + ) + return null; + const coordinates = feature?.geometry?.coordinates; + if (!Array.isArray(coordinates) || coordinates.length < 2) continue; + const [lon, lat] = coordinates.map(Number); + if (!Number.isFinite(lon) || !Number.isFinite(lat)) continue; + if (Math.abs(lat) > 90 || Math.abs(lon) > 180) return null; + const id = String(properties.location_id ?? feature.id ?? '').trim(); + if (!id) continue; + stations.set(id, { + id, + name: String(properties.location_name_en || id).trim(), + // ECCC's administrative zone: atl, que, ont, pnr, pyr. + zone: String(properties['eccc_administrative-zone'] || '').trim(), + lat, + lon, + }); + } + return stations; +} + +/** + * Join latest ECCC observations onto known stations into provider-shaped rows. + * + * The catalog is the only thing that supplies a reading its coordinates, so an + * observation without a matching station cannot be placed and is dropped. + * Readings older than OBSERVATION_MAX_AGE_MS are dropped too — a six-hour-old + * air-quality number presented as current is worse than no number. + * + * @param {object} geojson - FeatureCollection from `aqhi-observations-realtime`. + * @param {Map} stations - Known stations by location_id. + * @param {number} [now=Date.now()] - Clock seam for tests. + * @returns {Array|null} Normalized readings, or null when malformed. + */ +export function normalizeAqhiObservations(geojson, stations, now = Date.now()) { + if (!Array.isArray(geojson?.features)) return null; + if (!(stations instanceof Map)) return null; + const provider = AIR_QUALITY_PROVIDERS.eccc; + const newest = new Map(); + for (const feature of geojson.features) { + const properties = feature?.properties; + if ( + !properties || + typeof properties !== 'object' || + Array.isArray(properties) + ) + return null; + const stationId = String(properties.location_id ?? '').trim(); + const station = stations.get(stationId); + if (!station) continue; + + const value = aqhiDisplayValue(properties.aqhi); + if (value === null) continue; + const observedMs = Date.parse(properties.observation_datetime); + if (!Number.isFinite(observedMs)) continue; + if (now - observedMs > OBSERVATION_MAX_AGE_MS) continue; + + const previous = newest.get(stationId); + if (previous && previous.observedMs >= observedMs) continue; + const band = bandForValue(provider.id, value); + newest.set(stationId, { + id: `${provider.id}:${stationId}`, + provider: provider.id, + scale: provider.scale, + value, + band, + stationId, + name: station.name, + zone: station.zone, + lat: station.lat, + lon: station.lon, + observedMs, + color: airQualityColor(band), + label: valueLabel(provider.id, value), + risk: bandRiskText(band), + note: String(properties.special_notes_en || '').trim() || null, + }); + } + return [...newest.values()].sort( + (a, b) => b.value - a.value || a.stationId.localeCompare(b.stationId), + ); +} + +/** + * Build the source-owned presentation for one station label. + * @param {object} input + * @returns {object} + */ +export function createAirQualityOverlayEntry({ + id, + position, + title, + accent, + value, +}) { + return { + id: String(id), + position, + variant: 'label', + title, + accent, + // Worse air wins the label budget — the reading people need to see first. + priority: Math.round(Number(value) || 0) * 1000, + collisionGroup: 'ambient-label', + paintLane: 'ambient-label', + interactive: false, + edgeFade: 'keyhole', + horizonCull: true, + terrainOcclusion: false, + gapPx: 14, + verticalOnly: true, + placement: 'above', + }; +} + +/** Keep the worst readings, with stable identity as the tie-break. */ +export function selectAirQualityOverlayCohort( + entries, + limit = AIR_QUALITY_OVERLAY_COHORT_LIMIT, +) { + const cap = Math.max( + 0, + Math.min(AIR_QUALITY_OVERLAY_COHORT_LIMIT, Math.floor(Number(limit) || 0)), + ); + if (!Array.isArray(entries) || cap === 0) return []; + return entries + .slice() + .sort( + (a, b) => + b.priority - a.priority || String(a.id).localeCompare(String(b.id)), + ) + .slice(0, cap); +} + +/** + * The worst readings inside a lon/lat rectangle, for the readout card. + * + * A null rectangle means the camera could not resolve one (a fully oblique or + * space-level view); everything is then in scope rather than nothing, so the + * card degrades to the worst readings overall instead of going blank. + * + * @param {Array} readings + * @param {?{west:number, south:number, east:number, north:number}} rect - Degrees. + * @param {number} [limit=5] + * @returns {Array} + */ +export function readingsInView(readings, rect, limit = 5) { + if (!Array.isArray(readings)) return []; + const cap = Math.max(0, Math.floor(Number(limit) || 0)); + if (cap === 0) return []; + const inside = !rect + ? readings + : readings.filter((reading) => { + if (reading.lat < rect.south || reading.lat > rect.north) return false; + // A rectangle crossing the antimeridian has west > east. + return rect.west <= rect.east + ? reading.lon >= rect.west && reading.lon <= rect.east + : reading.lon >= rect.west || reading.lon <= rect.east; + }); + return inside + .slice() + .sort((a, b) => b.value - a.value || a.stationId.localeCompare(b.stationId)) + .slice(0, cap); +} + +/** + * Map one reading to a JSON-safe analyst record. Pure — no Cesium types. + * @param {object|null|undefined} raw + * @param {number} [index=0] + * @returns {object} + */ +export function mapAnalystRecord(raw, index = 0) { + const num = (v) => (Number.isFinite(v) ? v : null); + const text = (v) => { + const t = String(v ?? '').trim(); + return t || null; + }; + return { + id: text(raw?.id) || `AIR-${String(index).padStart(4, '0')}`, + provider: text(raw?.provider), + scale: text(raw?.scale), + value: num(raw?.value), + band: text(raw?.band), + name: text(raw?.name), + zone: text(raw?.zone), + risk: text(raw?.risk), + lat: num(raw?.lat), + lon: num(raw?.lon), + observedMs: num(raw?.observedMs), + }; +} diff --git a/src/layers/airQuality/model.test.mjs b/src/layers/airQuality/model.test.mjs new file mode 100644 index 000000000..56130c30e --- /dev/null +++ b/src/layers/airQuality/model.test.mjs @@ -0,0 +1,250 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + AIR_QUALITY_BANDS, + AIR_QUALITY_PROVIDERS, + AIR_QUALITY_UNKNOWN_COLOR, + OBSERVATION_MAX_AGE_MS, + airQualityColor, + aqhiDisplayValue, + bandForValue, + bandRiskText, + mapAnalystRecord, + normalizeAqhiObservations, + normalizeAqhiStations, + readingsInView, + selectAirQualityOverlayCohort, + valueLabel, +} from './model.js'; + +const NOW = Date.parse('2026-09-24T18:30:00Z'); +const RECENT = '2026-09-24T18:00:00Z'; +const collection = (features) => ({ type: 'FeatureCollection', features }); +const station = (id, name, zone, lon, lat) => ({ + type: 'Feature', + geometry: { type: 'Point', coordinates: [lon, lat] }, + properties: { + location_id: id, + location_name_en: name, + 'eccc_administrative-zone': zone, + }, +}); +const observation = (id, aqhi, when = RECENT) => ({ + type: 'Feature', + properties: { location_id: id, aqhi, observation_datetime: when }, +}); + +const STATIONS = collection([ + station('IAKID', 'Calgary', 'pnr', -114.0575, 51.0458), + station('OAAAA', 'Toronto', 'ont', -79.3832, 43.6532), + station('AAAAA', 'Halifax', 'atl', -63.5752, 44.6488), +]); + +// ── provider-shaped records ──────────────────────────────────────────────── + +test('readings carry the provider-neutral {provider, scale, value, band} shape', () => { + // Indices are not interchangeable numbers: AQHI is 1-10+ on health risk, + // US AQI is 0-500 on concentration. The shape names which index a value is on + // so a second network can be added without inventing a shared meaning. + const stations = normalizeAqhiStations(STATIONS); + const [row] = normalizeAqhiObservations( + collection([observation('IAKID', 2.32)]), + stations, + NOW, + ); + assert.equal(row.provider, 'eccc'); + assert.equal(row.scale, 'AQHI'); + assert.equal(row.value, 2); + assert.equal(row.band, 'low'); +}); + +test('bands map from a value on a named provider scale', () => { + assert.equal(bandForValue('eccc', 1), 'low'); + assert.equal(bandForValue('eccc', 3), 'low'); + assert.equal(bandForValue('eccc', 4), 'moderate'); + assert.equal(bandForValue('eccc', 7), 'high'); + assert.equal(bandForValue('eccc', 11), 'very-high'); + // An unregistered provider cannot be banded rather than being guessed at. + assert.equal(bandForValue('unknown-network', 5), null); + assert.equal(bandForValue('eccc', null), null); +}); + +test('the provider states its coverage plainly', () => { + assert.match(AIR_QUALITY_PROVIDERS.eccc.coverage, /Canada only/); + assert.equal(AIR_QUALITY_PROVIDERS.eccc.scale, 'AQHI'); +}); + +test("values above ten collapse to the scale's open-ended label", () => { + assert.equal(valueLabel('eccc', 10), '10'); + assert.equal(valueLabel('eccc', 11), '10+'); + assert.equal(valueLabel('eccc', null), '--'); +}); + +test('every band has a distinct colour, and an unknown band is neutral', () => { + const colors = AIR_QUALITY_BANDS.map((b) => b.color); + assert.equal(new Set(colors).size, colors.length); + assert.equal(airQualityColor('nonsense'), AIR_QUALITY_UNKNOWN_COLOR); + assert.equal(bandRiskText('high'), 'High health risk'); + assert.equal(bandRiskText(null), null); +}); + +// ── published form ───────────────────────────────────────────────────────── + +test('AQHI rounds to the published integer form and floors at 1', () => { + assert.equal(aqhiDisplayValue(1.08), 1); + assert.equal(aqhiDisplayValue(3.5), 4); + assert.equal(aqhiDisplayValue(0), 1); + assert.equal(aqhiDisplayValue('3'), 3); +}); + +test('an absent reading never becomes a confident AQHI 1', () => { + // Number(null) and Number('') are both 0, which would floor to 1. + for (const empty of [null, undefined, '', ' ', NaN, true, {}, [], -1]) { + assert.equal(aqhiDisplayValue(empty), null, `${JSON.stringify(empty)}`); + } +}); + +// ── snapshot validation ──────────────────────────────────────────────────── + +test('stations normalize into an id-keyed catalog carrying their zone', () => { + const stations = normalizeAqhiStations(STATIONS); + assert.equal(stations.size, 3); + assert.equal(stations.get('OAAAA').zone, 'ont'); +}); + +test('malformed payloads are rejected atomically', () => { + assert.equal(normalizeAqhiStations(null), null); + assert.equal(normalizeAqhiStations(collection([{ properties: null }])), null); + const stations = normalizeAqhiStations(STATIONS); + assert.equal(normalizeAqhiObservations(null, stations, NOW), null); + assert.equal( + normalizeAqhiObservations(collection([]), 'not a map', NOW), + null, + ); +}); + +test('an observation with no matching station is dropped', () => { + const stations = normalizeAqhiStations(STATIONS); + const rows = normalizeAqhiObservations( + collection([observation('IAKID', 2), observation('ZZZZZ', 9)]), + stations, + NOW, + ); + assert.equal(rows.length, 1); +}); + +test('a stale reading is dropped rather than shown as current', () => { + const stations = normalizeAqhiStations(STATIONS); + const old = new Date(NOW - OBSERVATION_MAX_AGE_MS - 60_000).toISOString(); + const rows = normalizeAqhiObservations( + collection([observation('IAKID', 3, old), observation('OAAAA', 4)]), + stations, + NOW, + ); + assert.deepEqual( + rows.map((r) => r.stationId), + ['OAAAA'], + ); +}); + +test('the newest reading per station wins and worst air sorts first', () => { + const stations = normalizeAqhiStations(STATIONS); + const rows = normalizeAqhiObservations( + collection([ + observation('IAKID', 3, '2026-09-24T16:00:00Z'), + observation('IAKID', 7, '2026-09-24T18:00:00Z'), + observation('OAAAA', 5), + ]), + stations, + NOW, + ); + assert.deepEqual( + rows.map((r) => r.value), + [7, 5], + ); +}); + +// ── readout selection ────────────────────────────────────────────────────── + +const reading = (id, value, lat, lon) => ({ + id, + stationId: id, + value, + lat, + lon, + name: id, +}); + +test('the readout lists the worst readings inside the view rectangle', () => { + const rows = [ + reading('west', 9, 51.0, -114.0), + reading('east', 4, 43.6, -79.3), + reading('mild', 2, 51.1, -114.1), + ]; + const inView = readingsInView( + rows, + { west: -115, south: 50, east: -113, north: 52 }, + 5, + ); + assert.deepEqual( + inView.map((r) => r.id), + ['west', 'mild'], + ); +}); + +test('an unresolvable view rectangle degrades to the worst overall, not to blank', () => { + // A space-level or fully oblique camera cannot produce a rectangle; the card + // should still say something useful. + const rows = [reading('a', 3, 0, 0), reading('b', 8, 10, 10)]; + assert.deepEqual( + readingsInView(rows, null, 5).map((r) => r.id), + ['b', 'a'], + ); + assert.deepEqual(readingsInView(rows, null, 0), []); + assert.deepEqual(readingsInView(null, null, 5), []); +}); + +test('a view rectangle crossing the antimeridian still selects correctly', () => { + // west > east is how a dateline-crossing rectangle is expressed. + const rows = [reading('near', 5, 0, 179), reading('far', 9, 0, 100)]; + const inView = readingsInView( + rows, + { west: 170, south: -10, east: -170, north: 10 }, + 5, + ); + assert.deepEqual( + inView.map((r) => r.id), + ['near'], + ); +}); + +test('the overlay cohort is capped and deterministic', () => { + const entries = Array.from({ length: 120 }, (_, i) => ({ + id: `s${i}`, + priority: i % 11, + })); + const cohort = selectAirQualityOverlayCohort(entries); + assert.ok(cohort.length <= 48); + assert.deepEqual( + cohort.map((e) => e.id), + selectAirQualityOverlayCohort(entries).map((e) => e.id), + ); +}); + +test('analyst records are JSON-safe and name the provider and scale', () => { + const stations = normalizeAqhiStations(STATIONS); + const [row] = normalizeAqhiObservations( + collection([observation('IAKID', 2)]), + stations, + NOW, + ); + const record = mapAnalystRecord(row, 0); + assert.deepEqual(JSON.parse(JSON.stringify(record)), record); + assert.equal(record.provider, 'eccc'); + assert.equal(record.scale, 'AQHI'); + const empty = mapAnalystRecord(undefined, 7); + assert.equal(empty.id, 'AIR-0007'); + for (const [key, value] of Object.entries(empty)) { + assert.notEqual(value, undefined, `${key} must not be undefined`); + } +}); diff --git a/src/layers/airQuality/ownership.test.mjs b/src/layers/airQuality/ownership.test.mjs new file mode 100644 index 000000000..5eeaae0e1 --- /dev/null +++ b/src/layers/airQuality/ownership.test.mjs @@ -0,0 +1,326 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createAirQualityLayer } from './index.js'; +import { createEcccAirQualitySource } from './source.js'; +import { layerFeedState } from '../../data/feedState.js'; +import { STALE_AFTER_MS } from './model.js'; + +function harness(source) { + const sources = []; + const events = []; + const viewer = { + dataSources: { + add(value) { + sources.push(value); + }, + remove(value) { + sources.splice(sources.indexOf(value), 1); + }, + }, + }; + const layer = createAirQualityLayer({ + source, + overlayHost: { + setEntries(...args) { + events.push(args); + }, + setVisible() {}, + clearSource() {}, + }, + }); + layer.init(viewer); + layer.enable(viewer); + return { layer, viewer, sources, events }; +} + +const reading = (id, value, overrides = {}) => ({ + id: `eccc:${id}`, + provider: 'eccc', + scale: 'AQHI', + value, + band: 'low', + stationId: id, + name: id, + zone: 'ont', + lat: 43.65, + lon: -79.38, + observedMs: 1_700_000_000_000, + color: '#4cc9f0', + label: String(value), + risk: 'Low health risk', + note: null, + ...overrides, +}); + +const staticSource = (rows) => ({ getSnapshot: async () => rows }); + +test('a layer requires both a snapshot source and an overlay host', () => { + assert.throws(() => createAirQualityLayer(), /snapshot source/); + assert.throws( + () => createAirQualityLayer({ source: staticSource([]) }), + /overlay host/, + ); +}); + +test('initialization is single-use and owns exactly one data source', () => { + const { layer, viewer, sources } = harness(staticSource([])); + assert.equal(sources.length, 1); + assert.throws(() => layer.init(viewer), /already initialized/); + layer.destroy(viewer); + assert.equal(sources.length, 0); +}); + +test('a refresh renders readings and publishes an overlay cohort', async () => { + const { layer, events } = harness( + staticSource([reading('A', 2), reading('B', 7)]), + ); + assert.equal(await layer.update(), true); + assert.equal(layer.getStats().count, 2); + assert.equal(layer.getStats().error, null); + assert.equal(events.at(-1)[0], 'air-quality'); +}); + +// ── the abort + enabled guard ────────────────────────────────────────────── + +test('a response landing after disable cannot redraw the scene', async () => { + // The failure this prevents: disable() clears the overlay, then an in-flight + // response resolves and repopulates it behind the user's back. + let release; + const gate = new Promise((resolve) => { + release = resolve; + }); + const { layer, events } = harness({ + getSnapshot: async ({ signal } = {}) => { + await gate; + signal?.throwIfAborted(); + return [reading('A', 5)]; + }, + }); + const pending = layer.update(); + layer.disable(); + const publishedBefore = events.length; + release(); + assert.equal(await pending, false, 'a late response must not be applied'); + assert.equal( + events.length, + publishedBefore, + 'no overlay write after disable', + ); + assert.equal(layer.getStats().count, 0); +}); + +test('disable aborts the request in flight', async () => { + let seenSignal = null; + const { layer } = harness({ + getSnapshot: async ({ signal } = {}) => { + seenSignal = signal; + await new Promise((resolve) => setTimeout(resolve, 40)); + signal?.throwIfAborted(); + return []; + }, + }); + const pending = layer.update(); + layer.disable(); + await pending; + assert.ok(seenSignal, 'a signal was passed through to the source'); + assert.equal(seenSignal.aborted, true); +}); + +test('a superseded refresh does not clobber the newer one', async () => { + let call = 0; + const { layer } = harness({ + getSnapshot: async ({ signal } = {}) => { + call += 1; + const mine = call; + await new Promise((resolve) => setTimeout(resolve, mine === 1 ? 40 : 1)); + signal?.throwIfAborted(); + return [reading(`call-${mine}`, mine)]; + }, + }); + const first = layer.update(); + const second = layer.update(); + const [firstResult, secondResult] = await Promise.all([first, second]); + assert.equal(firstResult, false, 'the superseded call reports no update'); + assert.equal(secondResult, true); + assert.equal(layer.getAnalystRecords()[0].name, 'call-2'); +}); + +test('update is a no-op while the layer is disabled', async () => { + const { layer } = harness(staticSource([reading('A', 2)])); + layer.disable(); + assert.equal(await layer.update(), false); + assert.equal(layer.getStats().count, 0); +}); + +// ── readout descriptor ───────────────────────────────────────────────────── + +test('the readout descriptor carries the band legend and coverage', async () => { + const { layer } = harness(staticSource([reading('A', 2), reading('B', 9)])); + await layer.update(); + const controls = layer.getRowControls(); + assert.equal(controls.readout, true); + assert.match(controls.summary.coverage, /Canada only/); + assert.equal(controls.legend.categorical, true); + assert.deepEqual(controls.legend.labels, [ + 'Low', + 'Moderate', + 'High', + 'Very High', + ]); + assert.equal(controls.legend.colors.length, controls.legend.labels.length); + // Worst first, and every item keyed. + assert.deepEqual( + controls.list.items.map((i) => i.label), + ['B', 'A'], + ); + assert.ok(controls.list.items.every((i) => i.id)); + assert.match(controls.summary.status, /2 stations reporting/); +}); + +test('the readout reports an error rather than a silent empty card', async () => { + const { layer } = harness({ + getSnapshot: async () => { + throw new Error('ECCC HTTP 503'); + }, + }); + await layer.update(); + assert.match(layer.getRowControls().summary.status, /503/); +}); + +test('row-control listeners are notified on refresh and teardown-safe', async () => { + const { layer, viewer } = harness(staticSource([reading('A', 2)])); + let calls = 0; + layer.setRowControlsListener(() => { + calls += 1; + }); + await layer.update(); + assert.ok(calls >= 1, 'a refresh notifies the card'); + layer.destroy(viewer); + const after = calls; + layer.setRowControlsListener(() => { + calls += 1; + }); + assert.equal(calls, after); +}); + +// ── source contract ──────────────────────────────────────────────────────── + +test('the source rejects a malformed station catalog and surfaces HTTP errors', async () => { + await assert.rejects( + () => + createEcccAirQualitySource({ + fetchImpl: async () => ({ + ok: true, + json: async () => ({ features: 'no' }), + }), + }).getSnapshot(), + /Malformed AQHI station/, + ); + await assert.rejects( + () => + createEcccAirQualitySource({ + fetchImpl: async () => ({ + ok: false, + status: 503, + json: async () => ({}), + }), + }).getSnapshot(), + /ECCC HTTP 503/, + ); +}); + +test('the station catalog is cached and observations ask for latest values only', async () => { + const urls = []; + const source = createEcccAirQualitySource({ + fetchImpl: async (url) => { + urls.push(url); + return { + ok: true, + json: async () => + url.includes('aqhi-stations') + ? { + features: [ + { + geometry: { type: 'Point', coordinates: [-79.38, 43.65] }, + properties: { location_id: 'A', location_name_en: 'A' }, + }, + ], + } + : { features: [] }, + }; + }, + now: () => 1_700_000_000_000, + }); + await source.getSnapshot(); + await source.getSnapshot(); + assert.equal(urls.filter((u) => u.includes('aqhi-stations')).length, 1); + // Without latest=true the collection returns the full history (~2,000 rows). + assert.match( + urls.find((u) => u.includes('observations')), + /latest=true/, + ); +}); + +// ── feed-state honesty ───────────────────────────────────────────────────── + +test('freshness is judged on observation time, not on when we last fetched', async () => { + // lastUpdate stays current every time the network answers, even when it + // answers with nothing new. Judging freshness on it would let the HUD and + // voice describe hours-old air quality as live. + const aged = Date.now() - STALE_AFTER_MS - 60_000; + const { layer } = harness( + staticSource([reading('A', 2, { observedMs: aged })]), + ); + await layer.update(); + const stats = layer.getStats(); + assert.ok( + stats.lastUpdate >= Date.now() - 5_000, + 'we did just receive a response', + ); + assert.equal(stats.observedAt, aged, 'but the observation itself is old'); + assert.equal(stats.stale, true); + assert.equal( + layerFeedState(stats), + 'stale', + 'provenance must not read this as nominal', + ); +}); + +test('a current reading reports nominal', async () => { + const { layer } = harness( + staticSource([reading('A', 2, { observedMs: Date.now() - 10 * 60_000 })]), + ); + await layer.update(); + const stats = layer.getStats(); + assert.equal(stats.stale, false); + assert.equal(layerFeedState(stats), 'nominal'); +}); + +test('an empty feed is not called stale', async () => { + // Nothing to be stale about; the chip should not invent a fault. + const { layer } = harness(staticSource([])); + await layer.update(); + assert.equal(layer.getStats().stale, false); + assert.equal(layer.getStats().observedAt, null); +}); + +test('a failed refresh over good readings reports degraded, not nominal', async () => { + let fail = false; + const { layer } = harness({ + getSnapshot: async () => { + if (fail) throw new Error('ECCC HTTP 503'); + return [reading('A', 2, { observedMs: Date.now() - 10 * 60_000 })]; + }, + }); + await layer.update(); + assert.equal(layerFeedState(layer.getStats()), 'nominal'); + fail = true; + await layer.update(); + const stats = layer.getStats(); + assert.ok(stats.count > 0, 'the previous readings are still displayed'); + assert.equal( + layerFeedState(stats), + 'degraded', + 'showing old data after a failure is degraded', + ); +}); diff --git a/src/layers/airQuality/source.js b/src/layers/airQuality/source.js new file mode 100644 index 000000000..b5f98894c --- /dev/null +++ b/src/layers/airQuality/source.js @@ -0,0 +1,78 @@ +import { normalizeAqhiStations, normalizeAqhiObservations } from './model.js'; + +const COLLECTIONS_ROOT = 'https://api.weather.gc.ca/collections'; +const STATIONS_URL = `${COLLECTIONS_ROOT}/aqhi-stations/items`; +const OBSERVATIONS_URL = `${COLLECTIONS_ROOT}/aqhi-observations-realtime/items`; + +/** Comfortably above Canada's ~134 stations without inviting a huge page. */ +const STATION_LIMIT = 500; +const OBSERVATION_LIMIT = 500; +/** Station geometry is near-static — a few changes a year. Refetch twice a day. */ +const STATION_TTL_MS = 12 * 60 * 60 * 1000; + +/** + * Request and validate a complete ECCC AQHI snapshot before it can replace + * displayed readings. The first registered air-quality provider. + * + * Two collections are involved. `aqhi-stations` is the catalog — it supplies + * every reading its coordinates and is cached for twelve hours because it + * barely changes. `aqhi-observations-realtime` is the hourly reading feed, + * requested with `latest=true` so the response is current values rather than + * the full observation history, which is roughly 2,000 rows nationally. + * + * The station catalog is only replaced when a refresh actually returns + * stations, so a transient empty response cannot blank the layer. + * + * @param {object} [options] + * @param {Function} [options.fetchImpl] - Injected fetch, for tests. + * @param {() => number} [options.now] - Clock seam, for tests. + * @returns {{getSnapshot: Function}} + */ +export function createEcccAirQualitySource({ + fetchImpl = (...args) => globalThis.fetch(...args), + now = () => Date.now(), +} = {}) { + /** @type {Map} */ + let stations = new Map(); + let stationsFetchedAt = 0; + + const getJson = async (url, signal) => { + signal?.throwIfAborted(); + const response = await fetchImpl(url, { + headers: { Accept: 'application/json' }, + signal, + }); + if (!response.ok) throw new Error(`ECCC HTTP ${response.status}`); + const payload = await response.json(); + signal?.throwIfAborted(); + return payload; + }; + + return { + async getSnapshot({ signal } = {}) { + const clock = now(); + if (stations.size === 0 || clock - stationsFetchedAt > STATION_TTL_MS) { + const payload = await getJson( + `${STATIONS_URL}?f=json&limit=${STATION_LIMIT}`, + signal, + ); + const next = normalizeAqhiStations(payload); + if (!next) throw new Error('Malformed AQHI station response'); + // An empty refresh must not wipe a good catalog. + if (next.size > 0) { + stations = next; + stationsFetchedAt = clock; + } + } + if (stations.size === 0) throw new Error('No AQHI stations available'); + + const payload = await getJson( + `${OBSERVATIONS_URL}?f=json&latest=true&limit=${OBSERVATION_LIMIT}`, + signal, + ); + const readings = normalizeAqhiObservations(payload, stations, clock); + if (!readings) throw new Error('Malformed AQHI observation response'); + return readings; + }, + }; +} diff --git a/src/overlays/worldOverlayAllocation.test.mjs b/src/overlays/worldOverlayAllocation.test.mjs index bbaa4f168..e29785d48 100644 --- a/src/overlays/worldOverlayAllocation.test.mjs +++ b/src/overlays/worldOverlayAllocation.test.mjs @@ -11,6 +11,7 @@ import { EARTHQUAKE_OVERLAY_COHORT_LIMIT } from '../data/earthquakes.js'; import { ROCKET_MISSION_AMBIENT_OVERLAY_COHORT_LIMIT } from '../data/rocketLaunches.js'; import { RADIO_OVERLAY_COHORT_LIMIT } from '../data/radio.js'; import { CABLE_REFERENCE_LABEL_WINNER_CAP } from '../data/telegeographySubmarineCables.js'; +import { AIR_QUALITY_OVERLAY_COHORT_LIMIT } from '../layers/airQuality/model.js'; import { isCalibratedAllocationRuntime } from '../../scripts/run-unit-tests.mjs'; /** @@ -34,6 +35,7 @@ import { isCalibratedAllocationRuntime } from '../../scripts/run-unit-tests.mjs' * ---------------------+---------+------------+---------+----------------+-------------+------------- * generic below cap | 60 | 60 | 60 | 3182 | 53.0 | 4,100 * generic above cap | 250 | 250 | 96 | 10022 | 40.1 | 13,000 + * air quality | 48 | 48 | 32 | 3900 | 81.3 | 5,200 * local infrastructure | 320 | 320 | 192 | 39478 | 123.4 | 49,000 * Phase 3 + FIRMS | 338 | 338 | 210 | 41220 | 122.0 | 53,600 * Phase 3 + vessels | 451 | 451 | 311 | 67252 | 149.1 | 87,500 @@ -259,6 +261,23 @@ const WORKLOADS = [ maxBytesPerFrame: 19_000, saturated: true, }, + { + // The air-quality station cohort on its own. Measured standalone rather + // than stacked on the Phase-5 host (the submarine-cable precedent): these + // stations are sparse and, with the single registered provider, + // continental, so what matters is its own cohort cost. + // + // Re-measured against the current worker: median 3,900 B/frame, max 4,114, + // and 85.7 B/candidate at the ceiling — inside the shared 154 B/candidate + // bound, so no per-source exception is taken. The 5,200 frame budget + // carries 26.4% headroom over the measured maximum. + name: 'with the air-quality station cohort live', + profile: 'air-quality', + entries: AIR_QUALITY_OVERLAY_COHORT_LIMIT, + candidates: AIR_QUALITY_OVERLAY_COHORT_LIMIT, + maxBytesPerFrame: 5_200, + saturated: true, + }, { name: 'with the Dense detection lane active over 5,000 observations', profile: 'phase6-detection', diff --git a/src/overlays/worldOverlayAllocation.worker.mjs b/src/overlays/worldOverlayAllocation.worker.mjs index ea463f080..0314ec41d 100644 --- a/src/overlays/worldOverlayAllocation.worker.mjs +++ b/src/overlays/worldOverlayAllocation.worker.mjs @@ -26,6 +26,12 @@ import { TRACKED_OVERLAY_SOURCE_OPTIONS, } from '../data/trackedReadout.js'; import { CCTV_AMBIENT_CARD_MAX } from '../data/cctvLod.js'; +import { + AIR_QUALITY_OVERLAY_COHORT_LIMIT, + AIR_QUALITY_OVERLAY_COLLISION_CAPACITY, + AIR_QUALITY_OVERLAY_SOURCE_ID, + createAirQualityOverlayEntry, +} from '../layers/airQuality/model.js'; import { CCTV_OVERLAY_SOURCE_ID, createCctvThumbnailOverlayEntry, @@ -1101,6 +1107,68 @@ function buildSubmarineCablesWorkload(count) { ); } +/** + * The air-quality station cohort at full size. + * + * Measured standalone (the submarine-cable precedent) rather than stacked on + * the Phase-5 host: air-quality stations are sparse and, with the single + * currently-registered provider, continental — so the question a budget has to + * answer is what its own cohort costs, not what it adds to a worldwide worst + * case it never shares a viewport with. The entry shape is the same static + * `ambient-label` the earthquake layer publishes, so the per-candidate ceiling + * is the meaningful gate. + */ +function buildAirQualityWorkload(count) { + if (count !== AIR_QUALITY_OVERLAY_COHORT_LIMIT) { + throw new Error( + `air-quality requires ${AIR_QUALITY_OVERLAY_COHORT_LIMIT} entries`, + ); + } + const workload = { + entries: [], + positions: [], + drifts: [], + registrations: [], + }; + const columns = Math.max(1, Math.ceil(Math.sqrt((count * 16) / 9))); + const rows = Math.max(1, Math.ceil(count / columns)); + const spanX = columns > 1 ? 1.68 / (columns - 1) : 0; + const spanY = rows > 1 ? 1.56 / (rows - 1) : 0; + const stations = []; + for (let index = 0; index < count; index++) { + const column = index % columns; + const row = Math.floor(index / columns); + const baseX = -0.84 + column * spanX; + const baseY = -0.78 + row * spanY; + const position = new Cesium.Cartesian3(baseX, baseY, 0); + workload.positions.push(position); + workload.drifts.push({ + baseX, + baseY, + phase: index * 0.27, + rate: 0.32 + (index % 6) * 0.04, + }); + const entry = createAirQualityOverlayEntry({ + id: `air-${index}`, + position, + title: `Station ${index} ${1 + (index % 10)}`, + accent: '#4cc9f0', + value: 1 + (index % 10), + }); + entry.horizonCull = false; + stations.push(entry); + } + workload.registrations.push({ + sourceId: AIR_QUALITY_OVERLAY_SOURCE_ID, + entries: stations, + options: { + cohortLimit: AIR_QUALITY_OVERLAY_COHORT_LIMIT, + collisionCapacity: AIR_QUALITY_OVERLAY_COLLISION_CAPACITY, + }, + }); + return workload; +} + function buildPhase6DetectionWorkload(count) { const random = makeRandom(0xd37ec710); const positions = []; @@ -1206,9 +1274,13 @@ function main() { ? buildPhase5RocketMissionWorkload(ENTRY_COUNT) : PROFILE === 'all-live-radio' ? buildAllLiveRadioWorkload(ENTRY_COUNT) - : PROFILE === 'submarine-cables' - ? buildSubmarineCablesWorkload(ENTRY_COUNT) - : buildWorkload(ENTRY_COUNT); + : PROFILE === 'air-quality' + ? buildAirQualityWorkload(ENTRY_COUNT) + : PROFILE === 'submarine-cables' + ? buildSubmarineCablesWorkload( + ENTRY_COUNT, + ) + : buildWorkload(ENTRY_COUNT); const { entries, positions, drifts } = workload; const solveIntervalMs = Number(process.env.GEV_ALLOC_SOLVE_MS) || 125; const detectionActive = !!workload.detectionLayer; diff --git a/src/sources/reference.js b/src/sources/reference.js index 4bdf7e9ac..34c776164 100644 --- a/src/sources/reference.js +++ b/src/sources/reference.js @@ -1,4 +1,5 @@ import { createUsgsEarthquakeSource } from '../layers/earthquakes/source.js'; +import { createEcccAirQualitySource } from '../layers/airQuality/source.js'; import { createWfigsPerimeterSource } from '../layers/perimeters/source.js'; import { createBundledCableSource } from '../layers/submarineCables/bundledSource.js'; @@ -6,6 +7,7 @@ import { createBundledCableSource } from '../layers/submarineCables/bundledSourc export function createReferenceSources() { return { earthquakes: createUsgsEarthquakeSource(), + 'air-quality': createEcccAirQualitySource(), 'fire-perimeters': createWfigsPerimeterSource(), cables: createBundledCableSource(), }; diff --git a/src/sources/reference.test.mjs b/src/sources/reference.test.mjs index 902a72c83..285ee6d66 100644 --- a/src/sources/reference.test.mjs +++ b/src/sources/reference.test.mjs @@ -14,13 +14,16 @@ test('reference factories retain compatibility without starting acquisition or s const second = createReferenceSources(); assert.deepEqual(Object.keys(first), [ 'earthquakes', + 'air-quality', 'fire-perimeters', 'cables', ]); assert.notEqual(first.earthquakes, second.earthquakes); + assert.notEqual(first['air-quality'], second['air-quality']); assert.notEqual(first['fire-perimeters'], second['fire-perimeters']); assert.notEqual(first.cables, second.cables); assert.equal(typeof first.earthquakes.getSnapshot, 'function'); + assert.equal(typeof first['air-quality'].getSnapshot, 'function'); assert.equal(typeof first['fire-perimeters'].getSnapshot, 'function'); assert.equal(typeof first.cables.fetch, 'function'); assert.equal(requests, 0); diff --git a/src/ui/layerPanel.js b/src/ui/layerPanel.js index 57fd595f0..81ced6cf8 100644 --- a/src/ui/layerPanel.js +++ b/src/ui/layerPanel.js @@ -51,6 +51,7 @@ const PANEL_GROUPS = [ { label: 'Weather', ids: [ + 'air-quality', 'wind', 'weather-radar', 'weather-satellite', From de4c8343ddd3ce3a164e0d44e63cf3c7e805f477 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 10:33:46 -0500 Subject: [PATCH 012/202] test: make Recent Imagery live gate panel-relative --- scripts/qa-recent-imagery.mjs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/qa-recent-imagery.mjs b/scripts/qa-recent-imagery.mjs index 1460ed3b5..f9601432f 100644 --- a/scripts/qa-recent-imagery.mjs +++ b/scripts/qa-recent-imagery.mjs @@ -308,15 +308,21 @@ const CONTROLS = { }; const measure = () => page.evaluate((controls) => { + const panel = document.getElementById('recent-imagery-panel'); const body = document.getElementById('recent-imagery-panel-body'); + const panelRect = panel?.getBoundingClientRect(); const out = { bodyScroll: body?.scrollTop ?? null, rects: {} }; for (const [name, selector] of Object.entries(controls)) { const rect = document.querySelector(selector)?.getBoundingClientRect(); - out.rects[name] = rect - ? [rect.left, rect.top, rect.width, rect.height].map( - (v) => Math.round(v * 10) / 10, - ) - : null; + out.rects[name] = + rect && panelRect + ? [ + rect.left - panelRect.left, + rect.top - panelRect.top, + rect.width, + rect.height, + ].map((v) => Math.round(v * 10) / 10) + : null; } return out; }, CONTROLS); @@ -1042,13 +1048,13 @@ try { ) <= target * 0.01, fitHeight, - 2_000, + 4_000, ); const zoomedHeight = await page.evaluate( () => window.__godsEyeView.viewer.camera.positionCartographic.height, ); check( - `ZOOM IN flies to ${Math.round(fitHeight / 1000)} km within 2 s`, + `ZOOM IN lands at ${Math.round(fitHeight / 1000)} km`, zoomed, `height=${Math.round(zoomedHeight)} target=${Math.round(fitHeight)}`, ); @@ -1258,7 +1264,7 @@ try { ); await wait(100); } - const tail = (key) => new Set(samples.slice(8).map((s) => s[key])); + const tail = (key) => new Set(samples.slice(-8).map((s) => s[key])); return { settled: tail('imagery').size === 1 && tail('cctv').size === 1, detail: `imagery=${samples.map((s) => s.imagery).join(',')} cctv=${samples.map((s) => s.cctv).join(',')}`, From 8d1f69a8731b6cab3fb95f1bf6d5cd05d89f7e85 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 10:34:28 -0500 Subject: [PATCH 013/202] ci: prove Recent Imagery gate repair --- .../recent-imagery-gate-repair-proof.yml | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/recent-imagery-gate-repair-proof.yml diff --git a/.github/workflows/recent-imagery-gate-repair-proof.yml b/.github/workflows/recent-imagery-gate-repair-proof.yml new file mode 100644 index 000000000..b613d9561 --- /dev/null +++ b/.github/workflows/recent-imagery-gate-repair-proof.yml @@ -0,0 +1,37 @@ +name: Recent Imagery gate repair proof + +on: + push: + branches: + - fix/recent-imagery-qa-relative-layout-20261003 + +permissions: + contents: read + +jobs: + browser-proof: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24.14.0 + cache: npm + - run: npm ci + - name: Start GEV + run: | + npm run dev -- --host 127.0.0.1 --port 4173 > /tmp/gev-dev.log 2>&1 & + for i in {1..90}; do + if curl -fsS http://127.0.0.1:4173/ >/dev/null; then + exit 0 + fi + sleep 1 + done + cat /tmp/gev-dev.log + exit 1 + - name: Recent Imagery live gate + run: node scripts/qa-recent-imagery.mjs + - name: Dev log on failure + if: failure() + run: cat /tmp/gev-dev.log From 8f5710e7cb65fae957cd703756ca88838a5a9c69 Mon Sep 17 00:00:00 2001 From: devv-shayan Date: Sun, 4 Oct 2026 17:24:30 +0500 Subject: [PATCH 014/202] fix(voice): track OAuth login lifecycle and storage readiness --- README.md | 6 + docs/CURRENT-STATE.md | 2 + scripts/qa-voice-auth.mjs | 291 ++++++++++++++++++++++++++ server/providers/openai.js | 9 +- server/providers/openai/codex-auth.js | 228 +++++++++++++++++--- server/providers/openai/realtime.js | 63 +----- src/codexOauthRealtime.test.mjs | 109 +++++++++- src/keySetup.js | 34 ++- src/keySetup.test.mjs | 36 ++++ src/tooling/localServices.test.mjs | 234 +++++++++++++++++++-- 10 files changed, 906 insertions(+), 106 deletions(-) create mode 100644 scripts/qa-voice-auth.mjs diff --git a/README.md b/README.md index 37a0b0553..16a2ebf7e 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,12 @@ _Why cockpit mode exists: you're riding a real aircraft over real terrain — an > Voice can use an **OpenAI API key** or, on the same machine, an opt-in signed-in **ChatGPT/Codex OAuth** session. The API-key path remains the default. Selecting ChatGPT OAuth in Provider Settings reuses an existing local sign-in or starts the Codex browser sign-in flow and selects OAuth when it completes. The OpenAI key still drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. +The OAuth option is experimental. OpenAI documents ChatGPT plan usage for the [Responses API](https://developers.openai.com/siwc/token-sharing-open-source/models-and-inference); a supported third-party Realtime contract for reusing Codex tokens has not been confirmed. A successful token mint does not establish billing or support terms. + +GEV reads Codex's file credentials but does not write or refresh them. It checks token shape and expiry, and reports expired sign-in as requiring re-authentication. The login button runs `codex login`, permits one pending attempt, and reports exit failures or a two-minute timeout. `CODEX_HOME` must match the directory that Codex uses. If `CODEX_AUTH_JSON` points elsewhere, sign in separately or fix that mismatch before using the button. Codex credential storage must use `auth.json`; Keychain-only and in-memory storage are not readable by GEV. See [Codex credential storage](https://learn.chatgpt.com/docs/auth#credential-storage). + +For a live voice check, run `node scripts/qa-voice-auth.mjs http://127.0.0.1:4189 oauth microphone.wav` against your local dev server. Use a WAV with three seconds of leading silence followed by a short spoken reply request. Replace `oauth` with `api-key` to check the API-key path. The check verifies audio input, a completed spoken reply, usage, resource cleanup, and reconnect. It saves screenshots and credential-free results under `.gev-logs/voice-auth/`. + Click **GEV MIC**, grant the microphone, and just talk. This is more than a voice-controlled remote: - **🧠 It knows what it's looking at.** The agent pulls live scene context before answering — including coordinates, street names, active layers, and view scale. Ask _"what city is this?"_ mid-flight and it knows. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index e768be94e..2834a5b1c 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3924,6 +3924,8 @@ and unreachable upstream (502/504) separately from road geometry. 3. **Every count names its scope in words** — "42 in your window", "8 in view", "about 30 within 250 km of Austin" — never a bare number. `analyst_query` returns `scopeLabel` so this is mechanical. Two different numbers with named scopes are not a contradiction. 4. **The loaded-data caveat is stated once when relevant**: counts cover loaded data, and the flights layer loads where you look (appended to `coverage.note` for radius/view scopes over viewport-loaded layers). - **Degradation**: API-key mode without `OPENAI_API_KEY` returns 503 as before. Selecting OAuth without a usable local ChatGPT/Codex sign-in starts the local `codex login` browser flow, polls for the resulting local token, and selects OAuth when sign-in completes. If the Codex login flow cannot start or does not complete, the rest of the app is unaffected. +- **Login lifecycle and storage**: one pending Codex login is shared by the login and status routes. Process failure, success without readable credentials, and a two-minute timeout each report a clear failure; timeout and server shutdown cancel the owned process. The launcher resolves `CODEX_HOME` before changing directories and refuses a mismatched `CODEX_AUTH_JSON`. GEV reads file credentials only, never writes or refreshes them, and does not require the vendor refresh token. Expired access tokens report `CODEX_OAUTH_REAUTH_REQUIRED`. OAuth status and login routes also use the same-site request gate. +- **Support boundary**: the local OAuth option is experimental. OpenAI's documented Sign in with ChatGPT plan-sharing inference endpoint is `/v1/responses`; support for third-party reuse of Codex credentials at `/v1/realtime/client_secrets` remains unconfirmed. Technical success does not establish a supported auth or billing contract. ### AI HUD Summary (June 2026) diff --git a/scripts/qa-voice-auth.mjs b/scripts/qa-voice-auth.mjs new file mode 100644 index 000000000..9c8cf0c1e --- /dev/null +++ b/scripts/qa-voice-auth.mjs @@ -0,0 +1,291 @@ +#!/usr/bin/env node +/** + * Live Realtime auth acceptance with a prerecorded Chromium microphone. + * Run against the local dev server after configuring the selected credential: + * node scripts/qa-voice-auth.mjs http://127.0.0.1:4189 oauth microphone.wav + * Use a short spoken reply request with three seconds of leading silence. + * Saves only usage, transport metadata, and screenshots; never credentials. + */ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import puppeteer from 'puppeteer'; + +const root = fileURLToPath(new URL('../', import.meta.url)); +const url = new URL(process.argv[2] || 'http://127.0.0.1:4189'); +const auth = process.argv[3] || 'oauth'; +assert.ok(['oauth', 'api-key'].includes(auth), 'Choose oauth or api-key'); +assert.ok( + ['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname), + 'Use a local server', +); +assert.ok(process.argv[4], 'Provide a prerecorded microphone WAV'); +const wav = path.resolve(process.argv[4]); +assert.ok(fs.existsSync(wav), 'Microphone WAV must exist'); +url.searchParams.set('welcome', '0'); +url.searchParams.set('setup', '1'); +const output = path.join(root, '.gev-logs', 'voice-auth'); +fs.mkdirSync(output, { recursive: true }); +const executablePath = + process.env.PUPPETEER_EXECUTABLE_PATH || + (fs.existsSync('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome') + ? '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' + : puppeteer.executablePath()); +const browser = await puppeteer.launch({ + headless: true, + executablePath, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--use-fake-device-for-media-stream', + `--use-file-for-fake-audio-capture=${wav}%noloop`, + '--autoplay-policy=no-user-gesture-required', + '--disable-background-timer-throttling', + '--window-size=1440,900', + ], +}); +const result = { + ok: false, + auth, + input: 'prerecorded microphone', + mints: [], + negotiations: [], +}; +let page; +const readSession = () => + page.evaluate(async () => { + const voice = window.__godsEyeView.voiceCommands; + const stats = voice.pc ? [...(await voice.pc.getStats()).values()] : []; + const input = stats.find( + (entry) => entry.type === 'outbound-rtp' && entry.kind === 'audio', + ); + const source = stats.find( + (entry) => entry.type === 'media-source' && entry.kind === 'audio', + ); + const audio = stats.find( + (entry) => entry.type === 'inbound-rtp' && entry.kind === 'audio', + ); + const meter = document.getElementById('gev-voice-cost-value'); + return { + ...window.__voiceAuthEvidence, + connection: voice.pc?.connectionState, + microphoneEnabled: voice.stream + ?.getAudioTracks() + .every((track) => track.enabled), + inputBytes: input?.bytesSent || 0, + inputAudioEnergy: source?.totalAudioEnergy ?? null, + outputBytes: audio?.bytesReceived || 0, + outputAudioEnergy: audio?.totalAudioEnergy || 0, + playbackTime: voice.audioEl?.currentTime || 0, + meter: { + text: meter?.textContent, + usage: meter?.title, + visible: Boolean(meter?.getClientRects().length), + }, + }; + }); +try { + await browser + .defaultBrowserContext() + .overridePermissions(url.origin, ['microphone']); + page = await browser.newPage(); + await page.setViewport({ width: 1440, height: 900 }); + page.on('response', (response) => { + const route = new URL(response.url()).pathname; + if (route === '/api/realtime/token') + result.mints.push({ + status: response.status(), + auth: response.headers()['x-gev-voice-auth'], + model: response.headers()['x-gev-voice-model'], + }); + if ( + route === '/v1/realtime/calls' && + response.request().method() === 'POST' + ) + result.negotiations.push({ status: response.status() }); + }); + // Observe the application's real transport. Do not replace fetch or replies. + await page.evaluateOnNewDocument(() => { + window.__voiceAuthEvidence = { + speechStopped: 0, + transcripts: [], + replies: [], + responses: [], + errors: [], + eventCounts: {}, + }; + const create = RTCPeerConnection.prototype.createDataChannel; + RTCPeerConnection.prototype.createDataChannel = function (...args) { + const channel = create.apply(this, args); + channel.addEventListener('message', ({ data }) => { + let event; + try { + event = JSON.parse(data); + } catch { + return; + } + const evidence = window.__voiceAuthEvidence; + evidence.eventCounts[event.type] = + (evidence.eventCounts[event.type] || 0) + 1; + if (event.type === 'input_audio_buffer.speech_stopped') + evidence.speechStopped += 1; + if ( + event.type === 'conversation.item.input_audio_transcription.completed' + ) + evidence.transcripts.push(event.transcript); + if (event.type === 'response.output_audio_transcript.done') + evidence.replies.push(event.transcript); + if (event.type === 'response.done') + evidence.responses.push({ + status: event.response?.status, + inputTokens: event.response?.usage?.input_tokens, + outputTokens: event.response?.usage?.output_tokens, + }); + if (event.type === 'error') + evidence.errors.push({ code: event.error?.code }); + }); + return channel; + }; + }); + await page.goto(url.href, { waitUntil: 'domcontentloaded', timeout: 30_000 }); + await page.waitForFunction( + () => + window.__godsEyeView?.voiceCommands && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 45_000 }, + ); + await page.waitForSelector('[data-cloud-voice-auth-toggle]', { + visible: true, + }); + await page.waitForFunction( + () => + getComputedStyle(document.getElementById('key-setup')).opacity === '1', + ); + const selected = await page.evaluate( + () => localStorage.getItem('godsEyeView.voice.cloudAuth') || 'api-key', + ); + if (selected !== auth) { + await page.click('[data-cloud-voice-auth-toggle]'); + await page.waitForFunction( + (mode) => localStorage.getItem('godsEyeView.voice.cloudAuth') === mode, + { timeout: 125_000 }, + auth, + ); + } + await page.click('[data-key-setup-close]'); + await page.click('#gev-voice-button'); + await page.waitForFunction( + () => + window.__godsEyeView.voiceCommands.dc?.readyState === 'open' || + window.__godsEyeView.voiceCommands.status === 'error', + { timeout: 35_000 }, + ); + assert.equal( + await page.evaluate( + () => window.__godsEyeView.voiceCommands.dc?.readyState, + ), + 'open', + 'Realtime channel must open', + ); + await page.waitForFunction( + () => + window.__voiceAuthEvidence.speechStopped > 0 && + window.__voiceAuthEvidence.responses.some( + (response) => + response.status === 'completed' && response.outputTokens > 0, + ), + { timeout: 45_000 }, + ); + await page.waitForFunction( + () => + window.__voiceAuthEvidence.eventCounts['output_audio_buffer.stopped'] > 0, + { timeout: 30_000 }, + ); + result.session = await readSession(); + assert.ok( + result.session.inputBytes > 0 && result.session.outputBytes > 0, + 'Audio must flow in both directions', + ); + assert.ok( + result.session.outputAudioEnergy > 0 && result.session.playbackTime > 0, + 'Reply audio must play', + ); + assert.equal( + result.session.meter.text === 'COST UNKNOWN', + auth === 'oauth', + 'Use the correct accounting display', + ); + await page.screenshot({ path: path.join(output, `${auth}-usage.png`) }); + await page.click('#key-setup-chip'); + await page.waitForSelector('[data-cloud-voice-auth-toggle]', { + visible: true, + }); + await page.waitForFunction( + () => + getComputedStyle(document.getElementById('key-setup')).opacity === '1', + ); + await page.screenshot({ path: path.join(output, `${auth}-settings.png`) }); + await page.click('[data-key-setup-close]'); + result.stopped = await page.evaluate(() => { + const voice = window.__godsEyeView.voiceCommands; + const pc = voice.pc; + const tracks = voice.stream?.getTracks() || []; + voice.session.stop(); + return ( + pc.connectionState === 'closed' && + tracks.every((track) => track.readyState === 'ended') + ); + }); + assert.ok(result.stopped, 'Stop must close the peer and microphone tracks'); + await page.click('#gev-voice-button'); + await page.waitForFunction( + () => + window.__godsEyeView.voiceCommands.dc?.readyState === 'open' || + window.__godsEyeView.voiceCommands.status === 'error', + { timeout: 35_000 }, + ); + assert.equal( + await page.evaluate( + () => window.__godsEyeView.voiceCommands.dc?.readyState, + ), + 'open', + 'Reconnect must work', + ); + assert.equal( + result.mints.length, + 2, + 'Reconnect must request a fresh client secret', + ); + assert.ok( + result.mints.every( + (mint) => + mint.status === 200 && + mint.auth === (auth === 'oauth' ? 'codex-oauth' : 'api-key'), + ), + ); + assert.ok(result.negotiations.every((entry) => entry.status === 201)); + result.ok = true; +} catch (error) { + result.error = error.message; + if (page) { + result.session = await readSession().catch(() => null); + result.detail = await page + .$eval('#gev-voice-detail', (element) => element.textContent) + .catch(() => null); + await page + .screenshot({ path: path.join(output, `${auth}-failure.png`) }) + .catch(() => {}); + } +} finally { + await page + ?.evaluate(() => window.__godsEyeView?.voiceCommands?.session?.stop()) + .catch(() => {}); + await browser.close(); + fs.writeFileSync( + path.join(output, `${auth}-result.json`), + `${JSON.stringify(result, null, 2)}\n`, + ); +} +console.log(JSON.stringify(result, null, 2)); +if (!result.ok) process.exitCode = 1; diff --git a/server/providers/openai.js b/server/providers/openai.js index 6e8c65281..ad2161d6e 100644 --- a/server/providers/openai.js +++ b/server/providers/openai.js @@ -7,6 +7,7 @@ import { createRealtimeTokenHandler, } from './openai/realtime.js'; import { sameSiteGated } from './common/same-site.js'; +import { createCodexOAuthLogin } from './openai/codex-auth.js'; /** * Vite plugin: OpenAI Realtime ephemeral client secret. @@ -19,6 +20,7 @@ function openAiRealtimeProxy({ annotationGuidance, realtime = {}, } = {}) { + const oauthLogin = createCodexOAuthLogin(realtime); function install(middlewares) { // Cost-bearing and log endpoints refuse cross-site browser requests // (see server/providers/common/same-site.js and SECURITY.md). @@ -38,12 +40,12 @@ function openAiRealtimeProxy({ middlewares.use( '/api/realtime/oauth-status', - createRealtimeOAuthStatusHandler(realtime), + sameSiteGated(createRealtimeOAuthStatusHandler({ oauthLogin })), ); middlewares.use( '/api/realtime/oauth-login', - createRealtimeOAuthLoginHandler(realtime), + sameSiteGated(createRealtimeOAuthLoginHandler({ oauthLogin })), ); } @@ -51,10 +53,13 @@ function openAiRealtimeProxy({ name: 'openai-realtime-proxy', configureServer(server) { install(server.middlewares); + server.httpServer?.once('close', oauthLogin.dispose); }, configurePreviewServer(server) { install(server.middlewares); + server.httpServer?.once('close', oauthLogin.dispose); }, + closeBundle: oauthLogin.dispose, }; } diff --git a/server/providers/openai/codex-auth.js b/server/providers/openai/codex-auth.js index 68c20e929..376830cf4 100644 --- a/server/providers/openai/codex-auth.js +++ b/server/providers/openai/codex-auth.js @@ -34,6 +34,12 @@ export function preferCodexOAuth(environment = process.env) { ); } +function codexOAuthError(message, code = 'CODEX_OAUTH_UNAVAILABLE') { + const error = new Error(message); + error.code = code; + return error; +} + export function readCodexOAuthAccessToken({ authPath = codexAuthPath(), readFile = readFileSync, @@ -46,10 +52,10 @@ export function readCodexOAuthAccessToken({ try { parsed = JSON.parse(readFile(authPath, 'utf8')); } catch { - throw new Error(unavailable); + throw codexOAuthError(unavailable); } if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - throw new Error(unavailable); + throw codexOAuthError(unavailable); } const mode = String(parsed.auth_mode || '') .trim() @@ -58,15 +64,8 @@ export function readCodexOAuthAccessToken({ ? mode === 'chatgpt' || mode === 'chatgptauthtokens' : typeof parsed.OPENAI_API_KEY !== 'string'; const token = parsed.tokens?.access_token; - const refresh = parsed.tokens?.refresh_token; - if ( - !chatGptMode || - typeof token !== 'string' || - !token.trim() || - typeof refresh !== 'string' || - !refresh.trim() - ) { - throw new Error(unavailable); + if (!chatGptMode || typeof token !== 'string' || !token.trim()) { + throw codexOAuthError(unavailable); } // This only checks token shape and expiry. OpenAI verifies the signature when @@ -76,33 +75,34 @@ export function readCodexOAuthAccessToken({ parts.length !== 3 || parts.some((part) => !/^[A-Za-z0-9_-]+$/.test(part)) ) { - throw new Error(unavailable); + throw codexOAuthError(unavailable); } let payload; try { payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')); } catch { - throw new Error(unavailable); + throw codexOAuthError(unavailable); } if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { - throw new Error(unavailable); + throw codexOAuthError(unavailable); } let expiresAtMs; if (Object.hasOwn(payload, 'exp')) { if (typeof payload.exp !== 'number' || !Number.isFinite(payload.exp)) { - throw new Error(unavailable); + throw codexOAuthError(unavailable); } expiresAtMs = payload.exp * 1000; } else { try { expiresAtMs = statFile(authPath).mtimeMs + 60 * 60 * 1000; } catch { - throw new Error(unavailable); + throw codexOAuthError(unavailable); } } if (!Number.isFinite(expiresAtMs) || expiresAtMs <= nowMs + 60_000) { - throw new Error( - 'ChatGPT sign-in has expired. Run codex login and try again.', + throw codexOAuthError( + 'ChatGPT sign-in has expired. Re-authenticate in Provider Settings or run codex login and try again.', + 'CODEX_OAUTH_REAUTH_REQUIRED', ); } return token; @@ -144,28 +144,202 @@ export function startCodexChatGptLogin({ home = os.homedir(), executable = resolveCodexExecutable({ environment, home }), } = {}) { + // Codex owns its credential storage. An explicit read path must agree with + // the CLI's write path; never silently sign in to another credential store. + const configuredHome = String(environment.CODEX_HOME || '').trim(); + const loginHome = configuredHome + ? path.resolve(configuredHome) + : path.join(home, '.codex'); + if ( + codexAuthPath({ environment, home }) !== path.join(loginHome, 'auth.json') + ) { + return Promise.reject( + codexOAuthError( + 'CODEX_AUTH_JSON does not match the Codex login storage path. Set CODEX_HOME to the matching directory or sign in to Codex separately.', + 'CODEX_OAUTH_STORAGE_MISMATCH', + ), + ); + } + return new Promise((resolve, reject) => { let child; + let spawned = false; + let finish; + const completion = new Promise((done) => { + finish = done; + }); + const fail = () => { + reject( + codexOAuthError( + 'Could not start ChatGPT sign-in. Check that Codex is installed, or set CODEX_BIN to its executable.', + 'CODEX_OAUTH_LOGIN_FAILED', + ), + ); + finish({ exitCode: null }); + }; try { child = spawnImpl(executable, ['login'], { cwd: home, detached: true, stdio: 'ignore', - env: environment, + env: { ...environment, CODEX_HOME: loginHome }, }); - } catch (error) { - reject(error); + } catch { + fail(); return; } - child.once('error', reject); + child.once('error', () => { + if (!spawned) fail(); + else finish({ exitCode: null }); + }); + child.once('exit', (exitCode) => finish({ exitCode })); child.once('spawn', () => { + spawned = true; child.unref?.(); - resolve({ executable }); + resolve({ + completion, + cancel() { + try { + child.kill?.('SIGTERM'); + } catch { + /* Already stopped. */ + } + }, + }); }); - }).catch((error) => { - throw new Error( - `Could not start ChatGPT sign-in with Codex: ${error?.message || error}`, - ); }); } + +/** One local login owner shared by the launch and status endpoints. */ +export function createCodexOAuthLogin({ + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), + startOAuthLogin = () => startCodexChatGptLogin(), + loginTimeoutMs = 120_000, + setTimer = setTimeout, + clearTimer = clearTimeout, +} = {}) { + let active = null; + let failure = null; + let disposed = false; + + function credentials() { + try { + if (resolveOAuthAccessToken()) return { available: true }; + } catch (error) { + const code = error?.code || 'CODEX_OAUTH_UNAVAILABLE'; + return { + available: false, + error: error?.message || 'ChatGPT sign-in is unavailable', + code, + reauthRequired: code === 'CODEX_OAUTH_REAUTH_REQUIRED', + }; + } + return { + available: false, + error: 'ChatGPT sign-in is unavailable', + code: 'CODEX_OAUTH_UNAVAILABLE', + reauthRequired: false, + }; + } + + function status() { + const current = credentials(); + if (current.available) return current; + if (failure) return { available: false, ...failure, loginFailed: true }; + return active ? { ...current, pending: true } : current; + } + + function finish(attempt, error) { + if (active !== attempt) return; + clearTimer(attempt.timer); + active = null; + failure = error; + } + + async function start() { + if (disposed) + throw codexOAuthError( + 'ChatGPT sign-in is unavailable because the server has stopped.', + 'CODEX_OAUTH_LOGIN_FAILED', + ); + if (credentials().available) + return { available: true, started: false, pending: false }; + if (active) return { available: false, started: false, pending: true }; + + // Reserve the attempt before awaiting spawn so concurrent POSTs share it. + const attempt = { process: null, timer: null }; + active = attempt; + failure = null; + attempt.timer = setTimer(() => { + finish(attempt, { + code: 'CODEX_OAUTH_LOGIN_TIMEOUT', + error: + 'ChatGPT sign-in timed out. Click USE CHATGPT OAUTH to try again.', + }); + attempt.process?.cancel(); + }, loginTimeoutMs); + attempt.timer?.unref?.(); + + try { + const login = await startOAuthLogin(); + if (!login?.completion || typeof login.cancel !== 'function') + throw new Error('Invalid login process'); + attempt.process = login; + if (active !== attempt) { + login.cancel(); + return { ...status(), started: false, pending: false }; + } + Promise.resolve(login.completion).then( + (result) => { + if (active !== attempt) return; + if (result?.exitCode !== 0) { + finish(attempt, { + code: 'CODEX_OAUTH_LOGIN_FAILED', + error: + 'ChatGPT sign-in did not complete. Try again in Provider Settings, or run codex login separately.', + }); + } else if (!credentials().available) { + finish(attempt, { + code: 'CODEX_OAUTH_STORAGE_UNAVAILABLE', + error: + 'Codex sign-in finished, but its configured auth.json has no usable ChatGPT token. Check CODEX_HOME and Codex credential storage. GEV can read file storage only.', + }); + } else finish(attempt, null); + }, + () => + finish(attempt, { + code: 'CODEX_OAUTH_LOGIN_FAILED', + error: + 'ChatGPT sign-in did not complete. Try again in Provider Settings.', + }), + ); + return { available: false, started: true, pending: true }; + } catch (error) { + const code = + error?.code === 'CODEX_OAUTH_STORAGE_MISMATCH' + ? error.code + : 'CODEX_OAUTH_LOGIN_FAILED'; + finish(attempt, { + code, + error: + code === 'CODEX_OAUTH_STORAGE_MISMATCH' + ? error.message + : 'Could not start ChatGPT sign-in. Check that Codex is installed, or set CODEX_BIN to its executable.', + }); + return { ...status(), started: false, pending: false }; + } + } + + return { + status, + start, + dispose() { + disposed = true; + if (!active) return; + const attempt = active; + finish(attempt, null); + attempt.process?.cancel(); + }, + }; +} diff --git a/server/providers/openai/realtime.js b/server/providers/openai/realtime.js index 44f0f9529..588f43c77 100644 --- a/server/providers/openai/realtime.js +++ b/server/providers/openai/realtime.js @@ -16,7 +16,7 @@ import { GEV_REALTIME_TOOLS } from './tools.js'; import { preferCodexOAuth, readCodexOAuthAccessToken, - startCodexChatGptLogin, + createCodexOAuthLogin, } from './codex-auth.js'; const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']); @@ -53,7 +53,7 @@ function resolveRealtimeAuthMode(query) { } function createRealtimeOAuthStatusHandler({ - resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), + oauthLogin = createCodexOAuthLogin(), } = {}) { return (req, res) => { res.setHeader('Cache-Control', 'no-store'); @@ -73,30 +73,14 @@ function createRealtimeOAuthStatusHandler({ ); return; } - try { - const token = resolveOAuthAccessToken(); - if (!token) throw new Error('ChatGPT sign-in is unavailable'); - res.statusCode = 200; - res.end(JSON.stringify({ available: true })); - } catch (error) { - res.statusCode = 200; - res.end( - JSON.stringify({ - available: false, - error: error?.message || 'ChatGPT sign-in is unavailable', - }), - ); - } + res.statusCode = 200; + res.end(JSON.stringify(oauthLogin.status())); }; } function createRealtimeOAuthLoginHandler({ - resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), - startOAuthLogin = () => startCodexChatGptLogin(), - now = () => Date.now(), - restartCooldownMs = 30_000, + oauthLogin = createCodexOAuthLogin(), } = {}) { - let lastStartedAt = 0; return async (req, res) => { res.setHeader('Cache-Control', 'no-store'); res.setHeader('Content-Type', 'application/json'); @@ -127,42 +111,15 @@ function createRealtimeOAuthLoginHandler({ } try { - if (resolveOAuthAccessToken()) { - res.statusCode = 200; - res.end( - JSON.stringify({ available: true, started: false, pending: false }), - ); - return; - } + const result = await oauthLogin.start(); + res.statusCode = result.available ? 200 : result.loginFailed ? 503 : 202; + res.end(JSON.stringify(result)); } catch { - // No usable local token yet; launch the supported Codex sign-in flow. - } - - const currentTime = now(); - if ( - lastStartedAt > 0 && - currentTime - lastStartedAt < Math.max(0, restartCooldownMs) - ) { - res.statusCode = 202; - res.end( - JSON.stringify({ available: false, started: false, pending: true }), - ); - return; - } - - try { - await startOAuthLogin(); - lastStartedAt = currentTime; - res.statusCode = 202; - res.end( - JSON.stringify({ available: false, started: true, pending: true }), - ); - } catch (error) { res.statusCode = 503; res.end( JSON.stringify({ available: false, - error: error?.message || 'Could not start ChatGPT sign-in', + error: 'Could not start ChatGPT sign-in', }), ); } @@ -242,7 +199,7 @@ function createRealtimeTokenHandler({ res.end( JSON.stringify({ error: error?.message || 'ChatGPT sign-in is unavailable', - code: 'CODEX_OAUTH_UNAVAILABLE', + code: error?.code || 'CODEX_OAUTH_UNAVAILABLE', }), ); return; diff --git a/src/codexOauthRealtime.test.mjs b/src/codexOauthRealtime.test.mjs index ffd2b169f..391082ff4 100644 --- a/src/codexOauthRealtime.test.mjs +++ b/src/codexOauthRealtime.test.mjs @@ -61,6 +61,19 @@ test('Codex OAuth reader accepts a current ChatGPT token and rejects unusable cr }), current, ); + assert.equal( + readCodexOAuthAccessToken({ + authPath: '/unused', + nowMs, + readFile: () => + JSON.stringify({ + auth_mode: 'chatgpt', + tokens: { access_token: current }, + }), + }), + current, + 'GEV does not require or consume the vendor refresh token', + ); for (const readFile of [ () => '{bad json', () => JSON.stringify({ tokens: {} }), @@ -70,11 +83,6 @@ test('Codex OAuth reader accepts a current ChatGPT token and rejects unusable cr OPENAI_API_KEY: 'sk-fixture', tokens: { access_token: current, refresh_token: 'refresh-fixture' }, }), - () => - JSON.stringify({ - auth_mode: 'chatgpt', - tokens: { access_token: current }, - }), () => auth('opaque-token'), () => auth('header.not-json.signature'), () => auth(jwt({ exp: 'tomorrow' })), @@ -92,13 +100,18 @@ test('Codex OAuth reader accepts a current ChatGPT token and rejects unusable cr test('Codex OAuth reader enforces expiry margin and bounds tokens without exp by file age', () => { for (const exp of [nowMs / 1000 - 1, nowMs / 1000 + 59]) { assert.throws( - () => + () => { readCodexOAuthAccessToken({ authPath: '/unused', nowMs, readFile: () => auth(jwt({ exp })), - }), - /sign-in has expired/, + }); + }, + (error) => { + assert.match(error.message, /sign-in has expired/); + assert.equal(error.code, 'CODEX_OAUTH_REAUTH_REQUIRED'); + return true; + }, ); } const noExp = jwt({ sub: 'fixture' }); @@ -162,10 +175,88 @@ test('Codex login resolves an installed executable before PATH and starts browse return child; }, }); - assert.deepEqual(await pending, { executable: '/fixture/codex' }); + const login = await pending; assert.equal(unref, 1); assert.equal(calls[0].command, '/fixture/codex'); assert.deepEqual(calls[0].args, ['login']); assert.equal(calls[0].options.detached, true); assert.equal(calls[0].options.stdio, 'ignore'); + assert.equal( + calls[0].options.env.CODEX_HOME, + path.join('/home/fixture', '.codex'), + ); + child.emit('exit', 1); + assert.deepEqual(await login.completion, { exitCode: 1 }); +}); + +test('Codex login refuses a read/write storage mismatch before spawning', async () => { + await assert.rejects( + startCodexChatGptLogin({ + environment: { CODEX_AUTH_JSON: '/tmp/another-auth.json' }, + home: '/home/fixture', + executable: 'codex', + spawnImpl: () => assert.fail('mismatched storage must not launch login'), + }), + { code: 'CODEX_OAUTH_STORAGE_MISMATCH' }, + ); +}); + +test('Codex login resolves relative CODEX_HOME before changing the child directory', async () => { + const child = new EventEmitter(); + const environment = { CODEX_HOME: './fixture-codex' }; + const login = await startCodexChatGptLogin({ + environment, + executable: 'codex', + spawnImpl(_command, _args, options) { + assert.equal( + options.env.CODEX_HOME, + path.resolve(environment.CODEX_HOME), + ); + queueMicrotask(() => child.emit('spawn')); + return child; + }, + }); + assert.equal( + environment.CODEX_HOME, + './fixture-codex', + 'caller environment is unchanged', + ); + child.emit('exit', 0); + assert.deepEqual(await login.completion, { exitCode: 0 }); +}); + +test('Codex login hides spawn errors and can cancel only its owned child', async () => { + const child = new EventEmitter(); + await assert.rejects( + startCodexChatGptLogin({ + executable: 'codex', + spawnImpl() { + queueMicrotask(() => + child.emit('error', new Error('private-token-fixture')), + ); + return child; + }, + }), + (error) => + error.code === 'CODEX_OAUTH_LOGIN_FAILED' && + !error.message.includes('private-token-fixture'), + ); + + const running = new EventEmitter(); + let cancelled = 0; + running.kill = (signal) => { + assert.equal(signal, 'SIGTERM'); + cancelled += 1; + running.emit('exit', null); + }; + const login = await startCodexChatGptLogin({ + executable: 'codex', + spawnImpl() { + queueMicrotask(() => running.emit('spawn')); + return running; + }, + }); + login.cancel(); + assert.equal(cancelled, 1); + assert.deepEqual(await login.completion, { exitCode: null }); }); diff --git a/src/keySetup.js b/src/keySetup.js index 5208c9063..ec8c1aa94 100644 --- a/src/keySetup.js +++ b/src/keySetup.js @@ -75,7 +75,13 @@ export async function waitForChatGptOAuth({ signal, }); const payload = await response.json().catch(() => ({})); + if (signal?.aborted) return false; if (response.ok && payload.available) return true; + if (!response.ok || payload.loginFailed) { + throw new Error( + payload.error || 'Could not check ChatGPT sign-in. Try again.', + ); + } if (now() >= deadline) break; await sleep(Math.max(0, pollMs), signal); } @@ -288,6 +294,7 @@ export async function initKeySetup({ const statusLine = root.querySelector('[data-key-setup-status]'); const defaultStatusText = statusLine?.textContent || ''; let busy = false; + let oauthBusy = false; let open = false; const render = (nextStatus) => { @@ -303,6 +310,10 @@ export async function initKeySetup({ rowsHost.textContent = ''; for (const key of status.keys || []) rowsHost.append(buildRow(documentRef, key)); + for (const button of rowsHost.querySelectorAll( + '[data-cloud-voice-auth-toggle]', + )) + button.disabled = oauthBusy; }; const visible = () => @@ -435,6 +446,7 @@ export async function initKeySetup({ '[data-cloud-voice-auth-toggle]', ); if (authButton && !disposed) { + if (oauthBusy) return; void (async () => { const current = readStoredCloudVoiceAuthMode(); if (current === 'oauth') { @@ -444,6 +456,7 @@ export async function initKeySetup({ return; } authButton.disabled = true; + oauthBusy = true; say('Checking local ChatGPT OAuth sign-in…'); try { const response = await doFetch('/api/realtime/oauth-status', { @@ -451,6 +464,11 @@ export async function initKeySetup({ signal: lifetime.signal, }); const payload = await response.json().catch(() => ({})); + if (lifetime.signal.aborted) return; + if (!response.ok) + throw new Error( + payload.error || 'Could not check ChatGPT sign-in.', + ); if (response.ok && payload.available) { writeStoredCloudVoiceAuthMode('oauth'); render(status); @@ -460,13 +478,18 @@ export async function initKeySetup({ return; } - say('Opening ChatGPT sign-in in your browser…'); + say( + payload.code === 'CODEX_OAUTH_REAUTH_REQUIRED' + ? 'ChatGPT sign-in expired. Opening sign-in again…' + : 'Opening ChatGPT sign-in in your browser…', + ); const loginResponse = await doFetch('/api/realtime/oauth-login', { method: 'POST', cache: 'no-store', signal: lifetime.signal, }); const loginPayload = await loginResponse.json().catch(() => ({})); + if (lifetime.signal.aborted) return; if (!loginResponse.ok && loginResponse.status !== 202) { say( loginPayload.error || @@ -491,9 +514,10 @@ export async function initKeySetup({ fetchImpl: doFetch, signal: lifetime.signal, }); + if (lifetime.signal.aborted) return; if (!available) { say( - 'ChatGPT sign-in is still pending. Finish it in the browser, then click USE CHATGPT OAUTH again.', + 'ChatGPT sign-in timed out. Click USE CHATGPT OAUTH to try again.', ); return; } @@ -506,7 +530,11 @@ export async function initKeySetup({ if (lifetime.signal.aborted) return; say(`OAuth check failed: ${error?.message || error}`); } finally { - authButton.disabled = false; + oauthBusy = false; + for (const button of rowsHost.querySelectorAll( + '[data-cloud-voice-auth-toggle]', + )) + button.disabled = false; } })(); return; diff --git a/src/keySetup.test.mjs b/src/keySetup.test.mjs index a03f2d556..4cdba1765 100644 --- a/src/keySetup.test.mjs +++ b/src/keySetup.test.mjs @@ -84,6 +84,42 @@ test('OAuth login polling stops as soon as local ChatGPT auth becomes available' assert.equal(checks, 3); }); +test('OAuth login polling stops on login or HTTP failure instead of waiting for timeout', async () => { + for (const result of [ + { ok: true, payload: { available: false, loginFailed: true, error: 'Sign-in failed.' } }, + { ok: false, payload: { error: 'Sign-in is local only.' } }, + ]) { + let checks = 0; + await assert.rejects(waitForChatGptOAuth({ + fetchImpl: async () => { checks += 1; return { ok: result.ok, json: async () => result.payload }; }, + sleep: () => assert.fail('failed login must stop polling'), + }), { message: result.payload.error }); + assert.equal(checks, 1); + } +}); + +test('OAuth login polling respects timeout and an aborted setup surface', async () => { + let clock = 0; + let checks = 0; + const poll = { fetchImpl: async () => { checks += 1; return { ok: true, json: async () => ({ available: false, pending: true }) }; } }; + assert.equal(await waitForChatGptOAuth({ ...poll, timeoutMs: 100, pollMs: 100, now: () => clock, sleep: async (ms) => { clock += ms; } }), false); + assert.equal(checks, 2); + const controller = new AbortController(); + controller.abort(); + assert.equal(await waitForChatGptOAuth({ ...poll, signal: controller.signal }), false); + assert.equal(checks, 2, 'aborted polling does not fetch'); +}); + +test('OAuth polling ignores a ready response received after the surface was aborted', async () => { + const controller = new AbortController(); + const available = await waitForChatGptOAuth({ + signal: controller.signal, + fetchImpl: async () => ({ ok: true, json: async () => { controller.abort(); return { available: true }; } }), + sleep: () => assert.fail('aborted polling must not continue'), + }); + assert.equal(available, false); +}); + test('the first Google key strips ONLY the keyless OSM basemap from the share hash', () => { const stripped = stripKeylessBasemapFromHash('lat=30.2&lon=-97.7&map=osm&style=normal'); assert.ok(stripped !== null); diff --git a/src/tooling/localServices.test.mjs b/src/tooling/localServices.test.mjs index 05296ec7e..f6f78f6b6 100644 --- a/src/tooling/localServices.test.mjs +++ b/src/tooling/localServices.test.mjs @@ -299,25 +299,72 @@ test('OAuth status reports local sign-in availability without returning the toke assert.equal(response.body.includes('fixture-oauth-secret'), false); }); -test('OAuth login endpoint launches local Codex sign-in once and never exposes credentials', async () => { - let starts = 0; - let clock = 1_000; +test('OAuth status and token minting report expired sign-in as re-auth required', async () => { + const expired = () => { + const error = new Error('ChatGPT sign-in has expired.'); + error.code = 'CODEX_OAUTH_REAUTH_REQUIRED'; + throw error; + }; const routes = install( openAiRealtimeProxy({ realtime: { - resolveOAuthAccessToken: () => { - throw new Error('not signed in'); - }, - startOAuthLogin: async () => { - starts += 1; - }, - now: () => clock, + resolveOAuthAccessToken: expired, + fetchImpl: () => assert.fail('expired OAuth must not reach OpenAI'), }, }), ); + + const status = await request(routes.get('/api/realtime/oauth-status')); + assert.equal(status.status, 200); + assert.deepEqual(status.json(), { + available: false, + error: 'ChatGPT sign-in has expired.', + code: 'CODEX_OAUTH_REAUTH_REQUIRED', + reauthRequired: true, + }); + + const token = await request(routes.get('/api/realtime/token'), { + url: '/?auth=oauth', + }); + assert.equal(token.status, 503); + assert.equal(token.json().code, 'CODEX_OAUTH_REAUTH_REQUIRED'); +}); + +test('OAuth login endpoint shares pending attempts and never exposes credentials', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let starts = 0; + let spawnReady; + let exit; + const plugin = openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => { + throw new Error('not signed in'); + }, + startOAuthLogin: async () => { + starts += 1; + await new Promise((resolve) => { + spawnReady = resolve; + }); + return { + completion: new Promise((resolve) => { + exit = resolve; + }), + cancel() {}, + }; + }, + }, + }); + t.after(plugin.closeBundle); + const routes = install(plugin); const handler = routes.get('/api/realtime/oauth-login'); - const started = await request(handler, { method: 'POST' }); + const firstRequest = request(handler, { method: 'POST' }); + const concurrent = await request(handler, { method: 'POST' }); + assert.equal(concurrent.status, 202); + assert.equal(concurrent.json().started, false); + assert.equal(starts, 1, 'reserve the attempt before awaiting spawn'); + spawnReady(); + const started = await firstRequest; assert.equal(started.status, 202); assert.deepEqual(started.json(), { available: false, @@ -326,7 +373,7 @@ test('OAuth login endpoint launches local Codex sign-in once and never exposes c }); assert.equal(starts, 1); - clock += 1_000; + t.mock.timers.tick(31_000); const duplicate = await request(handler, { method: 'POST' }); assert.equal(duplicate.status, 202); assert.equal(duplicate.json().pending, true); @@ -345,6 +392,169 @@ test('OAuth login endpoint launches local Codex sign-in once and never exposes c }); assert.equal(crossOrigin.status, 403); assert.equal(starts, 1); + + exit({ exitCode: 1 }); + await Promise.resolve(); + const failed = await request(routes.get('/api/realtime/oauth-status')); + assert.equal(failed.json().loginFailed, true); + assert.equal(failed.json().code, 'CODEX_OAUTH_LOGIN_FAILED'); +}); + +test('OAuth status follows process completion, permits retry, and checks credential storage', async (t) => { + let token = null; + let exit; + let starts = 0; + const plugin = openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => token, + startOAuthLogin: async () => { + starts += 1; + return { + completion: new Promise((resolve) => { + exit = resolve; + }), + cancel() {}, + }; + }, + }, + }); + t.after(plugin.closeBundle); + const routes = install(plugin); + const launch = () => + request(routes.get('/api/realtime/oauth-login'), { method: 'POST' }); + const status = () => request(routes.get('/api/realtime/oauth-status')); + await launch(); + exit({ exitCode: 0 }); + await Promise.resolve(); + assert.equal( + (await status()).json().code, + 'CODEX_OAUTH_STORAGE_UNAVAILABLE', + 'exit success alone is not usable sign-in', + ); + await launch(); + assert.equal(starts, 2, 'failed attempts can be retried immediately'); + token = 'fixture-private-token'; + exit({ exitCode: 0 }); + await Promise.resolve(); + const ready = await status(); + assert.deepEqual(ready.json(), { available: true }); + assert.equal(ready.body.includes(token), false); +}); + +test('OAuth login timeout and server shutdown cancel the owned process', async () => { + let timeout; + let cancelled = 0; + let cleared = 0; + const plugin = openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => null, + startOAuthLogin: async () => ({ + completion: new Promise(() => {}), + cancel() { + cancelled += 1; + }, + }), + setTimer(callback, duration) { + timeout = callback; + assert.equal(duration, 120_000); + return 1; + }, + clearTimer() { + cleared += 1; + }, + }, + }); + const routes = install(plugin); + const launch = () => + request(routes.get('/api/realtime/oauth-login'), { method: 'POST' }); + await launch(); + timeout(); + assert.equal(cancelled, 1); + const status = await request(routes.get('/api/realtime/oauth-status')); + assert.equal(status.json().code, 'CODEX_OAUTH_LOGIN_TIMEOUT'); + await launch(); + plugin.closeBundle(); + plugin.closeBundle(); + assert.equal(cancelled, 2, 'shutdown is idempotent'); + assert.equal(cleared, 2); +}); + +test('OAuth launch errors are safe and cross-origin status requests cannot read credentials', async () => { + let reads = 0; + const routes = install( + openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken() { + reads += 1; + return null; + }, + startOAuthLogin: async () => { + throw new Error('private-token-fixture'); + }, + }, + }), + ); + const remote = await request(routes.get('/api/realtime/oauth-status'), { + origin: 'https://example.com', + }); + assert.equal(remote.status, 403); + assert.equal(reads, 0); + const failed = await request(routes.get('/api/realtime/oauth-login'), { + method: 'POST', + }); + assert.equal(failed.status, 503); + assert.equal(failed.json().code, 'CODEX_OAUTH_LOGIN_FAILED'); + assert.equal(failed.body.includes('private-token-fixture'), false); +}); + +test('a login that spawns after timeout is cancelled and cannot replace a retry', async (t) => { + let timeout; + let releaseSpawn; + let cancelled = 0; + let launches = 0; + const plugin = openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => null, + startOAuthLogin: async () => { + launches += 1; + if (launches === 1) + await new Promise((resolve) => { + releaseSpawn = resolve; + }); + return { + completion: new Promise(() => {}), + cancel() { + cancelled += 1; + }, + }; + }, + setTimer(callback) { + timeout = callback; + return 1; + }, + clearTimer() {}, + }, + }); + t.after(plugin.closeBundle); + const routes = install(plugin); + const launch = () => + request(routes.get('/api/realtime/oauth-login'), { method: 'POST' }); + const first = launch(); + timeout(); + await launch(); + releaseSpawn(); + await first; + assert.equal(launches, 2); + assert.equal(cancelled, 1, 'late process must not become an orphan'); + const status = ( + await request(routes.get('/api/realtime/oauth-status')) + ).json(); + assert.equal(status.pending, true); + assert.equal( + status.loginFailed, + undefined, + 'the older attempt cannot fail the retry', + ); }); test('OAuth login endpoint selects an existing local sign-in without launching Codex', async () => { From 467c4e928d29aebd98359ac735d1d86f006a777a Mon Sep 17 00:00:00 2001 From: devv-shayan Date: Sun, 4 Oct 2026 21:41:22 +0500 Subject: [PATCH 015/202] test(voice): match API-key realtime auth response --- scripts/qa-voice-auth.mjs | 2 +- src/keySetup.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/qa-voice-auth.mjs b/scripts/qa-voice-auth.mjs index 9c8cf0c1e..800df9598 100644 --- a/scripts/qa-voice-auth.mjs +++ b/scripts/qa-voice-auth.mjs @@ -261,7 +261,7 @@ try { result.mints.every( (mint) => mint.status === 200 && - mint.auth === (auth === 'oauth' ? 'codex-oauth' : 'api-key'), + mint.auth === (auth === 'oauth' ? 'codex-oauth' : 'env'), ), ); assert.ok(result.negotiations.every((entry) => entry.status === 201)); diff --git a/src/keySetup.js b/src/keySetup.js index ec8c1aa94..1186e28a7 100644 --- a/src/keySetup.js +++ b/src/keySetup.js @@ -469,7 +469,7 @@ export async function initKeySetup({ throw new Error( payload.error || 'Could not check ChatGPT sign-in.', ); - if (response.ok && payload.available) { + if (payload.available) { writeStoredCloudVoiceAuthMode('oauth'); render(status); say( @@ -490,7 +490,7 @@ export async function initKeySetup({ }); const loginPayload = await loginResponse.json().catch(() => ({})); if (lifetime.signal.aborted) return; - if (!loginResponse.ok && loginResponse.status !== 202) { + if (!loginResponse.ok) { say( loginPayload.error || payload.error || From c6b43a365d3081e199ddce7b052b08b27568a62b Mon Sep 17 00:00:00 2001 From: daikaginza Date: Mon, 5 Oct 2026 16:50:30 +0700 Subject: [PATCH 016/202] fix(tiles): derive the tile origin from the configured TileJSON (#933) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `createVectorTileSource` took `tileJsonUrl` and `allowedOrigin` as independent options, and `createOpenFreeMapSource` hardcoded both to the default host. A caller that repointed only `tileJsonUrl` — the obvious single change — kept the default `allowedOrigin`, so the TileJSON's absolute upstream `tiles[0]` passed the origin check and every tile was fetched from the default host instead. No error, no warning, and nothing reached the configured mirror. `allowedOrigin` now defaults to the origin of `tileJsonUrl`, so repointing one option is enough, and a mirror that serves the default host's TileJSON verbatim fails with the existing `Invalid vector tile origin` instead of silently bypassing itself. The origin check is unchanged: a TileJSON still cannot point the client at a host the caller did not configure. All three existing callers pass `allowedOrigin` explicitly (`openFreeMap.js`, `alpr/source.js`, `traffic/flowSource.js`), and an explicit value still overrides the derived one, so behaviour is unchanged for every current call site. --- src/sources/openFreeMap.js | 11 +++++++-- src/sources/openFreeMap.test.mjs | 38 +++++++++++++++++++++++++++++ src/sources/vectorTiles.js | 18 +++++++++++++- src/sources/vectorTiles.test.mjs | 42 ++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 3 deletions(-) diff --git a/src/sources/openFreeMap.js b/src/sources/openFreeMap.js index 3f40af1e8..58208a1fd 100644 --- a/src/sources/openFreeMap.js +++ b/src/sources/openFreeMap.js @@ -214,11 +214,18 @@ function decodeLayers(bytes, z, x, y, layerNames) { return { roads, military }; } -/** Construct an immutable-version road/military tile source without starting I/O. */ +/** + * Construct an immutable-version road/military tile source without starting I/O. + * + * `allowedOrigin` is deliberately not set here: `createVectorTileSource` + * derives it from whichever `tileJsonUrl` is in effect, so a caller that + * repoints only `tileJsonUrl` gets tiles from that host or a named error — + * never a silent fall back to the default host (#933). Passing + * `allowedOrigin` explicitly still overrides both. + */ export function createOpenFreeMapSource(options = {}) { return createVectorTileSource({ tileJsonUrl: 'https://tiles.openfreemap.org/planet', - allowedOrigin: 'https://tiles.openfreemap.org', decode: decodeOpenFreeMapTile, maxEntries: 192, maxCacheBytes: 64 * 1024 * 1024, diff --git a/src/sources/openFreeMap.test.mjs b/src/sources/openFreeMap.test.mjs index f2a91098c..47a835917 100644 --- a/src/sources/openFreeMap.test.mjs +++ b/src/sources/openFreeMap.test.mjs @@ -320,3 +320,41 @@ test('Capitol, greenbelt and commercial property fixtures exclude non-public dri assert.ok(records.some((r) => r.area === area && !r.drivable)); } }); + +test('repointing tileJsonUrl at a mirror never falls back to the default tile host', async () => { + const urls = []; + const mirror = 'http://127.0.0.1:12348'; + const source = createOpenFreeMapSource({ + fetchImpl: async (url) => { + urls.push(url); + return url.endsWith('/planet') + ? Response.json({ tiles: [`${mirror}/planet/v/{z}/{x}/{y}.pbf`] }) + : new Response(fixture('ofm-austin-14-3743-6745.pbf')); + }, + tileJsonUrl: `${mirror}/planet`, + }); + await source.fetchBounds( + { south: 30.264, north: 30.271, west: -97.75, east: -97.74 }, + { zoom: 14 }, + ); + assert.ok(urls.length > 1, 'tiles were fetched'); + assert.deepEqual( + urls.filter((url) => !url.startsWith(mirror)), + [], + 'a repointed source must not fetch from tiles.openfreemap.org', + ); +}); + +test('a mirror that serves the default host TileJSON verbatim fails loudly', async () => { + // A transparent caching proxy returns upstream's TileJSON unchanged, whose + // tiles[0] is an absolute upstream URL. That must be an error, not 100% of + // tile traffic quietly leaving the configured mirror. + const source = createOpenFreeMapSource({ + fetchImpl: async () => + Response.json({ + tiles: ['https://tiles.openfreemap.org/planet/v/{z}/{x}/{y}.pbf'], + }), + tileJsonUrl: 'http://127.0.0.1:12348/planet', + }); + await assert.rejects(source.getMetadata(), { retryable: false }); +}); diff --git a/src/sources/vectorTiles.js b/src/sources/vectorTiles.js index 7eef0d3e7..6a90f4aaf 100644 --- a/src/sources/vectorTiles.js +++ b/src/sources/vectorTiles.js @@ -16,11 +16,27 @@ export function validTileBounds(box) { ); } +/** + * The origin a configured TileJSON is served from, or undefined when there is + * no absolute URL to read one off. Deriving `allowedOrigin` from it means + * repointing `tileJsonUrl` at a mirror cannot silently keep loading tiles from + * the old host: the origin check below then names the mismatch instead of + * passing it (#933). + */ +function tileJsonOrigin(tileJsonUrl) { + if (typeof tileJsonUrl !== 'string') return undefined; + try { + return new URL(tileJsonUrl).origin; + } catch { + return undefined; + } +} + /** A decoded XYZ cache with bounded workers, reads, retained bytes and request lifetimes. */ export function createVectorTileSource({ tileJsonUrl, template, - allowedOrigin, + allowedOrigin = tileJsonOrigin(tileJsonUrl), decode, fetchImpl = (...args) => globalThis.fetch(...args), maxTiles = 16, diff --git a/src/sources/vectorTiles.test.mjs b/src/sources/vectorTiles.test.mjs index bfc366964..92bde2f70 100644 --- a/src/sources/vectorTiles.test.mjs +++ b/src/sources/vectorTiles.test.mjs @@ -203,3 +203,45 @@ test('polygon-selected tiles keep nearest-first order and cached revisits issue 'over-budget selection is rejected before fetching', ); }); + +test('a configured tileJsonUrl sets the tile origin, and a foreign tiles[0] is still refused', async () => { + // Without an explicit allowedOrigin the mirror's own TileJSON is honoured… + const mirrored = createVectorTileSource({ + tileJsonUrl: 'http://127.0.0.1:12348/planet', + decode: () => [], + fetchImpl: async () => + Response.json({ tiles: ['http://127.0.0.1:12348/p/{z}/{x}/{y}.pbf'] }), + }); + assert.equal( + (await mirrored.getMetadata()).template, + 'http://127.0.0.1:12348/p/{z}/{x}/{y}.pbf', + ); + + // …while a TileJSON that names a different host is refused rather than + // silently fetched, which is what the origin check is for. + const foreign = createVectorTileSource({ + tileJsonUrl: 'http://127.0.0.1:12348/planet', + decode: () => [], + fetchImpl: async () => + Response.json({ + tiles: ['https://tiles.example/planet/{z}/{x}/{y}.pbf'], + }), + }); + await assert.rejects(foreign.getMetadata(), { retryable: false }); +}); + +test('an explicit allowedOrigin still wins over the one derived from tileJsonUrl', async () => { + const source = createVectorTileSource({ + tileJsonUrl: 'http://127.0.0.1:12348/planet', + allowedOrigin: 'https://tiles.example', + decode: () => [], + fetchImpl: async () => + Response.json({ + tiles: ['https://tiles.example/planet/{z}/{x}/{y}.pbf'], + }), + }); + assert.equal( + (await source.getMetadata()).template, + 'https://tiles.example/planet/{z}/{x}/{y}.pbf', + ); +}); From 1c3748e3bd6a186ecd2ef752b597ff762f4c2b59 Mon Sep 17 00:00:00 2001 From: Huanyi Xie <74809554+xiehuanyi@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:02:20 +0300 Subject: [PATCH 017/202] fix(satellites): recover TLE parsing after incomplete sets --- CHANGELOG.md | 4 ++++ docs/CURRENT-STATE.md | 4 ++++ src/sources/tle.js | 5 +++-- src/tools/queries/space.test.mjs | 21 +++++++++++++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39cf9b524..2ace48350 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +- Keep later satellites when a TLE catalog entry is missing an orbital line. + The parser skips the incomplete set and resumes at the next complete named + set, instead of losing the rest of the group (#906). + ## [0.2.1] — 2026-10-02 — God's Eye View through Your Agent (with security fixes and other features) - Ship each bundled data pack once. The region, marine, admin-boundary, diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 7c06724e2..19897526b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1049,6 +1049,10 @@ The Node-only `gods-eye-view/server/providers/space` export supplies factories; `sources/space` supplies fixed upstream URL builders with no I/O or environment access. Callers retain validation, transport and response policy. +The shared TLE text parser skips incomplete named sets and resumes at the next +complete set. A missing orbital line does not discard the later satellites in +the catalog; satellite layers and next-pass tools use the same parser. + ## Build configuration and local provider boundaries `vite.config.js` delegates to `server/standalone/vite.config.js`, which loads diff --git a/src/sources/tle.js b/src/sources/tle.js index 423651e6b..ac7aedc2c 100644 --- a/src/sources/tle.js +++ b/src/sources/tle.js @@ -1,6 +1,6 @@ /** * Parse three-line TLE catalog text into `{ name, line1, line2 }` entries. - * Blocks whose second and third lines are not TLE lines 1 and 2 are skipped. + * Malformed sets are skipped without dropping the valid sets that follow. */ export function parseTleText(text) { const lines = String(text) @@ -9,12 +9,13 @@ export function parseTleText(text) { .map((line) => line.trim()) .filter((line) => line.length > 0); const result = []; - for (let i = 0; i < lines.length - 2; i += 3) { + for (let i = 0; i < lines.length - 2; i++) { const name = lines[i]; const line1 = lines[i + 1]; const line2 = lines[i + 2]; if (line1.startsWith('1 ') && line2.startsWith('2 ')) { result.push({ name, line1, line2 }); + i += 2; } } return result; diff --git a/src/tools/queries/space.test.mjs b/src/tools/queries/space.test.mjs index 376cf54d7..f6c2a8df8 100644 --- a/src/tools/queries/space.test.mjs +++ b/src/tools/queries/space.test.mjs @@ -30,6 +30,27 @@ test('TLE text parses into named entries with catalog numbers', () => { assert.equal(tleCatalogNumber('1 xx'), null); }); +for (const missingLine of [1, 2]) { + test(`TLE parsing resumes after a set missing line ${missingLine}`, () => { + const sets = Array.from({ length: 8 }, (_, i) => { + const lines = TLE.trim().split('\n').slice(0, 3); + lines[0] = `SAT ${i}`; + return lines; + }); + sets[1].splice(missingLine, 1); + const entries = parseTleText(sets.flat().join('\r\n')); + assert.deepEqual( + entries.map((entry) => entry.name), + ['SAT 0', 'SAT 2', 'SAT 3', 'SAT 4', 'SAT 5', 'SAT 6', 'SAT 7'], + ); + assert.ok( + entries.every( + (entry) => entry.line1.startsWith('1 ') && entry.line2.startsWith('2 '), + ), + ); + }); +} + test('the next pass defaults to the ISS and reports times, peak and direction', async () => { const catalog = composeCatalog({ tools: coreTools, From 7485b16ded0158b804f1ac07da7e25b71eae0d5d Mon Sep 17 00:00:00 2001 From: daikaginza Date: Mon, 5 Oct 2026 17:12:41 +0700 Subject: [PATCH 018/202] fix(overpass): do not serve cached answers from a different upstream (#932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both disk caches fed by OVERPASS_UPSTREAMS key only on the query — the Overpass proxy on the sanitized request body, the installations proxy on the bbox — so an entry written against one instance is a hit for the same query against another. After an operator repoints OVERPASS_UPSTREAMS, the previous upstream's answers keep being served for 7 days, 30 for boundary-class queries, and indefinitely through the serve-stale path. An install that predates #742 has a cache written from the public instances that were the default then, and configuring your own instance does not displace it. Cached answers now carry a tag identifying the configured upstream set, and a disk read refuses an entry that does not match it, on the stale path too. With nothing configured any entry is still accepted: there is no instance to contradict, and serving last-good data at any age is what the not-configured path exists for. The tag is a hash of the resolved endpoint list, never the URLs; it is written to disk and compared on read, and is not sent to clients or surfaced in errors. One line is logged once per process when a foreign entry is ignored, naming the variable and what will happen, as local-receivers.js already does for a rejected feed. Cost worth naming: on upgrade every existing entry is untagged, so the first outage after upgrading serves nothing where it previously served stale data, until each query is fetched once from the configured instance. --- server/providers/military-installations.js | 13 +++- .../providers/military-installations/cache.js | 4 ++ server/providers/overpass/cache.js | 6 ++ server/providers/overpass/constants.js | 63 +++++++++++++++++-- server/providers/overpass/transport.js | 4 ++ src/overpassProxy.test.mjs | 52 ++++++++++++++- 6 files changed, 135 insertions(+), 7 deletions(-) diff --git a/server/providers/military-installations.js b/server/providers/military-installations.js index 08ad87d9f..7d252b8ed 100644 --- a/server/providers/military-installations.js +++ b/server/providers/military-installations.js @@ -9,7 +9,10 @@ import { fetchOverpassPayload, overpassNotConfigured, } from './overpass/transport.js'; -import { resolveOverpassUpstreams } from './overpass/constants.js'; +import { + resolveOverpassUpstreams, + resolveOverpassUpstreamTag, +} from './overpass/constants.js'; import { _militaryInstallationCache, trimMilitaryInstallationCache, @@ -74,7 +77,13 @@ function militaryInstallationsProxy() { retrievedAt: new Date().toISOString(), status: 'ready', }; - const entry = { payload, cachedAt: Date.now() }; + // Tagged with the configured upstream set so the disk layer does not serve + // this answer back after the operator repoints (see overpass/cache.js). + const entry = { + payload, + cachedAt: Date.now(), + upstreams: resolveOverpassUpstreamTag(), + }; _militaryInstallationCache.set(key, entry); trimMilitaryInstallationCache(); writeMilitaryInstallationDisk(key, entry); diff --git a/server/providers/military-installations/cache.js b/server/providers/military-installations/cache.js index c77627e29..4d77981e6 100644 --- a/server/providers/military-installations/cache.js +++ b/server/providers/military-installations/cache.js @@ -5,6 +5,7 @@ import { MILITARY_INSTALLATION_DISK_DIR, MILITARY_INSTALLATION_MAX_CACHE, } from './constants.js'; +import { cachedByConfiguredUpstream } from '../overpass/constants.js'; import path from 'node:path'; import { createHash } from 'node:crypto'; import { promises as fsp } from 'node:fs'; @@ -114,6 +115,9 @@ async function readMilitaryInstallationDisk( const entry = JSON.parse( await fsp.readFile(militaryInstallationDiskPath(cacheKey, dir), 'utf8'), ); + // Same reason as the Overpass disk cache: the key is the bbox alone, so an + // entry written against a different OVERPASS_UPSTREAMS is otherwise a hit. + if (!cachedByConfiguredUpstream(entry)) return null; if (!militaryInstallationDiskFresh(entry, maxAgeMs)) return null; return migrateMilitaryInstallationEntry(entry); } catch { diff --git a/server/providers/overpass/cache.js b/server/providers/overpass/cache.js index b4e53ab21..442f599a6 100644 --- a/server/providers/overpass/cache.js +++ b/server/providers/overpass/cache.js @@ -5,6 +5,7 @@ import { OVERPASS_DISK_DIR, OVERPASS_CACHE_MS, OVERPASS_CACHE_MAX_ENTRIES, + cachedByConfiguredUpstream, } from './constants.js'; import path from 'node:path'; import { createHash } from 'node:crypto'; @@ -47,6 +48,11 @@ async function readOverpassDisk(cacheKey, maxAgeMs) { // Older versions persisted 4xx refusals with normal data TTLs. Ignore // them on both fresh and stale reads so an upgrade can recover immediately. if (!overpassPayloadIsData(payload)) return null; + // The key is the query alone, so a file written by a different upstream — + // including the public instances that were the default before #742 — is a + // hit for the same query. Refuse it: the operator's configured instance + // must be what answers, on the stale path too. + if (!cachedByConfiguredUpstream(payload)) return null; if (Date.now() - payload.cachedAt > maxAgeMs) return null; return payload; } catch { diff --git a/server/providers/overpass/constants.js b/server/providers/overpass/constants.js index 40f244d11..2a0d4a3ef 100644 --- a/server/providers/overpass/constants.js +++ b/server/providers/overpass/constants.js @@ -1,4 +1,5 @@ import path from 'node:path'; +import { createHash } from 'node:crypto'; // --------------------------------------------------------------------------- // Overpass API proxy constants and cache state @@ -27,16 +28,68 @@ function parseOverpassUpstreams(raw) { return endpoints.slice(0, 8); } -let upstreamMemo = { raw: null, endpoints: [] }; +let upstreamMemo = { raw: null, endpoints: [], tag: 'none' }; + +/** + * Short, stable identifier for the configured upstream set. Cached responses + * carry it so an answer produced by a different instance is not served after + * the operator repoints `OVERPASS_UPSTREAMS` — the disk cache outlives the + * configuration change by 7 days, a month for boundary queries, and forever on + * the serve-stale path. It is a hash, never the URLs, and it is only ever + * compared on read: it is not sent to clients and does not appear in errors. + */ +function upstreamTag(endpoints) { + if (!endpoints.length) return 'none'; + return createHash('sha1') + .update(endpoints.join('\n')) + .digest('hex') + .slice(0, 12); +} /** Resolve after environment loading. Public Overpass instances are not used by default. */ function resolveOverpassUpstreams() { - const raw = process.env.OVERPASS_UPSTREAMS || ''; - if (upstreamMemo.raw !== raw) - upstreamMemo = { raw, endpoints: parseOverpassUpstreams(raw) }; + readUpstreamMemo(); return [...upstreamMemo.endpoints]; } +/** The configured-upstream tag stored alongside, and checked against, cache entries. */ +function resolveOverpassUpstreamTag() { + return readUpstreamMemo().tag; +} + +let reportedForeignCache = false; + +/** + * Was this cached entry produced by the upstream set configured now? + * + * With nothing configured there is no instance to contradict, and serving + * last-good data at any age is the point of the cache (the proxy's + * not-configured path depends on it), so any entry is accepted. Once an + * operator does configure an instance, only its own answers are served: an + * entry with no tag, or one from a different instance, is refused rather than + * matched on the query alone. Shared by both caches fed by these upstreams. + */ +function cachedByConfiguredUpstream(entry) { + const tag = resolveOverpassUpstreamTag(); + if (tag === 'none' || entry?.upstreams === tag) return true; + if (!reportedForeignCache) { + reportedForeignCache = true; + console.warn( + '[Overpass] ignoring cached responses produced by a different OVERPASS_UPSTREAMS; they will be refetched from the configured instance.', + ); + } + return false; +} + +function readUpstreamMemo() { + const raw = process.env.OVERPASS_UPSTREAMS || ''; + if (upstreamMemo.raw !== raw) { + const endpoints = parseOverpassUpstreams(raw); + upstreamMemo = { raw, endpoints, tag: upstreamTag(endpoints) }; + } + return upstreamMemo; +} + /** * TTL for FRESH cached Overpass responses (ms). Road geometry is static for * months — the original 45 s TTL forced a public-mirror round-trip on nearly @@ -153,6 +206,8 @@ export { OVERPASS_MAX_RESPONSE_BYTES, parseOverpassUpstreams, resolveOverpassUpstreams, + resolveOverpassUpstreamTag, + cachedByConfiguredUpstream, OVERPASS_USER_AGENT, OVERPASS_TIMEOUT_MS, }; diff --git a/server/providers/overpass/transport.js b/server/providers/overpass/transport.js index 86b9eae15..38c31bb08 100644 --- a/server/providers/overpass/transport.js +++ b/server/providers/overpass/transport.js @@ -1,6 +1,7 @@ import { OVERPASS_MAX_RESPONSE_BYTES, resolveOverpassUpstreams, + resolveOverpassUpstreamTag, OVERPASS_USER_AGENT, OVERPASS_TIMEOUT_MS, } from './constants.js'; @@ -187,6 +188,9 @@ async function fetchOverpassPayload( contentType: 'application/json', // Never retain a secret-bearing endpoint in cache or response metadata. endpoint: 'configured', + // Which configured instance produced this, as a hash rather than a URL, + // so a cached answer is not reused after the operator repoints. + upstreams: resolveOverpassUpstreamTag(), rateLimited: false, }; } catch { diff --git a/src/overpassProxy.test.mjs b/src/overpassProxy.test.mjs index 71b62aa2d..ed4539fda 100644 --- a/src/overpassProxy.test.mjs +++ b/src/overpassProxy.test.mjs @@ -12,6 +12,7 @@ import { createHash, randomUUID } from 'node:crypto'; import path from 'node:path'; import { Readable } from 'node:stream'; import createViteConfig, { fetchOverpassPayload, overpassPayloadIsData, readOverpassDisk } from '../vite.config.js'; +import { resolveOverpassUpstreamTag } from '../server/providers/overpass/constants.js'; const DATA = { status: 200, body: '{"elements":[]}' }; @@ -155,7 +156,13 @@ test('coalesced outage callers both receive last-good data, never a cached refus const directory = path.join(process.cwd(), '.gev-cache', 'overpass'); const file = path.join(directory, `${createHash('sha1').update(body).digest('hex')}.json`); await mkdir(directory, { recursive: true }); - const stale = { ...DATA, cachedAt: Date.now() - 40 * 86400000 }; + // Last-good data was fetched from the instance configured above, so it + // carries that instance's tag the way the transport now writes it. + const stale = { + ...DATA, + cachedAt: Date.now() - 40 * 86400000, + upstreams: resolveOverpassUpstreamTag(), + }; await writeFile(file, JSON.stringify(stale)); const entered = Promise.withResolvers(); const release = Promise.withResolvers(); @@ -188,3 +195,46 @@ test('coalesced outage callers both receive last-good data, never a cached refus } } }); + +test('a disk entry is not served after OVERPASS_UPSTREAMS is repointed', async (t) => { + const prior = process.env.OVERPASS_UPSTREAMS; + t.after(() => { + if (prior === undefined) delete process.env.OVERPASS_UPSTREAMS; + else process.env.OVERPASS_UPSTREAMS = prior; + }); + const key = `overpass-upstream-switch-${randomUUID()}`; + const directory = path.join(process.cwd(), '.gev-cache', 'overpass'); + const file = path.join( + directory, + `${createHash('sha1').update(key).digest('hex')}.json`, + ); + await mkdir(directory, { recursive: true }); + try { + // Written while one instance was configured... + process.env.OVERPASS_UPSTREAMS = 'https://first.example/api/interpreter'; + const first = resolveOverpassUpstreamTag(); + await writeFile( + file, + JSON.stringify({ ...DATA, cachedAt: Date.now(), upstreams: first }), + ); + assert.ok(await readOverpassDisk(key, 60000), 'its own entry is a hit'); + + // ...is not an answer once the operator points at a different one, on the + // stale path either: the key is the query alone, so it would otherwise hit. + process.env.OVERPASS_UPSTREAMS = 'https://second.example/api/interpreter'; + assert.notEqual(resolveOverpassUpstreamTag(), first, 'tag follows the env'); + assert.equal(await readOverpassDisk(key, 60000), null, 'fresh read refused'); + assert.equal( + await readOverpassDisk(key, Infinity), + null, + 'stale read refused', + ); + + // With nothing configured there is no instance to contradict, and serving + // last-good data is what the not-configured path exists for. + delete process.env.OVERPASS_UPSTREAMS; + assert.ok(await readOverpassDisk(key, Infinity), 'unconfigured still serves'); + } finally { + await unlink(file); + } +}); From 42543c1fa1f547db940dd46b0f5c781cff11be9a Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:13:53 +0200 Subject: [PATCH 019/202] feat(bikeshare): add Oslo, Bergen and Trondheim Bysykkel Add Norway's three Urban Sharing city-bike systems to the Bike Share layer and get_bike_share. The feeds are keyless GBFS on gbfs.urbansharing.com, which joins the proxy allowlist. The proxy now sends the Client-Identifier header the operators ask clients to send. Attributed under NLOD 2.0 in DATA_SOURCES.md and the data credits. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++ DATA_SOURCES.md | 1 + docs/CURRENT-STATE.md | 5 ++- server/providers/gbfs.js | 6 ++++ src/data/dataCredits.js | 7 ++++ src/data/gbfsProxy.test.mjs | 16 +++++++++ src/data/gbfsSource.js | 1 + src/layers/bikeshare/proxyRoute.test.mjs | 44 ++++++++++++++++++++++++ src/layers/bikeshare/registry.js | 44 ++++++++++++++++++++++++ src/tools/queries/mobility.test.mjs | 25 ++++++++++++++ 10 files changed, 154 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b433f8c06..a2fc2d1e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +- Add Norway's city bikes to the Bike Share layer: Oslo, Bergen and + Trondheim Bysykkel (Urban Sharing, ~440 stations). The feeds are keyless + GBFS on `gbfs.urbansharing.com`, which joins the proxy allowlist; the proxy + now sends the `Client-Identifier: gods-eye-view` header the operators ask + for. Attributed under NLOD 2.0. `get_bike_share` covers the three cities too. + - Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera list is one keyless GeoJSON request to the agency's OGC API view of its DATEX 3.1 CCTV table, so no DATEX account is needed. Every working camera diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 734e35a69..07782fd5e 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -56,6 +56,7 @@ How to read this: | **Open Calgary** (`data.calgary.ca`) | CCTV camera catalog + frames, Calgary | [Open Government Licence – City of Calgary](https://data.calgary.ca/stories/s/Open-Calgary-Terms-of-Use/u45n-7awa) — attribution REQUIRED | "Contains information licensed under the Open Government Licence – City of Calgary" | | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | +| **Oslo / Bergen / Trondheim Bysykkel** (Urban Sharing, `gbfs.urbansharing.com`) | Bikeshare availability, Oslo, Bergen and Trondheim city bikes. Keyless GBFS; the proxy sends the `Client-Identifier` header the operators ask for | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED ([Oslo](https://oslobysykkel.no/en/open-data/realtime), [Bergen](https://bergenbysykkel.no/en/apne-data/sanntid), [Trondheim](https://trondheimbysykkel.no/en/open-data/realtime)) | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel" | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | | **Metro Transit** (Metropolitan Council) | Transit layer live vehicles, Minneapolis–St Paul | Metro Transit / Metropolitan Council open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to Metro Transit / Metropolitan Council (courtesy text) | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d4fe022a8..8a25ccb8c 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -961,6 +961,9 @@ The station reader addresses the proxy as `/api/gbfs/`; the feed address is read from the path, and a query-string form is refused with a 400. GBFS refuses upstream redirects and enforces its 5 MiB response cap while streaming. The 12-second deadline includes reading the body, and rejected or stalled downloads are cancelled. Development and preview use the same handler. +Every upstream request carries `Client-Identifier: gods-eye-view`, which Urban +Sharing asks of clients of the Oslo, Bergen and Trondheim Bysykkel feeds +(`gbfs.urbansharing.com`); other operators ignore it. ## Remaining local service modules @@ -2922,7 +2925,7 @@ its criteria cannot be silently ignored. | CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Urban Sharing NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | | Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | diff --git a/server/providers/gbfs.js b/server/providers/gbfs.js index 9741bb4e6..f55299992 100644 --- a/server/providers/gbfs.js +++ b/server/providers/gbfs.js @@ -11,6 +11,9 @@ import { /** Upstream fetch timeout for GBFS requests (ms). */ const GBFS_PROXY_TIMEOUT_MS = 12000; +/** Value of the Client-Identifier header sent with every upstream request. */ +export const GBFS_CLIENT_IDENTIFIER = 'gods-eye-view'; + export const GBFS_MAX_BODY_BYTES = 5 * 1024 * 1024; // 5 MB function gbfsRedirectHost(location, requestUrl) { @@ -60,6 +63,9 @@ export async function fetchGbfsUpstream( headers: { Accept: 'application/json', 'User-Agent': 'gods-eye-view-gbfs-proxy/1.0', + // Urban Sharing (the Norwegian city-bike feeds) asks every client to + // name itself in this header; other operators ignore it. + 'Client-Identifier': GBFS_CLIENT_IDENTIFIER, }, redirect: 'manual', signal: controller.signal, diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 2967f0a2b..87358dafc 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -251,6 +251,13 @@ export const DATA_CREDITS = [ key: 'gbfs', html: 'Bikeshare availability: GBFS operator feeds (e.g. Austin BCycle)', }, + { + key: 'urbansharing-gbfs', + html: + 'City bikes (Norway): contains data under the ' + + 'Norwegian licence for Open Government data (NLOD) ' + + 'made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel', + }, { key: 'osrm-routing', // The service asks for its attribution to carry a "fix the map" link, so diff --git a/src/data/gbfsProxy.test.mjs b/src/data/gbfsProxy.test.mjs index 191cb57ba..af107b38e 100644 --- a/src/data/gbfsProxy.test.mjs +++ b/src/data/gbfsProxy.test.mjs @@ -1,6 +1,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { + GBFS_CLIENT_IDENTIFIER, GBFS_MAX_BODY_BYTES, fetchGbfsUpstream, } from '../../server/providers/gbfs.js'; @@ -56,6 +57,21 @@ test('GBFS upstream fetch passes a direct 200 through unchanged', async () => { }); }); +test('GBFS upstream fetch names the app in the Client-Identifier header', async () => { + let observedHeaders = null; + await fetchGbfsUpstream( + 'https://gbfs.urbansharing.com/oslobysykkel.no/station_status.json', + { + fetchImpl: async (_url, options) => { + observedHeaders = options.headers; + return new Response(STATION_BODY, { status: 200 }); + }, + }, + ); + assert.equal(GBFS_CLIENT_IDENTIFIER, 'gods-eye-view'); + assert.equal(observedHeaders['Client-Identifier'], GBFS_CLIENT_IDENTIFIER); +}); + test('GBFS upstream fetch forwards a non-redirect error status for the middleware to relay', async () => { const result = await fetchGbfsUpstream(STATION_URL, { fetchImpl: async () => new Response(null, { status: 404 }), diff --git a/src/data/gbfsSource.js b/src/data/gbfsSource.js index a15a806d9..f06fa24f2 100644 --- a/src/data/gbfsSource.js +++ b/src/data/gbfsSource.js @@ -8,6 +8,7 @@ const GBFS_ALLOWED_HOSTS = new Set([ 'austin.publicbikesystem.net', 'hon.publicbikesystem.net', 'chat.publicbikesystem.net', + 'gbfs.urbansharing.com', ]); /** diff --git a/src/layers/bikeshare/proxyRoute.test.mjs b/src/layers/bikeshare/proxyRoute.test.mjs index 88b98f31b..9158a6bdf 100644 --- a/src/layers/bikeshare/proxyRoute.test.mjs +++ b/src/layers/bikeshare/proxyRoute.test.mjs @@ -17,6 +17,7 @@ import { createModel } from './model.js'; import { CITY_BY_ID } from './registry.js'; const AUSTIN = CITY_BY_ID.get('austin-capmetro'); +const OSLO = CITY_BY_ID.get('oslo-bysykkel'); const STATION_INFORMATION = { data: { @@ -204,6 +205,49 @@ test('the proxy relays its cache policy and upstream host through the mounted ro ); }); +test('the Norwegian city-bike systems are served through the mounted route', async () => { + for (const [id, systemId] of [ + ['oslo-bysykkel', 'oslobysykkel.no'], + ['bergen-bysykkel', 'bergenbysykkel.no'], + ['trondheim-bysykkel', 'trondheimbysykkel.no'], + ]) { + const system = CITY_BY_ID.get(id); + assert.ok(system, `${id} is registered`); + assert.deepEqual(system.hosts, ['gbfs.urbansharing.com']); + assert.equal( + system.stationStatusUrl, + `https://gbfs.urbansharing.com/${systemId}/station_status.json`, + ); + } + + await withUpstream( + { + [OSLO.stationInformationUrl]: STATION_INFORMATION, + [OSLO.stationStatusUrl]: STATION_STATUS, + }, + async (requested) => { + const source = createBikeshareSource({ fetchImpl: mountGbfsProxy() }); + const info = await source.getStations(OSLO.stationInformationUrl); + const status = await source.getStations(OSLO.stationStatusUrl); + assert.deepEqual(requested, [ + OSLO.stationInformationUrl, + OSLO.stationStatusUrl, + ]); + assert.equal(info.data.stations.length, 1); + assert.equal(status.data.stations.length, 1); + + const response = await mountGbfsProxy()( + '/api/gbfs/' + encodeURIComponent(OSLO.stationStatusUrl), + ); + assert.equal(response.status, 200); + assert.equal( + response.headers.get('x-gbfs-upstream'), + 'gbfs.urbansharing.com', + ); + }, + ); +}); + test('the query-string shape the client used to send is refused by the mounted route', async () => { await withUpstream( { [AUSTIN.stationStatusUrl]: STATION_STATUS }, diff --git a/src/layers/bikeshare/registry.js b/src/layers/bikeshare/registry.js index ff752f664..37b7d2c4e 100644 --- a/src/layers/bikeshare/registry.js +++ b/src/layers/bikeshare/registry.js @@ -23,6 +23,26 @@ function bcycleEntry({ ...buildBcycleUrls(systemId), }; } +function urbanSharingEntry({ + id, + city, + centerLat, + centerLon, + systemId, + provider, + loadRadiusKm = 100, +}) { + return { + id, + city, + centerLat, + centerLon, + loadRadiusKm, + provider, + stationInformationUrl: `https://gbfs.urbansharing.com/${systemId}/station_information.json`, + stationStatusUrl: `https://gbfs.urbansharing.com/${systemId}/station_status.json`, + }; +} const RAW_GBFS_CITY_REGISTRY = [ { id: 'nyc-citibike', @@ -318,6 +338,30 @@ const RAW_GBFS_CITY_REGISTRY = [ centerLon: -119.6982, systemId: 'bcycle_santabarbara', }), + urbanSharingEntry({ + id: 'oslo-bysykkel', + city: 'Oslo, Norway', + centerLat: 59.9139, + centerLon: 10.7522, + systemId: 'oslobysykkel.no', + provider: 'Oslo Bysykkel', + }), + urbanSharingEntry({ + id: 'bergen-bysykkel', + city: 'Bergen, Norway', + centerLat: 60.3913, + centerLon: 5.3221, + systemId: 'bergenbysykkel.no', + provider: 'Bergen Bysykkel', + }), + urbanSharingEntry({ + id: 'trondheim-bysykkel', + city: 'Trondheim, Norway', + centerLat: 63.4305, + centerLon: 10.3951, + systemId: 'trondheimbysykkel.no', + provider: 'Trondheim Bysykkel', + }), ]; function normalizeRegistryEntry(entry) { const id = String(entry?.id || '') diff --git a/src/tools/queries/mobility.test.mjs b/src/tools/queries/mobility.test.mjs index af8142305..1626aa3dd 100644 --- a/src/tools/queries/mobility.test.mjs +++ b/src/tools/queries/mobility.test.mjs @@ -429,6 +429,31 @@ test('bike share names the systems it did not search', async () => { ); }); +test('bike share over Oslo searches Oslo Bysykkel only', async () => { + const requested = []; + const result = await composeCatalog({ + tools: coreTools, + services: { + bikeshare: { + systems: GBFS_CITY_REGISTRY, + getStations: async (url) => { + requested.push(url); + return { data: { stations: [] } }; + }, + }, + }, + }).call('get_bike_share', { + area: { lat: 59.9139, lon: 10.7522, radius_km: 5 }, + }); + assert.deepEqual(result.data.systems, ['Oslo, Norway']); + assert.deepEqual(result.data.systems_not_searched, []); + assert.ok( + requested.every((url) => + url.startsWith('https://gbfs.urbansharing.com/oslobysykkel.no/'), + ), + ); +}); + test('traffic counts only roads inside a radius area and reports missing tiles', async () => { const flow = (segments, partial = false) => ({ getStatus: async () => ({ hasKey: true }), From fa61955cf15cab33e55707aba55efc598f35d6e9 Mon Sep 17 00:00:00 2001 From: Trygve Hemminghytt <274629477+trygvehhh@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:02:37 +0200 Subject: [PATCH 020/202] =?UTF-8?q?feat(bikeshare):=20add=20Kolumbus,=20Fa?= =?UTF-8?q?rte=20and=20Lillestr=C3=B8m=20city=20bikes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover every Norwegian station-based city-bike system with an open GBFS feed: Kolumbus Bysykkel (Stavanger) and Farte Bysykkel (Skien/Porsgrunn) through Entur Mobility, and Bysykkel Lillestrøm through JCDecaux Cyclocity. api.entur.io and api.cyclocity.fr join the GBFS allowlist, each kept to its GBFS path prefix since both hosts front other APIs. Entur serves its feeds without a .json suffix, so the endpoint checks accept either form, and requests to Entur carry the ET-Client-Name header it requires. Attributed under NLOD (Entur) and Licence Ouverte (JCDecaux). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 ++++-- DATA_SOURCES.md | 2 + docs/CURRENT-STATE.md | 10 +++- server/providers/gbfs.js | 4 +- src/data/dataCredits.js | 13 ++++++ src/data/gbfsProxy.test.mjs | 20 ++++++++ src/data/gbfsSource.js | 59 ++++++++++++++++++++++-- src/layers/bikeshare/proxyRoute.test.mjs | 52 +++++++++++++++++++++ src/layers/bikeshare/registry.js | 58 +++++++++++++++++++++-- src/tools/queries/mobility.test.mjs | 28 +++++++++-- 10 files changed, 241 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2fc2d1e2..184e55230 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,15 @@ ## [Unreleased] - Add Norway's city bikes to the Bike Share layer: Oslo, Bergen and - Trondheim Bysykkel (Urban Sharing, ~440 stations). The feeds are keyless - GBFS on `gbfs.urbansharing.com`, which joins the proxy allowlist; the proxy - now sends the `Client-Identifier: gods-eye-view` header the operators ask - for. Attributed under NLOD 2.0. `get_bike_share` covers the three cities too. + Trondheim Bysykkel (Urban Sharing), Kolumbus Bysykkel in Stavanger and + Farte Bysykkel in Skien/Porsgrunn (Entur Mobility), and Bysykkel + Lillestrøm (JCDecaux), about 760 stations in all. Every feed is keyless + GBFS. `gbfs.urbansharing.com`, `api.entur.io` and `api.cyclocity.fr` join + the proxy allowlist; the last two are kept to their GBFS paths. The proxy + now sends `Client-Identifier: gods-eye-view` on every request and + `ET-Client-Name: gods-eye-view-bikeshare` to Entur, as the operators ask, + and accepts Entur's extensionless feed paths. Attributed under NLOD and + Licence Ouverte. `get_bike_share` covers the six systems too. - Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera list is one keyless GeoJSON request to the agency's OGC API view of its diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md index 07782fd5e..f7e283117 100644 --- a/DATA_SOURCES.md +++ b/DATA_SOURCES.md @@ -57,6 +57,8 @@ How to read this: | **Statens vegvesen** (catalog `ogckart-sn1.atlas.vegvesen.no`, frames `kamera.atlas.vegvesen.no`, HLS `kamera.vegvesen.no`) | CCTV camera catalog + frames + live HLS video, Norway road cameras | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) distributed by Statens vegvesen" | | **GBFS (Lyft / BCycle)** | Bikeshare availability | Per-feed (attribution-only) | Credit the operator (e.g. Austin BCycle) + its `license_url` | | **Oslo / Bergen / Trondheim Bysykkel** (Urban Sharing, `gbfs.urbansharing.com`) | Bikeshare availability, Oslo, Bergen and Trondheim city bikes. Keyless GBFS; the proxy sends the `Client-Identifier` header the operators ask for | [NLOD 2.0](https://data.norge.no/nlod/en/2.0) — attribution REQUIRED ([Oslo](https://oslobysykkel.no/en/open-data/realtime), [Bergen](https://bergenbysykkel.no/en/apne-data/sanntid), [Trondheim](https://trondheimbysykkel.no/en/open-data/realtime)) | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel" | +| **Kolumbus / Farte Bysykkel** (Entur Mobility, `api.entur.io/mobility/v2/gbfs`) | Bikeshare availability, Kolumbus Bysykkel (Stavanger/Nord-Jæren) and Farte Bysykkel (Skien/Porsgrunn). Keyless GBFS; the proxy sends the `ET-Client-Name` header Entur requires and is limited to Entur's GBFS paths | [NLOD](https://data.norge.no/nlod/en/2.0) per the [Entur Mobility API docs](https://developer.entur.org/pages-mobility-docs-mobility-v2) — attribution REQUIRED | "Contains data under the Norwegian licence for Open Government data (NLOD) made available by Entur (Kolumbus Bysykkel, Farte Bysykkel)" | +| **JCDecaux Cyclocity** (`api.cyclocity.fr`) | Bikeshare availability, Bysykkel Lillestrøm. Keyless GBFS; the proxy is limited to JCDecaux contract paths | [Licence Ouverte / Etalab](https://developer.jcdecaux.com/files/Open-Licence-fr.pdf) (the feed's `license_url`) — attribution REQUIRED | "Bysykkel Lillestrøm: JCDecaux, Licence Ouverte" | | **MBTA / MassDOT** (cdn.mbta.com) | Transit layer live vehicles and up to 15 minutes of recently observed positions, Greater Boston | MBTA / MassDOT open realtime vehicle data, used for the live view and up to fifteen minutes of recently observed positions; published for developer use and not against their use. | Attribution to MBTA / MassDOT (courtesy text) | | **CapMetro** (data.texas.gov) | Transit layer live vehicles, Austin TX | CapMetro open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to CapMetro (courtesy text) | | **Metro Transit** (Metropolitan Council) | Transit layer live vehicles, Minneapolis–St Paul | Metro Transit / Metropolitan Council open realtime vehicle data, used for the live view; published for developer use and not against their use. | Attribution to Metro Transit / Metropolitan Council (courtesy text) | diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 8a25ccb8c..55056d2ea 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -963,7 +963,13 @@ streaming. The 12-second deadline includes reading the body, and rejected or stalled downloads are cancelled. Development and preview use the same handler. Every upstream request carries `Client-Identifier: gods-eye-view`, which Urban Sharing asks of clients of the Oslo, Bergen and Trondheim Bysykkel feeds -(`gbfs.urbansharing.com`); other operators ignore it. +(`gbfs.urbansharing.com`); other operators ignore it. Requests to Entur +(`api.entur.io`, Kolumbus and Farte Bysykkel) also carry +`ET-Client-Name: gods-eye-view-bikeshare`. Entur serves its feeds without a +`.json` suffix, so the endpoint check accepts `station_information` and +`station_status` with or without it. Hosts that front more than GBFS are kept to +their GBFS tree: `api.entur.io` to `/mobility/v2/gbfs/` and `api.cyclocity.fr` +(Bysykkel Lillestrøm) to `/contracts/`. ## Remaining local service modules @@ -2925,7 +2931,7 @@ its criteria cannot be silently ignored. | CCTV | Austin + Caltrans (CA) + TfL London + Ontario 511 + Fintraffic (FI) + DriveBC (BC) + TxDOT (TX) + Estonia (Tallinn, Tarktee) + Live Traffic NSW + Open Calgary Open Data + Statens vegvesen (NO, stills + live HLS) + Street View fallback | `src/data/cctv.js` | `/api/cctv` | 10s (active) | | Radio | Radio Browser (public-domain station directory) | `src/data/radio.js` | `/api/radio/stations`, `/api/radio/click/:uuid` | 45 min directory refresh | | Transit 🚌 | Operator GTFS-Realtime VehiclePositions (7 keyless regions, `src/data/transitFeeds.js`) | `src/layers/transit/` via `src/app/layers/transit.js` | `/api/transit` | 15s (poll + delayed playback) | -| Bikeshare 🚲 | GBFS (Lyft + BCycle + Urban Sharing NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | +| Bikeshare 🚲 | GBFS (Lyft + BCycle + Urban Sharing, Entur, JCDecaux NO) | `src/data/bikeshare.js` | `/api/gbfs` | 60s | | Directions 🧭 | OSRM on FOSSGIS servers (OpenStreetMap) | `src/data/directions.js` | `/api/route` (`steps=1`) | on placement / mode change | | Datacenters ▣ | OSM extract (bundled) | `src/data/localLayers.js` | — | static | | Dams ▰ | OpenInfraMap/OSM extract (bundled) | `src/data/localLayers.js` | — | static | diff --git a/server/providers/gbfs.js b/server/providers/gbfs.js index f55299992..990642c8b 100644 --- a/server/providers/gbfs.js +++ b/server/providers/gbfs.js @@ -3,6 +3,7 @@ import { isAllowedGbfsHost, isAllowedGbfsPath, gbfsCacheControl, + gbfsUpstreamHeaders, } from '../../src/data/gbfsSource.js'; // --------------------------------------------------------------------------- @@ -66,6 +67,7 @@ export async function fetchGbfsUpstream( // Urban Sharing (the Norwegian city-bike feeds) asks every client to // name itself in this header; other operators ignore it. 'Client-Identifier': GBFS_CLIENT_IDENTIFIER, + ...gbfsUpstreamHeaders(new URL(url).hostname), }, redirect: 'manual', signal: controller.signal, @@ -187,7 +189,7 @@ export function gbfsProxy() { return; } - if (!isAllowedGbfsPath(upstreamUrl.pathname)) { + if (!isAllowedGbfsPath(upstreamUrl.pathname, upstreamUrl.hostname)) { res.writeHead(400, { 'Content-Type': 'application/json', 'Cache-Control': 'no-store', diff --git a/src/data/dataCredits.js b/src/data/dataCredits.js index 87358dafc..195d2487e 100644 --- a/src/data/dataCredits.js +++ b/src/data/dataCredits.js @@ -258,6 +258,19 @@ export const DATA_CREDITS = [ 'Norwegian licence for Open Government data (NLOD) ' + 'made available by Oslo Bysykkel, Bergen Bysykkel and Trondheim Bysykkel', }, + { + key: 'entur-mobility-gbfs', + html: + 'City bikes (Stavanger, Skien/Porsgrunn): contains data under the ' + + 'Norwegian licence for Open Government data (NLOD) ' + + 'made available by Entur (Kolumbus Bysykkel, Farte Bysykkel)', + }, + { + key: 'jcdecaux-gbfs', + html: + 'City bikes (Lillestrøm): Bysykkel Lillestrøm, JCDecaux — ' + + 'Licence Ouverte', + }, { key: 'osrm-routing', // The service asks for its attribution to carry a "fix the map" link, so diff --git a/src/data/gbfsProxy.test.mjs b/src/data/gbfsProxy.test.mjs index af107b38e..c95256137 100644 --- a/src/data/gbfsProxy.test.mjs +++ b/src/data/gbfsProxy.test.mjs @@ -72,6 +72,26 @@ test('GBFS upstream fetch names the app in the Client-Identifier header', async assert.equal(observedHeaders['Client-Identifier'], GBFS_CLIENT_IDENTIFIER); }); +test('GBFS upstream fetch adds ET-Client-Name for Entur and only for Entur', async () => { + const headersFor = async (url) => { + let observed = null; + await fetchGbfsUpstream(url, { + fetchImpl: async (_url, options) => { + observed = options.headers; + return new Response(STATION_BODY, { status: 200 }); + }, + }); + return observed; + }; + const entur = await headersFor( + 'https://api.entur.io/mobility/v2/gbfs/v2/kolumbusbysykkel/station_status', + ); + assert.equal(entur['ET-Client-Name'], 'gods-eye-view-bikeshare'); + assert.equal(entur['Client-Identifier'], GBFS_CLIENT_IDENTIFIER); + const lyft = await headersFor(STATION_URL); + assert.equal(lyft['ET-Client-Name'], undefined); +}); + test('GBFS upstream fetch forwards a non-redirect error status for the middleware to relay', async () => { const result = await fetchGbfsUpstream(STATION_URL, { fetchImpl: async () => new Response(null, { status: 404 }), diff --git a/src/data/gbfsSource.js b/src/data/gbfsSource.js index f06fa24f2..8d4980948 100644 --- a/src/data/gbfsSource.js +++ b/src/data/gbfsSource.js @@ -9,6 +9,30 @@ const GBFS_ALLOWED_HOSTS = new Set([ 'hon.publicbikesystem.net', 'chat.publicbikesystem.net', 'gbfs.urbansharing.com', + 'api.entur.io', + 'api.cyclocity.fr', +]); + +/** + * Hosts that serve more than GBFS keep the proxy to their GBFS tree: Entur's + * api.entur.io fronts every Entur API, and api.cyclocity.fr every JCDecaux + * contract endpoint. + */ +const GBFS_HOST_PATH_PREFIXES = new Map([ + ['api.entur.io', '/mobility/v2/gbfs/'], + ['api.cyclocity.fr', '/contracts/'], +]); + +/** + * Identifying headers an operator asks every GBFS client to send. Entur rate + * limits or blocks consumers that do not send ET-Client-Name, in the form + * "company-application". + */ +const GBFS_HOST_HEADERS = new Map([ + [ + 'api.entur.io', + Object.freeze({ 'ET-Client-Name': 'gods-eye-view-bikeshare' }), + ], ]); /** @@ -29,13 +53,40 @@ export function isAllowedGbfsHost(hostname) { } /** - * Only allow station_information.json and station_status.json endpoints. + * Only allow station_information and station_status endpoints. The `.json` + * suffix is optional because Entur serves its feeds without one. When a + * hostname is given and that host serves more than GBFS, the path must also + * sit under the host's GBFS prefix. * * @param {string} pathname + * @param {string} [hostname] * @returns {boolean} */ -export function isAllowedGbfsPath(pathname) { - return /\/station_(information|status)\.json$/i.test(String(pathname || '')); +export function isAllowedGbfsPath(pathname, hostname = '') { + const path = String(pathname || ''); + if (!/\/station_(information|status)(\.json)?$/i.test(path)) return false; + const prefix = GBFS_HOST_PATH_PREFIXES.get( + String(hostname || '') + .trim() + .toLowerCase(), + ); + return !prefix || path.startsWith(prefix); +} + +/** + * Extra request headers an upstream GBFS host asks clients to send. + * + * @param {string} hostname + * @returns {Record} + */ +export function gbfsUpstreamHeaders(hostname) { + return ( + GBFS_HOST_HEADERS.get( + String(hostname || '') + .trim() + .toLowerCase(), + ) || {} + ); } /** @@ -48,7 +99,7 @@ export function isAllowedGbfsPath(pathname) { * @returns {string} Cache-Control header value. */ export function gbfsCacheControl(pathname) { - if (/\/station_information\.json$/i.test(String(pathname || ''))) { + if (/\/station_information(\.json)?$/i.test(String(pathname || ''))) { return 'public, max-age=300'; } return 'no-store'; diff --git a/src/layers/bikeshare/proxyRoute.test.mjs b/src/layers/bikeshare/proxyRoute.test.mjs index 9158a6bdf..9bb04ced0 100644 --- a/src/layers/bikeshare/proxyRoute.test.mjs +++ b/src/layers/bikeshare/proxyRoute.test.mjs @@ -248,6 +248,58 @@ test('the Norwegian city-bike systems are served through the mounted route', asy ); }); +test('every Norwegian city-bike system is registered and reachable through the mounted route', async () => { + const expected = { + 'stavanger-kolumbus': + 'https://api.entur.io/mobility/v2/gbfs/v2/kolumbusbysykkel/station_status', + 'skien-farte': + 'https://api.entur.io/mobility/v2/gbfs/v2/fartebysykkel/station_status', + 'lillestrom-bysykkel': + 'https://api.cyclocity.fr/contracts/lillestrom/gbfs/station_status.json', + }; + const upstream = {}; + for (const [id, statusUrl] of Object.entries(expected)) { + const system = CITY_BY_ID.get(id); + assert.ok(system, `${id} is registered`); + assert.equal(system.stationStatusUrl, statusUrl); + upstream[system.stationInformationUrl] = STATION_INFORMATION; + upstream[system.stationStatusUrl] = STATION_STATUS; + } + + await withUpstream(upstream, async (requested) => { + const source = createBikeshareSource({ fetchImpl: mountGbfsProxy() }); + for (const id of Object.keys(expected)) { + const system = CITY_BY_ID.get(id); + const info = await source.getStations(system.stationInformationUrl); + const status = await source.getStations(system.stationStatusUrl); + assert.equal(info.data.stations.length, 1, `${id} information`); + assert.equal(status.data.stations.length, 1, `${id} status`); + } + assert.equal(requested.length, 6); + + // Entur's extensionless station_information keeps its 5-minute cache. + const kolumbus = CITY_BY_ID.get('stavanger-kolumbus'); + const response = await mountGbfsProxy()( + '/api/gbfs/' + encodeURIComponent(kolumbus.stationInformationUrl), + ); + assert.equal(response.headers.get('cache-control'), 'public, max-age=300'); + }); +}); + +test('the mounted route keeps multi-API hosts to their GBFS tree', async () => { + await withUpstream({}, async (requested) => { + const mounted = mountGbfsProxy(); + for (const target of [ + 'https://api.entur.io/journey-planner/v3/station_status', + 'https://api.cyclocity.fr/other/station_status.json', + ]) { + const response = await mounted('/api/gbfs/' + encodeURIComponent(target)); + assert.equal(response.status, 400, target); + } + assert.deepEqual(requested, []); + }); +}); + test('the query-string shape the client used to send is refused by the mounted route', async () => { await withUpstream( { [AUSTIN.stationStatusUrl]: STATION_STATUS }, diff --git a/src/layers/bikeshare/registry.js b/src/layers/bikeshare/registry.js index 37b7d2c4e..89508d42e 100644 --- a/src/layers/bikeshare/registry.js +++ b/src/layers/bikeshare/registry.js @@ -43,6 +43,27 @@ function urbanSharingEntry({ stationStatusUrl: `https://gbfs.urbansharing.com/${systemId}/station_status.json`, }; } +function enturEntry({ + id, + city, + centerLat, + centerLon, + systemId, + provider, + loadRadiusKm = 100, +}) { + const base = `https://api.entur.io/mobility/v2/gbfs/v2/${systemId}`; + return { + id, + city, + centerLat, + centerLon, + loadRadiusKm, + provider, + stationInformationUrl: `${base}/station_information`, + stationStatusUrl: `${base}/station_status`, + }; +} const RAW_GBFS_CITY_REGISTRY = [ { id: 'nyc-citibike', @@ -362,6 +383,35 @@ const RAW_GBFS_CITY_REGISTRY = [ systemId: 'trondheimbysykkel.no', provider: 'Trondheim Bysykkel', }), + enturEntry({ + id: 'stavanger-kolumbus', + city: 'Stavanger, Norway', + centerLat: 58.9, + centerLon: 5.72, + systemId: 'kolumbusbysykkel', + provider: 'Kolumbus Bysykkel', + }), + enturEntry({ + id: 'skien-farte', + city: 'Skien/Porsgrunn, Norway', + centerLat: 59.16, + centerLon: 9.64, + systemId: 'fartebysykkel', + provider: 'Farte Bysykkel', + loadRadiusKm: 40, + }), + { + id: 'lillestrom-bysykkel', + city: 'Lillestrøm, Norway', + centerLat: 59.9556, + centerLon: 11.0492, + loadRadiusKm: 40, + stationInformationUrl: + 'https://api.cyclocity.fr/contracts/lillestrom/gbfs/station_information.json', + stationStatusUrl: + 'https://api.cyclocity.fr/contracts/lillestrom/gbfs/station_status.json', + provider: 'Bysykkel Lillestrøm', + }, ]; function normalizeRegistryEntry(entry) { const id = String(entry?.id || '') @@ -390,11 +440,13 @@ function normalizeRegistryEntry(entry) { ) { throw new Error(`GBFS entry "${id}" must use https URLs`); } - // Validate that URLs end with expected GBFS endpoint filenames - if (!/\/station_information\.json$/i.test(stationInformationUrl.pathname)) { + // Validate that URLs end with expected GBFS endpoint names (Entur omits .json) + if ( + !/\/station_information(\.json)?$/i.test(stationInformationUrl.pathname) + ) { throw new Error(`GBFS entry "${id}" station_information URL is invalid`); } - if (!/\/station_status\.json$/i.test(stationStatusUrl.pathname)) { + if (!/\/station_status(\.json)?$/i.test(stationStatusUrl.pathname)) { throw new Error(`GBFS entry "${id}" station_status URL is invalid`); } diff --git a/src/tools/queries/mobility.test.mjs b/src/tools/queries/mobility.test.mjs index 1626aa3dd..07482946b 100644 --- a/src/tools/queries/mobility.test.mjs +++ b/src/tools/queries/mobility.test.mjs @@ -429,7 +429,7 @@ test('bike share names the systems it did not search', async () => { ); }); -test('bike share over Oslo searches Oslo Bysykkel only', async () => { +test('bike share over Oslo searches only the nearby Norwegian systems', async () => { const requested = []; const result = await composeCatalog({ tools: coreTools, @@ -445,15 +445,35 @@ test('bike share over Oslo searches Oslo Bysykkel only', async () => { }).call('get_bike_share', { area: { lat: 59.9139, lon: 10.7522, radius_km: 5 }, }); - assert.deepEqual(result.data.systems, ['Oslo, Norway']); + assert.deepEqual(result.data.systems.sort(), [ + 'Lillestrøm, Norway', + 'Oslo, Norway', + ]); assert.deepEqual(result.data.systems_not_searched, []); assert.ok( - requested.every((url) => - url.startsWith('https://gbfs.urbansharing.com/oslobysykkel.no/'), + requested.every( + (url) => + url.startsWith('https://gbfs.urbansharing.com/oslobysykkel.no/') || + url.startsWith('https://api.cyclocity.fr/contracts/lillestrom/'), ), ); }); +test('bike share over Stavanger searches Kolumbus Bysykkel', async () => { + const result = await composeCatalog({ + tools: coreTools, + services: { + bikeshare: { + systems: GBFS_CITY_REGISTRY, + getStations: async () => ({ data: { stations: [] } }), + }, + }, + }).call('get_bike_share', { + area: { lat: 58.97, lon: 5.7331, radius_km: 5 }, + }); + assert.deepEqual(result.data.systems, ['Stavanger, Norway']); +}); + test('traffic counts only roads inside a radius area and reports missing tiles', async () => { const flow = (segments, partial = false) => ({ getStatus: async () => ({ hasKey: true }), From a79f58b1a4bbb339eb1e016129e12f0380cbb29a Mon Sep 17 00:00:00 2001 From: devv-shayan Date: Tue, 6 Oct 2026 20:36:56 +0500 Subject: [PATCH 021/202] fix: preserve focus when changing voice auth --- CHANGELOG.md | 8 + docs/CURRENT-STATE.md | 1 + scripts/format-scope.json | 4 + scripts/qa-voice-auth-focus.mjs | 328 ++++++++++++++++++++++++++++ src/keySetup.js | 49 +++-- src/ui/styles/provider-settings.css | 2 +- 6 files changed, 370 insertions(+), 22 deletions(-) create mode 100644 scripts/qa-voice-auth-focus.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index b433f8c06..25415f601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## [Unreleased] +- Add an experimental ChatGPT/Codex OAuth option for cloud voice in Provider + Settings. API-key voice remains the default. Local sign-in can start from + the auth button and reports completion, failure, timeout, or expired + credentials. OAuth credentials stay on the server; local routes require + loopback access and same-site requests. OAuth sessions retain usage data + and show unknown cost. The auth button keeps keyboard focus after a mode + change, and pending sign-in does not discard text in the key fields. + - Add Norway's road cameras from Statens vegvesen as a CCTV pack. The camera list is one keyless GeoJSON request to the agency's OGC API view of its DATEX 3.1 CCTV table, so no DATEX account is needed. Every working camera diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 569ce2420..f7de72983 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3925,6 +3925,7 @@ and unreachable upstream (502/504) separately from road geometry. 4. **The loaded-data caveat is stated once when relevant**: counts cover loaded data, and the flights layer loads where you look (appended to `coverage.note` for radius/view scopes over viewport-loaded layers). - **Degradation**: API-key mode without `OPENAI_API_KEY` returns 503 as before. Selecting OAuth without a usable local ChatGPT/Codex sign-in starts the local `codex login` browser flow, polls for the resulting local token, and selects OAuth when sign-in completes. If the Codex login flow cannot start or does not complete, the rest of the app is unaffected. - **Login lifecycle and storage**: one pending Codex login is shared by the login and status routes. Process failure, success without readable credentials, and a two-minute timeout each report a clear failure; timeout and server shutdown cancel the owned process. The launcher resolves `CODEX_HOME` before changing directories and refuses a mismatched `CODEX_AUTH_JSON`. GEV reads file credentials only, never writes or refreshes them, and does not require the vendor refresh token. Expired access tokens report `CODEX_OAUTH_REAUTH_REQUIRED`. OAuth status and login routes also use the same-site request gate. +- **Auth button focus**: changing voice auth updates the existing controls in place. Enter, Space, and mouse selection keep focus on the auth button; Tab then follows the normal control order. Pending sign-in blocks repeated activation without removing the button from the focus order. Completion does not take focus from another control or discard text in the key fields. `scripts/qa-voice-auth-focus.mjs` checks these paths in Chromium with fixture sign-in responses. - **Support boundary**: the local OAuth option is experimental. OpenAI's documented Sign in with ChatGPT plan-sharing inference endpoint is `/v1/responses`; support for third-party reuse of Codex credentials at `/v1/realtime/client_secrets` remains unconfirmed. Technical success does not establish a supported auth or billing contract. ### AI HUD Summary (June 2026) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 662b2a43a..0d9530871 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -36,6 +36,10 @@ "src/main.js", "docs/APPLICATION.md", "scripts/qa-application.mjs", + "scripts/qa-voice-auth.mjs", + "scripts/qa-voice-auth-focus.mjs", + "src/codexOauthRealtime.test.mjs", + "src/voice/cloudVoiceAuth.test.mjs", "scripts/qa-terrain-429.mjs", "src/app/application.js", "src/app/application.test.mjs", diff --git a/scripts/qa-voice-auth-focus.mjs b/scripts/qa-voice-auth-focus.mjs new file mode 100644 index 000000000..6d131a22c --- /dev/null +++ b/scripts/qa-voice-auth-focus.mjs @@ -0,0 +1,328 @@ +#!/usr/bin/env node +/** + * Check auth-mode keyboard focus with the real settings template and controls. + * Run: node --test --test-reporter=spec scripts/qa-voice-auth-focus.mjs + * Sign-in responses are fixtures. No credentials or live login are used. + */ +import assert from 'node:assert/strict'; +import { existsSync } from 'node:fs'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { after, before, test } from 'node:test'; +import puppeteer from 'puppeteer'; + +const root = new URL('../', import.meta.url); +const template = await readFile( + new URL('src/ui/templates/provider-settings.html', root), + 'utf8', +); +const assets = new Set([ + '/src/keySetup.js', + '/src/voice/cloudVoiceAuth.js', + '/src/ui/surfaceKeyboard.js', + '/src/ui/styles/provider-settings.css', +]); +const observations = []; +let server; +let browser; +let origin; + +before(async () => { + server = createServer(async (req, res) => { + const pathname = new URL(req.url, 'http://localhost').pathname; + if (pathname === '/') { + res.setHeader('Content-Type', 'text/html; charset=utf-8'); + res.end(` + Voice auth keyboard focus test + + ${template}`); + return; + } + if (!assets.has(pathname)) { + res.statusCode = 404; + res.end(); + return; + } + try { + res.setHeader( + 'Content-Type', + pathname.endsWith('.css') ? 'text/css' : 'text/javascript', + ); + res.end(await readFile(new URL(pathname.slice(1), root))); + } catch { + res.statusCode = 500; + res.end('Could not load the test asset'); + } + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + origin = `http://127.0.0.1:${server.address().port}`; + const chrome = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'; + browser = await puppeteer.launch({ + headless: true, + executablePath: + process.env.PUPPETEER_EXECUTABLE_PATH || + (existsSync(chrome) ? chrome : await puppeteer.executablePath()), + }); +}); + +after(async () => { + await browser?.close(); + if (server?.listening) await new Promise((resolve) => server.close(resolve)); + const output = new URL('.gev-logs/voice-auth-focus/', root); + await mkdir(output, { recursive: true }); + await writeFile( + new URL('results.json', output), + `${JSON.stringify(observations, null, 2)}\n`, + ); +}); + +const scenarios = [ + { + name: 'OAuth to API key', + initial: 'oauth', + expected: 'api-key', + flow: 'unused', + }, + { + name: 'API key to existing OAuth', + initial: 'api-key', + expected: 'oauth', + flow: 'existing', + }, + { + name: 'API key to OAuth ready at login launch', + initial: 'api-key', + expected: 'oauth', + flow: 'login-ready', + }, + { + name: 'API key to OAuth ready after polling', + initial: 'api-key', + expected: 'oauth', + flow: 'poll-ready', + }, +]; + +async function mountSettings(page, scenario) { + await page.goto(`${origin}/?setup=1`, { + waitUntil: 'domcontentloaded', + }); + await page.evaluate(async ({ initial, flow }) => { + localStorage.clear(); + localStorage.setItem('godsEyeView.voice.cloudAuth', initial); + window.__focusTestCalls = []; + let statusReads = 0; + const fetchImpl = async (url) => { + window.__focusTestCalls.push(url); + let payload; + let status = 200; + if (url === '/api/setup/status') { + payload = { + setCount: 0, + total: 1, + keys: [ + { + id: 'openai', + title: 'OPENAI', + tier: 'metered', + set: false, + unlocks: 'Voice control', + envVars: ['OPENAI_API_KEY'], + getUrl: 'https://example.invalid/', + }, + ], + }; + } else if (url === '/api/realtime/oauth-status') { + statusReads += 1; + if (flow === 'deferred') { + return await new Promise((resolve) => { + window.__resolveOAuthStatus = (available) => + resolve( + new Response( + JSON.stringify({ available, error: 'Fixture failure' }), + { + status: available ? 200 : 503, + }, + ), + ); + }); + } + payload = { + available: + flow === 'existing' || (flow === 'poll-ready' && statusReads > 1), + }; + } else if (url === '/api/realtime/oauth-login') { + const available = flow === 'login-ready'; + status = available ? 200 : 202; + payload = { available, pending: !available, started: !available }; + } else { + throw new Error(`Unexpected request: ${url}`); + } + return new Response(JSON.stringify(payload), { status }); + }; + const { initKeySetup } = await import('/src/keySetup.js'); + window.__focusTestSettings = await initKeySetup({ fetchImpl }); + }, scenario); + await page.waitForFunction( + () => + document.getElementById('key-setup').classList.contains('visible') && + document.activeElement.matches('input[data-env-var]'), + ); +} + +for (const scenario of scenarios) { + for (const key of ['Enter', 'Space', 'Mouse click']) { + test(`${scenario.name} keeps keyboard focus after ${key}`, async (t) => { + const page = await browser.newPage(); + try { + await mountSettings(page, scenario); + + if (key !== 'Mouse click') { + // Tab from the initial key field to the real auth button. + await page.keyboard.press('Tab'); + assert.equal( + await page.evaluate(() => + document.activeElement.matches('[data-cloud-voice-auth-toggle]'), + ), + true, + 'Keyboard navigation must reach the auth toggle before activation', + ); + } + await page.evaluate(() => { + const toggle = document.querySelector( + '[data-cloud-voice-auth-toggle]', + ); + window.__originalFocusToggle = toggle; + window.__focusAtActivation = null; + toggle.addEventListener( + 'click', + () => { + window.__focusAtActivation = document.activeElement === toggle; + }, + { capture: true, once: true }, + ); + }); + if (key === 'Mouse click') { + await page.waitForFunction( + () => + getComputedStyle(document.getElementById('key-setup')).opacity === + '1', + ); + await page.click('[data-cloud-voice-auth-toggle]'); + } else { + await page.keyboard.press(key); + } + await page.waitForFunction( + (expected) => + localStorage.getItem('godsEyeView.voice.cloudAuth') === expected && + !document.querySelector('[data-cloud-voice-auth-toggle]') + .disabled && + document + .querySelector('[data-cloud-voice-auth-toggle]') + .getAttribute('aria-disabled') !== 'true', + {}, + scenario.expected, + ); + const selection = await page.evaluate(() => ({ + focusOnToggleAtActivation: window.__focusAtActivation, + focusOnToggle: + document.activeElement === + document.querySelector('[data-cloud-voice-auth-toggle]'), + activeTag: document.activeElement.tagName, + originalToggleConnected: window.__originalFocusToggle.isConnected, + requests: window.__focusTestCalls, + })); + await page.keyboard.press('Tab'); + const nextTab = await page.evaluate(() => ({ + focusOnSave: document.activeElement.matches('[data-key-setup-apply]'), + activeLabel: + document.activeElement.getAttribute('aria-label') || + document.activeElement.textContent.trim(), + })); + const observed = { scenario: scenario.name, key, selection, nextTab }; + observations.push(observed); + t.diagnostic(JSON.stringify(observed)); + assert.equal( + selection.focusOnToggle, + true, + 'Auth selection must keep focus on its usable toggle', + ); + assert.equal( + nextTab.focusOnSave, + true, + 'Tab must continue to SAVE KEYS rather than restart at Close', + ); + } finally { + await page.close(); + } + }); + } +} + +for (const available of [true, false]) { + test(`pending OAuth ${available ? 'success' : 'failure'} blocks repeats without moving focus or losing a key draft`, async () => { + const page = await browser.newPage(); + try { + await mountSettings(page, { initial: 'api-key', flow: 'deferred' }); + await page.type('input[data-env-var]', 'test-draft-only'); + await page.keyboard.press('Tab'); + await page.keyboard.press('Enter'); + await page.waitForFunction( + () => typeof window.__resolveOAuthStatus === 'function', + ); + assert.equal( + await page.evaluate(() => + document.activeElement.matches('[data-cloud-voice-auth-toggle]'), + ), + true, + ); + await page.keyboard.press('Enter'); + await page.keyboard.press('Space'); + assert.equal( + await page.evaluate( + () => + window.__focusTestCalls.filter( + (url) => url === '/api/realtime/oauth-status', + ).length, + ), + 1, + ); + await page.keyboard.press('Tab'); + assert.equal( + await page.evaluate(() => + document.activeElement.matches('[data-key-setup-apply]'), + ), + true, + ); + await page.evaluate( + (available) => window.__resolveOAuthStatus(available), + available, + ); + await page.waitForFunction( + () => + document + .querySelector('[data-cloud-voice-auth-toggle]') + .getAttribute('aria-disabled') === 'false', + ); + const observed = await page.evaluate(() => ({ + mode: localStorage.getItem('godsEyeView.voice.cloudAuth'), + focusOnSave: document.activeElement.matches('[data-key-setup-apply]'), + draft: document.querySelector('input[data-env-var]').value, + requests: window.__focusTestCalls, + })); + observations.push({ scenario: 'pending OAuth', available, observed }); + assert.equal(observed.mode, available ? 'oauth' : 'api-key'); + assert.equal( + observed.focusOnSave, + true, + 'Completing a request must not take focus from another control', + ); + assert.equal(observed.draft, 'test-draft-only'); + } finally { + await page.close(); + } + }); +} diff --git a/src/keySetup.js b/src/keySetup.js index 1186e28a7..1029d479b 100644 --- a/src/keySetup.js +++ b/src/keySetup.js @@ -111,6 +111,21 @@ export function stripKeylessBasemapFromHash(hash) { const TIER_DOTS = Object.freeze({ metered: '🔴', free: '🟡' }); +function updateCloudVoiceAuthControl(auth, busy = false) { + const mode = readStoredCloudVoiceAuthMode(); + const state = auth.querySelector('.key-setup-cloud-auth-state'); + const toggle = auth.querySelector('[data-cloud-voice-auth-toggle]'); + state.textContent = + mode === 'oauth' ? 'VOICE AUTH · CHATGPT OAUTH' : 'VOICE AUTH · API KEY'; + toggle.textContent = mode === 'oauth' ? 'USE API KEY' : 'USE CHATGPT OAUTH'; + toggle.title = + mode === 'oauth' + ? 'Use OPENAI_API_KEY for the next cloud voice session' + : 'Use the signed-in local ChatGPT/Codex OAuth session for the next cloud voice session'; + // Keep the control focusable while the delegated handler blocks repeats. + toggle.setAttribute('aria-disabled', String(busy)); +} + /** Build one key row. All content is our own registry text, set via textContent. */ function buildRow(documentRef, key) { const row = documentRef.createElement('section'); @@ -197,21 +212,14 @@ function buildRow(documentRef, key) { if (key.id === 'openai') { const auth = documentRef.createElement('div'); auth.className = 'key-setup-cloud-auth'; - const mode = readStoredCloudVoiceAuthMode(); const state = documentRef.createElement('span'); state.className = 'key-setup-cloud-auth-state'; - state.textContent = - mode === 'oauth' ? 'VOICE AUTH · CHATGPT OAUTH' : 'VOICE AUTH · API KEY'; const toggle = documentRef.createElement('button'); toggle.type = 'button'; toggle.className = 'key-setup-cloud-auth-toggle'; toggle.dataset.cloudVoiceAuthToggle = 'true'; - toggle.textContent = mode === 'oauth' ? 'USE API KEY' : 'USE CHATGPT OAUTH'; - toggle.title = - mode === 'oauth' - ? 'Use OPENAI_API_KEY for the next cloud voice session' - : 'Use the signed-in local ChatGPT/Codex OAuth session for the next cloud voice session'; auth.append(state, toggle); + updateCloudVoiceAuthControl(auth); row.append(auth); } return row; @@ -297,6 +305,11 @@ export async function initKeySetup({ let oauthBusy = false; let open = false; + const syncCloudVoiceAuth = () => { + for (const auth of rowsHost.querySelectorAll('.key-setup-cloud-auth')) + updateCloudVoiceAuthControl(auth, oauthBusy); + }; + const render = (nextStatus) => { if (disposed) return; status = nextStatus; @@ -310,10 +323,7 @@ export async function initKeySetup({ rowsHost.textContent = ''; for (const key of status.keys || []) rowsHost.append(buildRow(documentRef, key)); - for (const button of rowsHost.querySelectorAll( - '[data-cloud-voice-auth-toggle]', - )) - button.disabled = oauthBusy; + syncCloudVoiceAuth(); }; const visible = () => @@ -451,12 +461,12 @@ export async function initKeySetup({ const current = readStoredCloudVoiceAuthMode(); if (current === 'oauth') { writeStoredCloudVoiceAuthMode('api-key'); - render(status); + syncCloudVoiceAuth(); say('Cloud voice will use OPENAI_API_KEY on the next session.'); return; } - authButton.disabled = true; oauthBusy = true; + syncCloudVoiceAuth(); say('Checking local ChatGPT OAuth sign-in…'); try { const response = await doFetch('/api/realtime/oauth-status', { @@ -471,7 +481,7 @@ export async function initKeySetup({ ); if (payload.available) { writeStoredCloudVoiceAuthMode('oauth'); - render(status); + syncCloudVoiceAuth(); say( 'ChatGPT OAuth selected for cloud voice. Your API key stays saved and available.', ); @@ -500,7 +510,7 @@ export async function initKeySetup({ } if (loginPayload.available) { writeStoredCloudVoiceAuthMode('oauth'); - render(status); + syncCloudVoiceAuth(); say( 'ChatGPT OAuth selected for cloud voice. Your API key stays saved and available.', ); @@ -522,7 +532,7 @@ export async function initKeySetup({ return; } writeStoredCloudVoiceAuthMode('oauth'); - render(status); + syncCloudVoiceAuth(); say( 'ChatGPT sign-in complete. OAuth will be used for the next cloud voice session.', ); @@ -531,10 +541,7 @@ export async function initKeySetup({ say(`OAuth check failed: ${error?.message || error}`); } finally { oauthBusy = false; - for (const button of rowsHost.querySelectorAll( - '[data-cloud-voice-auth-toggle]', - )) - button.disabled = false; + if (!disposed) syncCloudVoiceAuth(); } })(); return; diff --git a/src/ui/styles/provider-settings.css b/src/ui/styles/provider-settings.css index 043e046d6..7563a886d 100644 --- a/src/ui/styles/provider-settings.css +++ b/src/ui/styles/provider-settings.css @@ -332,7 +332,7 @@ outline: none; } -.key-setup-cloud-auth-toggle:disabled { +.key-setup-cloud-auth-toggle[aria-disabled='true'] { cursor: wait; opacity: 0.5; } From f93755af488585bd745f75c7baa43bebbed5dc3e Mon Sep 17 00:00:00 2001 From: Bruno Gomes Date: Tue, 6 Oct 2026 16:13:45 -0300 Subject: [PATCH 022/202] fix(cctv): start live HLS on plain-HTTP LAN pages The client lease id came from crypto.randomUUID, which browsers only expose in secure contexts. Opened over http://:4173 (the HOST=0.0.0.0 opt-in), starting a live camera feed threw before playback. Fall back to a v4 UUID from crypto.getRandomValues, which the server's lease check accepts. --- CHANGELOG.md | 6 ++++ docs/CURRENT-STATE.md | 4 ++- src/layers/cctv/videoPlayback.js | 28 +++++++++++++++++- src/layers/cctv/videoPlayback.test.mjs | 39 +++++++++++++++++++++++++- 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b433f8c06..9fc367d84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. +- Live HLS cameras play on a page opened over plain HTTP from another machine + (`HOST=0.0.0.0`). The client lease id came from `crypto.randomUUID`, which + browsers only expose in secure contexts, so starting a live feed threw + before playback. The id now falls back to a v4 UUID built from + `crypto.getRandomValues`, which the server's lease check accepts. + ## [0.2.1] — 2026-10-02 — God's Eye View through Your Agent (with security fixes and other features) - Ship each bundled data pack once. The region, marine, admin-boundary, diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d4fe022a8..fde1373af 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4814,7 +4814,9 @@ its paint loop. Camera deactivation destroys the decoder, and fatal playback errors revert to the labeled frame fallback. Camera changes clear the old panel frame. hls.js is imported only when an HLS camera starts. Two live sessions may coexist with at most eight independent consumer leases each; no-access lease -expiry is 15 seconds. Closing one consumer leaves other consumers running. Encrypted/fMP4/byte-range playlists fail +expiry is 15 seconds. Closing one consumer leaves other consumers running. Lease +ids use `crypto.randomUUID` where the page is a secure context and an equivalent +v4 id from `crypto.getRandomValues` on plain-HTTP LAN pages. Encrypted/fMP4/byte-range playlists fail closed. Performance and live agency reliability require recorded validation. ## Wind forecast models diff --git a/src/layers/cctv/videoPlayback.js b/src/layers/cctv/videoPlayback.js index d59143b5a..48988c17c 100644 --- a/src/layers/cctv/videoPlayback.js +++ b/src/layers/cctv/videoPlayback.js @@ -1,3 +1,28 @@ +/** + * A v4 UUID for the client lease. `crypto.randomUUID` only exists in secure + * contexts, so a page opened over plain HTTP from another machine (the + * HOST=0.0.0.0 LAN opt-in) has none; `getRandomValues` exists in every context + * and yields the same shape the server's lease check accepts. + */ +export function createLeaseId(cryptoImpl = globalThis.crypto) { + if (typeof cryptoImpl.randomUUID === 'function') { + return cryptoImpl.randomUUID(); + } + const bytes = cryptoImpl.getRandomValues(new Uint8Array(16)); + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')); + return [ + hex.slice(0, 4), + hex.slice(4, 6), + hex.slice(6, 8), + hex.slice(8, 10), + hex.slice(10), + ] + .map((group) => group.join('')) + .join('-'); +} + /** One decoder per active camera; both surfaces consume this video element. */ export function attachCctvVideo( video, @@ -7,13 +32,14 @@ export function attachCctvVideo( loadHls = () => import('hls.js'), onFailure = () => {}, fetchImpl = globalThis.fetch, + cryptoImpl = globalThis.crypto, } = {}, ) { // Preserve replay for existing finite video feeds; live HLS must not loop. video.loop = feedType !== 'hls'; let disposed = false; let hls = null; - const leaseId = feedType === 'hls' ? globalThis.crypto.randomUUID() : null; + const leaseId = feedType === 'hls' ? createLeaseId(cryptoImpl) : null; const mediaUrl = feedType === 'hls' ? `${url}${url.includes('?') ? '&' : '?'}lease=${leaseId}` diff --git a/src/layers/cctv/videoPlayback.test.mjs b/src/layers/cctv/videoPlayback.test.mjs index 1f8c1c0bb..d1ef02539 100644 --- a/src/layers/cctv/videoPlayback.test.mjs +++ b/src/layers/cctv/videoPlayback.test.mjs @@ -1,6 +1,15 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { attachCctvVideo } from './videoPlayback.js'; +import { webcrypto } from 'node:crypto'; +import { attachCctvVideo, createLeaseId } from './videoPlayback.js'; + +const LEASE_V4 = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +/** What an insecure context exposes: getRandomValues but no randomUUID. */ +const insecureCrypto = { + getRandomValues: (array) => webcrypto.getRandomValues(array), +}; function video() { const v = new EventTarget(); Object.assign(v, { @@ -69,6 +78,34 @@ test('native HLS releases its client lease without response-header access', asyn assert.equal(releases[0].init.method, 'DELETE'); }); +test('lease ids are v4 UUIDs with or without crypto.randomUUID', () => { + assert.match(createLeaseId(), LEASE_V4); + const ids = new Set(); + for (let i = 0; i < 64; i++) ids.add(createLeaseId(insecureCrypto)); + assert.equal(ids.size, 64); + for (const id of ids) assert.match(id, LEASE_V4); +}); + +test('HLS starts on a plain-HTTP LAN page without crypto.randomUUID', async () => { + const source = video(); + source.canPlayType = () => 'probably'; + const playback = attachCctvVideo(source, '/api/cctv/media/a', 'hls', { + loadHls: async () => ({ default: { isSupported: () => false } }), + fetchImpl: async () => {}, + cryptoImpl: insecureCrypto, + }); + try { + await playback.ready; + const lease = new URL( + source.src, + 'http://192.168.1.10:4173/', + ).searchParams.get('lease'); + assert.match(lease, LEASE_V4); + } finally { + playback.dispose(); + } +}); + test('finite video feeds retain looping while live HLS does not loop', async () => { for (const feedType of ['mp4', 'webm', 'hls']) { const source = video(); From 57b8d8b23c249e05cc122e6b9f9cec9706757439 Mon Sep 17 00:00:00 2001 From: kapturner Date: Sun, 4 Oct 2026 08:26:18 -0500 Subject: [PATCH 023/202] feat: import GeoJSON, KML, KMZ and GPX files onto the globe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DISPLAY ▸ Import (and drop onto the globe) puts the person's own files on the map as Cesium data sources, clamped to the ground, one color per file, with a row to show/hide, fly to or remove each. Files stay in the browser for the session. KML/KMZ are sanitized before Cesium reads them (network links, remote icons/styles, models, photo/screen overlays and balloon HTML removed; KMZ images served from blob URLs), so an imported file cannot make the page fetch anything. KMZ is read with a small dependency-free zip reader on the platform DecompressionStream, with a zip-bomb limit. Bounded file size, entity count and number of files. Tests: src/data/geoFileImport.test.mjs, src/ui/geoFileImport.test.mjs. Gate: scripts/qa-geo-import.mjs. The import half of #890. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 + docs/CURRENT-STATE.md | 10 + index.html | 2 +- scripts/format-scope.json | 3 + scripts/package-boundaries.json | 2 + scripts/qa-geo-import.mjs | 402 +++++++++++++++++++++++ src/app/tools.js | 6 + src/data/geoFileImport.js | 432 +++++++++++++++++++++++++ src/data/geoFileImport.test.mjs | 215 ++++++++++++ src/ui/geoFileImport.js | 329 +++++++++++++++++++ src/ui/geoFileImport.test.mjs | 399 +++++++++++++++++++++++ src/ui/styles/controls.css | 64 ++++ src/ui/templates/display-controls.html | 9 + 13 files changed, 1879 insertions(+), 1 deletion(-) create mode 100644 scripts/qa-geo-import.mjs create mode 100644 src/data/geoFileImport.js create mode 100644 src/data/geoFileImport.test.mjs create mode 100644 src/ui/geoFileImport.js create mode 100644 src/ui/geoFileImport.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3876f79..36100f297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,13 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. +- Import your own GeoJSON, KML, KMZ and GPX files (DISPLAY ▸ Import, or drop + them onto the globe). Each file is drawn on the ground in its own color with + a row to show, hide, fly to or remove it. Files stay in the browser for the + session, and KML/KMZ network links, remote icons, remote styles and balloon + HTML are left out so an imported file cannot make the page fetch anything. + The import half of #890. + ## [0.2.1] — 2026-10-02 — God's Eye View through Your Agent (with security fixes and other features) - Ship each bundled data pack once. The region, marine, admin-boundary, diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d51be986b..4a3b6a77b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4106,6 +4106,16 @@ are omitted rather than framing the wrong part of the globe. - Pure half: `src/annotations/drawMode.js` (session, minimum vertices, double-click de-dup at 0.5 m, vertex ceiling, degenerate-geometry refusal, ring closing, dateline-safe length/area/centroid, hint text). Regression surface: `src/annotations/drawMode.test.mjs` and `src/data/inputOwnership.test.mjs` (pure), `src/annotations/drawToolBehaviour.test.mjs` (the tool driven against DOM and viewer doubles — leases, borrowed input actions, teardown, the Enter key, preview heights), `src/annotations/drawTool.test.mjs` (structural: where files live, which ids the markup carries, what the docs may claim), and the rendered harness `scripts/qa-draw-tool.mjs`. - The annotation tree is a declared ownership group: `./annotations` → `src/annotations/index.js` in `package.json`, with its module graph listed in `scripts/package-boundaries.json`, so an annotation module reaching outside that graph is a gate failure rather than a surprise in a layer's bundle. `drawMode.js`, `drawTool.js` and their tests are in `scripts/format-scope.json`. +### Import Your Own Files (October 2026) + +- DISPLAY ▸ **Import** puts GeoJSON (`.geojson`/`.json`), KML, KMZ and GPX files on the globe. Files come in through the Import button (a hidden ``) or by dropping them onto the globe; the format is taken from the extension, then from the first bytes. Each file becomes its own Cesium data source (`GeoJsonDataSource`, `KmlDataSource`, `GpxDataSource`), clamped to the ground, in its own color from the whiteboard palette, with a row to show/hide it, fly to it, or remove it. The first file of a batch that loads is flown to. Imported files last for the page session: a reload starts empty, and nothing is uploaded or stored. +- Runtime entry: `createApplicationTools` (`src/app/tools.js`) calls `initGeoFileImport({ viewer })` (`src/ui/geoFileImport.js`) after the draw tool and defers its `destroy()`, which removes every imported data source, revokes every blob URL, and gives back the drop listeners on the viewer container and the `window.__gevGeoImport` handle (`importFiles`, `list`, `setVisible`, `flyTo`, `remove`, `diagnostics`). Parsing lives in `src/data/geoFileImport.js`. +- **An imported file cannot make the page fetch anything.** Before `KmlDataSource` reads a KML/KMZ document, `sanitizeKmlDocument` removes `NetworkLink`, `NetworkLinkControl`, `ScreenOverlay`, `PhotoOverlay`, `Model` and `Tour`, every `href` that is not an image packed in the same KMZ or a `data:image/` URL (an icon then falls back to Cesium's default pin, and a `GroundOverlay` left without an image is dropped), every `styleUrl` that is not a local `#id`, and placemark `description`/`BalloonStyle`/`Snippet` HTML. KMZ images are served from blob URLs. The hint says when something was left out. `scripts/qa-geo-import.mjs` imports files that name a trap host in each of those places and fails on any request to it. +- **KMZ** is read by a small zip reader (`readZipEntries`: stored and deflated entries through the platform `DecompressionStream`; no encryption, no ZIP64). The main document is `doc.kml`, else the shallowest `.kml`; relative hrefs resolve only to entries inside the archive. +- **Bounds.** A file may be at most 50 MB (`MAX_IMPORT_BYTES`), a KMZ may inflate to at most 120 MB in total (`MAX_KMZ_INFLATED_BYTES`, the zip-bomb stop), one file may create at most 20,000 entities (`MAX_IMPORT_ENTITIES`), and at most 12 files are on the globe at once (`MAX_IMPORTED_FILES`; files over that are not read). A refused file names its reason in the hint and does not stop the rest of the batch; a file with nothing to draw is refused. +- File names are rendered with `textContent` only. +- Gate: `node scripts/qa-geo-import.mjs` (dev server up). Unit: `src/data/geoFileImport.test.mjs`, `src/ui/geoFileImport.test.mjs`. + ### Recent Imagery (September 2026) - The layer sits in the Data Layers panel's Cameras group beside Cameras (CCTV); Mapped ALPR Cameras moved to the Infrastructure group to make room. diff --git a/index.html b/index.html index 2e630c4a8..78d6e34ec 100644 --- a/index.html +++ b/index.html @@ -13,7 +13,7 @@ with the sources: src/materialSymbolsSubset.test.mjs fails when a source names a glyph this list is missing, because a missing glyph does not draw a box — the ligature never forms and the element renders the literal word. --> - + diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 7062704a7..da80c0749 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -26,6 +26,8 @@ "src/annotations/drawMode.test.mjs", "src/annotations/drawTool.js", "src/annotations/drawTool.test.mjs", + "src/data/geoFileImport.test.mjs", + "src/ui/geoFileImport.test.mjs", "src/annotations/adminOutline.test.mjs", "src/annotations/regionRing.test.mjs", "src/data/adminBoundaries.test.mjs", @@ -195,6 +197,7 @@ "src/styles/snow.js", "scripts/qa-visual-effects.mjs", "scripts/qa-draw-tool.mjs", + "scripts/qa-geo-import.mjs", "src/ui/mapSource.js", "src/ui/mapSourceControls.js", "src/ui/mapSourceControls.test.mjs", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index ce778ee95..52062c1b3 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1958,6 +1958,8 @@ "src/data/inputOwnership.js", "src/annotations/drawMode.js", "src/annotations/drawTool.js", + "src/data/geoFileImport.js", + "src/ui/geoFileImport.js", "src/data/routeSteps.js", "src/layers/directions/index.js", "src/data/directions.js", diff --git a/scripts/qa-geo-import.mjs b/scripts/qa-geo-import.mjs new file mode 100644 index 000000000..c1c8f6462 --- /dev/null +++ b/scripts/qa-geo-import.mjs @@ -0,0 +1,402 @@ +#!/usr/bin/env node +/** + * Rendered acceptance for DISPLAY ▸ Import: real files through the real file + * input and a real drop, parsed by Cesium in the page. + * + * Proves: GeoJSON, KML, KMZ (with a packed icon and ground overlay) and GPX + * each become one row and one data source; the imported shape actually paints + * on the globe and stops painting when hidden; a KML/KMZ never makes the page + * request anything named in it (network link, remote icon, remote style); a + * dropped file imports like a picked one; a file that is not geodata is refused + * with a reason and leaves nothing behind; Remove gives every data source back; + * no page errors. + * + * Usage: dev server up (`npm run dev`), then `node scripts/qa-geo-import.mjs`. + * QA_BASE_URL (default http://localhost:4173), QA_SHOT_DIR, QA_HEADFUL=1. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import puppeteer from 'puppeteer'; + +const BASE_URL = process.env.QA_BASE_URL || 'http://localhost:4173'; +const SHOT_DIR = process.env.QA_SHOT_DIR || 'qa-shots/geo-import'; +const HEADFUL = process.env.QA_HEADFUL === '1'; +// Golden Gate Park: a big, flat, green rectangle the test polygon can cover. +const VIEW = { lon: -122.4862, lat: 37.7694, height: 4200, pitch: -88 }; +const TRAP_HOST = 'gev-qa-trap.invalid'; + +/* ── fixtures ──────────────────────────────────────────────────────────── */ + +const ring = [ + [-122.511, 37.7645], + [-122.4545, 37.7645], + [-122.4545, 37.7745], + [-122.511, 37.7745], + [-122.511, 37.7645], +]; +const geojson = JSON.stringify({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + properties: { name: 'QA park' }, + geometry: { type: 'Polygon', coordinates: [ring] }, + }, + { + type: 'Feature', + properties: { name: 'QA pin' }, + geometry: { type: 'Point', coordinates: [-122.4862, 37.7694] }, + }, + ], +}); +const kml = (iconHref) => ` + + + + https://${TRAP_HOST}/link.kml + QA KML pin#s + ]]> + -122.47,37.77,0 + QA remote stylehttps://${TRAP_HOST}/styles.kml#x + -122.50,37.766 -122.46,37.772 + https://${TRAP_HOST}/overlay.png + 37.7737.76-122.46-122.47 + +`; +const gpx = ` + + QA waypoint + QA track + + +`; +// 1×1 PNG. +const PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==', + 'base64', +); +/** Stored (uncompressed) zip — enough for a KMZ fixture. */ +function storedZip(files) { + const parts = []; + const central = []; + let offset = 0; + for (const [name, data] of files) { + const n = Buffer.from(name); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt32LE(data.length, 18); + local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(n.length, 26); + const c = Buffer.alloc(46); + c.writeUInt32LE(0x02014b50, 0); + c.writeUInt32LE(data.length, 20); + c.writeUInt32LE(data.length, 24); + c.writeUInt16LE(n.length, 28); + c.writeUInt32LE(offset, 42); + parts.push(local, n, data); + central.push(c, n); + offset += 30 + n.length + data.length; + } + const dir = Buffer.concat(central); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(files.length, 8); + end.writeUInt16LE(files.length, 10); + end.writeUInt32LE(dir.length, 12); + end.writeUInt32LE(offset, 16); + return Buffer.concat([...parts, dir, end]); +} +const kmzDoc = kml('files/pin.png').replace( + `https://${TRAP_HOST}/overlay.png`, + 'files/overlay.png', +); + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-qa-geo-import-')); +const fixture = (name, data) => { + const file = path.join(dir, name); + fs.writeFileSync(file, data); + return file; +}; +const FILES = [ + fixture('park.geojson', geojson), + fixture('remote-bits.kml', kml(`https://${TRAP_HOST}/icon.png`)), + fixture( + 'packed.kmz', + storedZip([ + ['doc.kml', Buffer.from(kmzDoc)], + ['files/pin.png', PNG], + ['files/overlay.png', PNG], + ]), + ), + fixture('walk.gpx', gpx), +]; + +/* ── harness ───────────────────────────────────────────────────────────── */ + +const browser = await puppeteer.launch({ + headless: !HEADFUL, + args: [ + '--no-sandbox', + '--disable-background-timer-throttling', + '--disable-renderer-backgrounding', + ...(process.platform === 'darwin' + ? ['--use-angle=metal', '--enable-gpu'] + : ['--use-gl=angle', '--use-angle=swiftshader']), + ], +}); +const page = await browser.newPage(); +const errors = []; +const trapped = []; +page.on('pageerror', (error) => errors.push(`pageerror: ${error.message}`)); +page.on('request', (request) => { + if (request.url().includes(TRAP_HOST)) trapped.push(request.url()); +}); +let failures = 0; +function check(name, passed, detail = '') { + console.log( + `[${passed ? 'PASS' : 'FAIL'}] ${name}${detail ? ` — ${detail}` : ''}`, + ); + if (!passed) failures++; +} +fs.mkdirSync(SHOT_DIR, { recursive: true }); +const shot = (name) => + page.screenshot({ + path: path.join(SHOT_DIR, `${name}.jpg`), + type: 'jpeg', + quality: 82, + }); +const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const CLIP = { x: 380, y: 240, width: 480, height: 320 }; + +/** Pixels in the clip close to the first import color (#39d0ff). */ +async function cyanPixels() { + await page.evaluate(() => window.__godsEyeView.viewer.scene.requestRender()); + await wait(600); + const base64 = await page.screenshot({ + clip: CLIP, + type: 'png', + encoding: 'base64', + }); + return page.evaluate(async (b64) => { + const bitmap = await createImageBitmap( + await (await fetch(`data:image/png;base64,${b64}`)).blob(), + ); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext('2d'); + context.drawImage(bitmap, 0, 0); + const { data } = context.getImageData(0, 0, bitmap.width, bitmap.height); + let n = 0; + for (let i = 0; i < data.length; i += 4) { + const [r, g, b] = [data[i], data[i + 1], data[i + 2]]; + if (b > 150 && g > 120 && r < 140 && b - r > 60) n += 1; + } + return n; + }, base64); +} +const state = () => + page.evaluate(() => ({ + files: window.__gevGeoImport.list(), + dataSources: window.__godsEyeView.viewer.dataSources.length, + rows: document.querySelectorAll('#geo-import-list .geo-import-row').length, + hint: document.getElementById('geo-import-hint')?.textContent || '', + })); + +try { + await page.setViewport({ width: 1280, height: 860 }); + await page.goto(`${BASE_URL}/?welcome=0`, { waitUntil: 'domcontentloaded' }); + await page.waitForFunction( + () => + window.__gevGeoImport && + window.__godsEyeView?.viewer && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 90_000 }, + ); + await page.evaluate((view) => { + const viewer = window.__godsEyeView.viewer; + viewer.camera.cancelFlight?.(); + viewer.camera.setView({ + destination: window.__CESIUM__.Cartesian3.fromDegrees( + view.lon, + view.lat, + view.height, + ), + orientation: { heading: 0, pitch: (view.pitch * Math.PI) / 180, roll: 0 }, + }); + viewer.scene.requestRender(); + }, VIEW); + await wait(4000); + await page.evaluate(() => { + const rail = document.getElementById('pp-toggles'); + if (rail?.classList.contains('collapsed')) + document + .querySelector('.panel-collapse-btn[data-collapse-target="pp-toggles"]') + ?.click(); + }); + await page.waitForFunction( + () => + document.getElementById('geo-import-button')?.getBoundingClientRect() + .width > 0, + { timeout: 10_000 }, + ); + const before = await state(); + check( + 'Import is on the DISPLAY rail with an empty, hidden list', + before.rows === 0 && + !(await page.evaluate(() => + document + .getElementById('geo-import-list') + .classList.contains('visible'), + )), + ); + const cyanBefore = await cyanPixels(); + + // Picked through the real . + const input = await page.$('#geo-import-input'); + await input.uploadFile(...FILES); + const imported = await page + .waitForFunction(() => window.__gevGeoImport.list().length === 4, { + timeout: 60_000, + }) + .then( + () => true, + () => false, + ); + const after = await state(); + check( + 'four picked files become four rows and four data sources', + imported && + after.rows === 4 && + after.dataSources === before.dataSources + 4, + JSON.stringify( + after.files.map((f) => `${f.name}:${f.format}:${f.entityCount}`), + ), + ); + check( + 'formats are recognized', + after.files.map((f) => f.format).join() === 'geojson,kml,kmz,gpx', + ); + check( + 'the KML and KMZ had their outside references removed', + after.files[1].removed > 0 && after.files[2].removed > 0, + `removed ${after.files[1].removed}/${after.files[2].removed}`, + ); + check( + 'the hint says so', + /Added 4 files\..*left out/.test(after.hint), + after.hint, + ); + await page.evaluate((view) => { + const viewer = window.__godsEyeView.viewer; + viewer.camera.cancelFlight?.(); + viewer.camera.setView({ + destination: window.__CESIUM__.Cartesian3.fromDegrees( + view.lon, + view.lat, + view.height, + ), + orientation: { heading: 0, pitch: (view.pitch * Math.PI) / 180, roll: 0 }, + }); + }, VIEW); + await wait(2500); + const cyanShown = await cyanPixels(); + await shot('imported'); + check( + 'the imported polygon paints on the globe', + cyanShown > cyanBefore * 4, + `${cyanBefore} → ${cyanShown}`, + ); + + const parkId = after.files[0].id; + await page.click( + `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`, + ); + const cyanHidden = await cyanPixels(); + check( + 'hiding it stops it painting', + cyanHidden < cyanShown / 3, + `${cyanShown} → ${cyanHidden}`, + ); + await page.click( + `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`, + ); + check( + 'showing it again paints it again', + (await cyanPixels()) > cyanHidden * 4, + ); + + // Dropped onto the globe, as a person drags from their file manager. + await page.evaluate((text) => { + const container = window.__godsEyeView.viewer.container; + const transfer = new DataTransfer(); + transfer.items.add( + new File([text], 'dropped.geojson', { type: 'application/geo+json' }), + ); + for (const type of ['dragenter', 'dragover', 'drop']) + container.dispatchEvent( + new DragEvent(type, { + dataTransfer: transfer, + bubbles: true, + cancelable: true, + }), + ); + }, geojson); + const dropped = await page + .waitForFunction(() => window.__gevGeoImport.list().length === 5, { + timeout: 20_000, + }) + .then( + () => true, + () => false, + ); + check('a file dropped on the globe imports', dropped); + + // Not geodata: refused with a reason, nothing added. + await page.evaluate(() => { + const transfer = new DataTransfer(); + transfer.items.add( + new File(['just some notes'], 'notes.txt', { type: 'text/plain' }), + ); + window.__godsEyeView.viewer.container.dispatchEvent( + new DragEvent('drop', { + dataTransfer: transfer, + bubbles: true, + cancelable: true, + }), + ); + }); + await wait(800); + const refused = await state(); + check( + 'a file that is not geodata is refused with a reason', + refused.files.length === 5 && + /notes\.txt: Only GeoJSON, KML, KMZ and GPX/.test(refused.hint), + refused.hint, + ); + + // Remove every file through its row button. + for (const { id } of refused.files) + await page.click( + `#geo-import-list [data-import-id="${id}"] button[data-action="remove"]`, + ); + const cleared = await state(); + check( + 'Remove gives every data source back', + cleared.rows === 0 && cleared.dataSources === before.dataSources, + `${cleared.dataSources} vs ${before.dataSources}`, + ); + await wait(1500); + check( + 'nothing named inside the KML/KMZ was ever requested', + trapped.length === 0, + trapped.join(' '), + ); + check('no page errors', errors.length === 0, errors.join(' | ')); +} catch (error) { + check('harness ran to completion', false, error?.stack || String(error)); +} finally { + await browser.close(); + fs.rmSync(dir, { recursive: true, force: true }); +} +console.log(failures ? `\n${failures} check(s) failed` : '\nall checks passed'); +process.exit(failures ? 1 : 0); diff --git a/src/app/tools.js b/src/app/tools.js index 25d763e2d..76f8e7fe6 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -1,6 +1,7 @@ import { SceneDirector } from '../scenes/director.js'; import { initAnnotations } from '../annotations/index.js'; import { initDrawTool } from '../annotations/drawTool.js'; +import { initGeoFileImport } from '../ui/geoFileImport.js'; import { initImageryBoxTool } from '../ui/imageryBoxTool.js'; import { createRecentImageryPanel } from '../ui/recentImagery.js'; import { initGevVoiceCommands } from '../voice/gevRealtime.js'; @@ -56,6 +57,11 @@ export function createApplicationTools({ // lifetime rather than to whoever last pressed the button. const drawTool = initDrawTool({ viewer, annotations }); defer(() => drawTool?.destroy()); + // DISPLAY ▸ Import: the person's own GeoJSON/KML/KMZ/GPX files, each its + // own data source. They live for the page session, so the application + // lifetime removes them and the drop listeners on the viewer container. + const geoFileImport = initGeoFileImport({ viewer }); + defer(() => geoFileImport?.destroy()); // DATA ▸ Recent Imagery: the box tool claims the pointer like Draw and the // panel lives on the right rail, so both belong to the application // lifetime. The tileset lets the layer drape while the globe is hidden. diff --git a/src/data/geoFileImport.js b/src/data/geoFileImport.js new file mode 100644 index 000000000..ee38b7fa1 --- /dev/null +++ b/src/data/geoFileImport.js @@ -0,0 +1,432 @@ +/** + * Your own geodata on the globe: GeoJSON, KML, KMZ and GPX files read in the + * browser and handed to Cesium's own data sources. + * + * Nothing here talks to the network. The file never leaves the page, and a + * KML/KMZ file cannot make the page fetch anything either: network links, + * remote icons, remote styles, models and photo/screen overlays are removed + * before Cesium sees the document (`sanitizeKmlDocument`). Images packed + * inside a KMZ are served to Cesium from blob URLs the caller revokes. + * + * The bounded pieces — format detection, the KMZ (zip) reader, the size and + * count limits — are plain functions so they run under `node --test`; the + * Cesium loaders need a browser DOM and are exercised by + * `scripts/qa-geo-import.mjs`. + */ +import * as Cesium from 'cesium'; + +/** Largest file accepted, in bytes. */ +export const MAX_IMPORT_BYTES = 50 * 1024 * 1024; +/** Largest total a KMZ may inflate to — a zip bomb stops here. */ +export const MAX_KMZ_INFLATED_BYTES = 120 * 1024 * 1024; +/** Most files on the globe at once. */ +export const MAX_IMPORTED_FILES = 12; +/** Most entities one file may create before it is refused. */ +export const MAX_IMPORT_ENTITIES = 20_000; +/** What the file picker offers. */ +export const GEO_FILE_ACCEPT = '.geojson,.json,.kml,.kmz,.gpx'; + +const GEOJSON_TYPES = new Set([ + 'FeatureCollection', + 'Feature', + 'Point', + 'MultiPoint', + 'LineString', + 'MultiLineString', + 'Polygon', + 'MultiPolygon', + 'GeometryCollection', +]); +const KMZ_IMAGE_TYPES = Object.freeze({ + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + bmp: 'image/bmp', +}); +// KML elements that fetch, embed or execute something outside the file. +const KML_REMOVED_ELEMENTS = Object.freeze([ + 'NetworkLink', + 'NetworkLinkControl', + 'ScreenOverlay', + 'PhotoOverlay', + 'Model', + 'Tour', +]); + +/** An error whose message is meant for the person who chose the file. */ +export class GeoImportError extends Error { + constructor(message) { + super(message); + this.name = 'GeoImportError'; + } +} + +/** + * The format of a file, from its name first and its first bytes second. + * @param {string} name File name. + * @param {Uint8Array} [head] The first bytes of the file. + * @returns {'geojson'|'kml'|'kmz'|'gpx'|null} + */ +export function detectGeoFileFormat(name, head) { + const ext = /\.([a-z0-9]+)$/i.exec(String(name || ''))?.[1]?.toLowerCase(); + if (ext === 'geojson') return 'geojson'; + if (ext === 'kml' || ext === 'kmz' || ext === 'gpx') return ext; + if (!head?.length) return ext === 'json' ? 'geojson' : null; + if ( + head[0] === 0x50 && + head[1] === 0x4b && + head[2] === 0x03 && + head[3] === 0x04 + ) + return 'kmz'; + const text = new TextDecoder() + .decode(head.subarray(0, 1024)) + .replace(/^/, '') + .trimStart(); + if (text.startsWith('{')) return 'geojson'; + if (text.startsWith('<')) { + if (/]/i.test(text)) return 'kml'; + if (/]/i.test(text)) return 'gpx'; + } + return ext === 'json' ? 'geojson' : null; +} + +/** + * A short, printable name for the list: the file name without control + * characters, cut to 60 characters. + * @param {string} name + * @returns {string} + */ +export function importDisplayName(name) { + const clean = String(name || '') + .replace(/[\u0000-\u001f\u007f]/g, '') + .trim(); + if (!clean) return 'Imported file'; + return clean.length > 60 ? `${clean.slice(0, 57)}…` : clean; +} + +/** + * Parse and check GeoJSON text. Throws `GeoImportError` for anything that is + * not a GeoJSON object. + * @param {string} text + * @returns {object} + */ +export function parseGeoJsonText(text) { + let value; + try { + value = JSON.parse(String(text).replace(/^/, '')); + } catch { + throw new GeoImportError('That file is not valid JSON.'); + } + if (!value || typeof value !== 'object' || !GEOJSON_TYPES.has(value.type)) + throw new GeoImportError('That JSON file is not GeoJSON.'); + if (value.type === 'FeatureCollection' && !Array.isArray(value.features)) + throw new GeoImportError('That FeatureCollection has no features list.'); + return value; +} + +/** Inflate raw-deflate bytes with the platform's stream, stopping at `maxBytes`. */ +async function inflateRaw(bytes, maxBytes) { + const stream = new Blob([bytes]) + .stream() + .pipeThrough(new DecompressionStream('deflate-raw')); + const reader = stream.getReader(); + const chunks = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.length; + if (total > maxBytes) + throw new GeoImportError( + 'That KMZ unpacks to more than the import limit.', + ); + chunks.push(value); + } + } catch (error) { + await reader.cancel().catch(() => {}); + if (error instanceof GeoImportError) throw error; + throw new GeoImportError('That KMZ is damaged and could not be unpacked.'); + } + const out = new Uint8Array(total); + let at = 0; + for (const chunk of chunks) { + out.set(chunk, at); + at += chunk.length; + } + return out; +} + +/** + * Read the entries of a zip archive (stored or deflated; no encryption, no + * ZIP64). Directory entries are skipped. + * @param {Uint8Array} bytes The whole archive. + * @param {{maxInflatedBytes?: number}} [options] + * @returns {Promise>} Entry path → contents. + */ +export async function readZipEntries( + bytes, + { maxInflatedBytes = MAX_KMZ_INFLATED_BYTES } = {}, +) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const fail = (message = 'That KMZ is not a readable zip archive.') => { + throw new GeoImportError(message); + }; + // End of central directory: 22 bytes plus a comment of up to 64 KiB. + let eocd = -1; + for ( + let i = bytes.length - 22; + i >= Math.max(0, bytes.length - 22 - 0xffff); + i-- + ) { + if (view.getUint32(i, true) === 0x06054b50) { + eocd = i; + break; + } + } + if (eocd < 0) fail(); + const count = view.getUint16(eocd + 10, true); + const dirSize = view.getUint32(eocd + 12, true); + let at = view.getUint32(eocd + 16, true); + if (count === 0xffff || at === 0xffffffff || at + dirSize > eocd) + fail('That KMZ uses a zip format this import does not read.'); + const entries = new Map(); + const names = new TextDecoder(); + let inflated = 0; + for (let n = 0; n < count; n++) { + if (at + 46 > bytes.length || view.getUint32(at, true) !== 0x02014b50) + fail(); + const flags = view.getUint16(at + 8, true); + const method = view.getUint16(at + 10, true); + const compressed = view.getUint32(at + 20, true); + const size = view.getUint32(at + 24, true); + const nameLength = view.getUint16(at + 28, true); + const extraLength = view.getUint16(at + 30, true); + const commentLength = view.getUint16(at + 32, true); + const local = view.getUint32(at + 42, true); + const name = names.decode(bytes.subarray(at + 46, at + 46 + nameLength)); + at += 46 + nameLength + extraLength + commentLength; + if (name.endsWith('/')) continue; + if (flags & 0x1) fail('That KMZ is encrypted.'); + if (compressed === 0xffffffff || size === 0xffffffff) + fail('That KMZ uses a zip format this import does not read.'); + if (local + 30 > bytes.length || view.getUint32(local, true) !== 0x04034b50) + fail(); + const start = + local + + 30 + + view.getUint16(local + 26, true) + + view.getUint16(local + 28, true); + if (start + compressed > bytes.length) fail(); + const data = bytes.subarray(start, start + compressed); + let contents; + if (method === 0) contents = data; + else if (method === 8) + contents = await inflateRaw(data, maxInflatedBytes - inflated); + else fail('That KMZ uses a compression method this import does not read.'); + inflated += contents.length; + if (inflated > maxInflatedBytes) + fail('That KMZ unpacks to more than the import limit.'); + entries.set(name, contents); + } + return entries; +} + +/** + * The KML document inside a KMZ: `doc.kml` when present, else the first + * `.kml` entry nearest the archive root. + * @param {Map} entries + * @returns {string|null} Entry path. + */ +export function kmzMainDocument(entries) { + if (entries.has('doc.kml')) return 'doc.kml'; + const kml = [...entries.keys()] + .filter((name) => /\.kml$/i.test(name)) + .sort( + (a, b) => a.split('/').length - b.split('/').length || a.localeCompare(b), + ); + return kml[0] ?? null; +} + +/** + * Resolve a KMZ-relative href to an entry path, or null. + * @param {string} href As written in the KML. + * @param {string} base The KML entry's path. + * @param {Map} entries + * @returns {string|null} + */ +export function resolveKmzPath(href, base, entries) { + const raw = String(href || '').trim(); + if (!raw || /^[a-z][a-z0-9+.-]*:/i.test(raw) || raw.startsWith('//')) + return null; + let decoded; + try { + decoded = decodeURIComponent(raw.split(/[?#]/)[0]); + } catch { + return null; + } + const parts = base.split('/').slice(0, -1); + for (const part of decoded.replace(/\\/g, '/').split('/')) { + if (!part || part === '.') continue; + if (part === '..') parts.pop(); + else parts.push(part); + } + const path = parts.join('/'); + return entries.has(path) ? path : null; +} + +/** MIME type for a KMZ image entry, or null when it is not an image we serve. */ +export function kmzImageType(path) { + const ext = /\.([a-z0-9]+)$/i.exec(path)?.[1]?.toLowerCase(); + return KMZ_IMAGE_TYPES[ext] ?? null; +} + +const elementsNamed = (doc, localName) => + [...doc.getElementsByTagName('*')].filter((el) => el.localName === localName); + +/** + * Strip everything from a parsed KML document that would make the page reach + * outside the file. `resolveLocal(href)` returns a URL for an image packed in + * the same KMZ (or null); every other href is removed, so icons fall back to + * Cesium's default pin and a ground overlay without a local image is dropped. + * @param {Document} doc + * @param {(href: string) => string|null} [resolveLocal] + * @returns {{removed: number}} How many elements were taken out. + */ +export function sanitizeKmlDocument(doc, resolveLocal = () => null) { + let removed = 0; + const drop = (el) => { + if (el.parentNode) { + el.parentNode.removeChild(el); + removed += 1; + } + }; + for (const name of KML_REMOVED_ELEMENTS) + elementsNamed(doc, name).forEach(drop); + for (const href of elementsNamed(doc, 'href')) { + const value = (href.textContent || '').trim(); + const local = resolveLocal(value); + if (local) href.textContent = local; + else if (/^data:image\/(png|jpeg|gif|webp|bmp);/i.test(value)) continue; + else drop(href); + } + for (const overlay of elementsNamed(doc, 'GroundOverlay')) { + if (!elementsNamed(overlay, 'href').length) drop(overlay); + } + for (const styleUrl of elementsNamed(doc, 'styleUrl')) { + if (!(styleUrl.textContent || '').trim().startsWith('#')) drop(styleUrl); + } + // Placemark balloons are HTML; the app shows no info box, and keeping their + // /]]> -122.47,37.77,0 + QA palette pin#palette + -122.48,37.771,0 + QA moving + 2019-01-01T00:00:00Z2019-01-01T01:00:00Z + -122.49 37.767 0-122.475 37.772 0 QA remote stylehttps://${TRAP_HOST}/styles.kml#x -122.50,37.766 -122.46,37.772 https://${TRAP_HOST}/overlay.png @@ -67,8 +87,10 @@ const kml = (iconHref) => ` const gpx = ` QA waypoint - QA track - + QA track<img src="https://${TRAP_HOST}/gpx.png"> + + + `; // 1×1 PNG. @@ -149,8 +171,12 @@ const page = await browser.newPage(); const errors = []; const trapped = []; page.on('pageerror', (error) => errors.push(`pageerror: ${error.message}`)); +// Anything a file names, and the icon assets Cesium would load on a file's +// behalf (KML palette icons from maps.google.com, GeoJSON maki markers). +const FILE_FETCH = [TRAP_HOST, 'maps.google.com/mapfiles', '/maki/']; page.on('request', (request) => { - if (request.url().includes(TRAP_HOST)) trapped.push(request.url()); + if (FILE_FETCH.some((needle) => request.url().includes(needle))) + trapped.push(request.url()); }); let failures = 0; function check(name, passed, detail = '') { @@ -250,6 +276,23 @@ try { )), ); const cyanBefore = await cyanPixels(); + const clock = () => + page.evaluate(() => { + const c = window.__godsEyeView.viewer.clock; + const iso = (t) => window.__CESIUM__.JulianDate.toIso8601(t, 0); + return { + start: iso(c.startTime), + stop: iso(c.stopTime), + range: c.clockRange, + step: c.clockStep, + multiplier: c.multiplier, + animate: c.shouldAnimate, + // Wall-clock drift aside, the current time must not jump to 2018/2019. + year: Number(iso(c.currentTime).slice(0, 4)), + }; + }); + const clockBefore = await clock(); + const sourcesBefore = before.dataSources; // Picked through the real . const input = await page.$('#geo-import-input'); @@ -286,6 +329,65 @@ try { /Added 4 files\..*left out/.test(after.hint), after.hint, ); + const clockAfter = await clock(); + check( + "dated KML and GPX leave the app's clock alone", + JSON.stringify(clockAfter) === JSON.stringify(clockBefore) && + (await page.evaluate( + () => + window.__godsEyeView.viewer.clockTrackedDataSource?.clock === + undefined, + )), + `${JSON.stringify(clockBefore)} → ${JSON.stringify(clockAfter)}`, + ); + const inside = await page.evaluate((from) => { + const viewer = window.__godsEyeView.viewer; + const now = viewer.clock.currentTime; + const sources = []; + for (let i = from; i < viewer.dataSources.length; i++) + sources.push(viewer.dataSources.get(i)); + const entities = sources.flatMap((ds) => ds.entities.values); + const named = (name) => entities.find((e) => e.name === name); + const gpx = sources[3].entities.values; + return { + sources: sources.length, + clocks: sources.filter((ds) => ds.clock !== undefined).length, + described: entities.filter((e) => e.description !== undefined).length, + labelled: entities.filter((e) => e.label !== undefined).length, + withProperties: entities.filter((e) => e.properties !== undefined).length, + withKml: entities.filter((e) => e.kml !== undefined).length, + hidden: entities.filter((e) => !e.isAvailable(now)).length, + datedPinNow: Boolean(named('QA KML pin')?.isAvailable(now)), + movingIsLine: Boolean( + named('QA moving')?.polyline && !named('QA moving')?.position, + ), + gpxLines: gpx.filter((e) => e.polyline).length, + gpxPoints: gpx.filter((e) => e.billboard || e.point).length, + }; + }, sourcesBefore); + check( + 'no imported entity keeps a description, label, properties or KML data', + inside.sources === 4 && + inside.described + + inside.labelled + + inside.withProperties + + inside.withKml === + 0, + JSON.stringify(inside), + ); + check( + 'no imported source has a clock, and dated features draw now', + inside.clocks === 0 && + inside.hidden === 0 && + inside.datedPinNow && + inside.movingIsLine, + JSON.stringify(inside), + ); + check( + 'the GPX track keeps its two segments apart', + inside.gpxLines === 2 && inside.gpxPoints === 1, + JSON.stringify(inside), + ); await page.evaluate((view) => { const viewer = window.__godsEyeView.viewer; viewer.camera.cancelFlight?.(); @@ -325,6 +427,26 @@ try { (await cyanPixels()) > cyanHidden * 4, ); + // Keyboard: focus stays on the same control when the list is rebuilt. + const toggleSelector = `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`; + await page.focus(toggleSelector); + await page.keyboard.press('Enter'); + const focusAfterHide = await page.evaluate( + (sel) => document.activeElement === document.querySelector(sel), + toggleSelector, + ); + await page.keyboard.press('Enter'); + const focusAfterShow = await page.evaluate( + (sel) => + document.activeElement === document.querySelector(sel) && + document.activeElement.getAttribute('aria-pressed') === 'true', + toggleSelector, + ); + check( + 'keyboard focus stays on the toggle through hide and show', + focusAfterHide && focusAfterShow, + ); + // Dropped onto the globe, as a person drags from their file manager. await page.evaluate((text) => { const container = window.__godsEyeView.viewer.container; @@ -387,7 +509,7 @@ try { ); await wait(1500); check( - 'nothing named inside the KML/KMZ was ever requested', + 'nothing named inside a file, and no icon asset for one, was ever requested', trapped.length === 0, trapped.join(' '), ); diff --git a/src/app/tools.js b/src/app/tools.js index 76f8e7fe6..7acc4939e 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -60,7 +60,10 @@ export function createApplicationTools({ // DISPLAY ▸ Import: the person's own GeoJSON/KML/KMZ/GPX files, each its // own data source. They live for the page session, so the application // lifetime removes them and the drop listeners on the viewer container. - const geoFileImport = initGeoFileImport({ viewer }); + const geoFileImport = initGeoFileImport({ + viewer, + navigation: styleManager, + }); defer(() => geoFileImport?.destroy()); // DATA ▸ Recent Imagery: the box tool claims the pointer like Draw and the // panel lives on the right rail, so both belong to the application diff --git a/src/data/geoFileImport.js b/src/data/geoFileImport.js index ee38b7fa1..53d954957 100644 --- a/src/data/geoFileImport.js +++ b/src/data/geoFileImport.js @@ -8,6 +8,23 @@ * before Cesium sees the document (`sanitizeKmlDocument`). Images packed * inside a KMZ are served to Cesium from blob URLs the caller revokes. * + * What the file says about itself stays inert: descriptions, balloons, + * extended data and GeoJSON properties are dropped from every entity, names + * are kept only as short plain text, and no native Cesium label is drawn + * (`scrubImportedEntities`). GeoJSON and GPX draw in the row's color: + * simplestyle properties are dropped, including `marker-symbol`, so no icon + * asset is loaded on the file's behalf. KML and KMZ keep their own styles. + * + * An import never moves the app's shared clock: dated KML (TimeSpan, + * TimeStamp, gx:Track) is drawn for all time, and the data source exposes no + * clock for the viewer to adopt (`detachImportedTime`). GPX is read here + * rather than by Cesium (`gpxToGeoJson`), keeping each track segment as its + * own line and leaving times out. + * + * Size is checked before Cesium builds anything (`countGeoJsonGeometry`, + * `countKmlGeometry`): a file with too many features or vertices is refused + * up front instead of after the entities exist. + * * The bounded pieces — format detection, the KMZ (zip) reader, the size and * count limits — are plain functions so they run under `node --test`; the * Cesium loaders need a browser DOM and are exercised by @@ -23,6 +40,10 @@ export const MAX_KMZ_INFLATED_BYTES = 120 * 1024 * 1024; export const MAX_IMPORTED_FILES = 12; /** Most entities one file may create before it is refused. */ export const MAX_IMPORT_ENTITIES = 20_000; +/** Most vertices (points of every line, ring and marker) in one file. */ +export const MAX_IMPORT_POSITIONS = 250_000; +/** Longest plain-text name kept from a file's features. */ +export const MAX_IMPORT_NAME_LENGTH = 120; /** What the file picker offers. */ export const GEO_FILE_ACCEPT = '.geojson,.json,.kml,.kmz,.gpx'; @@ -127,6 +148,112 @@ export function parseGeoJsonText(text) { return value; } +/** + * Count features and vertices in a parsed GeoJSON object without recursion + * limits or array spreading, stopping as soon as a limit is passed. + * @param {object} geojson + * @param {{maxFeatures?: number, maxPositions?: number}} [limits] + * @returns {{features: number, positions: number, over: ''|'features'|'positions'}} + */ +export function countGeoJsonGeometry( + geojson, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + } = {}, +) { + let features = 0; + let positions = 0; + const stack = [geojson]; + while (stack.length) { + const node = stack.pop(); + if (!node || typeof node !== 'object') continue; + if (node.type === 'FeatureCollection') { + const list = Array.isArray(node.features) ? node.features : []; + for (let i = 0; i < list.length; i++) stack.push(list[i]); + continue; + } + if (node.type === 'Feature') { + features += 1; + if (features > maxFeatures) + return { features, positions, over: 'features' }; + stack.push(node.geometry); + continue; + } + if (node.type === 'GeometryCollection') { + const list = Array.isArray(node.geometries) ? node.geometries : []; + for (let i = 0; i < list.length; i++) stack.push(list[i]); + continue; + } + if (GEOJSON_TYPES.has(node.type)) { + // A bare geometry is one feature: Cesium makes one entity for it. + if (node === geojson) features += 1; + // Walk the coordinate arrays iteratively: a position is an array whose + // first item is a number. + const coords = [node.coordinates]; + while (coords.length) { + const c = coords.pop(); + if (!Array.isArray(c)) continue; + if (typeof c[0] === 'number') { + positions += 1; + if (positions > maxPositions) + return { features, positions, over: 'positions' }; + } else for (let i = 0; i < c.length; i++) coords.push(c[i]); + } + } + } + return { features, positions, over: '' }; +} + +/** The refusal for a file over a size limit, in the person's terms. */ +function overLimitError({ over, features, positions }) { + if (over === 'features') + return new GeoImportError( + `That file has more than ${MAX_IMPORT_ENTITIES.toLocaleString()} features (the limit).`, + ); + return new GeoImportError( + `That file has more than ${MAX_IMPORT_POSITIONS.toLocaleString()} points (the limit)${features ? ` across ${features.toLocaleString()} features` : ''}.`, + ); +} + +// simplestyle-spec keys Cesium honors. `marker-symbol` would load a maki icon +// asset on the file's behalf; the colors would override the row's color. +const GEOJSON_STYLE_KEYS = Object.freeze([ + 'marker-symbol', + 'marker-color', + 'marker-size', + 'stroke', + 'stroke-opacity', + 'stroke-width', + 'fill', + 'fill-opacity', +]); + +/** + * Drop GeoJSON simplestyle properties, so a GeoJSON file draws in its row's + * color like a GPX file and no icon is fetched for it. Mutates and returns + * the object. + * @param {object} geojson + * @returns {object} + */ +export function stripGeoJsonStyleProperties(geojson) { + const stack = [geojson]; + while (stack.length) { + const node = stack.pop(); + if (!node || typeof node !== 'object') continue; + if (node.type === 'FeatureCollection' && Array.isArray(node.features)) + for (let i = 0; i < node.features.length; i++) + stack.push(node.features[i]); + else if ( + node.type === 'Feature' && + node.properties && + typeof node.properties === 'object' + ) + for (const key of GEOJSON_STYLE_KEYS) delete node.properties[key]; + } + return geojson; +} + /** Inflate raw-deflate bytes with the platform's stream, stopping at `maxBytes`. */ async function inflateRaw(bytes, maxBytes) { const stream = new Blob([bytes]) @@ -326,6 +453,253 @@ export function sanitizeKmlDocument(doc, resolveLocal = () => null) { return { removed }; } +/** Child elements of `el` with this local name (any namespace). */ +const childrenNamed = (el, localName) => { + const out = []; + const kids = el?.childNodes || []; + for (let i = 0; i < kids.length; i++) + if (kids[i].nodeType === 1 && kids[i].localName === localName) + out.push(kids[i]); + return out; +}; +const childText = (el, localName) => + (childrenNamed(el, localName)[0]?.textContent || '').trim(); + +/** + * Count placemarks and vertices in a parsed KML document before Cesium reads + * it: every `` tuple and every `gx:coord`. + * @param {Document} doc + * @returns {{features: number, positions: number, over: ''|'features'|'positions'}} + */ +export function countKmlGeometry( + doc, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + } = {}, +) { + let features = 0; + let positions = 0; + for (const el of doc.getElementsByTagName('*')) { + const name = el.localName; + if (name === 'Placemark') { + features += 1; + if (features > maxFeatures) + return { features, positions, over: 'features' }; + } else if (name === 'coordinates') { + const text = el.textContent || ''; + // Tuples are separated by whitespace; count runs without splitting + // a large string into a large array. + let inTuple = false; + for (let i = 0; i < text.length; i++) { + const space = text.charCodeAt(i) <= 32; + if (!space && !inTuple) { + positions += 1; + if (positions > maxPositions) + return { features, positions, over: 'positions' }; + } + inTuple = !space; + } + } else if (name === 'coord') { + positions += 1; + if (positions > maxPositions) + return { features, positions, over: 'positions' }; + } + } + return { features, positions, over: '' }; +} + +/** + * Read a parsed GPX document into GeoJSON: waypoints become points, each + * route a line and each track one MultiLineString with a line per `` + * (segment breaks are gaps in the recording and stay gaps). Only names are + * kept, and no times, so a dated track draws without a clock. Throws + * `GeoImportError` past the size limits. + * @param {Document} doc + * @returns {object} A FeatureCollection. + */ +export function gpxToGeoJson( + doc, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + } = {}, +) { + const root = doc.documentElement; + if (root?.localName !== 'gpx') + throw new GeoImportError('That GPX file has no document.'); + const features = []; + let positions = 0; + const point = (el) => { + const lat = Number(el.getAttribute('lat')); + const lon = Number(el.getAttribute('lon')); + if ( + !Number.isFinite(lat) || + !Number.isFinite(lon) || + Math.abs(lat) > 90 || + Math.abs(lon) > 180 + ) + return null; + positions += 1; + if (positions > maxPositions) + throw overLimitError({ + over: 'positions', + features: features.length, + positions, + }); + return [lon, lat]; + }; + const add = (name, geometry) => { + if (features.length >= maxFeatures) + throw overLimitError({ + over: 'features', + features: features.length + 1, + positions, + }); + features.push({ + type: 'Feature', + properties: name ? { name } : {}, + geometry, + }); + }; + for (const wpt of childrenNamed(root, 'wpt')) { + const at = point(wpt); + if (at) add(childText(wpt, 'name'), { type: 'Point', coordinates: at }); + } + for (const rte of childrenNamed(root, 'rte')) { + const line = []; + for (const pt of childrenNamed(rte, 'rtept')) { + const at = point(pt); + if (at) line.push(at); + } + if (line.length >= 2) + add(childText(rte, 'name'), { type: 'LineString', coordinates: line }); + } + for (const trk of childrenNamed(root, 'trk')) { + const lines = []; + for (const seg of childrenNamed(trk, 'trkseg')) { + const line = []; + for (const pt of childrenNamed(seg, 'trkpt')) { + const at = point(pt); + if (at) line.push(at); + } + if (line.length >= 2) lines.push(line); + } + if (lines.length) + add(childText(trk, 'name'), { + type: 'MultiLineString', + coordinates: lines, + }); + } + return { type: 'FeatureCollection', features }; +} + +/** A feature name as short plain text (no control characters), or undefined. */ +export function importFeatureName(value) { + const text = String(value ?? '') + .replace(/[\u0000-\u001f\u007f]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + if (!text) return undefined; + return text.length > MAX_IMPORT_NAME_LENGTH + ? `${text.slice(0, MAX_IMPORT_NAME_LENGTH - 1)}…` + : text; +} + +/** + * Make every entity of an imported data source inert: no description, + * balloon, extended data or properties, no native label, and a name that is + * short plain text. Returns how many entities were touched. + * @param {{entities: {values: object[]}}} dataSource + * @returns {number} + */ +export function scrubImportedEntities(dataSource) { + const list = dataSource?.entities?.values || []; + for (let i = 0; i < list.length; i++) { + const entity = list[i]; + let name; + try { + name = importFeatureName( + typeof entity.name === 'string' ? entity.name : undefined, + ); + } catch { + name = undefined; + } + entity.name = name; + entity.description = undefined; + entity.label = undefined; + entity.properties = undefined; + if (entity.kml && typeof entity.kml === 'object') entity.kml = undefined; + } + return list.length; +} + +const TRACK_SAMPLES = 2000; + +/** + * Take time out of an imported data source so adding it cannot move the app's + * shared clock: the data source reports no clock (the viewer adopts the clock + * of every added source that has one), every entity is available for all + * time, and a time-dynamic position (a KML gx:Track) is drawn as the line it + * travels instead of a moving marker. + * @param {object} dataSource + * @param {{Cesium?: object, color?: object}} [options] + * @returns {object} The same data source. + */ +export function detachImportedTime( + dataSource, + { Cesium: C = Cesium, color } = {}, +) { + Object.defineProperty(dataSource, 'clock', { + configurable: true, + enumerable: false, + get: () => undefined, + set: () => {}, + }); + const list = dataSource?.entities?.values || []; + for (let i = 0; i < list.length; i++) { + const entity = list[i]; + const span = entity.availability; + const position = entity.position; + if (position && position.isConstant === false) { + const start = span?.start; + const stop = span?.stop; + const line = []; + if (start && stop && C.JulianDate.lessThan(start, stop)) { + const seconds = C.JulianDate.secondsDifference(stop, start); + for (let n = 0; n <= TRACK_SAMPLES; n++) { + const t = C.JulianDate.addSeconds( + start, + (seconds * n) / TRACK_SAMPLES, + new C.JulianDate(), + ); + const at = position.getValue(t); + if (at) line.push(at); + } + } + entity.path = undefined; + entity.model = undefined; + if (line.length >= 2 && !entity.polyline) { + entity.billboard = undefined; + entity.point = undefined; + entity.position = undefined; + entity.polyline = new C.PolylineGraphics({ + positions: line, + clampToGround: true, + width: 3, + material: color || C.Color.WHITE, + }); + } else { + entity.position = line.length + ? new C.ConstantPositionProperty(line[0]) + : undefined; + } + } + entity.availability = undefined; + } + return dataSource; +} + function parseXml(text, what) { const doc = new DOMParser().parseFromString( String(text).replace(/^/, ''), @@ -364,22 +738,27 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { let dataSource; let removed = 0; try { - if (format === 'geojson') { - dataSource = await Cesium.GeoJsonDataSource.load( - parseGeoJsonText(new TextDecoder().decode(bytes)), - { - clampToGround: true, - stroke, - fill: stroke.withAlpha(0.25), - strokeWidth: 3, - markerColor: stroke, - }, - ); - } else if (format === 'gpx') { - dataSource = await Cesium.GpxDataSource.load( - parseXml(new TextDecoder().decode(bytes), 'GPX'), - { clampToGround: true, trackColor: stroke, routeColor: stroke }, - ); + if (format === 'geojson' || format === 'gpx') { + let geojson; + if (format === 'gpx') { + geojson = gpxToGeoJson( + parseXml(new TextDecoder().decode(bytes), 'GPX'), + ); + } else { + geojson = parseGeoJsonText(new TextDecoder().decode(bytes)); + const size = countGeoJsonGeometry(geojson); + if (size.over) throw overLimitError(size); + stripGeoJsonStyleProperties(geojson); + } + dataSource = await Cesium.GeoJsonDataSource.load(geojson, { + clampToGround: true, + stroke, + fill: stroke.withAlpha(0.25), + strokeWidth: 3, + markerColor: stroke, + // No HTML description table is built from the file's properties. + describe: () => undefined, + }); } else { let text; let resolveLocal = () => null; @@ -402,6 +781,8 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { text = new TextDecoder().decode(bytes); } const doc = parseXml(text, format.toUpperCase()); + const size = countKmlGeometry(doc); + if (size.over) throw overLimitError(size); removed = sanitizeKmlDocument(doc, resolveLocal).removed; dataSource = await Cesium.KmlDataSource.load(doc, { camera: viewer.scene.camera, @@ -427,6 +808,11 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { revoke(); throw new GeoImportError('That file has nothing to draw.'); } - // GpxDataSource's `name` is read-only, so the display name travels beside it. + scrubImportedEntities(dataSource); + // Only KML has time-dynamic positions (GPX is read without times); its + // track line is drawn in KML's default white, like its other unstyled lines. + detachImportedTime(dataSource); + // The display name travels beside the data source (the file name, not + // anything the document says about itself). return { dataSource, name, format, entityCount, removed, revoke }; } diff --git a/src/data/geoFileImport.test.mjs b/src/data/geoFileImport.test.mjs index 522af2550..c589438fc 100644 --- a/src/data/geoFileImport.test.mjs +++ b/src/data/geoFileImport.test.mjs @@ -1,8 +1,17 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { deflateRawSync } from 'node:zlib'; +import * as Cesium from 'cesium'; import { GeoImportError, + MAX_IMPORT_POSITIONS, + countGeoJsonGeometry, + countKmlGeometry, + detachImportedTime, + gpxToGeoJson, + importFeatureName, + scrubImportedEntities, + stripGeoJsonStyleProperties, detectGeoFileFormat, importDisplayName, kmzImageType, @@ -213,3 +222,283 @@ test('KMZ hrefs resolve only to entries inside the archive', () => { assert.equal(kmzImageType('files/a.PNG'), 'image/png'); assert.equal(kmzImageType('files/a.svg'), null); }); + +/* ── a tiny element tree with the DOM surface the readers use ─────────── */ +function xmlEl(localName, attrs = {}, kids = []) { + const el = { + nodeType: 1, + localName, + childNodes: [], + getAttribute: (n) => (n in attrs ? String(attrs[n]) : null), + }; + for (const k of typeof kids === 'string' ? [] : kids) el.childNodes.push(k); + const text = typeof kids === 'string' ? kids : null; + Object.defineProperty(el, 'textContent', { + get: () => text ?? el.childNodes.map((c) => c.textContent).join(''), + }); + return el; +} +const xmlDoc = (root) => ({ + documentElement: root, + getElementsByTagName(name) { + assert.equal(name, '*'); + const out = []; + const walk = (el) => { + out.push(el); + el.childNodes.forEach(walk); + }; + walk(root); + return out; + }, +}); +const trkpt = (lat, lon, time) => + xmlEl('trkpt', { lat, lon }, time ? [xmlEl('time', {}, time)] : []); + +test('GPX keeps each track segment as its own line, names as text, no times', () => { + const doc = xmlDoc( + xmlEl('gpx', {}, [ + xmlEl('wpt', { lat: 37.77, lon: -122.49 }, [xmlEl('name', {}, 'Camp')]), + xmlEl('wpt', { lat: 99, lon: 0 }), // out of range: skipped + xmlEl('rte', {}, [ + xmlEl('name', {}, 'Route'), + xmlEl('rtept', { lat: 1, lon: 2 }), + xmlEl('rtept', { lat: 3, lon: 4 }), + ]), + xmlEl('trk', {}, [ + xmlEl('name', {}, 'Morning walk'), + xmlEl('trkseg', {}, [ + trkpt(10, 20, '2024-05-01T10:00:00Z'), + trkpt(10.1, 20.1, '2024-05-01T10:01:00Z'), + ]), + // A pause in the recording: a new segment, not a straight jump. + xmlEl('trkseg', {}, [ + trkpt(11, 21, '2024-05-01T11:00:00Z'), + trkpt(11.1, 21.1, '2024-05-01T11:01:00Z'), + trkpt(11.2, 21.2, '2024-05-01T11:02:00Z'), + ]), + xmlEl('trkseg', {}, [trkpt(12, 22)]), // one point: not a line + ]), + ]), + ); + const fc = gpxToGeoJson(doc); + assert.deepEqual( + fc.features.map((f) => [f.properties.name, f.geometry.type]), + [ + ['Camp', 'Point'], + ['Route', 'LineString'], + ['Morning walk', 'MultiLineString'], + ], + ); + assert.deepEqual(fc.features[2].geometry.coordinates, [ + [ + [20, 10], + [20.1, 10.1], + ], + [ + [21, 11], + [21.1, 11.1], + [21.2, 11.2], + ], + ]); + assert.doesNotMatch(JSON.stringify(fc), /2024|time/); + assert.throws(() => gpxToGeoJson(xmlDoc(xmlEl('kml'))), GeoImportError); +}); + +test('GPX past the point limit is refused while reading, before Cesium', () => { + const seg = xmlEl( + 'trkseg', + {}, + Array.from({ length: 30 }, (_, i) => trkpt(1 + i / 100, 2)), + ); + const doc = xmlDoc(xmlEl('gpx', {}, [xmlEl('trk', {}, [seg])])); + assert.throws( + () => gpxToGeoJson(doc, { maxPositions: 20 }), + (e) => e instanceof GeoImportError && /more than .* points/.test(e.message), + ); + assert.throws( + () => + gpxToGeoJson( + xmlDoc( + xmlEl('gpx', {}, [ + xmlEl('wpt', { lat: 1, lon: 1 }), + xmlEl('wpt', { lat: 2, lon: 2 }), + ]), + ), + { maxFeatures: 1 }, + ), + /more than .* features/, + ); +}); + +test('GeoJSON features and vertices are counted before loading, without recursion', () => { + const line = Array.from({ length: 50 }, (_, i) => [i / 10, 1]); + const fc = { + type: 'FeatureCollection', + features: [ + { type: 'Feature', geometry: { type: 'LineString', coordinates: line } }, + { + type: 'Feature', + geometry: { + type: 'GeometryCollection', + geometries: [ + { type: 'Point', coordinates: [1, 2] }, + { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 0], + ], + ], + }, + ], + }, + }, + { type: 'Feature', geometry: null }, + ], + }; + assert.deepEqual(countGeoJsonGeometry(fc), { + features: 3, + positions: 55, + over: '', + }); + assert.equal( + countGeoJsonGeometry(fc, { maxPositions: 54 }).over, + 'positions', + ); + assert.equal(countGeoJsonGeometry(fc, { maxFeatures: 2 }).over, 'features'); + assert.deepEqual( + countGeoJsonGeometry({ type: 'Point', coordinates: [1, 2] }), + { + features: 1, + positions: 1, + over: '', + }, + ); + // A very long line is counted iteratively and stops at the limit. + const huge = { + type: 'LineString', + coordinates: Array.from({ length: MAX_IMPORT_POSITIONS + 5 }, () => [0, 0]), + }; + assert.equal(countGeoJsonGeometry(huge).over, 'positions'); +}); + +test('KML placemarks and coordinate tuples are counted before loading', () => { + const coords = (n) => + xmlEl( + 'coordinates', + {}, + Array.from({ length: n }, (_, i) => `${i},1,0`).join('\n '), + ); + const doc = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('Placemark', {}, [xmlEl('LineString', {}, [coords(40)])]), + xmlEl('Placemark', {}, [ + xmlEl('Track', {}, [ + xmlEl('coord', {}, '1 2 0'), + xmlEl('coord', {}, '1 3 0'), + ]), + ]), + ]), + ); + assert.deepEqual(countKmlGeometry(doc), { + features: 2, + positions: 42, + over: '', + }); + assert.equal(countKmlGeometry(doc, { maxPositions: 41 }).over, 'positions'); + assert.equal(countKmlGeometry(doc, { maxFeatures: 1 }).over, 'features'); +}); + +test('simplestyle is dropped: no icon fetched, the row color is what draws', () => { + const fc = { + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + properties: { + 'marker-symbol': 'rail', + 'marker-color': '#f00', + stroke: '#0f0', + 'stroke-width': 9, + fill: '#00f', + 'fill-opacity': 1, + name: 'x', + }, + geometry: null, + }, + { type: 'Feature', properties: null, geometry: null }, + ], + }; + stripGeoJsonStyleProperties(fc); + assert.deepEqual(fc.features[0].properties, { name: 'x' }); +}); + +test('imported entities keep only a short plain-text name: no description, label or properties', () => { + const ds = new Cesium.CustomDataSource('t'); + const e = ds.entities.add({ + name: 'Trail\u0007 head\n' + 'x'.repeat(300), + description: '', + label: { text: 'Trail head' }, + properties: { note: '' }, + position: Cesium.Cartesian3.fromDegrees(1, 2), + }); + e.kml = { extendedData: { note: { value: '