From c5b1d41cbade33efd598286a104e49b9cd1d2952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Saggese?= Date: Mon, 21 Sep 2026 11:30:49 +0200 Subject: [PATCH 01/55] fix(search): geocode through the server, not the browser - Google's Geocoding web service refuses referrer-restricted keys, so the browser call forced GOOGLE_MAPS_API_KEY to ship unrestricted - add /api/google/geocode and /api/google/reverse-geocode, using the server key like Places and Street View already do - the browser key now needs the Map Tiles API only --- CHANGELOG.md | 9 ++ SECURITY.md | 6 +- docs/CURRENT-STATE.md | 19 ++- server/providers/local.js | 3 + server/providers/places.js | 7 +- server/providers/places/google-key.js | 33 +++- server/providers/places/google.js | 155 ++++++++++++++++++ src/annotations/annotationEngine.test.mjs | 2 +- src/annotations/annotationResolver.test.mjs | 20 +-- src/googleGeocodeProxy.test.mjs | 167 ++++++++++++++++++++ src/googleServerKey.test.mjs | 16 +- src/search/defaults.js | 23 +-- src/search/http.js | 15 +- src/search/placeSearch.test.mjs | 2 +- src/voice/gevActions.test.mjs | 6 +- 15 files changed, 436 insertions(+), 47 deletions(-) create mode 100644 src/googleGeocodeProxy.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 744f4be0a..526a89bc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +- Move Google geocoding behind the local server. Forward and reverse lookups now + go through `/api/google/geocode` and `/api/google/reverse-geocode` with the + server key, joining Places and Street View. Google's Geocoding web service + refuses referrer-restricted keys, so the previous browser call left + `SECURITY.md`'s advice and working search mutually exclusive: the bundled + browser key had to be unrestricted for search to work at all. That key now + needs the Map Tiles API only. Keyless sessions and the Photon/Nominatim + fallback order are unchanged. + - Enable responsive trackpad pinch zoom on the globe. Browser pixel-mode `Ctrl+wheel` pinch gestures now reach Cesium with bounded amplification, while ordinary wheel, line-mode and touch-pinch inputs retain their existing diff --git a/SECURITY.md b/SECURITY.md index a21ea99a4..661e8764f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -20,18 +20,18 @@ The golden rule: **secret-bearing API keys stay on the server side.** The dev/pr | `OPENAI_API_KEY` | Server only | Browser fetches a short-lived **ephemeral** Realtime session token from `/api/realtime/token`; the real key never ships | | `AISSTREAM_API_KEY` | Server only | Server holds the AISStream websocket; browser polls the same-origin `/api/ais-live` cache | | OpenSky OAuth (`OPENSKY_CLIENT_ID/SECRET`) | Server only | Server mints + refreshes the token behind `/api/opensky` | -| `GOOGLE_MAPS_SERVER_API_KEY` (optional, #33) | Server only | Server calls Places (`/api/google/nearby-places`, `/api/google/text-search`) and the Street View fallback with this key; falls back to `GOOGLE_MAPS_API_KEY` when unset | +| `GOOGLE_MAPS_SERVER_API_KEY` (optional, #33) | Server only | Server calls Places (`/api/google/nearby-places`, `/api/google/text-search`), Geocoding (`/api/google/geocode`, `/api/google/reverse-geocode`) and the Street View fallback with this key; falls back to `GOOGLE_MAPS_API_KEY` when unset | ### Two deliberately client-side keys — restrict them These are designed to be used directly in the browser (like a Mapbox public token). They are injected into the client bundle via Vite's `define`, so they **will** be visible in browser devtools. Scope and restrict them rather than trying to hide them: -1. **Google Maps API key** — loads Photorealistic 3D Tiles directly and powers GEV place search. **Restrict it** (HTTP referrer + API restriction to the required Google APIs) in the Google Cloud Console. An unrestricted key in a public deployment can be abused and billed to you. +1. **Google Maps API key** — loads Photorealistic 3D Tiles directly. **Restrict it** (HTTP referrer + API restriction to the required Google APIs) in the Google Cloud Console. An unrestricted key in a public deployment can be abused and billed to you. Restrict it to the **Map Tiles API only**: a referrer restriction applies to the Maps JavaScript API and the client SDKs, not to the web services, so a key carrying Geocoding under one is refused with `API keys with referer restrictions cannot be used with this API`. Search does not need it — geocoding goes through the server proxies above (#363). 2. **Cesium ion token** (`CESIUM_ION_TOKEN`, optional — for ion-hosted Google Photorealistic 3D Tiles, Bing world imagery, and world terrain) — used as `Cesium.Ion.defaultAccessToken` client-side. Use a public **`assets:read`** token with **URL restrictions** for any hosted deployment. The Community plan has eligibility and usage limits; a public token is not a secret, but it can still consume the account's quota. > The explicit browser `define` block in `build/vite.js` controls exactly what reaches the client: only these two keys. Everything else stays server-side. -**Places and Street View never needed to be on that list** (#33): they're called from the server-side proxies in the table above, which use `GOOGLE_MAPS_SERVER_API_KEY` when it's set. Splitting it from the browser-exposed key lets each key's Google Cloud restriction actually match what it does — the browser key referrer-restricted to the APIs the client loads, the server key IP-restricted (never a referrer, since it never leaves your server) to Places + Street View Static — instead of one key that has to be either over-permissioned or broken for one of its two jobs. A single shared `GOOGLE_MAPS_API_KEY` still works if you don't split them; it just has to cover every API both sides use. +**Places, Geocoding and Street View never needed to be on that list** (#33, #363): they're called from the server-side proxies in the table above, which use `GOOGLE_MAPS_SERVER_API_KEY` when it's set. Splitting it from the browser-exposed key lets each key's Google Cloud restriction actually match what it does — the browser key referrer-restricted to Map Tiles, the server key IP-restricted (never a referrer, since it never leaves your server) to Places + Geocoding + Street View Static — instead of one key that has to be either over-permissioned or broken for one of its jobs. A single shared `GOOGLE_MAPS_API_KEY` still works if you don't split them, but then it cannot carry a referrer restriction at all: the web services refuse one, and search would be the half that breaks. Never commit real keys. `.env` is gitignored; only `.env.example` (placeholder names) is tracked. On macOS `dev-fresh.sh` can read keys from the Keychain; plain Vite uses env vars or a local `.env`, and Pinokio uses its ignored app `ENVIRONMENT` file. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9c381ac0c..e038bfe20 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3445,7 +3445,24 @@ Providers normalize coordinates, canonical name, label, place types and optional bounds. Camera framing, nearby landmark recovery and footprint selection remain in their consumers, including the Capitol identity/containment safeguards. Radio keeps localized country names in labels rather than station filters. -Reverse geocoding and nearby/text-search endpoints retain their existing behavior. +Nearby/text-search endpoints retain their existing behavior. + +**Google geocoding is a server route, not a browser call** (#363). Forward and +reverse lookups go to `/api/google/geocode` and `/api/google/reverse-geocode`, +which carry `GOOGLE_MAPS_SERVER_API_KEY` (falling back to `GOOGLE_MAPS_API_KEY`) +and share the Places routes' opt-in `GEV_RATELIMIT_GOOGLE_PER_MIN` throttle. +Google's Geocoding **web service** refuses referrer-restricted keys, so the +previous browser call forced the bundled browser key — visible in devtools by +design — to be left unrestricted; it now needs the Map Tiles API only. Both +routes answer in Google's own `status`/`results` shape rather than the Places +routes' `places: []` contract, because the browser adapters normalize that shape +directly. The browser key still gates the provider client-side, so a keyless +session spends no request to learn it has no Google: the route's own keyless +answer reads as an unanswered provider, which would stop the chain caching the +negative Photon or Nominatim then produces. Its query is capped at 200 +characters, its reverse coordinates are validated against WGS84 bounds before +the request is paid for, and a malformed viewport bias is dropped rather than +refused, since a bias only ranks results. Only valid answers and definitive misses enter bounded caches; malformed replies, HTTP refusals and outages remain retryable. Searches share a 12-second total diff --git a/server/providers/local.js b/server/providers/local.js index 68f61a3cb..07e1a5bb8 100644 --- a/server/providers/local.js +++ b/server/providers/local.js @@ -66,6 +66,9 @@ export { LL2_CACHE_TTL_MS, launchLibraryRequestHeaders } from './space.js'; export { googlePlacesContextProxy } from './places.js'; export { googleServerApiKey } from './places.js'; export { keylessGooglePlacesResponse } from './places.js'; +export { keylessGoogleGeocodeResponse } from './places.js'; +export { validateGeocodeAddress } from './places.js'; +export { geocodeBounds } from './places.js'; export { adsbLolFallbackAnchor } from './aircraft/opensky.js'; export { readResponseTextCapped } from './common/http.js'; export { readResponseJsonCapped } from './common/http.js'; diff --git a/server/providers/places.js b/server/providers/places.js index 648be2aa8..6c36627e0 100644 --- a/server/providers/places.js +++ b/server/providers/places.js @@ -1,8 +1,13 @@ /** Place-search and route middleware for Node development servers. */ -export { googlePlacesContextProxy } from './places/google.js'; +export { + googlePlacesContextProxy, + validateGeocodeAddress, + geocodeBounds, +} from './places/google.js'; export { googleServerApiKey, keylessGooglePlacesResponse, + keylessGoogleGeocodeResponse, } from './places/google-key.js'; export { installRouteMiddleware } from './places/routes.js'; export { diff --git a/server/providers/places/google-key.js b/server/providers/places/google-key.js index 353bc9e24..a5c54b302 100644 --- a/server/providers/places/google-key.js +++ b/server/providers/places/google-key.js @@ -14,12 +14,33 @@ export function keylessGooglePlacesResponse(apiKey) { } /** - * Google API key for the SERVER-SIDE calls (Places nearby/text search, the - * CCTV Street View fallback). These never reach the browser, so this key can - * be restricted by server IP and scoped to Places API + Street View Static - * API — while GOOGLE_MAPS_API_KEY stays referrer-restricted to Map Tiles + - * Geocoding for the browser (#33). Splitting them is opt-in: unset, this - * falls back to the shared browser key and nothing changes. + * Geocoding speaks Google's own response shape rather than the `places: []` + * contract its two siblings use, because the browser adapters read `status` + * and `results` directly. The keyless capability reads the same way: a status + * that is not `OK` and no results, which the adapters already treat as "this + * provider did not answer" and pass to the next geocoder in the chain. + */ +export function keylessGoogleGeocodeResponse(apiKey) { + if (String(apiKey ?? '').trim()) return null; + return { + statusCode: 200, + payload: { + configured: false, + status: 'REQUEST_DENIED', + results: [], + error: null, + }, + }; +} + +/** + * Google API key for the SERVER-SIDE calls (Places nearby/text search, + * geocoding, the CCTV Street View fallback). These never reach the browser, so + * this key can be restricted by server IP and scoped to Places API + Geocoding + * API + Street View Static API — while GOOGLE_MAPS_API_KEY stays + * referrer-restricted to Map Tiles for the browser (#33, #363). Splitting them + * is opt-in: unset, this falls back to the shared browser key, which then has + * to carry every API both sides use and cannot take a referrer restriction. */ export function googleServerApiKey() { return resolveGoogleServerKey(process.env); diff --git a/server/providers/places/google.js b/server/providers/places/google.js index 1ee83a892..8895bc19d 100644 --- a/server/providers/places/google.js +++ b/server/providers/places/google.js @@ -1,6 +1,7 @@ import { googleServerApiKey, keylessGooglePlacesResponse, + keylessGoogleGeocodeResponse, } from './google-key.js'; import { makeOptInRateLimiter, clientKey } from '../common/rate-limit.js'; import { @@ -42,6 +43,37 @@ export function validatePlacesCoordinates(searchParams) { return { ok: true, latitude, longitude }; } +/** Longest accepted geocoder query. Real place names are far shorter. */ +const GEOCODE_MAX_QUERY = 200; + +/** Validate a free-text geocoder query before consuming request quota. */ +export function validateGeocodeAddress(searchParams) { + const address = String(searchParams.get('address') || '').trim(); + if (!address) return { ok: false, error: 'address is required' }; + if (address.length > GEOCODE_MAX_QUERY) + return { ok: false, error: 'address is too long' }; + return { ok: true, address }; +} + +/** + * Validate the optional viewport bias, which Google reads as + * `sw_lat,sw_lng|ne_lat,ne_lng`. An unparsed value is dropped rather than + * refused: the bias only ranks results, so a malformed one costs relevance, + * not an answer. + */ +export function geocodeBounds(searchParams) { + const raw = String(searchParams.get('bounds') || '').trim(); + if (!raw) return null; + const corners = raw.split('|'); + if (corners.length !== 2) return null; + const numbers = corners.flatMap((corner) => corner.split(',').map(Number)); + if (numbers.length !== 4 || !numbers.every(Number.isFinite)) return null; + const [swLat, swLon, neLat, neLon] = numbers; + if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null; + if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null; + return raw; +} + /** Nearby place labels and view-biased text search, with request-time key resolution. */ export function googlePlacesContextProxy({ resolveApiKey = googleServerApiKey, @@ -274,6 +306,129 @@ export function googlePlacesContextProxy({ ); } }); + + /** + * Geocoding belongs on the server, not in the page. Google's Geocoding web + * service refuses referrer-restricted keys, so calling it from the browser + * forces GOOGLE_MAPS_API_KEY — which ships in the bundle by design — to be + * left unrestricted (#363). Here the request carries the server key, and + * the browser key no longer needs the Geocoding API at all. + * + * Unlike the two Places routes above, these answer in Google's own shape + * (`status` + `results`) rather than the `places: []` contract: the browser + * normalizers read that shape directly, and keeping it means the proxy adds + * a hop without adding a translation nobody asked for. + */ + function installGeocodeRoute(path, readQuery) { + middlewares.use(path, async (req, res) => { + const refuse = (statusCode, error) => { + res.statusCode = statusCode; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ status: 'REQUEST_DENIED', results: [], error }), + ); + }; + + if (req.method !== 'GET') { + refuse(405, 'Method not allowed'); + return; + } + + // A keyless geocode has no provider cost, so it resolves before the + // paid-endpoint limiter can consume or exhaust quota (mirrors the + // Places routes above). + const apiKey = resolveApiKey(); + const keyless = keylessGoogleGeocodeResponse(apiKey); + if (keyless) { + res.statusCode = keyless.statusCode; + res.setHeader('Content-Type', 'application/json'); + res.setHeader('Cache-Control', 'no-store'); + res.end(JSON.stringify(keyless.payload)); + return; + } + + const requestUrl = new URL(req.url || '', 'http://localhost'); + const query = readQuery(requestUrl.searchParams); + if (!query.ok) { + refuse(400, query.error); + return; + } + + // Opt-in per-IP throttle (GEV_RATELIMIT_GOOGLE_PER_MIN), shared with + // the Places routes because both spend the same Google budget. + const _grl = googleRateLimiter(); + if (_grl && !_grl(clientKey(req))) { + res.statusCode = 429; + res.setHeader('Content-Type', 'application/json'); + res.setHeader('Retry-After', '5'); + res.end( + JSON.stringify({ + status: 'OVER_QUERY_LIMIT', + results: [], + error: 'Rate limit exceeded', + }), + ); + return; + } + + try { + const upstream = new URL( + endpoints.geocode || + 'https://maps.googleapis.com/maps/api/geocode/json', + ); + for (const [name, value] of Object.entries(query.params)) + upstream.searchParams.set(name, value); + upstream.searchParams.set('key', apiKey); + const response = await fetchImpl(upstream.toString(), { + redirect: 'error', + }); + const data = await response.json().catch(() => null); + + res.statusCode = response.ok ? 200 : response.status; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.setHeader('Cache-Control', 'private, max-age=300'); + res.end( + JSON.stringify( + data ?? { + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }, + ), + ); + } catch (error) { + res.statusCode = 502; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end( + JSON.stringify({ + status: 'UNKNOWN_ERROR', + results: [], + error: error?.message || 'Google Geocoding request failed', + }), + ); + } + }); + } + + installGeocodeRoute('/api/google/geocode', (searchParams) => { + const address = validateGeocodeAddress(searchParams); + if (!address.ok) return address; + const bounds = geocodeBounds(searchParams); + return { + ok: true, + params: { + address: address.address, + ...(bounds ? { bounds } : {}), + }, + }; + }); + + installGeocodeRoute('/api/google/reverse-geocode', (searchParams) => { + const coordinates = validatePlacesCoordinates(searchParams); + if (!coordinates.ok) return coordinates; + const { latitude, longitude } = coordinates; + return { ok: true, params: { latlng: `${latitude},${longitude}` } }; + }); } return { diff --git a/src/annotations/annotationEngine.test.mjs b/src/annotations/annotationEngine.test.mjs index 0d26974ae..cc43b612a 100644 --- a/src/annotations/annotationEngine.test.mjs +++ b/src/annotations/annotationEngine.test.mjs @@ -400,7 +400,7 @@ test('outline upgrade updates the rendered element in place without remove/add', }; let overpassCall = 0; globalThis.fetch = async (url) => { - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => ({ status: 'OK', results: [{ diff --git a/src/annotations/annotationResolver.test.mjs b/src/annotations/annotationResolver.test.mjs index 452d6997c..2c51213c1 100644 --- a/src/annotations/annotationResolver.test.mjs +++ b/src/annotations/annotationResolver.test.mjs @@ -200,7 +200,7 @@ test('ask-side admin bypass: "the Texas Capitol" recovers near-view despite a fa const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -235,7 +235,7 @@ test('ask-side admin bypass: explicit "state of Texas" skips recovery and proxim const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - assert.match(String(url), /^https:\/\/maps\.googleapis\.com\/maps\/api\/geocode/); + assert.match(String(url), /^\/api\/google\/geocode\?/); return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -273,7 +273,7 @@ for (const fixture of [ const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: fixture.lat, lon: fixture.lon, @@ -299,7 +299,7 @@ test('ask-side admin bypass: bare "Texas" remains on the guarded recovery path', const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => geocodePayload({ lat: 31.0000, lon: -99.0000, @@ -328,7 +328,7 @@ test('ask-side admin bypass: admin level 2/3 result types never grant a township const calls = []; installGoogleMocks(t, async (url) => { calls.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { const query = new URL(String(url)).searchParams.get('address'); return { json: async () => geocodePayload({ lat: 39.7817, @@ -380,7 +380,7 @@ function installCapitolMocks(t, elements, components = [ { long_name: 'Capitol Hill', types: ['neighborhood', 'political'] }, ]) { installGoogleMocks(t, async (url) => { - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { json: async () => ({ status: 'OK', results: [{ formatted_address: 'Washington, DC 20004, USA', types: ['establishment', 'landmark', 'point_of_interest', 'tourist_attraction'], @@ -458,7 +458,7 @@ test('keyless: a key that geocodes to nothing still anchors the annotation', asy const requests = []; installGoogleMocks(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (String(url).startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; return { ok: true, json: async () => ({ @@ -495,7 +495,7 @@ test('keyless: a key that geocodes to nothing still anchors the annotation', asy high: { latitude: 30.28, longitude: -97.68 }, }); // Google is asked first and exactly once; one Photon call answers it. - assert.equal(requests.filter((url) => url.includes('maps.googleapis.com')).length, 1); + assert.equal(requests.filter((url) => url.includes('/api/google/geocode')).length, 1); assert.equal(requests.filter((url) => url.includes('photon.komoot.io')).length, 1); }); @@ -512,7 +512,7 @@ test('keyless: OSM matching uses the feature\'s canonical name, not the user\'s installGoogleMocks(t, async (url, init) => { const href = String(url); - if (href.startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (href.startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; if (href.startsWith('/api/google/text-search')) return { ok: true, json: async () => ({ places: [] }) }; if (href.startsWith('https://photon.komoot.io/')) { return { @@ -575,7 +575,7 @@ test('keyless: a Photon outage is not remembered as "no such place"', async (t) const requests = []; installGoogleMocks(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) return GOOGLE_FOUND_NOTHING; + if (String(url).startsWith('/api/google/geocode')) return GOOGLE_FOUND_NOTHING; if (!photonReachable) throw new Error('network down'); return photonUp; }); diff --git a/src/googleGeocodeProxy.test.mjs b/src/googleGeocodeProxy.test.mjs new file mode 100644 index 000000000..2518d02cb --- /dev/null +++ b/src/googleGeocodeProxy.test.mjs @@ -0,0 +1,167 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + googlePlacesContextProxy, + validateGeocodeAddress, + geocodeBounds, +} from '../vite.config.js'; + +const UPSTREAM = 'https://geocode.test/json'; + +function installRoutes({ apiKey = 'server-secret', fetchImpl } = {}) { + const routes = new Map(); + googlePlacesContextProxy({ + resolveApiKey: () => apiKey, + fetchImpl, + endpoints: { geocode: UPSTREAM }, + }).configureServer({ + middlewares: { use: (name, handler) => routes.set(name, handler) }, + }); + return routes; +} + +function invokeRoute(handler, { method = 'GET', url = '/' } = {}) { + return new Promise((resolve, reject) => { + const headers = new Map(); + const res = { + statusCode: 200, + setHeader(name, value) { + headers.set(String(name).toLowerCase(), String(value)); + }, + end(body = '') { + resolve({ + statusCode: this.statusCode, + headers: Object.fromEntries(headers), + body: body ? JSON.parse(String(body)) : null, + }); + }, + }; + Promise.resolve( + handler( + { method, url, headers: {}, socket: { remoteAddress: '127.0.0.1' } }, + res, + ), + ).catch(reject); + }); +} + +test('geocoder query and viewport bias are validated before Google is paid', () => { + assert.deepEqual(validateGeocodeAddress(new URLSearchParams()), { + ok: false, + error: 'address is required', + }); + assert.deepEqual(validateGeocodeAddress(new URLSearchParams({ address: ' ' })), { + ok: false, + error: 'address is required', + }); + assert.deepEqual( + validateGeocodeAddress(new URLSearchParams({ address: 'x'.repeat(201) })), + { ok: false, error: 'address is too long' }, + ); + assert.deepEqual( + validateGeocodeAddress(new URLSearchParams({ address: ' Austin ' })), + { ok: true, address: 'Austin' }, + ); + + // A bias only ranks results, so an unusable one is dropped, not refused. + const bias = '30.1,-97.9|30.4,-97.6'; + assert.equal(geocodeBounds(new URLSearchParams({ bounds: bias })), bias); + for (const bounds of [ + '', + '30.1,-97.9', + '30.1,-97.9|30.4', + 'north,-97.9|30.4,-97.6', + '91,-97.9|30.4,-97.6', + '30.1,-181|30.4,-97.6', + ]) + assert.equal(geocodeBounds(new URLSearchParams({ bounds })), null); +}); + +test('the geocode routes keep the key server-side and answer in Google shape', async () => { + const upstream = []; + const routes = installRoutes({ + fetchImpl: async (url) => { + upstream.push(new URL(String(url))); + return { + ok: true, + json: async () => ({ status: 'OK', results: [{ place_id: 'fixture' }] }), + }; + }, + }); + const forward = routes.get('/api/google/geocode'); + const reverse = routes.get('/api/google/reverse-geocode'); + assert.equal(typeof forward, 'function'); + assert.equal(typeof reverse, 'function'); + + const forwarded = await invokeRoute(forward, { + url: '/?address=Austin&bounds=30.1,-97.9|30.4,-97.6', + }); + assert.equal(forwarded.statusCode, 200); + assert.deepEqual(forwarded.body, { + status: 'OK', + results: [{ place_id: 'fixture' }], + }); + assert.equal(upstream.at(-1).searchParams.get('address'), 'Austin'); + assert.equal(upstream.at(-1).searchParams.get('bounds'), '30.1,-97.9|30.4,-97.6'); + assert.equal(upstream.at(-1).searchParams.get('key'), 'server-secret'); + + const reversed = await invokeRoute(reverse, { url: '/?lat=30.27&lon=-97.74' }); + assert.equal(reversed.statusCode, 200); + assert.equal(upstream.at(-1).searchParams.get('latlng'), '30.27,-97.74'); + assert.equal(upstream.at(-1).searchParams.get('key'), 'server-secret'); + + // The key travels to Google, never back to the page that asked. + assert.ok(!JSON.stringify(forwarded.body).includes('server-secret')); + assert.ok(!JSON.stringify(reversed.body).includes('server-secret')); + + // A malformed bias is dropped rather than forwarded. + await invokeRoute(forward, { url: '/?address=Austin&bounds=north' }); + assert.equal(upstream.at(-1).searchParams.has('bounds'), false); +}); + +test('a bad or keyless geocode request never reaches Google', async () => { + const upstream = []; + const fetchImpl = async (url) => { + upstream.push(String(url)); + return { ok: true, json: async () => ({ status: 'OK', results: [] }) }; + }; + + const routes = installRoutes({ fetchImpl }); + for (const [path, url] of [ + ['/api/google/geocode', '/?address='], + ['/api/google/reverse-geocode', '/?lat=91&lon=-97.74'], + ]) { + const response = await invokeRoute(routes.get(path), { url }); + assert.equal(response.statusCode, 400); + assert.deepEqual(response.body.results, []); + assert.equal(response.body.status, 'REQUEST_DENIED'); + } + + for (const path of ['/api/google/geocode', '/api/google/reverse-geocode']) { + const rejected = await invokeRoute(routes.get(path), { + method: 'POST', + url: '/?address=Austin&lat=30&lon=-97', + }); + assert.equal(rejected.statusCode, 405); + } + + // Keyless is an empty capability, not an outage: a 200 the browser adapters + // read as "this provider did not answer", so the chain moves on. It resolves + // ahead of validation, like the Places routes: with nothing configured there + // is no request to validate and no quota to protect. + const keyless = installRoutes({ apiKey: ' ', fetchImpl }); + for (const path of ['/api/google/geocode', '/api/google/reverse-geocode']) + for (const url of ['/?address=Austin&lat=30&lon=-97', '/?address=&lat=91']) { + const response = await invokeRoute(keyless.get(path), { url }); + assert.equal(response.statusCode, 200); + assert.equal(response.headers['cache-control'], 'no-store'); + assert.deepEqual(response.body, { + configured: false, + status: 'REQUEST_DENIED', + results: [], + error: null, + }); + } + + assert.deepEqual(upstream, []); +}); diff --git a/src/googleServerKey.test.mjs b/src/googleServerKey.test.mjs index 0851afcd5..2789219f5 100644 --- a/src/googleServerKey.test.mjs +++ b/src/googleServerKey.test.mjs @@ -67,11 +67,19 @@ test('the Street View tool resolves per-variable overrides before preferring the } }); -test('both Places routes select the intended key and keep it out of responses', async (t) => { +test('every Google route selects the intended key and keeps it out of responses', async (t) => { const original = { server: process.env.GOOGLE_MAPS_SERVER_API_KEY, browser: process.env.GOOGLE_MAPS_API_KEY, limit: process.env.GEV_RATELIMIT_GOOGLE_PER_MIN }; const calls = []; t.mock.method(globalThis, 'fetch', async (url, options) => { - calls.push({ url, key: options.headers['X-Goog-Api-Key'] }); + // Places sends the key as a header; Geocoding's web service takes it as a + // query parameter. Either way it must be the server key, and never the + // browser's copy of it. + calls.push({ + url, + key: + options?.headers?.['X-Goog-Api-Key'] ?? + new URL(String(url)).searchParams.get('key'), + }); return Response.json({ places: [] }); }); try { @@ -88,12 +96,12 @@ test('both Places routes select the intended key and keep it out of responses', for (const install of ['configureServer', 'configurePreviewServer']) { const routes = new Map(); googlePlacesContextProxy()[install]({ middlewares: { use: (name, handler) => routes.set(name, handler) } }); - assert.equal(routes.size, 2); + assert.equal(routes.size, 4); for (const handler of routes.values()) { const before = calls.length; let body; const res = { setHeader() {}, end(value) { body = value; } }; - await handler({ method: 'GET', url: '/?lat=30&lon=-97&q=capitol', headers: {}, socket: { remoteAddress: '127.0.0.1' } }, res); + await handler({ method: 'GET', url: '/?lat=30&lon=-97&q=capitol&address=capitol', headers: {}, socket: { remoteAddress: '127.0.0.1' } }, res); assert.equal(res.statusCode, 200); assert.ok(!body.includes('server-secret')); if (expected) { diff --git a/src/search/defaults.js b/src/search/defaults.js index 2b0428025..247512405 100644 --- a/src/search/defaults.js +++ b/src/search/defaults.js @@ -40,18 +40,23 @@ export function createDefaultPlaceSearch({ ...(selected ? [selected] : [ + // Google answers through the local proxy, which holds the server + // key: its Geocoding web service refuses referrer-restricted keys, + // so calling it from here would force the bundled browser key to be + // left unrestricted (#363). That key still gates the provider, + // because a keyless session must not spend a request to learn it + // has no Google: the proxy's "not configured" reads as an + // unanswered provider, which stops the chain caching the negative + // Photon or Nominatim then produces. createGoogleGeocoder({ request(query, { bias, signal }) { - const key = resolveApiKey?.(); - if (!key) return null; - const url = new URL( - endpoints.geocode || - 'https://maps.googleapis.com/maps/api/geocode/json', + if (!resolveApiKey?.()) return null; + const params = new URLSearchParams({ address: query }); + if (bias) params.set('bounds', bias); + return fetchImpl( + `${endpoints.geocode || '/api/google/geocode'}?${params}`, + { signal }, ); - url.searchParams.set('address', query); - url.searchParams.set('key', key); - if (bias) url.searchParams.set('bounds', bias); - return fetchImpl(url.toString(), { signal }); }, }), createPhotonGeocoder({ fetchImpl, endpoint: endpoints.photon }), diff --git a/src/search/http.js b/src/search/http.js index 5b1642c67..5afa35b75 100644 --- a/src/search/http.js +++ b/src/search/http.js @@ -53,7 +53,9 @@ export function createHttpGeospatialProvider({ endpoints = {}, } = {}) { const urls = { - reverse: 'https://maps.googleapis.com/maps/api/geocode/json', + // Same-origin like its siblings below: Google's Geocoding web service + // refuses referrer-restricted keys, so the key stays on the server (#363). + reverse: '/api/google/reverse-geocode', textSearch: '/api/google/text-search', nearby: '/api/google/nearby-places', route: '/api/route', @@ -88,14 +90,11 @@ export function createHttpGeospatialProvider({ }, routeProfiles: ['foot', 'car', 'bike'], async reverseGeocode(latitude, longitude, options) { - const key = resolveApiKey?.(); - if (resolveApiKey && !key) return null; + // The browser key gates the provider without travelling with the + // request: the proxy supplies the key Google actually sees. + if (resolveApiKey && !resolveApiKey()) return null; return normalizeGoogleReverse( - await json( - urls.reverse, - { latlng: `${latitude},${longitude}`, ...(key ? { key } : {}) }, - options, - ), + await json(urls.reverse, { lat: latitude, lon: longitude }, options), ); }, async textSearch(query, { latitude, longitude, radiusM = 6000 }, options) { diff --git a/src/search/placeSearch.test.mjs b/src/search/placeSearch.test.mjs index e609f9da9..05e1aaea0 100644 --- a/src/search/placeSearch.test.mjs +++ b/src/search/placeSearch.test.mjs @@ -20,7 +20,7 @@ test('standalone geocoding falls back after Google connection, JSON and refusal for (const fail of [() => { throw new Error('offline'); }, () => new Response('invalid json'), () => Response.json({ status: 'REQUEST_DENIED' })]) { const urls = []; const service = createStandalonePlaceSearch({ resolveApiKey: () => 'fixture', fetchImpl: async (url) => { - urls.push(url); return url.includes('maps.googleapis.com') ? fail() : hit(); + urls.push(url); return url.includes('/api/google/geocode') ? fail() : hit(); } }); const result = await service.geocode('Hanoi'); assert.equal(result.place.name, 'Hà Nội'); diff --git a/src/voice/gevActions.test.mjs b/src/voice/gevActions.test.mjs index ba0ed9cf7..1d8d28c97 100644 --- a/src/voice/gevActions.test.mjs +++ b/src/voice/gevActions.test.mjs @@ -3082,7 +3082,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl const requests = []; installKeyedFetch(t, async (url) => { requests.push(String(url)); - if (String(url).startsWith('https://maps.googleapis.com/')) { + if (String(url).startsWith('/api/google/geocode')) { return { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) }; } assert.match(String(url), /^https:\/\/photon\.komoot\.io\/api\/\?/); @@ -3107,7 +3107,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl assert.ok(Math.abs(calls[0].criteria.anchor.lat - 20.9101) < 1e-9); assert.ok(Math.abs(calls[0].criteria.anchor.lon - 107.1839) < 1e-9); // Google is asked first and exactly once; Photon answers unbiased, in one call. - assert.equal(requests.filter((url) => url.includes('maps.googleapis.com')).length, 1); + assert.equal(requests.filter((url) => url.includes('/api/google/geocode')).length, 1); assert.equal(requests.filter((url) => url.includes('photon.komoot.io')).length, 1); assert.match(requests.at(-1), /[?&]q=H%E1%BA%A1\+Long\+Bay/); assert.doesNotMatch(requests.at(-1), /[?&](lat|lon|bbox)=/, 'a named radio location is not viewport-biased'); @@ -3120,7 +3120,7 @@ test('voice Radio: the keyless path applies no country filter the keyed path wou // "No Radio station matched" for exactly the places it just resolved, while a // keyed install placed a station. The label may carry it; the filter may not. const { calls, dataManager } = radioSelectionHarness(); - installKeyedFetch(t, async (url) => (String(url).startsWith('https://maps.googleapis.com/') + installKeyedFetch(t, async (url) => (String(url).startsWith('/api/google/geocode') ? { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) } : { ok: true, From 771c9e5632ae6f0177adc09bc4ab4c34b4698765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Saggese?= Date: Mon, 21 Sep 2026 11:56:33 +0200 Subject: [PATCH 02/55] fix(search): bound and sanitize the geocode proxy - reject blank viewport-bias components before Number() reads them as 0 - give the upstream call a 10s deadline and a 256 KiB capped body read - return fixed public text on failure instead of the upstream message --- server/providers/places/google.js | 60 +++++++++++++++++++++++-------- src/googleGeocodeProxy.test.mjs | 47 +++++++++++++++++++++--- 2 files changed, 88 insertions(+), 19 deletions(-) diff --git a/server/providers/places/google.js b/server/providers/places/google.js index 8895bc19d..a4aa48a78 100644 --- a/server/providers/places/google.js +++ b/server/providers/places/google.js @@ -4,6 +4,7 @@ import { keylessGoogleGeocodeResponse, } from './google-key.js'; import { makeOptInRateLimiter, clientKey } from '../common/rate-limit.js'; +import { readResponseJsonCapped } from '../common/http.js'; import { projectNearbyPlaces, projectTextSearchPlaces, @@ -46,6 +47,16 @@ export function validatePlacesCoordinates(searchParams) { /** Longest accepted geocoder query. Real place names are far shorter. */ const GEOCODE_MAX_QUERY = 200; +/** + * Upstream deadline and body ceiling for the geocode routes. The deadline stays + * armed through body consumption, so a stalled upstream cannot hold a handler + * open, and the cap bounds a chunked or length-omitted reply. A geocode answer + * is a handful of results; 256 KiB is far above any honest one. + */ +const GEOCODE_UPSTREAM_TIMEOUT_MS = 10_000; + +const GEOCODE_MAX_RESPONSE_BYTES = 256 * 1024; + /** Validate a free-text geocoder query before consuming request quota. */ export function validateGeocodeAddress(searchParams) { const address = String(searchParams.get('address') || '').trim(); @@ -66,8 +77,12 @@ export function geocodeBounds(searchParams) { if (!raw) return null; const corners = raw.split('|'); if (corners.length !== 2) return null; - const numbers = corners.flatMap((corner) => corner.split(',').map(Number)); - if (numbers.length !== 4 || !numbers.every(Number.isFinite)) return null; + const parts = corners.flatMap((corner) => corner.split(',')); + // A blank component must be refused before Number() turns it into 0 and a + // half-written corner like `30.4,` reads as the equator. + if (parts.length !== 4 || parts.some((part) => !part.trim())) return null; + const numbers = parts.map(Number); + if (!numbers.every(Number.isFinite)) return null; const [swLat, swLon, neLat, neLon] = numbers; if (Math.abs(swLat) > 90 || Math.abs(neLat) > 90) return null; if (Math.abs(swLon) > 180 || Math.abs(neLon) > 180) return null; @@ -379,31 +394,48 @@ export function googlePlacesContextProxy({ for (const [name, value] of Object.entries(query.params)) upstream.searchParams.set(name, value); upstream.searchParams.set('key', apiKey); + const deadline = AbortSignal.timeout(GEOCODE_UPSTREAM_TIMEOUT_MS); const response = await fetchImpl(upstream.toString(), { redirect: 'error', + signal: deadline, }); - const data = await response.json().catch(() => null); + // An unreadable, oversized or timed-out body is not an answer: it + // falls through to the same shaped failure as a refusal below. + const data = await readResponseJsonCapped( + response, + GEOCODE_MAX_RESPONSE_BYTES, + deadline, + ).catch(() => null); + + // A body we could not read is an upstream failure, not an answer with + // a status of its own, so it does not borrow the upstream's code. + if (!data) { + res.statusCode = 502; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.end( + JSON.stringify({ + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }), + ); + return; + } res.statusCode = response.ok ? 200 : response.status; res.setHeader('Content-Type', 'application/json; charset=utf-8'); res.setHeader('Cache-Control', 'private, max-age=300'); - res.end( - JSON.stringify( - data ?? { - status: 'UNKNOWN_ERROR', - results: [], - error: 'Google Geocoding request failed', - }, - ), - ); - } catch (error) { + res.end(JSON.stringify(data)); + } catch { + // Fixed public text: a transport, URL or runtime message would + // describe the server to the browser (SECURITY.md, sanitized errors). res.statusCode = 502; res.setHeader('Content-Type', 'application/json; charset=utf-8'); res.end( JSON.stringify({ status: 'UNKNOWN_ERROR', results: [], - error: error?.message || 'Google Geocoding request failed', + error: 'Google Geocoding request failed', }), ); } diff --git a/src/googleGeocodeProxy.test.mjs b/src/googleGeocodeProxy.test.mjs index 2518d02cb..98ffd876e 100644 --- a/src/googleGeocodeProxy.test.mjs +++ b/src/googleGeocodeProxy.test.mjs @@ -73,6 +73,10 @@ test('geocoder query and viewport bias are validated before Google is paid', () 'north,-97.9|30.4,-97.6', '91,-97.9|30.4,-97.6', '30.1,-181|30.4,-97.6', + // A blank component must not read as 0 — that is the equator, not a corner. + '30.1,-97.9|30.4,', + '30.1,|30.4,-97.6', + '30.1,-97.9| ,-97.6', ]) assert.equal(geocodeBounds(new URLSearchParams({ bounds })), null); }); @@ -82,10 +86,7 @@ test('the geocode routes keep the key server-side and answer in Google shape', a const routes = installRoutes({ fetchImpl: async (url) => { upstream.push(new URL(String(url))); - return { - ok: true, - json: async () => ({ status: 'OK', results: [{ place_id: 'fixture' }] }), - }; + return Response.json({ status: 'OK', results: [{ place_id: 'fixture' }] }); }, }); const forward = routes.get('/api/google/geocode'); @@ -123,7 +124,7 @@ test('a bad or keyless geocode request never reaches Google', async () => { const upstream = []; const fetchImpl = async (url) => { upstream.push(String(url)); - return { ok: true, json: async () => ({ status: 'OK', results: [] }) }; + return Response.json({ status: 'OK', results: [] }); }; const routes = installRoutes({ fetchImpl }); @@ -165,3 +166,39 @@ test('a bad or keyless geocode request never reaches Google', async () => { assert.deepEqual(upstream, []); }); + +test('an upstream failure reaches the browser as fixed public text', async () => { + const routes = installRoutes({ + fetchImpl: async () => { + throw new Error('connect ECONNREFUSED 10.0.0.7:443'); + }, + }); + for (const path of ['/api/google/geocode', '/api/google/reverse-geocode']) { + const response = await invokeRoute(routes.get(path), { + url: '/?address=Austin&lat=30&lon=-97', + }); + assert.equal(response.statusCode, 502); + assert.deepEqual(response.body, { + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }); + } +}); + +test('an oversized upstream body is dropped rather than buffered', async () => { + const routes = installRoutes({ + // Two hundred times the 256 KiB ceiling, declared honestly. + fetchImpl: async () => + Response.json({ status: 'OK', results: [{ pad: 'x'.repeat(52_428_800) }] }), + }); + const response = await invokeRoute(routes.get('/api/google/geocode'), { + url: '/?address=Austin', + }); + assert.equal(response.statusCode, 502); + assert.deepEqual(response.body, { + status: 'UNKNOWN_ERROR', + results: [], + error: 'Google Geocoding request failed', + }); +}); From 57b8d8b23c249e05cc122e6b9f9cec9706757439 Mon Sep 17 00:00:00 2001 From: kapturner Date: Sun, 4 Oct 2026 08:26:18 -0500 Subject: [PATCH 03/55] feat: import GeoJSON, KML, KMZ and GPX files onto the globe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DISPLAY ▸ Import (and drop onto the globe) puts the person's own files on the map as Cesium data sources, clamped to the ground, one color per file, with a row to show/hide, fly to or remove each. Files stay in the browser for the session. KML/KMZ are sanitized before Cesium reads them (network links, remote icons/styles, models, photo/screen overlays and balloon HTML removed; KMZ images served from blob URLs), so an imported file cannot make the page fetch anything. KMZ is read with a small dependency-free zip reader on the platform DecompressionStream, with a zip-bomb limit. Bounded file size, entity count and number of files. Tests: src/data/geoFileImport.test.mjs, src/ui/geoFileImport.test.mjs. Gate: scripts/qa-geo-import.mjs. The import half of #890. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 + docs/CURRENT-STATE.md | 10 + index.html | 2 +- scripts/format-scope.json | 3 + scripts/package-boundaries.json | 2 + scripts/qa-geo-import.mjs | 402 +++++++++++++++++++++++ src/app/tools.js | 6 + src/data/geoFileImport.js | 432 +++++++++++++++++++++++++ src/data/geoFileImport.test.mjs | 215 ++++++++++++ src/ui/geoFileImport.js | 329 +++++++++++++++++++ src/ui/geoFileImport.test.mjs | 399 +++++++++++++++++++++++ src/ui/styles/controls.css | 64 ++++ src/ui/templates/display-controls.html | 9 + 13 files changed, 1879 insertions(+), 1 deletion(-) create mode 100644 scripts/qa-geo-import.mjs create mode 100644 src/data/geoFileImport.js create mode 100644 src/data/geoFileImport.test.mjs create mode 100644 src/ui/geoFileImport.js create mode 100644 src/ui/geoFileImport.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3876f79..36100f297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,13 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. +- Import your own GeoJSON, KML, KMZ and GPX files (DISPLAY ▸ Import, or drop + them onto the globe). Each file is drawn on the ground in its own color with + a row to show, hide, fly to or remove it. Files stay in the browser for the + session, and KML/KMZ network links, remote icons, remote styles and balloon + HTML are left out so an imported file cannot make the page fetch anything. + The import half of #890. + ## [0.2.1] — 2026-10-02 — God's Eye View through Your Agent (with security fixes and other features) - Ship each bundled data pack once. The region, marine, admin-boundary, diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d51be986b..4a3b6a77b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4106,6 +4106,16 @@ are omitted rather than framing the wrong part of the globe. - Pure half: `src/annotations/drawMode.js` (session, minimum vertices, double-click de-dup at 0.5 m, vertex ceiling, degenerate-geometry refusal, ring closing, dateline-safe length/area/centroid, hint text). Regression surface: `src/annotations/drawMode.test.mjs` and `src/data/inputOwnership.test.mjs` (pure), `src/annotations/drawToolBehaviour.test.mjs` (the tool driven against DOM and viewer doubles — leases, borrowed input actions, teardown, the Enter key, preview heights), `src/annotations/drawTool.test.mjs` (structural: where files live, which ids the markup carries, what the docs may claim), and the rendered harness `scripts/qa-draw-tool.mjs`. - The annotation tree is a declared ownership group: `./annotations` → `src/annotations/index.js` in `package.json`, with its module graph listed in `scripts/package-boundaries.json`, so an annotation module reaching outside that graph is a gate failure rather than a surprise in a layer's bundle. `drawMode.js`, `drawTool.js` and their tests are in `scripts/format-scope.json`. +### Import Your Own Files (October 2026) + +- DISPLAY ▸ **Import** puts GeoJSON (`.geojson`/`.json`), KML, KMZ and GPX files on the globe. Files come in through the Import button (a hidden ``) or by dropping them onto the globe; the format is taken from the extension, then from the first bytes. Each file becomes its own Cesium data source (`GeoJsonDataSource`, `KmlDataSource`, `GpxDataSource`), clamped to the ground, in its own color from the whiteboard palette, with a row to show/hide it, fly to it, or remove it. The first file of a batch that loads is flown to. Imported files last for the page session: a reload starts empty, and nothing is uploaded or stored. +- Runtime entry: `createApplicationTools` (`src/app/tools.js`) calls `initGeoFileImport({ viewer })` (`src/ui/geoFileImport.js`) after the draw tool and defers its `destroy()`, which removes every imported data source, revokes every blob URL, and gives back the drop listeners on the viewer container and the `window.__gevGeoImport` handle (`importFiles`, `list`, `setVisible`, `flyTo`, `remove`, `diagnostics`). Parsing lives in `src/data/geoFileImport.js`. +- **An imported file cannot make the page fetch anything.** Before `KmlDataSource` reads a KML/KMZ document, `sanitizeKmlDocument` removes `NetworkLink`, `NetworkLinkControl`, `ScreenOverlay`, `PhotoOverlay`, `Model` and `Tour`, every `href` that is not an image packed in the same KMZ or a `data:image/` URL (an icon then falls back to Cesium's default pin, and a `GroundOverlay` left without an image is dropped), every `styleUrl` that is not a local `#id`, and placemark `description`/`BalloonStyle`/`Snippet` HTML. KMZ images are served from blob URLs. The hint says when something was left out. `scripts/qa-geo-import.mjs` imports files that name a trap host in each of those places and fails on any request to it. +- **KMZ** is read by a small zip reader (`readZipEntries`: stored and deflated entries through the platform `DecompressionStream`; no encryption, no ZIP64). The main document is `doc.kml`, else the shallowest `.kml`; relative hrefs resolve only to entries inside the archive. +- **Bounds.** A file may be at most 50 MB (`MAX_IMPORT_BYTES`), a KMZ may inflate to at most 120 MB in total (`MAX_KMZ_INFLATED_BYTES`, the zip-bomb stop), one file may create at most 20,000 entities (`MAX_IMPORT_ENTITIES`), and at most 12 files are on the globe at once (`MAX_IMPORTED_FILES`; files over that are not read). A refused file names its reason in the hint and does not stop the rest of the batch; a file with nothing to draw is refused. +- File names are rendered with `textContent` only. +- Gate: `node scripts/qa-geo-import.mjs` (dev server up). Unit: `src/data/geoFileImport.test.mjs`, `src/ui/geoFileImport.test.mjs`. + ### Recent Imagery (September 2026) - The layer sits in the Data Layers panel's Cameras group beside Cameras (CCTV); Mapped ALPR Cameras moved to the Infrastructure group to make room. diff --git a/index.html b/index.html index 2e630c4a8..78d6e34ec 100644 --- a/index.html +++ b/index.html @@ -13,7 +13,7 @@ with the sources: src/materialSymbolsSubset.test.mjs fails when a source names a glyph this list is missing, because a missing glyph does not draw a box — the ligature never forms and the element renders the literal word. --> - + diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 7062704a7..da80c0749 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -26,6 +26,8 @@ "src/annotations/drawMode.test.mjs", "src/annotations/drawTool.js", "src/annotations/drawTool.test.mjs", + "src/data/geoFileImport.test.mjs", + "src/ui/geoFileImport.test.mjs", "src/annotations/adminOutline.test.mjs", "src/annotations/regionRing.test.mjs", "src/data/adminBoundaries.test.mjs", @@ -195,6 +197,7 @@ "src/styles/snow.js", "scripts/qa-visual-effects.mjs", "scripts/qa-draw-tool.mjs", + "scripts/qa-geo-import.mjs", "src/ui/mapSource.js", "src/ui/mapSourceControls.js", "src/ui/mapSourceControls.test.mjs", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index ce778ee95..52062c1b3 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1958,6 +1958,8 @@ "src/data/inputOwnership.js", "src/annotations/drawMode.js", "src/annotations/drawTool.js", + "src/data/geoFileImport.js", + "src/ui/geoFileImport.js", "src/data/routeSteps.js", "src/layers/directions/index.js", "src/data/directions.js", diff --git a/scripts/qa-geo-import.mjs b/scripts/qa-geo-import.mjs new file mode 100644 index 000000000..c1c8f6462 --- /dev/null +++ b/scripts/qa-geo-import.mjs @@ -0,0 +1,402 @@ +#!/usr/bin/env node +/** + * Rendered acceptance for DISPLAY ▸ Import: real files through the real file + * input and a real drop, parsed by Cesium in the page. + * + * Proves: GeoJSON, KML, KMZ (with a packed icon and ground overlay) and GPX + * each become one row and one data source; the imported shape actually paints + * on the globe and stops painting when hidden; a KML/KMZ never makes the page + * request anything named in it (network link, remote icon, remote style); a + * dropped file imports like a picked one; a file that is not geodata is refused + * with a reason and leaves nothing behind; Remove gives every data source back; + * no page errors. + * + * Usage: dev server up (`npm run dev`), then `node scripts/qa-geo-import.mjs`. + * QA_BASE_URL (default http://localhost:4173), QA_SHOT_DIR, QA_HEADFUL=1. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import puppeteer from 'puppeteer'; + +const BASE_URL = process.env.QA_BASE_URL || 'http://localhost:4173'; +const SHOT_DIR = process.env.QA_SHOT_DIR || 'qa-shots/geo-import'; +const HEADFUL = process.env.QA_HEADFUL === '1'; +// Golden Gate Park: a big, flat, green rectangle the test polygon can cover. +const VIEW = { lon: -122.4862, lat: 37.7694, height: 4200, pitch: -88 }; +const TRAP_HOST = 'gev-qa-trap.invalid'; + +/* ── fixtures ──────────────────────────────────────────────────────────── */ + +const ring = [ + [-122.511, 37.7645], + [-122.4545, 37.7645], + [-122.4545, 37.7745], + [-122.511, 37.7745], + [-122.511, 37.7645], +]; +const geojson = JSON.stringify({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + properties: { name: 'QA park' }, + geometry: { type: 'Polygon', coordinates: [ring] }, + }, + { + type: 'Feature', + properties: { name: 'QA pin' }, + geometry: { type: 'Point', coordinates: [-122.4862, 37.7694] }, + }, + ], +}); +const kml = (iconHref) => ` + + + + https://${TRAP_HOST}/link.kml + QA KML pin#s + ]]> + -122.47,37.77,0 + QA remote stylehttps://${TRAP_HOST}/styles.kml#x + -122.50,37.766 -122.46,37.772 + https://${TRAP_HOST}/overlay.png + 37.7737.76-122.46-122.47 + +`; +const gpx = ` + + QA waypoint + QA track + + +`; +// 1×1 PNG. +const PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==', + 'base64', +); +/** Stored (uncompressed) zip — enough for a KMZ fixture. */ +function storedZip(files) { + const parts = []; + const central = []; + let offset = 0; + for (const [name, data] of files) { + const n = Buffer.from(name); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt32LE(data.length, 18); + local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(n.length, 26); + const c = Buffer.alloc(46); + c.writeUInt32LE(0x02014b50, 0); + c.writeUInt32LE(data.length, 20); + c.writeUInt32LE(data.length, 24); + c.writeUInt16LE(n.length, 28); + c.writeUInt32LE(offset, 42); + parts.push(local, n, data); + central.push(c, n); + offset += 30 + n.length + data.length; + } + const dir = Buffer.concat(central); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(files.length, 8); + end.writeUInt16LE(files.length, 10); + end.writeUInt32LE(dir.length, 12); + end.writeUInt32LE(offset, 16); + return Buffer.concat([...parts, dir, end]); +} +const kmzDoc = kml('files/pin.png').replace( + `https://${TRAP_HOST}/overlay.png`, + 'files/overlay.png', +); + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-qa-geo-import-')); +const fixture = (name, data) => { + const file = path.join(dir, name); + fs.writeFileSync(file, data); + return file; +}; +const FILES = [ + fixture('park.geojson', geojson), + fixture('remote-bits.kml', kml(`https://${TRAP_HOST}/icon.png`)), + fixture( + 'packed.kmz', + storedZip([ + ['doc.kml', Buffer.from(kmzDoc)], + ['files/pin.png', PNG], + ['files/overlay.png', PNG], + ]), + ), + fixture('walk.gpx', gpx), +]; + +/* ── harness ───────────────────────────────────────────────────────────── */ + +const browser = await puppeteer.launch({ + headless: !HEADFUL, + args: [ + '--no-sandbox', + '--disable-background-timer-throttling', + '--disable-renderer-backgrounding', + ...(process.platform === 'darwin' + ? ['--use-angle=metal', '--enable-gpu'] + : ['--use-gl=angle', '--use-angle=swiftshader']), + ], +}); +const page = await browser.newPage(); +const errors = []; +const trapped = []; +page.on('pageerror', (error) => errors.push(`pageerror: ${error.message}`)); +page.on('request', (request) => { + if (request.url().includes(TRAP_HOST)) trapped.push(request.url()); +}); +let failures = 0; +function check(name, passed, detail = '') { + console.log( + `[${passed ? 'PASS' : 'FAIL'}] ${name}${detail ? ` — ${detail}` : ''}`, + ); + if (!passed) failures++; +} +fs.mkdirSync(SHOT_DIR, { recursive: true }); +const shot = (name) => + page.screenshot({ + path: path.join(SHOT_DIR, `${name}.jpg`), + type: 'jpeg', + quality: 82, + }); +const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const CLIP = { x: 380, y: 240, width: 480, height: 320 }; + +/** Pixels in the clip close to the first import color (#39d0ff). */ +async function cyanPixels() { + await page.evaluate(() => window.__godsEyeView.viewer.scene.requestRender()); + await wait(600); + const base64 = await page.screenshot({ + clip: CLIP, + type: 'png', + encoding: 'base64', + }); + return page.evaluate(async (b64) => { + const bitmap = await createImageBitmap( + await (await fetch(`data:image/png;base64,${b64}`)).blob(), + ); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext('2d'); + context.drawImage(bitmap, 0, 0); + const { data } = context.getImageData(0, 0, bitmap.width, bitmap.height); + let n = 0; + for (let i = 0; i < data.length; i += 4) { + const [r, g, b] = [data[i], data[i + 1], data[i + 2]]; + if (b > 150 && g > 120 && r < 140 && b - r > 60) n += 1; + } + return n; + }, base64); +} +const state = () => + page.evaluate(() => ({ + files: window.__gevGeoImport.list(), + dataSources: window.__godsEyeView.viewer.dataSources.length, + rows: document.querySelectorAll('#geo-import-list .geo-import-row').length, + hint: document.getElementById('geo-import-hint')?.textContent || '', + })); + +try { + await page.setViewport({ width: 1280, height: 860 }); + await page.goto(`${BASE_URL}/?welcome=0`, { waitUntil: 'domcontentloaded' }); + await page.waitForFunction( + () => + window.__gevGeoImport && + window.__godsEyeView?.viewer && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 90_000 }, + ); + await page.evaluate((view) => { + const viewer = window.__godsEyeView.viewer; + viewer.camera.cancelFlight?.(); + viewer.camera.setView({ + destination: window.__CESIUM__.Cartesian3.fromDegrees( + view.lon, + view.lat, + view.height, + ), + orientation: { heading: 0, pitch: (view.pitch * Math.PI) / 180, roll: 0 }, + }); + viewer.scene.requestRender(); + }, VIEW); + await wait(4000); + await page.evaluate(() => { + const rail = document.getElementById('pp-toggles'); + if (rail?.classList.contains('collapsed')) + document + .querySelector('.panel-collapse-btn[data-collapse-target="pp-toggles"]') + ?.click(); + }); + await page.waitForFunction( + () => + document.getElementById('geo-import-button')?.getBoundingClientRect() + .width > 0, + { timeout: 10_000 }, + ); + const before = await state(); + check( + 'Import is on the DISPLAY rail with an empty, hidden list', + before.rows === 0 && + !(await page.evaluate(() => + document + .getElementById('geo-import-list') + .classList.contains('visible'), + )), + ); + const cyanBefore = await cyanPixels(); + + // Picked through the real . + const input = await page.$('#geo-import-input'); + await input.uploadFile(...FILES); + const imported = await page + .waitForFunction(() => window.__gevGeoImport.list().length === 4, { + timeout: 60_000, + }) + .then( + () => true, + () => false, + ); + const after = await state(); + check( + 'four picked files become four rows and four data sources', + imported && + after.rows === 4 && + after.dataSources === before.dataSources + 4, + JSON.stringify( + after.files.map((f) => `${f.name}:${f.format}:${f.entityCount}`), + ), + ); + check( + 'formats are recognized', + after.files.map((f) => f.format).join() === 'geojson,kml,kmz,gpx', + ); + check( + 'the KML and KMZ had their outside references removed', + after.files[1].removed > 0 && after.files[2].removed > 0, + `removed ${after.files[1].removed}/${after.files[2].removed}`, + ); + check( + 'the hint says so', + /Added 4 files\..*left out/.test(after.hint), + after.hint, + ); + await page.evaluate((view) => { + const viewer = window.__godsEyeView.viewer; + viewer.camera.cancelFlight?.(); + viewer.camera.setView({ + destination: window.__CESIUM__.Cartesian3.fromDegrees( + view.lon, + view.lat, + view.height, + ), + orientation: { heading: 0, pitch: (view.pitch * Math.PI) / 180, roll: 0 }, + }); + }, VIEW); + await wait(2500); + const cyanShown = await cyanPixels(); + await shot('imported'); + check( + 'the imported polygon paints on the globe', + cyanShown > cyanBefore * 4, + `${cyanBefore} → ${cyanShown}`, + ); + + const parkId = after.files[0].id; + await page.click( + `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`, + ); + const cyanHidden = await cyanPixels(); + check( + 'hiding it stops it painting', + cyanHidden < cyanShown / 3, + `${cyanShown} → ${cyanHidden}`, + ); + await page.click( + `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`, + ); + check( + 'showing it again paints it again', + (await cyanPixels()) > cyanHidden * 4, + ); + + // Dropped onto the globe, as a person drags from their file manager. + await page.evaluate((text) => { + const container = window.__godsEyeView.viewer.container; + const transfer = new DataTransfer(); + transfer.items.add( + new File([text], 'dropped.geojson', { type: 'application/geo+json' }), + ); + for (const type of ['dragenter', 'dragover', 'drop']) + container.dispatchEvent( + new DragEvent(type, { + dataTransfer: transfer, + bubbles: true, + cancelable: true, + }), + ); + }, geojson); + const dropped = await page + .waitForFunction(() => window.__gevGeoImport.list().length === 5, { + timeout: 20_000, + }) + .then( + () => true, + () => false, + ); + check('a file dropped on the globe imports', dropped); + + // Not geodata: refused with a reason, nothing added. + await page.evaluate(() => { + const transfer = new DataTransfer(); + transfer.items.add( + new File(['just some notes'], 'notes.txt', { type: 'text/plain' }), + ); + window.__godsEyeView.viewer.container.dispatchEvent( + new DragEvent('drop', { + dataTransfer: transfer, + bubbles: true, + cancelable: true, + }), + ); + }); + await wait(800); + const refused = await state(); + check( + 'a file that is not geodata is refused with a reason', + refused.files.length === 5 && + /notes\.txt: Only GeoJSON, KML, KMZ and GPX/.test(refused.hint), + refused.hint, + ); + + // Remove every file through its row button. + for (const { id } of refused.files) + await page.click( + `#geo-import-list [data-import-id="${id}"] button[data-action="remove"]`, + ); + const cleared = await state(); + check( + 'Remove gives every data source back', + cleared.rows === 0 && cleared.dataSources === before.dataSources, + `${cleared.dataSources} vs ${before.dataSources}`, + ); + await wait(1500); + check( + 'nothing named inside the KML/KMZ was ever requested', + trapped.length === 0, + trapped.join(' '), + ); + check('no page errors', errors.length === 0, errors.join(' | ')); +} catch (error) { + check('harness ran to completion', false, error?.stack || String(error)); +} finally { + await browser.close(); + fs.rmSync(dir, { recursive: true, force: true }); +} +console.log(failures ? `\n${failures} check(s) failed` : '\nall checks passed'); +process.exit(failures ? 1 : 0); diff --git a/src/app/tools.js b/src/app/tools.js index 25d763e2d..76f8e7fe6 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -1,6 +1,7 @@ import { SceneDirector } from '../scenes/director.js'; import { initAnnotations } from '../annotations/index.js'; import { initDrawTool } from '../annotations/drawTool.js'; +import { initGeoFileImport } from '../ui/geoFileImport.js'; import { initImageryBoxTool } from '../ui/imageryBoxTool.js'; import { createRecentImageryPanel } from '../ui/recentImagery.js'; import { initGevVoiceCommands } from '../voice/gevRealtime.js'; @@ -56,6 +57,11 @@ export function createApplicationTools({ // lifetime rather than to whoever last pressed the button. const drawTool = initDrawTool({ viewer, annotations }); defer(() => drawTool?.destroy()); + // DISPLAY ▸ Import: the person's own GeoJSON/KML/KMZ/GPX files, each its + // own data source. They live for the page session, so the application + // lifetime removes them and the drop listeners on the viewer container. + const geoFileImport = initGeoFileImport({ viewer }); + defer(() => geoFileImport?.destroy()); // DATA ▸ Recent Imagery: the box tool claims the pointer like Draw and the // panel lives on the right rail, so both belong to the application // lifetime. The tileset lets the layer drape while the globe is hidden. diff --git a/src/data/geoFileImport.js b/src/data/geoFileImport.js new file mode 100644 index 000000000..ee38b7fa1 --- /dev/null +++ b/src/data/geoFileImport.js @@ -0,0 +1,432 @@ +/** + * Your own geodata on the globe: GeoJSON, KML, KMZ and GPX files read in the + * browser and handed to Cesium's own data sources. + * + * Nothing here talks to the network. The file never leaves the page, and a + * KML/KMZ file cannot make the page fetch anything either: network links, + * remote icons, remote styles, models and photo/screen overlays are removed + * before Cesium sees the document (`sanitizeKmlDocument`). Images packed + * inside a KMZ are served to Cesium from blob URLs the caller revokes. + * + * The bounded pieces — format detection, the KMZ (zip) reader, the size and + * count limits — are plain functions so they run under `node --test`; the + * Cesium loaders need a browser DOM and are exercised by + * `scripts/qa-geo-import.mjs`. + */ +import * as Cesium from 'cesium'; + +/** Largest file accepted, in bytes. */ +export const MAX_IMPORT_BYTES = 50 * 1024 * 1024; +/** Largest total a KMZ may inflate to — a zip bomb stops here. */ +export const MAX_KMZ_INFLATED_BYTES = 120 * 1024 * 1024; +/** Most files on the globe at once. */ +export const MAX_IMPORTED_FILES = 12; +/** Most entities one file may create before it is refused. */ +export const MAX_IMPORT_ENTITIES = 20_000; +/** What the file picker offers. */ +export const GEO_FILE_ACCEPT = '.geojson,.json,.kml,.kmz,.gpx'; + +const GEOJSON_TYPES = new Set([ + 'FeatureCollection', + 'Feature', + 'Point', + 'MultiPoint', + 'LineString', + 'MultiLineString', + 'Polygon', + 'MultiPolygon', + 'GeometryCollection', +]); +const KMZ_IMAGE_TYPES = Object.freeze({ + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + bmp: 'image/bmp', +}); +// KML elements that fetch, embed or execute something outside the file. +const KML_REMOVED_ELEMENTS = Object.freeze([ + 'NetworkLink', + 'NetworkLinkControl', + 'ScreenOverlay', + 'PhotoOverlay', + 'Model', + 'Tour', +]); + +/** An error whose message is meant for the person who chose the file. */ +export class GeoImportError extends Error { + constructor(message) { + super(message); + this.name = 'GeoImportError'; + } +} + +/** + * The format of a file, from its name first and its first bytes second. + * @param {string} name File name. + * @param {Uint8Array} [head] The first bytes of the file. + * @returns {'geojson'|'kml'|'kmz'|'gpx'|null} + */ +export function detectGeoFileFormat(name, head) { + const ext = /\.([a-z0-9]+)$/i.exec(String(name || ''))?.[1]?.toLowerCase(); + if (ext === 'geojson') return 'geojson'; + if (ext === 'kml' || ext === 'kmz' || ext === 'gpx') return ext; + if (!head?.length) return ext === 'json' ? 'geojson' : null; + if ( + head[0] === 0x50 && + head[1] === 0x4b && + head[2] === 0x03 && + head[3] === 0x04 + ) + return 'kmz'; + const text = new TextDecoder() + .decode(head.subarray(0, 1024)) + .replace(/^/, '') + .trimStart(); + if (text.startsWith('{')) return 'geojson'; + if (text.startsWith('<')) { + if (/]/i.test(text)) return 'kml'; + if (/]/i.test(text)) return 'gpx'; + } + return ext === 'json' ? 'geojson' : null; +} + +/** + * A short, printable name for the list: the file name without control + * characters, cut to 60 characters. + * @param {string} name + * @returns {string} + */ +export function importDisplayName(name) { + const clean = String(name || '') + .replace(/[\u0000-\u001f\u007f]/g, '') + .trim(); + if (!clean) return 'Imported file'; + return clean.length > 60 ? `${clean.slice(0, 57)}…` : clean; +} + +/** + * Parse and check GeoJSON text. Throws `GeoImportError` for anything that is + * not a GeoJSON object. + * @param {string} text + * @returns {object} + */ +export function parseGeoJsonText(text) { + let value; + try { + value = JSON.parse(String(text).replace(/^/, '')); + } catch { + throw new GeoImportError('That file is not valid JSON.'); + } + if (!value || typeof value !== 'object' || !GEOJSON_TYPES.has(value.type)) + throw new GeoImportError('That JSON file is not GeoJSON.'); + if (value.type === 'FeatureCollection' && !Array.isArray(value.features)) + throw new GeoImportError('That FeatureCollection has no features list.'); + return value; +} + +/** Inflate raw-deflate bytes with the platform's stream, stopping at `maxBytes`. */ +async function inflateRaw(bytes, maxBytes) { + const stream = new Blob([bytes]) + .stream() + .pipeThrough(new DecompressionStream('deflate-raw')); + const reader = stream.getReader(); + const chunks = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.length; + if (total > maxBytes) + throw new GeoImportError( + 'That KMZ unpacks to more than the import limit.', + ); + chunks.push(value); + } + } catch (error) { + await reader.cancel().catch(() => {}); + if (error instanceof GeoImportError) throw error; + throw new GeoImportError('That KMZ is damaged and could not be unpacked.'); + } + const out = new Uint8Array(total); + let at = 0; + for (const chunk of chunks) { + out.set(chunk, at); + at += chunk.length; + } + return out; +} + +/** + * Read the entries of a zip archive (stored or deflated; no encryption, no + * ZIP64). Directory entries are skipped. + * @param {Uint8Array} bytes The whole archive. + * @param {{maxInflatedBytes?: number}} [options] + * @returns {Promise>} Entry path → contents. + */ +export async function readZipEntries( + bytes, + { maxInflatedBytes = MAX_KMZ_INFLATED_BYTES } = {}, +) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const fail = (message = 'That KMZ is not a readable zip archive.') => { + throw new GeoImportError(message); + }; + // End of central directory: 22 bytes plus a comment of up to 64 KiB. + let eocd = -1; + for ( + let i = bytes.length - 22; + i >= Math.max(0, bytes.length - 22 - 0xffff); + i-- + ) { + if (view.getUint32(i, true) === 0x06054b50) { + eocd = i; + break; + } + } + if (eocd < 0) fail(); + const count = view.getUint16(eocd + 10, true); + const dirSize = view.getUint32(eocd + 12, true); + let at = view.getUint32(eocd + 16, true); + if (count === 0xffff || at === 0xffffffff || at + dirSize > eocd) + fail('That KMZ uses a zip format this import does not read.'); + const entries = new Map(); + const names = new TextDecoder(); + let inflated = 0; + for (let n = 0; n < count; n++) { + if (at + 46 > bytes.length || view.getUint32(at, true) !== 0x02014b50) + fail(); + const flags = view.getUint16(at + 8, true); + const method = view.getUint16(at + 10, true); + const compressed = view.getUint32(at + 20, true); + const size = view.getUint32(at + 24, true); + const nameLength = view.getUint16(at + 28, true); + const extraLength = view.getUint16(at + 30, true); + const commentLength = view.getUint16(at + 32, true); + const local = view.getUint32(at + 42, true); + const name = names.decode(bytes.subarray(at + 46, at + 46 + nameLength)); + at += 46 + nameLength + extraLength + commentLength; + if (name.endsWith('/')) continue; + if (flags & 0x1) fail('That KMZ is encrypted.'); + if (compressed === 0xffffffff || size === 0xffffffff) + fail('That KMZ uses a zip format this import does not read.'); + if (local + 30 > bytes.length || view.getUint32(local, true) !== 0x04034b50) + fail(); + const start = + local + + 30 + + view.getUint16(local + 26, true) + + view.getUint16(local + 28, true); + if (start + compressed > bytes.length) fail(); + const data = bytes.subarray(start, start + compressed); + let contents; + if (method === 0) contents = data; + else if (method === 8) + contents = await inflateRaw(data, maxInflatedBytes - inflated); + else fail('That KMZ uses a compression method this import does not read.'); + inflated += contents.length; + if (inflated > maxInflatedBytes) + fail('That KMZ unpacks to more than the import limit.'); + entries.set(name, contents); + } + return entries; +} + +/** + * The KML document inside a KMZ: `doc.kml` when present, else the first + * `.kml` entry nearest the archive root. + * @param {Map} entries + * @returns {string|null} Entry path. + */ +export function kmzMainDocument(entries) { + if (entries.has('doc.kml')) return 'doc.kml'; + const kml = [...entries.keys()] + .filter((name) => /\.kml$/i.test(name)) + .sort( + (a, b) => a.split('/').length - b.split('/').length || a.localeCompare(b), + ); + return kml[0] ?? null; +} + +/** + * Resolve a KMZ-relative href to an entry path, or null. + * @param {string} href As written in the KML. + * @param {string} base The KML entry's path. + * @param {Map} entries + * @returns {string|null} + */ +export function resolveKmzPath(href, base, entries) { + const raw = String(href || '').trim(); + if (!raw || /^[a-z][a-z0-9+.-]*:/i.test(raw) || raw.startsWith('//')) + return null; + let decoded; + try { + decoded = decodeURIComponent(raw.split(/[?#]/)[0]); + } catch { + return null; + } + const parts = base.split('/').slice(0, -1); + for (const part of decoded.replace(/\\/g, '/').split('/')) { + if (!part || part === '.') continue; + if (part === '..') parts.pop(); + else parts.push(part); + } + const path = parts.join('/'); + return entries.has(path) ? path : null; +} + +/** MIME type for a KMZ image entry, or null when it is not an image we serve. */ +export function kmzImageType(path) { + const ext = /\.([a-z0-9]+)$/i.exec(path)?.[1]?.toLowerCase(); + return KMZ_IMAGE_TYPES[ext] ?? null; +} + +const elementsNamed = (doc, localName) => + [...doc.getElementsByTagName('*')].filter((el) => el.localName === localName); + +/** + * Strip everything from a parsed KML document that would make the page reach + * outside the file. `resolveLocal(href)` returns a URL for an image packed in + * the same KMZ (or null); every other href is removed, so icons fall back to + * Cesium's default pin and a ground overlay without a local image is dropped. + * @param {Document} doc + * @param {(href: string) => string|null} [resolveLocal] + * @returns {{removed: number}} How many elements were taken out. + */ +export function sanitizeKmlDocument(doc, resolveLocal = () => null) { + let removed = 0; + const drop = (el) => { + if (el.parentNode) { + el.parentNode.removeChild(el); + removed += 1; + } + }; + for (const name of KML_REMOVED_ELEMENTS) + elementsNamed(doc, name).forEach(drop); + for (const href of elementsNamed(doc, 'href')) { + const value = (href.textContent || '').trim(); + const local = resolveLocal(value); + if (local) href.textContent = local; + else if (/^data:image\/(png|jpeg|gif|webp|bmp);/i.test(value)) continue; + else drop(href); + } + for (const overlay of elementsNamed(doc, 'GroundOverlay')) { + if (!elementsNamed(overlay, 'href').length) drop(overlay); + } + for (const styleUrl of elementsNamed(doc, 'styleUrl')) { + if (!(styleUrl.textContent || '').trim().startsWith('#')) drop(styleUrl); + } + // Placemark balloons are HTML; the app shows no info box, and keeping their + // /]]> -122.47,37.77,0 + QA palette pin#palette + -122.48,37.771,0 + QA moving + 2019-01-01T00:00:00Z2019-01-01T01:00:00Z + -122.49 37.767 0-122.475 37.772 0 QA remote stylehttps://${TRAP_HOST}/styles.kml#x -122.50,37.766 -122.46,37.772 https://${TRAP_HOST}/overlay.png @@ -67,8 +87,10 @@ const kml = (iconHref) => ` const gpx = ` QA waypoint - QA track - + QA track<img src="https://${TRAP_HOST}/gpx.png"> + + + `; // 1×1 PNG. @@ -149,8 +171,12 @@ const page = await browser.newPage(); const errors = []; const trapped = []; page.on('pageerror', (error) => errors.push(`pageerror: ${error.message}`)); +// Anything a file names, and the icon assets Cesium would load on a file's +// behalf (KML palette icons from maps.google.com, GeoJSON maki markers). +const FILE_FETCH = [TRAP_HOST, 'maps.google.com/mapfiles', '/maki/']; page.on('request', (request) => { - if (request.url().includes(TRAP_HOST)) trapped.push(request.url()); + if (FILE_FETCH.some((needle) => request.url().includes(needle))) + trapped.push(request.url()); }); let failures = 0; function check(name, passed, detail = '') { @@ -250,6 +276,23 @@ try { )), ); const cyanBefore = await cyanPixels(); + const clock = () => + page.evaluate(() => { + const c = window.__godsEyeView.viewer.clock; + const iso = (t) => window.__CESIUM__.JulianDate.toIso8601(t, 0); + return { + start: iso(c.startTime), + stop: iso(c.stopTime), + range: c.clockRange, + step: c.clockStep, + multiplier: c.multiplier, + animate: c.shouldAnimate, + // Wall-clock drift aside, the current time must not jump to 2018/2019. + year: Number(iso(c.currentTime).slice(0, 4)), + }; + }); + const clockBefore = await clock(); + const sourcesBefore = before.dataSources; // Picked through the real . const input = await page.$('#geo-import-input'); @@ -286,6 +329,65 @@ try { /Added 4 files\..*left out/.test(after.hint), after.hint, ); + const clockAfter = await clock(); + check( + "dated KML and GPX leave the app's clock alone", + JSON.stringify(clockAfter) === JSON.stringify(clockBefore) && + (await page.evaluate( + () => + window.__godsEyeView.viewer.clockTrackedDataSource?.clock === + undefined, + )), + `${JSON.stringify(clockBefore)} → ${JSON.stringify(clockAfter)}`, + ); + const inside = await page.evaluate((from) => { + const viewer = window.__godsEyeView.viewer; + const now = viewer.clock.currentTime; + const sources = []; + for (let i = from; i < viewer.dataSources.length; i++) + sources.push(viewer.dataSources.get(i)); + const entities = sources.flatMap((ds) => ds.entities.values); + const named = (name) => entities.find((e) => e.name === name); + const gpx = sources[3].entities.values; + return { + sources: sources.length, + clocks: sources.filter((ds) => ds.clock !== undefined).length, + described: entities.filter((e) => e.description !== undefined).length, + labelled: entities.filter((e) => e.label !== undefined).length, + withProperties: entities.filter((e) => e.properties !== undefined).length, + withKml: entities.filter((e) => e.kml !== undefined).length, + hidden: entities.filter((e) => !e.isAvailable(now)).length, + datedPinNow: Boolean(named('QA KML pin')?.isAvailable(now)), + movingIsLine: Boolean( + named('QA moving')?.polyline && !named('QA moving')?.position, + ), + gpxLines: gpx.filter((e) => e.polyline).length, + gpxPoints: gpx.filter((e) => e.billboard || e.point).length, + }; + }, sourcesBefore); + check( + 'no imported entity keeps a description, label, properties or KML data', + inside.sources === 4 && + inside.described + + inside.labelled + + inside.withProperties + + inside.withKml === + 0, + JSON.stringify(inside), + ); + check( + 'no imported source has a clock, and dated features draw now', + inside.clocks === 0 && + inside.hidden === 0 && + inside.datedPinNow && + inside.movingIsLine, + JSON.stringify(inside), + ); + check( + 'the GPX track keeps its two segments apart', + inside.gpxLines === 2 && inside.gpxPoints === 1, + JSON.stringify(inside), + ); await page.evaluate((view) => { const viewer = window.__godsEyeView.viewer; viewer.camera.cancelFlight?.(); @@ -325,6 +427,26 @@ try { (await cyanPixels()) > cyanHidden * 4, ); + // Keyboard: focus stays on the same control when the list is rebuilt. + const toggleSelector = `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`; + await page.focus(toggleSelector); + await page.keyboard.press('Enter'); + const focusAfterHide = await page.evaluate( + (sel) => document.activeElement === document.querySelector(sel), + toggleSelector, + ); + await page.keyboard.press('Enter'); + const focusAfterShow = await page.evaluate( + (sel) => + document.activeElement === document.querySelector(sel) && + document.activeElement.getAttribute('aria-pressed') === 'true', + toggleSelector, + ); + check( + 'keyboard focus stays on the toggle through hide and show', + focusAfterHide && focusAfterShow, + ); + // Dropped onto the globe, as a person drags from their file manager. await page.evaluate((text) => { const container = window.__godsEyeView.viewer.container; @@ -387,7 +509,7 @@ try { ); await wait(1500); check( - 'nothing named inside the KML/KMZ was ever requested', + 'nothing named inside a file, and no icon asset for one, was ever requested', trapped.length === 0, trapped.join(' '), ); diff --git a/src/app/tools.js b/src/app/tools.js index 76f8e7fe6..7acc4939e 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -60,7 +60,10 @@ export function createApplicationTools({ // DISPLAY ▸ Import: the person's own GeoJSON/KML/KMZ/GPX files, each its // own data source. They live for the page session, so the application // lifetime removes them and the drop listeners on the viewer container. - const geoFileImport = initGeoFileImport({ viewer }); + const geoFileImport = initGeoFileImport({ + viewer, + navigation: styleManager, + }); defer(() => geoFileImport?.destroy()); // DATA ▸ Recent Imagery: the box tool claims the pointer like Draw and the // panel lives on the right rail, so both belong to the application diff --git a/src/data/geoFileImport.js b/src/data/geoFileImport.js index ee38b7fa1..53d954957 100644 --- a/src/data/geoFileImport.js +++ b/src/data/geoFileImport.js @@ -8,6 +8,23 @@ * before Cesium sees the document (`sanitizeKmlDocument`). Images packed * inside a KMZ are served to Cesium from blob URLs the caller revokes. * + * What the file says about itself stays inert: descriptions, balloons, + * extended data and GeoJSON properties are dropped from every entity, names + * are kept only as short plain text, and no native Cesium label is drawn + * (`scrubImportedEntities`). GeoJSON and GPX draw in the row's color: + * simplestyle properties are dropped, including `marker-symbol`, so no icon + * asset is loaded on the file's behalf. KML and KMZ keep their own styles. + * + * An import never moves the app's shared clock: dated KML (TimeSpan, + * TimeStamp, gx:Track) is drawn for all time, and the data source exposes no + * clock for the viewer to adopt (`detachImportedTime`). GPX is read here + * rather than by Cesium (`gpxToGeoJson`), keeping each track segment as its + * own line and leaving times out. + * + * Size is checked before Cesium builds anything (`countGeoJsonGeometry`, + * `countKmlGeometry`): a file with too many features or vertices is refused + * up front instead of after the entities exist. + * * The bounded pieces — format detection, the KMZ (zip) reader, the size and * count limits — are plain functions so they run under `node --test`; the * Cesium loaders need a browser DOM and are exercised by @@ -23,6 +40,10 @@ export const MAX_KMZ_INFLATED_BYTES = 120 * 1024 * 1024; export const MAX_IMPORTED_FILES = 12; /** Most entities one file may create before it is refused. */ export const MAX_IMPORT_ENTITIES = 20_000; +/** Most vertices (points of every line, ring and marker) in one file. */ +export const MAX_IMPORT_POSITIONS = 250_000; +/** Longest plain-text name kept from a file's features. */ +export const MAX_IMPORT_NAME_LENGTH = 120; /** What the file picker offers. */ export const GEO_FILE_ACCEPT = '.geojson,.json,.kml,.kmz,.gpx'; @@ -127,6 +148,112 @@ export function parseGeoJsonText(text) { return value; } +/** + * Count features and vertices in a parsed GeoJSON object without recursion + * limits or array spreading, stopping as soon as a limit is passed. + * @param {object} geojson + * @param {{maxFeatures?: number, maxPositions?: number}} [limits] + * @returns {{features: number, positions: number, over: ''|'features'|'positions'}} + */ +export function countGeoJsonGeometry( + geojson, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + } = {}, +) { + let features = 0; + let positions = 0; + const stack = [geojson]; + while (stack.length) { + const node = stack.pop(); + if (!node || typeof node !== 'object') continue; + if (node.type === 'FeatureCollection') { + const list = Array.isArray(node.features) ? node.features : []; + for (let i = 0; i < list.length; i++) stack.push(list[i]); + continue; + } + if (node.type === 'Feature') { + features += 1; + if (features > maxFeatures) + return { features, positions, over: 'features' }; + stack.push(node.geometry); + continue; + } + if (node.type === 'GeometryCollection') { + const list = Array.isArray(node.geometries) ? node.geometries : []; + for (let i = 0; i < list.length; i++) stack.push(list[i]); + continue; + } + if (GEOJSON_TYPES.has(node.type)) { + // A bare geometry is one feature: Cesium makes one entity for it. + if (node === geojson) features += 1; + // Walk the coordinate arrays iteratively: a position is an array whose + // first item is a number. + const coords = [node.coordinates]; + while (coords.length) { + const c = coords.pop(); + if (!Array.isArray(c)) continue; + if (typeof c[0] === 'number') { + positions += 1; + if (positions > maxPositions) + return { features, positions, over: 'positions' }; + } else for (let i = 0; i < c.length; i++) coords.push(c[i]); + } + } + } + return { features, positions, over: '' }; +} + +/** The refusal for a file over a size limit, in the person's terms. */ +function overLimitError({ over, features, positions }) { + if (over === 'features') + return new GeoImportError( + `That file has more than ${MAX_IMPORT_ENTITIES.toLocaleString()} features (the limit).`, + ); + return new GeoImportError( + `That file has more than ${MAX_IMPORT_POSITIONS.toLocaleString()} points (the limit)${features ? ` across ${features.toLocaleString()} features` : ''}.`, + ); +} + +// simplestyle-spec keys Cesium honors. `marker-symbol` would load a maki icon +// asset on the file's behalf; the colors would override the row's color. +const GEOJSON_STYLE_KEYS = Object.freeze([ + 'marker-symbol', + 'marker-color', + 'marker-size', + 'stroke', + 'stroke-opacity', + 'stroke-width', + 'fill', + 'fill-opacity', +]); + +/** + * Drop GeoJSON simplestyle properties, so a GeoJSON file draws in its row's + * color like a GPX file and no icon is fetched for it. Mutates and returns + * the object. + * @param {object} geojson + * @returns {object} + */ +export function stripGeoJsonStyleProperties(geojson) { + const stack = [geojson]; + while (stack.length) { + const node = stack.pop(); + if (!node || typeof node !== 'object') continue; + if (node.type === 'FeatureCollection' && Array.isArray(node.features)) + for (let i = 0; i < node.features.length; i++) + stack.push(node.features[i]); + else if ( + node.type === 'Feature' && + node.properties && + typeof node.properties === 'object' + ) + for (const key of GEOJSON_STYLE_KEYS) delete node.properties[key]; + } + return geojson; +} + /** Inflate raw-deflate bytes with the platform's stream, stopping at `maxBytes`. */ async function inflateRaw(bytes, maxBytes) { const stream = new Blob([bytes]) @@ -326,6 +453,253 @@ export function sanitizeKmlDocument(doc, resolveLocal = () => null) { return { removed }; } +/** Child elements of `el` with this local name (any namespace). */ +const childrenNamed = (el, localName) => { + const out = []; + const kids = el?.childNodes || []; + for (let i = 0; i < kids.length; i++) + if (kids[i].nodeType === 1 && kids[i].localName === localName) + out.push(kids[i]); + return out; +}; +const childText = (el, localName) => + (childrenNamed(el, localName)[0]?.textContent || '').trim(); + +/** + * Count placemarks and vertices in a parsed KML document before Cesium reads + * it: every `` tuple and every `gx:coord`. + * @param {Document} doc + * @returns {{features: number, positions: number, over: ''|'features'|'positions'}} + */ +export function countKmlGeometry( + doc, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + } = {}, +) { + let features = 0; + let positions = 0; + for (const el of doc.getElementsByTagName('*')) { + const name = el.localName; + if (name === 'Placemark') { + features += 1; + if (features > maxFeatures) + return { features, positions, over: 'features' }; + } else if (name === 'coordinates') { + const text = el.textContent || ''; + // Tuples are separated by whitespace; count runs without splitting + // a large string into a large array. + let inTuple = false; + for (let i = 0; i < text.length; i++) { + const space = text.charCodeAt(i) <= 32; + if (!space && !inTuple) { + positions += 1; + if (positions > maxPositions) + return { features, positions, over: 'positions' }; + } + inTuple = !space; + } + } else if (name === 'coord') { + positions += 1; + if (positions > maxPositions) + return { features, positions, over: 'positions' }; + } + } + return { features, positions, over: '' }; +} + +/** + * Read a parsed GPX document into GeoJSON: waypoints become points, each + * route a line and each track one MultiLineString with a line per `` + * (segment breaks are gaps in the recording and stay gaps). Only names are + * kept, and no times, so a dated track draws without a clock. Throws + * `GeoImportError` past the size limits. + * @param {Document} doc + * @returns {object} A FeatureCollection. + */ +export function gpxToGeoJson( + doc, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + } = {}, +) { + const root = doc.documentElement; + if (root?.localName !== 'gpx') + throw new GeoImportError('That GPX file has no document.'); + const features = []; + let positions = 0; + const point = (el) => { + const lat = Number(el.getAttribute('lat')); + const lon = Number(el.getAttribute('lon')); + if ( + !Number.isFinite(lat) || + !Number.isFinite(lon) || + Math.abs(lat) > 90 || + Math.abs(lon) > 180 + ) + return null; + positions += 1; + if (positions > maxPositions) + throw overLimitError({ + over: 'positions', + features: features.length, + positions, + }); + return [lon, lat]; + }; + const add = (name, geometry) => { + if (features.length >= maxFeatures) + throw overLimitError({ + over: 'features', + features: features.length + 1, + positions, + }); + features.push({ + type: 'Feature', + properties: name ? { name } : {}, + geometry, + }); + }; + for (const wpt of childrenNamed(root, 'wpt')) { + const at = point(wpt); + if (at) add(childText(wpt, 'name'), { type: 'Point', coordinates: at }); + } + for (const rte of childrenNamed(root, 'rte')) { + const line = []; + for (const pt of childrenNamed(rte, 'rtept')) { + const at = point(pt); + if (at) line.push(at); + } + if (line.length >= 2) + add(childText(rte, 'name'), { type: 'LineString', coordinates: line }); + } + for (const trk of childrenNamed(root, 'trk')) { + const lines = []; + for (const seg of childrenNamed(trk, 'trkseg')) { + const line = []; + for (const pt of childrenNamed(seg, 'trkpt')) { + const at = point(pt); + if (at) line.push(at); + } + if (line.length >= 2) lines.push(line); + } + if (lines.length) + add(childText(trk, 'name'), { + type: 'MultiLineString', + coordinates: lines, + }); + } + return { type: 'FeatureCollection', features }; +} + +/** A feature name as short plain text (no control characters), or undefined. */ +export function importFeatureName(value) { + const text = String(value ?? '') + .replace(/[\u0000-\u001f\u007f]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + if (!text) return undefined; + return text.length > MAX_IMPORT_NAME_LENGTH + ? `${text.slice(0, MAX_IMPORT_NAME_LENGTH - 1)}…` + : text; +} + +/** + * Make every entity of an imported data source inert: no description, + * balloon, extended data or properties, no native label, and a name that is + * short plain text. Returns how many entities were touched. + * @param {{entities: {values: object[]}}} dataSource + * @returns {number} + */ +export function scrubImportedEntities(dataSource) { + const list = dataSource?.entities?.values || []; + for (let i = 0; i < list.length; i++) { + const entity = list[i]; + let name; + try { + name = importFeatureName( + typeof entity.name === 'string' ? entity.name : undefined, + ); + } catch { + name = undefined; + } + entity.name = name; + entity.description = undefined; + entity.label = undefined; + entity.properties = undefined; + if (entity.kml && typeof entity.kml === 'object') entity.kml = undefined; + } + return list.length; +} + +const TRACK_SAMPLES = 2000; + +/** + * Take time out of an imported data source so adding it cannot move the app's + * shared clock: the data source reports no clock (the viewer adopts the clock + * of every added source that has one), every entity is available for all + * time, and a time-dynamic position (a KML gx:Track) is drawn as the line it + * travels instead of a moving marker. + * @param {object} dataSource + * @param {{Cesium?: object, color?: object}} [options] + * @returns {object} The same data source. + */ +export function detachImportedTime( + dataSource, + { Cesium: C = Cesium, color } = {}, +) { + Object.defineProperty(dataSource, 'clock', { + configurable: true, + enumerable: false, + get: () => undefined, + set: () => {}, + }); + const list = dataSource?.entities?.values || []; + for (let i = 0; i < list.length; i++) { + const entity = list[i]; + const span = entity.availability; + const position = entity.position; + if (position && position.isConstant === false) { + const start = span?.start; + const stop = span?.stop; + const line = []; + if (start && stop && C.JulianDate.lessThan(start, stop)) { + const seconds = C.JulianDate.secondsDifference(stop, start); + for (let n = 0; n <= TRACK_SAMPLES; n++) { + const t = C.JulianDate.addSeconds( + start, + (seconds * n) / TRACK_SAMPLES, + new C.JulianDate(), + ); + const at = position.getValue(t); + if (at) line.push(at); + } + } + entity.path = undefined; + entity.model = undefined; + if (line.length >= 2 && !entity.polyline) { + entity.billboard = undefined; + entity.point = undefined; + entity.position = undefined; + entity.polyline = new C.PolylineGraphics({ + positions: line, + clampToGround: true, + width: 3, + material: color || C.Color.WHITE, + }); + } else { + entity.position = line.length + ? new C.ConstantPositionProperty(line[0]) + : undefined; + } + } + entity.availability = undefined; + } + return dataSource; +} + function parseXml(text, what) { const doc = new DOMParser().parseFromString( String(text).replace(/^/, ''), @@ -364,22 +738,27 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { let dataSource; let removed = 0; try { - if (format === 'geojson') { - dataSource = await Cesium.GeoJsonDataSource.load( - parseGeoJsonText(new TextDecoder().decode(bytes)), - { - clampToGround: true, - stroke, - fill: stroke.withAlpha(0.25), - strokeWidth: 3, - markerColor: stroke, - }, - ); - } else if (format === 'gpx') { - dataSource = await Cesium.GpxDataSource.load( - parseXml(new TextDecoder().decode(bytes), 'GPX'), - { clampToGround: true, trackColor: stroke, routeColor: stroke }, - ); + if (format === 'geojson' || format === 'gpx') { + let geojson; + if (format === 'gpx') { + geojson = gpxToGeoJson( + parseXml(new TextDecoder().decode(bytes), 'GPX'), + ); + } else { + geojson = parseGeoJsonText(new TextDecoder().decode(bytes)); + const size = countGeoJsonGeometry(geojson); + if (size.over) throw overLimitError(size); + stripGeoJsonStyleProperties(geojson); + } + dataSource = await Cesium.GeoJsonDataSource.load(geojson, { + clampToGround: true, + stroke, + fill: stroke.withAlpha(0.25), + strokeWidth: 3, + markerColor: stroke, + // No HTML description table is built from the file's properties. + describe: () => undefined, + }); } else { let text; let resolveLocal = () => null; @@ -402,6 +781,8 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { text = new TextDecoder().decode(bytes); } const doc = parseXml(text, format.toUpperCase()); + const size = countKmlGeometry(doc); + if (size.over) throw overLimitError(size); removed = sanitizeKmlDocument(doc, resolveLocal).removed; dataSource = await Cesium.KmlDataSource.load(doc, { camera: viewer.scene.camera, @@ -427,6 +808,11 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { revoke(); throw new GeoImportError('That file has nothing to draw.'); } - // GpxDataSource's `name` is read-only, so the display name travels beside it. + scrubImportedEntities(dataSource); + // Only KML has time-dynamic positions (GPX is read without times); its + // track line is drawn in KML's default white, like its other unstyled lines. + detachImportedTime(dataSource); + // The display name travels beside the data source (the file name, not + // anything the document says about itself). return { dataSource, name, format, entityCount, removed, revoke }; } diff --git a/src/data/geoFileImport.test.mjs b/src/data/geoFileImport.test.mjs index 522af2550..c589438fc 100644 --- a/src/data/geoFileImport.test.mjs +++ b/src/data/geoFileImport.test.mjs @@ -1,8 +1,17 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { deflateRawSync } from 'node:zlib'; +import * as Cesium from 'cesium'; import { GeoImportError, + MAX_IMPORT_POSITIONS, + countGeoJsonGeometry, + countKmlGeometry, + detachImportedTime, + gpxToGeoJson, + importFeatureName, + scrubImportedEntities, + stripGeoJsonStyleProperties, detectGeoFileFormat, importDisplayName, kmzImageType, @@ -213,3 +222,283 @@ test('KMZ hrefs resolve only to entries inside the archive', () => { assert.equal(kmzImageType('files/a.PNG'), 'image/png'); assert.equal(kmzImageType('files/a.svg'), null); }); + +/* ── a tiny element tree with the DOM surface the readers use ─────────── */ +function xmlEl(localName, attrs = {}, kids = []) { + const el = { + nodeType: 1, + localName, + childNodes: [], + getAttribute: (n) => (n in attrs ? String(attrs[n]) : null), + }; + for (const k of typeof kids === 'string' ? [] : kids) el.childNodes.push(k); + const text = typeof kids === 'string' ? kids : null; + Object.defineProperty(el, 'textContent', { + get: () => text ?? el.childNodes.map((c) => c.textContent).join(''), + }); + return el; +} +const xmlDoc = (root) => ({ + documentElement: root, + getElementsByTagName(name) { + assert.equal(name, '*'); + const out = []; + const walk = (el) => { + out.push(el); + el.childNodes.forEach(walk); + }; + walk(root); + return out; + }, +}); +const trkpt = (lat, lon, time) => + xmlEl('trkpt', { lat, lon }, time ? [xmlEl('time', {}, time)] : []); + +test('GPX keeps each track segment as its own line, names as text, no times', () => { + const doc = xmlDoc( + xmlEl('gpx', {}, [ + xmlEl('wpt', { lat: 37.77, lon: -122.49 }, [xmlEl('name', {}, 'Camp')]), + xmlEl('wpt', { lat: 99, lon: 0 }), // out of range: skipped + xmlEl('rte', {}, [ + xmlEl('name', {}, 'Route'), + xmlEl('rtept', { lat: 1, lon: 2 }), + xmlEl('rtept', { lat: 3, lon: 4 }), + ]), + xmlEl('trk', {}, [ + xmlEl('name', {}, 'Morning walk'), + xmlEl('trkseg', {}, [ + trkpt(10, 20, '2024-05-01T10:00:00Z'), + trkpt(10.1, 20.1, '2024-05-01T10:01:00Z'), + ]), + // A pause in the recording: a new segment, not a straight jump. + xmlEl('trkseg', {}, [ + trkpt(11, 21, '2024-05-01T11:00:00Z'), + trkpt(11.1, 21.1, '2024-05-01T11:01:00Z'), + trkpt(11.2, 21.2, '2024-05-01T11:02:00Z'), + ]), + xmlEl('trkseg', {}, [trkpt(12, 22)]), // one point: not a line + ]), + ]), + ); + const fc = gpxToGeoJson(doc); + assert.deepEqual( + fc.features.map((f) => [f.properties.name, f.geometry.type]), + [ + ['Camp', 'Point'], + ['Route', 'LineString'], + ['Morning walk', 'MultiLineString'], + ], + ); + assert.deepEqual(fc.features[2].geometry.coordinates, [ + [ + [20, 10], + [20.1, 10.1], + ], + [ + [21, 11], + [21.1, 11.1], + [21.2, 11.2], + ], + ]); + assert.doesNotMatch(JSON.stringify(fc), /2024|time/); + assert.throws(() => gpxToGeoJson(xmlDoc(xmlEl('kml'))), GeoImportError); +}); + +test('GPX past the point limit is refused while reading, before Cesium', () => { + const seg = xmlEl( + 'trkseg', + {}, + Array.from({ length: 30 }, (_, i) => trkpt(1 + i / 100, 2)), + ); + const doc = xmlDoc(xmlEl('gpx', {}, [xmlEl('trk', {}, [seg])])); + assert.throws( + () => gpxToGeoJson(doc, { maxPositions: 20 }), + (e) => e instanceof GeoImportError && /more than .* points/.test(e.message), + ); + assert.throws( + () => + gpxToGeoJson( + xmlDoc( + xmlEl('gpx', {}, [ + xmlEl('wpt', { lat: 1, lon: 1 }), + xmlEl('wpt', { lat: 2, lon: 2 }), + ]), + ), + { maxFeatures: 1 }, + ), + /more than .* features/, + ); +}); + +test('GeoJSON features and vertices are counted before loading, without recursion', () => { + const line = Array.from({ length: 50 }, (_, i) => [i / 10, 1]); + const fc = { + type: 'FeatureCollection', + features: [ + { type: 'Feature', geometry: { type: 'LineString', coordinates: line } }, + { + type: 'Feature', + geometry: { + type: 'GeometryCollection', + geometries: [ + { type: 'Point', coordinates: [1, 2] }, + { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 0], + ], + ], + }, + ], + }, + }, + { type: 'Feature', geometry: null }, + ], + }; + assert.deepEqual(countGeoJsonGeometry(fc), { + features: 3, + positions: 55, + over: '', + }); + assert.equal( + countGeoJsonGeometry(fc, { maxPositions: 54 }).over, + 'positions', + ); + assert.equal(countGeoJsonGeometry(fc, { maxFeatures: 2 }).over, 'features'); + assert.deepEqual( + countGeoJsonGeometry({ type: 'Point', coordinates: [1, 2] }), + { + features: 1, + positions: 1, + over: '', + }, + ); + // A very long line is counted iteratively and stops at the limit. + const huge = { + type: 'LineString', + coordinates: Array.from({ length: MAX_IMPORT_POSITIONS + 5 }, () => [0, 0]), + }; + assert.equal(countGeoJsonGeometry(huge).over, 'positions'); +}); + +test('KML placemarks and coordinate tuples are counted before loading', () => { + const coords = (n) => + xmlEl( + 'coordinates', + {}, + Array.from({ length: n }, (_, i) => `${i},1,0`).join('\n '), + ); + const doc = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('Placemark', {}, [xmlEl('LineString', {}, [coords(40)])]), + xmlEl('Placemark', {}, [ + xmlEl('Track', {}, [ + xmlEl('coord', {}, '1 2 0'), + xmlEl('coord', {}, '1 3 0'), + ]), + ]), + ]), + ); + assert.deepEqual(countKmlGeometry(doc), { + features: 2, + positions: 42, + over: '', + }); + assert.equal(countKmlGeometry(doc, { maxPositions: 41 }).over, 'positions'); + assert.equal(countKmlGeometry(doc, { maxFeatures: 1 }).over, 'features'); +}); + +test('simplestyle is dropped: no icon fetched, the row color is what draws', () => { + const fc = { + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + properties: { + 'marker-symbol': 'rail', + 'marker-color': '#f00', + stroke: '#0f0', + 'stroke-width': 9, + fill: '#00f', + 'fill-opacity': 1, + name: 'x', + }, + geometry: null, + }, + { type: 'Feature', properties: null, geometry: null }, + ], + }; + stripGeoJsonStyleProperties(fc); + assert.deepEqual(fc.features[0].properties, { name: 'x' }); +}); + +test('imported entities keep only a short plain-text name: no description, label or properties', () => { + const ds = new Cesium.CustomDataSource('t'); + const e = ds.entities.add({ + name: 'Trail\u0007 head\n' + 'x'.repeat(300), + description: '', + label: { text: 'Trail head' }, + properties: { note: '' }, + position: Cesium.Cartesian3.fromDegrees(1, 2), + }); + e.kml = { extendedData: { note: { value: '