From 0da4d390eaf5b54cac0efeb269879d92e6a562bd Mon Sep 17 00:00:00 2001 From: daikaginza Date: Sat, 26 Sep 2026 17:36:59 +0700 Subject: [PATCH 01/95] docs: add the realtime backend to the voice ownership table The ownership table lists every owner except the backend, though realtimeBackend.js owns the two requests, their separate transports, the abort/timeout scope around both, and client-secret expiry. The method signatures are already documented with application construction, so link there rather than repeat them, and state only the ownership facts written nowhere: the connection is the backend's only consumer in the application, the resolved credential is opaque to it and is handed to negotiate whole, and client-secret expiry is refused inside the backend. --- docs/VOICE-OWNERSHIP.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/VOICE-OWNERSHIP.md b/docs/VOICE-OWNERSHIP.md index d5553df00..be7a6c26b 100644 --- a/docs/VOICE-OWNERSHIP.md +++ b/docs/VOICE-OWNERSHIP.md @@ -8,6 +8,7 @@ The session adapter, action runner and backend interfaces remain unchanged. | Owner | State and responsibility | | --- | --- | | `realtimeConnection.js` | Start generation, request cancellation, peer/data channel, acquired microphone stream and playback element | +| `realtimeBackend.js` | Token and SDP requests, their separate transports, the abort/timeout scope around both, and client-secret expiry | | `realtimeTurns.js` | Active and superseded responses, pending confirmations, deduplicated calls and action cancellation | | `realtimeRadio.js` | Speaker ducking, prepared playback, stronger-action reservations and playback observers | | `realtimeInput.js` | Physical Space gesture, microphone mute, input/output meters and their frame/listener lifetime | @@ -15,6 +16,15 @@ The session adapter, action runner and backend interfaces remain unchanged. | `realtimeViewport.js` | One retained image, bounded deletion identities and capture generation | | `realtimeDiagnostics.js` | Bounded error history and sanitized optional diagnostics | +The backend is replaceable per deployment: `realtimeController.js` accepts it as a +`backend` option and falls back to `createRealtimeBackend()`. Its two method +signatures live with [application construction](APPLICATION.md); what belongs here +is the ownership. The connection is the backend's only consumer in the application. +The resolved credential is opaque to the connection and is handed to `negotiate` +whole. Client-secret expiry is refused inside the backend, not by the connection. +`realtimeOwners.test.mjs` drives whole sessions through a substitute that +implements only those two methods. + Pure input policy, preferences and protocol-response policy have separate modules. Owners receive named readers, operations or focused collaborators. Cross-owner cancellation uses operations; response handling does not own Radio's reservations From 993b1bcdb1d7871a1f39b7cabc7c16387d93dd5d Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:29:19 -0500 Subject: [PATCH 02/95] docs: add Pinokio browser performance workaround --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 88690d52f..0ff6c32e1 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,14 @@ locked dependencies, finds a free local port, and opens the app. Version 8.2 fixes the launcher installation issue; [details from the Pinokio maintainer](https://pinokio.co/posts/01m1m4p9xxm3qw7dnnpj2wr93g). +**Pinokio opens but the globe is very choppy?** Before treating that as a GPU +compatibility or minimum-hardware problem, copy the local server URL shown by +Pinokio and open the same URL in your normal hardware-accelerated browser. A +Pinokio 8.2.0 Linux report reproduced severe stutter in Pinokio's embedded view +while the same running server was smooth in an external browser ([#365](https://github.com/bilawalsidhu/gods-eye-view/issues/365)). +That isolates the browser/compositor path from the app server and your machine; +it does not establish a minimum GPU requirement. + ### Path 2 — Terminal / coding agent Use **Node.js 24.x (24.14.0 or later) or 26.x**. The setup doctor warns about From 4ab530b8a83c6f56b34612c1ea5c9bc07c34e3ea Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:29:21 -0500 Subject: [PATCH 03/95] docs: distinguish embedded browser performance --- docs/PERFORMANCE.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 34b207f79..873ef8768 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -128,6 +128,12 @@ Use the same controls before attributing a difference to the application: 6. Record live object counts before attributing a difference to the client. 7. Treat a live-source outage as missing coverage, not as evidence of low client rendering cost. +8. Record whether the app is running in Pinokio's embedded view or in an + external browser. The same Pinokio 8.2.0 Linux server was reported as + severely stuttery in the embedded view and smooth when its local URL was + opened directly in a normal browser ([#365](https://github.com/bilawalsidhu/gods-eye-view/issues/365)). + Treat that as a browser/compositor-path difference until the GPU renderer + proves otherwise. ## What is not established yet From c87f007e34a627bc9554c4bfe8070267f9c5e0b5 Mon Sep 17 00:00:00 2001 From: daikaginza Date: Mon, 28 Sep 2026 11:39:19 +0700 Subject: [PATCH 04/95] fix(build): discover unit tests under server/, not only src/ npm test is the only suite CI runs, and its discovery walked src/ alone, so server/providers/transitHistory.test.mjs has never executed since it was added. Its 8 tests pass, so this was silent coverage loss rather than a hidden failure: the file reads like covered code in review and never ran anywhere. Add server/ to the searched roots, and pin the property with a test that walks the repository itself and compares the two sets, so a tree added later without updating the roots fails loudly instead of going quiet. Suite goes from 5090 to 5099 tests, 0 failures: the 8 tests in that file plus the regression test added here. --- scripts/run-unit-tests.mjs | 14 ++++++++-- src/unitTestRunner.test.mjs | 51 ++++++++++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/scripts/run-unit-tests.mjs b/scripts/run-unit-tests.mjs index 9bd7e70e0..7024ea38c 100644 --- a/scripts/run-unit-tests.mjs +++ b/scripts/run-unit-tests.mjs @@ -21,9 +21,19 @@ export function assertNode24AllocationRuntime(version = process.versions.node) { return version; } +/** + * Directories this runner searches for `*.test.mjs`, in stable order. + * + * `server/` belongs here because provider modules own tests of their own: a + * suite placed beside the module it covers must run, and `npm test` is the only + * thing CI invokes, so a directory missing from this list is a suite that never + * runs anywhere. `unitTestRunner.test.mjs` asserts the list covers every test + * file in the repository, so adding a tree without adding it here fails. + */ +export const UNIT_TEST_ROOTS = Object.freeze(['src', 'server']); + /** Discover repository unit tests in stable path order. */ export function discoverUnitTestFiles(root = process.cwd()) { - const sourceRoot = path.join(root, 'src'); const files = []; const visit = (directory) => { for (const entry of readdirSync(directory, { withFileTypes: true })) { @@ -34,7 +44,7 @@ export function discoverUnitTestFiles(root = process.cwd()) { } } }; - visit(sourceRoot); + for (const testRoot of UNIT_TEST_ROOTS) visit(path.join(root, testRoot)); return files.sort(); } diff --git a/src/unitTestRunner.test.mjs b/src/unitTestRunner.test.mjs index 002dcb796..201f6ed8e 100644 --- a/src/unitTestRunner.test.mjs +++ b/src/unitTestRunner.test.mjs @@ -1,14 +1,40 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { readFileSync, readdirSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { ALLOCATION_TEST_FILES, + UNIT_TEST_ROOTS, allocationTestArgs, assertNode24AllocationRuntime, buildUnitTestPlan, + discoverUnitTestFiles, isCalibratedAllocationRuntime, } from '../scripts/run-unit-tests.mjs'; +const repositoryRoot = fileURLToPath(new URL('..', import.meta.url)); + +/** Every `*.test.mjs` a contributor could add, found without consulting the runner. */ +function testFilesOnDisk(root) { + // Build outputs and dependencies are not places a contributor writes a test, + // and walking them would be slow enough to matter. Everything else counts. + const skip = new Set(['node_modules', 'dist', 'build', 'public']); + const found = []; + const visit = (directory) => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (entry.name.startsWith('.') || skip.has(entry.name)) continue; + const absolute = path.join(directory, entry.name); + if (entry.isDirectory()) visit(absolute); + else if (entry.isFile() && entry.name.endsWith('.test.mjs')) { + found.push(path.relative(root, absolute).split(path.sep).join('/')); + } + } + }; + visit(root); + return found.sort(); +} + test('unit runner serializes only GC-bracketed allocation microbenchmarks', () => { const ordinary = [ 'src/data/manager.test.mjs', @@ -37,6 +63,29 @@ test('unit runner serializes only GC-bracketed allocation microbenchmarks', () = ); }); +test('the runner discovers every test file in the repository', () => { + // `npm test` is the only suite CI runs, and it runs exactly what this + // function returns. A `*.test.mjs` outside the searched roots therefore never + // executes anywhere, while still reading — in review and to its author — like + // covered code. Compare the two sets rather than their sizes, so the failure + // names the files instead of only disagreeing about a count. + const discovered = discoverUnitTestFiles(repositoryRoot); + const onDisk = testFilesOnDisk(repositoryRoot); + const undiscovered = onDisk.filter((file) => !discovered.includes(file)); + assert.deepEqual( + undiscovered, + [], + `test files the runner never executes: ${undiscovered.join(', ')}`, + ); + assert.deepEqual(discovered, onDisk); + for (const file of discovered) { + assert.ok( + UNIT_TEST_ROOTS.some((root) => file.startsWith(`${root}/`)), + `${file} should sit under one of ${UNIT_TEST_ROOTS.join(', ')}`, + ); + } +}); + test('allocation runtime calibration is explicit and pinned to Node 24', () => { assert.equal(assertNode24AllocationRuntime('24.19.0'), '24.19.0'); assert.throws( From de4c8343ddd3ce3a164e0d44e63cf3c7e805f477 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 10:33:46 -0500 Subject: [PATCH 05/95] test: make Recent Imagery live gate panel-relative --- scripts/qa-recent-imagery.mjs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/qa-recent-imagery.mjs b/scripts/qa-recent-imagery.mjs index 1460ed3b5..f9601432f 100644 --- a/scripts/qa-recent-imagery.mjs +++ b/scripts/qa-recent-imagery.mjs @@ -308,15 +308,21 @@ const CONTROLS = { }; const measure = () => page.evaluate((controls) => { + const panel = document.getElementById('recent-imagery-panel'); const body = document.getElementById('recent-imagery-panel-body'); + const panelRect = panel?.getBoundingClientRect(); const out = { bodyScroll: body?.scrollTop ?? null, rects: {} }; for (const [name, selector] of Object.entries(controls)) { const rect = document.querySelector(selector)?.getBoundingClientRect(); - out.rects[name] = rect - ? [rect.left, rect.top, rect.width, rect.height].map( - (v) => Math.round(v * 10) / 10, - ) - : null; + out.rects[name] = + rect && panelRect + ? [ + rect.left - panelRect.left, + rect.top - panelRect.top, + rect.width, + rect.height, + ].map((v) => Math.round(v * 10) / 10) + : null; } return out; }, CONTROLS); @@ -1042,13 +1048,13 @@ try { ) <= target * 0.01, fitHeight, - 2_000, + 4_000, ); const zoomedHeight = await page.evaluate( () => window.__godsEyeView.viewer.camera.positionCartographic.height, ); check( - `ZOOM IN flies to ${Math.round(fitHeight / 1000)} km within 2 s`, + `ZOOM IN lands at ${Math.round(fitHeight / 1000)} km`, zoomed, `height=${Math.round(zoomedHeight)} target=${Math.round(fitHeight)}`, ); @@ -1258,7 +1264,7 @@ try { ); await wait(100); } - const tail = (key) => new Set(samples.slice(8).map((s) => s[key])); + const tail = (key) => new Set(samples.slice(-8).map((s) => s[key])); return { settled: tail('imagery').size === 1 && tail('cctv').size === 1, detail: `imagery=${samples.map((s) => s.imagery).join(',')} cctv=${samples.map((s) => s.cctv).join(',')}`, From 8d1f69a8731b6cab3fb95f1bf6d5cd05d89f7e85 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sat, 3 Oct 2026 10:34:28 -0500 Subject: [PATCH 06/95] ci: prove Recent Imagery gate repair --- .../recent-imagery-gate-repair-proof.yml | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .github/workflows/recent-imagery-gate-repair-proof.yml diff --git a/.github/workflows/recent-imagery-gate-repair-proof.yml b/.github/workflows/recent-imagery-gate-repair-proof.yml new file mode 100644 index 000000000..b613d9561 --- /dev/null +++ b/.github/workflows/recent-imagery-gate-repair-proof.yml @@ -0,0 +1,37 @@ +name: Recent Imagery gate repair proof + +on: + push: + branches: + - fix/recent-imagery-qa-relative-layout-20261003 + +permissions: + contents: read + +jobs: + browser-proof: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24.14.0 + cache: npm + - run: npm ci + - name: Start GEV + run: | + npm run dev -- --host 127.0.0.1 --port 4173 > /tmp/gev-dev.log 2>&1 & + for i in {1..90}; do + if curl -fsS http://127.0.0.1:4173/ >/dev/null; then + exit 0 + fi + sleep 1 + done + cat /tmp/gev-dev.log + exit 1 + - name: Recent Imagery live gate + run: node scripts/qa-recent-imagery.mjs + - name: Dev log on failure + if: failure() + run: cat /tmp/gev-dev.log From f93755af488585bd745f75c7baa43bebbed5dc3e Mon Sep 17 00:00:00 2001 From: Bruno Gomes Date: Tue, 6 Oct 2026 16:13:45 -0300 Subject: [PATCH 07/95] fix(cctv): start live HLS on plain-HTTP LAN pages The client lease id came from crypto.randomUUID, which browsers only expose in secure contexts. Opened over http://:4173 (the HOST=0.0.0.0 opt-in), starting a live camera feed threw before playback. Fall back to a v4 UUID from crypto.getRandomValues, which the server's lease check accepts. --- CHANGELOG.md | 6 ++++ docs/CURRENT-STATE.md | 4 ++- src/layers/cctv/videoPlayback.js | 28 +++++++++++++++++- src/layers/cctv/videoPlayback.test.mjs | 39 +++++++++++++++++++++++++- 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b433f8c06..9fc367d84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. +- Live HLS cameras play on a page opened over plain HTTP from another machine + (`HOST=0.0.0.0`). The client lease id came from `crypto.randomUUID`, which + browsers only expose in secure contexts, so starting a live feed threw + before playback. The id now falls back to a v4 UUID built from + `crypto.getRandomValues`, which the server's lease check accepts. + ## [0.2.1] — 2026-10-02 — God's Eye View through Your Agent (with security fixes and other features) - Ship each bundled data pack once. The region, marine, admin-boundary, diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d4fe022a8..fde1373af 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4814,7 +4814,9 @@ its paint loop. Camera deactivation destroys the decoder, and fatal playback errors revert to the labeled frame fallback. Camera changes clear the old panel frame. hls.js is imported only when an HLS camera starts. Two live sessions may coexist with at most eight independent consumer leases each; no-access lease -expiry is 15 seconds. Closing one consumer leaves other consumers running. Encrypted/fMP4/byte-range playlists fail +expiry is 15 seconds. Closing one consumer leaves other consumers running. Lease +ids use `crypto.randomUUID` where the page is a secure context and an equivalent +v4 id from `crypto.getRandomValues` on plain-HTTP LAN pages. Encrypted/fMP4/byte-range playlists fail closed. Performance and live agency reliability require recorded validation. ## Wind forecast models diff --git a/src/layers/cctv/videoPlayback.js b/src/layers/cctv/videoPlayback.js index d59143b5a..48988c17c 100644 --- a/src/layers/cctv/videoPlayback.js +++ b/src/layers/cctv/videoPlayback.js @@ -1,3 +1,28 @@ +/** + * A v4 UUID for the client lease. `crypto.randomUUID` only exists in secure + * contexts, so a page opened over plain HTTP from another machine (the + * HOST=0.0.0.0 LAN opt-in) has none; `getRandomValues` exists in every context + * and yields the same shape the server's lease check accepts. + */ +export function createLeaseId(cryptoImpl = globalThis.crypto) { + if (typeof cryptoImpl.randomUUID === 'function') { + return cryptoImpl.randomUUID(); + } + const bytes = cryptoImpl.getRandomValues(new Uint8Array(16)); + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, '0')); + return [ + hex.slice(0, 4), + hex.slice(4, 6), + hex.slice(6, 8), + hex.slice(8, 10), + hex.slice(10), + ] + .map((group) => group.join('')) + .join('-'); +} + /** One decoder per active camera; both surfaces consume this video element. */ export function attachCctvVideo( video, @@ -7,13 +32,14 @@ export function attachCctvVideo( loadHls = () => import('hls.js'), onFailure = () => {}, fetchImpl = globalThis.fetch, + cryptoImpl = globalThis.crypto, } = {}, ) { // Preserve replay for existing finite video feeds; live HLS must not loop. video.loop = feedType !== 'hls'; let disposed = false; let hls = null; - const leaseId = feedType === 'hls' ? globalThis.crypto.randomUUID() : null; + const leaseId = feedType === 'hls' ? createLeaseId(cryptoImpl) : null; const mediaUrl = feedType === 'hls' ? `${url}${url.includes('?') ? '&' : '?'}lease=${leaseId}` diff --git a/src/layers/cctv/videoPlayback.test.mjs b/src/layers/cctv/videoPlayback.test.mjs index 1f8c1c0bb..d1ef02539 100644 --- a/src/layers/cctv/videoPlayback.test.mjs +++ b/src/layers/cctv/videoPlayback.test.mjs @@ -1,6 +1,15 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { attachCctvVideo } from './videoPlayback.js'; +import { webcrypto } from 'node:crypto'; +import { attachCctvVideo, createLeaseId } from './videoPlayback.js'; + +const LEASE_V4 = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +/** What an insecure context exposes: getRandomValues but no randomUUID. */ +const insecureCrypto = { + getRandomValues: (array) => webcrypto.getRandomValues(array), +}; function video() { const v = new EventTarget(); Object.assign(v, { @@ -69,6 +78,34 @@ test('native HLS releases its client lease without response-header access', asyn assert.equal(releases[0].init.method, 'DELETE'); }); +test('lease ids are v4 UUIDs with or without crypto.randomUUID', () => { + assert.match(createLeaseId(), LEASE_V4); + const ids = new Set(); + for (let i = 0; i < 64; i++) ids.add(createLeaseId(insecureCrypto)); + assert.equal(ids.size, 64); + for (const id of ids) assert.match(id, LEASE_V4); +}); + +test('HLS starts on a plain-HTTP LAN page without crypto.randomUUID', async () => { + const source = video(); + source.canPlayType = () => 'probably'; + const playback = attachCctvVideo(source, '/api/cctv/media/a', 'hls', { + loadHls: async () => ({ default: { isSupported: () => false } }), + fetchImpl: async () => {}, + cryptoImpl: insecureCrypto, + }); + try { + await playback.ready; + const lease = new URL( + source.src, + 'http://192.168.1.10:4173/', + ).searchParams.get('lease'); + assert.match(lease, LEASE_V4); + } finally { + playback.dispose(); + } +}); + test('finite video feeds retain looping while live HLS does not loop', async () => { for (const feedType of ['mp4', 'webm', 'hls']) { const source = video(); From 57b8d8b23c249e05cc122e6b9f9cec9706757439 Mon Sep 17 00:00:00 2001 From: kapturner Date: Sun, 4 Oct 2026 08:26:18 -0500 Subject: [PATCH 08/95] feat: import GeoJSON, KML, KMZ and GPX files onto the globe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DISPLAY ▸ Import (and drop onto the globe) puts the person's own files on the map as Cesium data sources, clamped to the ground, one color per file, with a row to show/hide, fly to or remove each. Files stay in the browser for the session. KML/KMZ are sanitized before Cesium reads them (network links, remote icons/styles, models, photo/screen overlays and balloon HTML removed; KMZ images served from blob URLs), so an imported file cannot make the page fetch anything. KMZ is read with a small dependency-free zip reader on the platform DecompressionStream, with a zip-bomb limit. Bounded file size, entity count and number of files. Tests: src/data/geoFileImport.test.mjs, src/ui/geoFileImport.test.mjs. Gate: scripts/qa-geo-import.mjs. The import half of #890. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 + docs/CURRENT-STATE.md | 10 + index.html | 2 +- scripts/format-scope.json | 3 + scripts/package-boundaries.json | 2 + scripts/qa-geo-import.mjs | 402 +++++++++++++++++++++++ src/app/tools.js | 6 + src/data/geoFileImport.js | 432 +++++++++++++++++++++++++ src/data/geoFileImport.test.mjs | 215 ++++++++++++ src/ui/geoFileImport.js | 329 +++++++++++++++++++ src/ui/geoFileImport.test.mjs | 399 +++++++++++++++++++++++ src/ui/styles/controls.css | 64 ++++ src/ui/templates/display-controls.html | 9 + 13 files changed, 1879 insertions(+), 1 deletion(-) create mode 100644 scripts/qa-geo-import.mjs create mode 100644 src/data/geoFileImport.js create mode 100644 src/data/geoFileImport.test.mjs create mode 100644 src/ui/geoFileImport.js create mode 100644 src/ui/geoFileImport.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b3876f79..36100f297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,13 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. +- Import your own GeoJSON, KML, KMZ and GPX files (DISPLAY ▸ Import, or drop + them onto the globe). Each file is drawn on the ground in its own color with + a row to show, hide, fly to or remove it. Files stay in the browser for the + session, and KML/KMZ network links, remote icons, remote styles and balloon + HTML are left out so an imported file cannot make the page fetch anything. + The import half of #890. + ## [0.2.1] — 2026-10-02 — God's Eye View through Your Agent (with security fixes and other features) - Ship each bundled data pack once. The region, marine, admin-boundary, diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d51be986b..4a3b6a77b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4106,6 +4106,16 @@ are omitted rather than framing the wrong part of the globe. - Pure half: `src/annotations/drawMode.js` (session, minimum vertices, double-click de-dup at 0.5 m, vertex ceiling, degenerate-geometry refusal, ring closing, dateline-safe length/area/centroid, hint text). Regression surface: `src/annotations/drawMode.test.mjs` and `src/data/inputOwnership.test.mjs` (pure), `src/annotations/drawToolBehaviour.test.mjs` (the tool driven against DOM and viewer doubles — leases, borrowed input actions, teardown, the Enter key, preview heights), `src/annotations/drawTool.test.mjs` (structural: where files live, which ids the markup carries, what the docs may claim), and the rendered harness `scripts/qa-draw-tool.mjs`. - The annotation tree is a declared ownership group: `./annotations` → `src/annotations/index.js` in `package.json`, with its module graph listed in `scripts/package-boundaries.json`, so an annotation module reaching outside that graph is a gate failure rather than a surprise in a layer's bundle. `drawMode.js`, `drawTool.js` and their tests are in `scripts/format-scope.json`. +### Import Your Own Files (October 2026) + +- DISPLAY ▸ **Import** puts GeoJSON (`.geojson`/`.json`), KML, KMZ and GPX files on the globe. Files come in through the Import button (a hidden ``) or by dropping them onto the globe; the format is taken from the extension, then from the first bytes. Each file becomes its own Cesium data source (`GeoJsonDataSource`, `KmlDataSource`, `GpxDataSource`), clamped to the ground, in its own color from the whiteboard palette, with a row to show/hide it, fly to it, or remove it. The first file of a batch that loads is flown to. Imported files last for the page session: a reload starts empty, and nothing is uploaded or stored. +- Runtime entry: `createApplicationTools` (`src/app/tools.js`) calls `initGeoFileImport({ viewer })` (`src/ui/geoFileImport.js`) after the draw tool and defers its `destroy()`, which removes every imported data source, revokes every blob URL, and gives back the drop listeners on the viewer container and the `window.__gevGeoImport` handle (`importFiles`, `list`, `setVisible`, `flyTo`, `remove`, `diagnostics`). Parsing lives in `src/data/geoFileImport.js`. +- **An imported file cannot make the page fetch anything.** Before `KmlDataSource` reads a KML/KMZ document, `sanitizeKmlDocument` removes `NetworkLink`, `NetworkLinkControl`, `ScreenOverlay`, `PhotoOverlay`, `Model` and `Tour`, every `href` that is not an image packed in the same KMZ or a `data:image/` URL (an icon then falls back to Cesium's default pin, and a `GroundOverlay` left without an image is dropped), every `styleUrl` that is not a local `#id`, and placemark `description`/`BalloonStyle`/`Snippet` HTML. KMZ images are served from blob URLs. The hint says when something was left out. `scripts/qa-geo-import.mjs` imports files that name a trap host in each of those places and fails on any request to it. +- **KMZ** is read by a small zip reader (`readZipEntries`: stored and deflated entries through the platform `DecompressionStream`; no encryption, no ZIP64). The main document is `doc.kml`, else the shallowest `.kml`; relative hrefs resolve only to entries inside the archive. +- **Bounds.** A file may be at most 50 MB (`MAX_IMPORT_BYTES`), a KMZ may inflate to at most 120 MB in total (`MAX_KMZ_INFLATED_BYTES`, the zip-bomb stop), one file may create at most 20,000 entities (`MAX_IMPORT_ENTITIES`), and at most 12 files are on the globe at once (`MAX_IMPORTED_FILES`; files over that are not read). A refused file names its reason in the hint and does not stop the rest of the batch; a file with nothing to draw is refused. +- File names are rendered with `textContent` only. +- Gate: `node scripts/qa-geo-import.mjs` (dev server up). Unit: `src/data/geoFileImport.test.mjs`, `src/ui/geoFileImport.test.mjs`. + ### Recent Imagery (September 2026) - The layer sits in the Data Layers panel's Cameras group beside Cameras (CCTV); Mapped ALPR Cameras moved to the Infrastructure group to make room. diff --git a/index.html b/index.html index 2e630c4a8..78d6e34ec 100644 --- a/index.html +++ b/index.html @@ -13,7 +13,7 @@ with the sources: src/materialSymbolsSubset.test.mjs fails when a source names a glyph this list is missing, because a missing glyph does not draw a box — the ligature never forms and the element renders the literal word. --> - + diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 7062704a7..da80c0749 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -26,6 +26,8 @@ "src/annotations/drawMode.test.mjs", "src/annotations/drawTool.js", "src/annotations/drawTool.test.mjs", + "src/data/geoFileImport.test.mjs", + "src/ui/geoFileImport.test.mjs", "src/annotations/adminOutline.test.mjs", "src/annotations/regionRing.test.mjs", "src/data/adminBoundaries.test.mjs", @@ -195,6 +197,7 @@ "src/styles/snow.js", "scripts/qa-visual-effects.mjs", "scripts/qa-draw-tool.mjs", + "scripts/qa-geo-import.mjs", "src/ui/mapSource.js", "src/ui/mapSourceControls.js", "src/ui/mapSourceControls.test.mjs", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index ce778ee95..52062c1b3 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1958,6 +1958,8 @@ "src/data/inputOwnership.js", "src/annotations/drawMode.js", "src/annotations/drawTool.js", + "src/data/geoFileImport.js", + "src/ui/geoFileImport.js", "src/data/routeSteps.js", "src/layers/directions/index.js", "src/data/directions.js", diff --git a/scripts/qa-geo-import.mjs b/scripts/qa-geo-import.mjs new file mode 100644 index 000000000..c1c8f6462 --- /dev/null +++ b/scripts/qa-geo-import.mjs @@ -0,0 +1,402 @@ +#!/usr/bin/env node +/** + * Rendered acceptance for DISPLAY ▸ Import: real files through the real file + * input and a real drop, parsed by Cesium in the page. + * + * Proves: GeoJSON, KML, KMZ (with a packed icon and ground overlay) and GPX + * each become one row and one data source; the imported shape actually paints + * on the globe and stops painting when hidden; a KML/KMZ never makes the page + * request anything named in it (network link, remote icon, remote style); a + * dropped file imports like a picked one; a file that is not geodata is refused + * with a reason and leaves nothing behind; Remove gives every data source back; + * no page errors. + * + * Usage: dev server up (`npm run dev`), then `node scripts/qa-geo-import.mjs`. + * QA_BASE_URL (default http://localhost:4173), QA_SHOT_DIR, QA_HEADFUL=1. + */ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import puppeteer from 'puppeteer'; + +const BASE_URL = process.env.QA_BASE_URL || 'http://localhost:4173'; +const SHOT_DIR = process.env.QA_SHOT_DIR || 'qa-shots/geo-import'; +const HEADFUL = process.env.QA_HEADFUL === '1'; +// Golden Gate Park: a big, flat, green rectangle the test polygon can cover. +const VIEW = { lon: -122.4862, lat: 37.7694, height: 4200, pitch: -88 }; +const TRAP_HOST = 'gev-qa-trap.invalid'; + +/* ── fixtures ──────────────────────────────────────────────────────────── */ + +const ring = [ + [-122.511, 37.7645], + [-122.4545, 37.7645], + [-122.4545, 37.7745], + [-122.511, 37.7745], + [-122.511, 37.7645], +]; +const geojson = JSON.stringify({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + properties: { name: 'QA park' }, + geometry: { type: 'Polygon', coordinates: [ring] }, + }, + { + type: 'Feature', + properties: { name: 'QA pin' }, + geometry: { type: 'Point', coordinates: [-122.4862, 37.7694] }, + }, + ], +}); +const kml = (iconHref) => ` + + + + https://${TRAP_HOST}/link.kml + QA KML pin#s + ]]> + -122.47,37.77,0 + QA remote stylehttps://${TRAP_HOST}/styles.kml#x + -122.50,37.766 -122.46,37.772 + https://${TRAP_HOST}/overlay.png + 37.7737.76-122.46-122.47 + +`; +const gpx = ` + + QA waypoint + QA track + + +`; +// 1×1 PNG. +const PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==', + 'base64', +); +/** Stored (uncompressed) zip — enough for a KMZ fixture. */ +function storedZip(files) { + const parts = []; + const central = []; + let offset = 0; + for (const [name, data] of files) { + const n = Buffer.from(name); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt32LE(data.length, 18); + local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(n.length, 26); + const c = Buffer.alloc(46); + c.writeUInt32LE(0x02014b50, 0); + c.writeUInt32LE(data.length, 20); + c.writeUInt32LE(data.length, 24); + c.writeUInt16LE(n.length, 28); + c.writeUInt32LE(offset, 42); + parts.push(local, n, data); + central.push(c, n); + offset += 30 + n.length + data.length; + } + const dir = Buffer.concat(central); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(files.length, 8); + end.writeUInt16LE(files.length, 10); + end.writeUInt32LE(dir.length, 12); + end.writeUInt32LE(offset, 16); + return Buffer.concat([...parts, dir, end]); +} +const kmzDoc = kml('files/pin.png').replace( + `https://${TRAP_HOST}/overlay.png`, + 'files/overlay.png', +); + +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-qa-geo-import-')); +const fixture = (name, data) => { + const file = path.join(dir, name); + fs.writeFileSync(file, data); + return file; +}; +const FILES = [ + fixture('park.geojson', geojson), + fixture('remote-bits.kml', kml(`https://${TRAP_HOST}/icon.png`)), + fixture( + 'packed.kmz', + storedZip([ + ['doc.kml', Buffer.from(kmzDoc)], + ['files/pin.png', PNG], + ['files/overlay.png', PNG], + ]), + ), + fixture('walk.gpx', gpx), +]; + +/* ── harness ───────────────────────────────────────────────────────────── */ + +const browser = await puppeteer.launch({ + headless: !HEADFUL, + args: [ + '--no-sandbox', + '--disable-background-timer-throttling', + '--disable-renderer-backgrounding', + ...(process.platform === 'darwin' + ? ['--use-angle=metal', '--enable-gpu'] + : ['--use-gl=angle', '--use-angle=swiftshader']), + ], +}); +const page = await browser.newPage(); +const errors = []; +const trapped = []; +page.on('pageerror', (error) => errors.push(`pageerror: ${error.message}`)); +page.on('request', (request) => { + if (request.url().includes(TRAP_HOST)) trapped.push(request.url()); +}); +let failures = 0; +function check(name, passed, detail = '') { + console.log( + `[${passed ? 'PASS' : 'FAIL'}] ${name}${detail ? ` — ${detail}` : ''}`, + ); + if (!passed) failures++; +} +fs.mkdirSync(SHOT_DIR, { recursive: true }); +const shot = (name) => + page.screenshot({ + path: path.join(SHOT_DIR, `${name}.jpg`), + type: 'jpeg', + quality: 82, + }); +const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const CLIP = { x: 380, y: 240, width: 480, height: 320 }; + +/** Pixels in the clip close to the first import color (#39d0ff). */ +async function cyanPixels() { + await page.evaluate(() => window.__godsEyeView.viewer.scene.requestRender()); + await wait(600); + const base64 = await page.screenshot({ + clip: CLIP, + type: 'png', + encoding: 'base64', + }); + return page.evaluate(async (b64) => { + const bitmap = await createImageBitmap( + await (await fetch(`data:image/png;base64,${b64}`)).blob(), + ); + const canvas = new OffscreenCanvas(bitmap.width, bitmap.height); + const context = canvas.getContext('2d'); + context.drawImage(bitmap, 0, 0); + const { data } = context.getImageData(0, 0, bitmap.width, bitmap.height); + let n = 0; + for (let i = 0; i < data.length; i += 4) { + const [r, g, b] = [data[i], data[i + 1], data[i + 2]]; + if (b > 150 && g > 120 && r < 140 && b - r > 60) n += 1; + } + return n; + }, base64); +} +const state = () => + page.evaluate(() => ({ + files: window.__gevGeoImport.list(), + dataSources: window.__godsEyeView.viewer.dataSources.length, + rows: document.querySelectorAll('#geo-import-list .geo-import-row').length, + hint: document.getElementById('geo-import-hint')?.textContent || '', + })); + +try { + await page.setViewport({ width: 1280, height: 860 }); + await page.goto(`${BASE_URL}/?welcome=0`, { waitUntil: 'domcontentloaded' }); + await page.waitForFunction( + () => + window.__gevGeoImport && + window.__godsEyeView?.viewer && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 90_000 }, + ); + await page.evaluate((view) => { + const viewer = window.__godsEyeView.viewer; + viewer.camera.cancelFlight?.(); + viewer.camera.setView({ + destination: window.__CESIUM__.Cartesian3.fromDegrees( + view.lon, + view.lat, + view.height, + ), + orientation: { heading: 0, pitch: (view.pitch * Math.PI) / 180, roll: 0 }, + }); + viewer.scene.requestRender(); + }, VIEW); + await wait(4000); + await page.evaluate(() => { + const rail = document.getElementById('pp-toggles'); + if (rail?.classList.contains('collapsed')) + document + .querySelector('.panel-collapse-btn[data-collapse-target="pp-toggles"]') + ?.click(); + }); + await page.waitForFunction( + () => + document.getElementById('geo-import-button')?.getBoundingClientRect() + .width > 0, + { timeout: 10_000 }, + ); + const before = await state(); + check( + 'Import is on the DISPLAY rail with an empty, hidden list', + before.rows === 0 && + !(await page.evaluate(() => + document + .getElementById('geo-import-list') + .classList.contains('visible'), + )), + ); + const cyanBefore = await cyanPixels(); + + // Picked through the real . + const input = await page.$('#geo-import-input'); + await input.uploadFile(...FILES); + const imported = await page + .waitForFunction(() => window.__gevGeoImport.list().length === 4, { + timeout: 60_000, + }) + .then( + () => true, + () => false, + ); + const after = await state(); + check( + 'four picked files become four rows and four data sources', + imported && + after.rows === 4 && + after.dataSources === before.dataSources + 4, + JSON.stringify( + after.files.map((f) => `${f.name}:${f.format}:${f.entityCount}`), + ), + ); + check( + 'formats are recognized', + after.files.map((f) => f.format).join() === 'geojson,kml,kmz,gpx', + ); + check( + 'the KML and KMZ had their outside references removed', + after.files[1].removed > 0 && after.files[2].removed > 0, + `removed ${after.files[1].removed}/${after.files[2].removed}`, + ); + check( + 'the hint says so', + /Added 4 files\..*left out/.test(after.hint), + after.hint, + ); + await page.evaluate((view) => { + const viewer = window.__godsEyeView.viewer; + viewer.camera.cancelFlight?.(); + viewer.camera.setView({ + destination: window.__CESIUM__.Cartesian3.fromDegrees( + view.lon, + view.lat, + view.height, + ), + orientation: { heading: 0, pitch: (view.pitch * Math.PI) / 180, roll: 0 }, + }); + }, VIEW); + await wait(2500); + const cyanShown = await cyanPixels(); + await shot('imported'); + check( + 'the imported polygon paints on the globe', + cyanShown > cyanBefore * 4, + `${cyanBefore} → ${cyanShown}`, + ); + + const parkId = after.files[0].id; + await page.click( + `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`, + ); + const cyanHidden = await cyanPixels(); + check( + 'hiding it stops it painting', + cyanHidden < cyanShown / 3, + `${cyanShown} → ${cyanHidden}`, + ); + await page.click( + `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`, + ); + check( + 'showing it again paints it again', + (await cyanPixels()) > cyanHidden * 4, + ); + + // Dropped onto the globe, as a person drags from their file manager. + await page.evaluate((text) => { + const container = window.__godsEyeView.viewer.container; + const transfer = new DataTransfer(); + transfer.items.add( + new File([text], 'dropped.geojson', { type: 'application/geo+json' }), + ); + for (const type of ['dragenter', 'dragover', 'drop']) + container.dispatchEvent( + new DragEvent(type, { + dataTransfer: transfer, + bubbles: true, + cancelable: true, + }), + ); + }, geojson); + const dropped = await page + .waitForFunction(() => window.__gevGeoImport.list().length === 5, { + timeout: 20_000, + }) + .then( + () => true, + () => false, + ); + check('a file dropped on the globe imports', dropped); + + // Not geodata: refused with a reason, nothing added. + await page.evaluate(() => { + const transfer = new DataTransfer(); + transfer.items.add( + new File(['just some notes'], 'notes.txt', { type: 'text/plain' }), + ); + window.__godsEyeView.viewer.container.dispatchEvent( + new DragEvent('drop', { + dataTransfer: transfer, + bubbles: true, + cancelable: true, + }), + ); + }); + await wait(800); + const refused = await state(); + check( + 'a file that is not geodata is refused with a reason', + refused.files.length === 5 && + /notes\.txt: Only GeoJSON, KML, KMZ and GPX/.test(refused.hint), + refused.hint, + ); + + // Remove every file through its row button. + for (const { id } of refused.files) + await page.click( + `#geo-import-list [data-import-id="${id}"] button[data-action="remove"]`, + ); + const cleared = await state(); + check( + 'Remove gives every data source back', + cleared.rows === 0 && cleared.dataSources === before.dataSources, + `${cleared.dataSources} vs ${before.dataSources}`, + ); + await wait(1500); + check( + 'nothing named inside the KML/KMZ was ever requested', + trapped.length === 0, + trapped.join(' '), + ); + check('no page errors', errors.length === 0, errors.join(' | ')); +} catch (error) { + check('harness ran to completion', false, error?.stack || String(error)); +} finally { + await browser.close(); + fs.rmSync(dir, { recursive: true, force: true }); +} +console.log(failures ? `\n${failures} check(s) failed` : '\nall checks passed'); +process.exit(failures ? 1 : 0); diff --git a/src/app/tools.js b/src/app/tools.js index 25d763e2d..76f8e7fe6 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -1,6 +1,7 @@ import { SceneDirector } from '../scenes/director.js'; import { initAnnotations } from '../annotations/index.js'; import { initDrawTool } from '../annotations/drawTool.js'; +import { initGeoFileImport } from '../ui/geoFileImport.js'; import { initImageryBoxTool } from '../ui/imageryBoxTool.js'; import { createRecentImageryPanel } from '../ui/recentImagery.js'; import { initGevVoiceCommands } from '../voice/gevRealtime.js'; @@ -56,6 +57,11 @@ export function createApplicationTools({ // lifetime rather than to whoever last pressed the button. const drawTool = initDrawTool({ viewer, annotations }); defer(() => drawTool?.destroy()); + // DISPLAY ▸ Import: the person's own GeoJSON/KML/KMZ/GPX files, each its + // own data source. They live for the page session, so the application + // lifetime removes them and the drop listeners on the viewer container. + const geoFileImport = initGeoFileImport({ viewer }); + defer(() => geoFileImport?.destroy()); // DATA ▸ Recent Imagery: the box tool claims the pointer like Draw and the // panel lives on the right rail, so both belong to the application // lifetime. The tileset lets the layer drape while the globe is hidden. diff --git a/src/data/geoFileImport.js b/src/data/geoFileImport.js new file mode 100644 index 000000000..ee38b7fa1 --- /dev/null +++ b/src/data/geoFileImport.js @@ -0,0 +1,432 @@ +/** + * Your own geodata on the globe: GeoJSON, KML, KMZ and GPX files read in the + * browser and handed to Cesium's own data sources. + * + * Nothing here talks to the network. The file never leaves the page, and a + * KML/KMZ file cannot make the page fetch anything either: network links, + * remote icons, remote styles, models and photo/screen overlays are removed + * before Cesium sees the document (`sanitizeKmlDocument`). Images packed + * inside a KMZ are served to Cesium from blob URLs the caller revokes. + * + * The bounded pieces — format detection, the KMZ (zip) reader, the size and + * count limits — are plain functions so they run under `node --test`; the + * Cesium loaders need a browser DOM and are exercised by + * `scripts/qa-geo-import.mjs`. + */ +import * as Cesium from 'cesium'; + +/** Largest file accepted, in bytes. */ +export const MAX_IMPORT_BYTES = 50 * 1024 * 1024; +/** Largest total a KMZ may inflate to — a zip bomb stops here. */ +export const MAX_KMZ_INFLATED_BYTES = 120 * 1024 * 1024; +/** Most files on the globe at once. */ +export const MAX_IMPORTED_FILES = 12; +/** Most entities one file may create before it is refused. */ +export const MAX_IMPORT_ENTITIES = 20_000; +/** What the file picker offers. */ +export const GEO_FILE_ACCEPT = '.geojson,.json,.kml,.kmz,.gpx'; + +const GEOJSON_TYPES = new Set([ + 'FeatureCollection', + 'Feature', + 'Point', + 'MultiPoint', + 'LineString', + 'MultiLineString', + 'Polygon', + 'MultiPolygon', + 'GeometryCollection', +]); +const KMZ_IMAGE_TYPES = Object.freeze({ + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + bmp: 'image/bmp', +}); +// KML elements that fetch, embed or execute something outside the file. +const KML_REMOVED_ELEMENTS = Object.freeze([ + 'NetworkLink', + 'NetworkLinkControl', + 'ScreenOverlay', + 'PhotoOverlay', + 'Model', + 'Tour', +]); + +/** An error whose message is meant for the person who chose the file. */ +export class GeoImportError extends Error { + constructor(message) { + super(message); + this.name = 'GeoImportError'; + } +} + +/** + * The format of a file, from its name first and its first bytes second. + * @param {string} name File name. + * @param {Uint8Array} [head] The first bytes of the file. + * @returns {'geojson'|'kml'|'kmz'|'gpx'|null} + */ +export function detectGeoFileFormat(name, head) { + const ext = /\.([a-z0-9]+)$/i.exec(String(name || ''))?.[1]?.toLowerCase(); + if (ext === 'geojson') return 'geojson'; + if (ext === 'kml' || ext === 'kmz' || ext === 'gpx') return ext; + if (!head?.length) return ext === 'json' ? 'geojson' : null; + if ( + head[0] === 0x50 && + head[1] === 0x4b && + head[2] === 0x03 && + head[3] === 0x04 + ) + return 'kmz'; + const text = new TextDecoder() + .decode(head.subarray(0, 1024)) + .replace(/^/, '') + .trimStart(); + if (text.startsWith('{')) return 'geojson'; + if (text.startsWith('<')) { + if (/]/i.test(text)) return 'kml'; + if (/]/i.test(text)) return 'gpx'; + } + return ext === 'json' ? 'geojson' : null; +} + +/** + * A short, printable name for the list: the file name without control + * characters, cut to 60 characters. + * @param {string} name + * @returns {string} + */ +export function importDisplayName(name) { + const clean = String(name || '') + .replace(/[\u0000-\u001f\u007f]/g, '') + .trim(); + if (!clean) return 'Imported file'; + return clean.length > 60 ? `${clean.slice(0, 57)}…` : clean; +} + +/** + * Parse and check GeoJSON text. Throws `GeoImportError` for anything that is + * not a GeoJSON object. + * @param {string} text + * @returns {object} + */ +export function parseGeoJsonText(text) { + let value; + try { + value = JSON.parse(String(text).replace(/^/, '')); + } catch { + throw new GeoImportError('That file is not valid JSON.'); + } + if (!value || typeof value !== 'object' || !GEOJSON_TYPES.has(value.type)) + throw new GeoImportError('That JSON file is not GeoJSON.'); + if (value.type === 'FeatureCollection' && !Array.isArray(value.features)) + throw new GeoImportError('That FeatureCollection has no features list.'); + return value; +} + +/** Inflate raw-deflate bytes with the platform's stream, stopping at `maxBytes`. */ +async function inflateRaw(bytes, maxBytes) { + const stream = new Blob([bytes]) + .stream() + .pipeThrough(new DecompressionStream('deflate-raw')); + const reader = stream.getReader(); + const chunks = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.length; + if (total > maxBytes) + throw new GeoImportError( + 'That KMZ unpacks to more than the import limit.', + ); + chunks.push(value); + } + } catch (error) { + await reader.cancel().catch(() => {}); + if (error instanceof GeoImportError) throw error; + throw new GeoImportError('That KMZ is damaged and could not be unpacked.'); + } + const out = new Uint8Array(total); + let at = 0; + for (const chunk of chunks) { + out.set(chunk, at); + at += chunk.length; + } + return out; +} + +/** + * Read the entries of a zip archive (stored or deflated; no encryption, no + * ZIP64). Directory entries are skipped. + * @param {Uint8Array} bytes The whole archive. + * @param {{maxInflatedBytes?: number}} [options] + * @returns {Promise>} Entry path → contents. + */ +export async function readZipEntries( + bytes, + { maxInflatedBytes = MAX_KMZ_INFLATED_BYTES } = {}, +) { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const fail = (message = 'That KMZ is not a readable zip archive.') => { + throw new GeoImportError(message); + }; + // End of central directory: 22 bytes plus a comment of up to 64 KiB. + let eocd = -1; + for ( + let i = bytes.length - 22; + i >= Math.max(0, bytes.length - 22 - 0xffff); + i-- + ) { + if (view.getUint32(i, true) === 0x06054b50) { + eocd = i; + break; + } + } + if (eocd < 0) fail(); + const count = view.getUint16(eocd + 10, true); + const dirSize = view.getUint32(eocd + 12, true); + let at = view.getUint32(eocd + 16, true); + if (count === 0xffff || at === 0xffffffff || at + dirSize > eocd) + fail('That KMZ uses a zip format this import does not read.'); + const entries = new Map(); + const names = new TextDecoder(); + let inflated = 0; + for (let n = 0; n < count; n++) { + if (at + 46 > bytes.length || view.getUint32(at, true) !== 0x02014b50) + fail(); + const flags = view.getUint16(at + 8, true); + const method = view.getUint16(at + 10, true); + const compressed = view.getUint32(at + 20, true); + const size = view.getUint32(at + 24, true); + const nameLength = view.getUint16(at + 28, true); + const extraLength = view.getUint16(at + 30, true); + const commentLength = view.getUint16(at + 32, true); + const local = view.getUint32(at + 42, true); + const name = names.decode(bytes.subarray(at + 46, at + 46 + nameLength)); + at += 46 + nameLength + extraLength + commentLength; + if (name.endsWith('/')) continue; + if (flags & 0x1) fail('That KMZ is encrypted.'); + if (compressed === 0xffffffff || size === 0xffffffff) + fail('That KMZ uses a zip format this import does not read.'); + if (local + 30 > bytes.length || view.getUint32(local, true) !== 0x04034b50) + fail(); + const start = + local + + 30 + + view.getUint16(local + 26, true) + + view.getUint16(local + 28, true); + if (start + compressed > bytes.length) fail(); + const data = bytes.subarray(start, start + compressed); + let contents; + if (method === 0) contents = data; + else if (method === 8) + contents = await inflateRaw(data, maxInflatedBytes - inflated); + else fail('That KMZ uses a compression method this import does not read.'); + inflated += contents.length; + if (inflated > maxInflatedBytes) + fail('That KMZ unpacks to more than the import limit.'); + entries.set(name, contents); + } + return entries; +} + +/** + * The KML document inside a KMZ: `doc.kml` when present, else the first + * `.kml` entry nearest the archive root. + * @param {Map} entries + * @returns {string|null} Entry path. + */ +export function kmzMainDocument(entries) { + if (entries.has('doc.kml')) return 'doc.kml'; + const kml = [...entries.keys()] + .filter((name) => /\.kml$/i.test(name)) + .sort( + (a, b) => a.split('/').length - b.split('/').length || a.localeCompare(b), + ); + return kml[0] ?? null; +} + +/** + * Resolve a KMZ-relative href to an entry path, or null. + * @param {string} href As written in the KML. + * @param {string} base The KML entry's path. + * @param {Map} entries + * @returns {string|null} + */ +export function resolveKmzPath(href, base, entries) { + const raw = String(href || '').trim(); + if (!raw || /^[a-z][a-z0-9+.-]*:/i.test(raw) || raw.startsWith('//')) + return null; + let decoded; + try { + decoded = decodeURIComponent(raw.split(/[?#]/)[0]); + } catch { + return null; + } + const parts = base.split('/').slice(0, -1); + for (const part of decoded.replace(/\\/g, '/').split('/')) { + if (!part || part === '.') continue; + if (part === '..') parts.pop(); + else parts.push(part); + } + const path = parts.join('/'); + return entries.has(path) ? path : null; +} + +/** MIME type for a KMZ image entry, or null when it is not an image we serve. */ +export function kmzImageType(path) { + const ext = /\.([a-z0-9]+)$/i.exec(path)?.[1]?.toLowerCase(); + return KMZ_IMAGE_TYPES[ext] ?? null; +} + +const elementsNamed = (doc, localName) => + [...doc.getElementsByTagName('*')].filter((el) => el.localName === localName); + +/** + * Strip everything from a parsed KML document that would make the page reach + * outside the file. `resolveLocal(href)` returns a URL for an image packed in + * the same KMZ (or null); every other href is removed, so icons fall back to + * Cesium's default pin and a ground overlay without a local image is dropped. + * @param {Document} doc + * @param {(href: string) => string|null} [resolveLocal] + * @returns {{removed: number}} How many elements were taken out. + */ +export function sanitizeKmlDocument(doc, resolveLocal = () => null) { + let removed = 0; + const drop = (el) => { + if (el.parentNode) { + el.parentNode.removeChild(el); + removed += 1; + } + }; + for (const name of KML_REMOVED_ELEMENTS) + elementsNamed(doc, name).forEach(drop); + for (const href of elementsNamed(doc, 'href')) { + const value = (href.textContent || '').trim(); + const local = resolveLocal(value); + if (local) href.textContent = local; + else if (/^data:image\/(png|jpeg|gif|webp|bmp);/i.test(value)) continue; + else drop(href); + } + for (const overlay of elementsNamed(doc, 'GroundOverlay')) { + if (!elementsNamed(overlay, 'href').length) drop(overlay); + } + for (const styleUrl of elementsNamed(doc, 'styleUrl')) { + if (!(styleUrl.textContent || '').trim().startsWith('#')) drop(styleUrl); + } + // Placemark balloons are HTML; the app shows no info box, and keeping their + // /]]> -122.47,37.77,0 + QA palette pin#palette + -122.48,37.771,0 + QA moving + 2019-01-01T00:00:00Z2019-01-01T01:00:00Z + -122.49 37.767 0-122.475 37.772 0 QA remote stylehttps://${TRAP_HOST}/styles.kml#x -122.50,37.766 -122.46,37.772 https://${TRAP_HOST}/overlay.png @@ -67,8 +87,10 @@ const kml = (iconHref) => ` const gpx = ` QA waypoint - QA track - + QA track<img src="https://${TRAP_HOST}/gpx.png"> + + + `; // 1×1 PNG. @@ -149,8 +171,12 @@ const page = await browser.newPage(); const errors = []; const trapped = []; page.on('pageerror', (error) => errors.push(`pageerror: ${error.message}`)); +// Anything a file names, and the icon assets Cesium would load on a file's +// behalf (KML palette icons from maps.google.com, GeoJSON maki markers). +const FILE_FETCH = [TRAP_HOST, 'maps.google.com/mapfiles', '/maki/']; page.on('request', (request) => { - if (request.url().includes(TRAP_HOST)) trapped.push(request.url()); + if (FILE_FETCH.some((needle) => request.url().includes(needle))) + trapped.push(request.url()); }); let failures = 0; function check(name, passed, detail = '') { @@ -250,6 +276,23 @@ try { )), ); const cyanBefore = await cyanPixels(); + const clock = () => + page.evaluate(() => { + const c = window.__godsEyeView.viewer.clock; + const iso = (t) => window.__CESIUM__.JulianDate.toIso8601(t, 0); + return { + start: iso(c.startTime), + stop: iso(c.stopTime), + range: c.clockRange, + step: c.clockStep, + multiplier: c.multiplier, + animate: c.shouldAnimate, + // Wall-clock drift aside, the current time must not jump to 2018/2019. + year: Number(iso(c.currentTime).slice(0, 4)), + }; + }); + const clockBefore = await clock(); + const sourcesBefore = before.dataSources; // Picked through the real . const input = await page.$('#geo-import-input'); @@ -286,6 +329,65 @@ try { /Added 4 files\..*left out/.test(after.hint), after.hint, ); + const clockAfter = await clock(); + check( + "dated KML and GPX leave the app's clock alone", + JSON.stringify(clockAfter) === JSON.stringify(clockBefore) && + (await page.evaluate( + () => + window.__godsEyeView.viewer.clockTrackedDataSource?.clock === + undefined, + )), + `${JSON.stringify(clockBefore)} → ${JSON.stringify(clockAfter)}`, + ); + const inside = await page.evaluate((from) => { + const viewer = window.__godsEyeView.viewer; + const now = viewer.clock.currentTime; + const sources = []; + for (let i = from; i < viewer.dataSources.length; i++) + sources.push(viewer.dataSources.get(i)); + const entities = sources.flatMap((ds) => ds.entities.values); + const named = (name) => entities.find((e) => e.name === name); + const gpx = sources[3].entities.values; + return { + sources: sources.length, + clocks: sources.filter((ds) => ds.clock !== undefined).length, + described: entities.filter((e) => e.description !== undefined).length, + labelled: entities.filter((e) => e.label !== undefined).length, + withProperties: entities.filter((e) => e.properties !== undefined).length, + withKml: entities.filter((e) => e.kml !== undefined).length, + hidden: entities.filter((e) => !e.isAvailable(now)).length, + datedPinNow: Boolean(named('QA KML pin')?.isAvailable(now)), + movingIsLine: Boolean( + named('QA moving')?.polyline && !named('QA moving')?.position, + ), + gpxLines: gpx.filter((e) => e.polyline).length, + gpxPoints: gpx.filter((e) => e.billboard || e.point).length, + }; + }, sourcesBefore); + check( + 'no imported entity keeps a description, label, properties or KML data', + inside.sources === 4 && + inside.described + + inside.labelled + + inside.withProperties + + inside.withKml === + 0, + JSON.stringify(inside), + ); + check( + 'no imported source has a clock, and dated features draw now', + inside.clocks === 0 && + inside.hidden === 0 && + inside.datedPinNow && + inside.movingIsLine, + JSON.stringify(inside), + ); + check( + 'the GPX track keeps its two segments apart', + inside.gpxLines === 2 && inside.gpxPoints === 1, + JSON.stringify(inside), + ); await page.evaluate((view) => { const viewer = window.__godsEyeView.viewer; viewer.camera.cancelFlight?.(); @@ -325,6 +427,26 @@ try { (await cyanPixels()) > cyanHidden * 4, ); + // Keyboard: focus stays on the same control when the list is rebuilt. + const toggleSelector = `#geo-import-list [data-import-id="${parkId}"] button[data-action="toggle"]`; + await page.focus(toggleSelector); + await page.keyboard.press('Enter'); + const focusAfterHide = await page.evaluate( + (sel) => document.activeElement === document.querySelector(sel), + toggleSelector, + ); + await page.keyboard.press('Enter'); + const focusAfterShow = await page.evaluate( + (sel) => + document.activeElement === document.querySelector(sel) && + document.activeElement.getAttribute('aria-pressed') === 'true', + toggleSelector, + ); + check( + 'keyboard focus stays on the toggle through hide and show', + focusAfterHide && focusAfterShow, + ); + // Dropped onto the globe, as a person drags from their file manager. await page.evaluate((text) => { const container = window.__godsEyeView.viewer.container; @@ -387,7 +509,7 @@ try { ); await wait(1500); check( - 'nothing named inside the KML/KMZ was ever requested', + 'nothing named inside a file, and no icon asset for one, was ever requested', trapped.length === 0, trapped.join(' '), ); diff --git a/src/app/tools.js b/src/app/tools.js index 76f8e7fe6..7acc4939e 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -60,7 +60,10 @@ export function createApplicationTools({ // DISPLAY ▸ Import: the person's own GeoJSON/KML/KMZ/GPX files, each its // own data source. They live for the page session, so the application // lifetime removes them and the drop listeners on the viewer container. - const geoFileImport = initGeoFileImport({ viewer }); + const geoFileImport = initGeoFileImport({ + viewer, + navigation: styleManager, + }); defer(() => geoFileImport?.destroy()); // DATA ▸ Recent Imagery: the box tool claims the pointer like Draw and the // panel lives on the right rail, so both belong to the application diff --git a/src/data/geoFileImport.js b/src/data/geoFileImport.js index ee38b7fa1..53d954957 100644 --- a/src/data/geoFileImport.js +++ b/src/data/geoFileImport.js @@ -8,6 +8,23 @@ * before Cesium sees the document (`sanitizeKmlDocument`). Images packed * inside a KMZ are served to Cesium from blob URLs the caller revokes. * + * What the file says about itself stays inert: descriptions, balloons, + * extended data and GeoJSON properties are dropped from every entity, names + * are kept only as short plain text, and no native Cesium label is drawn + * (`scrubImportedEntities`). GeoJSON and GPX draw in the row's color: + * simplestyle properties are dropped, including `marker-symbol`, so no icon + * asset is loaded on the file's behalf. KML and KMZ keep their own styles. + * + * An import never moves the app's shared clock: dated KML (TimeSpan, + * TimeStamp, gx:Track) is drawn for all time, and the data source exposes no + * clock for the viewer to adopt (`detachImportedTime`). GPX is read here + * rather than by Cesium (`gpxToGeoJson`), keeping each track segment as its + * own line and leaving times out. + * + * Size is checked before Cesium builds anything (`countGeoJsonGeometry`, + * `countKmlGeometry`): a file with too many features or vertices is refused + * up front instead of after the entities exist. + * * The bounded pieces — format detection, the KMZ (zip) reader, the size and * count limits — are plain functions so they run under `node --test`; the * Cesium loaders need a browser DOM and are exercised by @@ -23,6 +40,10 @@ export const MAX_KMZ_INFLATED_BYTES = 120 * 1024 * 1024; export const MAX_IMPORTED_FILES = 12; /** Most entities one file may create before it is refused. */ export const MAX_IMPORT_ENTITIES = 20_000; +/** Most vertices (points of every line, ring and marker) in one file. */ +export const MAX_IMPORT_POSITIONS = 250_000; +/** Longest plain-text name kept from a file's features. */ +export const MAX_IMPORT_NAME_LENGTH = 120; /** What the file picker offers. */ export const GEO_FILE_ACCEPT = '.geojson,.json,.kml,.kmz,.gpx'; @@ -127,6 +148,112 @@ export function parseGeoJsonText(text) { return value; } +/** + * Count features and vertices in a parsed GeoJSON object without recursion + * limits or array spreading, stopping as soon as a limit is passed. + * @param {object} geojson + * @param {{maxFeatures?: number, maxPositions?: number}} [limits] + * @returns {{features: number, positions: number, over: ''|'features'|'positions'}} + */ +export function countGeoJsonGeometry( + geojson, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + } = {}, +) { + let features = 0; + let positions = 0; + const stack = [geojson]; + while (stack.length) { + const node = stack.pop(); + if (!node || typeof node !== 'object') continue; + if (node.type === 'FeatureCollection') { + const list = Array.isArray(node.features) ? node.features : []; + for (let i = 0; i < list.length; i++) stack.push(list[i]); + continue; + } + if (node.type === 'Feature') { + features += 1; + if (features > maxFeatures) + return { features, positions, over: 'features' }; + stack.push(node.geometry); + continue; + } + if (node.type === 'GeometryCollection') { + const list = Array.isArray(node.geometries) ? node.geometries : []; + for (let i = 0; i < list.length; i++) stack.push(list[i]); + continue; + } + if (GEOJSON_TYPES.has(node.type)) { + // A bare geometry is one feature: Cesium makes one entity for it. + if (node === geojson) features += 1; + // Walk the coordinate arrays iteratively: a position is an array whose + // first item is a number. + const coords = [node.coordinates]; + while (coords.length) { + const c = coords.pop(); + if (!Array.isArray(c)) continue; + if (typeof c[0] === 'number') { + positions += 1; + if (positions > maxPositions) + return { features, positions, over: 'positions' }; + } else for (let i = 0; i < c.length; i++) coords.push(c[i]); + } + } + } + return { features, positions, over: '' }; +} + +/** The refusal for a file over a size limit, in the person's terms. */ +function overLimitError({ over, features, positions }) { + if (over === 'features') + return new GeoImportError( + `That file has more than ${MAX_IMPORT_ENTITIES.toLocaleString()} features (the limit).`, + ); + return new GeoImportError( + `That file has more than ${MAX_IMPORT_POSITIONS.toLocaleString()} points (the limit)${features ? ` across ${features.toLocaleString()} features` : ''}.`, + ); +} + +// simplestyle-spec keys Cesium honors. `marker-symbol` would load a maki icon +// asset on the file's behalf; the colors would override the row's color. +const GEOJSON_STYLE_KEYS = Object.freeze([ + 'marker-symbol', + 'marker-color', + 'marker-size', + 'stroke', + 'stroke-opacity', + 'stroke-width', + 'fill', + 'fill-opacity', +]); + +/** + * Drop GeoJSON simplestyle properties, so a GeoJSON file draws in its row's + * color like a GPX file and no icon is fetched for it. Mutates and returns + * the object. + * @param {object} geojson + * @returns {object} + */ +export function stripGeoJsonStyleProperties(geojson) { + const stack = [geojson]; + while (stack.length) { + const node = stack.pop(); + if (!node || typeof node !== 'object') continue; + if (node.type === 'FeatureCollection' && Array.isArray(node.features)) + for (let i = 0; i < node.features.length; i++) + stack.push(node.features[i]); + else if ( + node.type === 'Feature' && + node.properties && + typeof node.properties === 'object' + ) + for (const key of GEOJSON_STYLE_KEYS) delete node.properties[key]; + } + return geojson; +} + /** Inflate raw-deflate bytes with the platform's stream, stopping at `maxBytes`. */ async function inflateRaw(bytes, maxBytes) { const stream = new Blob([bytes]) @@ -326,6 +453,253 @@ export function sanitizeKmlDocument(doc, resolveLocal = () => null) { return { removed }; } +/** Child elements of `el` with this local name (any namespace). */ +const childrenNamed = (el, localName) => { + const out = []; + const kids = el?.childNodes || []; + for (let i = 0; i < kids.length; i++) + if (kids[i].nodeType === 1 && kids[i].localName === localName) + out.push(kids[i]); + return out; +}; +const childText = (el, localName) => + (childrenNamed(el, localName)[0]?.textContent || '').trim(); + +/** + * Count placemarks and vertices in a parsed KML document before Cesium reads + * it: every `` tuple and every `gx:coord`. + * @param {Document} doc + * @returns {{features: number, positions: number, over: ''|'features'|'positions'}} + */ +export function countKmlGeometry( + doc, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + } = {}, +) { + let features = 0; + let positions = 0; + for (const el of doc.getElementsByTagName('*')) { + const name = el.localName; + if (name === 'Placemark') { + features += 1; + if (features > maxFeatures) + return { features, positions, over: 'features' }; + } else if (name === 'coordinates') { + const text = el.textContent || ''; + // Tuples are separated by whitespace; count runs without splitting + // a large string into a large array. + let inTuple = false; + for (let i = 0; i < text.length; i++) { + const space = text.charCodeAt(i) <= 32; + if (!space && !inTuple) { + positions += 1; + if (positions > maxPositions) + return { features, positions, over: 'positions' }; + } + inTuple = !space; + } + } else if (name === 'coord') { + positions += 1; + if (positions > maxPositions) + return { features, positions, over: 'positions' }; + } + } + return { features, positions, over: '' }; +} + +/** + * Read a parsed GPX document into GeoJSON: waypoints become points, each + * route a line and each track one MultiLineString with a line per `` + * (segment breaks are gaps in the recording and stay gaps). Only names are + * kept, and no times, so a dated track draws without a clock. Throws + * `GeoImportError` past the size limits. + * @param {Document} doc + * @returns {object} A FeatureCollection. + */ +export function gpxToGeoJson( + doc, + { + maxFeatures = MAX_IMPORT_ENTITIES, + maxPositions = MAX_IMPORT_POSITIONS, + } = {}, +) { + const root = doc.documentElement; + if (root?.localName !== 'gpx') + throw new GeoImportError('That GPX file has no document.'); + const features = []; + let positions = 0; + const point = (el) => { + const lat = Number(el.getAttribute('lat')); + const lon = Number(el.getAttribute('lon')); + if ( + !Number.isFinite(lat) || + !Number.isFinite(lon) || + Math.abs(lat) > 90 || + Math.abs(lon) > 180 + ) + return null; + positions += 1; + if (positions > maxPositions) + throw overLimitError({ + over: 'positions', + features: features.length, + positions, + }); + return [lon, lat]; + }; + const add = (name, geometry) => { + if (features.length >= maxFeatures) + throw overLimitError({ + over: 'features', + features: features.length + 1, + positions, + }); + features.push({ + type: 'Feature', + properties: name ? { name } : {}, + geometry, + }); + }; + for (const wpt of childrenNamed(root, 'wpt')) { + const at = point(wpt); + if (at) add(childText(wpt, 'name'), { type: 'Point', coordinates: at }); + } + for (const rte of childrenNamed(root, 'rte')) { + const line = []; + for (const pt of childrenNamed(rte, 'rtept')) { + const at = point(pt); + if (at) line.push(at); + } + if (line.length >= 2) + add(childText(rte, 'name'), { type: 'LineString', coordinates: line }); + } + for (const trk of childrenNamed(root, 'trk')) { + const lines = []; + for (const seg of childrenNamed(trk, 'trkseg')) { + const line = []; + for (const pt of childrenNamed(seg, 'trkpt')) { + const at = point(pt); + if (at) line.push(at); + } + if (line.length >= 2) lines.push(line); + } + if (lines.length) + add(childText(trk, 'name'), { + type: 'MultiLineString', + coordinates: lines, + }); + } + return { type: 'FeatureCollection', features }; +} + +/** A feature name as short plain text (no control characters), or undefined. */ +export function importFeatureName(value) { + const text = String(value ?? '') + .replace(/[\u0000-\u001f\u007f]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + if (!text) return undefined; + return text.length > MAX_IMPORT_NAME_LENGTH + ? `${text.slice(0, MAX_IMPORT_NAME_LENGTH - 1)}…` + : text; +} + +/** + * Make every entity of an imported data source inert: no description, + * balloon, extended data or properties, no native label, and a name that is + * short plain text. Returns how many entities were touched. + * @param {{entities: {values: object[]}}} dataSource + * @returns {number} + */ +export function scrubImportedEntities(dataSource) { + const list = dataSource?.entities?.values || []; + for (let i = 0; i < list.length; i++) { + const entity = list[i]; + let name; + try { + name = importFeatureName( + typeof entity.name === 'string' ? entity.name : undefined, + ); + } catch { + name = undefined; + } + entity.name = name; + entity.description = undefined; + entity.label = undefined; + entity.properties = undefined; + if (entity.kml && typeof entity.kml === 'object') entity.kml = undefined; + } + return list.length; +} + +const TRACK_SAMPLES = 2000; + +/** + * Take time out of an imported data source so adding it cannot move the app's + * shared clock: the data source reports no clock (the viewer adopts the clock + * of every added source that has one), every entity is available for all + * time, and a time-dynamic position (a KML gx:Track) is drawn as the line it + * travels instead of a moving marker. + * @param {object} dataSource + * @param {{Cesium?: object, color?: object}} [options] + * @returns {object} The same data source. + */ +export function detachImportedTime( + dataSource, + { Cesium: C = Cesium, color } = {}, +) { + Object.defineProperty(dataSource, 'clock', { + configurable: true, + enumerable: false, + get: () => undefined, + set: () => {}, + }); + const list = dataSource?.entities?.values || []; + for (let i = 0; i < list.length; i++) { + const entity = list[i]; + const span = entity.availability; + const position = entity.position; + if (position && position.isConstant === false) { + const start = span?.start; + const stop = span?.stop; + const line = []; + if (start && stop && C.JulianDate.lessThan(start, stop)) { + const seconds = C.JulianDate.secondsDifference(stop, start); + for (let n = 0; n <= TRACK_SAMPLES; n++) { + const t = C.JulianDate.addSeconds( + start, + (seconds * n) / TRACK_SAMPLES, + new C.JulianDate(), + ); + const at = position.getValue(t); + if (at) line.push(at); + } + } + entity.path = undefined; + entity.model = undefined; + if (line.length >= 2 && !entity.polyline) { + entity.billboard = undefined; + entity.point = undefined; + entity.position = undefined; + entity.polyline = new C.PolylineGraphics({ + positions: line, + clampToGround: true, + width: 3, + material: color || C.Color.WHITE, + }); + } else { + entity.position = line.length + ? new C.ConstantPositionProperty(line[0]) + : undefined; + } + } + entity.availability = undefined; + } + return dataSource; +} + function parseXml(text, what) { const doc = new DOMParser().parseFromString( String(text).replace(/^/, ''), @@ -364,22 +738,27 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { let dataSource; let removed = 0; try { - if (format === 'geojson') { - dataSource = await Cesium.GeoJsonDataSource.load( - parseGeoJsonText(new TextDecoder().decode(bytes)), - { - clampToGround: true, - stroke, - fill: stroke.withAlpha(0.25), - strokeWidth: 3, - markerColor: stroke, - }, - ); - } else if (format === 'gpx') { - dataSource = await Cesium.GpxDataSource.load( - parseXml(new TextDecoder().decode(bytes), 'GPX'), - { clampToGround: true, trackColor: stroke, routeColor: stroke }, - ); + if (format === 'geojson' || format === 'gpx') { + let geojson; + if (format === 'gpx') { + geojson = gpxToGeoJson( + parseXml(new TextDecoder().decode(bytes), 'GPX'), + ); + } else { + geojson = parseGeoJsonText(new TextDecoder().decode(bytes)); + const size = countGeoJsonGeometry(geojson); + if (size.over) throw overLimitError(size); + stripGeoJsonStyleProperties(geojson); + } + dataSource = await Cesium.GeoJsonDataSource.load(geojson, { + clampToGround: true, + stroke, + fill: stroke.withAlpha(0.25), + strokeWidth: 3, + markerColor: stroke, + // No HTML description table is built from the file's properties. + describe: () => undefined, + }); } else { let text; let resolveLocal = () => null; @@ -402,6 +781,8 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { text = new TextDecoder().decode(bytes); } const doc = parseXml(text, format.toUpperCase()); + const size = countKmlGeometry(doc); + if (size.over) throw overLimitError(size); removed = sanitizeKmlDocument(doc, resolveLocal).removed; dataSource = await Cesium.KmlDataSource.load(doc, { camera: viewer.scene.camera, @@ -427,6 +808,11 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { revoke(); throw new GeoImportError('That file has nothing to draw.'); } - // GpxDataSource's `name` is read-only, so the display name travels beside it. + scrubImportedEntities(dataSource); + // Only KML has time-dynamic positions (GPX is read without times); its + // track line is drawn in KML's default white, like its other unstyled lines. + detachImportedTime(dataSource); + // The display name travels beside the data source (the file name, not + // anything the document says about itself). return { dataSource, name, format, entityCount, removed, revoke }; } diff --git a/src/data/geoFileImport.test.mjs b/src/data/geoFileImport.test.mjs index 522af2550..c589438fc 100644 --- a/src/data/geoFileImport.test.mjs +++ b/src/data/geoFileImport.test.mjs @@ -1,8 +1,17 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { deflateRawSync } from 'node:zlib'; +import * as Cesium from 'cesium'; import { GeoImportError, + MAX_IMPORT_POSITIONS, + countGeoJsonGeometry, + countKmlGeometry, + detachImportedTime, + gpxToGeoJson, + importFeatureName, + scrubImportedEntities, + stripGeoJsonStyleProperties, detectGeoFileFormat, importDisplayName, kmzImageType, @@ -213,3 +222,283 @@ test('KMZ hrefs resolve only to entries inside the archive', () => { assert.equal(kmzImageType('files/a.PNG'), 'image/png'); assert.equal(kmzImageType('files/a.svg'), null); }); + +/* ── a tiny element tree with the DOM surface the readers use ─────────── */ +function xmlEl(localName, attrs = {}, kids = []) { + const el = { + nodeType: 1, + localName, + childNodes: [], + getAttribute: (n) => (n in attrs ? String(attrs[n]) : null), + }; + for (const k of typeof kids === 'string' ? [] : kids) el.childNodes.push(k); + const text = typeof kids === 'string' ? kids : null; + Object.defineProperty(el, 'textContent', { + get: () => text ?? el.childNodes.map((c) => c.textContent).join(''), + }); + return el; +} +const xmlDoc = (root) => ({ + documentElement: root, + getElementsByTagName(name) { + assert.equal(name, '*'); + const out = []; + const walk = (el) => { + out.push(el); + el.childNodes.forEach(walk); + }; + walk(root); + return out; + }, +}); +const trkpt = (lat, lon, time) => + xmlEl('trkpt', { lat, lon }, time ? [xmlEl('time', {}, time)] : []); + +test('GPX keeps each track segment as its own line, names as text, no times', () => { + const doc = xmlDoc( + xmlEl('gpx', {}, [ + xmlEl('wpt', { lat: 37.77, lon: -122.49 }, [xmlEl('name', {}, 'Camp')]), + xmlEl('wpt', { lat: 99, lon: 0 }), // out of range: skipped + xmlEl('rte', {}, [ + xmlEl('name', {}, 'Route'), + xmlEl('rtept', { lat: 1, lon: 2 }), + xmlEl('rtept', { lat: 3, lon: 4 }), + ]), + xmlEl('trk', {}, [ + xmlEl('name', {}, 'Morning walk'), + xmlEl('trkseg', {}, [ + trkpt(10, 20, '2024-05-01T10:00:00Z'), + trkpt(10.1, 20.1, '2024-05-01T10:01:00Z'), + ]), + // A pause in the recording: a new segment, not a straight jump. + xmlEl('trkseg', {}, [ + trkpt(11, 21, '2024-05-01T11:00:00Z'), + trkpt(11.1, 21.1, '2024-05-01T11:01:00Z'), + trkpt(11.2, 21.2, '2024-05-01T11:02:00Z'), + ]), + xmlEl('trkseg', {}, [trkpt(12, 22)]), // one point: not a line + ]), + ]), + ); + const fc = gpxToGeoJson(doc); + assert.deepEqual( + fc.features.map((f) => [f.properties.name, f.geometry.type]), + [ + ['Camp', 'Point'], + ['Route', 'LineString'], + ['Morning walk', 'MultiLineString'], + ], + ); + assert.deepEqual(fc.features[2].geometry.coordinates, [ + [ + [20, 10], + [20.1, 10.1], + ], + [ + [21, 11], + [21.1, 11.1], + [21.2, 11.2], + ], + ]); + assert.doesNotMatch(JSON.stringify(fc), /2024|time/); + assert.throws(() => gpxToGeoJson(xmlDoc(xmlEl('kml'))), GeoImportError); +}); + +test('GPX past the point limit is refused while reading, before Cesium', () => { + const seg = xmlEl( + 'trkseg', + {}, + Array.from({ length: 30 }, (_, i) => trkpt(1 + i / 100, 2)), + ); + const doc = xmlDoc(xmlEl('gpx', {}, [xmlEl('trk', {}, [seg])])); + assert.throws( + () => gpxToGeoJson(doc, { maxPositions: 20 }), + (e) => e instanceof GeoImportError && /more than .* points/.test(e.message), + ); + assert.throws( + () => + gpxToGeoJson( + xmlDoc( + xmlEl('gpx', {}, [ + xmlEl('wpt', { lat: 1, lon: 1 }), + xmlEl('wpt', { lat: 2, lon: 2 }), + ]), + ), + { maxFeatures: 1 }, + ), + /more than .* features/, + ); +}); + +test('GeoJSON features and vertices are counted before loading, without recursion', () => { + const line = Array.from({ length: 50 }, (_, i) => [i / 10, 1]); + const fc = { + type: 'FeatureCollection', + features: [ + { type: 'Feature', geometry: { type: 'LineString', coordinates: line } }, + { + type: 'Feature', + geometry: { + type: 'GeometryCollection', + geometries: [ + { type: 'Point', coordinates: [1, 2] }, + { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [1, 0], + [1, 1], + [0, 0], + ], + ], + }, + ], + }, + }, + { type: 'Feature', geometry: null }, + ], + }; + assert.deepEqual(countGeoJsonGeometry(fc), { + features: 3, + positions: 55, + over: '', + }); + assert.equal( + countGeoJsonGeometry(fc, { maxPositions: 54 }).over, + 'positions', + ); + assert.equal(countGeoJsonGeometry(fc, { maxFeatures: 2 }).over, 'features'); + assert.deepEqual( + countGeoJsonGeometry({ type: 'Point', coordinates: [1, 2] }), + { + features: 1, + positions: 1, + over: '', + }, + ); + // A very long line is counted iteratively and stops at the limit. + const huge = { + type: 'LineString', + coordinates: Array.from({ length: MAX_IMPORT_POSITIONS + 5 }, () => [0, 0]), + }; + assert.equal(countGeoJsonGeometry(huge).over, 'positions'); +}); + +test('KML placemarks and coordinate tuples are counted before loading', () => { + const coords = (n) => + xmlEl( + 'coordinates', + {}, + Array.from({ length: n }, (_, i) => `${i},1,0`).join('\n '), + ); + const doc = xmlDoc( + xmlEl('kml', {}, [ + xmlEl('Placemark', {}, [xmlEl('LineString', {}, [coords(40)])]), + xmlEl('Placemark', {}, [ + xmlEl('Track', {}, [ + xmlEl('coord', {}, '1 2 0'), + xmlEl('coord', {}, '1 3 0'), + ]), + ]), + ]), + ); + assert.deepEqual(countKmlGeometry(doc), { + features: 2, + positions: 42, + over: '', + }); + assert.equal(countKmlGeometry(doc, { maxPositions: 41 }).over, 'positions'); + assert.equal(countKmlGeometry(doc, { maxFeatures: 1 }).over, 'features'); +}); + +test('simplestyle is dropped: no icon fetched, the row color is what draws', () => { + const fc = { + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + properties: { + 'marker-symbol': 'rail', + 'marker-color': '#f00', + stroke: '#0f0', + 'stroke-width': 9, + fill: '#00f', + 'fill-opacity': 1, + name: 'x', + }, + geometry: null, + }, + { type: 'Feature', properties: null, geometry: null }, + ], + }; + stripGeoJsonStyleProperties(fc); + assert.deepEqual(fc.features[0].properties, { name: 'x' }); +}); + +test('imported entities keep only a short plain-text name: no description, label or properties', () => { + const ds = new Cesium.CustomDataSource('t'); + const e = ds.entities.add({ + name: 'Trail\u0007 head\n' + 'x'.repeat(300), + description: '', + label: { text: 'Trail head' }, + properties: { note: '' }, + position: Cesium.Cartesian3.fromDegrees(1, 2), + }); + e.kml = { extendedData: { note: { value: '