From 5ea6d813104f7388975191507e3ce4b3db387605 Mon Sep 17 00:00:00 2001 From: Mazeyar Moeini Feizabadi Date: Thu, 15 Oct 2026 04:10:41 +0200 Subject: [PATCH] fix(import): correct clamping and bound local-file rendering Normalize GeoJSON ground footprints, preserve explicit KML track heights, and tessellate ground paths before native loading. Validate coordinates, geometry types and decoded image budgets before allocating native resources; strip loader-sensitive metadata and settle deferred failures and cancellation. Render static pins in owned billboard collections instead of walking tens of thousands of point entities each frame. Refine nearby ground heights in batches, isolate imported pins from height picks, and release owned resources and point selections on removal. Use safe ray origins above a hidden coarse globe and retain valid mesh heights outside the current footprint. Frame complete geometry with viewport-aware nadir arrivals, including walls and rotated overlays; respect camera gestures and accept document file drops. Strengthen real-loader, native 3D Tiles, paint, clock and lifecycle coverage. Addresses both P1 and all 25 P2 findings in the follow-up review of #1122. --- CHANGELOG.md | 27 +- docs/CURRENT-STATE.md | 19 +- scripts/format-scope.json | 1 + scripts/package-boundaries.json | 11 +- scripts/qa-geo-import-large.mjs | 64 ++- scripts/qa-geo-import-regressions.mjs | 564 +++++++++++++++++++++++ scripts/qa-geo-import.mjs | 67 ++- src/app/tools.js | 2 + src/data/cameraFraming.js | 15 + src/data/geoFileImport.js | 589 +++++++++++++++++++------ src/data/geoFileImport.test.mjs | 318 +++++++++++-- src/data/geoImportImages.js | 74 ++++ src/data/geoImportPins.js | 128 ++++++ src/locations.js | 14 +- src/ui/flyToDataSource.js | 170 ++++--- src/ui/flyToDataSource.test.mjs | 93 +++- src/ui/geoFileImport.js | 110 +++-- src/ui/geoFileImport.test.mjs | 87 +++- src/ui/templates/display-controls.html | 2 +- 19 files changed, 2023 insertions(+), 332 deletions(-) create mode 100644 scripts/qa-geo-import-regressions.mjs create mode 100644 src/data/cameraFraming.js create mode 100644 src/data/geoImportImages.js create mode 100644 src/data/geoImportPins.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 01df3e208..7c9ff39c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -211,25 +211,14 @@ largest cities) and `CCTV_VEGVESEN_VIDEO=0` keeps stills only. Attributed under NLOD 2.0. -- Import your own GeoJSON, KML, KMZ and GPX files (DISPLAY ▸ Import, or drop - them onto the globe). GeoJSON and GPX files are drawn on the ground in the - color shown on their row; KML and KMZ keep the styles written in them. Each - file gets a row to show, hide, fly to or remove it. Files stay in the - browser for the session. An import cannot make the page fetch anything (KML - network links, remote icons and styles are left out), keeps only plain-text - names from the file (no descriptions, balloons or native labels), never - moves the app's clock (dated tracks draw for all time), and keeps GPX track - segments apart. Size limits are checked before anything is built. The import - half of #890. Thanks to Chase Mills (@kapturner). Multipart geometry limits - apply before loading, recorded KML tracks retain their original vertices, - and queued imports respect navigation made while waiting. Repeated KML styles - are shared before the style limit is applied, allowing large files with many - copies of the same style to load without changing their graphics. Blocked - remote icons now fall back to local pins instead of leaving point markers - invisible, including USGS earthquake exports. Generated pins share one image - in the GPU atlas; unused time/view metadata is pruned before loading, and - obsolete track resampling code is removed. KML vertex counting follows - the loader's whitespace rules. +- Import your own GeoJSON, KML, KMZ and GPX files from DISPLAY or by dropping + them onto the app. Show, hide, fly to or remove each file for the session. + GeoJSON/GPX draw on the ground; KML retains its styles and explicit heights. + Large repeated-style exports use shared local pins, while recorded track + vertices and segment gaps stay intact. Imports respect newer camera gestures, + leave the shared clock alone, and report malformed or oversized files. + File bytes stay on the device; rendered overlays may appear in shared map + screenshots. Thanks to [Chase Mills](https://github.com/kapturner) (#925). - Live HLS cameras play on a page opened over plain HTTP from another machine (`HOST=0.0.0.0`). The client lease id came from `crypto.randomUUID`, which diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 945e7b545..4d1e9c63c 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4449,16 +4449,15 @@ are omitted rather than framing the wrong part of the globe. ### Import Your Own Files (October 2026) -- DISPLAY ▸ **Import** puts GeoJSON (`.geojson`/`.json`), KML, KMZ and GPX files on the globe. Files come in through the Import button (a hidden ``) or by dropping them onto the globe; the format is taken from the extension, then from the first bytes. Each file becomes its own Cesium data source, clamped to the ground, with a row to show/hide it, fly to it, or remove it. GeoJSON and GPX are drawn in the row's color from the whiteboard palette (simplestyle properties in GeoJSON are dropped); KML and KMZ keep the styles written in them, and their row shows a hatched swatch instead of a color. GPX is read by `gpxToGeoJson` and loaded as GeoJSON: waypoints are points, each route a line, and each track one `MultiLineString` with a line per ``, so a pause in a recording stays a gap. Imported files last for the page session: a reload starts empty, and nothing is uploaded or stored. -- **Navigation.** The row's Fly-to is an immediate navigation through the application shell (`runImmediateNavigation('file', …)`): Cockpit refuses it, and the current follow/track owner is released first. The fly after an import is a deferred navigation: `beginDeferredLocationNavigation()` stamps it when the pick or drop is submitted, before queued batches wait and `reassertDeferredLocationNavigation()` re-checks it just before flying, so a camera gesture, search, Cockpit or any newer navigation while files load wins and the import does not fly. Fly-to frames the features' longitudes and latitudes at the surface (thinned to 5,000 points by index, never spread into a call) and lets the camera ground guard lift the eye once the terrain has loaded (`src/ui/flyToDataSource.js`); `viewer.flyTo` is not used, because before terrain streams in it frames ground-clamped points kilometers below the surface. -- **Clock.** An import never changes the app's shared clock. The viewer adopts the clock of every data source added to it, so each imported data source reports no clock (`detachImportedTime`); every entity is available for all time. Unused KML `TimeSpan`/`TimeStamp` and `LookAt`/`Camera` metadata is removed before loader construction, and KML `gx:Track` / `gx:MultiTrack` geometry is converted to static lines before Cesium loads it, preserving the recorded vertices and segment gaps without expanding them through time sampling. GPX times are not read at all. -- Runtime entry: `createApplicationTools` (`src/app/tools.js`) calls `initGeoFileImport({ viewer, navigation: styleManager })` (`src/ui/geoFileImport.js`) after the draw tool and defers its `destroy()`, which removes every imported data source, revokes every blob URL, and gives back the drop listeners on the viewer container and the `window.__gevGeoImport` handle (`importFiles`, `list`, `setVisible`, `flyTo`, `remove`, `diagnostics`). Parsing lives in `src/data/geoFileImport.js`. -- **An imported file cannot make the page fetch anything.** Before `KmlDataSource` reads a KML/KMZ document, `sanitizeKmlDocument` removes `NetworkLink`, `NetworkLinkControl`, `ScreenOverlay`, `PhotoOverlay`, `Model` and `Tour`, every `href` that is not an image packed in the same KMZ or a `data:image/` URL (a blocked `IconStyle` icon definition is removed so the marker falls back to Cesium's default pin, drawn locally; an intentionally empty icon remains empty, and a `GroundOverlay` left without an image is dropped), every `styleUrl` that is not a local `#id`, and placemark `description`/`BalloonStyle`/`Snippet` HTML and `ExtendedData` before metadata processing. GeoJSON `marker-symbol` (a maki icon asset) is dropped. KMZ images are served from blob URLs, one per packed image however many hrefs name it (`kmzImageResolver`). The hint says when something was left out. `scripts/qa-geo-import.mjs` imports files that name a trap host in each of those places and fails on any request that leaves the app's own origin. -- **Imported metadata stays inert.** After loading, `scrubImportedEntities` removes every entity's description, native label, properties and KML extended data, and keeps the name only as plain text without control characters, at most 120 characters. GeoJSON is loaded with a `describe` that builds no HTML. The file name is shown with `textContent` only. Entity normalization batches collection notifications and converts constant generated pin canvases to one shared data URL per image, avoiding a duplicate GPU texture entry per marker while preserving authored image URLs and dynamic image properties. -- **KMZ** is read by a small zip reader (`readZipEntries`: stored and deflated entries through the platform `DecompressionStream`; no encryption, no ZIP64). The main document is `doc.kml`, else the shallowest `.kml`; relative hrefs resolve only to entries inside the archive. -- **Bounds.** A file may be at most 50 MiB (`MAX_IMPORT_BYTES`), a KMZ may inflate to at most 120 MiB in total (`MAX_KMZ_INFLATED_BYTES`, the zip-bomb stop), one file may create at most 20,000 Cesium entities, including multipart geometries and KML parent containers (`MAX_IMPORT_ENTITIES`) and 250,000 vertices (`MAX_IMPORT_POSITIONS`), and a KML/KMZ at most 2,000 normalized style definitions (including named StyleMaps) and distinct surviving image links (`MAX_KML_LINKS`). Identical KML styles are shared after metadata and remote-reference cleanup, preserving inline overrides and local style references; repeated copies do not exhaust the style budget. Folder containers count toward the entity budget. Files of 20–30 MB fit within the byte limit; drawing complexity is bounded separately. Geometry is counted before Cesium builds anything (`countGeoJsonGeometry`, `countKmlGeometry`, and while reading GPX), and the entity count is checked again after loading. At most 12 files are on the globe at once (`MAX_IMPORTED_FILES`): imports run one batch at a time, so two picks or drops in quick succession cannot pass the cap; files over it are not read. A refused file names its reason in the hint and does not stop the rest of the batch; a file with nothing to draw is refused. -- **Keyboard.** Hide, show and remove rebuild the list without losing focus: the same control on the same row keeps it, a removed row hands it to the next row (else the Import button). -- Gate: `node scripts/qa-geo-import.mjs` (dev server up). Large-file gate: `node scripts/qa-geo-import-large.mjs [--file /absolute/path/to/sample.kml]` tests a 27.6 MB repeated-style fixture, compares its graphics with an unshared Cesium control, and records matched frame times on the actual WebGL renderer. Add `--expect-local-pins` for a USGS fixture to verify marker images and painted show/hide output. Unit: `src/data/geoFileImport.test.mjs`, `src/ui/geoFileImport.test.mjs`, `src/ui/flyToDataSource.test.mjs`. +- DISPLAY ▸ **Import** reads GeoJSON (`.geojson`/`.json`), KML, KMZ and GPX from the picker or a file drop anywhere in the app. Each file gets show/hide, Fly-to and Remove controls. GeoJSON and GPX use the row's color; KML/KMZ retain their styles and show a hatched swatch. The file bytes remain on the device for the page session. Rendered overlays can appear in map screenshots sent through voice or other existing screenshot tools. +- **Ground and height.** GeoJSON XYZ coordinates become ground footprints. KML ground lines/rings are tessellated; static recorded tracks preserve their original vertices, segment gaps and explicit altitude modes. KML with absolute/relative altitude keeps that meaning. Ground pins resolve in batches of 256 against loaded terrain and, in the nearby camera footprint on photoreal maps, `clampToHeightMostDetailed`, then draw in one owned native BillboardCollection per file, with static positions and no per-frame point Entity visualizer or clamping callbacks. Point metadata remains in `dataSource.importedEntities` for framing/style inspection; non-point geometry stays in the normal source collection. Pin collections mount only after source admission, follow file show/hide, and are destroyed on removal. Settled camera/map-source changes and completed terrain loads refresh visible files through the same serial queue; distant points retain their latest available height. No detailed worldwide terrain sweep is started. Nearby mesh sampling can add requests/time; configured photoreal providers require separate validation. The depth test is disabled for ground pins, matching Draw's marker visibility policy. +- **Navigation.** Row Fly-to uses `runImmediateNavigation('file', …)`; Cockpit can refuse it and existing follow/track ownership is released first. Automatic Fly-to is stamped when a batch is submitted and reasserted before flying. Pointer/wheel gestures and newer navigation supersede it, including time spent in the queue. Framing uses every finite visible point/path/polygon-hole/wall vertex plus rotated overlay bounds, with viewport-aware range and a nadir view. The target uses available terrain/cached ground height and the ground guard on arrival. Empty footprints never fall back to deferred `viewer.flyTo`. +- **Clock.** Imports expose no source clock and no entity availability. KML times/view metadata are removed before loading; recorded tracks become static lines. GPX segments remain separate and GPX times are ignored. The shared clock stays unchanged. +- **Input handling.** Coordinates and GeoJSON geometry types are checked before Cesium allocation. Only short plain names reach GeoJSON property bags; file IDs, custom CRS and styling properties are removed. KML feature/geometry IDs are removed while Style/StyleMap references remain. Local style references are trimmed, incomplete style pairs removed, hidden-folder flags normalized to the import's show control, and unsupported prototype-named elements removed. UTF-16 BOMs and common KML namespace export defects are handled. Empty drawable content is refused. A loader failure or timeout does not hold the next file indefinitely; teardown aborts pending waits. +- **References and images.** File-authored remote links/icons/styles, network links, models and screen/photo overlays are removed. Packed KMZ and embedded PNG/JPEG/GIF/WebP/BMP images remain local; decoded dimensions are checked before image decoding against a total 16-megapixel and 8,192-pixel-side budget per file. Blocked IconStyle images fall back to local pins; authored empty icons remain empty. Colored overlays without an Icon survive; overlays with a removed external image are dropped. Metadata, native labels and descriptions are removed, and generated pins share one lossless image URL per canvas. +- **Bounds.** File size ≤50 MiB; inflated KMZ ≤120 MiB; ≤20,000 entities (including multipart children and Folder containers); ≤250,000 positions; ≤2,000 normalized style definitions and distinct image links; ≤12 files per session. Exact repeated styles share definitions without changing inheritance or inline overrides. Files of 20–30 MB fit the byte budget, subject to geometry/image limits. Refusals name a reason and batches continue; with multiple failures the hint shows the first reason. +- **Lifecycle and keyboard.** `createApplicationTools` wires `initGeoFileImport` with viewer, navigation, ground services and map-source subscription. Destroy aborts pending work, unsubscribes, removes data sources, revokes blob URLs, removes document/canvas listeners and releases `window.__gevGeoImport`. List rebuilds retain keyboard focus; removing a row hands focus to the next row, else the previous row, else Import. +- **Validation.** `node scripts/qa-geo-import.mjs` covers all four formats and real painted show/hide. Run `node scripts/qa-geo-import-regressions.mjs` against both dev and preview for XYZ/track heights, failure recovery, namespace/BOM support, portrait framing and panel drops. `node scripts/qa-geo-import-large.mjs [--file /absolute/path/sample.kml] [--expect-local-pins]` checks a 27.6 MB repeated-style file, native style/image equivalence, large-file painted output and frame times. It records the WebGL renderer; software-rendered runs do not establish GPU performance. Unit tests live beside the data/controller/framing modules. ### Recent Imagery (September 2026) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 6eb2d1bb7..32f3104e6 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -202,6 +202,7 @@ "scripts/qa-visual-effects.mjs", "scripts/qa-draw-tool.mjs", "scripts/qa-geo-import.mjs", + "scripts/qa-geo-import-regressions.mjs", "scripts/qa-geo-import-large.mjs", "src/ui/mapSource.js", "src/ui/mapSourceControls.js", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index e2f138e7a..8599feecc 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -2228,7 +2228,10 @@ "src/tools/functions.js", "src/voice/actionSchemas.js", "src/view/index.js", - "src/app/embed.js" + "src/app/embed.js", + "src/data/cameraFraming.js", + "src/data/geoImportPins.js", + "src/data/geoImportImages.js" ], "external": [ "hls.js", @@ -2763,7 +2766,8 @@ "src/data/local_data/osm_military_names/names.json", "src/layers/installations/namedMarkers.js", "src/data/infrastructureOverlayEntry.js", - "src/sources/infrastructureData.js" + "src/sources/infrastructureData.js", + "src/data/cameraFraming.js" ], "external": [ "hls.js", @@ -2828,7 +2832,8 @@ "src/sources/overpassFeatures.js", "src/sources/featureSource.js", "src/sources/capability.js", - "src/data/cameraArrival.js" + "src/data/cameraArrival.js", + "src/data/cameraFraming.js" ], "external": [ "cesium", diff --git a/scripts/qa-geo-import-large.mjs b/scripts/qa-geo-import-large.mjs index bdebdeb70..679ea14a1 100644 --- a/scripts/qa-geo-import-large.mjs +++ b/scripts/qa-geo-import-large.mjs @@ -1,5 +1,5 @@ #!/usr/bin/env node -/** Large KML acceptance: style equivalence, 25 MB import and optional real file. +/** Large KML acceptance: style equivalence, 27.6 MB import and optional real file. * QA_BASE_URL defaults to http://localhost:4173. Optional --file /path/file.kml. * For a USGS fixture with blocked icons, add --expect-local-pins to check paint. */ @@ -7,13 +7,14 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import puppeteer from 'puppeteer'; +import { hookRenderErrors, readRenderErrors } from './qa-browserEvidence.mjs'; const base = process.env.QA_BASE_URL || 'http://localhost:4173'; const supplied = process.argv.includes('--file') ? path.resolve(process.argv[process.argv.indexOf('--file') + 1]) : null; const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-large-kml-')); -const file = path.join(directory, 'metadata-rich-25mb.kml'); +const file = path.join(directory, 'metadata-rich-27mb.kml'); const style = ''; const notes = 'A local observation with supporting detail. '.repeat(120); @@ -64,6 +65,7 @@ const browser = await puppeteer.launch({ : ['--no-sandbox', '--use-gl=angle', '--use-angle=swiftshader'], }); const page = await browser.newPage(); +await hookRenderErrors(page); const errors = []; const fileRequests = []; page.on('pageerror', (e) => errors.push(e.message)); @@ -93,6 +95,7 @@ try { const C = window.__CESIUM__; for (const id of dataManager.layers.keys()) await dataManager.setEnabled(id, false); + viewer.clock.shouldAnimate = false; viewer.imageryLayers.removeAll(); viewer.terrainProvider = new C.EllipsoidTerrainProvider(); viewer.camera.cancelFlight(); @@ -111,14 +114,27 @@ try { }; const color = (p) => p?.getValue(now)?.toCssColorString(); const positions = (p) => p?.getValue(now)?.map((v) => [v.x, v.y, v.z]); - return source.entities.values + const renderedPins = new Map(); + const collection = source.importedBillboards; + if (collection) + for (let i = 0; i < collection.length; i++) { + const pin = collection.get(i); + renderedPins.set(pin.id, pin); + } + const actual = (e, key, fallback) => { + const pin = renderedPins.get(e); + return pin ? pin[key] : (e.billboard?.[key]?.getValue(now) ?? fallback); + }; + return (source.importedEntities || source.entities.values) .map((e) => ({ name: e.name, - point: e.position?.getValue(now), - scale: e.billboard?.scale?.getValue(now), - rotation: e.billboard?.rotation?.getValue(now), - image: imageValue(e.billboard?.image), - color: color(e.billboard?.color), + point: renderedPins.get(e)?.position || e.position?.getValue(now), + scale: e.billboard ? actual(e, 'scale', 1) : undefined, + rotation: e.billboard ? actual(e, 'rotation', 0) : undefined, + image: renderedPins.get(e)?.image || imageValue(e.billboard?.image), + color: e.billboard + ? actual(e, 'color', C.Color.WHITE).toCssColorString() + : undefined, width: e.polyline?.width?.getValue(now), lineColor: color(e.polyline?.material?.color), line: positions(e.polyline?.positions), @@ -143,7 +159,8 @@ try { canvas: viewer.scene.canvas, clampToGround: true, }); - for (const e of source.entities.values) e.label = undefined; + for (const e of source.importedEntities || source.entities.values) + e.label = undefined; return source; }; }); @@ -159,7 +176,8 @@ try { same: JSON.stringify(window.__qaStyleSnapshot(control)) === JSON.stringify(window.__qaStyleSnapshot(candidate)), - entities: candidate.entities.values.length, + entities: (candidate.importedEntities || candidate.entities.values) + .length, }; await window.__gevGeoImport.remove(outcomes[0]?.id); return result; @@ -235,18 +253,21 @@ try { const before = await page.evaluate( () => window.__godsEyeView.viewer.dataSources.length, ); + const priorIds = await page.evaluate(() => + window.__gevGeoImport.list().map((f) => f.id), + ); const started = performance.now(); await (await page.$('#geo-import-input')).uploadFile(filename); const name = path.basename(filename); const loaded = await page .waitForFunction( - (name) => - window.__gevGeoImport.list().some((f) => f.name === name) || + ({ name, priorIds }) => + window.__gevGeoImport.list().some((f) => !priorIds.includes(f.id)) || document .getElementById('geo-import-hint') .textContent.startsWith(`${name}:`), { timeout: 120000 }, - name, + { name, priorIds }, ) .then( () => true, @@ -257,7 +278,7 @@ try { hint: document.getElementById('geo-import-hint').textContent, sources: window.__godsEyeView.viewer.dataSources.length, })); - const item = state.files.find((f) => f.name === name); + const item = state.files.find((f) => !priorIds.includes(f.id)); check( `picked ${name} loads`, loaded && @@ -294,6 +315,7 @@ try { return [ C.JulianDate.toIso8601(v.clock.startTime), C.JulianDate.toIso8601(v.clock.stopTime), + C.JulianDate.toIso8601(v.clock.currentTime), v.clock.multiplier, ]; }); @@ -304,6 +326,7 @@ try { return [ C.JulianDate.toIso8601(v.clock.startTime), C.JulianDate.toIso8601(v.clock.stopTime), + C.JulianDate.toIso8601(v.clock.currentTime), v.clock.multiplier, ]; }); @@ -316,9 +339,9 @@ try { const { viewer } = window.__godsEyeView, C = window.__CESIUM__; const source = viewer.dataSources.get(viewer.dataSources.length - 1); - const points = source.entities.values.filter( - (e) => e.position && e.billboard, - ); + const points = ( + source.importedEntities || source.entities.values + ).filter((e) => e.position && e.billboard); const images = points.filter((e) => e.billboard.image?.getValue(viewer.clock.currentTime), ); @@ -406,7 +429,12 @@ try { fileRequests.length === 0, fileRequests.length, ); - check('no uncaught page errors', errors.length === 0, JSON.stringify(errors)); + errors.push(...(await readRenderErrors(page))); + check( + 'no uncaught page or render errors', + errors.length === 0, + JSON.stringify(errors), + ); check( 'all imported sources are removed', await page.evaluate(() => window.__gevGeoImport.list().length === 0), diff --git a/scripts/qa-geo-import-regressions.mjs b/scripts/qa-geo-import-regressions.mjs new file mode 100644 index 000000000..ba5a040a2 --- /dev/null +++ b/scripts/qa-geo-import-regressions.mjs @@ -0,0 +1,564 @@ +#!/usr/bin/env node +/** Import regressions through the real UI controller and Cesium loaders. + * Run against dev and preview: QA_BASE_URL=http://localhost:4173 node scripts/qa-geo-import-regressions.mjs + */ +import puppeteer from 'puppeteer'; +import * as Cesium from 'cesium'; +import { hookRenderErrors, readRenderErrors } from './qa-browserEvidence.mjs'; + +const browser = await puppeteer.launch({ headless: true }); +const page = await browser.newPage(); +const meshBytes = Buffer.alloc(60); +[ + -2000, 2000, -2000, 2000, 2000, -2000, -2000, 2000, 2000, 2000, 2000, 2000, +].forEach((value, i) => meshBytes.writeFloatLE(value, i * 4)); +[0, 1, 2, 2, 1, 3].forEach((value, i) => + meshBytes.writeUInt16LE(value, 48 + i * 2), +); +const fixture = { + '/__qa_import_surface/tileset.json': { + asset: { version: '1.1' }, + geometricError: 0, + root: { + transform: Cesium.Matrix4.toArray( + Cesium.Transforms.eastNorthUpToFixedFrame( + Cesium.Cartesian3.fromDegrees(-122.5, 37.7), + ), + ), + boundingVolume: { box: [0, 0, 2000, 2000, 0, 0, 0, 2000, 0, 0, 0, 100] }, + geometricError: 0, + content: { uri: 'surface.gltf' }, + }, + }, + '/__qa_import_surface/surface.gltf': { + asset: { version: '2.0' }, + scene: 0, + scenes: [{ nodes: [0] }], + nodes: [{ mesh: 0 }], + meshes: [ + { + primitives: [{ attributes: { POSITION: 0 }, indices: 1, material: 0 }], + }, + ], + materials: [{ doubleSided: true }], + buffers: [ + { + byteLength: 60, + uri: `data:application/octet-stream;base64,${meshBytes.toString('base64')}`, + }, + ], + bufferViews: [ + { buffer: 0, byteOffset: 0, byteLength: 48, target: 34962 }, + { buffer: 0, byteOffset: 48, byteLength: 12, target: 34963 }, + ], + accessors: [ + { + bufferView: 0, + componentType: 5126, + count: 4, + type: 'VEC3', + min: [-2000, 2000, -2000], + max: [2000, 2000, 2000], + }, + { bufferView: 1, componentType: 5123, count: 6, type: 'SCALAR' }, + ], + }, +}; +await page.setRequestInterception(true); +page.on('request', (request) => { + const body = fixture[new URL(request.url()).pathname]; + if (body) + void request.respond({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(body), + }); + else void request.continue(); +}); +const errors = [], + requests = []; +page.on('pageerror', (e) => errors.push(e.message)); +page.on('console', (m) => { + if (/Ignoring clampToGround/.test(m.text())) errors.push(m.text()); +}); +await hookRenderErrors(page); +let failed = 0; +try { + await page.setViewport({ width: 390, height: 844 }); + await page.goto( + `${process.env.QA_BASE_URL || 'http://localhost:4173'}/?welcome=0`, + { waitUntil: 'domcontentloaded' }, + ); + await page.waitForFunction( + () => + window.__gevGeoImport && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 90000 }, + ); + await page.evaluate(async () => { + const { viewer, dataManager } = window.__godsEyeView, + C = window.__CESIUM__; + for (const id of dataManager.layers.keys()) + await dataManager.setEnabled(id, false); + viewer.imageryLayers.removeAll(); + viewer.terrainProvider = new C.EllipsoidTerrainProvider(); + viewer.clock.shouldAnimate = false; + viewer.camera.cancelFlight(); + viewer.camera.setView({ + destination: C.Cartesian3.fromDegrees(-122.5, 37.7, 100000), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + }); + page.on('request', (r) => { + if (!/^(data:|blob:)/.test(r.url())) requests.push(r.url()); + }); + const results = await page.evaluate(async () => { + const { viewer } = window.__godsEyeView, + C = window.__CESIUM__, + tool = window.__gevGeoImport; + const results = [], + now = C.JulianDate.clone(viewer.clock.currentTime); + const check = (name, pass, detail = '') => + results.push({ name, pass: Boolean(pass), detail }); + const wrap = (body, attrs = '') => + `${body}`; + const point = '-122.5,37.7,0'; + const load = async (value, name) => { + const content = + typeof value === 'object' && !ArrayBuffer.isView(value) + ? JSON.stringify(value) + : value; + const [outcome] = await tool.importFiles([new File([content], name)]); + return { + outcome, + source: outcome.ok + ? viewer.dataSources.get(viewer.dataSources.length - 1) + : null, + }; + }; + const remove = (r) => { + if (r.outcome.ok) tool.remove(r.outcome.id); + }; + let r = await load( + { + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + id: 'kartaview-owned', + properties: { + name: 'XYZ', + addProperty: 1, + 'marker-symbol': 'rail', + }, + geometry: { type: 'Point', coordinates: [-122.5, 37.7, 100] }, + }, + { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [-122.5, 37.7, 10], + [-122.4, 37.7, 10], + [-122.4, 37.8, 10], + [-122.5, 37.7, 10], + ], + ], + }, + }, + ], + }, + 'xyz.geojson', + ); + const pin = (r.source?.importedEntities || r.source?.entities.values).find( + (e) => e.billboard, + ), + polygon = (r.source?.importedEntities || r.source?.entities.values).find( + (e) => e.polygon, + ); + check( + 'XYZ ground footprint and isolated entity IDs', + r.outcome.ok && + pin.id !== 'kartaview-owned' && + Math.abs( + C.Cartographic.fromCartesian(pin.position.getValue(now)).height, + ) < 0.01 && + polygon.polygon.perPositionHeight?.getValue(now) !== true, + ); + check( + 'ground pins have no persistent clamping callbacks', + pin?.billboard.heightReference?.getValue(now) === + C.HeightReference.NONE && + pin.billboard.disableDepthTestDistance.getValue(now) === Infinity, + ); + remove(r); + r = await load( + wrap( + 'ground-122.5 37.7 0-122.4 37.8 0' + + 'absoluteabsolute-122.5 37.7 10000-122.4 37.8 10000' + + 'line-122.5,37.7 -122.4,37.8', + ), + 'tracks.kml', + ); + const find = (name) => + (r.source?.importedEntities || r.source?.entities.values).find( + (e) => e.name === name, + ); + check( + 'ground tracks and untessellated authored lines clamp', + find('ground')?.polyline.clampToGround?.getValue(now) && + find('line')?.polyline.clampToGround?.getValue(now), + ); + check( + 'explicit track altitude and original vertices survive', + find('absolute')?.polyline.positions.getValue(now).length === 2 && + find('absolute') + .polyline.positions.getValue(now) + .every( + (p) => + Math.abs(C.Cartographic.fromCartesian(p).height - 10000) < 0.01, + ), + ); + remove(r); + r = await load( + wrap( + ` #s ${point}`, + ), + 'local-style.kml', + ); + check('whitespace around a local style remains local', r.outcome.ok); + remove(r); + r = await load( + wrap( + `normalhttps://gev-qa-trap.invalid/style${point}`, + ), + 'incomplete-style.kml', + ); + check('removing external style does not hang the queue', r.outcome.ok); + remove(r); + r = await load( + wrap( + `0${point}`, + ), + 'hidden-folder.kml', + ); + check( + 'import show controls own previously hidden KML content', + r.outcome.ok && + (r.source.importedEntities || r.source.entities.values).every( + (e) => e.isShowing, + ), + ); + remove(r); + r = await load( + wrap( + 'ff00ff0037.837.7-122.4-122.5', + ), + 'colored.kml', + ); + check( + 'a colored overlay without an icon is drawable', + r.outcome.ok && + (r.source.importedEntities || r.source.entities.values).some( + (e) => e.rectangle, + ), + ); + remove(r); + const xml = wrap(`${point}`); + const utf16 = new Uint8Array(2 + xml.length * 2); + utf16.set([255, 254]); + for (let i = 0; i < xml.length; i++) { + utf16[2 + i * 2] = xml.charCodeAt(i) & 255; + utf16[3 + i * 2] = xml.charCodeAt(i) >> 8; + } + r = await load(utf16, 'utf16.kml'); + check('UTF-16 BOM is decoded', r.outcome.ok); + remove(r); + r = await load( + wrap( + `${point}`, + 'xmlns:gx="http://www.google.com/kml/ext/2.2" xsi:schemaLocation="test"', + ), + 'namespaces.kml', + ); + check( + 'ordinary duplicate and missing export namespaces are repaired', + r.outcome.ok, + ); + remove(r); + for (const [name, value] of [ + ['null geometry', { type: 'Feature', geometry: null }], + ['invalid coordinates', { type: 'Point', coordinates: ['bad', 37] }], + [ + 'missing feature type', + { + type: 'FeatureCollection', + features: [{ geometry: { type: 'Point', coordinates: [0, 0] } }], + }, + ], + [ + 'nested unsupported geometry', + { type: 'Feature', geometry: { type: 'Topology', arcs: [] } }, + ], + ]) { + r = await load(value, 'refused.geojson'); + check(`${name} is refused without stopping rendering`, !r.outcome.ok); + remove(r); + } + r = await load( + wrap( + 'NaN,37,0', + ), + 'invalid.kml', + ); + check('non-finite KML coordinates are refused', !r.outcome.ok); + remove(r); + r = await load( + { + type: 'LineString', + coordinates: [ + [-122.6, 37.7], + [-122.4, 37.7], + ], + }, + 'portrait.geojson', + ); + await new Promise((resolve) => setTimeout(resolve, 2300)); + const positions = (r.source?.importedEntities || + r.source?.entities.values)[0].polyline.positions.getValue(now); + const screen = positions?.map((p) => + C.SceneTransforms.worldToWindowCoordinates(viewer.scene, p), + ); + check( + 'portrait flight includes both route endpoints', + screen?.every( + (p) => p && p.x > 0 && p.x < innerWidth && p.y > 0 && p.y < innerHeight, + ), + screen?.map((p) => p && [p.x, p.y]), + ); + remove(r); + r = await load( + { + type: 'MultiPoint', + coordinates: [ + [-74, 40.7], + [139.7, 35.7], + ], + }, + 'intercontinental.geojson', + ); + await new Promise((resolve) => setTimeout(resolve, 2300)); + const intercontinental = ( + r.source?.importedEntities || r.source?.entities.values + ) + .filter((e) => e.position) + .map((e) => e.position.getValue(now)); + const occluder = new C.EllipsoidalOccluder( + C.Ellipsoid.WGS84, + viewer.camera.positionWC, + ); + check( + 'intercontinental points remain in frame and in front of the horizon', + intercontinental?.every((p) => { + const s = C.SceneTransforms.worldToWindowCoordinates(viewer.scene, p); + return ( + occluder.isPointVisible(p) && + s && + s.x > 0 && + s.x < innerWidth && + s.y > 0 && + s.y < innerHeight + ); + }), + ); + remove(r); + r = await load( + wrap( + 'absolute14.9,52.37,3000 5.12,52.09,30005.12,52.09', + ), + 'wall.kml', + ); + await new Promise((resolve) => setTimeout(resolve, 2300)); + check( + 'a wall contributes both endpoints to the flight', + r.outcome.ok && + (r.source.importedEntities || r.source.entities.values) + .find((e) => e.wall) + ?.wall.positions.getValue(now) + .every((p) => { + const s = C.SceneTransforms.worldToWindowCoordinates( + viewer.scene, + p, + ); + return ( + s && s.x > 0 && s.x < innerWidth && s.y > 0 && s.y < innerHeight + ); + }), + ); + remove(r); + r = await load( + { + type: 'MultiPoint', + coordinates: [ + [0, 0], + [180, 0], + ], + }, + 'antipodal.geojson', + ); + await new Promise((resolve) => setTimeout(resolve, 1800)); + check( + 'symmetric global extents keep a finite camera', + r.outcome.ok && + Number.isFinite(viewer.camera.positionCartographic.height), + ); + remove(r); + viewer.camera.cancelFlight(); + viewer.camera.setView({ + destination: C.Cartesian3.fromDegrees(-122.5, 37.7, 100000), + orientation: { heading: 0, pitch: -Math.PI / 2, roll: 0 }, + }); + const mesh = await C.Cesium3DTileset.fromUrl( + `${location.origin}/__qa_import_surface/tileset.json`, + ); + viewer.scene.primitives.add(mesh); + // This fixture installs a mesh outside the app's map-source lifecycle. + // Let the new camera/tiles reach a rendered frame before testing sampling. + await new Promise((resolve) => { + let frames = 0; + const remove = viewer.scene.postRender.addEventListener(() => { + if (++frames >= 2) { + remove(); + resolve(); + } else viewer.scene.requestRender(); + }); + viewer.scene.requestRender(); + }); + const fixtureDeadline = performance.now() + 5000; + while (!mesh.tilesLoaded && performance.now() < fixtureDeadline) { + viewer.scene.requestRender(); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + const globeShown = viewer.scene.globe.show; + viewer.scene.globe.show = false; + + r = await load( + { type: 'Point', coordinates: [-122.5, 37.7, 0] }, + 'mesh-point.geojson', + ); + const meshPoint = (r.source?.importedEntities || r.source?.entities.values) + .find((e) => e.billboard) + ?.position.getValue(now); + const meshHeight = + meshPoint && C.Cartographic.fromCartesian(meshPoint).height; + check( + 'native 3D Tiles ground sampling keeps pins on the rendered mesh', + r.outcome.ok && Math.abs(meshHeight - 2000) < 2, + { + meshHeight, + tilesLoaded: mesh.tilesLoaded, + globeShown: viewer.scene.globe.show, + }, + ); + tool.flyTo(r.outcome.id); + await new Promise((resolve) => setTimeout(resolve, 2300)); + const meshScreen = + meshPoint && + C.SceneTransforms.worldToWindowCoordinates(viewer.scene, meshPoint); + check( + 'a high-surface arrival frames the elevated pin', + meshScreen && + Math.abs(meshScreen.x - innerWidth / 2) < 20 && + Math.abs(meshScreen.y - innerHeight / 2) < 20, + { + screen: meshScreen && [meshScreen.x, meshScreen.y], + camera: { + height: viewer.camera.positionCartographic.height, + pitch: viewer.camera.pitch, + }, + meshHeight, + }, + ); + const groundSource = r.source; + const elevated = await load( + wrap( + 'absolute-122.5,37.7,5000', + ), + 'elevated-pin.kml', + ); + await new Promise((resolve) => setTimeout(resolve, 2600)); + check( + 'a higher imported pin never becomes the sampled ground', + elevated.outcome.ok && + Math.abs( + C.Cartographic.fromCartesian( + groundSource.importedBillboards.get(0).position, + ).height - 2000, + ) < 2, + ); + remove(elevated); + remove(r); + viewer.scene.primitives.remove(mesh); + viewer.scene.globe.show = globeShown; + const before = tool.list().length; + const transfer = new DataTransfer(); + transfer.items.add( + new File( + [JSON.stringify({ type: 'Point', coordinates: [-122.5, 37.7] })], + 'panel-drop.geojson', + ), + ); + const event = new DragEvent('drop', { + bubbles: true, + cancelable: true, + dataTransfer: transfer, + }); + document.getElementById('geo-import-button').dispatchEvent(event); + for (let i = 0; i < 100 && tool.list().length === before; i++) + await new Promise((resolve) => setTimeout(resolve, 20)); + check( + 'a panel drop imports and prevents the browser default', + event.defaultPrevented && tool.list().length === before + 1, + ); + for (const file of tool.list()) tool.remove(file.id); + check( + 'the shared clock stays exactly unchanged', + C.JulianDate.equals(now, viewer.clock.currentTime), + ); + check('no file sources remain', tool.list().length === 0); + return results; + }); + for (const r of results) { + console.log( + `[${r.pass ? 'PASS' : 'FAIL'}] ${r.name}`, + JSON.stringify(r.detail), + ); + if (!r.pass) failed++; + } + const renderErrors = await readRenderErrors(page); + if (errors.length || renderErrors.length) { + failed++; + console.log( + '[FAIL] browser/render errors', + JSON.stringify({ errors, renderErrors }), + ); + } + // Terrain belongs to the base map; imported files may sample that provider. + const unexpected = requests.filter( + (url) => + new URL(url).pathname !== '/logo.svg' && + !new URL(url).pathname.startsWith('/__qa_import_surface/') && + !/\/api\/|\.terrain(?:\?|$)|reearth|cesium|arcgis|openstreetmap|\.pbf(?:\?|$)/i.test( + url, + ), + ); + console.log( + `[${unexpected.length ? 'FAIL' : 'PASS'}] no unexpected file-directed requests`, + JSON.stringify(unexpected), + ); + if (unexpected.length) failed++; +} finally { + await browser.close(); +} +process.exitCode = failed ? 1 : 0; diff --git a/scripts/qa-geo-import.mjs b/scripts/qa-geo-import.mjs index b76e67623..074eb8fbd 100644 --- a/scripts/qa-geo-import.mjs +++ b/scripts/qa-geo-import.mjs @@ -23,6 +23,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import puppeteer from 'puppeteer'; +import { hookRenderErrors, readRenderErrors } from './qa-browserEvidence.mjs'; const BASE_URL = process.env.QA_BASE_URL || 'http://localhost:4173'; const SHOT_DIR = process.env.QA_SHOT_DIR || 'qa-shots/geo-import'; @@ -53,12 +54,12 @@ const geojson = JSON.stringify({ 'fill-opacity': 1, description: ``, }, - geometry: { type: 'Polygon', coordinates: [ring] }, + geometry: { type: 'Polygon', coordinates: [ring.map((p) => [...p, 0])] }, }, { type: 'Feature', properties: { name: 'QA pin', 'marker-symbol': 'rail' }, - geometry: { type: 'Point', coordinates: [-122.4862, 37.7694] }, + geometry: { type: 'Point', coordinates: [-122.4862, 37.7694, 0] }, }, ], }); @@ -172,9 +173,14 @@ const browser = await puppeteer.launch({ ], }); const page = await browser.newPage(); +await hookRenderErrors(page); const errors = []; const trapped = []; page.on('pageerror', (error) => errors.push(`pageerror: ${error.message}`)); +page.on('console', (message) => { + if (/Ignoring clampToGround/.test(message.text())) + errors.push(message.text()); +}); // Anything a file names, and the icon assets Cesium would load on a file's // behalf (KML palette icons from maps.google.com, GeoJSON maki markers). const FILE_FETCH = [TRAP_HOST, 'maps.google.com/mapfiles', '/maki/']; @@ -219,7 +225,7 @@ async function cyanPixels() { let n = 0; for (let i = 0; i < data.length; i += 4) { const [r, g, b] = [data[i], data[i + 1], data[i + 2]]; - if (b > 150 && g > 120 && r < 140 && b - r > 60) n += 1; + if (b > 45 && g > 30 && r < 100 && b - r > 30) n += 1; } return n; }, base64); @@ -242,8 +248,14 @@ try { document.getElementById('loading-screen')?.classList.contains('hidden'), { timeout: 90_000 }, ); - await page.evaluate((view) => { + await page.evaluate(async (view) => { const viewer = window.__godsEyeView.viewer; + for (const id of window.__godsEyeView.dataManager.layers.keys()) + await window.__godsEyeView.dataManager.setEnabled(id, false); + viewer.imageryLayers.removeAll(); + viewer.terrainProvider = new window.__CESIUM__.EllipsoidTerrainProvider(); + viewer.scene.globe.baseColor = window.__CESIUM__.Color.BLACK; + viewer.scene.globe.enableLighting = false; viewer.camera.cancelFlight?.(); viewer.camera.setView({ destination: window.__CESIUM__.Cartesian3.fromDegrees( @@ -280,6 +292,9 @@ try { )), ); const cyanBefore = await cyanPixels(); + await page.evaluate(() => { + window.__godsEyeView.viewer.clock.shouldAnimate = false; + }); const clock = () => page.evaluate(() => { const c = window.__godsEyeView.viewer.clock; @@ -292,7 +307,7 @@ try { multiplier: c.multiplier, animate: c.shouldAnimate, // Wall-clock drift aside, the current time must not jump to 2018/2019. - year: Number(iso(c.currentTime).slice(0, 4)), + currentTime: window.__CESIUM__.JulianDate.toIso8601(c.currentTime), }; }); const clockBefore = await clock(); @@ -350,9 +365,12 @@ try { const sources = []; for (let i = from; i < viewer.dataSources.length; i++) sources.push(viewer.dataSources.get(i)); - const entities = sources.flatMap((ds) => ds.entities.values); + const entities = sources.flatMap( + (ds) => ds.importedEntities || ds.entities.values, + ); const named = (name) => entities.find((e) => e.name === name); - const gpx = sources[3].entities.values; + const gpx = sources[3].importedEntities || sources[3].entities.values; + const kmz = sources[2].importedEntities || sources[2].entities.values; return { sources: sources.length, clocks: sources.filter((ds) => ds.clock !== undefined).length, @@ -370,11 +388,26 @@ try { ), movingVertices: named('QA moving')?.polyline?.positions?.getValue(now)?.length, - splitTracks: sources[1].entities.values + splitTracks: (sources[1].importedEntities || sources[1].entities.values) .filter((e) => e.parent?.name === 'QA split tracks' && e.polyline) .map((e) => e.polyline.positions.getValue(now).length), gpxLines: gpx.filter((e) => e.polyline).length, gpxPoints: gpx.filter((e) => e.billboard || e.point).length, + kmzLocalImage: kmz.some((e) => + String(e.billboard?.image?.getValue(now) || '').startsWith('blob:'), + ), + kmzOverlay: kmz.some((e) => e.rectangle), + groundedLines: sources + .slice(1, 3) + .every((ds) => + (ds.importedEntities || ds.entities.values) + .filter((e) => e.polyline) + .every((e) => e.polyline.clampToGround?.getValue(now)), + ), + xyzPolygon: + (sources[0].importedEntities || sources[0].entities.values) + .find((e) => e.polygon) + ?.polygon.perPositionHeight?.getValue(now) !== true, }; }, sourcesBefore); check( @@ -411,6 +444,19 @@ try { inside.gpxLines === 2 && inside.gpxPoints === 1, JSON.stringify(inside), ); + check( + 'KMZ retains its own packed image and overlay', + inside.kmzLocalImage && inside.kmzOverlay, + ); + check( + 'XYZ polygons and KML/KMZ lines use ground geometry', + inside.xyzPolygon && inside.groundedLines, + ); + await page.evaluate((from) => { + for (const entity of window.__godsEyeView.viewer.dataSources.get(from) + .entities.values) + if (entity.billboard) entity.show = false; + }, sourcesBefore); await page.evaluate((view) => { const viewer = window.__godsEyeView.viewer; viewer.camera.cancelFlight?.(); @@ -428,7 +474,7 @@ try { await shot('imported'); check( 'the imported polygon paints on the globe', - cyanShown > cyanBefore * 4, + cyanShown > cyanBefore * 4 && cyanShown > cyanBefore + 5000, `${cyanBefore} → ${cyanShown}`, ); @@ -536,7 +582,8 @@ try { trapped.length === 0, trapped.join(' '), ); - check('no page errors', errors.length === 0, errors.join(' | ')); + errors.push(...(await readRenderErrors(page))); + check('no page or rendering errors', errors.length === 0, errors.join(' | ')); } catch (error) { check('harness ran to completion', false, error?.stack || String(error)); } finally { diff --git a/src/app/tools.js b/src/app/tools.js index 7acc4939e..7b3ea83c9 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -63,6 +63,8 @@ export function createApplicationTools({ const geoFileImport = initGeoFileImport({ viewer, navigation: styleManager, + ground: operations.surface?.groundFloor, + subscribeMapSource: (listener) => mapStackController?.subscribe?.(listener), }); defer(() => geoFileImport?.destroy()); // DATA ▸ Recent Imagery: the box tool claims the pointer like Draw and the diff --git a/src/data/cameraFraming.js b/src/data/cameraFraming.js new file mode 100644 index 000000000..a2d38b5ec --- /dev/null +++ b/src/data/cameraFraming.js @@ -0,0 +1,15 @@ +import * as Cesium from 'cesium'; + +/** Camera range with breathing room in both viewport dimensions. */ +export function rangeForBoundingSphere(viewer, radius) { + const frustum = viewer.camera?.frustum; + const verticalFov = Number(frustum?.fov) || Cesium.Math.toRadians(60); + const aspectRatio = Math.max(0.5, Number(frustum?.aspectRatio) || 1); + const horizontalFov = 2 * Math.atan(Math.tan(verticalFov / 2) * aspectRatio); + const limitingFov = Math.min(verticalFov, horizontalFov); + // Target 57% sphere occupancy so perspective still leaves at least 40% + // measured roof-to-base breathing room in ordinary oblique building views. + const occupiedViewportFraction = 0.57; + const desiredAngularRadius = (limitingFov * occupiedViewportFraction) / 2; + return (radius / Math.sin(desiredAngularRadius)) * 1.05; +} diff --git a/src/data/geoFileImport.js b/src/data/geoFileImport.js index 9c0e3688b..0ceaf9ef5 100644 --- a/src/data/geoFileImport.js +++ b/src/data/geoFileImport.js @@ -1,36 +1,7 @@ -/** - * Your own geodata on the globe: GeoJSON, KML, KMZ and GPX files read in the - * browser and handed to Cesium's own data sources. - * - * Nothing here talks to the network. The file never leaves the page, and a - * KML/KMZ file cannot make the page fetch anything either: network links, - * remote icons, remote styles, models and photo/screen overlays are removed - * before Cesium sees the document (`sanitizeKmlDocument`). Images packed - * inside a KMZ are served to Cesium from blob URLs the caller revokes. - * - * What the file says about itself stays inert: descriptions, balloons, - * extended data and GeoJSON properties are dropped from every entity, names - * are kept only as short plain text, and no native Cesium label is drawn - * (`scrubImportedEntities`). GeoJSON and GPX draw in the row's color: - * simplestyle properties are dropped, including `marker-symbol`, so no icon - * asset is loaded on the file's behalf. KML and KMZ keep their own styles. - * - * An import never moves the app's shared clock: dated KML (TimeSpan, - * TimeStamp, gx:Track) is drawn for all time, and the data source exposes no - * clock for the viewer to adopt (`detachImportedTime`). GPX is read here - * rather than by Cesium (`gpxToGeoJson`), keeping each track segment as its - * own line and leaving times out. - * - * Size is checked before Cesium builds anything (`countGeoJsonGeometry`, - * `countKmlGeometry`): a file with too many features or vertices is refused - * up front instead of after the entities exist. - * - * The bounded pieces — format detection, the KMZ (zip) reader, the size and - * count limits — are plain functions so they run under `node --test`; the - * Cesium loaders need a browser DOM and are exercised by - * `scripts/qa-geo-import.mjs`. - */ +/** Session-local geodata loading with bounded geometry and local-only file references. */ import * as Cesium from 'cesium'; +import { createImportImageBudget } from './geoImportImages.js'; +import { createImportedPins } from './geoImportPins.js'; /** Largest file accepted, in bytes. */ export const MAX_IMPORT_BYTES = 50 * 1024 * 1024; @@ -84,6 +55,10 @@ const KML_REMOVED_ELEMENTS = new Set([ 'TimeStamp', 'LookAt', 'Camera', + 'visibility', + 'viewRefreshMode', + 'viewFormat', + 'httpQuery', ]); /** An error whose message is meant for the person who chose the file. */ @@ -186,7 +161,13 @@ export function countGeoJsonGeometry( if (!node || typeof node !== 'object') continue; if (node.type === 'FeatureCollection') { const list = Array.isArray(node.features) ? node.features : []; - for (let i = 0; i < list.length; i++) stack.push(list[i]); + for (let i = 0; i < list.length; i++) { + if (list[i]?.type !== 'Feature') + throw new GeoImportError( + 'Every GeoJSON feature must have type Feature.', + ); + stack.push(list[i]); + } continue; } if (node.type === 'Feature') { @@ -200,11 +181,27 @@ export function countGeoJsonGeometry( continue; } if (node.type === 'GeometryCollection') { - const list = Array.isArray(node.geometries) ? node.geometries : []; - for (let i = 0; i < list.length; i++) stack.push(list[i]); + const list = node.geometries; + if (!Array.isArray(list)) + throw new GeoImportError('That GeoJSON has invalid geometries.'); + for (let i = 0; i < list.length; i++) { + if (!list[i] || ['Feature', 'FeatureCollection'].includes(list[i].type)) + throw new GeoImportError( + 'That GeoJSON contains an unsupported geometry.', + ); + stack.push(list[i]); + } continue; } - if (GEOJSON_TYPES.has(node.type)) { + const depth = { + Point: 0, + MultiPoint: 1, + LineString: 1, + MultiLineString: 2, + Polygon: 2, + MultiPolygon: 3, + }[node.type]; + if (depth !== undefined) { // Multipart geometries create one Cesium entity per component, even // when the document wraps them in just one Feature. entities += node.type.startsWith('Multi') @@ -215,16 +212,30 @@ export function countGeoJsonGeometry( if (entities > maxEntities) return result('entities'); // Walk the coordinate arrays iteratively: a position is an array whose // first item is a number. - const coords = [node.coordinates]; + const coords = [[node.coordinates, depth]]; while (coords.length) { - const c = coords.pop(); - if (!Array.isArray(c)) continue; - if (typeof c[0] === 'number') { + const [c, remaining] = coords.pop(); + if (!Array.isArray(c)) + throw new GeoImportError('That GeoJSON has invalid coordinates.'); + if (remaining === 0) { + if ( + c.length < 2 || + c.length > 3 || + !c.every(Number.isFinite) || + Math.abs(c[0]) > 180 || + Math.abs(c[1]) > 90 || + (c.length === 3 && Math.abs(c[2]) > 1_000_000) + ) + throw new GeoImportError('That GeoJSON has invalid coordinates.'); positions += 1; if (positions > maxPositions) return result('positions'); - } else for (let i = 0; i < c.length; i++) coords.push(c[i]); + } else + for (let i = 0; i < c.length; i++) coords.push([c[i], remaining - 1]); } - } + } else + throw new GeoImportError( + 'That GeoJSON contains an unsupported geometry.', + ); } return result(); } @@ -248,42 +259,32 @@ function overLimitError({ over, features }) { ); } -// simplestyle-spec keys Cesium honors. `marker-symbol` would load a maki icon -// asset on the file's behalf; the colors would override the row's color. -const GEOJSON_STYLE_KEYS = Object.freeze([ - 'marker-symbol', - 'marker-color', - 'marker-size', - 'stroke', - 'stroke-opacity', - 'stroke-width', - 'fill', - 'fill-opacity', -]); - -/** - * Drop GeoJSON simplestyle properties, so a GeoJSON file draws in its row's - * color like a GPX file and no icon is fetched for it. Mutates and returns - * the object. - * @param {object} geojson - * @returns {object} - */ -export function stripGeoJsonStyleProperties(geojson) { +// The import uses WGS84 ground footprints and a short name only. Remove +// loader-sensitive metadata before Cesium allocates properties or entity IDs. +function prepareGeoJson(geojson) { const stack = [geojson]; while (stack.length) { const node = stack.pop(); - if (!node || typeof node !== 'object') continue; - if (node.type === 'FeatureCollection' && Array.isArray(node.features)) - for (let i = 0; i < node.features.length; i++) - stack.push(node.features[i]); - else if ( - node.type === 'Feature' && - node.properties && - typeof node.properties === 'object' - ) - for (const key of GEOJSON_STYLE_KEYS) delete node.properties[key]; + if (!node) continue; + delete node.crs; + delete node.id; + if (node.type === 'FeatureCollection') { + for (const feature of node.features) stack.push(feature); + } else if (node.type === 'Feature') { + const name = importFeatureName(node.properties?.name); + node.properties = name ? { name } : {}; + stack.push(node.geometry); + } else if (node.type === 'GeometryCollection') { + for (const geometry of node.geometries) stack.push(geometry); + } else { + const coords = [node.coordinates]; + while (coords.length) { + const value = coords.pop(); + if (typeof value[0] === 'number') value.length = 2; + else for (const child of value) coords.push(child); + } + } } - return geojson; } /** Inflate raw-deflate bytes with the platform's stream, stopping at `maxBytes`. */ @@ -498,14 +499,26 @@ export function sanitizeKmlDocument(doc, resolveLocal = () => null) { // every discarded element type. Strip unused metadata before the loader // constructs descriptions, availability intervals and camera objects. for (const el of [...doc.getElementsByTagName('*')]) - if (KML_REMOVED_ELEMENTS.has(el.localName)) drop(el); + if ( + KML_REMOVED_ELEMENTS.has(el.localName) || + el.localName in Object.prototype + ) + drop(el); + else if (!['Style', 'StyleMap'].includes(el.localName)) + el.removeAttribute('id'); for (const href of elementsNamed(doc, 'href')) { const value = (href.textContent || '').trim(); + if (!value) continue; const local = resolveLocal(value); if (local) href.textContent = local; else if (/^data:image\/(png|jpeg|gif|webp|bmp);/i.test(value)) continue; else { const icon = href.parentNode; + const overlay = icon?.parentNode; + if (overlay?.localName === 'GroundOverlay') { + drop(overlay); + continue; + } // An empty Icon explicitly hides the billboard in Cesium. Remove the // blocked icon definition so the loader supplies its local default pin. // Authored empty Icons (with no blocked href) still mean no marker. @@ -516,12 +529,18 @@ export function sanitizeKmlDocument(doc, resolveLocal = () => null) { ); } } - for (const overlay of elementsNamed(doc, 'GroundOverlay')) { - if (!elementsNamed(overlay, 'href').length) drop(overlay); - } for (const styleUrl of elementsNamed(doc, 'styleUrl')) { - if (!(styleUrl.textContent || '').trim().startsWith('#')) drop(styleUrl); + const value = (styleUrl.textContent || '').trim(); + if (!value.startsWith('#')) drop(styleUrl); + else styleUrl.textContent = value; } + for (const pair of elementsNamed(doc, 'Pair')) + if ( + !childText(pair, 'key') || + (!childrenNamed(pair, 'styleUrl').length && + !childrenNamed(pair, 'Style').length) + ) + drop(pair); // Draw recorded KML paths as their original static vertices. Letting the // loader build time-dynamic tracks would expand each into thousands of // interpolated points later and exceed the checked geometry budget. @@ -539,6 +558,8 @@ export function sanitizeKmlDocument(doc, resolveLocal = () => null) { ); coordinates.textContent = recordedCoordinates.join(' '); line.appendChild(coordinates); + for (const mode of childrenNamed(track, 'altitudeMode')) + line.appendChild(mode.cloneNode(true)); track.parentNode?.replaceChild(line, track); } for (const tracks of elementsNamed(doc, 'MultiTrack')) { @@ -550,6 +571,28 @@ export function sanitizeKmlDocument(doc, resolveLocal = () => null) { geometry.appendChild(line); tracks.parentNode?.replaceChild(geometry, tracks); } + for (const line of [ + ...elementsNamed(doc, 'LineString'), + ...elementsNamed(doc, 'LinearRing'), + ]) { + const mode = childText(line, 'altitudeMode'); + if ( + !mode || + mode === 'clampToGround' || + mode === 'clampToSeaFloor' || + !childrenNamed(line, 'tessellate').length + ) { + let tessellate = childrenNamed(line, 'tessellate')[0]; + if (!tessellate) { + tessellate = doc.createElementNS( + 'http://www.opengis.net/kml/2.2', + 'tessellate', + ); + line.appendChild(tessellate); + } + tessellate.textContent = '1'; + } + } return { removed }; } @@ -708,6 +751,22 @@ export function countKmlGeometry( ]); for (const el of doc.getElementsByTagName('*')) { const name = el.localName; + if ( + ['north', 'south', 'east', 'west', 'rotation', 'altitude'].includes(name) + ) { + const value = Number(el.textContent); + const limit = ['north', 'south'].includes(name) + ? 90 + : name === 'altitude' + ? 1_000_000 + : 360; + if ( + !el.textContent.trim() || + !Number.isFinite(value) || + Math.abs(value) > limit + ) + throw new GeoImportError('That KML has invalid geometry bounds.'); + } // Cesium creates entities for folders, a parent for every placemark, and // a child for each geometry in a MultiGeometry, including nested containers. if ( @@ -738,21 +797,13 @@ export function countKmlGeometry( if (features > maxFeatures) return result('features'); } else if (name === 'coordinates') { const text = el.textContent || ''; - // Tuples are separated by whitespace; count runs without splitting - // a large string into a large array. - let inTuple = false; - for (let i = 0; i < text.length; i++) { - const code = text.charCodeAt(i); - // Match Cesium's whitespace splitting, keeping the common ASCII - // path cheap and avoiding a large token array during preflight. - const space = code <= 32 || (code > 127 && /\s/.test(text[i])); - if (!space && !inTuple) { - positions += 1; - if (positions > maxPositions) return result('positions'); - } - inTuple = !space; + for (const match of text.matchAll(/\S+/g)) { + checkKmlPosition(match[0].split(',')); + positions += 1; + if (positions > maxPositions) return result('positions'); } } else if (name === 'coord') { + checkKmlPosition(el.textContent.trim().split(/\s+/)); positions += 1; if (positions > maxPositions) return result('positions'); } @@ -760,6 +811,20 @@ export function countKmlGeometry( return result(''); } +function checkKmlPosition(tuple) { + const numbers = tuple.map(Number); + if ( + tuple.length < 2 || + tuple.length > 3 || + tuple.some((v) => !v.trim()) || + !numbers.every(Number.isFinite) || + Math.abs(numbers[0]) > 180 || + Math.abs(numbers[1]) > 90 || + (numbers.length === 3 && Math.abs(numbers[2]) > 1_000_000) + ) + throw new GeoImportError('That KML has invalid coordinates.'); +} + /** * Read a parsed GPX document into GeoJSON: waypoints become points, each * route a line and each track one MultiLineString with a line per `` @@ -891,6 +956,11 @@ export function scrubImportedEntities(dataSource) { entity.label = undefined; entity.properties = undefined; if (entity.kml && typeof entity.kml === 'object') entity.kml = undefined; + if ( + entity.billboard?.heightReference?.getValue?.() === + Cesium.HeightReference.CLAMP_TO_GROUND + ) + entity.billboard.disableDepthTestDistance = Number.POSITIVE_INFINITY; const property = entity.billboard?.image; if (property?.isConstant === false) continue; const image = property?.getValue?.(); @@ -934,6 +1004,25 @@ export function detachImportedTime(dataSource) { } function parseXml(text, what) { + // Match the namespace repairs Cesium applies to ordinary exported KML. + if (what !== 'GPX') + text = String(text).replace(/]*>/i, (root) => { + const seen = new Set(); + root = root.replace( + /\s+xmlns:([\w.-]+)\s*=\s*(["'])[^"']*\2/g, + (declaration, prefix) => { + if (seen.has(prefix)) return ''; + seen.add(prefix); + return declaration; + }, + ); + if (!seen.has('xsi') && /[<\s]xsi:/.test(text)) + root = root.replace( + />$/, + ' xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">', + ); + return root; + }); const doc = new DOMParser().parseFromString( String(text).replace(/^/, ''), 'application/xml', @@ -943,21 +1032,224 @@ function parseXml(text, what) { return doc; } +function decodeXml(bytes) { + const encoding = + bytes[0] === 0xff && bytes[1] === 0xfe + ? 'utf-16le' + : bytes[0] === 0xfe && bytes[1] === 0xff + ? 'utf-16be' + : 'utf-8'; + return new TextDecoder(encoding).decode(bytes); +} + +function checkLocalImages(doc, entries = new Map(), main = '') { + const charge = createImportImageBudget(); + const checked = new Set(); + for (const href of elementsNamed(doc, 'href')) { + const value = href.textContent.trim(); + if (checked.has(value)) continue; + checked.add(value); + const path = resolveKmzPath(value, main, entries); + try { + if (path && kmzImageType(path)) charge(path, entries.get(path)); + else if (/^data:image\/(png|jpeg|gif|webp|bmp);/i.test(value)) { + const comma = value.indexOf(','); + const header = value.slice(0, comma); + const body = value.slice(comma + 1); + const raw = /;base64$/i.test(header) + ? atob(body) + : decodeURIComponent(body); + charge( + value, + Uint8Array.from(raw, (c) => c.charCodeAt(0)), + ); + } + } catch { + throw new GeoImportError( + 'That file has an unreadable image or exceeds the 16-megapixel image limit.', + ); + } + } +} + +// Cesium also processes KML in deferred tasks; an exception there does not +// reject its load promise. Keep a failed file from blocking the serial queue. +async function loadKml(doc, viewer, signal) { + let removeError; + try { + return await boundedImport( + Promise.race([ + Cesium.KmlDataSource.load(doc, { + camera: viewer.scene.camera, + canvas: viewer.scene.canvas, + clampToGround: true, + }), + new Promise((_, reject) => { + const failed = () => { + reject(new GeoImportError('That KML could not be read.')); + }; + globalThis.addEventListener?.('error', failed); + removeError = () => globalThis.removeEventListener?.('error', failed); + }), + ]), + signal, + ); + } finally { + removeError?.(); + } +} + +// Resolve a static file's ground pins in batches before they are visualized. +// Per-billboard CLAMP_TO_GROUND subscriptions scale poorly on large imports. +const groundPins = new WeakMap(); +async function settleGroundPins(dataSource, viewer, signal) { + let pins = groundPins.get(dataSource); + if (!pins) { + pins = []; + for (const entity of dataSource.entities.values) { + const reference = entity.billboard?.heightReference?.getValue?.(); + if ( + reference === Cesium.HeightReference.CLAMP_TO_GROUND || + reference === Cesium.HeightReference.RELATIVE_TO_GROUND + ) { + const position = entity.position?.getValue?.(Cesium.JulianDate.now()); + if (!position) continue; + const cartographic = Cesium.Cartographic.fromCartesian(position); + if (!cartographic) + throw new GeoImportError('That file has invalid point coordinates.'); + pins.push({ + entity, + original: position, + cartographic, + offset: + reference === Cesium.HeightReference.RELATIVE_TO_GROUND + ? cartographic.height + : 0, + }); + entity.billboard.heightReference = Cesium.HeightReference.NONE; + entity.billboard.disableDepthTestDistance = Number.POSITIVE_INFINITY; + } + } + groundPins.set(dataSource, pins); + } + for (let at = 0; at < pins.length; at += 256) { + signal?.throwIfAborted(); + const batch = pins.slice(at, at + 256); + const positions = batch.map( + ({ cartographic: c }) => + new Cesium.Cartographic(c.longitude, c.latitude, 0), + ); + const mesh = + viewer.scene?.globe?.show === false && + viewer.scene?.clampToHeightSupported; + // A coarse rendered ellipsoid mesh can report a height kilometres below + // zero. Its provider's exact height is zero even while a 3D map hides it; + // a buried starting point can make the downward mesh ray miss the surface. + const ellipsoid = + viewer.terrainProvider instanceof Cesium.EllipsoidTerrainProvider; + for (let i = 0; i < positions.length; i++) { + const sampled = ellipsoid + ? 0 + : viewer.scene?.globe?.getHeight?.(positions[i]); + positions[i].height = mesh + ? (batch[i].groundHeight ?? + Math.max(0, Number.isFinite(sampled) ? sampled : 0)) + : Number.isFinite(sampled) + ? sampled + : batch[i].groundHeight || 0; + } + const points = positions.map((c) => + Cesium.Cartesian3.fromRadians(c.longitude, c.latitude, c.height), + ); + const height = viewer.camera?.positionCartographic?.height; + if (mesh && Number.isFinite(height) && height < 250_000) { + const rectangle = viewer.camera.computeViewRectangle?.(); + const indices = []; + for (let i = 0; i < positions.length; i++) + if (rectangle && Cesium.Rectangle.contains(rectangle, positions[i])) + indices.push(i); + if (indices.length) { + const resolved = await boundedImport( + viewer.scene.clampToHeightMostDetailed( + indices.map((i) => { + const c = positions[i]; + return Cesium.Cartesian3.fromRadians( + c.longitude, + c.latitude, + Math.max(0, c.height), + ); + }), + ), + signal, + ); + for (let i = 0; i < indices.length; i++) + if (resolved[i]) points[indices[i]] = resolved[i]; + } + } + signal?.throwIfAborted(); + for (let i = 0; i < batch.length; i++) { + const item = batch[i]; + const c = points[i] && Cesium.Cartographic.fromCartesian(points[i]); + if (!c) + throw new GeoImportError( + 'Ground heights are not available for that file. Try again when the map has loaded.', + ); + item.groundHeight = c.height; + c.height += item.offset; + item.entity.position = + Math.abs(c.height - item.cartographic.height) < 1e-6 + ? item.original + : Cesium.Cartesian3.fromRadians(c.longitude, c.latitude, c.height); + } + if (at + batch.length < pins.length) + await new Promise((resolve) => setTimeout(resolve, 0)); + } +} + +async function boundedImport(promise, signal) { + let timer, abort; + try { + signal?.throwIfAborted(); + return await Promise.race([ + promise, + new Promise((_, reject) => { + abort = () => reject(signal.reason); + signal?.addEventListener('abort', abort, { once: true }); + timer = setTimeout( + () => reject(new GeoImportError('That file took too long to read.')), + 60_000, + ); + }), + ]); + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + } +} + /** - * Load one file into a Cesium data source, clamped to the ground. + * Load a ground footprint, retaining explicit KML altitude modes. Call mount after + * adding the source; revoke owns pin and blob cleanup. * @param {File|Blob & {name?: string}} file * @param {{viewer: Cesium.Viewer, color?: string}} options `color` is a CSS color. * @returns {Promise<{dataSource: Cesium.DataSource, name: string, - * format: string, entityCount: number, removed: number, revoke: Function}>} + * format: string, entityCount: number, removed: number, mount: Function, revoke: Function}>} */ -export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { +export async function loadGeoFile( + file, + { viewer, color = '#39d0ff', signal } = {}, +) { + signal = signal + ? AbortSignal.any([signal, AbortSignal.timeout(60_000)]) + : AbortSignal.timeout(60_000); + signal?.throwIfAborted(); if (!file || !Number.isFinite(file.size)) throw new GeoImportError('Nothing to import.'); if (file.size > MAX_IMPORT_BYTES) throw new GeoImportError( - `That file is larger than ${MAX_IMPORT_BYTES / 1024 / 1024} MB.`, + `That file is larger than ${MAX_IMPORT_BYTES / 1024 / 1024} MiB.`, ); - const bytes = new Uint8Array(await file.arrayBuffer()); + let bytes = new Uint8Array(await boundedImport(file.arrayBuffer(), signal)); const format = detectGeoFileFormat(file.name, bytes.subarray(0, 1024)); if (!format) throw new GeoImportError( @@ -965,8 +1257,11 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { ); const stroke = Cesium.Color.fromCssColorString(color); const urls = []; - const revoke = () => + let pins; + const revoke = () => { + pins?.destroy(); urls.splice(0).forEach((url) => URL.revokeObjectURL(url)); + }; const name = importDisplayName(file.name); let dataSource; let removed = 0; @@ -974,56 +1269,62 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { if (format === 'geojson' || format === 'gpx') { let geojson; if (format === 'gpx') { - geojson = gpxToGeoJson( - parseXml(new TextDecoder().decode(bytes), 'GPX'), - ); + geojson = gpxToGeoJson(parseXml(decodeXml(bytes), 'GPX')); } else { geojson = parseGeoJsonText(new TextDecoder().decode(bytes)); - stripGeoJsonStyleProperties(geojson); } const size = countGeoJsonGeometry(geojson); if (size.over) throw overLimitError(size); - dataSource = await Cesium.GeoJsonDataSource.load(geojson, { - clampToGround: true, - stroke, - fill: stroke.withAlpha(0.25), - strokeWidth: 3, - markerColor: stroke, - // No HTML description table is built from the file's properties. - describe: () => undefined, - }); + if (!size.positions) + throw new GeoImportError('That file has nothing to draw.'); + prepareGeoJson(geojson); + bytes = null; + dataSource = await boundedImport( + Cesium.GeoJsonDataSource.load(geojson, { + clampToGround: true, + stroke, + fill: stroke.withAlpha(0.25), + strokeWidth: 3, + markerColor: stroke, + // No HTML description table is built from the file's properties. + describe: () => undefined, + }), + signal, + ); } else { let text; + let entries; + let main; let resolveLocal = () => null; if (format === 'kmz') { - const entries = await readZipEntries(bytes); - const main = kmzMainDocument(entries); + entries = await readZipEntries(bytes); + main = kmzMainDocument(entries); if (!main) throw new GeoImportError('That KMZ holds no KML document.'); - text = new TextDecoder().decode(entries.get(main)); + text = decodeXml(entries.get(main)); resolveLocal = kmzImageResolver(entries, main, (url) => urls.push(url)); } else { - text = new TextDecoder().decode(bytes); + text = decodeXml(bytes); } const doc = parseXml(text, format.toUpperCase()); + text = null; + bytes = null; // Bound geometry before transformations, then count the styles and // image links that survive sanitizing and sharing repeated styles. const size = countKmlGeometry(doc, { maxLinks: Infinity }); if (size.over) throw overLimitError(size); + checkLocalImages(doc, entries, main); removed = sanitizeKmlDocument(doc, resolveLocal).removed; + entries?.clear(); deduplicateKmlStyles(doc); const normalizedSize = countKmlGeometry(doc); if (normalizedSize.over) throw overLimitError(normalizedSize); - dataSource = await Cesium.KmlDataSource.load(doc, { - camera: viewer.scene.camera, - canvas: viewer.scene.canvas, - clampToGround: true, - }); + dataSource = await loadKml(doc, viewer, signal); } } catch (error) { revoke(); if (error instanceof GeoImportError) throw error; throw new GeoImportError( - `That ${format.toUpperCase()} file could not be read${error?.message ? `: ${error.message}` : '.'}`, + `That ${format.toUpperCase()} file could not be read.`, ); } const entityCount = dataSource.entities.values.length; @@ -1033,15 +1334,57 @@ export async function loadGeoFile(file, { viewer, color = '#39d0ff' }) { `That file has ${entityCount.toLocaleString()} map objects; the limit is ${MAX_IMPORT_ENTITIES.toLocaleString()}.`, ); } - if (entityCount === 0) { + const now = Cesium.JulianDate.now(); + const drawable = dataSource.entities.values.some( + (e) => + e.billboard?.image?.getValue(now) || + e.point || + e.rectangle || + e.polyline?.positions?.getValue(now)?.length >= 2 || + e.wall?.positions?.getValue(now)?.length >= 2 || + e.polygon?.hierarchy?.getValue(now)?.positions?.length >= 3, + ); + if (!drawable) { revoke(); throw new GeoImportError('That file has nothing to draw.'); } scrubImportedEntities(dataSource); + try { + await settleGroundPins(dataSource, viewer, signal); + } catch (error) { + revoke(); + throw error instanceof GeoImportError + ? error + : new GeoImportError( + 'Ground heights could not be read. Try that file again.', + ); + } // Keep the clock invariant explicit even though unused KML dates have // already been stripped and all supported geometries are static. detachImportedTime(dataSource); + try { + pins = createImportedPins(dataSource, viewer); + } catch { + revoke(); + throw new GeoImportError('That file has invalid point graphics.'); + } // The display name travels beside the data source (the file name, not // anything the document says about itself). - return { dataSource, name, format, entityCount, removed, revoke }; + return { + dataSource, + name, + format, + entityCount, + removed, + revoke, + mount: () => pins.mount(), + refreshGround: (refreshSignal) => + settleGroundPins( + dataSource, + viewer, + refreshSignal + ? AbortSignal.any([refreshSignal, AbortSignal.timeout(15_000)]) + : AbortSignal.timeout(15_000), + ).then(() => pins.sync()), + }; } diff --git a/src/data/geoFileImport.test.mjs b/src/data/geoFileImport.test.mjs index 2769fbe44..af02ec222 100644 --- a/src/data/geoFileImport.test.mjs +++ b/src/data/geoFileImport.test.mjs @@ -4,6 +4,7 @@ import { deflateRawSync } from 'node:zlib'; import * as Cesium from 'cesium'; import { GeoImportError, + loadGeoFile, MAX_IMPORT_POSITIONS, MAX_IMPORT_ENTITIES, countGeoJsonGeometry, @@ -14,7 +15,6 @@ import { importFeatureName, scrubImportedEntities, sanitizeKmlDocument, - stripGeoJsonStyleProperties, detectGeoFileFormat, importDisplayName, kmzImageResolver, @@ -24,6 +24,298 @@ import { readZipEntries, resolveKmzPath, } from './geoFileImport.js'; +import { createImportImageBudget, importImageSize } from './geoImportImages.js'; + +test('real loader grounds XYZ GeoJSON and strips loader-sensitive metadata before allocation', async () => { + const previous = Cesium.PinBuilder.prototype.fromColor; + Cesium.PinBuilder.prototype.fromColor = () => ({ + toDataURL: () => 'data:image/png;base64,local', + tagName: 'CANVAS', + }); + try { + const object = { + type: 'FeatureCollection', + crs: { type: 'name', properties: { name: 'unsupported' } }, + features: [ + { + type: 'Feature', + id: 'kartaview-owned', + properties: { name: 'Test', addProperty: 1, 'marker-symbol': 'rail' }, + geometry: { type: 'Point', coordinates: [1, 2, 999] }, + }, + { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [1, 2, 0], + [2, 2, 0], + [2, 3, 0], + [1, 2, 0], + ], + ], + }, + }, + ], + }; + const loaded = await loadGeoFile( + new File([JSON.stringify(object)], 'xyz.geojson'), + { + viewer: { + scene: { + frameState: { mode: Cesium.SceneMode.SCENE3D }, + primitives: new Cesium.PrimitiveCollection(), + globe: { getHeight: () => 1234 }, + }, + }, + }, + ); + const [point, polygon] = + loaded.dataSource.importedEntities || loaded.dataSource.entities.values; + assert.notEqual(point.id, 'kartaview-owned'); + assert.equal(point.name, 'Test'); + assert.equal(point.properties, undefined); + assert.equal( + point.billboard.heightReference.getValue(), + Cesium.HeightReference.NONE, + ); + assert.equal(point.billboard.disableDepthTestDistance.getValue(), Infinity); + assert.ok( + Math.abs( + Cesium.Cartographic.fromCartesian( + point.position.getValue(Cesium.JulianDate.now()), + ).height - 1234, + ) < 1e-6, + ); + assert.notEqual(polygon.polygon.perPositionHeight?.getValue(), true); + assert.equal(loaded.dataSource.clock, undefined); + assert.equal( + loaded.dataSource.entities.values.length, + 1, + 'only polygon geometry remains in the per-frame entity collection', + ); + assert.equal(loaded.dataSource.importedBillboards.length, 1); + const update = Cesium.BillboardCollection.prototype.update; + let passes = 0; + Cesium.BillboardCollection.prototype.update = () => passes++; + try { + const collection = loaded.dataSource.importedBillboards; + collection.update({ passes: { pick: true, offscreen: true } }); + assert.equal(passes, 0, 'height picks ignore imported markers'); + collection.update({ passes: { pick: true, offscreen: false } }); + collection.update({ passes: { render: true, offscreen: true } }); + assert.equal(passes, 2, 'ordinary picks and screenshots include markers'); + } finally { + Cesium.BillboardCollection.prototype.update = update; + } + + loaded.mount(); + loaded.dataSource.show = false; + assert.equal(loaded.dataSource.importedBillboards.show, false); + loaded.dataSource.show = true; + await loaded.refreshGround(); + assert.equal( + point.billboard.heightReference.getValue(), + Cesium.HeightReference.NONE, + ); + loaded.revoke(); + assert.ok(loaded.dataSource.importedBillboards.isDestroyed()); + } finally { + Cesium.PinBuilder.prototype.fromColor = previous; + } +}); + +test('real loader refuses empty files and malformed nested geometry before allocating entities', async () => { + for (const object of [ + { type: 'Feature', geometry: null }, + { + type: 'FeatureCollection', + features: [{ geometry: { type: 'Point', coordinates: [1, 2] } }], + }, + { type: 'Feature', geometry: { type: 'Topology', arcs: [] } }, + ...[ + [1, '2'], + [Infinity, 0], + [0, 91], + [0, 0, 1e100], + ].map((coordinates) => ({ type: 'Point', coordinates })), + ]) + await assert.rejects( + loadGeoFile(new File([JSON.stringify(object)], 'bad.geojson'), { + viewer: {}, + }), + GeoImportError, + ); +}); + +test('KML tracks retain their explicit altitude and ground lines are tessellated', () => { + const track = xmlEl('Track', {}, [ + xmlEl('altitudeMode', {}, 'absolute'), + xmlEl('coord', {}, '1 2 10000'), + xmlEl('coord', {}, '2 3 10000'), + ]); + const line = xmlEl('LineString', {}, [xmlEl('coordinates', {}, '1,2 2,3')]); + const ground = xmlEl('Track', {}, [ + xmlEl('coord', {}, '1 2 0'), + xmlEl('coord', {}, '2 3 0'), + ]); + const root = xmlEl('kml', {}, [track, line, ground]); + sanitizeKmlDocument(xmlDoc(root)); + assert.equal( + root.childNodes[0].childNodes.find((n) => n.localName === 'altitudeMode') + .textContent, + 'absolute', + ); + assert.equal( + root.childNodes[1].childNodes.find((n) => n.localName === 'tessellate') + .textContent, + '1', + ); + assert.equal( + root.childNodes[2].childNodes.find((n) => n.localName === 'tessellate') + .textContent, + '1', + ); +}); + +test('sanitizer normalizes styles, removes incomplete pairs and keeps colored overlays', () => { + const local = xmlEl('styleUrl', {}, ' #s '); + const pair = xmlEl('Pair', {}, [ + xmlEl('key', {}, 'normal'), + xmlEl('styleUrl', {}, 'https://example.invalid/s'), + ]); + const overlay = xmlEl('GroundOverlay', {}, [xmlEl('color', {}, 'ff00ff00')]); + const hidden = xmlEl('visibility', {}, '0'); + const id = xmlEl('Placemark', { id: 'kartaview-123' }); + const root = xmlEl('kml', {}, [ + local, + xmlEl('StyleMap', { id: 's' }, [pair]), + overlay, + hidden, + id, + xmlEl('toString'), + ]); + sanitizeKmlDocument(xmlDoc(root)); + assert.equal(local.textContent, '#s'); + assert.equal(pair.parentNode, null); + assert.ok(root.childNodes.includes(overlay)); + assert.equal(hidden.parentNode, null); + assert.equal(id.getAttribute('id'), null); + assert.ok(!root.childNodes.some((n) => n.localName === 'toString')); +}); + +test('image budget rejects large decoded rasters before decoding and shares repeated images', () => { + const png = (w, h) => { + const b = new Uint8Array(24); + const v = new DataView(b.buffer); + v.setUint32(0, 0x89504e47); + b.set([73, 72, 68, 82], 12); + v.setUint32(16, w); + v.setUint32(20, h); + return b; + }; + assert.deepEqual(importImageSize(png(4096, 4096)), [4096, 4096]); + const charge = createImportImageBudget(); + charge('shared', png(4096, 4096)); + charge('shared', png(4096, 4096)); + assert.throws(() => charge('other', png(1, 1)), /limit/); + assert.throws( + () => createImportImageBudget()('huge', png(16384, 16384)), + /limit/, + ); +}); + +test('pending file reads can be cancelled without waiting for the loader timeout', async () => { + const controller = new AbortController(); + const file = { + name: 'waiting.geojson', + size: 1, + arrayBuffer: () => new Promise(() => {}), + }; + const promise = loadGeoFile(file, { viewer: {}, signal: controller.signal }); + controller.abort(new Error('cancelled by teardown')); + await assert.rejects(promise, /cancelled by teardown/); +}); + +test('a hidden coarse globe never seeds a mesh-clamp ray below the surface', async () => { + const previous = Cesium.PinBuilder.prototype.fromColor; + Cesium.PinBuilder.prototype.fromColor = () => ({ + tagName: 'CANVAS', + toDataURL: () => 'data:image/png;base64,local', + }); + const starts = []; + try { + for (const terrainProvider of [new Cesium.EllipsoidTerrainProvider(), {}]) { + const viewer = { + terrainProvider, + camera: { + positionCartographic: { height: 100000 }, + computeViewRectangle: () => Cesium.Rectangle.MAX_VALUE, + }, + scene: { + frameState: { mode: Cesium.SceneMode.SCENE3D }, + globe: { show: false, getHeight: () => -12311 }, + clampToHeightSupported: true, + clampToHeightMostDetailed: async (points) => + points.map((p) => { + const c = Cesium.Cartographic.fromCartesian(p); + starts.push(c.height); + return c.height < -9000 + ? undefined + : Cesium.Cartesian3.fromRadians(c.longitude, c.latitude, 2000); + }), + }, + }; + const loaded = await loadGeoFile( + new File( + [JSON.stringify({ type: 'Point', coordinates: [1, 2] })], + 'surface.geojson', + ), + { viewer }, + ); + assert.ok( + Math.abs(starts.at(-1)) < 1e-6, + `ray started at ${starts.at(-1)}`, + ); + assert.ok( + Math.abs( + Cesium.Cartographic.fromCartesian( + loaded.dataSource.importedBillboards.get(0).position, + ).height - 2000, + ) < 1e-6, + ); + const sampled = starts.length; + viewer.camera.computeViewRectangle = () => undefined; + await loaded.refreshGround(); + assert.equal( + starts.length, + sampled, + 'distant points do not start a detailed mesh sweep', + ); + assert.ok( + Math.abs( + Cesium.Cartographic.fromCartesian( + loaded.dataSource.importedBillboards.get(0).position, + ).height - 2000, + ) < 1e-6, + 'the last valid mesh height survives outside the camera footprint', + ); + const point = loaded.dataSource.importedEntities[0]; + const other = new Cesium.Entity({ id: 'unrelated-aircraft' }); + viewer.trackedEntity = starts.length === 1 ? other : point; + viewer.selectedEntity = point; + loaded.revoke(); + assert.equal(viewer.selectedEntity, undefined); + assert.equal( + viewer.trackedEntity, + starts.length === 1 ? other : undefined, + ); + } + } finally { + Cesium.PinBuilder.prototype.fromColor = previous; + } +}); const bytes = (text) => new TextEncoder().encode(text); @@ -756,30 +1048,6 @@ test('a KMZ image gets one blob URL however many hrefs name it', () => { assert.deepEqual(kept, ['blob:test/1', 'blob:test/2']); }); -test('simplestyle is dropped: no icon fetched, the row color is what draws', () => { - const fc = { - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - properties: { - 'marker-symbol': 'rail', - 'marker-color': '#f00', - stroke: '#0f0', - 'stroke-width': 9, - fill: '#00f', - 'fill-opacity': 1, - name: 'x', - }, - geometry: null, - }, - { type: 'Feature', properties: null, geometry: null }, - ], - }; - stripGeoJsonStyleProperties(fc); - assert.deepEqual(fc.features[0].properties, { name: 'x' }); -}); - test('imported entities keep only a short plain-text name: no description, label or properties', () => { const ds = new Cesium.CustomDataSource('t'); const e = ds.entities.add({ diff --git a/src/data/geoImportImages.js b/src/data/geoImportImages.js new file mode 100644 index 000000000..86f8e98e2 --- /dev/null +++ b/src/data/geoImportImages.js @@ -0,0 +1,74 @@ +// Inspect encoded headers before allowing the browser to decode local images. +export const MAX_IMPORT_IMAGE_PIXELS = 16 * 1024 * 1024; +const MAX_SIDE = 8192; + +/** Width and height from a supported raster header; no image is decoded. */ +export function importImageSize(bytes) { + const v = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const ascii = (at, n) => String.fromCharCode(...bytes.subarray(at, at + n)); + const u24 = (at) => bytes[at] | (bytes[at + 1] << 8) | (bytes[at + 2] << 16); + if ( + bytes.length >= 24 && + v.getUint32(0) === 0x89504e47 && + ascii(12, 4) === 'IHDR' + ) + return [v.getUint32(16), v.getUint32(20)]; + if (bytes.length >= 10 && /^GIF8[79]a$/.test(ascii(0, 6))) + return [v.getUint16(6, true), v.getUint16(8, true)]; + if (bytes.length >= 26 && ascii(0, 2) === 'BM') { + if (v.getUint32(14, true) === 12) + return [v.getUint16(18, true), v.getUint16(20, true)]; + return [Math.abs(v.getInt32(18, true)), Math.abs(v.getInt32(22, true))]; + } + if (bytes.length >= 30 && ascii(0, 4) === 'RIFF' && ascii(8, 4) === 'WEBP') { + const kind = ascii(12, 4); + if (kind === 'VP8X') return [1 + u24(24), 1 + u24(27)]; + if (kind === 'VP8 ' && u24(23) === 0x2a019d) + return [v.getUint16(26, true) & 0x3fff, v.getUint16(28, true) & 0x3fff]; + if (kind === 'VP8L' && bytes[20] === 0x2f) { + const bits = v.getUint32(21, true); + return [(bits & 0x3fff) + 1, ((bits >>> 14) & 0x3fff) + 1]; + } + } + if (bytes.length >= 4 && v.getUint16(0) === 0xffd8) { + for (let at = 2; at + 4 <= bytes.length;) { + if (bytes[at++] !== 0xff) break; + while (bytes[at] === 0xff) at++; + const marker = bytes[at++]; + if (marker === 0xd9 || marker === 0xda) break; + if (marker === 1 || (marker >= 0xd0 && marker <= 0xd7)) continue; + const length = v.getUint16(at); + if (length < 2 || at + length > bytes.length) break; + if ( + [ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, + 0xce, 0xcf, + ].includes(marker) && + length >= 8 + ) + return [v.getUint16(at + 5), v.getUint16(at + 3)]; + at += length; + } + } + return null; +} + +/** Charge each distinct encoded image once against the file's decoded budget. */ +export function createImportImageBudget() { + const seen = new Set(); + let pixels = 0; + return (key, bytes) => { + if (seen.has(key)) return; + const size = importImageSize(bytes); + if ( + !size || + size.some((n) => !Number.isInteger(n) || n <= 0 || n > MAX_SIDE) || + pixels + size[0] * size[1] > MAX_IMPORT_IMAGE_PIXELS + ) + throw new Error( + 'Local images exceed the decoded image limit or have an unreadable header.', + ); + pixels += size[0] * size[1]; + seen.add(key); + }; +} diff --git a/src/data/geoImportPins.js b/src/data/geoImportPins.js new file mode 100644 index 000000000..1828b5df7 --- /dev/null +++ b/src/data/geoImportPins.js @@ -0,0 +1,128 @@ +import * as Cesium from 'cesium'; + +const STYLE = [ + 'image', + 'scale', + 'rotation', + 'alignedAxis', + 'horizontalOrigin', + 'verticalOrigin', + 'color', + 'eyeOffset', + 'pixelOffset', + 'width', + 'height', + 'scaleByDistance', + 'translucencyByDistance', + 'pixelOffsetScaleByDistance', + 'distanceDisplayCondition', + 'sizeInMeters', + 'disableDepthTestDistance', + 'imageSubRegion', +]; + +class ImportedBillboards extends Cesium.BillboardCollection { + update(frameState) { + // Height queries should find the map, never another imported marker. + // Ordinary click picks and rendered screenshots still include the pins. + if (frameState.passes.pick && frameState.passes.offscreen) return; + super.update(frameState); + } +} + +/** Own static imported pins without a per-frame Entity visualizer walk. */ +export function createImportedPins(dataSource, viewer) { + const all = dataSource.entities.values.slice(); + const now = Cesium.JulianDate.now(); + const pins = all.filter((entity) => entity.billboard && entity.position); + if (!pins.length) return { mount() {}, sync() {}, destroy() {} }; + const collection = new ImportedBillboards({ scene: viewer.scene }); + const billboards = []; + try { + for (const entity of pins) { + const position = entity.position.getValue(now); + if ( + !position || + ![position.x, position.y, position.z].every(Number.isFinite) + ) + throw new Error('Invalid imported point position.'); + const options = { + id: entity, + position, + show: + entity.isShowing && entity.billboard.show?.getValue(now) !== false, + heightReference: Cesium.HeightReference.NONE, + }; + for (const key of STYLE) { + const value = entity.billboard[key]?.getValue(now); + if (value !== undefined) options[key] = value; + } + billboards.push(collection.add(options)); + } + } catch (error) { + collection.destroy(); + throw error; + } + // Rebuild once before the source is added to DataSourceDisplay. Removing + // one entity at a time would itself make this transition quadratic. + dataSource.entities.suspendEvents(); + try { + dataSource.entities.removeAll(); + for (const entity of all) + if (!entity.billboard || !entity.position) + dataSource.entities.add(entity); + } finally { + dataSource.entities.resumeEvents(); + } + Object.defineProperty(dataSource, 'importedEntities', { + value: all, + configurable: true, + }); + Object.defineProperty(dataSource, 'importedBillboards', { + value: collection, + configurable: true, + }); + let show = dataSource.show, + mounted = false, + destroyed = false; + Object.defineProperty(dataSource, 'show', { + configurable: true, + get: () => show, + set(value) { + show = Boolean(value); + dataSource.entities.show = show; + collection.show = show; + }, + }); + collection.show = show; + return { + mount() { + if (destroyed || mounted) return; + viewer.scene.primitives?.add(collection); + mounted = true; + }, + sync() { + if (destroyed) return; + for (let i = 0; i < pins.length; i++) { + billboards[i].position = pins[i].position.getValue(now); + billboards[i].show = + pins[i].isShowing && pins[i].billboard.show?.getValue(now) !== false; + } + viewer.scene.requestRender?.(); + }, + destroy() { + if (destroyed) return; + destroyed = true; + if (viewer.trackedEntity && pins.includes(viewer.trackedEntity)) + viewer.trackedEntity = undefined; + if (viewer.selectedEntity && pins.includes(viewer.selectedEntity)) + viewer.selectedEntity = undefined; + if (mounted && !viewer.scene.primitives?.isDestroyed?.()) + viewer.scene.primitives?.remove(collection); + if (!collection.isDestroyed()) collection.destroy(); + all.length = 0; + pins.length = 0; + billboards.length = 0; + }, + }; +} diff --git a/src/locations.js b/src/locations.js index 02bd853fb..035f8114c 100644 --- a/src/locations.js +++ b/src/locations.js @@ -1,3 +1,4 @@ +import { rangeForBoundingSphere } from './data/cameraFraming.js'; import { isUnavailableCapability } from './sources/capability.js'; import { createOverpassFeatureSource } from './sources/overpassFeatures.js'; import { applicationServices } from './services/application.js'; @@ -1377,19 +1378,6 @@ function buildingBoundingRadius(bounds) { return Math.hypot(halfHeight, halfWidth, halfDepth) * 1.18; } -function rangeForBoundingSphere(viewer, radius) { - const frustum = viewer.camera.frustum; - const verticalFov = Number(frustum?.fov) || Cesium.Math.toRadians(60); - const aspectRatio = Math.max(0.5, Number(frustum?.aspectRatio) || 1); - const horizontalFov = 2 * Math.atan(Math.tan(verticalFov / 2) * aspectRatio); - const limitingFov = Math.min(verticalFov, horizontalFov); - // Target 57% sphere occupancy so perspective still leaves at least 40% - // measured roof-to-base breathing room in ordinary oblique building views. - const occupiedViewportFraction = 0.57; - const desiredAngularRadius = (limitingFov * occupiedViewportFraction) / 2; - return (radius / Math.sin(desiredAngularRadius)) * 1.05; -} - function buildingPitch(bounds) { if (!bounds) return -25; const footprint = Math.max(bounds.width, bounds.depth); diff --git a/src/ui/flyToDataSource.js b/src/ui/flyToDataSource.js index f7358dc2b..bf4b9336a 100644 --- a/src/ui/flyToDataSource.js +++ b/src/ui/flyToDataSource.js @@ -1,92 +1,122 @@ -/** - * Fly to everything in an imported file without - * trusting Cesium's own framing of ground-clamped points: before the - * destination's terrain has streamed in, a clamped billboard reports a - * bounding sphere kilometers below the surface, and `viewer.flyTo` follows it - * underground. Instead the footprint is framed from the features' longitudes - * and latitudes, and the camera ground guard lifts the eye once the real - * surface can be measured. - */ +/** Frame the complete finite footprint of a session-local file. */ import * as Cesium from 'cesium'; import { guardCameraAboveGround } from '../cameraGroundGuard.js'; +import { rangeForBoundingSphere } from '../data/cameraFraming.js'; +import { cancelCameraArrival } from '../data/cameraArrival.js'; -const SAMPLE_TARGET_POINTS = 5000; - -/** - * Longitude/latitude pairs of every entity's position, line and polygon, - * sampling long paths toward `SAMPLE_TARGET_POINTS` while retaining each - * geometry's endpoints. Separate point features stay in the footprint so - * distant outliers are not lost. Long tracks are walked by index, never - * spread into a call, so a file with hundreds of thousands of vertices cannot - * overflow the stack; only the kept points are converted. - */ +/** All visible positions, path vertices, polygon rings, walls and overlay bounds. */ export function dataSourcePoints(dataSource, time = Cesium.JulianDate.now()) { - const parts = []; - let total = 0; - for (const entity of dataSource?.entities?.values ?? []) { - const at = entity.position?.getValue?.(time); - if (at) { - parts.push([at]); - total += 1; - } - const line = entity.polyline?.positions?.getValue?.(time); - if (Array.isArray(line) && line.length) { - parts.push(line); - total += line.length; - } - const ring = entity.polygon?.hierarchy?.getValue?.(time)?.positions; - if (Array.isArray(ring) && ring.length) { - parts.push(ring); - total += ring.length; - } - } - const stride = Math.max(1, Math.ceil(total / SAMPLE_TARGET_POINTS)); const points = []; - let index = 0; - for (const part of parts) { - // Keep every stride-th vertex across all parts, plus each part's ends so - // a short line is never skipped entirely. - for (let i = 0; i < part.length; i++, index++) { - if (index % stride !== 0 && i !== 0 && i !== part.length - 1) continue; - const carto = Cesium.Cartographic.fromCartesian(part[i]); - if (!carto) continue; + const add = (position, grounded = false) => { + if ( + !position || + ![position.x, position.y, position.z].every(Number.isFinite) + ) + return; + const c = Cesium.Cartographic.fromCartesian(position); + if (c && Number.isFinite(c.longitude) && Number.isFinite(c.latitude)) points.push([ - Cesium.Math.toDegrees(carto.longitude), - Cesium.Math.toDegrees(carto.latitude), + Cesium.Math.toDegrees(c.longitude), + Cesium.Math.toDegrees(c.latitude), + grounded ? 0 : c.height, ]); + }; + if (dataSource?.show === false) return points; + for (const entity of dataSource?.importedEntities ?? + dataSource?.entities?.values ?? + []) { + if (entity.isShowing === false) continue; + add( + entity.position?.getValue?.(time), + entity.billboard?.heightReference?.getValue?.(time) === + Cesium.HeightReference.CLAMP_TO_GROUND, + ); + for (const graphic of [entity.polyline, entity.wall]) + for (const position of graphic?.positions?.getValue?.(time) || []) + add(position, graphic.clampToGround?.getValue?.(time) === true); + const rings = [entity.polygon?.hierarchy?.getValue?.(time)]; + while (rings.length) { + const ring = rings.pop(); + for (const position of ring?.positions || []) + add( + position, + entity.polygon?.perPositionHeight?.getValue?.(time) !== true, + ); + for (const hole of ring?.holes || []) rings.push(hole); + } + const rectangle = entity.rectangle?.coordinates?.getValue?.(time); + if (rectangle) { + const bounds = Cesium.RectangleGeometry.computeRectangle({ + rectangle, + rotation: entity.rectangle.rotation?.getValue?.(time) || 0, + }); + for (const corner of [ + 'southwest', + 'southeast', + 'northwest', + 'northeast', + ]) { + const c = Cesium.Rectangle[corner](bounds); + if (Number.isFinite(c.longitude) && Number.isFinite(c.latitude)) + points.push([ + Cesium.Math.toDegrees(c.longitude), + Cesium.Math.toDegrees(c.latitude), + entity.rectangle.height?.getValue?.(time) || 0, + ]); + } } } return points; } -/** - * @param {Cesium.Viewer} viewer - * @param {Cesium.DataSource} dataSource - * @param {{duration?: number}} [options] - * @returns {boolean} Whether there was anything to fly to. - */ -export function flyToDataSource(viewer, dataSource, { duration = 1.5 } = {}) { +/** Fly synchronously to the footprint; later navigation always owns its camera. */ +export function flyToDataSource( + viewer, + dataSource, + { duration = 1.5, ground = null } = {}, +) { const points = dataSourcePoints(dataSource); - if (!points.length) { - viewer.flyTo?.(dataSource, { duration }); - return false; - } - const sphere = Cesium.BoundingSphere.fromPoints( - points.map(([lon, lat]) => Cesium.Cartesian3.fromDegrees(lon, lat, 0)), + if (!points.length) return false; + cancelCameraArrival(viewer); + const positions = points.map(([lon, lat, height]) => + Cesium.Cartesian3.fromDegrees(lon, lat, height), ); - const center = Cesium.Cartographic.fromCartesian(sphere.center); + const sphere = Cesium.BoundingSphere.fromPoints(positions); + const center = + Cesium.Cartographic.fromCartesian(sphere.center) || + Cesium.Cartographic.fromDegrees(points[0][0], points[0][1]); + const lat = Cesium.Math.toDegrees(center.latitude), + lon = Cesium.Math.toDegrees(center.longitude); + const terrain = viewer.scene?.globe?.getHeight?.(center); + const cached = ground?.cachedGroundFloor?.(lat, lon); + const height = + Number.isFinite(terrain) && terrain > 0 + ? terrain + : Number.isFinite(cached) + ? cached + : Number.isFinite(terrain) + ? terrain + : 0; + ground?.warmGroundFloor?.([{ lat, lon }]); + sphere.center = Cesium.Cartesian3.fromRadians( + center.longitude, + center.latitude, + Math.max(0, height, center.height), + ); + sphere.radius = 0; + for (const position of positions) + sphere.radius = Math.max( + sphere.radius, + Cesium.Cartesian3.distance(sphere.center, position), + ); viewer.camera?.flyToBoundingSphere?.(sphere, { offset: new Cesium.HeadingPitchRange( viewer.camera.heading || 0, - Cesium.Math.toRadians(-45), - Math.max(800, sphere.radius * 2.8), + -Math.PI / 2, + Math.max(800, rangeForBoundingSphere(viewer, sphere.radius)), ), duration, - complete: () => - guardCameraAboveGround(viewer, { - lat: Cesium.Math.toDegrees(center.latitude), - lon: Cesium.Math.toDegrees(center.longitude), - }), + complete: () => guardCameraAboveGround(viewer, { lat, lon }), }); return true; } diff --git a/src/ui/flyToDataSource.test.mjs b/src/ui/flyToDataSource.test.mjs index 926a9511e..62f6073a6 100644 --- a/src/ui/flyToDataSource.test.mjs +++ b/src/ui/flyToDataSource.test.mjs @@ -1,6 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import * as Cesium from 'cesium'; +import { cancelCameraArrival } from '../data/cameraArrival.js'; import { dataSourcePoints, flyToDataSource } from './flyToDataSource.js'; function source() { @@ -55,7 +56,11 @@ test('the camera frames the footprint at the surface, never a buried center', () flyToDataSource(viewer, new Cesium.CustomDataSource('empty')), false, ); - assert.equal(calls.at(-1), 'viewer.flyTo'); + assert.equal( + calls.length, + 1, + 'empty sources never start deferred navigation', + ); }); test('a very long track is framed without spreading it into a call', () => { @@ -69,8 +74,90 @@ test('a very long track is framed without spreading it into a call', () => { ds.entities.add({ polyline: { positions: line } }); ds.entities.add({ position: Cesium.Cartesian3.fromDegrees(-110, 41) }); const points = dataSourcePoints(ds); - assert.ok(points.length <= 5002, `${points.length} points kept`); + assert.equal(points.length, n + 1, 'every vertex contributes to framing'); const lons = points.map(([lon]) => lon); - assert.ok(Math.min(...lons) < -119.99 && Math.max(...lons) > -110.01); + assert.ok( + lons.reduce((a, b) => Math.min(a, b), Infinity) < -119.99 && + lons.reduce((a, b) => Math.max(a, b), -Infinity) > -110.01, + ); assert.ok(lons.some((lon) => Math.abs(lon - -118.000005) < 0.01)); }); + +test('framing includes a sparse interior outlier, walls and rotated overlays', () => { + const ds = new Cesium.CustomDataSource('bounds'); + const line = Array.from({ length: 200031 }, () => + Cesium.Cartesian3.fromDegrees(0, 0), + ); + line[12345] = Cesium.Cartesian3.fromDegrees(20, 10); + ds.entities.add({ polyline: { positions: line } }); + ds.entities.add({ + wall: { positions: Cesium.Cartesian3.fromDegreesArray([30, 10, 31, 11]) }, + }); + ds.entities.add({ + rectangle: { + coordinates: Cesium.Rectangle.fromDegrees(40, 0, 41, 1), + rotation: Math.PI / 4, + }, + }); + const points = dataSourcePoints(ds); + assert.ok(points.some(([lon]) => Math.abs(lon - 20) < 1e-6)); + assert.ok(points.some(([lon]) => Math.abs(lon - 31) < 1e-6)); + assert.ok(points.some(([lon]) => lon > 41)); +}); + +test('portrait framing accounts for FOV, high ground and symmetric extents', () => { + const ds = new Cesium.CustomDataSource('view'); + ds.entities.add({ + polyline: { + positions: Cesium.Cartesian3.fromDegreesArray([-105, 39, -104.9, 39]), + }, + }); + const calls = []; + const viewer = { + scene: { globe: { getHeight: () => 2600 } }, + camera: { + frustum: { fov: Math.PI / 3, aspectRatio: 390 / 844 }, + flyToBoundingSphere: (sphere, options) => calls.push({ sphere, options }), + }, + }; + flyToDataSource(viewer, ds); + assert.ok( + Math.abs( + Cesium.Cartographic.fromCartesian(calls[0].sphere.center).height - 2600, + ) < 1e-6, + ); + assert.equal(calls[0].options.offset.pitch, -Math.PI / 2); + viewer.camera.frustum.aspectRatio = 16 / 9; + flyToDataSource(viewer, ds); + assert.ok(calls[0].options.offset.range > calls[1].options.offset.range); + const symmetric = { + entities: { + values: [ + { position: { getValue: () => new Cesium.Cartesian3(6378137, 0, 0) } }, + { position: { getValue: () => new Cesium.Cartesian3(-6378137, 0, 0) } }, + ], + }, + }; + assert.doesNotThrow(() => flyToDataSource(viewer, symmetric)); + assert.ok(Number.isFinite(calls.at(-1).options.offset.range)); + assert.doesNotThrow(() => calls.at(-1).options.complete()); + cancelCameraArrival(viewer); +}); + +test('non-finite positions and hidden features never enter the camera footprint', () => { + const position = (p) => ({ position: { getValue: () => p } }); + const points = dataSourcePoints({ + entities: { + values: [ + position(new Cesium.Cartesian3(NaN, 1, 2)), + { + ...position(Cesium.Cartesian3.fromDegrees(10, 10)), + isShowing: false, + }, + position(Cesium.Cartesian3.fromDegrees(20, 20)), + ], + }, + }); + assert.equal(points.length, 1); + assert.ok(Math.abs(points[0][0] - 20) < 1e-6); +}); diff --git a/src/ui/geoFileImport.js b/src/ui/geoFileImport.js index 1a5992893..ae3266608 100644 --- a/src/ui/geoFileImport.js +++ b/src/ui/geoFileImport.js @@ -1,24 +1,4 @@ -/** - * DISPLAY ▸ Import: put your own GeoJSON, KML, KMZ or GPX files on the globe. - * - * Files come in through the Import button or by dropping them onto the globe. - * Each one becomes its own Cesium data source, clamped to the ground, with a - * row in the panel to show or hide it, fly to it, or remove it. Files are read - * in the browser and never uploaded; see `src/data/geoFileImport.js` for what - * is accepted and what is stripped from KML before Cesium reads it. - * - * Imported files last for the page session: a reload starts empty. - * - * Imports run one batch at a time (a second pick or drop waits for the - * first), so the file cap is checked against what is really on the globe. - * Fly-to goes through the app's navigation policy (`navigation`, the - * application shell): the button is an immediate navigation, and the fly - * after an import is a deferred one, stamped when the import starts and - * re-asserted just before flying, so a camera move, Cockpit or any newer - * navigation in between wins over a slow file. - * `destroy()` removes every imported data source, revokes every blob URL and - * gives back every listener and the `window.__gevGeoImport` handle. - */ +/** Session-local file controls, serial loading and application-owned navigation. */ import { GEO_FILE_ACCEPT, GeoImportError, @@ -44,6 +24,8 @@ export function initGeoFileImport({ document: doc = globalThis.document, flyToSource = flyToDataSource, navigation = null, + ground = null, + subscribeMapSource = null, }) { const button = doc?.getElementById('geo-import-button'); const input = doc?.getElementById('geo-import-input'); @@ -58,6 +40,9 @@ export function initGeoFileImport({ let colorAt = 0; let destroyed = false; let dragDepth = 0; + let cameraIntent = 0; + const lifetime = new AbortController(); + let removeMapSource; const listen = (target, type, listener, options) => { if (!target?.addEventListener) return; @@ -68,6 +53,52 @@ export function initGeoFileImport({ if (hint) hint.textContent = text || ''; }; const render = () => viewer.scene?.requestRender?.(); + let groundTimer, + groundQueued = false, + groundAgain = false; + const groundRemovers = []; + const scheduleGround = () => { + if (destroyed || !imports.length) return; + if (groundQueued) { + groundAgain = true; + return; + } + clearTimeout(groundTimer); + groundTimer = setTimeout(() => { + groundQueued = true; + queue = queue + .catch(() => {}) + .then(async () => { + try { + for (const item of imports) { + if (destroyed) return; + if (item.visible) await item.refreshGround?.(lifetime.signal); + } + render(); + } catch { + if (!destroyed) + setHint( + 'Ground heights could not be refreshed. Try the file again when the map has loaded.', + ); + } finally { + groundQueued = false; + if (groundAgain) { + groundAgain = false; + scheduleGround(); + } + } + }); + }, 200); + }; + removeMapSource = subscribeMapSource?.(scheduleGround); + const ownEvent = (event, listener) => { + const remove = event?.addEventListener?.(listener); + if (remove) groundRemovers.push(remove); + }; + ownEvent(viewer.camera?.moveEnd, scheduleGround); + ownEvent(viewer.scene?.globe?.tileLoadProgressEvent, (pending) => { + if (!pending) scheduleGround(); + }); /** Which row button has focus, so a re-render can give it back. */ function focusedAction() { @@ -108,7 +139,7 @@ export function initGeoFileImport({ name.className = 'geo-import-name'; // textContent, never markup: the name comes from the file system. name.textContent = item.name; - name.title = `${item.name} — ${item.format.toUpperCase()}, ${item.entityCount.toLocaleString()} feature${item.entityCount === 1 ? '' : 's'}${item.ownStyles ? ', drawn with its own styles' : ''}`; + name.title = `${item.name} — ${item.format.toUpperCase()}, ${item.entityCount.toLocaleString()} map object${item.entityCount === 1 ? '' : 's'}${item.ownStyles ? ', drawn with its own styles' : ''}`; const actions = [ [ item.visible ? 'visibility' : 'visibility_off', @@ -149,6 +180,7 @@ export function initGeoFileImport({ if (!item) return false; item.visible = Boolean(visible); item.dataSource.show = item.visible; + if (item.visible) scheduleGround(); renderList(); render(); return true; @@ -158,7 +190,7 @@ export function initGeoFileImport({ // destination's terrain has loaded (see ./flyToDataSource.js). const fly = (item) => { if (!item.visible) setVisible(item.id, true); - flyToSource(viewer, item.dataSource); + flyToSource(viewer, item.dataSource, { ground }); return true; }; @@ -216,12 +248,13 @@ export function initGeoFileImport({ // Capture the user's intent when the files are chosen, including time // spent waiting for an earlier batch. Later navigation must still win. const generation = beginAutoFly(); - const run = queue.then(() => importBatch(list, generation)); + const intent = ++cameraIntent; + const run = queue.then(() => importBatch(list, generation, intent)); queue = run.catch(() => {}); return run; } - async function importBatch(list, generation) { + async function importBatch(list, generation, intent) { const outcomes = []; if (destroyed || !list.length) return outcomes; for (const file of list) { @@ -239,7 +272,7 @@ export function initGeoFileImport({ const color = COLORS[colorAt % COLORS.length]; let loaded; try { - loaded = await load(file, { viewer, color }); + loaded = await load(file, { viewer, color, signal: lifetime.signal }); } catch (error) { outcomes.push({ name: fileName, @@ -247,7 +280,7 @@ export function initGeoFileImport({ error: error instanceof GeoImportError || error?.name === 'GeoImportError' ? error.message - : `That file could not be read${error?.message ? `: ${error.message}` : '.'}`, + : 'That file could not be read.', }); continue; } @@ -268,6 +301,7 @@ export function initGeoFileImport({ visible: true, dataSource: loaded.dataSource, revoke: loaded.revoke, + refreshGround: loaded.refreshGround, }; await viewer.dataSources.add(item.dataSource); if (destroyed) { @@ -275,6 +309,7 @@ export function initGeoFileImport({ item.revoke?.(); break; } + loaded.mount?.(); imports.push(item); renderList(); render(); @@ -288,6 +323,7 @@ export function initGeoFileImport({ // result, unless something else took the camera meanwhile. if ( outcomes.filter((o) => o.ok).length === 1 && + intent === cameraIntent && autoFlyStillOwns(generation) ) fly(item); @@ -317,6 +353,15 @@ export function initGeoFileImport({ // ---- drop onto the globe ---------------------------------------------- const container = viewer.container; + const gesture = () => { + cameraIntent += 1; + }; + listen(viewer.scene?.canvas || container, 'pointerdown', gesture, { + passive: true, + }); + listen(viewer.scene?.canvas || container, 'wheel', gesture, { + passive: true, + }); const hasFiles = (event) => [...(event.dataTransfer?.types || [])].includes('Files'); const setDragging = (on) => @@ -327,7 +372,7 @@ export function initGeoFileImport({ dragDepth += 1; setDragging(true); }); - listen(container, 'dragover', (event) => { + listen(doc, 'dragover', (event) => { if (!hasFiles(event)) return; event.preventDefault(); if (event.dataTransfer) event.dataTransfer.dropEffect = 'copy'; @@ -337,7 +382,7 @@ export function initGeoFileImport({ dragDepth = Math.max(0, dragDepth - 1); if (!dragDepth) setDragging(false); }); - listen(container, 'drop', (event) => { + listen(doc, 'drop', (event) => { if (!hasFiles(event)) return; event.preventDefault(); dragDepth = 0; @@ -371,6 +416,10 @@ export function initGeoFileImport({ destroy() { if (destroyed) return; destroyed = true; + lifetime.abort(); + removeMapSource?.(); + clearTimeout(groundTimer); + for (const remove of groundRemovers) remove(); for (const item of imports.splice(0)) { viewer.dataSources?.remove?.(item.dataSource, true); item.revoke?.(); @@ -408,7 +457,6 @@ export function summarize(outcomes) { (n, o) => n + (o.ok ? o.removed || 0 : 0), 0, ); - if (added && stripped) - parts.push('KML descriptions and external references were left out.'); + if (added && stripped) parts.push('Some KML content was left out.'); return parts.join(' '); } diff --git a/src/ui/geoFileImport.test.mjs b/src/ui/geoFileImport.test.mjs index f854af8c9..510883cb1 100644 --- a/src/ui/geoFileImport.test.mjs +++ b/src/ui/geoFileImport.test.mjs @@ -134,7 +134,7 @@ class FakeElement { } } -function setup({ load, navigation, flyToSource } = {}) { +function setup({ load, navigation, flyToSource, subscribeMapSource } = {}) { focus.active = null; const els = { 'geo-import-button': new FakeElement('button', 'geo-import-button'), @@ -150,6 +150,7 @@ function setup({ load, navigation, flyToSource } = {}) { return focus.active; }, }; + Object.setPrototypeOf(document, new FakeElement('document')); const added = []; const removed = []; const flights = []; @@ -189,7 +190,8 @@ function setup({ load, navigation, flyToSource } = {}) { load: load || defaultLoad, document, ...(navigation ? { navigation } : {}), - ...(flyToSource ? { flyToSource } : {}), + ...(subscribeMapSource ? { subscribeMapSource } : {}), + flyToSource: flyToSource || ((_viewer, ds) => flights.push(ds)), }); return { tool, @@ -301,7 +303,7 @@ test('a refused file says why and does not stop the rest of the batch', async () [false, false, true], ); assert.equal(out[0].error, 'That KML file is not valid XML.'); - assert.match(out[1].error, /could not be read: boom/); + assert.match(out[1].error, /could not be read\./); assert.match( t.els['geo-import-hint'].textContent, /^Added 1 file\. 2 files could not be added \(bad\.kml: That KML file is not valid XML\.\)/, @@ -329,13 +331,13 @@ test('dropping files on the globe imports them; other drags are left alone', asy const c = t.viewer.container; const files = { types: ['Files'], files: [file('drop.kmz')] }; let ev = { dataTransfer: { types: ['text/plain'] } }; - c.emit('dragover', ev); + t.document.emit('dragover', ev); c.emit('dragenter', { dataTransfer: files }); assert.ok(t.document.body.classList.contains('gev-geo-import-drag')); c.emit('dragleave', { dataTransfer: files }); assert.ok(!t.document.body.classList.contains('gev-geo-import-drag')); c.emit('dragenter', { dataTransfer: files }); - c.emit('drop', { dataTransfer: files }); + t.document.emit('drop', { dataTransfer: files }); assert.ok(!t.document.body.classList.contains('gev-geo-import-drag')); await new Promise((r) => setImmediate(r)); assert.deepEqual( @@ -347,7 +349,7 @@ test('dropping files on the globe imports them; other drags are left alone', asy test('stripped KML content is mentioned once, for the batch that had it', () => { assert.equal( summarize([{ name: 'a.kml', ok: true, removed: 3 }]), - 'Added 1 file. KML descriptions and external references were left out.', + 'Added 1 file. Some KML content was left out.', ); assert.equal( summarize([{ name: 'b.geojson', ok: true, removed: 0 }]), @@ -478,6 +480,79 @@ function navigationDouble({ immediate = true, reassert = true } = {}) { }; } +test('a direct camera gesture cancels the fly-to of a file still being read', async () => { + for (const gesture of ['pointerdown', 'wheel']) { + let release; + const t = setup({ + load: async () => { + await new Promise((r) => (release = r)); + return { + name: 'slow.geojson', + format: 'geojson', + entityCount: 1, + dataSource: {}, + }; + }, + }); + const result = t.tool.importFiles([file('slow.geojson')]); + await new Promise((r) => setImmediate(r)); + t.viewer.container.emit(gesture); + release(); + await result; + assert.equal(t.added.length, 1); + assert.equal(t.flights.length, 0); + t.tool.destroy(); + assert.equal(t.document.listeners.length, 0); + } +}); + +test('dropping on panel controls is accepted and file drags prevent browser navigation', async () => { + const t = setup(); + let prevented = 0; + const event = { + target: t.els['geo-import-button'], + dataTransfer: { types: ['Files'], files: [file('panel.kml')] }, + preventDefault: () => prevented++, + }; + t.document.emit('dragover', event); + t.document.emit('drop', event); + await new Promise((r) => setImmediate(r)); + assert.equal(prevented, 2); + assert.equal(t.tool.list()[0].name, 'panel.kml'); + t.tool.destroy(); + assert.equal(t.document.listeners.length, 0); +}); + +test('ground refresh coalesces map changes and releases its subscription on teardown', async () => { + let changed, + unsubscribed = 0, + refreshed = 0; + const t = setup({ + subscribeMapSource: (listener) => { + changed = listener; + return () => unsubscribed++; + }, + load: async () => ({ + name: 'point.geojson', + format: 'geojson', + entityCount: 1, + dataSource: {}, + refreshGround: async () => refreshed++, + }), + }); + await t.tool.importFiles([file('point.geojson')]); + changed(); + changed(); + changed(); + await new Promise((resolve) => setTimeout(resolve, 300)); + assert.equal(refreshed, 1); + t.tool.destroy(); + changed(); + await new Promise((resolve) => setTimeout(resolve, 250)); + assert.equal(refreshed, 1); + assert.equal(unsubscribed, 1); +}); + test('Fly to is an immediate navigation that Cockpit can refuse', async () => { const flown = []; const nav = navigationDouble({ immediate: false }); diff --git a/src/ui/templates/display-controls.html b/src/ui/templates/display-controls.html index c9da1c022..79e1f7fd7 100644 --- a/src/ui/templates/display-controls.html +++ b/src/ui/templates/display-controls.html @@ -148,7 +148,7 @@
-