diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 10d769fb0..e554b3496 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5705,6 +5705,7 @@ request viewbox and continues searching without that geographic preference. - **Input handling.** Coordinates and GeoJSON geometry types are checked before Cesium allocation. Only short plain names reach GeoJSON property bags; file IDs, custom CRS and styling properties are removed. KML feature/geometry IDs are removed while Style/StyleMap references remain. Local style references are trimmed, incomplete style pairs removed, hidden-folder flags normalized to the import's show control, and unsupported prototype-named elements removed. UTF-16 BOMs and common KML namespace export defects are handled. Empty drawable content is refused. A loader failure or timeout does not hold the next file indefinitely; teardown aborts pending waits. - **References and images.** File-authored remote links/icons/styles, network links, models and screen/photo overlays are removed. Packed KMZ and embedded PNG/JPEG/GIF/WebP/BMP images remain local; decoded dimensions are checked before image decoding against a total 16-megapixel and 8,192-pixel-side budget per file. Blocked IconStyle images fall back to local pins; authored empty icons remain empty. Colored overlays without an Icon survive; overlays with a removed external image are dropped. Metadata, native labels and descriptions are removed, and generated pins share one lossless image URL per canvas. - **Bounds.** File size ≤50 MiB; inflated KMZ ≤120 MiB; ≤20,000 entities (including multipart children and Folder containers); ≤250,000 positions; ≤2,000 normalized style definitions and distinct image links; ≤12 files per session. Exact repeated styles share definitions without changing inheritance or inline overrides. Files of 20–30 MB fit the byte budget, subject to geometry/image limits. Refusals name a reason and batches continue; with multiple failures the hint shows the first reason. +- **Code layout.** `src/data/geoFileImport.js` reads, bounds and sanitizes the files and no longer imports or drives Cesium itself; it hands the parsed file to the Cesium-bound part. The Cesium-bound part (loading the parsed file as a Cesium data source, scrubbing and grounding its entities, its pin collection and its footprint for Fly-to) is `src/data/geoImportPins.js`, which stays on the Cesium allow-list; `src/data/geoImportShared.js` holds the error, limits, deadline and feature names both use. - **Lifecycle and keyboard.** `createApplicationTools` wires `initGeoFileImport` with viewer, navigation, ground services and map-source subscription. Destroy aborts pending work, unsubscribes, removes data sources, revokes blob URLs, removes document/canvas listeners and releases `window.__gevGeoImport`. List rebuilds retain keyboard focus; removing a row hands focus to the next row, else the previous row, else Import. - **Validation.** `node scripts/qa-geo-import.mjs` covers all four formats and real painted show/hide. Run `node scripts/qa-geo-import-regressions.mjs` against both dev and preview for XYZ/track heights, failure recovery, namespace/BOM support, portrait framing and panel drops. `node scripts/qa-geo-import-large.mjs [--file /absolute/path/sample.kml] [--expect-local-pins]` checks a 27.6 MB repeated-style file, native style/image equivalence, large-file painted output and frame times. It records the WebGL renderer; software-rendered runs do not establish GPU performance. Unit tests live beside the data/controller/framing modules. diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index dd9029743..1dfb92e58 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -3758,6 +3758,7 @@ "src/annotations/drawMode.js", "src/annotations/drawTool.js", "src/data/geoFileImport.js", + "src/data/geoImportShared.js", "src/ui/flyToDataSource.js", "src/ui/geoFileImport.js", "src/data/routeSteps.js", @@ -4749,6 +4750,7 @@ "src/layers/streetLevel/providers/mapillary/policy.js", "src/data/geoImportImages.js", "src/data/geoImportPins.js", + "src/data/geoImportShared.js", "src/data/geoFileImport.js", "src/ui/stores/imagerySplit.js", "src/ui/vueRuntime.js", @@ -5867,6 +5869,7 @@ "src/layers/ghostMaps/source.js", "src/sources/httpBody.js", "src/data/geoFileImport.js", + "src/data/geoImportShared.js", "src/i18n/marker.js", "src/i18n/index.js", "src/i18n/locale.js", diff --git a/src/data/geoFileImport.js b/src/data/geoFileImport.js index 7ddaef93f..8779118b1 100644 --- a/src/data/geoFileImport.js +++ b/src/data/geoFileImport.js @@ -1,9 +1,19 @@ /** Session-local geodata loading with bounded geometry and local-only file references. */ -import * as Cesium from 'cesium'; import { createImportImageBudget } from './geoImportImages.js'; -import { createImportedPins } from './geoImportPins.js'; +import { + loadImportedGeoJson, + loadImportedKml, + settleImportedSource, +} from './geoImportPins.js'; +import { + GeoImportError, + MAX_IMPORT_ENTITIES, + boundedImport, + importFeatureName, +} from './geoImportShared.js'; import { msg } from '../i18n/marker.js'; -import { heightsFor } from '../render/cesium/heights.js'; + +export { GeoImportError, MAX_IMPORT_ENTITIES, importFeatureName }; /** Largest file accepted, in bytes. */ const MAX_IMPORT_BYTES = 50 * 1024 * 1024; @@ -11,14 +21,10 @@ const MAX_IMPORT_BYTES = 50 * 1024 * 1024; const MAX_KMZ_INFLATED_BYTES = 120 * 1024 * 1024; /** Most files on the globe at once. */ export const MAX_IMPORTED_FILES = 12; -/** Most entities one file may create before it is refused. */ -export const MAX_IMPORT_ENTITIES = 20_000; /** Most vertices (points of every line, ring and marker) in one file. */ export const MAX_IMPORT_POSITIONS = 250_000; /** Most normalized style definitions and local image links in one KML/KMZ. */ const MAX_KML_LINKS = 2_000; -/** Longest plain-text name kept from a file's features. */ -const MAX_IMPORT_NAME_LENGTH = 120; /** What the file picker offers. */ export const GEO_FILE_ACCEPT = '.geojson,.json,.kml,.kmz,.gpx'; @@ -63,21 +69,6 @@ const KML_REMOVED_ELEMENTS = new Set([ 'httpQuery', ]); -/** - * An error whose message is meant for the person who chose the file. The - * message stays English; `template` and `values`, when given, let the UI show - * it in the active language. Numbers stay numbers in `values`: the UI formats - * them in that language's numerals. - */ -export class GeoImportError extends Error { - constructor(message, template = message, values = {}) { - super(message); - this.name = 'GeoImportError'; - this.template = template; - this.values = values; - } -} - /** * The format of a file, from its name first and its first bytes second. * @param {string} name File name. @@ -953,95 +944,6 @@ export function gpxToGeoJson( return { type: 'FeatureCollection', features }; } -/** A feature name as short plain text (no control characters), or undefined. */ -export function importFeatureName(value) { - const text = String(value ?? '') - .replace(/[\u0000-\u001f\u007f]/g, ' ') - .replace(/\s+/g, ' ') - .trim(); - if (!text) return undefined; - return text.length > MAX_IMPORT_NAME_LENGTH - ? `${text.slice(0, MAX_IMPORT_NAME_LENGTH - 1)}…` - : text; -} - -/** - * Make every entity of an imported data source inert: no description, - * balloon, extended data or properties, no native label, and a name that is - * short plain text. Generated pin canvases share a data URL so Cesium uploads - * each image once, rather than allocating a texture entry for every entity. - * Returns how many entities were touched. - * @param {{entities: {values: object[]}}} dataSource - * @returns {number} - */ -export function scrubImportedEntities(dataSource) { - const collection = dataSource?.entities; - const list = collection?.values || []; - const images = new Map(); - collection?.suspendEvents?.(); - try { - for (let i = 0; i < list.length; i++) { - const entity = list[i]; - let name; - try { - name = importFeatureName( - typeof entity.name === 'string' ? entity.name : undefined, - ); - } catch { - name = undefined; - } - entity.name = name; - entity.description = undefined; - entity.label = undefined; - entity.properties = undefined; - if (entity.kml && typeof entity.kml === 'object') entity.kml = undefined; - if ( - entity.billboard?.heightReference?.getValue?.() === - Cesium.HeightReference.CLAMP_TO_GROUND - ) - entity.billboard.disableDepthTestDistance = Number.POSITIVE_INFINITY; - const property = entity.billboard?.image; - if (property?.isConstant === false) continue; - const image = property?.getValue?.(); - if (image?.tagName === 'CANVAS') { - let url = images.get(image); - if (!url) { - url = image.toDataURL(); - images.set(image, url); - } - entity.billboard.image = url; - } - } - } finally { - collection?.resumeEvents?.(); - } - return list.length; -} - -/** - * Take time out of an imported data source so adding it cannot move the app's - * shared clock: the data source reports no clock (the viewer adopts the clock - * of every added source that has one), every entity is available for all - * time. KML tracks have already become their original static vertices in - * `sanitizeKmlDocument`, so no sampled-position fallback is needed here. - * @param {object} dataSource - * @returns {object} The same data source. - */ -export function detachImportedTime(dataSource) { - Object.defineProperty(dataSource, 'clock', { - configurable: true, - enumerable: false, - get: () => undefined, - set: () => {}, - }); - const list = dataSource?.entities?.values || []; - for (let i = 0; i < list.length; i++) { - const entity = list[i]; - entity.availability = undefined; - } - return dataSource; -} - /** Repair only the first KML opening tag, scanning its delimiter once. */ export function repairKmlRootNamespaces(value, what = 'KML') { const text = String(value); @@ -1196,173 +1098,12 @@ function checkLocalImages(doc, entries = new Map(), main = '') { } } -// Cesium also processes KML in deferred tasks; an exception there does not -// reject its load promise. Keep a failed file from blocking the serial queue. -async function loadKml(doc, viewer, signal) { - let removeError; - try { - return await boundedImport( - Promise.race([ - Cesium.KmlDataSource.load(doc, { - camera: viewer.scene.camera, - canvas: viewer.scene.canvas, - clampToGround: true, - }), - new Promise((_, reject) => { - const failed = () => { - reject(new GeoImportError(msg('That KML could not be read.'))); - }; - globalThis.addEventListener?.('error', failed); - removeError = () => globalThis.removeEventListener?.('error', failed); - }), - ]), - signal, - ); - } finally { - removeError?.(); - } -} - -// Resolve a static file's ground pins in batches before they are visualized. -// Per-billboard CLAMP_TO_GROUND subscriptions scale poorly on large imports. -const groundPins = new WeakMap(); -async function settleGroundPins(dataSource, viewer, signal) { - let pins = groundPins.get(dataSource); - if (!pins) { - pins = []; - for (const entity of dataSource.entities.values) { - const reference = entity.billboard?.heightReference?.getValue?.(); - if ( - reference === Cesium.HeightReference.CLAMP_TO_GROUND || - reference === Cesium.HeightReference.RELATIVE_TO_GROUND - ) { - const position = entity.position?.getValue?.(Cesium.JulianDate.now()); - if (!position) continue; - const cartographic = Cesium.Cartographic.fromCartesian(position); - if (!cartographic) - throw new GeoImportError( - msg('That file has invalid point coordinates.'), - ); - pins.push({ - entity, - original: position, - cartographic, - offset: - reference === Cesium.HeightReference.RELATIVE_TO_GROUND - ? cartographic.height - : 0, - }); - entity.billboard.heightReference = Cesium.HeightReference.NONE; - entity.billboard.disableDepthTestDistance = Number.POSITIVE_INFINITY; - } - } - groundPins.set(dataSource, pins); - } - for (let at = 0; at < pins.length; at += 256) { - signal?.throwIfAborted(); - const batch = pins.slice(at, at + 256); - const positions = batch.map( - ({ cartographic: c }) => - new Cesium.Cartographic(c.longitude, c.latitude, 0), - ); - const mesh = - viewer.scene?.globe?.show === false && - viewer.scene?.clampToHeightSupported; - // A coarse rendered ellipsoid mesh can report a height kilometres below - // zero. Its provider's exact height is zero even while a 3D map hides it; - // a buried starting point can make the downward mesh ray miss the surface. - const ellipsoid = - viewer.terrainProvider instanceof Cesium.EllipsoidTerrainProvider; - const heights = viewer.scene ? heightsFor(viewer.scene) : null; - for (let i = 0; i < positions.length; i++) { - const sampled = ellipsoid - ? 0 - : heights?.now(positions[i], { against: 'terrain' }); - positions[i].height = mesh - ? (batch[i].groundHeight ?? - Math.max(0, Number.isFinite(sampled) ? sampled : 0)) - : Number.isFinite(sampled) - ? sampled - : batch[i].groundHeight || 0; - } - const points = positions.map((c) => - Cesium.Cartesian3.fromRadians(c.longitude, c.latitude, c.height), - ); - const height = viewer.camera?.positionCartographic?.height; - if (mesh && Number.isFinite(height) && height < 250_000) { - const rectangle = viewer.camera.computeViewRectangle?.(); - const indices = []; - for (let i = 0; i < positions.length; i++) - if (rectangle && Cesium.Rectangle.contains(rectangle, positions[i])) - indices.push(i); - if (indices.length) { - const resolved = await boundedImport( - heights.clampBatch( - indices.map((i) => { - const c = positions[i]; - return Cesium.Cartesian3.fromRadians( - c.longitude, - c.latitude, - Math.max(0, c.height), - ); - }), - ), - signal, - ); - for (let i = 0; i < indices.length; i++) - if (resolved[i]) points[indices[i]] = resolved[i]; - } - } - signal?.throwIfAborted(); - for (let i = 0; i < batch.length; i++) { - const item = batch[i]; - const c = points[i] && Cesium.Cartographic.fromCartesian(points[i]); - if (!c) - throw new GeoImportError( - msg( - 'Ground heights are not available for that file. Try again when the map has loaded.', - ), - ); - item.groundHeight = c.height; - c.height += item.offset; - item.entity.position = - Math.abs(c.height - item.cartographic.height) < 1e-6 - ? item.original - : Cesium.Cartesian3.fromRadians(c.longitude, c.latitude, c.height); - } - if (at + batch.length < pins.length) - await new Promise((resolve) => setTimeout(resolve, 0)); - } -} - -async function boundedImport(promise, signal) { - let timer, abort; - try { - signal?.throwIfAborted(); - return await Promise.race([ - promise, - new Promise((_, reject) => { - abort = () => reject(signal.reason); - signal?.addEventListener('abort', abort, { once: true }); - timer = setTimeout( - () => - reject(new GeoImportError(msg('That file took too long to read.'))), - 60_000, - ); - }), - ]); - } finally { - clearTimeout(timer); - signal?.removeEventListener('abort', abort); - } -} - /** * Load a ground footprint, retaining explicit KML altitude modes. Call mount after * adding the source; revoke owns pin and blob cleanup. * @param {File|Blob & {name?: string}} file - * @param {{viewer: Cesium.Viewer, color?: string}} options `color` is a CSS color. - * @returns {Promise<{dataSource: Cesium.DataSource, name: string, + * @param {{viewer: object, color?: string}} options `color` is a CSS color. + * @returns {Promise<{dataSource: object, name: string, * format: string, entityCount: number, removed: number, mount: Function, revoke: Function}>} */ export async function loadGeoFile( @@ -1387,11 +1128,10 @@ export async function loadGeoFile( throw new GeoImportError( msg('Only GeoJSON, KML, KMZ and GPX files can be imported.'), ); - const stroke = Cesium.Color.fromCssColorString(color); const urls = []; - let pins; + let drawn; const revoke = () => { - pins?.destroy(); + drawn?.pins.destroy(); urls.splice(0).forEach((url) => URL.revokeObjectURL(url)); }; const name = importDisplayName(file.name); @@ -1411,18 +1151,7 @@ export async function loadGeoFile( throw new GeoImportError(msg('That file has nothing to draw.')); prepareGeoJson(geojson); bytes = null; - dataSource = await boundedImport( - Cesium.GeoJsonDataSource.load(geojson, { - clampToGround: true, - stroke, - fill: stroke.withAlpha(0.25), - strokeWidth: 3, - markerColor: stroke, - // No HTML description table is built from the file's properties. - describe: () => undefined, - }), - signal, - ); + dataSource = await loadImportedGeoJson(geojson, { color, signal }); } else { let text; let entries; @@ -1451,7 +1180,7 @@ export async function loadGeoFile( deduplicateKmlStyles(doc); const normalizedSize = countKmlGeometry(doc); if (normalizedSize.over) throw overLimitError(normalizedSize); - dataSource = await loadKml(doc, viewer, signal); + dataSource = await loadImportedKml(doc, viewer, signal); } } catch (error) { revoke(); @@ -1462,51 +1191,11 @@ export async function loadGeoFile( { format: format.toUpperCase() }, ); } - const entityCount = dataSource.entities.values.length; - if (entityCount > MAX_IMPORT_ENTITIES) { - revoke(); - throw new GeoImportError( - `That file has ${entityCount.toLocaleString()} map objects; the limit is ${MAX_IMPORT_ENTITIES.toLocaleString()}.`, - msg('That file has {count} map objects; the limit is {limit}.'), - { - count: entityCount, - limit: MAX_IMPORT_ENTITIES, - }, - ); - } - const now = Cesium.JulianDate.now(); - const drawable = dataSource.entities.values.some( - (e) => - e.billboard?.image?.getValue(now) || - e.point || - e.rectangle || - e.polyline?.positions?.getValue(now)?.length >= 2 || - e.wall?.positions?.getValue(now)?.length >= 2 || - e.polygon?.hierarchy?.getValue(now)?.positions?.length >= 3, - ); - if (!drawable) { - revoke(); - throw new GeoImportError(msg('That file has nothing to draw.')); - } - scrubImportedEntities(dataSource); try { - await settleGroundPins(dataSource, viewer, signal); + drawn = await settleImportedSource(dataSource, viewer, signal); } catch (error) { revoke(); - throw error instanceof GeoImportError - ? error - : new GeoImportError( - msg('Ground heights could not be read. Try that file again.'), - ); - } - // Keep the clock invariant explicit even though unused KML dates have - // already been stripped and all supported geometries are static. - detachImportedTime(dataSource); - try { - pins = createImportedPins(dataSource, viewer); - } catch { - revoke(); - throw new GeoImportError(msg('That file has invalid point graphics.')); + throw error; } // The display name travels beside the data source (the file name, not // anything the document says about itself). @@ -1514,82 +1203,10 @@ export async function loadGeoFile( dataSource, name, format, - entityCount, + entityCount: drawn.entityCount, removed, revoke, - mount: () => pins.mount(), - refreshGround: (refreshSignal) => - settleGroundPins( - dataSource, - viewer, - refreshSignal - ? AbortSignal.any([refreshSignal, AbortSignal.timeout(15_000)]) - : AbortSignal.timeout(15_000), - ).then(() => pins.sync()), + mount: () => drawn.pins.mount(), + refreshGround: drawn.refreshGround, }; } - -/** All visible positions, path vertices, polygon rings, walls and overlay bounds. */ -export function dataSourcePoints(dataSource, time = Cesium.JulianDate.now()) { - const points = []; - const add = (position, grounded = false) => { - if ( - !position || - ![position.x, position.y, position.z].every(Number.isFinite) - ) - return; - const c = Cesium.Cartographic.fromCartesian(position); - if (c && Number.isFinite(c.longitude) && Number.isFinite(c.latitude)) - points.push([ - Cesium.Math.toDegrees(c.longitude), - Cesium.Math.toDegrees(c.latitude), - grounded ? 0 : c.height, - ]); - }; - if (dataSource?.show === false) return points; - for (const entity of dataSource?.importedEntities ?? - dataSource?.entities?.values ?? - []) { - if (entity.isShowing === false) continue; - add( - entity.position?.getValue?.(time), - entity.billboard?.heightReference?.getValue?.(time) === - Cesium.HeightReference.CLAMP_TO_GROUND, - ); - for (const graphic of [entity.polyline, entity.wall]) - for (const position of graphic?.positions?.getValue?.(time) || []) - add(position, graphic.clampToGround?.getValue?.(time) === true); - const rings = [entity.polygon?.hierarchy?.getValue?.(time)]; - while (rings.length) { - const ring = rings.pop(); - for (const position of ring?.positions || []) - add( - position, - entity.polygon?.perPositionHeight?.getValue?.(time) !== true, - ); - for (const hole of ring?.holes || []) rings.push(hole); - } - const rectangle = entity.rectangle?.coordinates?.getValue?.(time); - if (rectangle) { - const bounds = Cesium.RectangleGeometry.computeRectangle({ - rectangle, - rotation: entity.rectangle.rotation?.getValue?.(time) || 0, - }); - for (const corner of [ - 'southwest', - 'southeast', - 'northwest', - 'northeast', - ]) { - const c = Cesium.Rectangle[corner](bounds); - if (Number.isFinite(c.longitude) && Number.isFinite(c.latitude)) - points.push([ - Cesium.Math.toDegrees(c.longitude), - Cesium.Math.toDegrees(c.latitude), - entity.rectangle.height?.getValue?.(time) || 0, - ]); - } - } - } - return points; -} diff --git a/src/data/geoFileImport.test.mjs b/src/data/geoFileImport.test.mjs index aac478a07..52a1c6165 100644 --- a/src/data/geoFileImport.test.mjs +++ b/src/data/geoFileImport.test.mjs @@ -12,10 +12,8 @@ import { countGeoJsonGeometry, countKmlGeometry, deduplicateKmlStyles, - detachImportedTime, gpxToGeoJson, importFeatureName, - scrubImportedEntities, sanitizeKmlDocument, detectGeoFileFormat, importDisplayName, @@ -26,6 +24,7 @@ import { readZipEntries, resolveKmzPath, } from './geoFileImport.js'; +import { detachImportedTime, scrubImportedEntities } from './geoImportPins.js'; import { createImportImageBudget, importImageSize } from './geoImportImages.js'; test('real loader grounds XYZ GeoJSON and strips loader-sensitive metadata before allocation', async () => { diff --git a/src/data/geoImportPins.js b/src/data/geoImportPins.js index d024a4d60..0fe38d2c9 100644 --- a/src/data/geoImportPins.js +++ b/src/data/geoImportPins.js @@ -1,5 +1,18 @@ +/** + * The Cesium-bound part of the file import: what `geoFileImport.js` parsed + * and bounded, loaded as a Cesium data source, scrubbed, grounded, its pins + * drawn in one billboard collection, and its footprint read back. + */ import { followedOf, stopFollowing } from './followed.js'; import * as Cesium from 'cesium'; +import { + GeoImportError, + MAX_IMPORT_ENTITIES, + boundedImport, + importFeatureName, +} from './geoImportShared.js'; +import { msg } from '../i18n/marker.js'; +import { heightsFor } from '../render/cesium/heights.js'; const STYLE = [ 'image', @@ -128,3 +141,371 @@ export function createImportedPins(dataSource, viewer) { }, }; } + +/** + * Prepared GeoJSON as a Cesium data source: ground-clamped, one colour for + * the file's lines, fills and markers, and no description tables. + * @param {object} geojson + * @param {{color: string, signal?: AbortSignal}} options `color` is a CSS color. + */ +export function loadImportedGeoJson(geojson, { color, signal }) { + const stroke = Cesium.Color.fromCssColorString(color); + return boundedImport( + Cesium.GeoJsonDataSource.load(geojson, { + clampToGround: true, + stroke, + fill: stroke.withAlpha(0.25), + strokeWidth: 3, + markerColor: stroke, + // No HTML description table is built from the file's properties. + describe: () => undefined, + }), + signal, + ); +} + +/** + * A sanitized KML document as a Cesium data source. Cesium also processes + * KML in deferred tasks; an exception there does not reject its load + * promise. Keep a failed file from blocking the serial queue. + */ +export async function loadImportedKml(doc, viewer, signal) { + let removeError; + try { + return await boundedImport( + Promise.race([ + Cesium.KmlDataSource.load(doc, { + camera: viewer.scene.camera, + canvas: viewer.scene.canvas, + clampToGround: true, + }), + new Promise((_, reject) => { + const failed = () => { + reject(new GeoImportError(msg('That KML could not be read.'))); + }; + globalThis.addEventListener?.('error', failed); + removeError = () => globalThis.removeEventListener?.('error', failed); + }), + ]), + signal, + ); + } finally { + removeError?.(); + } +} + +/** + * Check, scrub and ground a loaded data source and gather its pins. The + * refusals a person reads are `GeoImportError`s; the caller revokes what it + * owns on any failure. + * @returns {Promise<{entityCount: number, pins: object, refreshGround: Function}>} + */ +export async function settleImportedSource(dataSource, viewer, signal) { + const entityCount = dataSource.entities.values.length; + if (entityCount > MAX_IMPORT_ENTITIES) + throw new GeoImportError( + `That file has ${entityCount.toLocaleString()} map objects; the limit is ${MAX_IMPORT_ENTITIES.toLocaleString()}.`, + msg('That file has {count} map objects; the limit is {limit}.'), + { + count: entityCount, + limit: MAX_IMPORT_ENTITIES, + }, + ); + const now = Cesium.JulianDate.now(); + const drawable = dataSource.entities.values.some( + (e) => + e.billboard?.image?.getValue(now) || + e.point || + e.rectangle || + e.polyline?.positions?.getValue(now)?.length >= 2 || + e.wall?.positions?.getValue(now)?.length >= 2 || + e.polygon?.hierarchy?.getValue(now)?.positions?.length >= 3, + ); + if (!drawable) + throw new GeoImportError(msg('That file has nothing to draw.')); + scrubImportedEntities(dataSource); + try { + await settleGroundPins(dataSource, viewer, signal); + } catch (error) { + throw error instanceof GeoImportError + ? error + : new GeoImportError( + msg('Ground heights could not be read. Try that file again.'), + ); + } + // Keep the clock invariant explicit even though unused KML dates have + // already been stripped and all supported geometries are static. + detachImportedTime(dataSource); + let pins; + try { + pins = createImportedPins(dataSource, viewer); + } catch { + throw new GeoImportError(msg('That file has invalid point graphics.')); + } + return { + entityCount, + pins, + refreshGround: (refreshSignal) => + settleGroundPins( + dataSource, + viewer, + refreshSignal + ? AbortSignal.any([refreshSignal, AbortSignal.timeout(15_000)]) + : AbortSignal.timeout(15_000), + ).then(() => pins.sync()), + }; +} + +/** + * Make every entity of an imported data source inert: no description, + * balloon, extended data or properties, no native label, and a name that is + * short plain text. Generated pin canvases share a data URL so Cesium uploads + * each image once, rather than allocating a texture entry for every entity. + * Returns how many entities were touched. + * @param {{entities: {values: object[]}}} dataSource + * @returns {number} + */ +export function scrubImportedEntities(dataSource) { + const collection = dataSource?.entities; + const list = collection?.values || []; + const images = new Map(); + collection?.suspendEvents?.(); + try { + for (let i = 0; i < list.length; i++) { + const entity = list[i]; + let name; + try { + name = importFeatureName( + typeof entity.name === 'string' ? entity.name : undefined, + ); + } catch { + name = undefined; + } + entity.name = name; + entity.description = undefined; + entity.label = undefined; + entity.properties = undefined; + if (entity.kml && typeof entity.kml === 'object') entity.kml = undefined; + if ( + entity.billboard?.heightReference?.getValue?.() === + Cesium.HeightReference.CLAMP_TO_GROUND + ) + entity.billboard.disableDepthTestDistance = Number.POSITIVE_INFINITY; + const property = entity.billboard?.image; + if (property?.isConstant === false) continue; + const image = property?.getValue?.(); + if (image?.tagName === 'CANVAS') { + let url = images.get(image); + if (!url) { + url = image.toDataURL(); + images.set(image, url); + } + entity.billboard.image = url; + } + } + } finally { + collection?.resumeEvents?.(); + } + return list.length; +} + +/** + * Take time out of an imported data source so adding it cannot move the app's + * shared clock: the data source reports no clock (the viewer adopts the clock + * of every added source that has one), every entity is available for all + * time. KML tracks have already become their original static vertices in + * `sanitizeKmlDocument`, so no sampled-position fallback is needed here. + * @param {object} dataSource + * @returns {object} The same data source. + */ +export function detachImportedTime(dataSource) { + Object.defineProperty(dataSource, 'clock', { + configurable: true, + enumerable: false, + get: () => undefined, + set: () => {}, + }); + const list = dataSource?.entities?.values || []; + for (let i = 0; i < list.length; i++) { + const entity = list[i]; + entity.availability = undefined; + } + return dataSource; +} + +// Resolve a static file's ground pins in batches before they are visualized. +// Per-billboard CLAMP_TO_GROUND subscriptions scale poorly on large imports. +const groundPins = new WeakMap(); +async function settleGroundPins(dataSource, viewer, signal) { + let pins = groundPins.get(dataSource); + if (!pins) { + pins = []; + for (const entity of dataSource.entities.values) { + const reference = entity.billboard?.heightReference?.getValue?.(); + if ( + reference === Cesium.HeightReference.CLAMP_TO_GROUND || + reference === Cesium.HeightReference.RELATIVE_TO_GROUND + ) { + const position = entity.position?.getValue?.(Cesium.JulianDate.now()); + if (!position) continue; + const cartographic = Cesium.Cartographic.fromCartesian(position); + if (!cartographic) + throw new GeoImportError( + msg('That file has invalid point coordinates.'), + ); + pins.push({ + entity, + original: position, + cartographic, + offset: + reference === Cesium.HeightReference.RELATIVE_TO_GROUND + ? cartographic.height + : 0, + }); + entity.billboard.heightReference = Cesium.HeightReference.NONE; + entity.billboard.disableDepthTestDistance = Number.POSITIVE_INFINITY; + } + } + groundPins.set(dataSource, pins); + } + for (let at = 0; at < pins.length; at += 256) { + signal?.throwIfAborted(); + const batch = pins.slice(at, at + 256); + const positions = batch.map( + ({ cartographic: c }) => + new Cesium.Cartographic(c.longitude, c.latitude, 0), + ); + const mesh = + viewer.scene?.globe?.show === false && + viewer.scene?.clampToHeightSupported; + // A coarse rendered ellipsoid mesh can report a height kilometres below + // zero. Its provider's exact height is zero even while a 3D map hides it; + // a buried starting point can make the downward mesh ray miss the surface. + const ellipsoid = + viewer.terrainProvider instanceof Cesium.EllipsoidTerrainProvider; + const heights = viewer.scene ? heightsFor(viewer.scene) : null; + for (let i = 0; i < positions.length; i++) { + const sampled = ellipsoid + ? 0 + : heights?.now(positions[i], { against: 'terrain' }); + positions[i].height = mesh + ? (batch[i].groundHeight ?? + Math.max(0, Number.isFinite(sampled) ? sampled : 0)) + : Number.isFinite(sampled) + ? sampled + : batch[i].groundHeight || 0; + } + const points = positions.map((c) => + Cesium.Cartesian3.fromRadians(c.longitude, c.latitude, c.height), + ); + const height = viewer.camera?.positionCartographic?.height; + if (mesh && Number.isFinite(height) && height < 250_000) { + const rectangle = viewer.camera.computeViewRectangle?.(); + const indices = []; + for (let i = 0; i < positions.length; i++) + if (rectangle && Cesium.Rectangle.contains(rectangle, positions[i])) + indices.push(i); + if (indices.length) { + const resolved = await boundedImport( + heights.clampBatch( + indices.map((i) => { + const c = positions[i]; + return Cesium.Cartesian3.fromRadians( + c.longitude, + c.latitude, + Math.max(0, c.height), + ); + }), + ), + signal, + ); + for (let i = 0; i < indices.length; i++) + if (resolved[i]) points[indices[i]] = resolved[i]; + } + } + signal?.throwIfAborted(); + for (let i = 0; i < batch.length; i++) { + const item = batch[i]; + const c = points[i] && Cesium.Cartographic.fromCartesian(points[i]); + if (!c) + throw new GeoImportError( + msg( + 'Ground heights are not available for that file. Try again when the map has loaded.', + ), + ); + item.groundHeight = c.height; + c.height += item.offset; + item.entity.position = + Math.abs(c.height - item.cartographic.height) < 1e-6 + ? item.original + : Cesium.Cartesian3.fromRadians(c.longitude, c.latitude, c.height); + } + if (at + batch.length < pins.length) + await new Promise((resolve) => setTimeout(resolve, 0)); + } +} + +/** All visible positions, path vertices, polygon rings, walls and overlay bounds. */ +export function dataSourcePoints(dataSource, time = Cesium.JulianDate.now()) { + const points = []; + const add = (position, grounded = false) => { + if ( + !position || + ![position.x, position.y, position.z].every(Number.isFinite) + ) + return; + const c = Cesium.Cartographic.fromCartesian(position); + if (c && Number.isFinite(c.longitude) && Number.isFinite(c.latitude)) + points.push([ + Cesium.Math.toDegrees(c.longitude), + Cesium.Math.toDegrees(c.latitude), + grounded ? 0 : c.height, + ]); + }; + if (dataSource?.show === false) return points; + for (const entity of dataSource?.importedEntities ?? + dataSource?.entities?.values ?? + []) { + if (entity.isShowing === false) continue; + add( + entity.position?.getValue?.(time), + entity.billboard?.heightReference?.getValue?.(time) === + Cesium.HeightReference.CLAMP_TO_GROUND, + ); + for (const graphic of [entity.polyline, entity.wall]) + for (const position of graphic?.positions?.getValue?.(time) || []) + add(position, graphic.clampToGround?.getValue?.(time) === true); + const rings = [entity.polygon?.hierarchy?.getValue?.(time)]; + while (rings.length) { + const ring = rings.pop(); + for (const position of ring?.positions || []) + add( + position, + entity.polygon?.perPositionHeight?.getValue?.(time) !== true, + ); + for (const hole of ring?.holes || []) rings.push(hole); + } + const rectangle = entity.rectangle?.coordinates?.getValue?.(time); + if (rectangle) { + const bounds = Cesium.RectangleGeometry.computeRectangle({ + rectangle, + rotation: entity.rectangle.rotation?.getValue?.(time) || 0, + }); + for (const corner of [ + 'southwest', + 'southeast', + 'northwest', + 'northeast', + ]) { + const c = Cesium.Rectangle[corner](bounds); + if (Number.isFinite(c.longitude) && Number.isFinite(c.latitude)) + points.push([ + Cesium.Math.toDegrees(c.longitude), + Cesium.Math.toDegrees(c.latitude), + entity.rectangle.height?.getValue?.(time) || 0, + ]); + } + } + } + return points; +} diff --git a/src/data/geoImportShared.js b/src/data/geoImportShared.js new file mode 100644 index 000000000..a2305cb8d --- /dev/null +++ b/src/data/geoImportShared.js @@ -0,0 +1,57 @@ +/** What the file import's parsing and drawing parts share: its error, its limits, its deadline and feature names. */ +import { msg } from '../i18n/marker.js'; + +/** Most entities one file may create before it is refused. */ +export const MAX_IMPORT_ENTITIES = 20_000; +/** Longest plain-text name kept from a file's features. */ +const MAX_IMPORT_NAME_LENGTH = 120; + +/** + * An error whose message is meant for the person who chose the file. The + * message stays English; `template` and `values`, when given, let the UI show + * it in the active language. Numbers stay numbers in `values`: the UI formats + * them in that language's numerals. + */ +export class GeoImportError extends Error { + constructor(message, template = message, values = {}) { + super(message); + this.name = 'GeoImportError'; + this.template = template; + this.values = values; + } +} + +/** A feature name as short plain text (no control characters), or undefined. */ +export function importFeatureName(value) { + const text = String(value ?? '') + .replace(/[\u0000-\u001f\u007f]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + if (!text) return undefined; + return text.length > MAX_IMPORT_NAME_LENGTH + ? `${text.slice(0, MAX_IMPORT_NAME_LENGTH - 1)}…` + : text; +} + +/** `promise`, or a refusal when the import is aborted or takes over a minute. */ +export async function boundedImport(promise, signal) { + let timer, abort; + try { + signal?.throwIfAborted(); + return await Promise.race([ + promise, + new Promise((_, reject) => { + abort = () => reject(signal.reason); + signal?.addEventListener('abort', abort, { once: true }); + timer = setTimeout( + () => + reject(new GeoImportError(msg('That file took too long to read.'))), + 60_000, + ); + }), + ]); + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + } +} diff --git a/src/render/cesiumAllowList.json b/src/render/cesiumAllowList.json index bfa0a2790..c118c1d64 100644 --- a/src/render/cesiumAllowList.json +++ b/src/render/cesiumAllowList.json @@ -13,7 +13,6 @@ "src/data/cctvGizmo.js", "src/data/detection.js", "src/data/focusDeemphasis.js", - "src/data/geoFileImport.js", "src/data/geoImportPins.js", "src/data/iconOrientation.js", "src/data/iconSegment.js", diff --git a/src/ui/flyToDataSource.test.mjs b/src/ui/flyToDataSource.test.mjs index 141d9f594..e0c70f0d2 100644 --- a/src/ui/flyToDataSource.test.mjs +++ b/src/ui/flyToDataSource.test.mjs @@ -5,7 +5,7 @@ import { cancelCameraArrival, ownCameraArrival, } from '../data/cameraArrival.js'; -import { dataSourcePoints } from '../data/geoFileImport.js'; +import { dataSourcePoints } from '../data/geoImportPins.js'; import { bindFakeCesiumCamera } from '../testSupport/fakeCameraPort.mjs'; import { fakeSceneQuery } from '../testSupport/fakeCesiumScene.mjs'; import { flyToFootprint } from './flyToDataSource.js'; diff --git a/src/ui/geoFileImport.js b/src/ui/geoFileImport.js index ee204c2ae..56b0ef08b 100644 --- a/src/ui/geoFileImport.js +++ b/src/ui/geoFileImport.js @@ -2,9 +2,9 @@ import { GeoImportError, MAX_IMPORTED_FILES, - dataSourcePoints, loadGeoFile, } from '../data/geoFileImport.js'; +import { dataSourcePoints } from '../data/geoImportPins.js'; import { formatNumber, t, tn, translateKnown } from '../i18n/index.js'; import { flyToFootprint } from './flyToDataSource.js';