diff --git a/changelog.d/1297-s2-ghostmaps-layer.md b/changelog.d/1297-s2-ghostmaps-layer.md index e79cc6eff..938f5f89b 100644 --- a/changelog.d/1297-s2-ghostmaps-layer.md +++ b/changelog.d/1297-s2-ghostmaps-layer.md @@ -7,3 +7,6 @@ products that changed (Black Pine Defense). - Clear a temporary GhostMaps upstream error and notify open pages when the repository becomes reachable again, including when no new edition was published. +- Release packed-image resources when a load is cancelled or fails, respect + injected or omitted GhostMaps sources, and wait for background cache warm-up + before removing provider test fixtures. diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 52fdb1e78..72f6db4b8 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5133,6 +5133,7 @@ are omitted rather than framing the wrong part of the globe. - The layer reloads only products whose blob changed. The new edition's data source is added before the old one is removed. A product that leaves the catalog leaves the globe. - The row reads `LIVE` while the stream is open. A five-minute catalog read is the backstop. - **Files.** `/api/ghostmaps/file/{product}` serves only catalog products. Files are downloaded by commit-pinned raw URLs, so a CDN cannot return an older edition. They are cached on disk by blob sha, and blobs no current product uses are pruned. A request whose `?v=` names the current blob is `immutable`. Repository paths never reach the page. +- **Source ownership.** The application catalog receives `sources['s2-ghostmaps']` through the shared source contracts; an omitted or null source stays unavailable without opening a stream or fetching. The standalone source factory supplies the same-origin implementation and accepts replacements. Packed-image URLs are released on cancelled or failed product construction, and remain owned by a successfully loaded product until it is removed. - **Rendering.** The page parses the KMZ with the import module's zip, KMZ and XML helpers into one `CustomDataSource` per product. - Points become ground-clamped billboards with the depth test off, so they never sink into terrain or 3D tiles. Because the Google 3D map hides the globe, nothing else would stop far-side pins drawing through the planet. Pins are therefore horizon-culled with Street Level's `createHorizonCull` before their first frame and again whenever the camera moves. - Lines and polygon outlines are ground-clamped polylines. diff --git a/scripts/format-scope.json b/scripts/format-scope.json index 71d51d253..879ef83a6 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -884,6 +884,7 @@ "src/layers/ghostMaps/source.js", "src/layers/ghostMaps/catalog.test.mjs", "src/layers/ghostMaps/records.test.mjs", + "src/layers/ghostMaps/lifecycle.test.mjs", "scripts/qa-ghostmaps.mjs", "src/layers/cyclones/index.js", "src/layers/cyclones/rendering.js", diff --git a/server/providers/ghostMaps.test.mjs b/server/providers/ghostMaps.test.mjs index 5ddef6aa5..9723436d2 100644 --- a/server/providers/ghostMaps.test.mjs +++ b/server/providers/ghostMaps.test.mjs @@ -65,6 +65,17 @@ async function withCache(fn) { } } +// checkNow announces the catalog before background warm-up finishes. Join the +// same file reads before changing editions or removing the temporary cache. +async function warmDefaults(watcher) { + await Promise.all( + watcher + .catalog() + .products.filter((product) => product.defaultOn) + .map((product) => watcher.file(product.id)), + ); +} + test('a new commit is announced once, naming only the products that changed', () => withCache(async (cacheDir) => { const { repo, fetchImpl } = fakeGitHub(); @@ -78,6 +89,7 @@ test('a new commit is announced once, naming only the products that changed', () watcher.subscribe((catalog) => events.push(catalog)); assert.equal(await watcher.checkNow(), true); + await warmDefaults(watcher); const first = watcher.catalog(); assert.equal(first.commit, sha('1')); assert.equal(first.message, 'Oct 18 Update'); @@ -105,6 +117,7 @@ test('a new commit is announced once, naming only the products that changed', () }, ]); assert.equal(await watcher.checkNow(), true); + await warmDefaults(watcher); assert.equal(events.length, 2); const cip = events[1].products.find((p) => p.id === 'cip'); assert.equal(cip.version, sha('c')); @@ -165,7 +178,9 @@ test('the catalog survives a restart from disk, and upstream failures are report withCache(async (cacheDir) => { const { fetchImpl } = fakeGitHub(); const quiet = { log: () => {}, warn: () => {} }; - await createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet }).checkNow(); + const online = createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet }); + await online.checkNow(); + await warmDefaults(online); const offline = createGhostMapsWatcher({ fetchImpl: async () => new Response('down', { status: 503 }), cacheDir, diff --git a/src/app/constructCatalog.js b/src/app/constructCatalog.js index 7c149ab59..59edc02ad 100644 --- a/src/app/constructCatalog.js +++ b/src/app/constructCatalog.js @@ -127,7 +127,7 @@ export function createApplicationCatalog({ createApplicationGnssIntegrity({ source: sources['gnss-integrity'], }), - createApplicationGhostMaps(), + createApplicationGhostMaps({ source: sources['s2-ghostmaps'] }), createApplicationAlpr({ surface, source: sources.alpr }), createApplicationStreetLevel({ surface, diff --git a/src/app/layers/ghostMaps.js b/src/app/layers/ghostMaps.js index 5e373c3d8..ccca18850 100644 --- a/src/app/layers/ghostMaps.js +++ b/src/app/layers/ghostMaps.js @@ -1,8 +1,5 @@ import * as Cesium from 'cesium'; -import { - createGhostMapsLayer, - createGhostMapsSource, -} from '../../layers/ghostMaps/index.js'; +import { createGhostMapsLayer } from '../../layers/ghostMaps/index.js'; import * as picking from '../../data/pickRegistry.js'; import * as overlays from '../../overlays/worldOverlay.js'; import { isPointerFree } from '../../data/inputOwnership.js'; @@ -10,7 +7,6 @@ import { isPointerFree } from '../../data/inputOwnership.js'; /** Wire the S2 Underground GhostMaps layer into the application catalog. */ export function createApplicationGhostMaps(options) { return createGhostMapsLayer({ - source: createGhostMapsSource(), overlayHost: { setEntries: overlays.setOverlayEntries, setVisible: overlays.setOverlaySourceVisible, diff --git a/src/app/sourceComposition.js b/src/app/sourceComposition.js index 0fc619142..442598658 100644 --- a/src/app/sourceComposition.js +++ b/src/app/sourceComposition.js @@ -48,6 +48,7 @@ export const CATALOG_SOURCE_CONTRACTS = Object.freeze({ 'fire-perimeters': contract(['fire-perimeters'], ['getSnapshot']), 'air-quality': contract(['air-quality'], ['getSnapshot']), 'gnss-integrity': contract(['gnss-integrity'], ['getSnapshot']), + 's2-ghostmaps': contract(['s2-ghostmaps'], ['getCatalog', 'getFile']), cables: contract(['telegeography-submarine-cables'], ['fetch']), transit: contract(['transit'], ['requestSnapshot', 'getHistory']), }); diff --git a/src/app/sourceComposition.test.mjs b/src/app/sourceComposition.test.mjs index bfb031026..5baf01dab 100644 --- a/src/app/sourceComposition.test.mjs +++ b/src/app/sourceComposition.test.mjs @@ -86,6 +86,70 @@ test('standalone defaults allow exact replacements and explicit removal, and rej ); }); +test('GhostMaps obeys source omission, replacement and explicit removal', async (t) => { + let fetches = 0; + t.mock.method(globalThis, 'fetch', async () => { + fetches++; + throw new Error('Unexpected standalone acquisition'); + }); + const missing = catalog({}, t); + assert.equal(missing.getSourceAvailability('s2-ghostmaps')?.available, false); + const manager = new LayerLifecycle( + {}, + { getSourceAvailability: missing.getSourceAvailability }, + ); + manager.register(missing.get('s2-ghostmaps')); + assert.equal(await manager.setEnabled('s2-ghostmaps', true), false); + assert.equal(manager.layers.get('s2-ghostmaps').initialized, false); + assert.equal( + createStandaloneLayerSources({ 's2-ghostmaps': null })['s2-ghostmaps'], + null, + ); + + let reads = 0; + let subscriptions = 0; + let unsubscribed = 0; + const source = { + getCatalog: async () => { + reads++; + return { commit: 'a'.repeat(40), products: [] }; + }, + getFile: async () => { + throw new Error('Empty catalog must not read files'); + }, + subscribe: () => { + subscriptions++; + return () => { + unsubscribed++; + }; + }, + }; + const supplied = createStandaloneLayerSources({ 's2-ghostmaps': source }); + assert.equal(supplied['s2-ghostmaps'], source); + const configured = catalog(supplied, t); + assert.equal( + configured.getSourceAvailability('s2-ghostmaps').available, + true, + ); + const previous = Object.getOwnPropertyDescriptor(globalThis, 'document'); + globalThis.document = { addEventListener() {}, removeEventListener() {} }; + t.after(() => { + if (previous) Object.defineProperty(globalThis, 'document', previous); + else delete globalThis.document; + }); + const layer = configured.get('s2-ghostmaps'); + t.after(() => layer.destroy()); + layer.init({ scene: { requestRender() {} } }); + layer.enable(); + await layer.update(); + layer.disable(); + assert.equal(reads, 1); + assert.equal(subscriptions, 1); + assert.equal(unsubscribed, 1); + assert.equal(layer.getStats().commit, 'a'.repeat(40)); + assert.equal(fetches, 0); +}); + test('empty Street Level uses the shared unavailable lifecycle without initialization', async (t) => { const actual = catalog({}, t); const manager = new LayerLifecycle( diff --git a/src/layers/ghostMaps/index.js b/src/layers/ghostMaps/index.js index b0aa7c88b..01d564d05 100644 --- a/src/layers/ghostMaps/index.js +++ b/src/layers/ghostMaps/index.js @@ -195,7 +195,9 @@ export function createGhostMapsLayer({ /** Build one product's data source off-scene, then swap it in. */ async function load(product, signal) { const bytes = await source.getFile(product.id, product.version, { signal }); + signal.throwIfAborted(); const entries = await readZipEntries(bytes); + signal.throwIfAborted(); const main = kmzMainDocument(entries); if (!main) throw new Error(msg('That KMZ holds no KML document.')); const urls = []; @@ -210,117 +212,118 @@ export function createGhostMapsLayer({ ? resolvePacked(href) : null; }; - let parsed; try { - parsed = readGhostMapsKml( + const parsed = readGhostMapsKml( parseXml(decodeXml(entries.get(main)), 'KML'), resolveImage, ); - } catch (error) { - urls.forEach((url) => URL.revokeObjectURL(url)); - throw error; - } - entries.clear(); - signal.throwIfAborted(); - const color = PRODUCT_COLORS[product.id] || ACCENT; - const dataSource = new Cesium.CustomDataSource( - `${GHOSTMAPS_LAYER_ID}:${product.id}`, - ); - const features = []; - const pins = []; - dataSource.entities.suspendEvents(); - parsed.features.forEach((feature, index) => { - const key = `${product.id}:${index}`; - const { style } = feature; - feature.geometries.forEach((geometry, part) => { - const id = `${PICK_PREFIX}${key}:${part}`; - if (geometry.type === 'point') { - const position = Cesium.Cartesian3.fromDegrees(...geometry.position); - const entity = dataSource.entities.add({ - id, - position, - billboard: { - image: style.icon || markerImage(color), - scale: Math.min( - 1.5, - Math.max(0.35, (style.iconScale ?? 1) * 0.7), - ), - color: - style.icon && style.iconColor - ? cesiumColor(style.iconColor) - : undefined, - heightReference: Cesium.HeightReference.CLAMP_TO_GROUND, - verticalOrigin: Cesium.VerticalOrigin.CENTER, - // Ground-clamped icons would sink into terrain and 3D tiles; the - // horizon cull hides the far side instead (see `_horizon`). - disableDepthTestDistance: Number.POSITIVE_INFINITY, - }, - }); - pins.push({ - position, - get show() { - return entity.show; - }, - set show(value) { - entity.show = value; - }, - }); - } else if (geometry.type === 'line') { - dataSource.entities.add({ - id, - polyline: { - positions: Cesium.Cartesian3.fromDegreesArray( - geometry.positions.flat(), - ), - width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)), - material: cesiumColor(style.lineColor, color), - clampToGround: true, - }, - }); - } else { - const ring = (points) => - Cesium.Cartesian3.fromDegreesArray(points.flat()); - dataSource.entities.add({ - id, - polygon: { - hierarchy: new Cesium.PolygonHierarchy( - ring(geometry.outer), - geometry.holes.map( - (hole) => new Cesium.PolygonHierarchy(ring(hole)), + entries.clear(); + signal.throwIfAborted(); + const color = PRODUCT_COLORS[product.id] || ACCENT; + const dataSource = new Cesium.CustomDataSource( + `${GHOSTMAPS_LAYER_ID}:${product.id}`, + ); + const features = []; + const pins = []; + dataSource.entities.suspendEvents(); + parsed.features.forEach((feature, index) => { + const key = `${product.id}:${index}`; + const { style } = feature; + feature.geometries.forEach((geometry, part) => { + const id = `${PICK_PREFIX}${key}:${part}`; + if (geometry.type === 'point') { + const position = Cesium.Cartesian3.fromDegrees( + ...geometry.position, + ); + const entity = dataSource.entities.add({ + id, + position, + billboard: { + image: style.icon || markerImage(color), + scale: Math.min( + 1.5, + Math.max(0.35, (style.iconScale ?? 1) * 0.7), ), - ), - material: - style.fill === false - ? Cesium.Color.TRANSPARENT - : cesiumColor( - style.polyColor, - color, - style.polyColor ? undefined : 0.25, - ), - classificationType: Cesium.ClassificationType.BOTH, - }, - }); - if (style.outline !== false) + color: + style.icon && style.iconColor + ? cesiumColor(style.iconColor) + : undefined, + heightReference: Cesium.HeightReference.CLAMP_TO_GROUND, + verticalOrigin: Cesium.VerticalOrigin.CENTER, + // Ground-clamped icons would sink into terrain and 3D tiles; the + // horizon cull hides the far side instead (see `_horizon`). + disableDepthTestDistance: Number.POSITIVE_INFINITY, + }, + }); + pins.push({ + position, + get show() { + return entity.show; + }, + set show(value) { + entity.show = value; + }, + }); + } else if (geometry.type === 'line') { dataSource.entities.add({ - id: `${id}:outline`, + id, polyline: { - positions: ring([...geometry.outer, geometry.outer[0]]), - width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)), + positions: Cesium.Cartesian3.fromDegreesArray( + geometry.positions.flat(), + ), + width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)), material: cesiumColor(style.lineColor, color), clampToGround: true, }, }); - } + } else { + const ring = (points) => + Cesium.Cartesian3.fromDegreesArray(points.flat()); + dataSource.entities.add({ + id, + polygon: { + hierarchy: new Cesium.PolygonHierarchy( + ring(geometry.outer), + geometry.holes.map( + (hole) => new Cesium.PolygonHierarchy(ring(hole)), + ), + ), + material: + style.fill === false + ? Cesium.Color.TRANSPARENT + : cesiumColor( + style.polyColor, + color, + style.polyColor ? undefined : 0.25, + ), + classificationType: Cesium.ClassificationType.BOTH, + }, + }); + if (style.outline !== false) + dataSource.entities.add({ + id: `${id}:outline`, + polyline: { + positions: ring([...geometry.outer, geometry.outer[0]]), + width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)), + material: cesiumColor(style.lineColor, color), + clampToGround: true, + }, + }); + } + }); + features.push({ + name: feature.name, + folder: feature.folder, + description: feature.description, + anchor: anchorOf(feature.geometries[0]), + }); }); - features.push({ - name: feature.name, - folder: feature.folder, - description: feature.description, - anchor: anchorOf(feature.geometries[0]), - }); - }); - dataSource.entities.resumeEvents(); - return { version: product.version, dataSource, urls, features, pins }; + dataSource.entities.resumeEvents(); + return { version: product.version, dataSource, urls, features, pins }; + } catch (error) { + urls.forEach((url) => URL.revokeObjectURL(url)); + throw error; + } } /** Bring the globe in line with the catalog and the chip selection. */ diff --git a/src/layers/ghostMaps/lifecycle.test.mjs b/src/layers/ghostMaps/lifecycle.test.mjs new file mode 100644 index 000000000..51d546017 --- /dev/null +++ b/src/layers/ghostMaps/lifecycle.test.mjs @@ -0,0 +1,180 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import * as Cesium from 'cesium'; +import { createGhostMapsLayer } from './index.js'; + +function element(localName, children = [], value = '', attributes = {}) { + const node = { + localName, + children, + get textContent() { + return value + children.map((child) => child.textContent).join(''); + }, + getAttribute: (key) => attributes[key] ?? null, + getElementsByTagName(name) { + return children.flatMap((child) => [ + ...(name === '*' || name === child.localName ? [child] : []), + ...child.getElementsByTagName(name), + ]); + }, + }; + for (const child of children) child.parentElement = node; + return node; +} + +function documentFixture() { + return element('document', [ + element('kml', [ + element('Document', [ + element( + 'Style', + [ + element('IconStyle', [ + element('Icon', [element('href', [], 'icon.png')]), + ]), + ], + '', + { id: 'pin' }, + ), + element('Placemark', [ + element('name', [], 'Test report'), + element('styleUrl', [], '#pin'), + element('Point', [element('coordinates', [], '12,34')]), + ]), + ]), + ]), + ]); +} + +// A stored KMZ with a local icon; the XML DOM double supplies its one point. +function fixtureKmz() { + const files = [ + ['doc.kml', ''], + ['icon.png', 'packed icon'], + ]; + const local = []; + const central = []; + let offset = 0; + for (const [name, value] of files) { + const key = Buffer.from(name); + const bytes = Buffer.from(value); + const header = Buffer.alloc(30); + header.writeUInt32LE(0x04034b50); + header.writeUInt32LE(bytes.length, 18); + header.writeUInt32LE(bytes.length, 22); + header.writeUInt16LE(key.length, 26); + const directory = Buffer.alloc(46); + directory.writeUInt32LE(0x02014b50); + directory.writeUInt32LE(bytes.length, 20); + directory.writeUInt32LE(bytes.length, 24); + directory.writeUInt16LE(key.length, 28); + directory.writeUInt32LE(offset, 42); + local.push(header, key, bytes); + central.push(directory, key); + offset += header.length + key.length + bytes.length; + } + const directory = Buffer.concat(central); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50); + end.writeUInt16LE(files.length, 8); + end.writeUInt16LE(files.length, 10); + end.writeUInt32LE(directory.length, 12); + end.writeUInt32LE(offset, 16); + return new Uint8Array(Buffer.concat([...local, directory, end])); +} + +function fixture(t, { beforeFileReturns, onCreateUrl } = {}) { + const previous = Object.getOwnPropertyDescriptor(globalThis, 'DOMParser'); + globalThis.DOMParser = class { + parseFromString() { + return documentFixture(); + } + }; + t.after(() => { + if (previous) Object.defineProperty(globalThis, 'DOMParser', previous); + else delete globalThis.DOMParser; + }); + const created = []; + const revoked = []; + t.mock.method(URL, 'createObjectURL', () => { + const url = `blob:local-${created.length}`; + created.push(url); + onCreateUrl?.(layer); + return url; + }); + t.mock.method(URL, 'revokeObjectURL', (url) => revoked.push(url)); + t.mock.method(console, 'warn', () => {}); + const mounted = new Set(); + const viewer = { + scene: { requestRender() {} }, + dataSources: { + async add(dataSource) { + mounted.add(dataSource); + return dataSource; + }, + remove(dataSource) { + return mounted.delete(dataSource); + }, + }, + }; + const layer = createGhostMapsLayer({ + storage: null, + source: { + getCatalog: async () => ({ + products: [{ id: 'cip', version: 'a'.repeat(40), defaultOn: true }], + }), + getFile: async () => { + beforeFileReturns?.(layer); + return fixtureKmz(); + }, + }, + }); + layer.init(viewer); + layer.enable(); + t.after(() => layer.destroy()); + return { layer, viewer, mounted, created, revoked }; +} + +test('cancelling a file read does not allocate packed-image URLs', async (t) => { + const { layer, viewer, mounted, created, revoked } = fixture(t, { + beforeFileReturns: (layer) => queueMicrotask(() => layer.disable()), + }); + await layer.update(viewer); + assert.equal(created.length, 0); + assert.equal(revoked.length, 0); + assert.equal(mounted.size, 0); +}); + +test('cancellation during parsing releases every packed-image URL', async (t) => { + const { layer, viewer, mounted, created, revoked } = fixture(t, { + onCreateUrl: (layer) => layer.disable(), + }); + await layer.update(viewer); + assert.equal(created.length, 1); + assert.deepEqual(revoked, created); + assert.equal(mounted.size, 0); +}); + +test('entity construction failure releases parsed image URLs', async (t) => { + const { layer, viewer, mounted, created, revoked } = fixture(t); + t.mock.method(Cesium.EntityCollection.prototype, 'add', () => { + throw new Error('fixture entity failure'); + }); + await layer.update(viewer); + assert.equal(created.length, 1); + assert.deepEqual(revoked, created); + assert.equal(mounted.size, 0); + assert.equal(layer.getStats().error, 'fixture entity failure'); +}); + +test('successful load keeps icons until the product is disabled', async (t) => { + const { layer, viewer, mounted, created, revoked } = fixture(t); + await layer.update(viewer); + assert.equal(created.length, 1); + assert.equal(revoked.length, 0); + assert.equal(mounted.size, 1); + assert.equal(layer.getStats().count, 1); + layer.disable(); + assert.deepEqual(revoked, created); + assert.equal(mounted.size, 0); +}); diff --git a/src/standalone/layerSources.js b/src/standalone/layerSources.js index 935351685..693ddd47f 100644 --- a/src/standalone/layerSources.js +++ b/src/standalone/layerSources.js @@ -19,6 +19,7 @@ import { createWeatherSource } from '../layers/weather/source.js'; import { createCycloneSource } from '../layers/cyclones/source.js'; import { createWindSource } from '../layers/wind/source.js'; import { createFirmsSource } from '../layers/firms/source.js'; +import { createGhostMapsSource } from '../layers/ghostMaps/source.js'; import { createMapillarySource } from '../layers/streetLevel/providers/mapillary/source.js'; import { createReferenceSources } from '../sources/reference.js'; export { createReferenceSources as createStandaloneReferenceSources } from '../sources/reference.js'; @@ -50,6 +51,7 @@ export function createStandaloneLayerSources(overrides = {}) { launches: createLaunchSource(), alpr: createAlprTileSource(), firms: createFirmsSource(), + 's2-ghostmaps': createGhostMapsSource(), wind: createWindSource(), weather: createWeatherSource(), cyclones: createCycloneSource(),