fix: address GhostMaps resource and source ownership review findings

changelog.d/1297-s2-ghostmaps-layer.md

@@ -7,3 +7,6 @@
   products that changed (Black Pine Defense).
 - Clear a temporary GhostMaps upstream error and notify open pages when the
   repository becomes reachable again, including when no new edition was published.
+- Release packed-image resources when a load is cancelled or fails, respect
+  injected or omitted GhostMaps sources, and wait for background cache warm-up
+  before removing provider test fixtures.

docs/CURRENT-STATE.md

@@ -5133,6 +5133,7 @@ are omitted rather than framing the wrong part of the globe.
   - The layer reloads only products whose blob changed. The new edition's data source is added before the old one is removed. A product that leaves the catalog leaves the globe.
   - The row reads `LIVE` while the stream is open. A five-minute catalog read is the backstop.
 - **Files.** `/api/ghostmaps/file/{product}` serves only catalog products. Files are downloaded by commit-pinned raw URLs, so a CDN cannot return an older edition. They are cached on disk by blob sha, and blobs no current product uses are pruned. A request whose `?v=` names the current blob is `immutable`. Repository paths never reach the page.
+- **Source ownership.** The application catalog receives `sources['s2-ghostmaps']` through the shared source contracts; an omitted or null source stays unavailable without opening a stream or fetching. The standalone source factory supplies the same-origin implementation and accepts replacements. Packed-image URLs are released on cancelled or failed product construction, and remain owned by a successfully loaded product until it is removed.
 - **Rendering.** The page parses the KMZ with the import module's zip, KMZ and XML helpers into one `CustomDataSource` per product.
   - Points become ground-clamped billboards with the depth test off, so they never sink into terrain or 3D tiles. Because the Google 3D map hides the globe, nothing else would stop far-side pins drawing through the planet. Pins are therefore horizon-culled with Street Level's `createHorizonCull` before their first frame and again whenever the camera moves.
   - Lines and polygon outlines are ground-clamped polylines.

scripts/format-scope.json

@@ -884,6 +884,7 @@
   "src/layers/ghostMaps/source.js",
   "src/layers/ghostMaps/catalog.test.mjs",
   "src/layers/ghostMaps/records.test.mjs",
+  "src/layers/ghostMaps/lifecycle.test.mjs",
   "scripts/qa-ghostmaps.mjs",
   "src/layers/cyclones/index.js",
   "src/layers/cyclones/rendering.js",

server/providers/ghostMaps.test.mjs

@@ -65,6 +65,17 @@ async function withCache(fn) {
   }
 }
 
+// checkNow announces the catalog before background warm-up finishes. Join the
+// same file reads before changing editions or removing the temporary cache.
+async function warmDefaults(watcher) {
+  await Promise.all(
+    watcher
+      .catalog()
+      .products.filter((product) => product.defaultOn)
+      .map((product) => watcher.file(product.id)),
+  );
+}
+
 test('a new commit is announced once, naming only the products that changed', () =>
   withCache(async (cacheDir) => {
     const { repo, fetchImpl } = fakeGitHub();
@@ -78,6 +89,7 @@ test('a new commit is announced once, naming only the products that changed', ()
     watcher.subscribe((catalog) => events.push(catalog));
 
     assert.equal(await watcher.checkNow(), true);
+    await warmDefaults(watcher);
     const first = watcher.catalog();
     assert.equal(first.commit, sha('1'));
     assert.equal(first.message, 'Oct 18 Update');
@@ -105,6 +117,7 @@ test('a new commit is announced once, naming only the products that changed', ()
       },
     ]);
     assert.equal(await watcher.checkNow(), true);
+    await warmDefaults(watcher);
     assert.equal(events.length, 2);
     const cip = events[1].products.find((p) => p.id === 'cip');
     assert.equal(cip.version, sha('c'));
@@ -165,7 +178,9 @@ test('the catalog survives a restart from disk, and upstream failures are report
   withCache(async (cacheDir) => {
     const { fetchImpl } = fakeGitHub();
     const quiet = { log: () => {}, warn: () => {} };
-    await createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet }).checkNow();
+    const online = createGhostMapsWatcher({ fetchImpl, cacheDir, ...quiet });
+    await online.checkNow();
+    await warmDefaults(online);
     const offline = createGhostMapsWatcher({
       fetchImpl: async () => new Response('down', { status: 503 }),
       cacheDir,

src/app/constructCatalog.js

@@ -127,7 +127,7 @@ export function createApplicationCatalog({
         createApplicationGnssIntegrity({
           source: sources['gnss-integrity'],
         }),
-        createApplicationGhostMaps(),
+        createApplicationGhostMaps({ source: sources['s2-ghostmaps'] }),
         createApplicationAlpr({ surface, source: sources.alpr }),
         createApplicationStreetLevel({
           surface,

src/app/layers/ghostMaps.js

@@ -1,8 +1,5 @@
 import * as Cesium from 'cesium';
-import {
-  createGhostMapsLayer,
-  createGhostMapsSource,
-} from '../../layers/ghostMaps/index.js';
+import { createGhostMapsLayer } from '../../layers/ghostMaps/index.js';
 import * as picking from '../../data/pickRegistry.js';
 import * as overlays from '../../overlays/worldOverlay.js';
 import { isPointerFree } from '../../data/inputOwnership.js';
@@ -10,7 +7,6 @@ import { isPointerFree } from '../../data/inputOwnership.js';
 /** Wire the S2 Underground GhostMaps layer into the application catalog. */
 export function createApplicationGhostMaps(options) {
   return createGhostMapsLayer({
-    source: createGhostMapsSource(),
     overlayHost: {
       setEntries: overlays.setOverlayEntries,
       setVisible: overlays.setOverlaySourceVisible,

src/app/sourceComposition.js

@@ -48,6 +48,7 @@ export const CATALOG_SOURCE_CONTRACTS = Object.freeze({
   'fire-perimeters': contract(['fire-perimeters'], ['getSnapshot']),
   'air-quality': contract(['air-quality'], ['getSnapshot']),
   'gnss-integrity': contract(['gnss-integrity'], ['getSnapshot']),
+  's2-ghostmaps': contract(['s2-ghostmaps'], ['getCatalog', 'getFile']),
   cables: contract(['telegeography-submarine-cables'], ['fetch']),
   transit: contract(['transit'], ['requestSnapshot', 'getHistory']),
 });

src/app/sourceComposition.test.mjs

@@ -86,6 +86,70 @@ test('standalone defaults allow exact replacements and explicit removal, and rej
   );
 });
 
+test('GhostMaps obeys source omission, replacement and explicit removal', async (t) => {
+  let fetches = 0;
+  t.mock.method(globalThis, 'fetch', async () => {
+    fetches++;
+    throw new Error('Unexpected standalone acquisition');
+  });
+  const missing = catalog({}, t);
+  assert.equal(missing.getSourceAvailability('s2-ghostmaps')?.available, false);
+  const manager = new LayerLifecycle(
+    {},
+    { getSourceAvailability: missing.getSourceAvailability },
+  );
+  manager.register(missing.get('s2-ghostmaps'));
+  assert.equal(await manager.setEnabled('s2-ghostmaps', true), false);
+  assert.equal(manager.layers.get('s2-ghostmaps').initialized, false);
+  assert.equal(
+    createStandaloneLayerSources({ 's2-ghostmaps': null })['s2-ghostmaps'],
+    null,
+  );
+
+  let reads = 0;
+  let subscriptions = 0;
+  let unsubscribed = 0;
+  const source = {
+    getCatalog: async () => {
+      reads++;
+      return { commit: 'a'.repeat(40), products: [] };
+    },
+    getFile: async () => {
+      throw new Error('Empty catalog must not read files');
+    },
+    subscribe: () => {
+      subscriptions++;
+      return () => {
+        unsubscribed++;
+      };
+    },
+  };
+  const supplied = createStandaloneLayerSources({ 's2-ghostmaps': source });
+  assert.equal(supplied['s2-ghostmaps'], source);
+  const configured = catalog(supplied, t);
+  assert.equal(
+    configured.getSourceAvailability('s2-ghostmaps').available,
+    true,
+  );
+  const previous = Object.getOwnPropertyDescriptor(globalThis, 'document');
+  globalThis.document = { addEventListener() {}, removeEventListener() {} };
+  t.after(() => {
+    if (previous) Object.defineProperty(globalThis, 'document', previous);
+    else delete globalThis.document;
+  });
+  const layer = configured.get('s2-ghostmaps');
+  t.after(() => layer.destroy());
+  layer.init({ scene: { requestRender() {} } });
+  layer.enable();
+  await layer.update();
+  layer.disable();
+  assert.equal(reads, 1);
+  assert.equal(subscriptions, 1);
+  assert.equal(unsubscribed, 1);
+  assert.equal(layer.getStats().commit, 'a'.repeat(40));
+  assert.equal(fetches, 0);
+});
+
 test('empty Street Level uses the shared unavailable lifecycle without initialization', async (t) => {
   const actual = catalog({}, t);
   const manager = new LayerLifecycle(

src/layers/ghostMaps/index.js

@@ -195,7 +195,9 @@ export function createGhostMapsLayer({
   /** Build one product's data source off-scene, then swap it in. */
   async function load(product, signal) {
     const bytes = await source.getFile(product.id, product.version, { signal });
+    signal.throwIfAborted();
     const entries = await readZipEntries(bytes);
+    signal.throwIfAborted();
     const main = kmzMainDocument(entries);
     if (!main) throw new Error(msg('That KMZ holds no KML document.'));
     const urls = [];
@@ -210,117 +212,118 @@ export function createGhostMapsLayer({
         ? resolvePacked(href)
         : null;
     };
-    let parsed;
     try {
-      parsed = readGhostMapsKml(
+      const parsed = readGhostMapsKml(
         parseXml(decodeXml(entries.get(main)), 'KML'),
         resolveImage,
       );
-    } catch (error) {
-      urls.forEach((url) => URL.revokeObjectURL(url));
-      throw error;
-    }
-    entries.clear();
-    signal.throwIfAborted();
-    const color = PRODUCT_COLORS[product.id] || ACCENT;
-    const dataSource = new Cesium.CustomDataSource(
-      `${GHOSTMAPS_LAYER_ID}:${product.id}`,
-    );
-    const features = [];
-    const pins = [];
-    dataSource.entities.suspendEvents();
-    parsed.features.forEach((feature, index) => {
-      const key = `${product.id}:${index}`;
-      const { style } = feature;
-      feature.geometries.forEach((geometry, part) => {
-        const id = `${PICK_PREFIX}${key}:${part}`;
-        if (geometry.type === 'point') {
-          const position = Cesium.Cartesian3.fromDegrees(...geometry.position);
-          const entity = dataSource.entities.add({
-            id,
-            position,
-            billboard: {
-              image: style.icon || markerImage(color),
-              scale: Math.min(
-                1.5,
-                Math.max(0.35, (style.iconScale ?? 1) * 0.7),
-              ),
-              color:
-                style.icon && style.iconColor
-                  ? cesiumColor(style.iconColor)
-                  : undefined,
-              heightReference: Cesium.HeightReference.CLAMP_TO_GROUND,
-              verticalOrigin: Cesium.VerticalOrigin.CENTER,
-              // Ground-clamped icons would sink into terrain and 3D tiles; the
-              // horizon cull hides the far side instead (see `_horizon`).
-              disableDepthTestDistance: Number.POSITIVE_INFINITY,
-            },
-          });
-          pins.push({
-            position,
-            get show() {
-              return entity.show;
-            },
-            set show(value) {
-              entity.show = value;
-            },
-          });
-        } else if (geometry.type === 'line') {
-          dataSource.entities.add({
-            id,
-            polyline: {
-              positions: Cesium.Cartesian3.fromDegreesArray(
-                geometry.positions.flat(),
-              ),
-              width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)),
-              material: cesiumColor(style.lineColor, color),
-              clampToGround: true,
-            },
-          });
-        } else {
-          const ring = (points) =>
-            Cesium.Cartesian3.fromDegreesArray(points.flat());
-          dataSource.entities.add({
-            id,
-            polygon: {
-              hierarchy: new Cesium.PolygonHierarchy(
-                ring(geometry.outer),
-                geometry.holes.map(
-                  (hole) => new Cesium.PolygonHierarchy(ring(hole)),
+      entries.clear();
+      signal.throwIfAborted();
+      const color = PRODUCT_COLORS[product.id] || ACCENT;
+      const dataSource = new Cesium.CustomDataSource(
+        `${GHOSTMAPS_LAYER_ID}:${product.id}`,
+      );
+      const features = [];
+      const pins = [];
+      dataSource.entities.suspendEvents();
+      parsed.features.forEach((feature, index) => {
+        const key = `${product.id}:${index}`;
+        const { style } = feature;
+        feature.geometries.forEach((geometry, part) => {
+          const id = `${PICK_PREFIX}${key}:${part}`;
+          if (geometry.type === 'point') {
+            const position = Cesium.Cartesian3.fromDegrees(
+              ...geometry.position,
+            );
+            const entity = dataSource.entities.add({
+              id,
+              position,
+              billboard: {
+                image: style.icon || markerImage(color),
+                scale: Math.min(
+                  1.5,
+                  Math.max(0.35, (style.iconScale ?? 1) * 0.7),
                 ),
-              ),
-              material:
-                style.fill === false
-                  ? Cesium.Color.TRANSPARENT
-                  : cesiumColor(
-                      style.polyColor,
-                      color,
-                      style.polyColor ? undefined : 0.25,
-                    ),
-              classificationType: Cesium.ClassificationType.BOTH,
-            },
-          });
-          if (style.outline !== false)
+                color:
+                  style.icon && style.iconColor
+                    ? cesiumColor(style.iconColor)
+                    : undefined,
+                heightReference: Cesium.HeightReference.CLAMP_TO_GROUND,
+                verticalOrigin: Cesium.VerticalOrigin.CENTER,
+                // Ground-clamped icons would sink into terrain and 3D tiles; the
+                // horizon cull hides the far side instead (see `_horizon`).
+                disableDepthTestDistance: Number.POSITIVE_INFINITY,
+              },
+            });
+            pins.push({
+              position,
+              get show() {
+                return entity.show;
+              },
+              set show(value) {
+                entity.show = value;
+              },
+            });
+          } else if (geometry.type === 'line') {
             dataSource.entities.add({
-              id: `${id}:outline`,
+              id,
               polyline: {
-                positions: ring([...geometry.outer, geometry.outer[0]]),
-                width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)),
+                positions: Cesium.Cartesian3.fromDegreesArray(
+                  geometry.positions.flat(),
+                ),
+                width: Math.min(8, Math.max(1.5, style.lineWidth ?? 2)),
                 material: cesiumColor(style.lineColor, color),
                 clampToGround: true,
               },
             });
-        }
+          } else {
+            const ring = (points) =>
+              Cesium.Cartesian3.fromDegreesArray(points.flat());
+            dataSource.entities.add({
+              id,
+              polygon: {
+                hierarchy: new Cesium.PolygonHierarchy(
+                  ring(geometry.outer),
+                  geometry.holes.map(
+                    (hole) => new Cesium.PolygonHierarchy(ring(hole)),
+                  ),
+                ),
+                material:
+                  style.fill === false
+                    ? Cesium.Color.TRANSPARENT
+                    : cesiumColor(
+                        style.polyColor,
+                        color,
+                        style.polyColor ? undefined : 0.25,
+                      ),
+                classificationType: Cesium.ClassificationType.BOTH,
+              },
+            });
+            if (style.outline !== false)
+              dataSource.entities.add({
+                id: `${id}:outline`,
+                polyline: {
+                  positions: ring([...geometry.outer, geometry.outer[0]]),
+                  width: Math.min(6, Math.max(1.5, style.lineWidth ?? 2)),
+                  material: cesiumColor(style.lineColor, color),
+                  clampToGround: true,
+                },
+              });
+          }
+        });
+        features.push({
+          name: feature.name,
+          folder: feature.folder,
+          description: feature.description,
+          anchor: anchorOf(feature.geometries[0]),
+        });
       });
-      features.push({
-        name: feature.name,
-        folder: feature.folder,
-        description: feature.description,
-        anchor: anchorOf(feature.geometries[0]),
-      });
-    });
-    dataSource.entities.resumeEvents();
-    return { version: product.version, dataSource, urls, features, pins };
+      dataSource.entities.resumeEvents();
+      return { version: product.version, dataSource, urls, features, pins };
+    } catch (error) {
+      urls.forEach((url) => URL.revokeObjectURL(url));
+      throw error;
+    }
   }
 
   /** Bring the globe in line with the catalog and the chip selection. */

src/layers/ghostMaps/lifecycle.test.mjs

@@ -0,0 +1,180 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import * as Cesium from 'cesium';
+import { createGhostMapsLayer } from './index.js';
+
+function element(localName, children = [], value = '', attributes = {}) {
+  const node = {
+    localName,
+    children,
+    get textContent() {
+      return value + children.map((child) => child.textContent).join('');
+    },
+    getAttribute: (key) => attributes[key] ?? null,
+    getElementsByTagName(name) {
+      return children.flatMap((child) => [
+        ...(name === '*' || name === child.localName ? [child] : []),
+        ...child.getElementsByTagName(name),
+      ]);
+    },
+  };
+  for (const child of children) child.parentElement = node;
+  return node;
+}
+
+function documentFixture() {
+  return element('document', [
+    element('kml', [
+      element('Document', [
+        element(
+          'Style',
+          [
+            element('IconStyle', [
+              element('Icon', [element('href', [], 'icon.png')]),
+            ]),
+          ],
+          '',
+          { id: 'pin' },
+        ),
+        element('Placemark', [
+          element('name', [], 'Test report'),
+          element('styleUrl', [], '#pin'),
+          element('Point', [element('coordinates', [], '12,34')]),
+        ]),
+      ]),
+    ]),
+  ]);
+}
+
+// A stored KMZ with a local icon; the XML DOM double supplies its one point.
+function fixtureKmz() {
+  const files = [
+    ['doc.kml', '<kml/>'],
+    ['icon.png', 'packed icon'],
+  ];
+  const local = [];
+  const central = [];
+  let offset = 0;
+  for (const [name, value] of files) {
+    const key = Buffer.from(name);
+    const bytes = Buffer.from(value);
+    const header = Buffer.alloc(30);
+    header.writeUInt32LE(0x04034b50);
+    header.writeUInt32LE(bytes.length, 18);
+    header.writeUInt32LE(bytes.length, 22);
+    header.writeUInt16LE(key.length, 26);
+    const directory = Buffer.alloc(46);
+    directory.writeUInt32LE(0x02014b50);
+    directory.writeUInt32LE(bytes.length, 20);
+    directory.writeUInt32LE(bytes.length, 24);
+    directory.writeUInt16LE(key.length, 28);
+    directory.writeUInt32LE(offset, 42);
+    local.push(header, key, bytes);
+    central.push(directory, key);
+    offset += header.length + key.length + bytes.length;
+  }
+  const directory = Buffer.concat(central);
+  const end = Buffer.alloc(22);
+  end.writeUInt32LE(0x06054b50);
+  end.writeUInt16LE(files.length, 8);
+  end.writeUInt16LE(files.length, 10);
+  end.writeUInt32LE(directory.length, 12);
+  end.writeUInt32LE(offset, 16);
+  return new Uint8Array(Buffer.concat([...local, directory, end]));
+}
+
+function fixture(t, { beforeFileReturns, onCreateUrl } = {}) {
+  const previous = Object.getOwnPropertyDescriptor(globalThis, 'DOMParser');
+  globalThis.DOMParser = class {
+    parseFromString() {
+      return documentFixture();
+    }
+  };
+  t.after(() => {
+    if (previous) Object.defineProperty(globalThis, 'DOMParser', previous);
+    else delete globalThis.DOMParser;
+  });
+  const created = [];
+  const revoked = [];
+  t.mock.method(URL, 'createObjectURL', () => {
+    const url = `blob:local-${created.length}`;
+    created.push(url);
+    onCreateUrl?.(layer);
+    return url;
+  });
+  t.mock.method(URL, 'revokeObjectURL', (url) => revoked.push(url));
+  t.mock.method(console, 'warn', () => {});
+  const mounted = new Set();
+  const viewer = {
+    scene: { requestRender() {} },
+    dataSources: {
+      async add(dataSource) {
+        mounted.add(dataSource);
+        return dataSource;
+      },
+      remove(dataSource) {
+        return mounted.delete(dataSource);
+      },
+    },
+  };
+  const layer = createGhostMapsLayer({
+    storage: null,
+    source: {
+      getCatalog: async () => ({
+        products: [{ id: 'cip', version: 'a'.repeat(40), defaultOn: true }],
+      }),
+      getFile: async () => {
+        beforeFileReturns?.(layer);
+        return fixtureKmz();
+      },
+    },
+  });
+  layer.init(viewer);
+  layer.enable();
+  t.after(() => layer.destroy());
+  return { layer, viewer, mounted, created, revoked };
+}
+
+test('cancelling a file read does not allocate packed-image URLs', async (t) => {
+  const { layer, viewer, mounted, created, revoked } = fixture(t, {
+    beforeFileReturns: (layer) => queueMicrotask(() => layer.disable()),
+  });
+  await layer.update(viewer);
+  assert.equal(created.length, 0);
+  assert.equal(revoked.length, 0);
+  assert.equal(mounted.size, 0);
+});
+
+test('cancellation during parsing releases every packed-image URL', async (t) => {
+  const { layer, viewer, mounted, created, revoked } = fixture(t, {
+    onCreateUrl: (layer) => layer.disable(),
+  });
+  await layer.update(viewer);
+  assert.equal(created.length, 1);
+  assert.deepEqual(revoked, created);
+  assert.equal(mounted.size, 0);
+});
+
+test('entity construction failure releases parsed image URLs', async (t) => {
+  const { layer, viewer, mounted, created, revoked } = fixture(t);
+  t.mock.method(Cesium.EntityCollection.prototype, 'add', () => {
+    throw new Error('fixture entity failure');
+  });
+  await layer.update(viewer);
+  assert.equal(created.length, 1);
+  assert.deepEqual(revoked, created);
+  assert.equal(mounted.size, 0);
+  assert.equal(layer.getStats().error, 'fixture entity failure');
+});
+
+test('successful load keeps icons until the product is disabled', async (t) => {
+  const { layer, viewer, mounted, created, revoked } = fixture(t);
+  await layer.update(viewer);
+  assert.equal(created.length, 1);
+  assert.equal(revoked.length, 0);
+  assert.equal(mounted.size, 1);
+  assert.equal(layer.getStats().count, 1);
+  layer.disable();
+  assert.deepEqual(revoked, created);
+  assert.equal(mounted.size, 0);
+});

src/standalone/layerSources.js

@@ -19,6 +19,7 @@ import { createWeatherSource } from '../layers/weather/source.js';
 import { createCycloneSource } from '../layers/cyclones/source.js';
 import { createWindSource } from '../layers/wind/source.js';
 import { createFirmsSource } from '../layers/firms/source.js';
+import { createGhostMapsSource } from '../layers/ghostMaps/source.js';
 import { createMapillarySource } from '../layers/streetLevel/providers/mapillary/source.js';
 import { createReferenceSources } from '../sources/reference.js';
 export { createReferenceSources as createStandaloneReferenceSources } from '../sources/reference.js';
@@ -50,6 +51,7 @@ export function createStandaloneLayerSources(overrides = {}) {
     launches: createLaunchSource(),
     alpr: createAlprTileSource(),
     firms: createFirmsSource(),
+    's2-ghostmaps': createGhostMapsSource(),
     wind: createWindSource(),
     weather: createWeatherSource(),
     cyclones: createCycloneSource(),