perf(shipwrecks): draw the wreck markers from one billboard collection, on the wreck, and rebuild them only on new data

- The ~8,000 markers were one Cesium entity each; they are now one
  BillboardCollection (as FIRMS, vessels and wildlife sightings do), ordered
  at the bottom of the shared sprite stack so moving traffic draws over it.
- eyeOffset.z was the death toll in metres, so a 1,500-death wreck's marker
  sat 1.5 km toward the camera and clipped or drifted when close. It is gone;
  markers are added by rising toll, so a bigger toll still draws on top.
- The page asks every 6 h but the proxy changes data at most daily: the proxy
  now sends a version that changes with either source's data, and the layer
  rebuilds only when it does.
- Adds a controller test: enable/disable, teardown, draw order, unchanged
  snapshots, a late response after turning off, and the period closing an
  open card.

scripts/format-scope.json

@@ -1279,6 +1279,7 @@
   "src/layers/shipwrecks/cards.test.mjs",
   "src/layers/shipwrecks/eras.test.mjs",
   "src/layers/shipwrecks/period.test.mjs",
+  "src/layers/shipwrecks/index.test.mjs",
   "src/data/shipwrecksProxy.test.mjs",
   "src/app/sourceComposition.js",
   "src/app/sourceComposition.test.mjs",

server/providers/shipwrecks.js

@@ -143,7 +143,12 @@ export function shipwrecksProxy({
       };
     }
     return {
-      value: { fetchedAt, sources: sourceFlags, records: snapshot.records },
+      value: {
+        fetchedAt,
+        version: key,
+        sources: sourceFlags,
+        records: snapshot.records,
+      },
       stale,
     };
   }

src/data/shipwrecksProxy.test.mjs

@@ -189,10 +189,12 @@ test('one source failing keeps its own last records while the other refreshes',
   const handler = handlerOf(
     shipwrecksProxy({ fetchImpl: upstream.fetchImpl, now: () => clock }),
   );
-  await request(handler);
+  const first = await request(handler);
+  assert.equal((await request(handler)).body.version, first.body.version);
   upstream.state.wikidata = 'down';
   clock = 25 * HOUR;
   const later = await request(handler);
+  assert.notEqual(later.body.version, first.body.version, 'new EMODnet data');
   assert.equal(later.body.records.length, 2);
   assert.deepEqual(later.body.sources, { emodnet: true, wikidata: true });
   assert.equal(later.body.stale, true);

src/data/spriteOrder.js

@@ -1,5 +1,6 @@
 /** Bottom-to-top order for near-plane-clamped contact sprite collections. */
 export const SPRITE_LAYER_ORDER = Object.freeze([
+  'shipwrecks',
   'cctv',
   'firms',
   'wildlife-sightings',

src/layers/shipwrecks/index.js

@@ -1,5 +1,10 @@
 import * as Cesium from 'cesium';
 import { formatNumber, msg, t, tn } from '../../i18n/index.js';
+import {
+  registerSpriteCollection,
+  restoreSpriteOrder,
+  unregisterSpriteCollection,
+} from '../../data/spriteOrder.js';
 import { createHorizonCull } from '../streetLevel/view.js';
 import { ERA_COLORS, ERA_LABELS, describeShipwreck } from './cards.js';
 import { SHIPWRECK_ERAS, eraOf } from './eras.js';
@@ -61,12 +66,19 @@ export function createShipwrecksLayer({
   picking = null,
   pointer = null,
   openExternal = null,
+  // One collection for every wreck marker (thousands of them): far cheaper to
+  // draw and rebuild than an entity each. Tests pass a fake.
+  createMarkers = (scene) =>
+    new Cesium.BillboardCollection({
+      scene,
+      blendOption: Cesium.BlendOption.TRANSLUCENT,
+    }),
 } = {}) {
   if (typeof source?.getSnapshot !== 'function')
     throw new TypeError('Shipwrecks require a snapshot source');
   let _viewer = null;
   let _enabled = false;
-  let _dataSource = null;
+  let _markers = null;
   let _request = null;
   let _records = new Map();
   let _pins = [];
@@ -102,7 +114,8 @@ export function createShipwrecksLayer({
   }
 
   function applyPin(pin) {
-    pin.entity.show = pin.inFront && visible(pin.record);
+    const show = pin.inFront && visible(pin.record);
+    if (pin.billboard.show !== show) pin.billboard.show = show;
   }
 
   /** Re-apply the period to every marker and the open card. */
@@ -118,30 +131,27 @@ export function createShipwrecksLayer({
     _extent = yearExtent(records);
     _range = periodRange(_period, _extent);
     _pins = [];
-    if (!_dataSource) return;
-    _dataSource.entities.suspendEvents();
-    _dataSource.entities.removeAll();
-    for (const record of records) {
+    if (!_markers) return;
+    _markers.removeAll();
+    // Within one collection later markers draw on top: add by rising death
+    // toll so a bigger toll covers smaller neighbours.
+    const ordered = [...records].sort(
+      (a, b) => (a.casualties || 0) - (b.casualties || 0),
+    );
+    for (const record of ordered) {
       const position = Cesium.Cartesian3.fromDegrees(record.lon, record.lat);
-      const entity = _dataSource.entities.add({
+      const size = markerPixelSize(record.casualties);
+      const billboard = _markers.add({
         id: `${PICK_PREFIX}${record.id}`,
         position,
-        billboard: {
-          image: _images[eraOf(record)],
-          width: markerPixelSize(record.casualties),
-          height: markerPixelSize(record.casualties),
-          heightReference: Cesium.HeightReference.CLAMP_TO_GROUND,
-          disableDepthTestDistance: Number.POSITIVE_INFINITY,
-          // Bigger toll, drawn on top of smaller neighbours.
-          eyeOffset: new Cesium.Cartesian3(
-            0,
-            0,
-            -Math.min(record.casualties || 0, 5000),
-          ),
-        },
+        image: _images[eraOf(record)],
+        width: size,
+        height: size,
+        heightReference: Cesium.HeightReference.CLAMP_TO_GROUND,
+        disableDepthTestDistance: Number.POSITIVE_INFINITY,
       });
       const pin = {
-        entity,
+        billboard,
         record,
         position,
         inFront: true,
@@ -156,7 +166,6 @@ export function createShipwrecksLayer({
       applyPin(pin);
       _pins.push(pin);
     }
-    _dataSource.entities.resumeEvents();
     _horizon.update();
     if (_selectedId && !_records.has(_selectedId)) clearSelection();
     else if (_selectedId) publishCard();
@@ -320,14 +329,16 @@ export function createShipwrecksLayer({
     init(viewer) {
       if (_viewer) throw new Error('Shipwrecks layer is already initialized');
       _viewer = viewer;
-      _dataSource = new Cesium.CustomDataSource(SHIPWRECKS_LAYER_ID);
-      _dataSource.show = false;
-      viewer.dataSources.add(_dataSource);
+      _markers = createMarkers(viewer.scene);
+      _markers.show = false;
+      viewer.scene.primitives.add(_markers);
+      registerSpriteCollection(SHIPWRECKS_LAYER_ID, _markers);
     },
 
     enable() {
       _enabled = true;
-      if (_dataSource) _dataSource.show = true;
+      if (_markers) _markers.show = true;
+      restoreSpriteOrder(_viewer);
       installClickHandler();
       if (_pins.length) _horizon.update();
     },
@@ -336,7 +347,7 @@ export function createShipwrecksLayer({
       _enabled = false;
       _request?.abort();
       _request = null;
-      if (_dataSource) _dataSource.show = false;
+      if (_markers) _markers.show = false;
       removeClickHandler();
       clearSelection();
       _horizon.stop();
@@ -344,7 +355,7 @@ export function createShipwrecksLayer({
     },
 
     async update() {
-      if (!_enabled || !_dataSource) return false;
+      if (!_enabled || !_markers) return false;
       _request?.abort();
       const request = new AbortController();
       _request = request;
@@ -352,6 +363,12 @@ export function createShipwrecksLayer({
         const snapshot = await source.getSnapshot({ signal: request.signal });
         if (request.signal.aborted || _request !== request || !_enabled)
           return false;
+        // The page checks every 6 h; the proxy refreshes once a day. Only a
+        // new snapshot rebuilds the markers.
+        const unchanged =
+          snapshot.version !== null &&
+          snapshot.version === _snapshot?.version &&
+          _pins.length > 0;
         _snapshot = snapshot;
         _lastError =
           snapshot.sources.emodnet && snapshot.sources.wikidata
@@ -359,7 +376,7 @@ export function createShipwrecksLayer({
             : snapshot.sources.emodnet
               ? msg('Wikidata unavailable; showing EMODnet only')
               : msg('EMODnet unavailable; showing Wikidata only');
-        rebuild(snapshot.records);
+        if (!unchanged) rebuild(snapshot.records);
         notifyRow();
         return true;
       } catch (error) {
@@ -376,9 +393,12 @@ export function createShipwrecksLayer({
 
     destroy(viewer = _viewer) {
       layer.disable();
-      if (_dataSource) {
-        viewer?.dataSources?.remove(_dataSource, true);
-        _dataSource = null;
+      if (_markers) {
+        unregisterSpriteCollection(SHIPWRECKS_LAYER_ID, _markers);
+        const primitives = viewer?.scene?.primitives;
+        if (primitives && !primitives.isDestroyed?.())
+          primitives.remove(_markers);
+        _markers = null;
       }
       _records = new Map();
       _pins = [];

src/layers/shipwrecks/index.test.mjs

@@ -0,0 +1,210 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import * as Cesium from 'cesium';
+import * as picking from '../../data/pickRegistry.js';
+import { createShipwrecksLayer, markerPixelSize } from './index.js';
+
+const wreck = (id, extra = {}) => ({
+  id,
+  source: 'wikidata',
+  name: id,
+  lat: 50,
+  lon: -1,
+  year: 1912,
+  date: null,
+  casualties: null,
+  ...extra,
+});
+
+const snapshot = (records, version = 'v1') => ({
+  records,
+  fetchedAt: 1,
+  stale: false,
+  version,
+  sources: { emodnet: true, wikidata: true },
+});
+
+function deferred() {
+  let resolve;
+  const promise = new Promise((done) => {
+    resolve = done;
+  });
+  return { promise, resolve };
+}
+
+/** A fake billboard collection that records what the layer adds. */
+function fakeMarkers() {
+  const collection = {
+    show: true,
+    billboards: [],
+    rebuilds: 0,
+    add(options) {
+      const billboard = { show: true, ...options };
+      collection.billboards.push(billboard);
+      return billboard;
+    },
+    removeAll() {
+      collection.rebuilds++;
+      collection.billboards = [];
+    },
+  };
+  return collection;
+}
+
+function harness({ snapshots } = {}) {
+  const markers = fakeMarkers();
+  const primitives = [];
+  const cards = [];
+  const handlers = [];
+  const opened = [];
+  let pickResult = null;
+  const viewer = {
+    camera: { positionWC: Cesium.Cartesian3.fromDegrees(-1, 50, 2e6) },
+    scene: {
+      canvas: {},
+      requestRender() {},
+      pick: () => pickResult,
+      preRender: { addEventListener: () => () => {} },
+      primitives: {
+        add: (primitive) => primitives.push(primitive),
+        remove: (primitive) =>
+          primitives.splice(primitives.indexOf(primitive), 1),
+        raiseToTop() {},
+        contains: (primitive) => primitives.includes(primitive),
+      },
+    },
+  };
+  const queue = [...(snapshots || [])];
+  const requests = [];
+  const layer = createShipwrecksLayer({
+    source: {
+      getSnapshot({ signal }) {
+        requests.push(signal);
+        const next = queue.shift();
+        return next instanceof Promise ? next : Promise.resolve(next);
+      },
+    },
+    createMarkers: () => markers,
+    overlayHost: {
+      setEntries: (_source, entries) => cards.push(entries),
+      setVisible() {},
+      hitTest: () => null,
+    },
+    screenSpaceEventHandlerFactory: () => {
+      const handler = {
+        setInputAction(callback) {
+          handler.click = callback;
+        },
+        destroy() {
+          handler.destroyed = true;
+        },
+      };
+      handlers.push(handler);
+      return handler;
+    },
+    picking,
+    pointer: { isPointerFree: () => true },
+    openExternal: (url) => opened.push(url),
+  });
+  layer.init(viewer);
+  return {
+    layer,
+    markers,
+    primitives,
+    cards,
+    handlers,
+    requests,
+    opened,
+    pick(id) {
+      pickResult = id ? { id: `shipwreck:${id}` } : null;
+    },
+    lastCard: () => cards.at(-1),
+  };
+}
+
+test('markers live in one collection, shown only while the layer is on', async () => {
+  const h = harness({ snapshots: [snapshot([wreck('a'), wreck('b')])] });
+  assert.deepEqual(h.primitives, [h.markers]);
+  assert.equal(h.markers.show, false);
+  h.layer.enable();
+  assert.equal(h.markers.show, true);
+  assert.equal(await h.layer.update(), true);
+  assert.equal(h.markers.billboards.length, 2);
+  assert.equal(h.layer.getStats().count, 2);
+  h.layer.disable();
+  assert.equal(h.markers.show, false);
+  assert.equal(h.handlers[0].destroyed, true);
+  h.layer.destroy();
+  assert.deepEqual(h.primitives, [], 'the collection leaves the scene');
+});
+
+test('markers sit on the wreck: no eye offset, bigger toll drawn last', async () => {
+  const h = harness({
+    snapshots: [
+      snapshot([
+        wreck('big', { casualties: 1500 }),
+        wreck('none'),
+        wreck('small', { casualties: 3 }),
+      ]),
+    ],
+  });
+  h.layer.enable();
+  await h.layer.update();
+  const ids = h.markers.billboards.map((billboard) => billboard.id);
+  assert.deepEqual(ids, ['shipwreck:none', 'shipwreck:small', 'shipwreck:big']);
+  for (const billboard of h.markers.billboards)
+    assert.equal(billboard.eyeOffset, undefined);
+  assert.equal(h.markers.billboards[2].width, markerPixelSize(1500));
+});
+
+test('an unchanged snapshot does not rebuild the markers', async () => {
+  const records = [wreck('a')];
+  const h = harness({
+    snapshots: [
+      snapshot(records, 'v1'),
+      snapshot(records, 'v1'),
+      snapshot([wreck('a'), wreck('b')], 'v2'),
+    ],
+  });
+  h.layer.enable();
+  await h.layer.update();
+  await h.layer.update();
+  assert.equal(h.markers.rebuilds, 1);
+  await h.layer.update();
+  assert.equal(h.markers.rebuilds, 2);
+  assert.equal(h.markers.billboards.length, 2);
+});
+
+test('a response that arrives after the layer is turned off is ignored', async () => {
+  const late = deferred();
+  const h = harness({ snapshots: [late.promise] });
+  h.layer.enable();
+  const pending = h.layer.update();
+  h.layer.disable();
+  assert.equal(h.requests[0].aborted, true);
+  late.resolve(snapshot([wreck('a')]));
+  assert.equal(await pending, false);
+  assert.equal(h.markers.billboards.length, 0);
+});
+
+test('a click opens the card; a period that excludes the wreck closes it', async () => {
+  const h = harness({
+    snapshots: [
+      snapshot([wreck('old', { year: 1850 }), wreck('new', { year: 1990 })]),
+    ],
+  });
+  h.layer.enable();
+  await h.layer.update();
+  h.pick('old');
+  h.handlers[0].click({ position: { x: 1, y: 1 } });
+  const [card] = h.lastCard();
+  assert.equal(card.id, 'shipwreck-card:old');
+  h.layer.setParams({ from: 1900, to: 1990 });
+  assert.deepEqual(h.lastCard(), [], 'the hidden wreck loses its card');
+  const shown = h.markers.billboards.filter((billboard) => billboard.show);
+  assert.deepEqual(
+    shown.map((billboard) => billboard.id),
+    ['shipwreck:new'],
+  );
+  assert.equal(h.layer.getStats().count, 1);
+});

src/layers/shipwrecks/source.js

@@ -25,6 +25,8 @@ export function createShipwrecksSource({
           ? payload.fetchedAt
           : null,
         stale: payload.stale === true,
+        // Changes whenever either source's data does; null from an older proxy.
+        version: typeof payload.version === 'string' ? payload.version : null,
         sources: {
           emodnet: payload.sources?.emodnet !== false,
           wikidata: payload.sources?.wikidata !== false,