fix(shipwrecks): the page expires each source's wrecks by its own clock, from the age the proxy reports

The page compared the proxy's fetchedAt with its own clock, so a skewed
browser clock cleared or kept wrecks wrongly; a snapshot without a timestamp
never expired; and one old source cleared the other's fresh wrecks. The proxy
now reports each source's age on its own clock and its seven-day limit; the
page turns them into per-source deadlines when the snapshot arrives (seven
days from arrival when no age is given) and, on a failed refresh, removes only
the sources past theirs. A refused short refresh is logged.

docs/CURRENT-STATE.md

@@ -164,8 +164,9 @@ one that failed is asked again after 15 minutes. Each source keeps its own last
 good records: a source that fails, answers with nothing usable, or answers with
 less than half of its last good set keeps serving its earlier records as stale
 for up to seven days, and never removes the other source's wrecks. When the
-page cannot refresh, the wrecks it shows are marked stale, and it clears them
-once they are more than seven days old. EMODnet rows the portal took from
+page cannot refresh, the wrecks it shows are marked stale, and it removes each
+source's wrecks once they pass seven days, counted from the age the proxy
+reported (so the page and server clocks are never compared). EMODnet rows the portal took from
 wrecksite.eu are skipped (that site is not openly licensed). A Wikidata wreck
 with the same name within 3 km of an EMODnet wreck is merged into one record
 that keeps the article link and takes EMODnet's year, sinking text, depth and

server/providers/shipwrecks.js

@@ -120,16 +120,25 @@ export function shipwrecksProxy({
       if (result.status === 'fulfilled' && !shrunk)
         state.good = { records: result.value, fetchedAt: done };
       else state.retryAt = done + RETRY_MS;
+      if (shrunk)
+        console.warn(
+          `[shipwrecks] ${names[i]} refresh refused: ${result.value.length} ` +
+            `records against ${state.good.records.length} last good`,
+        );
     });
   }
 
   /** The served snapshot: every source with usable records, merged once. */
   function compose(at) {
     const sourceFlags = {};
+    // How old each source's records are, on this clock, so the page can
+    // expire them by its own clock without comparing the two.
+    const ages = {};
     let stale = false;
     let fetchedAt = null;
     for (const [name, state] of Object.entries(sources)) {
       sourceFlags[name] = usable(state, at);
+      ages[name] = sourceFlags[name] ? at - state.good.fetchedAt : null;
       if (!sourceFlags[name]) continue;
       if (!fresh(state, at)) stale = true;
       fetchedAt = Math.min(fetchedAt ?? Infinity, state.good.fetchedAt);
@@ -155,6 +164,8 @@ export function shipwrecksProxy({
         fetchedAt,
         version: key,
         sources: sourceFlags,
+        ages,
+        maxAgeMs: STALE_MAX_MS,
         records: snapshot.records,
       },
       stale,

src/data/shipwrecksProxy.test.mjs

@@ -199,6 +199,8 @@ test('one source failing keeps its own last records while the other refreshes',
   assert.deepEqual(later.body.sources, { emodnet: true, wikidata: true });
   assert.equal(later.body.stale, true);
   assert.equal(upstream.callsTo('emodnet'), 2, 'EMODnet was refreshed');
+  assert.deepEqual(later.body.ages, { emodnet: 0, wikidata: 25 * HOUR });
+  assert.equal(later.body.maxAgeMs, 7 * 24 * HOUR);
 });
 
 test('a failed source waits before it is asked again', async () => {

src/layers/shipwrecks/index.js

@@ -31,6 +31,22 @@ const CARD_HOST_OPTIONS = Object.freeze({
 const MAX_CARD_LINES = 14;
 /** Wrecks not refreshed for this long leave the map (the proxy's own limit). */
 const MAX_SHOWN_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+
+/**
+ * When each source's wrecks leave the map, on this page's clock: the age the
+ * proxy reports (its own clock) is subtracted from the limit, so the two
+ * clocks are never compared. With no age reported, the snapshot counts as
+ * fresh when it arrives.
+ */
+export function sourceDeadlines(snapshot, at) {
+  const limit = snapshot.maxAgeMs ?? MAX_SHOWN_AGE_MS;
+  const deadlines = {};
+  for (const name of ['emodnet', 'wikidata']) {
+    if (snapshot.sources?.[name] === false) continue;
+    deadlines[name] = at + limit - (snapshot.ages?.[name] ?? 0);
+  }
+  return deadlines;
+}
 /**
  * Marker size in pixels: grows with the death toll, from 16 px (no deaths known) to 36 px (about a thousand or more).
  */
@@ -67,9 +83,9 @@ export function createShipwrecksLayer({
   picking = null,
   pointer = null,
   openExternal = null,
+  now = () => Date.now(),
   // One collection for every wreck marker (thousands of them): far cheaper to
   // draw and rebuild than an entity each. Tests pass a fake.
-  now = () => Date.now(),
   createMarkers = (scene) =>
     new Cesium.BillboardCollection({
       scene,
@@ -85,6 +101,7 @@ export function createShipwrecksLayer({
   let _records = new Map();
   let _pins = [];
   let _snapshot = null;
+  let _deadlines = {};
   let _lastError = null;
   let _clickHandler = null;
   let _rowListener = null;
@@ -385,6 +402,7 @@ export function createShipwrecksLayer({
           snapshot.version === _snapshot?.version &&
           _pins.length > 0;
         _snapshot = snapshot;
+        _deadlines = sourceDeadlines(snapshot, now());
         _lastError =
           snapshot.sources.emodnet && snapshot.sources.wikidata
             ? null
@@ -399,14 +417,27 @@ export function createShipwrecksLayer({
           return false;
         console.warn('[Data:Shipwrecks] Fetch error:', error);
         _lastError = error?.message || msg('Shipwrecks unavailable');
-        // What is on the map was not refreshed: say so, and drop it once it
-        // is older than the proxy itself would serve.
+        // What is on the map was not refreshed: say so, and drop each
+        // source's wrecks once they are older than the proxy would serve.
         if (_snapshot) {
-          const age = now() - (_snapshot.fetchedAt ?? now());
-          if (age > MAX_SHOWN_AGE_MS) {
+          const at = now();
+          const expired = Object.keys(_deadlines).filter(
+            (name) => _deadlines[name] <= at,
+          );
+          for (const name of expired) delete _deadlines[name];
+          if (!Object.keys(_deadlines).length) {
             _snapshot = null;
             rebuild([]);
-          } else _snapshot = { ..._snapshot, stale: true };
+          } else {
+            _snapshot = { ..._snapshot, stale: true };
+            if (expired.length) {
+              _snapshot.records = _snapshot.records.filter(
+                (record) => !expired.includes(record.source),
+              );
+              _snapshot.version = null;
+              rebuild(_snapshot.records);
+            }
+          }
         }
         notifyRow();
         return false;
@@ -427,6 +458,7 @@ export function createShipwrecksLayer({
       _records = new Map();
       _pins = [];
       _snapshot = null;
+      _deadlines = {};
       _viewer = null;
       _rowListener = null;
     },

src/layers/shipwrecks/index.test.mjs

@@ -291,3 +291,60 @@ test('a failed refresh marks the shown wrecks stale, and clears them after seven
   assert.equal(h.layer.getStats().count, 0, 'expired wrecks leave the map');
   assert.equal(h.markers.billboards.length, 0);
 });
+
+test('expiry runs on the page clock, per source, even without timestamps', async () => {
+  const DAY = 24 * 60 * 60 * 1000;
+  const down = () => new Error('Shipwrecks unavailable (HTTP 502)');
+  // A page clock far from the proxy's: only the reported ages count.
+  let clock = 5e12;
+  const h = harness({
+    now: () => clock,
+    snapshots: [
+      {
+        ...snapshot([
+          wreck('fresh', { source: 'emodnet' }),
+          wreck('old', { source: 'wikidata' }),
+        ]),
+        fetchedAt: null,
+        ages: { emodnet: 0, wikidata: 6.5 * DAY },
+        maxAgeMs: 7 * DAY,
+      },
+      down(),
+      down(),
+      down(),
+    ],
+  });
+  h.layer.enable();
+  await h.layer.update();
+  clock += 0.25 * DAY;
+  await h.layer.update();
+  assert.equal(h.layer.getStats().count, 2, 'both still inside their limit');
+  clock += 0.5 * DAY;
+  await h.layer.update();
+  assert.deepEqual(
+    h.markers.billboards.map((billboard) => billboard.id),
+    ['shipwreck:fresh'],
+    'only the source past its limit leaves',
+  );
+  assert.equal(h.layer.getStats().stale, true);
+  clock += 7 * DAY;
+  await h.layer.update();
+  assert.equal(h.layer.getStats().count, 0);
+});
+
+test('a snapshot with no timestamp or ages still expires seven days after it arrived', async () => {
+  const DAY = 24 * 60 * 60 * 1000;
+  let clock = 100;
+  const h = harness({
+    now: () => clock,
+    snapshots: [
+      { ...snapshot([wreck('a')]), fetchedAt: null },
+      new Error('down'),
+    ],
+  });
+  h.layer.enable();
+  await h.layer.update();
+  clock += 8 * DAY;
+  await h.layer.update();
+  assert.equal(h.layer.getStats().count, 0);
+});

src/layers/shipwrecks/source.js

@@ -1,6 +1,8 @@
 import { readResponseJsonCapped } from '../../sources/httpBody.js';
 import { normalizeShipwreckSnapshot } from './records.js';
 
+const age = (value) => (Number.isFinite(value) && value >= 0 ? value : null);
+
 /** Same-origin source for the proxy's EMODnet + Wikidata shipwreck snapshot. */
 export function createShipwrecksSource({
   fetchImpl = (...args) => globalThis.fetch(...args),
@@ -27,6 +29,12 @@ export function createShipwrecksSource({
         stale: payload.stale === true,
         // Changes whenever either source's data does; null from an older proxy.
         version: typeof payload.version === 'string' ? payload.version : null,
+        // Each source's age on the proxy's clock (null when unknown).
+        ages: {
+          emodnet: age(payload.ages?.emodnet),
+          wikidata: age(payload.ages?.wikidata),
+        },
+        maxAgeMs: age(payload.maxAgeMs),
         sources: {
           emodnet: payload.sources?.emodnet !== false,
           wikidata: payload.sources?.wikidata !== false,