From 7704de50756ab5f9caeb515e00b18023626ae82a Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Tue, 10 Nov 2026 13:38:31 -0800 Subject: [PATCH] fix(shipwrecks): the page expires each source's wrecks by its own clock, from the age the proxy reports The page compared the proxy's fetchedAt with its own clock, so a skewed browser clock cleared or kept wrecks wrongly; a snapshot without a timestamp never expired; and one old source cleared the other's fresh wrecks. The proxy now reports each source's age on its own clock and its seven-day limit; the page turns them into per-source deadlines when the snapshot arrives (seven days from arrival when no age is given) and, on a failed refresh, removes only the sources past theirs. A refused short refresh is logged. --- docs/CURRENT-STATE.md | 5 ++- server/providers/shipwrecks.js | 11 ++++++ src/data/shipwrecksProxy.test.mjs | 2 + src/layers/shipwrecks/index.js | 44 ++++++++++++++++++--- src/layers/shipwrecks/index.test.mjs | 57 ++++++++++++++++++++++++++++ src/layers/shipwrecks/source.js | 8 ++++ 6 files changed, 119 insertions(+), 8 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 4832c32a1..c7b57785e 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -164,8 +164,9 @@ one that failed is asked again after 15 minutes. Each source keeps its own last good records: a source that fails, answers with nothing usable, or answers with less than half of its last good set keeps serving its earlier records as stale for up to seven days, and never removes the other source's wrecks. When the -page cannot refresh, the wrecks it shows are marked stale, and it clears them -once they are more than seven days old. EMODnet rows the portal took from +page cannot refresh, the wrecks it shows are marked stale, and it removes each +source's wrecks once they pass seven days, counted from the age the proxy +reported (so the page and server clocks are never compared). EMODnet rows the portal took from wrecksite.eu are skipped (that site is not openly licensed). A Wikidata wreck with the same name within 3 km of an EMODnet wreck is merged into one record that keeps the article link and takes EMODnet's year, sinking text, depth and diff --git a/server/providers/shipwrecks.js b/server/providers/shipwrecks.js index 8a5fe275f..2d38b6c96 100644 --- a/server/providers/shipwrecks.js +++ b/server/providers/shipwrecks.js @@ -120,16 +120,25 @@ export function shipwrecksProxy({ if (result.status === 'fulfilled' && !shrunk) state.good = { records: result.value, fetchedAt: done }; else state.retryAt = done + RETRY_MS; + if (shrunk) + console.warn( + `[shipwrecks] ${names[i]} refresh refused: ${result.value.length} ` + + `records against ${state.good.records.length} last good`, + ); }); } /** The served snapshot: every source with usable records, merged once. */ function compose(at) { const sourceFlags = {}; + // How old each source's records are, on this clock, so the page can + // expire them by its own clock without comparing the two. + const ages = {}; let stale = false; let fetchedAt = null; for (const [name, state] of Object.entries(sources)) { sourceFlags[name] = usable(state, at); + ages[name] = sourceFlags[name] ? at - state.good.fetchedAt : null; if (!sourceFlags[name]) continue; if (!fresh(state, at)) stale = true; fetchedAt = Math.min(fetchedAt ?? Infinity, state.good.fetchedAt); @@ -155,6 +164,8 @@ export function shipwrecksProxy({ fetchedAt, version: key, sources: sourceFlags, + ages, + maxAgeMs: STALE_MAX_MS, records: snapshot.records, }, stale, diff --git a/src/data/shipwrecksProxy.test.mjs b/src/data/shipwrecksProxy.test.mjs index e0bdc8216..e66d16a40 100644 --- a/src/data/shipwrecksProxy.test.mjs +++ b/src/data/shipwrecksProxy.test.mjs @@ -199,6 +199,8 @@ test('one source failing keeps its own last records while the other refreshes', assert.deepEqual(later.body.sources, { emodnet: true, wikidata: true }); assert.equal(later.body.stale, true); assert.equal(upstream.callsTo('emodnet'), 2, 'EMODnet was refreshed'); + assert.deepEqual(later.body.ages, { emodnet: 0, wikidata: 25 * HOUR }); + assert.equal(later.body.maxAgeMs, 7 * 24 * HOUR); }); test('a failed source waits before it is asked again', async () => { diff --git a/src/layers/shipwrecks/index.js b/src/layers/shipwrecks/index.js index c4599303c..a16a728e1 100644 --- a/src/layers/shipwrecks/index.js +++ b/src/layers/shipwrecks/index.js @@ -31,6 +31,22 @@ const CARD_HOST_OPTIONS = Object.freeze({ const MAX_CARD_LINES = 14; /** Wrecks not refreshed for this long leave the map (the proxy's own limit). */ const MAX_SHOWN_AGE_MS = 7 * 24 * 60 * 60 * 1000; + +/** + * When each source's wrecks leave the map, on this page's clock: the age the + * proxy reports (its own clock) is subtracted from the limit, so the two + * clocks are never compared. With no age reported, the snapshot counts as + * fresh when it arrives. + */ +export function sourceDeadlines(snapshot, at) { + const limit = snapshot.maxAgeMs ?? MAX_SHOWN_AGE_MS; + const deadlines = {}; + for (const name of ['emodnet', 'wikidata']) { + if (snapshot.sources?.[name] === false) continue; + deadlines[name] = at + limit - (snapshot.ages?.[name] ?? 0); + } + return deadlines; +} /** * Marker size in pixels: grows with the death toll, from 16 px (no deaths known) to 36 px (about a thousand or more). */ @@ -67,9 +83,9 @@ export function createShipwrecksLayer({ picking = null, pointer = null, openExternal = null, + now = () => Date.now(), // One collection for every wreck marker (thousands of them): far cheaper to // draw and rebuild than an entity each. Tests pass a fake. - now = () => Date.now(), createMarkers = (scene) => new Cesium.BillboardCollection({ scene, @@ -85,6 +101,7 @@ export function createShipwrecksLayer({ let _records = new Map(); let _pins = []; let _snapshot = null; + let _deadlines = {}; let _lastError = null; let _clickHandler = null; let _rowListener = null; @@ -385,6 +402,7 @@ export function createShipwrecksLayer({ snapshot.version === _snapshot?.version && _pins.length > 0; _snapshot = snapshot; + _deadlines = sourceDeadlines(snapshot, now()); _lastError = snapshot.sources.emodnet && snapshot.sources.wikidata ? null @@ -399,14 +417,27 @@ export function createShipwrecksLayer({ return false; console.warn('[Data:Shipwrecks] Fetch error:', error); _lastError = error?.message || msg('Shipwrecks unavailable'); - // What is on the map was not refreshed: say so, and drop it once it - // is older than the proxy itself would serve. + // What is on the map was not refreshed: say so, and drop each + // source's wrecks once they are older than the proxy would serve. if (_snapshot) { - const age = now() - (_snapshot.fetchedAt ?? now()); - if (age > MAX_SHOWN_AGE_MS) { + const at = now(); + const expired = Object.keys(_deadlines).filter( + (name) => _deadlines[name] <= at, + ); + for (const name of expired) delete _deadlines[name]; + if (!Object.keys(_deadlines).length) { _snapshot = null; rebuild([]); - } else _snapshot = { ..._snapshot, stale: true }; + } else { + _snapshot = { ..._snapshot, stale: true }; + if (expired.length) { + _snapshot.records = _snapshot.records.filter( + (record) => !expired.includes(record.source), + ); + _snapshot.version = null; + rebuild(_snapshot.records); + } + } } notifyRow(); return false; @@ -427,6 +458,7 @@ export function createShipwrecksLayer({ _records = new Map(); _pins = []; _snapshot = null; + _deadlines = {}; _viewer = null; _rowListener = null; }, diff --git a/src/layers/shipwrecks/index.test.mjs b/src/layers/shipwrecks/index.test.mjs index 92cb05733..ce2fe992c 100644 --- a/src/layers/shipwrecks/index.test.mjs +++ b/src/layers/shipwrecks/index.test.mjs @@ -291,3 +291,60 @@ test('a failed refresh marks the shown wrecks stale, and clears them after seven assert.equal(h.layer.getStats().count, 0, 'expired wrecks leave the map'); assert.equal(h.markers.billboards.length, 0); }); + +test('expiry runs on the page clock, per source, even without timestamps', async () => { + const DAY = 24 * 60 * 60 * 1000; + const down = () => new Error('Shipwrecks unavailable (HTTP 502)'); + // A page clock far from the proxy's: only the reported ages count. + let clock = 5e12; + const h = harness({ + now: () => clock, + snapshots: [ + { + ...snapshot([ + wreck('fresh', { source: 'emodnet' }), + wreck('old', { source: 'wikidata' }), + ]), + fetchedAt: null, + ages: { emodnet: 0, wikidata: 6.5 * DAY }, + maxAgeMs: 7 * DAY, + }, + down(), + down(), + down(), + ], + }); + h.layer.enable(); + await h.layer.update(); + clock += 0.25 * DAY; + await h.layer.update(); + assert.equal(h.layer.getStats().count, 2, 'both still inside their limit'); + clock += 0.5 * DAY; + await h.layer.update(); + assert.deepEqual( + h.markers.billboards.map((billboard) => billboard.id), + ['shipwreck:fresh'], + 'only the source past its limit leaves', + ); + assert.equal(h.layer.getStats().stale, true); + clock += 7 * DAY; + await h.layer.update(); + assert.equal(h.layer.getStats().count, 0); +}); + +test('a snapshot with no timestamp or ages still expires seven days after it arrived', async () => { + const DAY = 24 * 60 * 60 * 1000; + let clock = 100; + const h = harness({ + now: () => clock, + snapshots: [ + { ...snapshot([wreck('a')]), fetchedAt: null }, + new Error('down'), + ], + }); + h.layer.enable(); + await h.layer.update(); + clock += 8 * DAY; + await h.layer.update(); + assert.equal(h.layer.getStats().count, 0); +}); diff --git a/src/layers/shipwrecks/source.js b/src/layers/shipwrecks/source.js index 7aaaa49b7..1928fd4ec 100644 --- a/src/layers/shipwrecks/source.js +++ b/src/layers/shipwrecks/source.js @@ -1,6 +1,8 @@ import { readResponseJsonCapped } from '../../sources/httpBody.js'; import { normalizeShipwreckSnapshot } from './records.js'; +const age = (value) => (Number.isFinite(value) && value >= 0 ? value : null); + /** Same-origin source for the proxy's EMODnet + Wikidata shipwreck snapshot. */ export function createShipwrecksSource({ fetchImpl = (...args) => globalThis.fetch(...args), @@ -27,6 +29,12 @@ export function createShipwrecksSource({ stale: payload.stale === true, // Changes whenever either source's data does; null from an older proxy. version: typeof payload.version === 'string' ? payload.version : null, + // Each source's age on the proxy's clock (null when unknown). + ages: { + emodnet: age(payload.ages?.emodnet), + wikidata: age(payload.ages?.wikidata), + }, + maxAgeMs: age(payload.maxAgeMs), sources: { emodnet: payload.sources?.emodnet !== false, wikidata: payload.sources?.wikidata !== false,