diff --git a/server/providers/shipwrecks.js b/server/providers/shipwrecks.js index e680b18a0..8a5fe275f 100644 --- a/server/providers/shipwrecks.js +++ b/server/providers/shipwrecks.js @@ -15,6 +15,8 @@ const REFRESH_MS = 24 * 60 * 60 * 1000; const STALE_MAX_MS = 7 * REFRESH_MS; /** After a source fails, it is not asked again for this long. */ const RETRY_MS = 15 * 60 * 1000; +/** A refresh with fewer than this share of the last good records is refused. */ +const MIN_KEEP_RATIO = 0.5; // Only the columns the card uses; ship_char/obj_desc histories can name people // and are not read beyond a short vessel type. const EMODNET_PROPERTIES = [ @@ -109,7 +111,13 @@ export function shipwrecksProxy({ const done = now(); results.forEach((result, i) => { const state = sources[names[i]]; - if (result.status === 'fulfilled') + // A valid answer far smaller than the last good set (a truncated or + // half-built upstream) is a failure too, until the old set expires. + const shrunk = + result.status === 'fulfilled' && + usable(state, done) && + result.value.length < state.good.records.length * MIN_KEEP_RATIO; + if (result.status === 'fulfilled' && !shrunk) state.good = { records: result.value, fetchedAt: done }; else state.retryAt = done + RETRY_MS; }); diff --git a/src/data/shipwrecksProxy.test.mjs b/src/data/shipwrecksProxy.test.mjs index 7793ef145..e0bdc8216 100644 --- a/src/data/shipwrecksProxy.test.mjs +++ b/src/data/shipwrecksProxy.test.mjs @@ -292,3 +292,39 @@ test('an upstream body over its cap is a failed source, read only to the cap', a assert.ok(cancelled, 'the oversized body is cancelled'); assert.ok(sent <= 34 * MIB, `read ${sent / MIB} MiB of a 32 MiB cap`); }); + +test('a refresh far smaller than the last good set is refused until it expires', async () => { + let clock = 0; + const many = { + results: { + bindings: Array.from({ length: 10 }, (_, i) => ({ + item: { value: `http://www.wikidata.org/entity/Q${100 + i}` }, + itemLabel: { value: `Wreck ${i}` }, + coord: { value: `Point(${i} 40)` }, + })), + }, + }; + let wikidata = many; + const handler = handlerOf( + shipwrecksProxy({ + fetchImpl: async (url) => + Response.json( + String(url).includes('query.wikidata.org') ? wikidata : EMODNET, + ), + now: () => clock, + }), + ); + assert.equal((await request(handler)).body.records.length, 11); + wikidata = WIKIDATA; + clock = 25 * HOUR; + const refused = await request(handler); + assert.equal(refused.body.records.length, 11, 'one wreck is not ten'); + assert.equal(refused.body.stale, true); + clock = 8 * 24 * HOUR; + const accepted = await request(handler); + assert.equal( + accepted.body.records.length, + 2, + 'taken once the old set expired', + ); +});