mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 08:12:24 +00:00
The page compared the proxy's fetchedAt with its own clock, so a skewed browser clock cleared or kept wrecks wrongly; a snapshot without a timestamp never expired; and one old source cleared the other's fresh wrecks. The proxy now reports each source's age on its own clock and its seven-day limit; the page turns them into per-source deadlines when the snapshot arrives (seven days from arrival when no age is given) and, on a failed refresh, removes only the sources past theirs. A refused short refresh is logged.
333 lines
11 KiB
JavaScript
333 lines
11 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { shipwrecksProxy } from '../../server/providers/shipwrecks.js';
|
|
|
|
const EMODNET = {
|
|
type: 'FeatureCollection',
|
|
features: [
|
|
{
|
|
type: 'Feature',
|
|
id: 'heritageshipwrecks.1',
|
|
geometry: { type: 'Point', coordinates: [-0.92, 50.74] },
|
|
properties: { name: 'HMS/m A1', sink_yr: '1911', country: 'UK' },
|
|
},
|
|
],
|
|
};
|
|
|
|
const WIKIDATA = {
|
|
results: {
|
|
bindings: [
|
|
{
|
|
item: { value: 'http://www.wikidata.org/entity/Q25173' },
|
|
itemLabel: { value: 'Titanic' },
|
|
coord: { value: 'Point(-49.9469 41.7325)' },
|
|
},
|
|
],
|
|
},
|
|
};
|
|
|
|
/**
|
|
* Upstream fake. Each source answers 'ok', 'down' (HTTP 503) or 'malformed'
|
|
* (HTTP 200 with a body that is not that source's data); change `state` to
|
|
* switch between requests.
|
|
*/
|
|
function fakeFetch({ wikidataFails = false, emodnetFails = false } = {}) {
|
|
const calls = [];
|
|
const state = {
|
|
emodnet: emodnetFails ? 'down' : 'ok',
|
|
wikidata: wikidataFails ? 'down' : 'ok',
|
|
};
|
|
const fetchImpl = async (url, options) => {
|
|
calls.push({ url: String(url), options });
|
|
const href = String(url);
|
|
const name = href.includes('query.wikidata.org')
|
|
? 'wikidata'
|
|
: href.includes('ows.emodnet-humanactivities.eu')
|
|
? 'emodnet'
|
|
: null;
|
|
if (!name) return new Response('', { status: 404 });
|
|
if (state[name] === 'down') return new Response('', { status: 503 });
|
|
if (state[name] === 'malformed') return Response.json({});
|
|
return Response.json(name === 'wikidata' ? WIKIDATA : EMODNET);
|
|
};
|
|
const callsTo = (name) =>
|
|
calls.filter(({ url }) =>
|
|
url.includes(
|
|
name === 'wikidata'
|
|
? 'query.wikidata.org'
|
|
: 'ows.emodnet-humanactivities',
|
|
),
|
|
).length;
|
|
return { fetchImpl, calls, state, callsTo };
|
|
}
|
|
|
|
const HOUR = 60 * 60 * 1000;
|
|
|
|
function handlerOf(plugin) {
|
|
let handler;
|
|
plugin.configureServer({
|
|
middlewares: { use: (_path, fn) => (handler = fn) },
|
|
});
|
|
return handler;
|
|
}
|
|
|
|
function request(handler, { method = 'GET', url = '/' } = {}) {
|
|
return new Promise((resolve) => {
|
|
const res = {
|
|
destroyed: false,
|
|
writeHead(status, headers) {
|
|
this.status = status;
|
|
this.headers = headers;
|
|
},
|
|
end(body) {
|
|
resolve({
|
|
status: this.status,
|
|
headers: this.headers,
|
|
body: JSON.parse(body),
|
|
});
|
|
},
|
|
};
|
|
handler(
|
|
{ method, url, headers: {}, socket: { remoteAddress: '127.0.0.1' } },
|
|
res,
|
|
);
|
|
});
|
|
}
|
|
|
|
test('serves one cached snapshot from fixed upstreams', async () => {
|
|
const { fetchImpl, calls } = fakeFetch();
|
|
const handler = handlerOf(shipwrecksProxy({ fetchImpl }));
|
|
const first = await request(handler);
|
|
assert.equal(first.status, 200);
|
|
assert.equal(first.body.records.length, 2);
|
|
const a1 = first.body.records.find((record) => record.name === 'HMS/m A1');
|
|
assert.equal(a1.year, 1911);
|
|
assert.deepEqual(first.body.sources, { emodnet: true, wikidata: true });
|
|
for (const { url } of calls)
|
|
assert.match(
|
|
url,
|
|
/^https:\/\/(ows\.emodnet-humanactivities\.eu\/wfs\?|query\.wikidata\.org\/sparql\?)/,
|
|
);
|
|
const count = calls.length;
|
|
await request(handler);
|
|
assert.equal(calls.length, count, 'second request is served from cache');
|
|
});
|
|
|
|
test('one failing source still serves the other; both failing is a 502', async () => {
|
|
const partial = handlerOf(
|
|
shipwrecksProxy({
|
|
fetchImpl: fakeFetch({ wikidataFails: true }).fetchImpl,
|
|
}),
|
|
);
|
|
const ok = await request(partial);
|
|
assert.equal(ok.status, 200);
|
|
assert.deepEqual(ok.body.sources, { emodnet: true, wikidata: false });
|
|
|
|
const down = handlerOf(
|
|
shipwrecksProxy({
|
|
fetchImpl: fakeFetch({ wikidataFails: true, emodnetFails: true })
|
|
.fetchImpl,
|
|
}),
|
|
);
|
|
assert.equal((await request(down)).status, 502);
|
|
});
|
|
|
|
test('a failed refresh serves the last snapshot as stale', async () => {
|
|
let clock = 0;
|
|
let fail = false;
|
|
const { fetchImpl: good } = fakeFetch();
|
|
const { fetchImpl: bad } = fakeFetch({
|
|
wikidataFails: true,
|
|
emodnetFails: true,
|
|
});
|
|
const handler = handlerOf(
|
|
shipwrecksProxy({
|
|
fetchImpl: (...args) => (fail ? bad : good)(...args),
|
|
now: () => clock,
|
|
}),
|
|
);
|
|
await request(handler);
|
|
fail = true;
|
|
clock = 25 * 60 * 60 * 1000;
|
|
const stale = await request(handler);
|
|
assert.equal(stale.status, 200);
|
|
assert.equal(stale.body.stale, true);
|
|
assert.equal(stale.headers['X-Data-Stale'], 'true');
|
|
});
|
|
|
|
test('rejects other methods and routes', async () => {
|
|
const handler = handlerOf(
|
|
shipwrecksProxy({ fetchImpl: fakeFetch().fetchImpl }),
|
|
);
|
|
assert.equal((await request(handler, { method: 'POST' })).status, 405);
|
|
assert.equal((await request(handler, { url: '/x' })).status, 404);
|
|
});
|
|
|
|
test("a refresh that gets nothing usable keeps every source's last good wrecks", async () => {
|
|
let clock = 0;
|
|
const upstream = fakeFetch();
|
|
const handler = handlerOf(
|
|
shipwrecksProxy({ fetchImpl: upstream.fetchImpl, now: () => clock }),
|
|
);
|
|
const first = await request(handler);
|
|
assert.equal(first.body.records.length, 2);
|
|
// A day later EMODnet is down and Wikidata answers 200 with no results.
|
|
upstream.state.emodnet = 'down';
|
|
upstream.state.wikidata = 'malformed';
|
|
clock = 25 * HOUR;
|
|
const later = await request(handler);
|
|
assert.equal(later.status, 200);
|
|
assert.equal(later.body.records.length, 2, 'no wreck is lost');
|
|
assert.deepEqual(later.body.sources, { emodnet: true, wikidata: true });
|
|
assert.equal(later.body.stale, true);
|
|
assert.equal(later.body.fetchedAt, 0, 'the age of the oldest data served');
|
|
});
|
|
|
|
test('one source failing keeps its own last records while the other refreshes', async () => {
|
|
let clock = 0;
|
|
const upstream = fakeFetch();
|
|
const handler = handlerOf(
|
|
shipwrecksProxy({ fetchImpl: upstream.fetchImpl, now: () => clock }),
|
|
);
|
|
const first = await request(handler);
|
|
assert.equal((await request(handler)).body.version, first.body.version);
|
|
upstream.state.wikidata = 'down';
|
|
clock = 25 * HOUR;
|
|
const later = await request(handler);
|
|
assert.notEqual(later.body.version, first.body.version, 'new EMODnet data');
|
|
assert.equal(later.body.records.length, 2);
|
|
assert.deepEqual(later.body.sources, { emodnet: true, wikidata: true });
|
|
assert.equal(later.body.stale, true);
|
|
assert.equal(upstream.callsTo('emodnet'), 2, 'EMODnet was refreshed');
|
|
assert.deepEqual(later.body.ages, { emodnet: 0, wikidata: 25 * HOUR });
|
|
assert.equal(later.body.maxAgeMs, 7 * 24 * HOUR);
|
|
});
|
|
|
|
test('a failed source waits before it is asked again', async () => {
|
|
let clock = 0;
|
|
const upstream = fakeFetch();
|
|
const handler = handlerOf(
|
|
shipwrecksProxy({ fetchImpl: upstream.fetchImpl, now: () => clock }),
|
|
);
|
|
await request(handler);
|
|
upstream.state.emodnet = 'down';
|
|
upstream.state.wikidata = 'down';
|
|
clock = 25 * HOUR;
|
|
await request(handler);
|
|
const asked = upstream.calls.length;
|
|
for (let i = 0; i < 10; i += 1) {
|
|
clock += 60 * 1000;
|
|
const res = await request(handler);
|
|
assert.equal(res.status, 200);
|
|
assert.equal(res.body.stale, true);
|
|
}
|
|
assert.equal(upstream.calls.length, asked, 'no upstream call while waiting');
|
|
clock = 25 * HOUR + 16 * 60 * 1000;
|
|
upstream.state.emodnet = 'ok';
|
|
upstream.state.wikidata = 'ok';
|
|
const back = await request(handler);
|
|
assert.equal(
|
|
upstream.calls.length,
|
|
asked + 2,
|
|
'both are retried after the wait',
|
|
);
|
|
assert.equal(back.body.stale, undefined);
|
|
});
|
|
|
|
test('with no data yet, failures are retried only after the wait', async () => {
|
|
let clock = 0;
|
|
const upstream = fakeFetch({ emodnetFails: true, wikidataFails: true });
|
|
const handler = handlerOf(
|
|
shipwrecksProxy({ fetchImpl: upstream.fetchImpl, now: () => clock }),
|
|
);
|
|
assert.equal((await request(handler)).status, 502);
|
|
const asked = upstream.calls.length;
|
|
clock += 5 * 60 * 1000;
|
|
assert.equal((await request(handler)).status, 502);
|
|
assert.equal(upstream.calls.length, asked);
|
|
clock += 11 * 60 * 1000;
|
|
assert.equal((await request(handler)).status, 502);
|
|
assert.equal(upstream.calls.length, asked * 2);
|
|
});
|
|
|
|
test('a source older than seven days is no longer served', async () => {
|
|
let clock = 0;
|
|
const upstream = fakeFetch();
|
|
const handler = handlerOf(
|
|
shipwrecksProxy({ fetchImpl: upstream.fetchImpl, now: () => clock }),
|
|
);
|
|
await request(handler);
|
|
upstream.state.emodnet = 'down';
|
|
upstream.state.wikidata = 'down';
|
|
clock = 7 * 24 * HOUR;
|
|
assert.equal((await request(handler)).status, 200, 'seven days is served');
|
|
clock = 7 * 24 * HOUR + 16 * 60 * 1000;
|
|
assert.equal((await request(handler)).status, 502, 'older is dropped');
|
|
});
|
|
|
|
test('an upstream body over its cap is a failed source, read only to the cap', async () => {
|
|
const MIB = 1024 * 1024;
|
|
const chunk = new Uint8Array(MIB).fill(0x20);
|
|
let sent = 0;
|
|
let cancelled = false;
|
|
const endless = () =>
|
|
new ReadableStream({
|
|
pull(controller) {
|
|
sent += chunk.byteLength;
|
|
controller.enqueue(chunk);
|
|
},
|
|
cancel() {
|
|
cancelled = true;
|
|
},
|
|
});
|
|
const handler = handlerOf(
|
|
shipwrecksProxy({
|
|
fetchImpl: async (url) =>
|
|
String(url).includes('query.wikidata.org')
|
|
? Response.json(WIKIDATA)
|
|
: new Response(endless()),
|
|
}),
|
|
);
|
|
const res = await request(handler);
|
|
assert.equal(res.status, 200);
|
|
assert.deepEqual(res.body.sources, { emodnet: false, wikidata: true });
|
|
assert.ok(cancelled, 'the oversized body is cancelled');
|
|
assert.ok(sent <= 34 * MIB, `read ${sent / MIB} MiB of a 32 MiB cap`);
|
|
});
|
|
|
|
test('a refresh far smaller than the last good set is refused until it expires', async () => {
|
|
let clock = 0;
|
|
const many = {
|
|
results: {
|
|
bindings: Array.from({ length: 10 }, (_, i) => ({
|
|
item: { value: `http://www.wikidata.org/entity/Q${100 + i}` },
|
|
itemLabel: { value: `Wreck ${i}` },
|
|
coord: { value: `Point(${i} 40)` },
|
|
})),
|
|
},
|
|
};
|
|
let wikidata = many;
|
|
const handler = handlerOf(
|
|
shipwrecksProxy({
|
|
fetchImpl: async (url) =>
|
|
Response.json(
|
|
String(url).includes('query.wikidata.org') ? wikidata : EMODNET,
|
|
),
|
|
now: () => clock,
|
|
}),
|
|
);
|
|
assert.equal((await request(handler)).body.records.length, 11);
|
|
wikidata = WIKIDATA;
|
|
clock = 25 * HOUR;
|
|
const refused = await request(handler);
|
|
assert.equal(refused.body.records.length, 11, 'one wreck is not ten');
|
|
assert.equal(refused.body.stale, true);
|
|
clock = 8 * 24 * HOUR;
|
|
const accepted = await request(handler);
|
|
assert.equal(
|
|
accepted.body.records.length,
|
|
2,
|
|
'taken once the old set expired',
|
|
);
|
|
});
|