Blame a GDELT export only while GDELT answers, and fix the review's notes

An export is given up on (never published, unreadable, or failing three
times) only while GDELT itself answers: its list of exports read, or
another export of the same round read. An outage that answers 404, or junk,
for everything now gives up on nothing, and a failing file earns one strike
a round rather than three within the first seconds' rounds.

Docs: News's share token is 9; territories without a country code are named
as themselves too; the changelog fragment credits one pull request. Unused
exports dropped from the rules and stale scaffolding from the ledger tests.
This commit is contained in:
Denis Groshev committed 2026-11-06 00:45:33 +01:00
1 parent 66ed0386a6
commit 1e78ac931c
9 files changed
+107 -40

No files matched your search

+68 -2
View File
@@ -200,6 +200,8 @@ function upstream(answers = {}) {
);
if (answers.corrupt?.includes(href))
return new Response('not a zip at all', { status: 200 });
if (answers.failing?.includes(href))
return new Response('busy', { status: 503 });
const gkg = href.endsWith('.gkg.csv.zip');
const listed = answers.exports ?? EXPORTS;
const answer = gkg
@@ -669,7 +671,12 @@ test('exports never published are given up on, not looked for back to back', asy
const quarter = exportName('20261105114500');
const { ask, calls, clock, settle } = keyed({
env: {},
answers: { exports: { [quarter]: EXPORTS[quarter] }, unlisted: 404 },
// GDELT's list names the newest; every other export it answers 404.
answers: {
exports: { [quarter]: EXPORTS[quarter] },
unlisted: 404,
lastupdate: '20261105120000',
},
});
assert.equal((await ask('/hotspots')).body.hotspots[0].counts.last24h, 3);
await settle();
@@ -943,7 +950,7 @@ test('no export later than the newest GDELT lists is asked for (its CDN keeps a
);
});
test('a town a disputed area is named after is named alone, on both sides of its line', () => {
test('a town a disputed area is named after is named as the town alone', () => {
assert.equal(
gdeltPlace({
lat: 31.7667,
@@ -986,3 +993,62 @@ test('GDELT down for a few minutes gives up on no export: they are all read once
),
);
});
test('a GDELT outage answering 404, its list down too, gives up on no export', async () => {
const answers = { exports: {}, unlisted: 404 };
const { ask, clock, settle } = keyed({ env: {}, answers });
// Long enough for the stories' exports to be 90 minutes old.
for (let i = 0; i < 20; i++) {
await ask('/hotspots');
await settle();
clock.now += 5 * MINUTE;
}
delete answers.exports;
delete answers.unlisted;
for (let i = 0; i < 3; i++) {
clock.now += 5 * MINUTE;
await ask('/hotspots');
await settle();
}
const back = await ask('/hotspots');
assert.equal(back.status, 200);
assert.equal(back.body.hotspots[0].counts.last24h, 4);
});
test('a host answering junk for every file gives up on none of them', async () => {
const answers = {
corrupt: [exportName('20261105114500'), exportName('20261105113000')],
exports: {},
unlisted: 404,
};
const { ask, clock, settle } = keyed({ env: {}, answers });
await ask('/hotspots');
await settle();
answers.corrupt = [];
delete answers.exports;
delete answers.unlisted;
clock.now += 6 * MINUTE;
await ask('/hotspots');
await settle();
const back = await ask('/hotspots');
assert.equal(back.body.hotspots[0].counts.last24h, 4);
});
test('a failing file gets one strike a round, not three within seconds', async () => {
const flaky = exportName('20261105114500');
const answers = { failing: [flaky] };
const { ask, calls, clock, settle } = keyed({ env: {}, answers });
// The startup fill runs several rounds a few seconds apart.
await ask('/hotspots');
await settle();
// GDELT answers again for it before three rounds five minutes apart.
answers.failing = [];
clock.now += 6 * MINUTE;
await ask('/hotspots');
await settle();
const back = await ask('/hotspots');
assert.equal(back.body.hotspots[0].counts.last24h, 4);
assert.ok(
gdeltCalls(calls).filter((c) => String(c.url) === flaky).length >= 2,
);
});