Blame a GDELT export only while GDELT answers, and fix the review's notes

An export is given up on (never published, unreadable, or failing three
times) only while GDELT itself answers: its list of exports read, or
another export of the same round read. An outage that answers 404, or junk,
for everything now gives up on nothing, and a failing file earns one strike
a round rather than three within the first seconds' rounds.

Docs: News's share token is 9; territories without a country code are named
as themselves too; the changelog fragment credits one pull request. Unused
exports dropped from the rules and stale scaffolding from the ledger tests.

server/providers/news/categorize.js

@@ -8,7 +8,7 @@
 // as the fallback.
 
 /** Text as the rules match it: folded case, one apostrophe, no vowel marks. */
-export function normaliseForMatch(text) {
+function normaliseForMatch(text) {
   return String(text || '')
     .normalize('NFKC')
     .toLowerCase()
@@ -553,11 +553,6 @@ const COMPILED = Object.entries(RULES).map(([category, stems]) => ({
   }),
 }));
 
-/**
- * Categorise a report from its title and summary.
- * @returns {{category: string, tags: string[]}}
- */
-export const RELEVANCE_LEVELS = Object.freeze(['core', 'related', 'minor']);
 const EVENT_CATEGORIES = new Set([
   'strike',
   'air-defence',
@@ -771,7 +766,7 @@ const ELECTION =
 const PHYSICAL =
   /(?<![\p{L}])(?:kill|dead(?![\p{L}])|fighting|clash|air ?strike|strikes?(?![\p{L}])|struck|destroy|downed|ablaze|on fire|catch(?:es)? fire|caught fire|naval mine|landmine|shot down|sunk|sank|deaths?|injur|wounded|casualt|troops|soldier|missile|drone|rocket|shell(?:ing|ed)|artiller|explosi|blast|bomb|gunfire|shot(?![\p{L}])|shooting|magnitude|richter|tremor|evacuat|flood(?:ed|ing|waters)|убит|погиб|ранен|ракет|дрон|обстрел|загин|поранен|обстріл|قتل|جرح|صاروخ|הרג|פצוע|טיל)/u;
 /** The categories of a physical event (an alert is a threat, not yet one). */
-export const EVENT_KINDS = new Set([
+const EVENT_KINDS = new Set([
   'strike',
   'air-defence',
   'ground',
@@ -780,6 +775,10 @@ export const EVENT_KINDS = new Set([
   'infrastructure',
 ]);
 
+/**
+ * Categorise a report from its title and summary.
+ * @returns {{category: string, tags: string[]}}
+ */
 export function categorizeText({ title = '', summary = '' } = {}) {
   if (isAlertText(title)) return { category: 'alert', tags: ['alert'] };
   const t = rephrase(normaliseForMatch(title));

server/providers/news/keyed.js

@@ -55,8 +55,9 @@ const DAY_MS = 24 * HOUR_MS;
 // newest first, a dozen a read (the first day's 96 fill in over the next
 // reads, a few seconds apart, while each reads something new). One listed
 // but missing (404) an hour and a half after its time is taken as never
-// published; one that fails GDELT_EXPORT_TRIES times, or cannot be read at
-// all, is given up on, so one bad file never holds the rest back. The first
+// published; one that fails GDELT_EXPORT_TRIES times (a try a round), or
+// cannot be read at all, is given up on, so one bad file never holds the
+// rest back. Only while GDELT itself answers is a file given up on. The first
 // answer waits at most GDELT_FIRST_MS for the first exports.
 const GDELT_FRESH_MS = 5 * 60_000;
 const GDELT_EXPORTS_PER_READ = 12;
@@ -283,13 +284,13 @@ export function createKeyedNews({
     read: new Set(),
     // Exports still missing well after their time: not looked for again.
     gone: new Set(),
-    // Failed reads of each export not read yet.
+    // Failed reads of each export not read yet: `{tries, at}`.
     failures: new Map(),
     // Exports read among the newest whose GKG titles are not in yet, with
     // the tries made: `{tries, triedAt}`.
     untitled: new Map(),
     titling: null,
-    // When a read last brought an export in, with nothing failing.
+    // When a read last brought an export in.
     readAt: null,
     checkedAt: null,
     failedAt: null,
@@ -503,6 +504,8 @@ export function createKeyedNews({
     let failed = null;
     let added = 0;
     const failing = [];
+    const unreadable = [];
+    const unpublished = [];
     for (let i = 0; i < missing.length; i += GDELT_PARALLEL)
       await Promise.all(
         missing.slice(i, i + GDELT_PARALLEL).map(async (url) => {
@@ -511,13 +514,11 @@ export function createKeyedNews({
             events = await fetchExport(url, { fetchImpl });
           } catch (error) {
             failed = error;
-            // An unreadable file stays so.
-            if (error.unreadable) gdelt.gone.add(url);
-            else failing.push(url);
+            (error.unreadable ? unreadable : failing).push(url);
             return;
           }
           if (!events) {
-            if (at - exportAt(url) >= GDELT_GONE_MS) gdelt.gone.add(url);
+            if (at - exportAt(url) >= GDELT_GONE_MS) unpublished.push(url);
             return;
           }
           // The events count now; the titles follow (readTitles).
@@ -531,12 +532,19 @@ export function createKeyedNews({
         }),
       );
     if (failed) log(`[news] gdelt read failed: ${failed.message}`);
-    // A file that fails while others read is that file's fault, and gets a
-    // few tries; when none read, GDELT itself is down and no file is blamed.
+    // A file is blamed only while GDELT itself answers (its list read, or
+    // another export of the round read): then a 404 long after its time is
+    // never published, an unreadable file stays so, and a failing one gets a
+    // few tries, one a GDELT_FRESH_MS. With GDELT down, no file is blamed.
+    const answering = added > 0 || latest !== null;
+    if (answering)
+      for (const url of [...unpublished, ...unreadable]) gdelt.gone.add(url);
     if (added)
       for (const url of failing) {
-        const tries = (gdelt.failures.get(url) || 0) + 1;
-        gdelt.failures.set(url, tries);
+        const prior = gdelt.failures.get(url);
+        if (prior && now() - prior.at < GDELT_FRESH_MS) continue;
+        const tries = (prior?.tries || 0) + 1;
+        gdelt.failures.set(url, { tries, at: now() });
         if (tries >= GDELT_EXPORT_TRIES) gdelt.gone.add(url);
       }
     // Nothing read and a failure: tried again after RETRY_MS, not after

server/providers/news/keyed.test.mjs

@@ -200,6 +200,8 @@ function upstream(answers = {}) {
         );
       if (answers.corrupt?.includes(href))
         return new Response('not a zip at all', { status: 200 });
+      if (answers.failing?.includes(href))
+        return new Response('busy', { status: 503 });
       const gkg = href.endsWith('.gkg.csv.zip');
       const listed = answers.exports ?? EXPORTS;
       const answer = gkg
@@ -669,7 +671,12 @@ test('exports never published are given up on, not looked for back to back', asy
   const quarter = exportName('20261105114500');
   const { ask, calls, clock, settle } = keyed({
     env: {},
-    answers: { exports: { [quarter]: EXPORTS[quarter] }, unlisted: 404 },
+    // GDELT's list names the newest; every other export it answers 404.
+    answers: {
+      exports: { [quarter]: EXPORTS[quarter] },
+      unlisted: 404,
+      lastupdate: '20261105120000',
+    },
   });
   assert.equal((await ask('/hotspots')).body.hotspots[0].counts.last24h, 3);
   await settle();
@@ -943,7 +950,7 @@ test('no export later than the newest GDELT lists is asked for (its CDN keeps a
   );
 });
 
-test('a town a disputed area is named after is named alone, on both sides of its line', () => {
+test('a town a disputed area is named after is named as the town alone', () => {
   assert.equal(
     gdeltPlace({
       lat: 31.7667,
@@ -986,3 +993,62 @@ test('GDELT down for a few minutes gives up on no export: they are all read once
     ),
   );
 });
+
+test('a GDELT outage answering 404, its list down too, gives up on no export', async () => {
+  const answers = { exports: {}, unlisted: 404 };
+  const { ask, clock, settle } = keyed({ env: {}, answers });
+  // Long enough for the stories' exports to be 90 minutes old.
+  for (let i = 0; i < 20; i++) {
+    await ask('/hotspots');
+    await settle();
+    clock.now += 5 * MINUTE;
+  }
+  delete answers.exports;
+  delete answers.unlisted;
+  for (let i = 0; i < 3; i++) {
+    clock.now += 5 * MINUTE;
+    await ask('/hotspots');
+    await settle();
+  }
+  const back = await ask('/hotspots');
+  assert.equal(back.status, 200);
+  assert.equal(back.body.hotspots[0].counts.last24h, 4);
+});
+
+test('a host answering junk for every file gives up on none of them', async () => {
+  const answers = {
+    corrupt: [exportName('20261105114500'), exportName('20261105113000')],
+    exports: {},
+    unlisted: 404,
+  };
+  const { ask, clock, settle } = keyed({ env: {}, answers });
+  await ask('/hotspots');
+  await settle();
+  answers.corrupt = [];
+  delete answers.exports;
+  delete answers.unlisted;
+  clock.now += 6 * MINUTE;
+  await ask('/hotspots');
+  await settle();
+  const back = await ask('/hotspots');
+  assert.equal(back.body.hotspots[0].counts.last24h, 4);
+});
+
+test('a failing file gets one strike a round, not three within seconds', async () => {
+  const flaky = exportName('20261105114500');
+  const answers = { failing: [flaky] };
+  const { ask, calls, clock, settle } = keyed({ env: {}, answers });
+  // The startup fill runs several rounds a few seconds apart.
+  await ask('/hotspots');
+  await settle();
+  // GDELT answers again for it before three rounds five minutes apart.
+  answers.failing = [];
+  clock.now += 6 * MINUTE;
+  await ask('/hotspots');
+  await settle();
+  const back = await ask('/hotspots');
+  assert.equal(back.body.hotspots[0].counts.last24h, 4);
+  assert.ok(
+    gdeltCalls(calls).filter((c) => String(c.url) === flaky).length >= 2,
+  );
+});