fix(news): one-hour World News API retention, refunded reads, pins, disputed areas, counts

Keep World News API results an hour at most and say caching needs its
permission; refund a read that never reached it; match Local by any pin;
assign a disputed area by a point inside it; untitled GDELT stories are
minor; hotspot counts leave out minor reports.

Co-Authored-By: Claude Fable 5.2 <[email protected]>

server/providers/news/areas.js

@@ -181,7 +181,11 @@ export function countryPolygons(entry) {
         ...holes,
         ...areas
           .flatMap((area) => area.rings)
-          .filter((ring) => pointInRing(outer, ring[0][1], ring[0][0])),
+          // By a point inside it: its corners lie on the neighbours' borders.
+          .filter((ring) => {
+            const inside = ringInterior(ring);
+            return inside && pointInRing(outer, inside.lat, inside.lon);
+          }),
       ],
     })),
   );
@@ -196,10 +200,17 @@ export function countryAnchor(entry) {
   const largest = geometries(entry).sort((a, b) => b.areaKm2 - a.areaKm2)[0];
   const { label, polygons, bbox } = largest;
   if (containsAnchor(polygons, label.lat, label.lon)) return label;
-  const lat = (bbox[1] + bbox[3]) / 2;
   const [outer] = polygons.reduce((a, b) =>
     b[0].length > a[0].length ? b : a,
   );
+  return ringInterior(outer, (bbox[1] + bbox[3]) / 2) || label;
+}
+
+/**
+ * A point inside a ring: the middle of its widest span at `lat` (its middle
+ * latitude by default); null when the ring is not crossed there.
+ */
+function ringInterior(outer, lat = middleLat(outer)) {
   const crossings = [];
   for (let i = 0, j = outer.length - 1; i < outer.length; j = i++) {
     const [x1, y1] = outer[j];
@@ -212,5 +223,10 @@ export function countryAnchor(entry) {
   for (let i = 0; i + 1 < crossings.length; i += 2)
     if (!best || crossings[i + 1] - crossings[i] > best[1] - best[0])
       best = [crossings[i], crossings[i + 1]];
-  return best ? { lat, lon: (best[0] + best[1]) / 2 } : label;
+  return best ? { lat, lon: (best[0] + best[1]) / 2 } : null;
 }
+
+const middleLat = (ring) => {
+  const lats = ring.map(([, y]) => y);
+  return (Math.min(...lats) + Math.max(...lats)) / 2;
+};

server/providers/news/gdeltEvents.js

@@ -423,14 +423,17 @@ export function eventReports(events, { titleOf = () => null } = {}) {
     // The article's words decide whether it is conflict, as any headline's
     // would: GDELT codes an election story as "fighting" too. GDELT's code
     // names the kind only where the words read as conflict without naming
-    // one, or there are no words.
+    // one. A story with no words at all has GDELT's code alone to go by,
+    // which is not enough to list it under Conflicts: it is minor (kept
+    // for Local) until a page title comes in.
     const read = own ? categorizeText({ title: own }) : null;
     const tags = read?.tags || [];
     const relevance = read
       ? relevanceOf({ category: read.category, tags, title: own })
-      : 'related';
-    const category =
-      read && read.category !== 'other'
+      : 'minor';
+    const category = !read
+      ? eventCategory
+      : read.category !== 'other'
         ? read.category
         : relevance === 'minor'
           ? 'other'

server/providers/news/gdeltEvents.test.mjs

@@ -119,10 +119,11 @@ test("an article is one report, at its most-mentioned event, in its address's wo
   assert.equal(drones.publisher, 'wire.example');
   assert.equal(drones.source.attribution, 'GDELT Project');
   assert.equal(drones.side, 'unknown', 'no actor is said');
-  // No words in the address: GDELT's coding, related, at least severity 1.
+  // No words in the address: GDELT's coding alone, so minor (not listed
+  // under Conflicts), at least severity 1.
   assert.equal(bare.title, 'Fighting reported in Ukraine');
   assert.equal(bare.category, 'ground');
-  assert.equal(bare.relevance, 'related');
+  assert.equal(bare.relevance, 'minor');
   assert.equal(bare.severity, 1);
   assert.equal(bare.geo.precision, 'country');
 });

server/providers/news/keyed.js

@@ -84,7 +84,9 @@ const RETRY_MS = 60_000;
 const FIRST_ANSWER_MS = 10_000;
 const GDELT_STALE_MS = 2 * HOUR_MS;
 const LIVE_STALE_MS = HOUR_MS;
-const COUNTRY_STALE_MS = 6 * HOUR_MS;
+// World News API's terms allow a cache of at most an hour, deleted after:
+// a country's news is kept no longer, served or not (dropExpiredCountries).
+const COUNTRY_STALE_MS = HOUR_MS;
 const MAX_COUNTRIES = 64;
 // World News API reads a UTC day, across all countries: each is about 1.5
 // of the free plan's 50 daily points.
@@ -310,16 +312,18 @@ export function createKeyedNews({
   const refused = new Set();
   // World News API reads of the UTC day; `spent` once it answered 402.
   const budget = { day: null, reads: 0, spent: false, logged: false };
-  // The area (disputed area or country) each placed Conflicts report lies
-  // in, looked up once.
+  // The areas (disputed area or country) of a placed Conflicts report's
+  // pins, its first and the others, looked up once.
   const reportAreas = new WeakMap();
-  const reportArea = (report) => {
+  const areasOf = (report) => {
     if (!reportAreas.has(report))
       reportAreas.set(
         report,
-        report.geo
-          ? (areaAt(report.geo.lat, report.geo.lon)?.key ?? null)
-          : null,
+        new Set(
+          [report.geo, ...(report.locations || [])]
+            .filter(Boolean)
+            .map((pin) => areaAt(pin.lat, pin.lon)?.key),
+        ),
       );
     return reportAreas.get(report);
   };
@@ -690,6 +694,15 @@ export function createKeyedNews({
     return false;
   }
 
+  /** World News API's news an hour old is deleted, as its terms ask. */
+  function dropExpiredCountries() {
+    for (const [code, entry] of countries)
+      if (entry.value && now() - entry.readAt >= COUNTRY_STALE_MS) {
+        const { value, readAt, ...rest } = entry;
+        countries.set(code, rest);
+      }
+  }
+
   async function countryReports(country) {
     const key = keyOf('worldnewsapi');
     if (!key) return { missing: 'worldnewsapi' };
@@ -708,7 +721,7 @@ export function createKeyedNews({
             quota: true,
           });
         try {
-          return (
+          const reports = (
             await fetchWorldNews({
               key: k,
               country: country.code,
@@ -716,9 +729,14 @@ export function createKeyedNews({
               now: now(),
             })
           ).map((article) => worldNewsReport(article, country));
+          // Deleted an hour on, whether or not anything is asked then.
+          setTimeout(dropExpiredCountries, COUNTRY_STALE_MS + 1000).unref?.();
+          return reports;
         } catch (error) {
           // Out of points: nothing more is asked today.
           if (error.status === 402) budget.spent = true;
+          // Never reached (offline, timed out): no read of the day's.
+          if (error.unreached && budget.reads > 0) budget.reads -= 1;
           throw error;
         }
       },
@@ -743,7 +761,10 @@ export function createKeyedNews({
     if (refused) return refused;
     const hours = Number(params.get('hours')) || 24;
     const since = new Date(now() - hours * HOUR_MS).toISOString();
-    const shown = read.reports.filter((r) => r.publishedAt >= since);
+    // The reports Conflicts lists: a minor one is in no count.
+    const shown = read.reports.filter(
+      (r) => r.publishedAt >= since && r.relevance !== 'minor',
+    );
     return {
       status: 200,
       body: {
@@ -825,8 +846,8 @@ export function createKeyedNews({
     ]);
     const refused = refusal(national) || (!country && refusal(conflicts));
     if (refused) return refused;
-    const inside = (conflicts.reports || []).filter(
-      (r) => reportArea(r) === area.key,
+    const inside = (conflicts.reports || []).filter((r) =>
+      areasOf(r).has(area.key),
     );
     const reports = [...national.reports, ...inside].sort((a, b) =>
       b.publishedAt.localeCompare(a.publishedAt),
@@ -870,6 +891,7 @@ export function createKeyedNews({
 
   return {
     async answer(path, params) {
+      dropExpiredCountries();
       if (path === '/hotspots') return hotspots(params);
       if (path === '/feed') return feed(params);
       if (path === '/outline') return outline(params);

server/providers/news/keyed.test.mjs

@@ -13,6 +13,8 @@ import {
   normalizeHotspotsPayload,
   normalizeOutlinePayload,
 } from '../../../src/layers/news/records.js';
+import { inPolygons } from '../../../src/layers/news/model.js';
+import { areaAt, countryPolygons } from './areas.js';
 import { exportsOfDay } from './gdeltEvents.js';
 import { eventRow, gkgRow, zipTable } from './gdeltFixtures.mjs';
 
@@ -277,8 +279,8 @@ test('without any key Conflicts are the day of GDELT events, one worldwide hotsp
     payload.hotspots.map((h) => [h.id, h.catchAll]),
     [[WORLDWIDE_HOTSPOT, true]],
   );
-  assert.equal(payload.hotspots[0].last24h, 4);
-  assert.equal(payload.hotspots[0].mapped24h, 4);
+  assert.equal(payload.hotspots[0].last24h, 2);
+  assert.equal(payload.hotspots[0].mapped24h, 2);
   assert.equal(payload.maxWindowHours, 24);
   // The newest dozen first; the rest of the day after, in the background.
   assert.deepEqual(
@@ -315,9 +317,9 @@ test('a GDELT article is a report at its place, in its words, credited to GDELT'
   // names it.
   assert.deepEqual(
     reports.map((r) => r.id),
-    ['gdelt-2', 'gdelt-5', 'gdelt-4'],
+    ['gdelt-2', 'gdelt-5'],
   );
-  const [redSea, kharkiv, bare] = reports;
+  const [redSea, kharkiv] = reports;
   // Its page title, from the GKG export.
   assert.equal(
     kharkiv.title,
@@ -337,8 +339,13 @@ test('a GDELT article is a report at its place, in its words, credited to GDELT'
   );
   assert.equal(redSea.geo.place, 'Yemen');
   assert.equal(redSea.geo.precision, 'country');
+  // An address without words has GDELT's coding alone: minor, so not in
+  // Conflicts, until a page title says what it is.
+  const bare = (await ask('/feed', 'hotspot=all&today=1')).body.reports.find(
+    (r) => r.id === 'gdelt-4',
+  );
   assert.equal(bare.title, 'Fighting reported in Ukraine');
-  assert.equal(bare.relevance, 'related');
+  assert.equal(bare.relevance, 'minor');
   // The council story GDELT coded as fighting is minor: not in Conflicts.
   assert.ok(!reports.some((r) => r.url.includes('city-council')));
 });
@@ -346,7 +353,7 @@ test('a GDELT article is a report at its place, in its words, credited to GDELT'
 test('with a Liveuamap key its mapped items join GDELT in the one hotspot', async () => {
   const { ask, calls, settle } = keyed();
   const answer = await ask('/hotspots');
-  assert.equal(answer.body.hotspots[0].counts.last24h, 7);
+  assert.equal(answer.body.hotspots[0].counts.last24h, 5);
   const live = calls.find((c) => host(c) === 'a.liveuamap.com').url;
   assert.equal(live.searchParams.get('a'), 'mpts');
   assert.equal(live.searchParams.get('key'), 'live-key');
@@ -376,21 +383,22 @@ test('with a Liveuamap key its mapped items join GDELT in the one hotspot', asyn
   );
   // Counts over a shorter window.
   const short = await ask('/hotspots', 'hours=2');
-  assert.equal(short.body.hotspots[0].counts.last24h, 6);
+  assert.equal(short.body.hotspots[0].counts.last24h, 4);
   await settle();
 });
 
 test('the Conflicts feed pages by time and filters by category and hotspot', async () => {
   const { ask, settle } = keyed({ env: {} });
-  const all = 'hotspot=all&today=1&relevance=core,related&events=1';
+  // Every report of the day, the minor ones too.
+  const all = 'hotspot=all&today=1&events=1';
+  // A page ends between two times: both of 11:45 come together, though
+  // the page asked for one.
   const first = await ask('/feed', `${all}&limit=1`);
   assert.deepEqual(
     first.body.reports.map((r) => r.id),
-    ['gdelt-2'],
+    ['gdelt-2', 'gdelt-3'],
   );
   assert.equal(first.body.nextBefore, '2026-11-05T11:45:00.000Z');
-  // A page ends between two times: both of 11:30 come together, though
-  // the page asked for one.
   const next = await ask(
     '/feed',
     `${all}&limit=1&before=${first.body.nextBefore}`,
@@ -436,7 +444,7 @@ test('an export not published yet is looked for again, and the day is kept a day
   await ask('/hotspots');
   await settle();
   const later = await ask('/hotspots');
-  assert.equal(later.body.hotspots[0].counts.last24h, 5, 'added, not replaced');
+  assert.equal(later.body.hotspots[0].counts.last24h, 3, 'added, not replaced');
   assert.equal(
     gdeltCalls(calls)
       .slice(read)
@@ -588,7 +596,7 @@ test('a refused Liveuamap key leaves GDELT; World News API refusing names its ke
   const refused = keyed({ answers: { liveuamap: 401, worldnews: 403 }, logs });
   const conflicts = await refused.ask('/hotspots');
   assert.equal(conflicts.status, 200);
-  assert.equal(conflicts.body.hotspots[0].counts.last24h, 4);
+  assert.equal(conflicts.body.hotspots[0].counts.last24h, 2);
   assert.deepEqual(await refused.ask('/feed', 'at=31.95,35.93'), {
     status: 503,
     body: { error: 'key_rejected', key: 'worldnewsapi' },
@@ -632,7 +640,7 @@ test('a failing GDELT is tried again after a minute, and what it read is kept',
   const kept = await flaky.ask('/hotspots');
   await flaky.settle();
   assert.equal(kept.status, 200);
-  assert.equal(kept.body.hotspots[0].counts.last24h, 4);
+  assert.equal(kept.body.hotspots[0].counts.last24h, 2);
 });
 
 test('a new key drops what the old one read', async () => {
@@ -678,7 +686,7 @@ test('exports never published are given up on, not looked for back to back', asy
       lastupdate: '20261105120000',
     },
   });
-  assert.equal((await ask('/hotspots')).body.hotspots[0].counts.last24h, 3);
+  assert.equal((await ask('/hotspots')).body.hotspots[0].counts.last24h, 2);
   await settle();
   // A round that brings nothing in is the last until the next five minutes.
   let seen = gdeltCalls(calls).length;
@@ -720,7 +728,7 @@ test('GDELT failing for two hours answers 502; until then what it read is served
   clock.now += 60 * MINUTE;
   const hour = await ask('/hotspots');
   assert.equal(hour.status, 200);
-  assert.equal(hour.body.hotspots[0].counts.last24h, 4);
+  assert.equal(hour.body.hotspots[0].counts.last24h, 2);
   clock.now += 61 * MINUTE;
   assert.deepEqual(await ask('/hotspots'), UPSTREAM_ERROR);
   assert.deepEqual(await ask('/feed', CONFLICTS), UPSTREAM_ERROR);
@@ -736,7 +744,7 @@ test('a failed GKG read keeps the events, titled by their words, and is tried ag
   await settle();
   const read = await ask('/feed', CONFLICTS);
   assert.equal(read.status, 200);
-  assert.equal(read.body.reports.length, 3);
+  assert.equal(read.body.reports.length, 2);
   assert.equal(
     kharkiv(read).title,
     'Russian drones attack Kharkiv overnight three killed',
@@ -858,7 +866,7 @@ test('a read in flight with a replaced key is not the answer for the new one', a
   env.LIVEUAMAP_API_KEY = 'new';
   delete answers.liveuamap;
   const after = await ask('/hotspots');
-  assert.equal(after.body.hotspots[0].counts.last24h, 7);
+  assert.equal(after.body.hotspots[0].counts.last24h, 5);
   assert.deepEqual(
     calls
       .filter((c) => host(c) === 'a.liveuamap.com')
@@ -868,7 +876,7 @@ test('a read in flight with a replaced key is not the answer for the new one', a
   release();
   await before;
   // The old key's late answer does not replace the new one's.
-  assert.equal((await ask('/hotspots')).body.hotspots[0].counts.last24h, 7);
+  assert.equal((await ask('/hotspots')).body.hotspots[0].counts.last24h, 5);
   await settle();
 });
 
@@ -922,7 +930,7 @@ test('one unreadable export is given up on and never holds the rest back', async
   // copy of the Kharkiv story among them).
   const first = await ask('/hotspots');
   assert.equal(first.status, 200);
-  assert.equal(first.body.hotspots[0].counts.last24h, 2);
+  assert.equal(first.body.hotspots[0].counts.last24h, 1);
   await settle();
   clock.now += 6 * MINUTE;
   assert.equal((await ask('/hotspots')).status, 200);
@@ -986,7 +994,7 @@ test('GDELT down for a few minutes gives up on no export: they are all read once
   const back = await ask('/hotspots');
   await settle();
   assert.equal(back.status, 200);
-  assert.equal(back.body.hotspots[0].counts.last24h, 4);
+  assert.equal(back.body.hotspots[0].counts.last24h, 2);
   assert.ok(
     gdeltCalls(calls).some(
       (c) => String(c.url) === exportName('20261105114500'),
@@ -1012,7 +1020,7 @@ test('a GDELT outage answering 404, its list down too, gives up on no export', a
   }
   const back = await ask('/hotspots');
   assert.equal(back.status, 200);
-  assert.equal(back.body.hotspots[0].counts.last24h, 4);
+  assert.equal(back.body.hotspots[0].counts.last24h, 2);
 });
 
 test('a host answering junk for every file gives up on none of them', async () => {
@@ -1031,7 +1039,7 @@ test('a host answering junk for every file gives up on none of them', async () =
   await ask('/hotspots');
   await settle();
   const back = await ask('/hotspots');
-  assert.equal(back.body.hotspots[0].counts.last24h, 4);
+  assert.equal(back.body.hotspots[0].counts.last24h, 2);
 });
 
 test('a failing file gets one strike a round, not three within seconds', async () => {
@@ -1047,7 +1055,7 @@ test('a failing file gets one strike a round, not three within seconds', async (
   await ask('/hotspots');
   await settle();
   const back = await ask('/hotspots');
-  assert.equal(back.body.hotspots[0].counts.last24h, 4);
+  assert.equal(back.body.hotspots[0].counts.last24h, 2);
   assert.ok(
     gdeltCalls(calls).filter((c) => String(c.url) === flaky).length >= 2,
   );
@@ -1186,3 +1194,105 @@ test(
     release();
   },
 );
+
+test("World News API's news is kept an hour at most, also while it cannot be read again", async () => {
+  const answers = {};
+  const { ask, clock, settle } = keyed({ answers });
+  const local = 'hotspot=all&at=31.95,35.93';
+  const own = (answer) =>
+    answer.body.reports.filter((r) => r.id.startsWith('wna-')).length;
+  assert.equal(own(await ask('/feed', local)), 2);
+  // Its terms: a cache of at most an hour, then deleted.
+  answers.worldnews = 500;
+  clock.now += 59 * MINUTE;
+  assert.equal(own(await ask('/feed', local)), 2);
+  clock.now += 2 * MINUTE;
+  assert.deepEqual(await ask('/feed', local), UPSTREAM_ERROR);
+  await settle();
+});
+
+test('a World News API read that never reached it is not counted against the day', async () => {
+  const answers = {
+    worldnews: () => {
+      throw new TypeError('fetch failed');
+    },
+  };
+  const { ask, clock, settle } = keyed({
+    env: { WORLDNEWSAPI_KEY: 'wna-key', WORLDNEWSAPI_DAILY_READS: '2' },
+    answers,
+  });
+  const paris = () => ask('/feed', 'hotspot=all&at=48.85,2.35');
+  for (let i = 0; i < 3; i++) {
+    assert.deepEqual(await paris(), UPSTREAM_ERROR);
+    clock.now += 2 * MINUTE;
+  }
+  // Back online: the day's reads are all still there.
+  answers.worldnews = WORLD_NEWS;
+  assert.equal((await paris()).status, 200);
+  // An answer, even a failing one, is a read: two of them spend the day.
+  answers.worldnews = 500;
+  const berlin = () => ask('/feed', 'hotspot=all&at=52.52,13.4');
+  assert.deepEqual(await berlin(), UPSTREAM_ERROR);
+  clock.now += 2 * MINUTE;
+  assert.deepEqual(await berlin(), KEY_QUOTA);
+  await settle();
+});
+
+test('Local lists a Conflicts report by any of its pins, not its first alone', async () => {
+  const url =
+    'https://news.example.com/2026/11/05/drones-hit-kharkiv-and-belgorod-overnight';
+  const exports = {
+    [exportName('20261105114500')]: [
+      eventRow({ id: 1, code: '195', mentions: 9, url }),
+      eventRow({
+        id: 2,
+        code: '195',
+        mentions: 2,
+        place: "Belgorod, Belgorodskaya Oblast', Russia",
+        lat: 50.6,
+        lon: 36.6,
+        url,
+      }),
+    ],
+  };
+  const { ask, settle } = keyed({
+    env: { WORLDNEWSAPI_KEY: 'wna-key' },
+    answers: { exports, worldnews: { news: [] } },
+  });
+  const urls = async (at) =>
+    (await ask('/feed', `hotspot=all&at=${at}`)).body.reports.map((r) => r.url);
+  assert.deepEqual(await urls('49.99,36.23'), [url]);
+  assert.deepEqual(await urls('50.6,36.6'), [url]);
+  await settle();
+});
+
+test('the hotspot counts the reports Conflicts lists: no minor ones', async () => {
+  const { ask, settle } = keyed({ env: {} });
+  const listed = (
+    await ask('/feed', 'hotspot=all&today=1&relevance=core,related&limit=60')
+  ).body.reports;
+  const { counts } = (await ask('/hotspots')).body.hotspots[0];
+  assert.equal(counts.last24h, listed.length);
+  assert.equal(counts.mapped24h, listed.filter((r) => r.geo).length);
+  // The council story is among the day's reports, and minor.
+  const all = (await ask('/feed', 'hotspot=all&today=1&limit=60')).body.reports;
+  assert.ok(all.length > listed.length);
+  await settle();
+});
+
+test('a disputed area is cut out of the country it lies in, not of the neighbour its first corner touches', () => {
+  const outline = (lat, lon) => countryPolygons(areaAt(lat, lon).country);
+  for (const [country, at, name, inside] of [
+    ['Israel', [32.08, 34.78], 'Golan Heights', [33.0, 35.75]],
+    ['Moldova', [47.0, 28.85], 'Transnistria', [47.2, 29.5]],
+    ['Egypt', [30.0, 31.2], 'Halayib', [22.5, 35.5]],
+  ]) {
+    const [lat, lon] = inside;
+    assert.equal(areaAt(lat, lon).kind, 'disputed', name);
+    assert.equal(
+      inPolygons(outline(...at), { lat, lon }),
+      false,
+      `${country} without ${name}`,
+    );
+  }
+});

server/providers/news/upstream.js

@@ -76,7 +76,13 @@ const finiteNumber = (value) => {
 export class UpstreamError extends Error {
   constructor(
     message,
-    { status = null, rejected = false, quota = false, unreadable = false } = {},
+    {
+      status = null,
+      rejected = false,
+      quota = false,
+      unreadable = false,
+      unreached = false,
+    } = {},
   ) {
     super(message);
     this.name = 'UpstreamError';
@@ -89,6 +95,8 @@ export class UpstreamError extends Error {
     // The answer itself cannot be read (corrupt, or over a size cap): asking
     // again gets the same.
     this.unreadable = unreadable;
+    // The request got no answer at all (offline, timed out).
+    this.unreached = unreached;
   }
 }
 
@@ -111,6 +119,7 @@ async function getText(
       cause?.name === 'TimeoutError'
         ? `${what} timed out`
         : `${what} unreachable`,
+      { unreached: true },
     );
   }
   if (!response.ok) {