fix(news): keep a refused key refused during a slow read, and read Local when Conflicts fails

A slow re-read of a refused key read as unavailable until it landed; it
now stays refused. The first-read wait is 10 s, for a margin under the
browser's 15 s. A failed Conflicts read no longer stops Local's refresh,
which the server reads on its own. Tests cover GDELT's wait, the refused
key and a caller abort past the timeout, and fail rather than hang; the
docs say what a section serves meanwhile.

server/providers/news/keyed.js

@@ -79,9 +79,9 @@ const COUNTRY_FRESH_MS = 30 * 60_000;
 // A failed read is not tried again for a minute; what was read before is
 // served meanwhile, up to these ages (GDELT's: since an export last came in).
 const RETRY_MS = 60_000;
-// A read with nothing to serve yet is waited for at most this long, under the
-// browser's own 15 s: it goes on, and the next ask is served from it.
-const FIRST_ANSWER_MS = 12_000;
+// A read with nothing to serve yet is waited for at most this long, well under
+// the browser's own 15 s: it goes on, and the next ask is served from it.
+const FIRST_ANSWER_MS = 10_000;
 const GDELT_STALE_MS = 2 * HOUR_MS;
 const LIVE_STALE_MS = HOUR_MS;
 const COUNTRY_STALE_MS = 6 * HOUR_MS;
@@ -270,7 +270,8 @@ const spentKey = (key) => ({ status: 503, body: { error: 'key_quota', key } });
  * for `/hotspots`, `/feed` and `/outline`, whose queries the proxy has
  * already checked against the client's allowlist. `later(run, ms)` runs
  * the next GDELT read of a day still filling in (an unref'd timer).
- * @param {{env?: () => object, fetchImpl?: typeof fetch, now?: () => number, log?: (message: string) => void, later?: (run: () => void, ms: number) => void}} [options]
+ * `firstAnswerMs` bounds the wait for a read with nothing to serve yet.
+ * @param {{env?: () => object, fetchImpl?: typeof fetch, now?: () => number, log?: (message: string) => void, later?: (run: () => void, ms: number) => void, firstAnswerMs?: number}} [options]
  */
 export function createKeyedNews({
   env = () => process.env,
@@ -376,7 +377,11 @@ export function createKeyedNews({
           .finally(() => inFlight.delete(flight)),
       );
     if (prior.value && at - prior.readAt < staleMs) return prior;
-    return firstAnswer(inFlight.get(flight), { ...prior, error: null });
+    // Meanwhile a refused key stays refused; an older failure is not this one.
+    return firstAnswer(inFlight.get(flight), {
+      ...prior,
+      error: prior.error?.rejected ? prior.error : null,
+    });
   }
 
   /** An entry stored as the most recently used (the countries are capped). */
@@ -386,7 +391,6 @@ export function createKeyedNews({
     while (store.size > MAX_COUNTRIES) store.delete(store.keys().next().value);
   }
 
-  /** An entry's reports while they may be served, else why there are none. */
   /** `read`, waited for at most firstAnswerMs; then `fallback`, as it goes on. */
   async function firstAnswer(read, fallback) {
     let timer;
@@ -403,6 +407,7 @@ export function createKeyedNews({
     }
   }
 
+  /** An entry's reports while they may be served, else why there are none. */
   function served(entry, staleMs, id) {
     if (entry.value && now() - entry.readAt < staleMs)
       return { reports: entry.value, readAt: entry.readAt };

server/providers/news/keyed.test.mjs

@@ -1083,34 +1083,106 @@ test("a country's own news keeps World News API's category and no severity, what
   assert.equal(read('Parliament session opens', null).category, 'other');
 });
 
-test('a first read slower than the browser waits answers unavailable, and the next ask is served from it', async () => {
-  const fake = upstream();
-  let release;
-  const held = new Promise((resolve) => {
-    release = resolve;
-  });
-  const news = createKeyedNews({
-    env: () => ({ WORLDNEWSAPI_KEY: 'wna-key' }),
-    fetchImpl: async (url, init) => {
-      if (new URL(url).host === 'api.worldnewsapi.com') await held;
-      return fake.fetchImpl(url, init);
-    },
-    now: () => NOW,
-    log: () => {},
-    later: () => {},
-    firstAnswerMs: 20,
-  });
-  const local = () =>
-    news.answer('/feed', new URLSearchParams('hotspot=all&at=31.95,35.93'));
-  assert.deepEqual(await local(), UPSTREAM_ERROR);
-  // The read went on, and is not asked again.
-  release();
-  await new Promise((resolve) => setTimeout(resolve, 5));
-  const served = await local();
-  assert.equal(served.status, 200);
-  assert.deepEqual(
-    served.body.reports.map((r) => r.id),
-    ['wna-9001', 'wna-9002'],
-  );
-  assert.equal(count(fake.calls, 'api.worldnewsapi.com'), 1);
-});
+test(
+  'a first read slower than the browser waits answers unavailable, and the next ask is served from it',
+  { timeout: 5_000 },
+  async () => {
+    const fake = upstream();
+    let release;
+    const held = new Promise((resolve) => {
+      release = resolve;
+    });
+    const news = createKeyedNews({
+      env: () => ({ WORLDNEWSAPI_KEY: 'wna-key' }),
+      fetchImpl: async (url, init) => {
+        if (new URL(url).host === 'api.worldnewsapi.com') await held;
+        return fake.fetchImpl(url, init);
+      },
+      now: () => NOW,
+      log: () => {},
+      later: () => {},
+      firstAnswerMs: 20,
+    });
+    const local = () =>
+      news.answer('/feed', new URLSearchParams('hotspot=all&at=31.95,35.93'));
+    assert.deepEqual(await local(), UPSTREAM_ERROR);
+    // The read went on, and is not asked again.
+    release();
+    await new Promise((resolve) => setTimeout(resolve, 5));
+    const served = await local();
+    assert.equal(served.status, 200);
+    assert.deepEqual(
+      served.body.reports.map((r) => r.id),
+      ['wna-9001', 'wna-9002'],
+    );
+    assert.equal(count(fake.calls, 'api.worldnewsapi.com'), 1);
+  },
+);
+
+test(
+  'a first GDELT read slower than the browser waits answers unavailable, and the next is served from it',
+  { timeout: 5_000 },
+  async () => {
+    const fake = upstream();
+    let release;
+    const held = new Promise((resolve) => {
+      release = resolve;
+    });
+    const news = createKeyedNews({
+      env: () => ({}),
+      fetchImpl: async (url, init) => {
+        if (new URL(url).host === 'data.gdeltproject.org') await held;
+        return fake.fetchImpl(url, init);
+      },
+      now: () => NOW,
+      log: () => {},
+      later: () => {},
+      firstAnswerMs: 20,
+    });
+    const hotspots = () => news.answer('/hotspots', new URLSearchParams());
+    assert.deepEqual(await hotspots(), UPSTREAM_ERROR);
+    release();
+    await new Promise((resolve) => setTimeout(resolve, 20));
+    assert.equal((await hotspots()).status, 200);
+  },
+);
+
+test(
+  'a refused key stays refused while its next read is slow',
+  { timeout: 5_000 },
+  async () => {
+    let release;
+    const held = new Promise((resolve) => {
+      release = resolve;
+    });
+    let reads = 0;
+    const clock = { now: NOW };
+    const fake = upstream({
+      worldnews: async () => {
+        reads++;
+        if (reads > 1) await held;
+        return new Response('{"message":"bad key"}', { status: 401 });
+      },
+    });
+    const news = createKeyedNews({
+      env: () => ({ WORLDNEWSAPI_KEY: 'wna-key' }),
+      fetchImpl: fake.fetchImpl,
+      now: () => clock.now,
+      log: () => {},
+      later: () => {},
+      firstAnswerMs: 20,
+    });
+    const local = () =>
+      news.answer('/feed', new URLSearchParams('hotspot=all&at=31.95,35.93'));
+    const refused = {
+      status: 503,
+      body: { error: 'key_rejected', key: 'worldnewsapi' },
+    };
+    assert.deepEqual(await local(), refused);
+    // Past the retry wait it is read again, slowly: still refused meanwhile.
+    clock.now += 61_000;
+    assert.deepEqual(await local(), refused);
+    assert.equal(reads, 2);
+    release();
+  },
+);