fix(wildlife): wait for the longest effective delay among every failed request

One error was chosen before each failure's delay was known. An antimeridian
box 400 hid the other box's 503 Retry-After; a grid tile 400 with Retry-After
set a cooldown through the snapshot metadata; a short Retry-After on one tile
undercut a sibling 5xx's default backoff; and a huge Retry-After made the
deadline Infinity.

Every failed request now keeps its status and Retry-After. One function gives
each failure's delay: 4xx other than 429 none, 429 and 5xx their Retry-After
or the default backoff. The layer waits for the longest, and shows the
failure that set it. Retry-After is capped at one day, the proxy's own
daily-budget deadline, and non-finite values are ignored. Each publish of
one acquisition derives the default backoff from the backoff before it
started, so repeated tile publishes no longer double it.

src/layers/wildlifeSightings/acquisition.test.mjs

@@ -1072,3 +1072,98 @@ for (const [name, respond] of [
     assert.equal(h.state.renderedRows.length, 2);
   });
 }
+
+// Every failed request keeps its own timing. The layer waits for the longest
+// effective delay: 4xx other than 429 never waits, 429/5xx wait for their
+// Retry-After or the default backoff. The shown error explains that wait.
+async function settleThroughSource(fetchImpl, bounds = salishSea) {
+  const h = harness(
+    createWildlifeSightingsSource({ now: () => NOW, fetchImpl }),
+  );
+  h.bounds(bounds);
+  h.state.enabled = false;
+  h.layer.enable();
+  const first = h.layer.update();
+  await h.advance(300);
+  await first;
+  await new Promise((resolve) => setImmediate(resolve));
+  await new Promise((resolve) => setImmediate(resolve));
+  return h;
+}
+const tileX = (url) =>
+  Number(new URL(url, 'https://example.test').searchParams.get('x'));
+const observations = () =>
+  Response.json({ rows: [inatRow], totalResults: 1, fetchedAt: NOW });
+
+test('an antimeridian box 400 never hides the other box 503 Retry-After', async () => {
+  for (const westFails of [true, false]) {
+    const h = await settleThroughSource(
+      async (url) => {
+        if (url.startsWith('/api/inaturalist/grid?')) return emptyGrid();
+        const west =
+          Number(
+            new URL(url, 'https://example.test').searchParams.get('west'),
+          ) > 0;
+        return west === westFails
+          ? new Response('', { status: 400 })
+          : unavailable();
+      },
+      { south: 10, west: 179, north: 12, east: -179 },
+    );
+    assert.equal(h.state.error, 'iNaturalist HTTP 503');
+    assert.equal(h.state.retryAt, NOW + 300 + 120_000);
+    assert.equal(h.layer.getStats().retryInSec, 120);
+  }
+});
+
+test('a grid tile 400 with Retry-After beside successful tiles creates no cooldown', async () => {
+  let tiles = new Set();
+  const h = await settleThroughSource(async (url) => {
+    if (!url.startsWith('/api/inaturalist/grid?')) return observations();
+    tiles.add(tileX(url));
+    return tileX(url) === Math.min(...tiles) && tiles.size === 1
+      ? new Response('', { status: 400, headers: { 'Retry-After': '120' } })
+      : emptyGrid();
+  });
+  assert.ok(tiles.size > 1, 'the view spans several grid tiles');
+  assert.equal(h.state.error, 'iNaturalist HTTP 400');
+  assert.equal(h.state.retryAt, 0);
+  assert.equal(h.layer.getStats().retryInSec, undefined);
+});
+
+test('a short grid Retry-After never undercuts a sibling 5xx default backoff', async () => {
+  let failed = 0;
+  const h = await settleThroughSource(async (url) => {
+    if (!url.startsWith('/api/inaturalist/grid?')) return observations();
+    failed++;
+    if (failed === 1)
+      return new Response('', { status: 503, headers: { 'Retry-After': '2' } });
+    if (failed === 2) return new Response('', { status: 502 });
+    return emptyGrid();
+  });
+  assert.ok(failed >= 2);
+  assert.equal(h.state.error, 'iNaturalist HTTP 502');
+  assert.equal(h.state.retryAt, NOW + 300 + 30_000);
+  assert.equal(h.layer.getStats().retryInSec, 30);
+});
+
+test('an enormous Retry-After is capped at one day and a non-finite one uses the default backoff', async () => {
+  for (const [header, delay] of [
+    ['1' + '0'.repeat(306), 86_400_000],
+    ['1' + '0'.repeat(400), 30_000],
+  ]) {
+    const h = await settleThroughSource(async (url) =>
+      url.startsWith('/api/inaturalist/grid?')
+        ? emptyGrid()
+        : url.startsWith('/api/acartia')
+          ? Response.json({ rows: [], fetchedAt: NOW })
+          : new Response('', {
+              status: 503,
+              headers: { 'Retry-After': header },
+            }),
+    );
+    assert.equal(h.state.error, 'iNaturalist HTTP 503');
+    assert.ok(Number.isFinite(h.state.retryAt));
+    assert.equal(h.state.retryAt, NOW + 300 + delay, header.length);
+  }
+});

src/layers/wildlifeSightings/lifecycle.js

@@ -2,7 +2,10 @@ import { msg } from '../../i18n/marker.js';
 import { getWildlifeHeightBand, rowInWildlifeBoxes } from './model.js';
 import {
   combineWildlifeGridSnapshots,
-  wildlifeFailureRetryable,
+  preferredWildlifeFailure,
+  wildlifeFailureDelayMs,
+  wildlifeFailures,
+  wildlifeRetryAfterMs,
 } from './source.js';
 import { wildlifeMeasure } from './performance.js';
 import { isRecentWildlifeSighting } from './records.js';
@@ -12,11 +15,7 @@ import {
   wildlifeGridTiles,
   wildlifeGridCellId,
 } from './gridViewport.js';
-import {
-  wildlifeViewportBoxes,
-  snapWildlifeBox,
-  retryDelayMs,
-} from './viewport.js';
+import { wildlifeViewportBoxes, snapWildlifeBox } from './viewport.js';
 import {
   WILDLIFE_DEBOUNCE_MS,
   WILDLIFE_MAX_ROWS,
@@ -95,25 +94,25 @@ export function createLifecycle({ state, services, parts }) {
     state.pendingParams = null;
   }
 
-  function applyRetry(errors, retryAfterSec = null) {
-    // Only upstream failures impose cooldown; network/schema errors retry on
-    // pan. Several sources can fail together: wait for the longest of them.
-    const failures = [errors].flat().filter(wildlifeFailureRetryable);
-    const snapshotDelay =
-      Number.isFinite(retryAfterSec) && retryAfterSec >= 0
-        ? retryAfterSec * 1000
-        : null;
-    if (!failures.length && snapshotDelay === null) {
+  function applyRetry(
+    errors,
+    retryAfterSec = null,
+    previousMs = state.failures || 0,
+  ) {
+    // Only 429/5xx failures impose cooldown; client, network and schema
+    // errors retry on pan. Several requests can fail together: wait for the
+    // longest effective delay among them.
+    const delays = [
+      ...wildlifeFailures(errors).map((error) =>
+        wildlifeFailureDelayMs(error, previousMs),
+      ),
+      wildlifeRetryAfterMs(retryAfterSec),
+    ].filter((delay) => delay !== null);
+    if (!delays.length) {
       state.failures = 0;
       state.retryAt = 0;
       return;
     }
-    const delays = failures.map((error) =>
-      Number.isFinite(error.retryAfterSec) && error.retryAfterSec >= 0
-        ? error.retryAfterSec * 1000
-        : retryDelayMs(state.failures || 0),
-    );
-    if (snapshotDelay !== null) delays.push(snapshotDelay);
     const backoff = Math.max(...delays);
     state.failures = backoff;
     state.retryAt = services.now() + backoff;
@@ -282,6 +281,7 @@ export function createLifecycle({ state, services, parts }) {
     request,
     signal,
     epoch,
+    previousFailures,
     replaceRows = true,
   }) {
     const current = currentQuery();
@@ -398,10 +398,14 @@ export function createLifecycle({ state, services, parts }) {
             value.upstreamStatus === 429 ||
             value.upstreamStatus >= 500),
       );
+      // The filter above admits only 429/5xx or an unmarked stale answer.
       const staleError = failedStale.length
         ? Object.assign(
             new Error(msg('Wildlife sightings source unavailable')),
             {
+              upstreamStatus: Math.max(
+                ...failedStale.map((value) => value.upstreamStatus || 503),
+              ),
               retryAfterSec: Math.max(
                 ...failedStale.map((value) => value.retryAfterSec),
               ),
@@ -410,8 +414,8 @@ export function createLifecycle({ state, services, parts }) {
         : null;
       const inat = snapshot.sources?.inaturalist;
       const acartia = snapshot.sources?.acartia;
-      // Keep each failed source's status and Retry-After; the displayed error
-      // is the one that explains the cooldown.
+      // Keep each failed request's status and Retry-After; the displayed
+      // error is the one whose effective delay sets the cooldown.
       const sourceErrors = [
         gridError,
         staleError,
@@ -433,14 +437,10 @@ export function createLifecycle({ state, services, parts }) {
             })
           : null,
       ].filter(Boolean);
-      const sourceError =
-        sourceErrors.find(
-          (error) =>
-            wildlifeFailureRetryable(error) ||
-            Number.isFinite(error.retryAfterSec),
-        ) ||
-        sourceErrors[0] ||
-        null;
+      const sourceError = preferredWildlifeFailure(
+        sourceErrors,
+        previousFailures,
+      );
       state.stale =
         !sourceError && (snapshot.stale === true || grid?.stale === true);
       const fetchedAt = [snapshot.fetchedAt, grid?.fetchedAt].filter(
@@ -467,12 +467,13 @@ export function createLifecycle({ state, services, parts }) {
         applyRetry(
           sourceErrors,
           retryAfter.length ? Math.max(...retryAfter) : null,
+          previousFailures,
         );
       } else {
         state.pendingParams = null;
         state.failedParams = null;
         state.retryAt = retryAfter.length
-          ? services.now() + Math.max(...retryAfter) * 1000
+          ? services.now() + wildlifeRetryAfterMs(Math.max(...retryAfter))
           : 0;
         state.failures = 0;
       }
@@ -632,6 +633,9 @@ export function createLifecycle({ state, services, parts }) {
       state.request?.abort();
       const request = new AbortController();
       const requestLifetime = lifetimeEpoch;
+      // One acquisition publishes several times (tiles, rows, final grid).
+      // Each derives its default backoff from the backoff before it started.
+      const previousFailures = state.failures || 0;
       const supersededResult = () =>
         state.enabled &&
         !state.destroyed &&
@@ -699,6 +703,7 @@ export function createLifecycle({ state, services, parts }) {
           request,
           signal,
           epoch: requestLifetime,
+          previousFailures,
         });
       }
       function finishTiles() {
@@ -804,11 +809,12 @@ export function createLifecycle({ state, services, parts }) {
         continueTiles();
         state.stale = false;
         state.error =
-          error?.message || msg('Wildlife sightings source unavailable');
+          preferredWildlifeFailure(error, previousFailures)?.message ||
+          msg('Wildlife sightings source unavailable');
         failParams(params);
         state.lastQueryKey = null;
         requestQueryKey = null;
-        applyRetry(error);
+        applyRetry(error, null, previousFailures);
         console.warn('[Data:WildlifeSightings] Update failed:', state.error);
         return true;
       } finally {

src/layers/wildlifeSightings/policy.js

@@ -74,6 +74,9 @@ export { WILDLIFE_MAX_VIEW_DEGREES } from './bounds.js';
 export const WILDLIFE_VIEW_PADDING = 0.3;
 export const WILDLIFE_RETRY_MIN_MS = 30_000;
 export const WILDLIFE_RETRY_MAX_MS = 240_000;
+// Cap a Retry-After so one reply can never stall refresh forever. The proxy's
+// daily-budget reply waits until the next UTC midnight, so allow one day.
+export const WILDLIFE_RETRY_AFTER_MAX_SEC = 86_400;
 
 // Faster polling would reread the proxy's five-minute cache.
 export const WILDLIFE_UPDATE_INTERVAL_MS = 5 * 60_000;

src/layers/wildlifeSightings/source.js

@@ -1,8 +1,12 @@
 import { readResponseJsonCapped } from '../../sources/httpBody.js';
 import { mergeSightings } from './records.js';
 import { wildlifeMeasure } from './performance.js';
-import { ACARTIA_GUARD_BOX, TAXA_PRESETS } from './policy.js';
-import { boxIntersects } from './viewport.js';
+import {
+  ACARTIA_GUARD_BOX,
+  TAXA_PRESETS,
+  WILDLIFE_RETRY_AFTER_MAX_SEC,
+} from './policy.js';
+import { boxIntersects, retryDelayMs } from './viewport.js';
 import {
   WILDLIFE_GRID_MAX_ZOOM,
   WILDLIFE_GRID_TILE_CAP,
@@ -72,14 +76,19 @@ function retryAfterSeconds(value, nowMs) {
   const text = value.trim();
   if (/^\d+(?:\.\d+)?$/.test(text)) {
     const seconds = Number(text);
-    return Number.isFinite(seconds) ? Math.ceil(seconds) : null;
+    return Number.isFinite(seconds)
+      ? Math.min(WILDLIFE_RETRY_AFTER_MAX_SEC, Math.ceil(seconds))
+      : null;
   }
   // Reject malformed numeric values rather than letting Date.parse interpret
   // them as calendar dates. HTTP dates include a named weekday or month.
   if (!/[a-z]/i.test(text)) return null;
   const deadline = Date.parse(text);
   return Number.isFinite(deadline) && Number.isFinite(nowMs)
-    ? Math.max(0, Math.ceil((deadline - nowMs) / 1000))
+    ? Math.min(
+        WILDLIFE_RETRY_AFTER_MAX_SEC,
+        Math.max(0, Math.ceil((deadline - nowMs) / 1000)),
+      )
     : null;
 }
 
@@ -119,17 +128,56 @@ export function wildlifeFailureRetryable(error) {
   return status === 429 || status >= 500;
 }
 
-/** Prefer the failure that carries retry timing, then the longest wait. */
-export function preferredWildlifeFailure(errors) {
-  const failures = errors.filter(Boolean);
-  const retryable = failures.filter(wildlifeFailureRetryable);
-  if (!retryable.length) return failures[0] || null;
-  return retryable.reduce((best, error) =>
-    (Number.isFinite(error.retryAfterSec) ? error.retryAfterSec : -1) >
-    (Number.isFinite(best.retryAfterSec) ? best.retryAfterSec : -1)
-      ? error
-      : best,
-  );
+/** Return a Retry-After in milliseconds, capped at one day, or null when
+ * it is missing, negative or not finite. */
+export function wildlifeRetryAfterMs(seconds) {
+  return Number.isFinite(seconds) && seconds >= 0
+    ? Math.min(WILDLIFE_RETRY_AFTER_MAX_SEC, seconds) * 1000
+    : null;
+}
+
+/** Return how long a failure asks the layer to wait, or null for none:
+ * 429 and 5xx wait for their Retry-After, else the default backoff after
+ * `previousMs`; other failures may retry on the next view. */
+export function wildlifeFailureDelayMs(error, previousMs = 0) {
+  if (!error || !wildlifeFailureRetryable(error)) return null;
+  return wildlifeRetryAfterMs(error.retryAfterSec) ?? retryDelayMs(previousMs);
+}
+
+/** Flatten failures, expanding a group into every request that failed. */
+export function wildlifeFailures(errors) {
+  return [errors]
+    .flat()
+    .filter(Boolean)
+    .flatMap((error) =>
+      Array.isArray(error.failures) && error.failures.length
+        ? error.failures
+        : [error],
+    );
+}
+
+/** Return the failure with the longest effective delay (the first failure
+ * when none waits), carrying every failure so the caller can recompute. */
+export function preferredWildlifeFailure(errors, previousMs = 0) {
+  const failures = wildlifeFailures(errors);
+  if (!failures.length) return null;
+  let chosen = failures[0];
+  let longest = wildlifeFailureDelayMs(chosen, previousMs) ?? -1;
+  for (const error of failures.slice(1)) {
+    const delay = wildlifeFailureDelayMs(error, previousMs) ?? -1;
+    if (delay > longest) [chosen, longest] = [error, delay];
+  }
+  if (failures.length === 1) return chosen;
+  // Grid errors are shared between consumers of one tile request; describe
+  // the group with a new error rather than mutating any of them.
+  return Object.assign(new Error(chosen.message, { cause: chosen }), {
+    ...(chosen.status ? { status: chosen.status } : {}),
+    ...(chosen.upstreamStatus ? { upstreamStatus: chosen.upstreamStatus } : {}),
+    ...(Number.isFinite(chosen.retryAfterSec)
+      ? { retryAfterSec: chosen.retryAfterSec }
+      : {}),
+    failures,
+  });
 }
 
 /** Merge occupied cells incrementally; neighboring UTFGrid hit tiles can
@@ -145,9 +193,12 @@ export function combineWildlifeGridSnapshots(snapshots, error = null) {
   const totalResults = cells.reduce((sum, cell) => sum + cell.cellCount, 0);
   if (!Number.isSafeInteger(totalResults))
     throw new Error('Malformed iNaturalist grid response');
+  // A failure's Retry-After counts only when its status asks for a wait.
   const retryAfter = [
     ...snapshots.map((snapshot) => snapshot.retryAfterSec),
-    error?.retryAfterSec,
+    ...wildlifeFailures(error)
+      .filter(wildlifeFailureRetryable)
+      .map((failure) => failure.retryAfterSec),
   ].filter(Number.isFinite);
   const stale = snapshots.some((snapshot) => snapshot.stale === true);
   return {
@@ -514,14 +565,16 @@ export function createWildlifeSightingsSource({
     signal?.throwIfAborted();
     const inatSnapshots = [];
     const failedBoxes = [];
-    let inatError = null;
+    const inatErrors = [];
     inatResult.value.forEach((result, index) => {
       if (result.status === 'fulfilled') inatSnapshots.push(result.value);
       else {
         failedBoxes.push(requestBoxes[index]);
-        inatError ??= result.reason;
+        inatErrors.push(result.reason);
       }
     });
+    // Keep every failed box; the layer waits for the longest of them.
+    const inatError = preferredWildlifeFailure(inatErrors);
     // Fail outright only when nothing succeeded; otherwise draw what arrived
     // and report the failed source in its per-source status.
     const acartiaError =