fix(wildlife): keep each failed source's retry timing after a partial failure

A partial failure reset the layer's cooldown, so a camera move retried at
once. Acartia's error was discarded, an absent X-Upstream-Status hid the
HTTP 503 (the proxy's daily-budget reply), and a grid schema error masked
the observation 503. Each failed source now keeps its status and
Retry-After, the HTTP status counts when no upstream status is sent, and the
layer waits for the longest 429/5xx cooldown among them. The shown error is
the one that explains the cooldown.

docs/CURRENT-STATE.md

@@ -124,7 +124,11 @@ with retry timing; logs exclude request URLs and upstream exception text.
 A partial failure still draws what arrived: a failed iNaturalist request, or
 one failed half of an antimeridian view, keeps a good Acartia reply and the
 other half, retains that area's last iNaturalist rows and reports DEGRADED.
-The update fails outright only when every requested source failed.
+The update fails outright only when every requested source failed. Each failed
+source keeps its HTTP status and `Retry-After` (the HTTP status counts when no
+`X-Upstream-Status` is sent), and the next request waits out the longest 429 or
+5xx cooldown among them. One update requests both sources, so the cooldown
+holds the whole layer; a fully successful refresh clears it.
 
 `/api/acartia/sightings` adds Salish Sea marine mammals within 45.5–50.5° N,
 126–121° W, cached ten minutes and stale for at most 24 h. A valid empty

src/layers/wildlifeSightings/acquisition.test.mjs

@@ -833,10 +833,10 @@ test('an empty tile retires its old zoom cells while uncovered density survives
   assert.equal(h.state.gridTotal, 2);
 });
 
-test('only marked upstream failures create cooldown; local pacing, network and schema errors can retry on the next view', async () => {
+test('only 429 and 5xx failures create cooldown; client, network and schema errors can retry on the next view', async () => {
   for (const error of [
-    Object.assign(new Error('iNaturalist HTTP 503'), {
-      status: 503,
+    Object.assign(new Error('iNaturalist HTTP 400'), {
+      status: 400,
       upstreamStatus: 0,
       retryAfterSec: 30,
     }),
@@ -873,6 +873,19 @@ test('only marked upstream failures create cooldown; local pacing, network and s
     2,
     'actual Retry-After is honored without imposing a 30s local floor',
   );
+  // The proxy's daily-budget 503 has no upstream status. Its HTTP status and
+  // Retry-After still govern the next request.
+  const budget = harness({
+    getSnapshot: async () => {
+      throw Object.assign(new Error('iNaturalist HTTP 503'), {
+        status: 503,
+        upstreamStatus: 0,
+        retryAfterSec: 3600,
+      });
+    },
+  });
+  await budget.layer.update();
+  assert.equal(budget.layer.getStats().retryInSec, 3600);
 });
 
 test('stale cached tiles with local pacing metadata keep their display and do not enter cooldown', async () => {
@@ -942,3 +955,120 @@ test('an explicit upstream Retry-After zero retries immediately while a missing
     }
   }
 });
+
+const salishSea = { south: 47.5, west: -122.5, north: 47.7, east: -122.2 };
+const inatRow = {
+  stableId: 'inat:301',
+  origin: 'inaturalist',
+  lat: 47.6,
+  lon: -122.3,
+  observedAt: NOW - 3_600_000,
+  iconic: 'Mammalia',
+};
+const acartiaRow = {
+  stableId: 'acartia:301',
+  origin: 'acartia',
+  lat: 47.6,
+  lon: -122.35,
+  observedAt: NOW - 3_600_000,
+  cetacean: true,
+  quality: 'research',
+  provider: { name: 'Provider', entity: null },
+};
+const emptyGrid = () =>
+  Response.json({ cells: [], totalResults: 0, fetchedAt: NOW });
+const unavailable = (headers = {}) =>
+  new Response('', {
+    status: 503,
+    headers: { 'Retry-After': '120', ...headers },
+  });
+
+// Each case runs the real source and lifecycle. A healthy source draws its
+// rows while the failed source's Retry-After holds the next acquisition.
+for (const [name, respond] of [
+  [
+    'Acartia alone returns 503',
+    (url) =>
+      url.startsWith('/api/acartia')
+        ? unavailable()
+        : url.startsWith('/api/inaturalist/grid?')
+          ? emptyGrid()
+          : Response.json({
+              rows: [inatRow],
+              totalResults: 1,
+              fetchedAt: NOW,
+            }),
+  ],
+  [
+    'iNaturalist returns 503 without X-Upstream-Status',
+    (url) =>
+      url.startsWith('/api/acartia')
+        ? Response.json({ rows: [acartiaRow], fetchedAt: NOW })
+        : url.startsWith('/api/inaturalist/grid?')
+          ? emptyGrid()
+          : unavailable(),
+  ],
+  [
+    'iNaturalist returns 503 while every grid tile fails its schema',
+    (url) =>
+      url.startsWith('/api/acartia')
+        ? Response.json({ rows: [acartiaRow], fetchedAt: NOW })
+        : url.startsWith('/api/inaturalist/grid?')
+          ? Response.json({ cells: 'malformed' })
+          : unavailable({ 'X-Upstream-Status': '503' }),
+  ],
+]) {
+  test(`a partial failure keeps its Retry-After: ${name}`, async () => {
+    let healthy = false;
+    let requests = 0;
+    const source = createWildlifeSightingsSource({
+      now: () => NOW,
+      fetchImpl: async (url) => {
+        if (!url.startsWith('/api/inaturalist/grid?')) requests++;
+        if (healthy)
+          return url.startsWith('/api/acartia')
+            ? Response.json({ rows: [acartiaRow], fetchedAt: NOW })
+            : url.startsWith('/api/inaturalist/grid?')
+              ? emptyGrid()
+              : Response.json({
+                  rows: [inatRow],
+                  totalResults: 1,
+                  fetchedAt: NOW,
+                });
+        return respond(url);
+      },
+    });
+    const h = harness(source);
+    h.bounds(salishSea);
+    h.state.enabled = false;
+    h.layer.enable();
+    const first = h.layer.update();
+    await h.advance(300);
+    await first;
+    await new Promise((resolve) => setImmediate(resolve));
+    assert.equal(h.state.renderedRows.length, 1, 'the healthy source draws');
+    assert.ok(h.state.error, 'the failed source is reported');
+    // The source clock stays at NOW; the layer settled 300 ms of camera time.
+    assert.equal(h.state.retryAt, NOW + 300 + 120_000);
+    assert.equal(h.layer.getStats().retryInSec, 120);
+    const attempted = requests;
+    h.bounds({ ...salishSea, north: 47.8 });
+    h.moveEnd();
+    assert.equal([...h.timers.values()][0]?.delay, 120_000);
+    await h.advance(119_999);
+    await h.layer.update();
+    assert.equal(
+      requests,
+      attempted,
+      'neither a camera move nor manager polling skips the cooldown',
+    );
+    healthy = true;
+    await h.advance(1);
+    await new Promise((resolve) => setImmediate(resolve));
+    assert.ok(requests > attempted, 'acquisition resumes at the deadline');
+    assert.equal(h.state.error, null);
+    assert.equal(h.state.retryAt, 0, 'a fully successful refresh clears it');
+    assert.equal(h.layer.getStats().retryInSec, undefined);
+    assert.equal(h.state.renderedRows.length, 2);
+  });
+}

src/layers/wildlifeSightings/lifecycle.js

@@ -1,6 +1,9 @@
 import { msg } from '../../i18n/marker.js';
 import { getWildlifeHeightBand, rowInWildlifeBoxes } from './model.js';
-import { combineWildlifeGridSnapshots } from './source.js';
+import {
+  combineWildlifeGridSnapshots,
+  wildlifeFailureRetryable,
+} from './source.js';
 import { wildlifeMeasure } from './performance.js';
 import { isRecentWildlifeSighting } from './records.js';
 import {
@@ -92,25 +95,26 @@ export function createLifecycle({ state, services, parts }) {
     state.pendingParams = null;
   }
 
-  function applyRetry(error, retryAfterSec = null) {
-    // Only upstream failures impose cooldown; network/schema errors retry on pan.
-    const status =
-      error?.upstreamStatus ??
-      error?.status ??
-      Number(/HTTP (\d{3})/.exec(error?.message || '')?.[1]);
-    const retryDelays = [error?.retryAfterSec, retryAfterSec].filter(
-      (value) => Number.isFinite(value) && value >= 0,
-    );
-    const retryFromSnapshot =
-      Number.isFinite(retryAfterSec) && retryAfterSec >= 0;
-    if (!(status === 429 || status >= 500) && !retryFromSnapshot) {
+  function applyRetry(errors, retryAfterSec = null) {
+    // Only upstream failures impose cooldown; network/schema errors retry on
+    // pan. Several sources can fail together: wait for the longest of them.
+    const failures = [errors].flat().filter(wildlifeFailureRetryable);
+    const snapshotDelay =
+      Number.isFinite(retryAfterSec) && retryAfterSec >= 0
+        ? retryAfterSec * 1000
+        : null;
+    if (!failures.length && snapshotDelay === null) {
       state.failures = 0;
       state.retryAt = 0;
       return;
     }
-    const backoff = retryDelays.length
-      ? Math.max(...retryDelays) * 1000
-      : retryDelayMs(state.failures || 0);
+    const delays = failures.map((error) =>
+      Number.isFinite(error.retryAfterSec) && error.retryAfterSec >= 0
+        ? error.retryAfterSec * 1000
+        : retryDelayMs(state.failures || 0),
+    );
+    if (snapshotDelay !== null) delays.push(snapshotDelay);
+    const backoff = Math.max(...delays);
     state.failures = backoff;
     state.retryAt = services.now() + backoff;
   }
@@ -404,16 +408,39 @@ export function createLifecycle({ state, services, parts }) {
             },
           )
         : null;
-      const sourceError =
-        gridError ||
-        staleError ||
-        (snapshot.sources?.inaturalist?.ok === false
-          ? snapshot.sources.inaturalist.error ||
+      const inat = snapshot.sources?.inaturalist;
+      const acartia = snapshot.sources?.acartia;
+      // Keep each failed source's status and Retry-After; the displayed error
+      // is the one that explains the cooldown.
+      const sourceErrors = [
+        gridError,
+        staleError,
+        inat?.ok === false
+          ? inat.error ||
             new Error(msg('Wildlife sightings source unavailable'))
-          : null) ||
-        (snapshot.sources?.acartia?.ok === false
-          ? new Error(msg('Acartia feed unavailable'))
-          : null);
+          : null,
+        acartia?.ok === false
+          ? Object.assign(new Error(msg('Acartia feed unavailable')), {
+              ...(acartia.error?.status
+                ? { status: acartia.error.status }
+                : {}),
+              ...(acartia.error?.upstreamStatus
+                ? { upstreamStatus: acartia.error.upstreamStatus }
+                : {}),
+              ...(Number.isFinite(acartia.error?.retryAfterSec)
+                ? { retryAfterSec: acartia.error.retryAfterSec }
+                : {}),
+            })
+          : null,
+      ].filter(Boolean);
+      const sourceError =
+        sourceErrors.find(
+          (error) =>
+            wildlifeFailureRetryable(error) ||
+            Number.isFinite(error.retryAfterSec),
+        ) ||
+        sourceErrors[0] ||
+        null;
       state.stale =
         !sourceError && (snapshot.stale === true || grid?.stale === true);
       const fetchedAt = [snapshot.fetchedAt, grid?.fetchedAt].filter(
@@ -438,7 +465,7 @@ export function createLifecycle({ state, services, parts }) {
       if (sourceError) {
         failParams(params);
         applyRetry(
-          sourceError,
+          sourceErrors,
           retryAfter.length ? Math.max(...retryAfter) : null,
         );
       } else {

src/layers/wildlifeSightings/source.js

@@ -102,6 +102,36 @@ function validProjectedRows(rows, origin) {
   );
 }
 
+/** Return the status a failure reports. The proxy marks upstream failures
+ * with X-Upstream-Status; without it the HTTP status is the signal. */
+function wildlifeFailureStatus(error) {
+  return (
+    error?.upstreamStatus ||
+    error?.status ||
+    Number(/HTTP (\d{3})/.exec(error?.message || '')?.[1]) ||
+    0
+  );
+}
+
+/** Whether a failure asks clients to wait before the next request. */
+export function wildlifeFailureRetryable(error) {
+  const status = wildlifeFailureStatus(error);
+  return status === 429 || status >= 500;
+}
+
+/** Prefer the failure that carries retry timing, then the longest wait. */
+export function preferredWildlifeFailure(errors) {
+  const failures = errors.filter(Boolean);
+  const retryable = failures.filter(wildlifeFailureRetryable);
+  if (!retryable.length) return failures[0] || null;
+  return retryable.reduce((best, error) =>
+    (Number.isFinite(error.retryAfterSec) ? error.retryAfterSec : -1) >
+    (Number.isFinite(best.retryAfterSec) ? best.retryAfterSec : -1)
+      ? error
+      : best,
+  );
+}
+
 /** Merge occupied cells incrementally; neighboring UTFGrid hit tiles can
  * report the same geographic cell, so source ids remain the dedupe key. */
 export function combineWildlifeGridSnapshots(snapshots, error = null) {
@@ -421,7 +451,7 @@ export function createWildlifeSightingsSource({
       ),
     );
     signal?.throwIfAborted();
-    const failure = failures.find(Boolean);
+    const failure = preferredWildlifeFailure(failures);
     if (!snapshots.some(Boolean) && failure) throw failure;
     return combineWildlifeGridSnapshots(snapshots.filter(Boolean), failure);
   }
@@ -494,11 +524,15 @@ export function createWildlifeSightingsSource({
     });
     // Fail outright only when nothing succeeded; otherwise draw what arrived
     // and report the failed source in its per-source status.
+    const acartiaError =
+      useAcartia && acartiaResult.status === 'rejected'
+        ? acartiaResult.reason
+        : null;
     if (
       !inatSnapshots.length &&
       (!useAcartia || acartiaResult.status !== 'fulfilled')
     )
-      throw inatError;
+      throw preferredWildlifeFailure([inatError, acartiaError]);
     const inatRows = inatSnapshots.flatMap((snapshot) => snapshot.rows);
     const acartiaSnapshot =
       acartiaResult.status === 'fulfilled' ? acartiaResult.value : null;
@@ -553,12 +587,13 @@ export function createWildlifeSightingsSource({
           ...(inatError ? { error: inatError, failedBoxes } : {}),
         },
         acartia: {
-          ok: !useAcartia || acartiaResult.status === 'fulfilled',
+          ok: !acartiaError,
           stale: acartiaStale,
           fetchedAt: Number.isFinite(acartiaSnapshot?.fetchedAt)
             ? acartiaSnapshot.fetchedAt
             : null,
           count: drawnAcartia,
+          ...(acartiaError ? { error: acartiaError } : {}),
         },
       },
     };