From dac9b5627d629655304f11eddd5e45f861f512b9 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Thu, 5 Nov 2026 16:42:45 -0800 Subject: [PATCH] fix(wildlife): wait for the longest effective delay among every failed request One error was chosen before each failure's delay was known. An antimeridian box 400 hid the other box's 503 Retry-After; a grid tile 400 with Retry-After set a cooldown through the snapshot metadata; a short Retry-After on one tile undercut a sibling 5xx's default backoff; and a huge Retry-After made the deadline Infinity. Every failed request now keeps its status and Retry-After. One function gives each failure's delay: 4xx other than 429 none, 429 and 5xx their Retry-After or the default backoff. The layer waits for the longest, and shows the failure that set it. Retry-After is capped at one day, the proxy's own daily-budget deadline, and non-finite values are ignored. Each publish of one acquisition derives the default backoff from the backoff before it started, so repeated tile publishes no longer double it. --- .../wildlifeSightings/acquisition.test.mjs | 95 +++++++++++++++++++ src/layers/wildlifeSightings/lifecycle.js | 74 ++++++++------- src/layers/wildlifeSightings/policy.js | 3 + src/layers/wildlifeSightings/source.js | 89 +++++++++++++---- 4 files changed, 209 insertions(+), 52 deletions(-) diff --git a/src/layers/wildlifeSightings/acquisition.test.mjs b/src/layers/wildlifeSightings/acquisition.test.mjs index d343ad9b0..b4ac47813 100644 --- a/src/layers/wildlifeSightings/acquisition.test.mjs +++ b/src/layers/wildlifeSightings/acquisition.test.mjs @@ -1072,3 +1072,98 @@ for (const [name, respond] of [ assert.equal(h.state.renderedRows.length, 2); }); } + +// Every failed request keeps its own timing. The layer waits for the longest +// effective delay: 4xx other than 429 never waits, 429/5xx wait for their +// Retry-After or the default backoff. The shown error explains that wait. +async function settleThroughSource(fetchImpl, bounds = salishSea) { + const h = harness( + createWildlifeSightingsSource({ now: () => NOW, fetchImpl }), + ); + h.bounds(bounds); + h.state.enabled = false; + h.layer.enable(); + const first = h.layer.update(); + await h.advance(300); + await first; + await new Promise((resolve) => setImmediate(resolve)); + await new Promise((resolve) => setImmediate(resolve)); + return h; +} +const tileX = (url) => + Number(new URL(url, 'https://example.test').searchParams.get('x')); +const observations = () => + Response.json({ rows: [inatRow], totalResults: 1, fetchedAt: NOW }); + +test('an antimeridian box 400 never hides the other box 503 Retry-After', async () => { + for (const westFails of [true, false]) { + const h = await settleThroughSource( + async (url) => { + if (url.startsWith('/api/inaturalist/grid?')) return emptyGrid(); + const west = + Number( + new URL(url, 'https://example.test').searchParams.get('west'), + ) > 0; + return west === westFails + ? new Response('', { status: 400 }) + : unavailable(); + }, + { south: 10, west: 179, north: 12, east: -179 }, + ); + assert.equal(h.state.error, 'iNaturalist HTTP 503'); + assert.equal(h.state.retryAt, NOW + 300 + 120_000); + assert.equal(h.layer.getStats().retryInSec, 120); + } +}); + +test('a grid tile 400 with Retry-After beside successful tiles creates no cooldown', async () => { + let tiles = new Set(); + const h = await settleThroughSource(async (url) => { + if (!url.startsWith('/api/inaturalist/grid?')) return observations(); + tiles.add(tileX(url)); + return tileX(url) === Math.min(...tiles) && tiles.size === 1 + ? new Response('', { status: 400, headers: { 'Retry-After': '120' } }) + : emptyGrid(); + }); + assert.ok(tiles.size > 1, 'the view spans several grid tiles'); + assert.equal(h.state.error, 'iNaturalist HTTP 400'); + assert.equal(h.state.retryAt, 0); + assert.equal(h.layer.getStats().retryInSec, undefined); +}); + +test('a short grid Retry-After never undercuts a sibling 5xx default backoff', async () => { + let failed = 0; + const h = await settleThroughSource(async (url) => { + if (!url.startsWith('/api/inaturalist/grid?')) return observations(); + failed++; + if (failed === 1) + return new Response('', { status: 503, headers: { 'Retry-After': '2' } }); + if (failed === 2) return new Response('', { status: 502 }); + return emptyGrid(); + }); + assert.ok(failed >= 2); + assert.equal(h.state.error, 'iNaturalist HTTP 502'); + assert.equal(h.state.retryAt, NOW + 300 + 30_000); + assert.equal(h.layer.getStats().retryInSec, 30); +}); + +test('an enormous Retry-After is capped at one day and a non-finite one uses the default backoff', async () => { + for (const [header, delay] of [ + ['1' + '0'.repeat(306), 86_400_000], + ['1' + '0'.repeat(400), 30_000], + ]) { + const h = await settleThroughSource(async (url) => + url.startsWith('/api/inaturalist/grid?') + ? emptyGrid() + : url.startsWith('/api/acartia') + ? Response.json({ rows: [], fetchedAt: NOW }) + : new Response('', { + status: 503, + headers: { 'Retry-After': header }, + }), + ); + assert.equal(h.state.error, 'iNaturalist HTTP 503'); + assert.ok(Number.isFinite(h.state.retryAt)); + assert.equal(h.state.retryAt, NOW + 300 + delay, header.length); + } +}); diff --git a/src/layers/wildlifeSightings/lifecycle.js b/src/layers/wildlifeSightings/lifecycle.js index 3b2a2999a..0fe430fbe 100644 --- a/src/layers/wildlifeSightings/lifecycle.js +++ b/src/layers/wildlifeSightings/lifecycle.js @@ -2,7 +2,10 @@ import { msg } from '../../i18n/marker.js'; import { getWildlifeHeightBand, rowInWildlifeBoxes } from './model.js'; import { combineWildlifeGridSnapshots, - wildlifeFailureRetryable, + preferredWildlifeFailure, + wildlifeFailureDelayMs, + wildlifeFailures, + wildlifeRetryAfterMs, } from './source.js'; import { wildlifeMeasure } from './performance.js'; import { isRecentWildlifeSighting } from './records.js'; @@ -12,11 +15,7 @@ import { wildlifeGridTiles, wildlifeGridCellId, } from './gridViewport.js'; -import { - wildlifeViewportBoxes, - snapWildlifeBox, - retryDelayMs, -} from './viewport.js'; +import { wildlifeViewportBoxes, snapWildlifeBox } from './viewport.js'; import { WILDLIFE_DEBOUNCE_MS, WILDLIFE_MAX_ROWS, @@ -95,25 +94,25 @@ export function createLifecycle({ state, services, parts }) { state.pendingParams = null; } - function applyRetry(errors, retryAfterSec = null) { - // Only upstream failures impose cooldown; network/schema errors retry on - // pan. Several sources can fail together: wait for the longest of them. - const failures = [errors].flat().filter(wildlifeFailureRetryable); - const snapshotDelay = - Number.isFinite(retryAfterSec) && retryAfterSec >= 0 - ? retryAfterSec * 1000 - : null; - if (!failures.length && snapshotDelay === null) { + function applyRetry( + errors, + retryAfterSec = null, + previousMs = state.failures || 0, + ) { + // Only 429/5xx failures impose cooldown; client, network and schema + // errors retry on pan. Several requests can fail together: wait for the + // longest effective delay among them. + const delays = [ + ...wildlifeFailures(errors).map((error) => + wildlifeFailureDelayMs(error, previousMs), + ), + wildlifeRetryAfterMs(retryAfterSec), + ].filter((delay) => delay !== null); + if (!delays.length) { state.failures = 0; state.retryAt = 0; return; } - const delays = failures.map((error) => - Number.isFinite(error.retryAfterSec) && error.retryAfterSec >= 0 - ? error.retryAfterSec * 1000 - : retryDelayMs(state.failures || 0), - ); - if (snapshotDelay !== null) delays.push(snapshotDelay); const backoff = Math.max(...delays); state.failures = backoff; state.retryAt = services.now() + backoff; @@ -282,6 +281,7 @@ export function createLifecycle({ state, services, parts }) { request, signal, epoch, + previousFailures, replaceRows = true, }) { const current = currentQuery(); @@ -398,10 +398,14 @@ export function createLifecycle({ state, services, parts }) { value.upstreamStatus === 429 || value.upstreamStatus >= 500), ); + // The filter above admits only 429/5xx or an unmarked stale answer. const staleError = failedStale.length ? Object.assign( new Error(msg('Wildlife sightings source unavailable')), { + upstreamStatus: Math.max( + ...failedStale.map((value) => value.upstreamStatus || 503), + ), retryAfterSec: Math.max( ...failedStale.map((value) => value.retryAfterSec), ), @@ -410,8 +414,8 @@ export function createLifecycle({ state, services, parts }) { : null; const inat = snapshot.sources?.inaturalist; const acartia = snapshot.sources?.acartia; - // Keep each failed source's status and Retry-After; the displayed error - // is the one that explains the cooldown. + // Keep each failed request's status and Retry-After; the displayed + // error is the one whose effective delay sets the cooldown. const sourceErrors = [ gridError, staleError, @@ -433,14 +437,10 @@ export function createLifecycle({ state, services, parts }) { }) : null, ].filter(Boolean); - const sourceError = - sourceErrors.find( - (error) => - wildlifeFailureRetryable(error) || - Number.isFinite(error.retryAfterSec), - ) || - sourceErrors[0] || - null; + const sourceError = preferredWildlifeFailure( + sourceErrors, + previousFailures, + ); state.stale = !sourceError && (snapshot.stale === true || grid?.stale === true); const fetchedAt = [snapshot.fetchedAt, grid?.fetchedAt].filter( @@ -467,12 +467,13 @@ export function createLifecycle({ state, services, parts }) { applyRetry( sourceErrors, retryAfter.length ? Math.max(...retryAfter) : null, + previousFailures, ); } else { state.pendingParams = null; state.failedParams = null; state.retryAt = retryAfter.length - ? services.now() + Math.max(...retryAfter) * 1000 + ? services.now() + wildlifeRetryAfterMs(Math.max(...retryAfter)) : 0; state.failures = 0; } @@ -632,6 +633,9 @@ export function createLifecycle({ state, services, parts }) { state.request?.abort(); const request = new AbortController(); const requestLifetime = lifetimeEpoch; + // One acquisition publishes several times (tiles, rows, final grid). + // Each derives its default backoff from the backoff before it started. + const previousFailures = state.failures || 0; const supersededResult = () => state.enabled && !state.destroyed && @@ -699,6 +703,7 @@ export function createLifecycle({ state, services, parts }) { request, signal, epoch: requestLifetime, + previousFailures, }); } function finishTiles() { @@ -804,11 +809,12 @@ export function createLifecycle({ state, services, parts }) { continueTiles(); state.stale = false; state.error = - error?.message || msg('Wildlife sightings source unavailable'); + preferredWildlifeFailure(error, previousFailures)?.message || + msg('Wildlife sightings source unavailable'); failParams(params); state.lastQueryKey = null; requestQueryKey = null; - applyRetry(error); + applyRetry(error, null, previousFailures); console.warn('[Data:WildlifeSightings] Update failed:', state.error); return true; } finally { diff --git a/src/layers/wildlifeSightings/policy.js b/src/layers/wildlifeSightings/policy.js index 4af9d7d17..f1b6df277 100644 --- a/src/layers/wildlifeSightings/policy.js +++ b/src/layers/wildlifeSightings/policy.js @@ -74,6 +74,9 @@ export { WILDLIFE_MAX_VIEW_DEGREES } from './bounds.js'; export const WILDLIFE_VIEW_PADDING = 0.3; export const WILDLIFE_RETRY_MIN_MS = 30_000; export const WILDLIFE_RETRY_MAX_MS = 240_000; +// Cap a Retry-After so one reply can never stall refresh forever. The proxy's +// daily-budget reply waits until the next UTC midnight, so allow one day. +export const WILDLIFE_RETRY_AFTER_MAX_SEC = 86_400; // Faster polling would reread the proxy's five-minute cache. export const WILDLIFE_UPDATE_INTERVAL_MS = 5 * 60_000; diff --git a/src/layers/wildlifeSightings/source.js b/src/layers/wildlifeSightings/source.js index aa218f34e..e2e3f1b80 100644 --- a/src/layers/wildlifeSightings/source.js +++ b/src/layers/wildlifeSightings/source.js @@ -1,8 +1,12 @@ import { readResponseJsonCapped } from '../../sources/httpBody.js'; import { mergeSightings } from './records.js'; import { wildlifeMeasure } from './performance.js'; -import { ACARTIA_GUARD_BOX, TAXA_PRESETS } from './policy.js'; -import { boxIntersects } from './viewport.js'; +import { + ACARTIA_GUARD_BOX, + TAXA_PRESETS, + WILDLIFE_RETRY_AFTER_MAX_SEC, +} from './policy.js'; +import { boxIntersects, retryDelayMs } from './viewport.js'; import { WILDLIFE_GRID_MAX_ZOOM, WILDLIFE_GRID_TILE_CAP, @@ -72,14 +76,19 @@ function retryAfterSeconds(value, nowMs) { const text = value.trim(); if (/^\d+(?:\.\d+)?$/.test(text)) { const seconds = Number(text); - return Number.isFinite(seconds) ? Math.ceil(seconds) : null; + return Number.isFinite(seconds) + ? Math.min(WILDLIFE_RETRY_AFTER_MAX_SEC, Math.ceil(seconds)) + : null; } // Reject malformed numeric values rather than letting Date.parse interpret // them as calendar dates. HTTP dates include a named weekday or month. if (!/[a-z]/i.test(text)) return null; const deadline = Date.parse(text); return Number.isFinite(deadline) && Number.isFinite(nowMs) - ? Math.max(0, Math.ceil((deadline - nowMs) / 1000)) + ? Math.min( + WILDLIFE_RETRY_AFTER_MAX_SEC, + Math.max(0, Math.ceil((deadline - nowMs) / 1000)), + ) : null; } @@ -119,17 +128,56 @@ export function wildlifeFailureRetryable(error) { return status === 429 || status >= 500; } -/** Prefer the failure that carries retry timing, then the longest wait. */ -export function preferredWildlifeFailure(errors) { - const failures = errors.filter(Boolean); - const retryable = failures.filter(wildlifeFailureRetryable); - if (!retryable.length) return failures[0] || null; - return retryable.reduce((best, error) => - (Number.isFinite(error.retryAfterSec) ? error.retryAfterSec : -1) > - (Number.isFinite(best.retryAfterSec) ? best.retryAfterSec : -1) - ? error - : best, - ); +/** Return a Retry-After in milliseconds, capped at one day, or null when + * it is missing, negative or not finite. */ +export function wildlifeRetryAfterMs(seconds) { + return Number.isFinite(seconds) && seconds >= 0 + ? Math.min(WILDLIFE_RETRY_AFTER_MAX_SEC, seconds) * 1000 + : null; +} + +/** Return how long a failure asks the layer to wait, or null for none: + * 429 and 5xx wait for their Retry-After, else the default backoff after + * `previousMs`; other failures may retry on the next view. */ +export function wildlifeFailureDelayMs(error, previousMs = 0) { + if (!error || !wildlifeFailureRetryable(error)) return null; + return wildlifeRetryAfterMs(error.retryAfterSec) ?? retryDelayMs(previousMs); +} + +/** Flatten failures, expanding a group into every request that failed. */ +export function wildlifeFailures(errors) { + return [errors] + .flat() + .filter(Boolean) + .flatMap((error) => + Array.isArray(error.failures) && error.failures.length + ? error.failures + : [error], + ); +} + +/** Return the failure with the longest effective delay (the first failure + * when none waits), carrying every failure so the caller can recompute. */ +export function preferredWildlifeFailure(errors, previousMs = 0) { + const failures = wildlifeFailures(errors); + if (!failures.length) return null; + let chosen = failures[0]; + let longest = wildlifeFailureDelayMs(chosen, previousMs) ?? -1; + for (const error of failures.slice(1)) { + const delay = wildlifeFailureDelayMs(error, previousMs) ?? -1; + if (delay > longest) [chosen, longest] = [error, delay]; + } + if (failures.length === 1) return chosen; + // Grid errors are shared between consumers of one tile request; describe + // the group with a new error rather than mutating any of them. + return Object.assign(new Error(chosen.message, { cause: chosen }), { + ...(chosen.status ? { status: chosen.status } : {}), + ...(chosen.upstreamStatus ? { upstreamStatus: chosen.upstreamStatus } : {}), + ...(Number.isFinite(chosen.retryAfterSec) + ? { retryAfterSec: chosen.retryAfterSec } + : {}), + failures, + }); } /** Merge occupied cells incrementally; neighboring UTFGrid hit tiles can @@ -145,9 +193,12 @@ export function combineWildlifeGridSnapshots(snapshots, error = null) { const totalResults = cells.reduce((sum, cell) => sum + cell.cellCount, 0); if (!Number.isSafeInteger(totalResults)) throw new Error('Malformed iNaturalist grid response'); + // A failure's Retry-After counts only when its status asks for a wait. const retryAfter = [ ...snapshots.map((snapshot) => snapshot.retryAfterSec), - error?.retryAfterSec, + ...wildlifeFailures(error) + .filter(wildlifeFailureRetryable) + .map((failure) => failure.retryAfterSec), ].filter(Number.isFinite); const stale = snapshots.some((snapshot) => snapshot.stale === true); return { @@ -514,14 +565,16 @@ export function createWildlifeSightingsSource({ signal?.throwIfAborted(); const inatSnapshots = []; const failedBoxes = []; - let inatError = null; + const inatErrors = []; inatResult.value.forEach((result, index) => { if (result.status === 'fulfilled') inatSnapshots.push(result.value); else { failedBoxes.push(requestBoxes[index]); - inatError ??= result.reason; + inatErrors.push(result.reason); } }); + // Keep every failed box; the layer waits for the longest of them. + const inatError = preferredWildlifeFailure(inatErrors); // Fail outright only when nothing succeeded; otherwise draw what arrived // and report the failed source in its per-source status. const acartiaError =