diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 96d85431f..5166e4a2a 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -124,7 +124,11 @@ with retry timing; logs exclude request URLs and upstream exception text. A partial failure still draws what arrived: a failed iNaturalist request, or one failed half of an antimeridian view, keeps a good Acartia reply and the other half, retains that area's last iNaturalist rows and reports DEGRADED. -The update fails outright only when every requested source failed. +The update fails outright only when every requested source failed. Each failed +source keeps its HTTP status and `Retry-After` (the HTTP status counts when no +`X-Upstream-Status` is sent), and the next request waits out the longest 429 or +5xx cooldown among them. One update requests both sources, so the cooldown +holds the whole layer; a fully successful refresh clears it. `/api/acartia/sightings` adds Salish Sea marine mammals within 45.5–50.5° N, 126–121° W, cached ten minutes and stale for at most 24 h. A valid empty diff --git a/src/layers/wildlifeSightings/acquisition.test.mjs b/src/layers/wildlifeSightings/acquisition.test.mjs index ea1d6fc1a..d343ad9b0 100644 --- a/src/layers/wildlifeSightings/acquisition.test.mjs +++ b/src/layers/wildlifeSightings/acquisition.test.mjs @@ -833,10 +833,10 @@ test('an empty tile retires its old zoom cells while uncovered density survives assert.equal(h.state.gridTotal, 2); }); -test('only marked upstream failures create cooldown; local pacing, network and schema errors can retry on the next view', async () => { +test('only 429 and 5xx failures create cooldown; client, network and schema errors can retry on the next view', async () => { for (const error of [ - Object.assign(new Error('iNaturalist HTTP 503'), { - status: 503, + Object.assign(new Error('iNaturalist HTTP 400'), { + status: 400, upstreamStatus: 0, retryAfterSec: 30, }), @@ -873,6 +873,19 @@ test('only marked upstream failures create cooldown; local pacing, network and s 2, 'actual Retry-After is honored without imposing a 30s local floor', ); + // The proxy's daily-budget 503 has no upstream status. Its HTTP status and + // Retry-After still govern the next request. + const budget = harness({ + getSnapshot: async () => { + throw Object.assign(new Error('iNaturalist HTTP 503'), { + status: 503, + upstreamStatus: 0, + retryAfterSec: 3600, + }); + }, + }); + await budget.layer.update(); + assert.equal(budget.layer.getStats().retryInSec, 3600); }); test('stale cached tiles with local pacing metadata keep their display and do not enter cooldown', async () => { @@ -942,3 +955,120 @@ test('an explicit upstream Retry-After zero retries immediately while a missing } } }); + +const salishSea = { south: 47.5, west: -122.5, north: 47.7, east: -122.2 }; +const inatRow = { + stableId: 'inat:301', + origin: 'inaturalist', + lat: 47.6, + lon: -122.3, + observedAt: NOW - 3_600_000, + iconic: 'Mammalia', +}; +const acartiaRow = { + stableId: 'acartia:301', + origin: 'acartia', + lat: 47.6, + lon: -122.35, + observedAt: NOW - 3_600_000, + cetacean: true, + quality: 'research', + provider: { name: 'Provider', entity: null }, +}; +const emptyGrid = () => + Response.json({ cells: [], totalResults: 0, fetchedAt: NOW }); +const unavailable = (headers = {}) => + new Response('', { + status: 503, + headers: { 'Retry-After': '120', ...headers }, + }); + +// Each case runs the real source and lifecycle. A healthy source draws its +// rows while the failed source's Retry-After holds the next acquisition. +for (const [name, respond] of [ + [ + 'Acartia alone returns 503', + (url) => + url.startsWith('/api/acartia') + ? unavailable() + : url.startsWith('/api/inaturalist/grid?') + ? emptyGrid() + : Response.json({ + rows: [inatRow], + totalResults: 1, + fetchedAt: NOW, + }), + ], + [ + 'iNaturalist returns 503 without X-Upstream-Status', + (url) => + url.startsWith('/api/acartia') + ? Response.json({ rows: [acartiaRow], fetchedAt: NOW }) + : url.startsWith('/api/inaturalist/grid?') + ? emptyGrid() + : unavailable(), + ], + [ + 'iNaturalist returns 503 while every grid tile fails its schema', + (url) => + url.startsWith('/api/acartia') + ? Response.json({ rows: [acartiaRow], fetchedAt: NOW }) + : url.startsWith('/api/inaturalist/grid?') + ? Response.json({ cells: 'malformed' }) + : unavailable({ 'X-Upstream-Status': '503' }), + ], +]) { + test(`a partial failure keeps its Retry-After: ${name}`, async () => { + let healthy = false; + let requests = 0; + const source = createWildlifeSightingsSource({ + now: () => NOW, + fetchImpl: async (url) => { + if (!url.startsWith('/api/inaturalist/grid?')) requests++; + if (healthy) + return url.startsWith('/api/acartia') + ? Response.json({ rows: [acartiaRow], fetchedAt: NOW }) + : url.startsWith('/api/inaturalist/grid?') + ? emptyGrid() + : Response.json({ + rows: [inatRow], + totalResults: 1, + fetchedAt: NOW, + }); + return respond(url); + }, + }); + const h = harness(source); + h.bounds(salishSea); + h.state.enabled = false; + h.layer.enable(); + const first = h.layer.update(); + await h.advance(300); + await first; + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(h.state.renderedRows.length, 1, 'the healthy source draws'); + assert.ok(h.state.error, 'the failed source is reported'); + // The source clock stays at NOW; the layer settled 300 ms of camera time. + assert.equal(h.state.retryAt, NOW + 300 + 120_000); + assert.equal(h.layer.getStats().retryInSec, 120); + const attempted = requests; + h.bounds({ ...salishSea, north: 47.8 }); + h.moveEnd(); + assert.equal([...h.timers.values()][0]?.delay, 120_000); + await h.advance(119_999); + await h.layer.update(); + assert.equal( + requests, + attempted, + 'neither a camera move nor manager polling skips the cooldown', + ); + healthy = true; + await h.advance(1); + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(requests > attempted, 'acquisition resumes at the deadline'); + assert.equal(h.state.error, null); + assert.equal(h.state.retryAt, 0, 'a fully successful refresh clears it'); + assert.equal(h.layer.getStats().retryInSec, undefined); + assert.equal(h.state.renderedRows.length, 2); + }); +} diff --git a/src/layers/wildlifeSightings/lifecycle.js b/src/layers/wildlifeSightings/lifecycle.js index 6da425c7a..3b2a2999a 100644 --- a/src/layers/wildlifeSightings/lifecycle.js +++ b/src/layers/wildlifeSightings/lifecycle.js @@ -1,6 +1,9 @@ import { msg } from '../../i18n/marker.js'; import { getWildlifeHeightBand, rowInWildlifeBoxes } from './model.js'; -import { combineWildlifeGridSnapshots } from './source.js'; +import { + combineWildlifeGridSnapshots, + wildlifeFailureRetryable, +} from './source.js'; import { wildlifeMeasure } from './performance.js'; import { isRecentWildlifeSighting } from './records.js'; import { @@ -92,25 +95,26 @@ export function createLifecycle({ state, services, parts }) { state.pendingParams = null; } - function applyRetry(error, retryAfterSec = null) { - // Only upstream failures impose cooldown; network/schema errors retry on pan. - const status = - error?.upstreamStatus ?? - error?.status ?? - Number(/HTTP (\d{3})/.exec(error?.message || '')?.[1]); - const retryDelays = [error?.retryAfterSec, retryAfterSec].filter( - (value) => Number.isFinite(value) && value >= 0, - ); - const retryFromSnapshot = - Number.isFinite(retryAfterSec) && retryAfterSec >= 0; - if (!(status === 429 || status >= 500) && !retryFromSnapshot) { + function applyRetry(errors, retryAfterSec = null) { + // Only upstream failures impose cooldown; network/schema errors retry on + // pan. Several sources can fail together: wait for the longest of them. + const failures = [errors].flat().filter(wildlifeFailureRetryable); + const snapshotDelay = + Number.isFinite(retryAfterSec) && retryAfterSec >= 0 + ? retryAfterSec * 1000 + : null; + if (!failures.length && snapshotDelay === null) { state.failures = 0; state.retryAt = 0; return; } - const backoff = retryDelays.length - ? Math.max(...retryDelays) * 1000 - : retryDelayMs(state.failures || 0); + const delays = failures.map((error) => + Number.isFinite(error.retryAfterSec) && error.retryAfterSec >= 0 + ? error.retryAfterSec * 1000 + : retryDelayMs(state.failures || 0), + ); + if (snapshotDelay !== null) delays.push(snapshotDelay); + const backoff = Math.max(...delays); state.failures = backoff; state.retryAt = services.now() + backoff; } @@ -404,16 +408,39 @@ export function createLifecycle({ state, services, parts }) { }, ) : null; - const sourceError = - gridError || - staleError || - (snapshot.sources?.inaturalist?.ok === false - ? snapshot.sources.inaturalist.error || + const inat = snapshot.sources?.inaturalist; + const acartia = snapshot.sources?.acartia; + // Keep each failed source's status and Retry-After; the displayed error + // is the one that explains the cooldown. + const sourceErrors = [ + gridError, + staleError, + inat?.ok === false + ? inat.error || new Error(msg('Wildlife sightings source unavailable')) - : null) || - (snapshot.sources?.acartia?.ok === false - ? new Error(msg('Acartia feed unavailable')) - : null); + : null, + acartia?.ok === false + ? Object.assign(new Error(msg('Acartia feed unavailable')), { + ...(acartia.error?.status + ? { status: acartia.error.status } + : {}), + ...(acartia.error?.upstreamStatus + ? { upstreamStatus: acartia.error.upstreamStatus } + : {}), + ...(Number.isFinite(acartia.error?.retryAfterSec) + ? { retryAfterSec: acartia.error.retryAfterSec } + : {}), + }) + : null, + ].filter(Boolean); + const sourceError = + sourceErrors.find( + (error) => + wildlifeFailureRetryable(error) || + Number.isFinite(error.retryAfterSec), + ) || + sourceErrors[0] || + null; state.stale = !sourceError && (snapshot.stale === true || grid?.stale === true); const fetchedAt = [snapshot.fetchedAt, grid?.fetchedAt].filter( @@ -438,7 +465,7 @@ export function createLifecycle({ state, services, parts }) { if (sourceError) { failParams(params); applyRetry( - sourceError, + sourceErrors, retryAfter.length ? Math.max(...retryAfter) : null, ); } else { diff --git a/src/layers/wildlifeSightings/source.js b/src/layers/wildlifeSightings/source.js index 8bd41bf1d..aa218f34e 100644 --- a/src/layers/wildlifeSightings/source.js +++ b/src/layers/wildlifeSightings/source.js @@ -102,6 +102,36 @@ function validProjectedRows(rows, origin) { ); } +/** Return the status a failure reports. The proxy marks upstream failures + * with X-Upstream-Status; without it the HTTP status is the signal. */ +function wildlifeFailureStatus(error) { + return ( + error?.upstreamStatus || + error?.status || + Number(/HTTP (\d{3})/.exec(error?.message || '')?.[1]) || + 0 + ); +} + +/** Whether a failure asks clients to wait before the next request. */ +export function wildlifeFailureRetryable(error) { + const status = wildlifeFailureStatus(error); + return status === 429 || status >= 500; +} + +/** Prefer the failure that carries retry timing, then the longest wait. */ +export function preferredWildlifeFailure(errors) { + const failures = errors.filter(Boolean); + const retryable = failures.filter(wildlifeFailureRetryable); + if (!retryable.length) return failures[0] || null; + return retryable.reduce((best, error) => + (Number.isFinite(error.retryAfterSec) ? error.retryAfterSec : -1) > + (Number.isFinite(best.retryAfterSec) ? best.retryAfterSec : -1) + ? error + : best, + ); +} + /** Merge occupied cells incrementally; neighboring UTFGrid hit tiles can * report the same geographic cell, so source ids remain the dedupe key. */ export function combineWildlifeGridSnapshots(snapshots, error = null) { @@ -421,7 +451,7 @@ export function createWildlifeSightingsSource({ ), ); signal?.throwIfAborted(); - const failure = failures.find(Boolean); + const failure = preferredWildlifeFailure(failures); if (!snapshots.some(Boolean) && failure) throw failure; return combineWildlifeGridSnapshots(snapshots.filter(Boolean), failure); } @@ -494,11 +524,15 @@ export function createWildlifeSightingsSource({ }); // Fail outright only when nothing succeeded; otherwise draw what arrived // and report the failed source in its per-source status. + const acartiaError = + useAcartia && acartiaResult.status === 'rejected' + ? acartiaResult.reason + : null; if ( !inatSnapshots.length && (!useAcartia || acartiaResult.status !== 'fulfilled') ) - throw inatError; + throw preferredWildlifeFailure([inatError, acartiaError]); const inatRows = inatSnapshots.flatMap((snapshot) => snapshot.rows); const acartiaSnapshot = acartiaResult.status === 'fulfilled' ? acartiaResult.value : null; @@ -553,12 +587,13 @@ export function createWildlifeSightingsSource({ ...(inatError ? { error: inatError, failedBoxes } : {}), }, acartia: { - ok: !useAcartia || acartiaResult.status === 'fulfilled', + ok: !acartiaError, stale: acartiaStale, fetchedAt: Number.isFinite(acartiaSnapshot?.fetchedAt) ? acartiaSnapshot.fetchedAt : null, count: drawnAcartia, + ...(acartiaError ? { error: acartiaError } : {}), }, }, };