fix(wildlife): wait for the longest effective delay among every failed request

One error was chosen before each failure's delay was known. An antimeridian
box 400 hid the other box's 503 Retry-After; a grid tile 400 with Retry-After
set a cooldown through the snapshot metadata; a short Retry-After on one tile
undercut a sibling 5xx's default backoff; and a huge Retry-After made the
deadline Infinity.

Every failed request now keeps its status and Retry-After. One function gives
each failure's delay: 4xx other than 429 none, 429 and 5xx their Retry-After
or the default backoff. The layer waits for the longest, and shows the
failure that set it. Retry-After is capped at one day, the proxy's own
daily-budget deadline, and non-finite values are ignored. Each publish of
one acquisition derives the default backoff from the backoff before it
started, so repeated tile publishes no longer double it.
This commit is contained in:
Sameh Khamis committed 2026-11-05 16:42:45 -08:00
1 parent 42fa974ee3
commit dac9b5627d
4 files changed
+209 -52

No files matched your search

@@ -1072,3 +1072,98 @@ for (const [name, respond] of [
assert.equal(h.state.renderedRows.length, 2);
});
}
// Every failed request keeps its own timing. The layer waits for the longest
// effective delay: 4xx other than 429 never waits, 429/5xx wait for their
// Retry-After or the default backoff. The shown error explains that wait.
async function settleThroughSource(fetchImpl, bounds = salishSea) {
const h = harness(
createWildlifeSightingsSource({ now: () => NOW, fetchImpl }),
);
h.bounds(bounds);
h.state.enabled = false;
h.layer.enable();
const first = h.layer.update();
await h.advance(300);
await first;
await new Promise((resolve) => setImmediate(resolve));
await new Promise((resolve) => setImmediate(resolve));
return h;
}
const tileX = (url) =>
Number(new URL(url, 'https://example.test').searchParams.get('x'));
const observations = () =>
Response.json({ rows: [inatRow], totalResults: 1, fetchedAt: NOW });
test('an antimeridian box 400 never hides the other box 503 Retry-After', async () => {
for (const westFails of [true, false]) {
const h = await settleThroughSource(
async (url) => {
if (url.startsWith('/api/inaturalist/grid?')) return emptyGrid();
const west =
Number(
new URL(url, 'https://example.test').searchParams.get('west'),
) > 0;
return west === westFails
? new Response('', { status: 400 })
: unavailable();
},
{ south: 10, west: 179, north: 12, east: -179 },
);
assert.equal(h.state.error, 'iNaturalist HTTP 503');
assert.equal(h.state.retryAt, NOW + 300 + 120_000);
assert.equal(h.layer.getStats().retryInSec, 120);
}
});
test('a grid tile 400 with Retry-After beside successful tiles creates no cooldown', async () => {
let tiles = new Set();
const h = await settleThroughSource(async (url) => {
if (!url.startsWith('/api/inaturalist/grid?')) return observations();
tiles.add(tileX(url));
return tileX(url) === Math.min(...tiles) && tiles.size === 1
? new Response('', { status: 400, headers: { 'Retry-After': '120' } })
: emptyGrid();
});
assert.ok(tiles.size > 1, 'the view spans several grid tiles');
assert.equal(h.state.error, 'iNaturalist HTTP 400');
assert.equal(h.state.retryAt, 0);
assert.equal(h.layer.getStats().retryInSec, undefined);
});
test('a short grid Retry-After never undercuts a sibling 5xx default backoff', async () => {
let failed = 0;
const h = await settleThroughSource(async (url) => {
if (!url.startsWith('/api/inaturalist/grid?')) return observations();
failed++;
if (failed === 1)
return new Response('', { status: 503, headers: { 'Retry-After': '2' } });
if (failed === 2) return new Response('', { status: 502 });
return emptyGrid();
});
assert.ok(failed >= 2);
assert.equal(h.state.error, 'iNaturalist HTTP 502');
assert.equal(h.state.retryAt, NOW + 300 + 30_000);
assert.equal(h.layer.getStats().retryInSec, 30);
});
test('an enormous Retry-After is capped at one day and a non-finite one uses the default backoff', async () => {
for (const [header, delay] of [
['1' + '0'.repeat(306), 86_400_000],
['1' + '0'.repeat(400), 30_000],
]) {
const h = await settleThroughSource(async (url) =>
url.startsWith('/api/inaturalist/grid?')
? emptyGrid()
: url.startsWith('/api/acartia')
? Response.json({ rows: [], fetchedAt: NOW })
: new Response('', {
status: 503,
headers: { 'Retry-After': header },
}),
);
assert.equal(h.state.error, 'iNaturalist HTTP 503');
assert.ok(Number.isFinite(h.state.retryAt));
assert.equal(h.state.retryAt, NOW + 300 + delay, header.length);
}
});