From d7986309b6d0b89c8d493378e96f209780f08768 Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:24:41 -0700 Subject: [PATCH 1/6] fix(terrain): pause upstream calls after a 429 A rate-limited upstream got asked again by every page request for new points (about once a second while the camera moves), each with its own retries, which kept the limit in force and logged "refresh incomplete" each time. After a 429 the proxy now leaves the upstream alone for its Retry-After (at most 10 minutes), or 60 s when it sends none, serving cached points meanwhile. The warning that starts the pause says "(HTTP 429; pausing upstream for N s)"; requests refused during the pause don't log. Co-Authored-By: Claude Opus 5.5 --- server/providers/terrain.js | 44 ++++++++++++++++--- src/tooling/environmentProviders.test.mjs | 52 +++++++++++++++++++++++ 2 files changed, 91 insertions(+), 5 deletions(-) diff --git a/server/providers/terrain.js b/server/providers/terrain.js index f4817f602..4d122d7ac 100644 --- a/server/providers/terrain.js +++ b/server/providers/terrain.js @@ -6,6 +6,7 @@ import { parseTerrainPoints, resolveTerrainHeightRequest, terrainPointKey, + terrainRetryAfterMs, validTerrainResult, } from '../../src/data/terrainHeightsProxy.js'; import { makeRateLimiter, clientKey } from './common/rate-limit.js'; @@ -55,6 +56,15 @@ export function terrainHeightsProxy(options = {}) { // 2026-08-21). 64 points costs 5.6-12 s, inside the timeout at both ends of // that range. const UPSTREAM_CHUNK = 64; + // After a 429 the upstream is left alone for its Retry-After (at most + // RATE_LIMIT_PAUSE_MAX_MS, so a far-off date can't stop terrain until a + // restart), or RATE_LIMIT_PAUSE_MS when it gives none. Every page poll for + // new points (about once a second while the camera moves) otherwise asks + // again with its own retries, which keeps the limit in force and logs each + // time. + const RATE_LIMIT_PAUSE_MS = 60_000; + const RATE_LIMIT_PAUSE_MAX_MS = 10 * 60_000; + let rateLimitedUntil = 0; const MAX_POINTS = UPSTREAM_CHUNK; /** * Entry ceiling for the point cache, and with it the file written from it. @@ -211,15 +221,31 @@ export function terrainHeightsProxy(options = {}) { * @returns {Promise>} */ async function fetchUpstreamAll(points) { + if (Date.now() < rateLimitedUntil) { + // Rate-limited: asking again only extends the limit. Cached points are + // still served; the rest resolve on the page through its geoid. + const error = new Error('HTTP 429'); + error.coolingDown = true; + throw error; + } const results = []; let firstError = null; for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) { const chunk = points.slice(i, i + UPSTREAM_CHUNK); let chunkResults = []; - try { - chunkResults = await fetchTerrainChunkWithRetry(chunk); - } catch (error) { - firstError = firstError || error; + if (Date.now() >= rateLimitedUntil) { + try { + chunkResults = await fetchTerrainChunkWithRetry(chunk); + } catch (error) { + firstError = firstError || error; + if (error?.message === 'HTTP 429') + rateLimitedUntil = + Date.now() + + Math.min( + terrainRetryAfterMs(error.retryAfter) ?? RATE_LIMIT_PAUSE_MS, + RATE_LIMIT_PAUSE_MAX_MS, + ); + } } // Keep later chunks aligned even if a malformed upstream response omits // trailing positions. The resolver will reject each null individually. @@ -292,9 +318,17 @@ export function terrainHeightsProxy(options = {}) { evictOverflow(); diskDirty = true; } - if (outcome.upstreamError) { + if (outcome.upstreamError?.coolingDown) { + // Logged once, when the pause began. + } else if (outcome.upstreamError) { + const pause = + outcome.upstreamError.message === 'HTTP 429' && + Date.now() < rateLimitedUntil + ? ` (HTTP 429; pausing upstream for ${Math.ceil((rateLimitedUntil - Date.now()) / 1000)} s)` + : ''; console.warn( '[terrain-heights-proxy] refresh incomplete' + + pause + ' — serving stale points when available', ); } else if (outcome.absentPoints > 0) { diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs index 5e786e408..dc0bc6ab7 100644 --- a/src/tooling/environmentProviders.test.mjs +++ b/src/tooling/environmentProviders.test.mjs @@ -411,3 +411,55 @@ test('terrain middleware retains successful chunks around a failure and retries ); assert.equal(requests.length, 4); }); + +test('after a 429 the terrain proxy leaves the upstream alone and logs once', async (t) => { + isolate(t); + const warnings = []; + t.mock.method(console, 'warn', (...args) => warnings.push(args.join(' '))); + let calls = 0; + t.mock.method(globalThis, 'fetch', async () => { + calls += 1; + return new Response('', { status: 429 }); + }); + const request = install(terrainHeightsProxy()); + await request('/?points=1,2'); + const afterFirst = calls; + assert.ok(afterFirst >= 1); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /HTTP 429; pausing upstream for 60 s/); + // New points during the pause: no upstream call, no new log line. + const paused = await request('/?points=3,4'); + assert.equal(calls, afterFirst); + assert.equal(paused.status, 502); + assert.equal(warnings.length, 1); +}); + +test('a far-off Retry-After pauses the terrain upstream for at most 10 minutes', async (t) => { + isolate(t); + t.mock.method(console, 'warn', () => {}); + t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 }); + let limited = true; + t.mock.method(globalThis, 'fetch', async () => + limited + ? new Response('', { status: 429, headers: { 'retry-after': '86400' } }) + : Response.json({ results: [{ ellipsoid: 100 }] }), + ); + const request = install(terrainHeightsProxy()); + // The first request spends its retry budget on 429s, then the pause begins. + let settled = false; + const first = request('/?points=1,2').then((res) => { + settled = true; + return res; + }); + while (!settled) { + await new Promise((resolve) => setImmediate(resolve)); + t.mock.timers.tick(1_000); + } + assert.equal((await first).status, 502); + limited = false; + const pausedAt = Date.now(); + t.mock.timers.setTime(pausedAt + 9 * 60_000); + assert.equal((await request('/?points=3,4')).status, 502, 'still paused'); + t.mock.timers.setTime(pausedAt + 10 * 60_000 + 1); + assert.equal((await request('/?points=3,4')).status, 200, 'pause over'); +}); From d1bdae20d3d3df101ae74882e4eaaad717168ef7 Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:38:20 -0700 Subject: [PATCH 2/6] docs(changelog): note #1402 Co-Authored-By: Claude Opus 5.5 --- changelog.d/1402-terrain-429-pause.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/1402-terrain-429-pause.md diff --git a/changelog.d/1402-terrain-429-pause.md b/changelog.d/1402-terrain-429-pause.md new file mode 100644 index 000000000..fb7e14f10 --- /dev/null +++ b/changelog.d/1402-terrain-429-pause.md @@ -0,0 +1,3 @@ +- The terrain proxy stops asking the upstream for a while after a 429 + (its Retry-After, at most 10 minutes, or 60 s) instead of retrying on every + page request (quanticsoul4772, #1402). From 3ac986882c46451c7270dc2fb133bea43cb2b15d Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:56:47 -0700 Subject: [PATCH 3/6] test(terrain): bound the 429 pause test's settle loop Co-Authored-By: Claude Opus 5.5 --- src/tooling/environmentProviders.test.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs index dc0bc6ab7..52dcd8351 100644 --- a/src/tooling/environmentProviders.test.mjs +++ b/src/tooling/environmentProviders.test.mjs @@ -451,10 +451,12 @@ test('a far-off Retry-After pauses the terrain upstream for at most 10 minutes', settled = true; return res; }); - while (!settled) { + // Bounded: the retry budget is 10 s, so 60 simulated seconds is plenty. + for (let step = 0; step < 60 && !settled; step += 1) { await new Promise((resolve) => setImmediate(resolve)); t.mock.timers.tick(1_000); } + assert.ok(settled, 'the first request settles within its retry budget'); assert.equal((await first).status, 502); limited = false; const pausedAt = Date.now(); From 923211c24ac5f38d0ac04a2ab554792385d472e6 Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:08:26 -0700 Subject: [PATCH 4/6] fix(terrain): key the 429 pause on status, never shorten it, and document it Review follow-ups: the upstream error now carries its HTTP status and the pause checks error.status instead of the message text; concurrent 429s extend the pause with Math.max rather than a shorter Retry-After cutting a longer one short; the per-chunk re-check says it is for multi-chunk requests; the first 429 test runs its backoff on mocked timers. CURRENT-STATE describes the pause. Co-Authored-By: Claude Opus 5.5 --- docs/CURRENT-STATE.md | 6 +++++- server/providers/terrain.js | 20 +++++++++++++------- src/data/terrainHeightsProxy.js | 1 + src/tooling/environmentProviders.test.mjs | 12 +++++++++++- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index cc4e7b861..d0ead090b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3005,7 +3005,11 @@ down | auth-failed` (plus the unchanged `missing-key`/`unsupported`) with > usable per point, while an uncached absent height still returns 502 rather > than becoming a fabricated ground value. A position the upstream answers > with a null ellipsoid is reported as an absent height rather than a refresh -> failure, and is left uncached so a later poll re-asks it. Client geoid +> failure, and is left uncached so a later poll re-asks it. After a 429 that +> outlasts its retries, the proxy pauses upstream calls for the `Retry-After` +> (60 seconds when none, at most 10 minutes): cached and stale points are still +> served, uncached points get 502 and use the page's geoid fallback, and the +> pause is logged once by the request that started it. Client geoid > fallbacks wait 60 seconds before retrying and self-heal to Re:Earth on the > first later successful fetch. Smaller chunks reduce timeout risk; complete > camera batches still wait for their sequential requests, and unresolved diff --git a/server/providers/terrain.js b/server/providers/terrain.js index 4d122d7ac..b217cd900 100644 --- a/server/providers/terrain.js +++ b/server/providers/terrain.js @@ -225,6 +225,7 @@ export function terrainHeightsProxy(options = {}) { // Rate-limited: asking again only extends the limit. Cached points are // still served; the rest resolve on the page through its geoid. const error = new Error('HTTP 429'); + error.status = 429; error.coolingDown = true; throw error; } @@ -233,18 +234,23 @@ export function terrainHeightsProxy(options = {}) { for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) { const chunk = points.slice(i, i + UPSTREAM_CHUNK); let chunkResults = []; + // Re-checked per chunk for when a request spans several chunks again; + // today requests are capped at one chunk (MAX_POINTS). if (Date.now() >= rateLimitedUntil) { try { chunkResults = await fetchTerrainChunkWithRetry(chunk); } catch (error) { firstError = firstError || error; - if (error?.message === 'HTTP 429') - rateLimitedUntil = + // Concurrent 429s extend the pause, never cut it short. + if (error?.status === 429) + rateLimitedUntil = Math.max( + rateLimitedUntil, Date.now() + - Math.min( - terrainRetryAfterMs(error.retryAfter) ?? RATE_LIMIT_PAUSE_MS, - RATE_LIMIT_PAUSE_MAX_MS, - ); + Math.min( + terrainRetryAfterMs(error.retryAfter) ?? RATE_LIMIT_PAUSE_MS, + RATE_LIMIT_PAUSE_MAX_MS, + ), + ); } } // Keep later chunks aligned even if a malformed upstream response omits @@ -322,7 +328,7 @@ export function terrainHeightsProxy(options = {}) { // Logged once, when the pause began. } else if (outcome.upstreamError) { const pause = - outcome.upstreamError.message === 'HTTP 429' && + outcome.upstreamError.status === 429 && Date.now() < rateLimitedUntil ? ` (HTTP 429; pausing upstream for ${Math.ceil((rateLimitedUntil - Date.now()) / 1000)} s)` : ''; diff --git a/src/data/terrainHeightsProxy.js b/src/data/terrainHeightsProxy.js index c83b30a87..562d4d7c1 100644 --- a/src/data/terrainHeightsProxy.js +++ b/src/data/terrainHeightsProxy.js @@ -145,6 +145,7 @@ export async function fetchTerrainChunkWithRetry( const res = await fetchImpl(url, signal ? { signal } : {}); if (!res.ok) { const error = new Error(`HTTP ${res.status}`); + error.status = res.status; error.retryable = res.status === 429 || res.status >= 500; error.retryAfter = res.headers?.get?.('retry-after') ?? null; throw error; diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs index 52dcd8351..e541a8069 100644 --- a/src/tooling/environmentProviders.test.mjs +++ b/src/tooling/environmentProviders.test.mjs @@ -421,8 +421,18 @@ test('after a 429 the terrain proxy leaves the upstream alone and logs once', as calls += 1; return new Response('', { status: 429 }); }); + t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 }); const request = install(terrainHeightsProxy()); - await request('/?points=1,2'); + // The retries' backoff runs on mocked time. + let settled = false; + const first = request('/?points=1,2').then(() => { + settled = true; + }); + for (let step = 0; step < 60 && !settled; step += 1) { + await new Promise((resolve) => setImmediate(resolve)); + t.mock.timers.tick(1_000); + } + await first; const afterFirst = calls; assert.ok(afterFirst >= 1); assert.equal(warnings.length, 1); From 5410c973072acd958130da7d7d3ff16281eac3a0 Mon Sep 17 00:00:00 2001 From: quanticsoul4772 <71096528+quanticsoul4772@users.noreply.github.com> Date: Sun, 25 Oct 2026 18:55:17 -0700 Subject: [PATCH 5/6] test(terrain): fail the first 429 test instead of hanging if its request never settles Co-Authored-By: Claude Opus 5.5 --- src/tooling/environmentProviders.test.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs index e541a8069..45940e00b 100644 --- a/src/tooling/environmentProviders.test.mjs +++ b/src/tooling/environmentProviders.test.mjs @@ -432,6 +432,7 @@ test('after a 429 the terrain proxy leaves the upstream alone and logs once', as await new Promise((resolve) => setImmediate(resolve)); t.mock.timers.tick(1_000); } + assert.ok(settled, 'the first request settles within its retry budget'); await first; const afterFirst = calls; assert.ok(afterFirst >= 1); From c1820aea4950b259605f4335e35e4fea231e6aa5 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Wed, 28 Oct 2026 14:09:12 +0000 Subject: [PATCH 6/6] fix(terrain): end retries early on a long Retry-After; cover the 429 pause A Retry-After longer than the 10 s retry budget used to sleep the whole budget out and then give up, so the proxy's 429 pause started 10 s late and every new-point poll in that window still went upstream and logged. The retry loop now stops at once in that case. Tests: the retry loop no longer sleeps for an unreachable Retry-After; fully cached points, including stale ones, are served during the pause with no upstream call; a 503 does not pause; a shorter concurrent Retry-After never cuts the pause short. CURRENT-STATE now says which requests are still served and which log. --- changelog.d/1402-terrain-429-pause.md | 4 +- docs/CURRENT-STATE.md | 11 +-- server/providers/terrain.js | 3 +- src/data/terrainHeightsProxy.js | 6 +- src/data/terrainHeightsProxy.test.mjs | 30 ++++++++ src/tooling/environmentProviders.test.mjs | 88 +++++++++++++++++++++++ 6 files changed, 135 insertions(+), 7 deletions(-) diff --git a/changelog.d/1402-terrain-429-pause.md b/changelog.d/1402-terrain-429-pause.md index fb7e14f10..05b18680b 100644 --- a/changelog.d/1402-terrain-429-pause.md +++ b/changelog.d/1402-terrain-429-pause.md @@ -1,3 +1,5 @@ - The terrain proxy stops asking the upstream for a while after a 429 (its Retry-After, at most 10 minutes, or 60 s) instead of retrying on every - page request (quanticsoul4772, #1402). + page request, and a Retry-After longer than its 10 s retry budget now ends + the retries at once instead of sleeping the budget out (quanticsoul4772, + #1402). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index df37c6805..a8f3646c8 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3078,11 +3078,14 @@ down | auth-failed` (plus the unchanged `missing-key`/`unsupported`) with > usable per point, while an uncached absent height still returns 502 rather > than becoming a fabricated ground value. A position the upstream answers > with a null ellipsoid is reported as an absent height rather than a refresh -> failure, and is left uncached so a later poll re-asks it. After a 429 that +> failure, and is left uncached so a later poll re-asks it. A `Retry-After` +> longer than the 10 s retry budget ends the retries at once. After a 429 that > outlasts its retries, the proxy pauses upstream calls for the `Retry-After` -> (60 seconds when none, at most 10 minutes): cached and stale points are still -> served, uncached points get 502 and use the page's geoid fallback, and the -> pause is logged once by the request that started it. Client geoid +> (60 seconds when none, at most 10 minutes). A request whose points are all +> cached, fresh or stale, is still served; a request with any uncached point +> gets 502 and the page uses its geoid fallback for that chunk. Each request +> that was waiting on the upstream when the 429 came back logs one warning; +> requests refused during the pause do not log. Client geoid > fallbacks wait 60 seconds before retrying and self-heal to Re:Earth on the > first later successful fetch. Smaller chunks reduce timeout risk; complete > camera batches still wait for their sequential requests, and unresolved diff --git a/server/providers/terrain.js b/server/providers/terrain.js index b217cd900..35606308a 100644 --- a/server/providers/terrain.js +++ b/server/providers/terrain.js @@ -209,7 +209,8 @@ export function terrainHeightsProxy(options = {}) { * * The first try keeps its empirically required 30s timeout; network errors, * 429, and 5xx receive up to three jittered retries sharing a 10s added-time - * budget, with Retry-After honored within that bound. + * budget, with Retry-After honored within that bound; a longer Retry-After + * ends the retries at once so the 429 pause below starts without delay. * * A chunk that still fails yields nulls for its own positions rather than * rejecting: with several chunks per request, throwing would discard every diff --git a/src/data/terrainHeightsProxy.js b/src/data/terrainHeightsProxy.js index 562d4d7c1..f06ff3671 100644 --- a/src/data/terrainHeightsProxy.js +++ b/src/data/terrainHeightsProxy.js @@ -107,7 +107,10 @@ export function terrainRetryAfterMs(value, nowMs = Date.now()) { /** * Fetch one upstream chunk with jittered exponential backoff. Network errors, * 429, and 5xx retry; other HTTP failures and malformed successes fail fast. - * Retry sleeps and retry attempts share a 10s budget after attempt one. + * Retry sleeps and retry attempts share a 10s budget after attempt one. A + * Retry-After longer than the budget left ends the retries at once: sleeping + * out the budget cannot reach the time the upstream asked for, and it would + * delay the caller's own back-off (the proxy's 429 pause) by up to 10 s. * * Dependencies are injectable for deterministic offline tests. * @param {Array<[number, number]>} points @@ -168,6 +171,7 @@ export async function fetchTerrainChunkWithRetry( if (remaining <= 0) break; const retryAfterMs = terrainRetryAfterMs(error?.retryAfter, now()); + if (retryAfterMs != null && retryAfterMs > remaining) break; const backoffMs = Math.round( 350 * 2 ** attempt * (0.75 + random() * 0.5), ); diff --git a/src/data/terrainHeightsProxy.test.mjs b/src/data/terrainHeightsProxy.test.mjs index 50e45b732..c9e143337 100644 --- a/src/data/terrainHeightsProxy.test.mjs +++ b/src/data/terrainHeightsProxy.test.mjs @@ -91,6 +91,36 @@ test('429 retry honors Retry-After before the next attempt', async () => { assert.equal(results[0].id, 'recovered'); }); +test('a Retry-After longer than the retry budget ends the retries without sleeping', async () => { + let clock = 3_000_000; + let attempts = 0; + const sleeps = []; + await assert.rejects( + fetchTerrainChunkWithRetry([[1, 1]], { + now: () => clock, + random: () => 0.5, + makeSignal: () => undefined, + sleep: async (ms) => { + sleeps.push(ms); + clock += ms; + }, + fetchImpl: async () => { + attempts += 1; + return { + ok: false, + status: 429, + headers: { + get: (name) => (name.toLowerCase() === 'retry-after' ? '30' : null), + }, + }; + }, + }), + (error) => error.status === 429 && error.retryAfter === '30', + ); + assert.equal(attempts, 1); + assert.deepEqual(sleeps, [], 'no sleep that cannot reach the Retry-After'); +}); + test('permanently failing upstream exhausts retries and returns 502 without fake results', async () => { let clock = 2_000_000; let attempts = 0; diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs index 45940e00b..d2f827838 100644 --- a/src/tooling/environmentProviders.test.mjs +++ b/src/tooling/environmentProviders.test.mjs @@ -476,3 +476,91 @@ test('a far-off Retry-After pauses the terrain upstream for at most 10 minutes', t.mock.timers.setTime(pausedAt + 10 * 60_000 + 1); assert.equal((await request('/?points=3,4')).status, 200, 'pause over'); }); + +/** Drive mocked timers until `promise` settles; the retry budget is 10 s. */ +async function settleOnMockedTime(t, promise) { + let settled = false; + const tracked = promise.finally(() => { + settled = true; + }); + for (let step = 0; step < 60 && !settled; step += 1) { + await new Promise((resolve) => setImmediate(resolve)); + t.mock.timers.tick(1_000); + } + assert.ok(settled, 'the request settles within its retry budget'); + return tracked; +} + +test('during a terrain 429 pause fully cached points, even stale ones, are still served', async (t) => { + isolate(t); + t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 }); + let calls = 0; + let limited = false; + t.mock.method(globalThis, 'fetch', async () => { + calls += 1; + return limited + ? new Response('', { status: 429, headers: { 'retry-after': '300' } }) + : Response.json({ results: [{ ellipsoid: 100 }] }); + }); + const request = install(terrainHeightsProxy()); + assert.equal((await request('/?points=5,6')).status, 200); + // Past the 30-day TTL, so 5,6 is stale and would normally be refreshed. + t.mock.timers.setTime(Date.now() + 31 * 24 * 3600_000); + limited = true; + const first = await settleOnMockedTime(t, request('/?points=1,2')); + assert.equal(first.status, 502); + assert.equal(calls, 2, 'a Retry-After past the budget is not retried'); + const stale = await request('/?points=5,6'); + assert.equal(stale.status, 200); + assert.deepEqual(json(stale).results, [{ ellipsoid: 100 }]); + assert.equal((await request('/?points=5,6;7,8')).status, 502); + assert.equal(calls, 2, 'no upstream call during the pause'); +}); + +test('a terrain upstream failure other than 429 does not pause later requests', async (t) => { + isolate(t); + t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 }); + let calls = 0; + t.mock.method(globalThis, 'fetch', async () => { + calls += 1; + return new Response('', { status: 503 }); + }); + const request = install(terrainHeightsProxy()); + const first = await settleOnMockedTime(t, request('/?points=1,2')); + assert.equal(first.status, 502); + const afterFirst = calls; + const second = await settleOnMockedTime(t, request('/?points=3,4')); + assert.equal(second.status, 502); + assert.ok(calls > afterFirst, 'new points still go upstream after a 503'); +}); + +test('a shorter concurrent Retry-After never cuts a terrain 429 pause short', async (t) => { + isolate(t); + t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 }); + let calls = 0; + t.mock.method(globalThis, 'fetch', async (raw) => { + calls += 1; + const points = new URL(raw).searchParams.get('points'); + if (points.startsWith('0.')) + return Response.json({ results: [{ ellipsoid: 1 }] }); + const retryAfter = points.startsWith('1.') ? '300' : '30'; + return new Response('', { + status: 429, + headers: { 'retry-after': retryAfter }, + }); + }); + const request = install(terrainHeightsProxy()); + // Load the disk cache first so both requests below reach the upstream in + // order: the 300 s answer lands first, the 30 s answer after it. + assert.equal((await request('/?points=0,0')).status, 200); + const [long, short] = await settleOnMockedTime( + t, + Promise.all([request('/?points=1,2'), request('/?points=3,4')]), + ); + assert.equal(long.status, 502); + assert.equal(short.status, 502); + const afterBoth = calls; + t.mock.timers.setTime(Date.now() + 120_000); + assert.equal((await request('/?points=5,6')).status, 502); + assert.equal(calls, afterBoth, 'still paused by the 300 s Retry-After'); +});