diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index cc4e7b861..d0ead090b 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3005,7 +3005,11 @@ down | auth-failed` (plus the unchanged `missing-key`/`unsupported`) with > usable per point, while an uncached absent height still returns 502 rather > than becoming a fabricated ground value. A position the upstream answers > with a null ellipsoid is reported as an absent height rather than a refresh -> failure, and is left uncached so a later poll re-asks it. Client geoid +> failure, and is left uncached so a later poll re-asks it. After a 429 that +> outlasts its retries, the proxy pauses upstream calls for the `Retry-After` +> (60 seconds when none, at most 10 minutes): cached and stale points are still +> served, uncached points get 502 and use the page's geoid fallback, and the +> pause is logged once by the request that started it. Client geoid > fallbacks wait 60 seconds before retrying and self-heal to Re:Earth on the > first later successful fetch. Smaller chunks reduce timeout risk; complete > camera batches still wait for their sequential requests, and unresolved diff --git a/server/providers/terrain.js b/server/providers/terrain.js index 4d122d7ac..b217cd900 100644 --- a/server/providers/terrain.js +++ b/server/providers/terrain.js @@ -225,6 +225,7 @@ export function terrainHeightsProxy(options = {}) { // Rate-limited: asking again only extends the limit. Cached points are // still served; the rest resolve on the page through its geoid. const error = new Error('HTTP 429'); + error.status = 429; error.coolingDown = true; throw error; } @@ -233,18 +234,23 @@ export function terrainHeightsProxy(options = {}) { for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) { const chunk = points.slice(i, i + UPSTREAM_CHUNK); let chunkResults = []; + // Re-checked per chunk for when a request spans several chunks again; + // today requests are capped at one chunk (MAX_POINTS). if (Date.now() >= rateLimitedUntil) { try { chunkResults = await fetchTerrainChunkWithRetry(chunk); } catch (error) { firstError = firstError || error; - if (error?.message === 'HTTP 429') - rateLimitedUntil = + // Concurrent 429s extend the pause, never cut it short. + if (error?.status === 429) + rateLimitedUntil = Math.max( + rateLimitedUntil, Date.now() + - Math.min( - terrainRetryAfterMs(error.retryAfter) ?? RATE_LIMIT_PAUSE_MS, - RATE_LIMIT_PAUSE_MAX_MS, - ); + Math.min( + terrainRetryAfterMs(error.retryAfter) ?? RATE_LIMIT_PAUSE_MS, + RATE_LIMIT_PAUSE_MAX_MS, + ), + ); } } // Keep later chunks aligned even if a malformed upstream response omits @@ -322,7 +328,7 @@ export function terrainHeightsProxy(options = {}) { // Logged once, when the pause began. } else if (outcome.upstreamError) { const pause = - outcome.upstreamError.message === 'HTTP 429' && + outcome.upstreamError.status === 429 && Date.now() < rateLimitedUntil ? ` (HTTP 429; pausing upstream for ${Math.ceil((rateLimitedUntil - Date.now()) / 1000)} s)` : ''; diff --git a/src/data/terrainHeightsProxy.js b/src/data/terrainHeightsProxy.js index c83b30a87..562d4d7c1 100644 --- a/src/data/terrainHeightsProxy.js +++ b/src/data/terrainHeightsProxy.js @@ -145,6 +145,7 @@ export async function fetchTerrainChunkWithRetry( const res = await fetchImpl(url, signal ? { signal } : {}); if (!res.ok) { const error = new Error(`HTTP ${res.status}`); + error.status = res.status; error.retryable = res.status === 429 || res.status >= 500; error.retryAfter = res.headers?.get?.('retry-after') ?? null; throw error; diff --git a/src/tooling/environmentProviders.test.mjs b/src/tooling/environmentProviders.test.mjs index 52dcd8351..e541a8069 100644 --- a/src/tooling/environmentProviders.test.mjs +++ b/src/tooling/environmentProviders.test.mjs @@ -421,8 +421,18 @@ test('after a 429 the terrain proxy leaves the upstream alone and logs once', as calls += 1; return new Response('', { status: 429 }); }); + t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 }); const request = install(terrainHeightsProxy()); - await request('/?points=1,2'); + // The retries' backoff runs on mocked time. + let settled = false; + const first = request('/?points=1,2').then(() => { + settled = true; + }); + for (let step = 0; step < 60 && !settled; step += 1) { + await new Promise((resolve) => setImmediate(resolve)); + t.mock.timers.tick(1_000); + } + await first; const afterFirst = calls; assert.ok(afterFirst >= 1); assert.equal(warnings.length, 1);