Merge pull request #1402 from quanticsoul4772/upstream/terrain-429-pause

fix(terrain): pause upstream calls after a 429

changelog.d/1402-terrain-429-pause.md

@@ -0,0 +1,5 @@
+- The terrain proxy stops asking the upstream for a while after a 429
+  (its Retry-After, at most 10 minutes, or 60 s) instead of retrying on every
+  page request, and a Retry-After longer than its 10 s retry budget now ends
+  the retries at once instead of sleeping the budget out (quanticsoul4772,
+  #1402).

docs/CURRENT-STATE.md

@@ -3087,7 +3087,14 @@ down | auth-failed` (plus the unchanged `missing-key`/`unsupported`) with
 >   usable per point, while an uncached absent height still returns 502 rather
 >   than becoming a fabricated ground value. A position the upstream answers
 >   with a null ellipsoid is reported as an absent height rather than a refresh
->   failure, and is left uncached so a later poll re-asks it. Client geoid
+>   failure, and is left uncached so a later poll re-asks it. A `Retry-After`
+>   longer than the 10 s retry budget ends the retries at once. After a 429 that
+>   outlasts its retries, the proxy pauses upstream calls for the `Retry-After`
+>   (60 seconds when none, at most 10 minutes). A request whose points are all
+>   cached, fresh or stale, is still served; a request with any uncached point
+>   gets 502 and the page uses its geoid fallback for that chunk. Each request
+>   that was waiting on the upstream when the 429 came back logs one warning;
+>   requests refused during the pause do not log. Client geoid
 >   fallbacks wait 60 seconds before retrying and self-heal to Re:Earth on the
 >   first later successful fetch. Smaller chunks reduce timeout risk; complete
 >   camera batches still wait for their sequential requests, and unresolved

server/providers/terrain.js

@@ -6,6 +6,7 @@ import {
   parseTerrainPoints,
   resolveTerrainHeightRequest,
   terrainPointKey,
+  terrainRetryAfterMs,
   validTerrainResult,
 } from '../../src/data/terrainHeightsProxy.js';
 import { makeRateLimiter, clientKey } from './common/rate-limit.js';
@@ -55,6 +56,15 @@ export function terrainHeightsProxy(options = {}) {
   // 2026-08-21). 64 points costs 5.6-12 s, inside the timeout at both ends of
   // that range.
   const UPSTREAM_CHUNK = 64;
+  // After a 429 the upstream is left alone for its Retry-After (at most
+  // RATE_LIMIT_PAUSE_MAX_MS, so a far-off date can't stop terrain until a
+  // restart), or RATE_LIMIT_PAUSE_MS when it gives none. Every page poll for
+  // new points (about once a second while the camera moves) otherwise asks
+  // again with its own retries, which keeps the limit in force and logs each
+  // time.
+  const RATE_LIMIT_PAUSE_MS = 60_000;
+  const RATE_LIMIT_PAUSE_MAX_MS = 10 * 60_000;
+  let rateLimitedUntil = 0;
   const MAX_POINTS = UPSTREAM_CHUNK;
   /**
    * Entry ceiling for the point cache, and with it the file written from it.
@@ -199,7 +209,8 @@ export function terrainHeightsProxy(options = {}) {
    *
    * The first try keeps its empirically required 30s timeout; network errors,
    * 429, and 5xx receive up to three jittered retries sharing a 10s added-time
-   * budget, with Retry-After honored within that bound.
+   * budget, with Retry-After honored within that bound; a longer Retry-After
+   * ends the retries at once so the 429 pause below starts without delay.
    *
    * A chunk that still fails yields nulls for its own positions rather than
    * rejecting: with several chunks per request, throwing would discard every
@@ -211,15 +222,37 @@ export function terrainHeightsProxy(options = {}) {
    * @returns {Promise<Array<object>>}
    */
   async function fetchUpstreamAll(points) {
+    if (Date.now() < rateLimitedUntil) {
+      // Rate-limited: asking again only extends the limit. Cached points are
+      // still served; the rest resolve on the page through its geoid.
+      const error = new Error('HTTP 429');
+      error.status = 429;
+      error.coolingDown = true;
+      throw error;
+    }
     const results = [];
     let firstError = null;
     for (let i = 0; i < points.length; i += UPSTREAM_CHUNK) {
       const chunk = points.slice(i, i + UPSTREAM_CHUNK);
       let chunkResults = [];
-      try {
+      // Re-checked per chunk for when a request spans several chunks again;
-        chunkResults = await fetchTerrainChunkWithRetry(chunk);
+      // today requests are capped at one chunk (MAX_POINTS).
-      } catch (error) {
+      if (Date.now() >= rateLimitedUntil) {
-        firstError = firstError || error;
+        try {
+          chunkResults = await fetchTerrainChunkWithRetry(chunk);
+        } catch (error) {
+          firstError = firstError || error;
+          // Concurrent 429s extend the pause, never cut it short.
+          if (error?.status === 429)
+            rateLimitedUntil = Math.max(
+              rateLimitedUntil,
+              Date.now() +
+                Math.min(
+                  terrainRetryAfterMs(error.retryAfter) ?? RATE_LIMIT_PAUSE_MS,
+                  RATE_LIMIT_PAUSE_MAX_MS,
+                ),
+            );
+        }
       }
       // Keep later chunks aligned even if a malformed upstream response omits
       // trailing positions. The resolver will reject each null individually.
@@ -292,9 +325,17 @@ export function terrainHeightsProxy(options = {}) {
           evictOverflow();
           diskDirty = true;
         }
-        if (outcome.upstreamError) {
+        if (outcome.upstreamError?.coolingDown) {
+          // Logged once, when the pause began.
+        } else if (outcome.upstreamError) {
+          const pause =
+            outcome.upstreamError.status === 429 &&
+            Date.now() < rateLimitedUntil
+              ? ` (HTTP 429; pausing upstream for ${Math.ceil((rateLimitedUntil - Date.now()) / 1000)} s)`
+              : '';
           console.warn(
             '[terrain-heights-proxy] refresh incomplete' +
+              pause +
               ' — serving stale points when available',
           );
         } else if (outcome.absentPoints > 0) {

src/data/terrainHeightsProxy.js

@@ -107,7 +107,10 @@ export function terrainRetryAfterMs(value, nowMs = Date.now()) {
 /**
  * Fetch one upstream chunk with jittered exponential backoff. Network errors,
  * 429, and 5xx retry; other HTTP failures and malformed successes fail fast.
- * Retry sleeps and retry attempts share a 10s budget after attempt one.
+ * Retry sleeps and retry attempts share a 10s budget after attempt one. A
+ * Retry-After longer than the budget left ends the retries at once: sleeping
+ * out the budget cannot reach the time the upstream asked for, and it would
+ * delay the caller's own back-off (the proxy's 429 pause) by up to 10 s.
  *
  * Dependencies are injectable for deterministic offline tests.
  * @param {Array<[number, number]>} points
@@ -145,6 +148,7 @@ export async function fetchTerrainChunkWithRetry(
       const res = await fetchImpl(url, signal ? { signal } : {});
       if (!res.ok) {
         const error = new Error(`HTTP ${res.status}`);
+        error.status = res.status;
         error.retryable = res.status === 429 || res.status >= 500;
         error.retryAfter = res.headers?.get?.('retry-after') ?? null;
         throw error;
@@ -167,6 +171,7 @@ export async function fetchTerrainChunkWithRetry(
       if (remaining <= 0) break;
 
       const retryAfterMs = terrainRetryAfterMs(error?.retryAfter, now());
+      if (retryAfterMs != null && retryAfterMs > remaining) break;
       const backoffMs = Math.round(
         350 * 2 ** attempt * (0.75 + random() * 0.5),
       );

src/data/terrainHeightsProxy.test.mjs

@@ -91,6 +91,36 @@ test('429 retry honors Retry-After before the next attempt', async () => {
   assert.equal(results[0].id, 'recovered');
 });
 
+test('a Retry-After longer than the retry budget ends the retries without sleeping', async () => {
+  let clock = 3_000_000;
+  let attempts = 0;
+  const sleeps = [];
+  await assert.rejects(
+    fetchTerrainChunkWithRetry([[1, 1]], {
+      now: () => clock,
+      random: () => 0.5,
+      makeSignal: () => undefined,
+      sleep: async (ms) => {
+        sleeps.push(ms);
+        clock += ms;
+      },
+      fetchImpl: async () => {
+        attempts += 1;
+        return {
+          ok: false,
+          status: 429,
+          headers: {
+            get: (name) => (name.toLowerCase() === 'retry-after' ? '30' : null),
+          },
+        };
+      },
+    }),
+    (error) => error.status === 429 && error.retryAfter === '30',
+  );
+  assert.equal(attempts, 1);
+  assert.deepEqual(sleeps, [], 'no sleep that cannot reach the Retry-After');
+});
+
 test('permanently failing upstream exhausts retries and returns 502 without fake results', async () => {
   let clock = 2_000_000;
   let attempts = 0;

src/tooling/environmentProviders.test.mjs

@@ -411,3 +411,156 @@ test('terrain middleware retains successful chunks around a failure and retries
   );
   assert.equal(requests.length, 4);
 });
+
+test('after a 429 the terrain proxy leaves the upstream alone and logs once', async (t) => {
+  isolate(t);
+  const warnings = [];
+  t.mock.method(console, 'warn', (...args) => warnings.push(args.join(' ')));
+  let calls = 0;
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    return new Response('', { status: 429 });
+  });
+  t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 });
+  const request = install(terrainHeightsProxy());
+  // The retries' backoff runs on mocked time.
+  let settled = false;
+  const first = request('/?points=1,2').then(() => {
+    settled = true;
+  });
+  for (let step = 0; step < 60 && !settled; step += 1) {
+    await new Promise((resolve) => setImmediate(resolve));
+    t.mock.timers.tick(1_000);
+  }
+  assert.ok(settled, 'the first request settles within its retry budget');
+  await first;
+  const afterFirst = calls;
+  assert.ok(afterFirst >= 1);
+  assert.equal(warnings.length, 1);
+  assert.match(warnings[0], /HTTP 429; pausing upstream for 60 s/);
+  // New points during the pause: no upstream call, no new log line.
+  const paused = await request('/?points=3,4');
+  assert.equal(calls, afterFirst);
+  assert.equal(paused.status, 502);
+  assert.equal(warnings.length, 1);
+});
+
+test('a far-off Retry-After pauses the terrain upstream for at most 10 minutes', async (t) => {
+  isolate(t);
+  t.mock.method(console, 'warn', () => {});
+  t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 });
+  let limited = true;
+  t.mock.method(globalThis, 'fetch', async () =>
+    limited
+      ? new Response('', { status: 429, headers: { 'retry-after': '86400' } })
+      : Response.json({ results: [{ ellipsoid: 100 }] }),
+  );
+  const request = install(terrainHeightsProxy());
+  // The first request spends its retry budget on 429s, then the pause begins.
+  let settled = false;
+  const first = request('/?points=1,2').then((res) => {
+    settled = true;
+    return res;
+  });
+  // Bounded: the retry budget is 10 s, so 60 simulated seconds is plenty.
+  for (let step = 0; step < 60 && !settled; step += 1) {
+    await new Promise((resolve) => setImmediate(resolve));
+    t.mock.timers.tick(1_000);
+  }
+  assert.ok(settled, 'the first request settles within its retry budget');
+  assert.equal((await first).status, 502);
+  limited = false;
+  const pausedAt = Date.now();
+  t.mock.timers.setTime(pausedAt + 9 * 60_000);
+  assert.equal((await request('/?points=3,4')).status, 502, 'still paused');
+  t.mock.timers.setTime(pausedAt + 10 * 60_000 + 1);
+  assert.equal((await request('/?points=3,4')).status, 200, 'pause over');
+});
+
+/** Drive mocked timers until `promise` settles; the retry budget is 10 s. */
+async function settleOnMockedTime(t, promise) {
+  let settled = false;
+  const tracked = promise.finally(() => {
+    settled = true;
+  });
+  for (let step = 0; step < 60 && !settled; step += 1) {
+    await new Promise((resolve) => setImmediate(resolve));
+    t.mock.timers.tick(1_000);
+  }
+  assert.ok(settled, 'the request settles within its retry budget');
+  return tracked;
+}
+
+test('during a terrain 429 pause fully cached points, even stale ones, are still served', async (t) => {
+  isolate(t);
+  t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 });
+  let calls = 0;
+  let limited = false;
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    return limited
+      ? new Response('', { status: 429, headers: { 'retry-after': '300' } })
+      : Response.json({ results: [{ ellipsoid: 100 }] });
+  });
+  const request = install(terrainHeightsProxy());
+  assert.equal((await request('/?points=5,6')).status, 200);
+  // Past the 30-day TTL, so 5,6 is stale and would normally be refreshed.
+  t.mock.timers.setTime(Date.now() + 31 * 24 * 3600_000);
+  limited = true;
+  const first = await settleOnMockedTime(t, request('/?points=1,2'));
+  assert.equal(first.status, 502);
+  assert.equal(calls, 2, 'a Retry-After past the budget is not retried');
+  const stale = await request('/?points=5,6');
+  assert.equal(stale.status, 200);
+  assert.deepEqual(json(stale).results, [{ ellipsoid: 100 }]);
+  assert.equal((await request('/?points=5,6;7,8')).status, 502);
+  assert.equal(calls, 2, 'no upstream call during the pause');
+});
+
+test('a terrain upstream failure other than 429 does not pause later requests', async (t) => {
+  isolate(t);
+  t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 });
+  let calls = 0;
+  t.mock.method(globalThis, 'fetch', async () => {
+    calls += 1;
+    return new Response('', { status: 503 });
+  });
+  const request = install(terrainHeightsProxy());
+  const first = await settleOnMockedTime(t, request('/?points=1,2'));
+  assert.equal(first.status, 502);
+  const afterFirst = calls;
+  const second = await settleOnMockedTime(t, request('/?points=3,4'));
+  assert.equal(second.status, 502);
+  assert.ok(calls > afterFirst, 'new points still go upstream after a 503');
+});
+
+test('a shorter concurrent Retry-After never cuts a terrain 429 pause short', async (t) => {
+  isolate(t);
+  t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: 1_000_000 });
+  let calls = 0;
+  t.mock.method(globalThis, 'fetch', async (raw) => {
+    calls += 1;
+    const points = new URL(raw).searchParams.get('points');
+    if (points.startsWith('0.'))
+      return Response.json({ results: [{ ellipsoid: 1 }] });
+    const retryAfter = points.startsWith('1.') ? '300' : '30';
+    return new Response('', {
+      status: 429,
+      headers: { 'retry-after': retryAfter },
+    });
+  });
+  const request = install(terrainHeightsProxy());
+  // Load the disk cache first so both requests below reach the upstream in
+  // order: the 300 s answer lands first, the 30 s answer after it.
+  assert.equal((await request('/?points=0,0')).status, 200);
+  const [long, short] = await settleOnMockedTime(
+    t,
+    Promise.all([request('/?points=1,2'), request('/?points=3,4')]),
+  );
+  assert.equal(long.status, 502);
+  assert.equal(short.status, 502);
+  const afterBoth = calls;
+  t.mock.timers.setTime(Date.now() + 120_000);
+  assert.equal((await request('/?points=5,6')).status, 502);
+  assert.equal(calls, afterBoth, 'still paused by the 300 s Retry-After');
+});