fix(osm): stop using public Overpass by default (#648) (#742)

Traffic roads come from OpenFreeMap vector tiles, or TomTom flow tiles with a
key (Hybrid by default). Military areas use OpenFreeMap polygons with a bundled
worldwide name index; wide views show named points. ALPR reads an hourly
OpenStreetMap extract. Cockpit place context and country, state and US county
outlines use bundled Natural Earth and US Census data. Overpass is used only
when an operator configures their own endpoint.

Also: camera arrivals no longer land below ground, keyless terrain retries on
429, bounded relation geometry for configured Overpass, one shared
OpenStreetMap credit, and separate tile and API transports for embedders.

Co-authored-by: theGaryLarson <[email protected]>
Co-authored-by: Milan Khanal <[email protected]>

src/maps/sourceFactories.test.mjs

@@ -24,9 +24,15 @@ test('Esri uses Re:Earth without keys and preserves ion terrain when configured'
     assert.equal(esri.terrain.id, 'keyless');
     assert.equal(esri.terrain.create, createKeylessTerrain);
     await esri.terrain.create();
-    assert.deepEqual(calls, [
+    // A Resource, not a bare URL: the retry policy rides on it (terrainRetry.js).
+    assert.equal(calls.length, 1);
+    assert.ok(calls[0] instanceof Cesium.Resource);
+    assert.equal(
+      calls[0].url,
       'https://terrain.reearth.land/cesium-mesh/ellipsoid',
-    ]);
+    );
+    assert.equal(calls[0].retryAttempts, 3);
+    assert.equal(typeof calls[0].retryCallback, 'function');
     assert.equal(
       keyless.sources.find(({ descriptor }) => descriptor.id === 'photoreal')
         .available,

src/maps/terrain.js

@@ -1,4 +1,9 @@
 import * as Cesium from 'cesium';
+import { createTerrainRetryPolicy } from './terrainRetry.js';
+
+/** Re:Earth / Mapterhorn ellipsoidal quantized mesh, CC BY 4.0. */
+export const KEYLESS_TERRAIN_URL =
+  'https://terrain.reearth.land/cesium-mesh/ellipsoid';
 
 /** These factories are lazy: a hidden globe must not trigger terrain loading. */
 export async function createWorldTerrain(accessToken, { signal } = {}) {
@@ -17,12 +22,36 @@ export async function createWorldTerrain(accessToken, { signal } = {}) {
   };
 }
 
+function logThrottle({ statusCode, delayMs, inWindow }) {
+  // One line per cooldown window, not per tile: a burst throttles dozens.
+  if (inWindow !== 1) return;
+  console.warn(
+    `[MapStack] Re:Earth terrain answered ${statusCode}; retrying throttled tiles after ${Math.round(delayMs)} ms`,
+  );
+}
+
+/**
+ * The Re:Earth resource with the tile retry policy attached. Cesium copies
+ * `retryCallback`/`retryAttempts` onto every derived resource, so layer.json
+ * and each `{z}/{x}/{y}.terrain` fetch inherit it (see `terrainRetry.js`).
+ * @param {ReturnType<typeof createTerrainRetryPolicy>} [policy]
+ * @returns {Cesium.Resource}
+ */
+export function createKeylessTerrainResource(
+  policy = createTerrainRetryPolicy({ onThrottle: logThrottle }),
+) {
+  return new Cesium.Resource({
+    url: KEYLESS_TERRAIN_URL,
+    retryCallback: policy.retryCallback,
+    retryAttempts: policy.retryAttempts,
+  });
+}
+
 export async function createKeylessTerrain() {
   try {
-    // Re:Earth / Mapterhorn ellipsoidal quantized mesh, CC BY 4.0.
     return {
       provider: await Cesium.CesiumTerrainProvider.fromUrl(
-        'https://terrain.reearth.land/cesium-mesh/ellipsoid',
+        createKeylessTerrainResource(),
       ),
     };
   } catch (error) {

src/maps/terrainRetry.js

@@ -0,0 +1,154 @@
+/**
+ * Retry policy for keyless terrain tiles.
+ *
+ * Re:Earth serves its quantized-mesh tiles straight to the browser — nothing
+ * in this repository proxies them — and its edge answers a zoom-out burst
+ * with HTTP 429 for a few dozen tiles at once. Cesium marks each throttled
+ * tile FAILED and upsamples the parent mesh, so the terrain stays coarse until
+ * the quadtree happens to ask for that tile again.
+ *
+ * Cesium's `Resource` carries a retry contract (`retryCallback` +
+ * `retryAttempts`) that `getDerivedResource` copies onto every tile fetch, and
+ * the callback may return a promise: the request is re-issued only once it
+ * resolves `true`. That is the hook this policy uses to insert the wait.
+ *
+ * Policy: throttled (429) and transient gateway (502/503/504) replies retry up
+ * to three times with exponential backoff. Every retry waits behind ONE shared
+ * cooldown per policy, so a burst of throttled tiles reissues as a jittered
+ * trickle after the window instead of as a second burst that gets throttled
+ * again. `Retry-After` extends the cooldown when the upstream exposes it.
+ * Anything else (404 outside coverage, network errors, aborts) is left to
+ * Cesium's own handling so the flat-terrain fallback stays prompt. A tile
+ * whose request was cancelled while it waited is not re-requested, and one
+ * attempt waits at most the maximum backoff plus the jitter spread.
+ */
+
+export const TERRAIN_RETRY_ATTEMPTS = 3;
+export const TERRAIN_RETRY_STATUSES = Object.freeze([429, 502, 503, 504]);
+export const TERRAIN_RETRY_BASE_MS = 750;
+export const TERRAIN_RETRY_MAX_MS = 15_000;
+export const TERRAIN_RETRY_SPREAD_MS = 1_500;
+
+const defaultWait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+/** Cesium `RequestState.CANCELLED`, kept local so this policy stays Cesium-free. */
+const REQUEST_CANCELLED = 4;
+
+/** A tile the quadtree no longer wants must not be re-requested. */
+function requestCancelled(resource) {
+  const request = resource?.request;
+  return request?.cancelled === true || request?.state === REQUEST_CANCELLED;
+}
+
+/**
+ * Parses a `Retry-After` header (delay-seconds or HTTP-date) into a wait in
+ * milliseconds relative to `nowMs`. Returns null for absent or unparseable
+ * values; a date already in the past yields 0.
+ * @param {string|number|null|undefined} value
+ * @param {number} nowMs
+ * @returns {number|null}
+ */
+export function parseRetryAfter(value, nowMs) {
+  if (value == null) return null;
+  const text = String(value).trim();
+  if (!text) return null;
+  if (/^\d+$/.test(text)) return Number(text) * 1000;
+  const at = Date.parse(text);
+  if (!Number.isFinite(at)) return null;
+  return Math.max(0, at - nowMs);
+}
+
+function headerValue(headers, name) {
+  if (!headers || typeof headers !== 'object') return null;
+  const wanted = name.toLowerCase();
+  for (const [key, value] of Object.entries(headers)) {
+    if (key.toLowerCase() === wanted) return value;
+  }
+  return null;
+}
+
+/**
+ * Builds the retry policy handed to a Cesium `Resource`.
+ *
+ * @param {object} [options]
+ * @param {() => number} [options.now] clock (ms); injectable for tests
+ * @param {(ms: number) => Promise<void>} [options.wait] sleeper; injectable
+ * @param {() => number} [options.random] jitter source in [0, 1)
+ * @param {number} [options.attempts] retries per tile before giving up
+ * @param {(info: {statusCode: number, attempt: number, delayMs: number,
+ *   inWindow: number}) => void} [options.onThrottle] observer, called once per
+ *   throttled reply; `inWindow` is 1 for the reply that opened a cooldown
+ * @returns {{
+ *   retryCallback: (resource: object, error: object) => Promise<boolean>,
+ *   retryAttempts: number,
+ *   shouldRetry: (error: object) => boolean,
+ *   cooldownUntil: () => number,
+ * }}
+ */
+export function createTerrainRetryPolicy({
+  now = () => Date.now(),
+  wait = defaultWait,
+  random = Math.random,
+  attempts = TERRAIN_RETRY_ATTEMPTS,
+  onThrottle = null,
+} = {}) {
+  /** @type {WeakMap<object, number>} attempts already spent per tile resource. */
+  const attemptsByResource = new WeakMap();
+  let cooldownUntil = 0;
+  let inWindow = 0;
+
+  const shouldRetry = (error) =>
+    TERRAIN_RETRY_STATUSES.includes(error?.statusCode);
+
+  /**
+   * Cesium `Resource.RetryCallback`: resolves `true` once the tile may be
+   * re-requested, `false` to let the failure stand.
+   */
+  async function retryCallback(resource, error) {
+    if (!shouldRetry(error) || requestCancelled(resource)) return false;
+    const attempt = attemptsByResource.get(resource) ?? 0;
+    attemptsByResource.set(resource, attempt + 1);
+    const started = now();
+    const backoff = Math.min(
+      TERRAIN_RETRY_MAX_MS,
+      TERRAIN_RETRY_BASE_MS * 2 ** attempt,
+    );
+    const retryAfter = parseRetryAfter(
+      headerValue(error?.responseHeaders, 'retry-after'),
+      started,
+    );
+    const delay = Math.min(
+      TERRAIN_RETRY_MAX_MS,
+      Math.max(backoff, retryAfter ?? 0),
+    );
+    if (cooldownUntil <= started) inWindow = 0;
+    inWindow += 1;
+    if (started + delay > cooldownUntil) cooldownUntil = started + delay;
+    onThrottle?.({
+      statusCode: error.statusCode,
+      attempt: attempt + 1,
+      delayMs: cooldownUntil - started,
+      inWindow,
+    });
+    const jitter = random() * TERRAIN_RETRY_SPREAD_MS;
+    // Re-check after each sleep: replies that arrive while this tile waits can
+    // push the shared window out, and the retry must stay behind it. One
+    // attempt never waits longer than the maximum backoff plus the spread,
+    // however far later replies push the window. The iteration cap only
+    // guards against a sleeper that cannot advance time.
+    const deadline = started + TERRAIN_RETRY_MAX_MS + TERRAIN_RETRY_SPREAD_MS;
+    for (let i = 0; i < 32; i += 1) {
+      const remaining = Math.min(cooldownUntil + jitter, deadline) - now();
+      if (remaining <= 0) break;
+      await wait(remaining);
+      if (requestCancelled(resource)) return false;
+    }
+    return !requestCancelled(resource);
+  }
+
+  return {
+    retryCallback,
+    retryAttempts: attempts,
+    shouldRetry,
+    cooldownUntil: () => cooldownUntil,
+  };
+}

src/maps/terrainRetry.test.mjs

@@ -0,0 +1,277 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import * as Cesium from 'cesium';
+import {
+  TERRAIN_RETRY_ATTEMPTS,
+  TERRAIN_RETRY_BASE_MS,
+  TERRAIN_RETRY_MAX_MS,
+  TERRAIN_RETRY_SPREAD_MS,
+  createTerrainRetryPolicy,
+  parseRetryAfter,
+} from './terrainRetry.js';
+import {
+  KEYLESS_TERRAIN_URL,
+  createKeylessTerrainResource,
+} from './terrain.js';
+
+const START = 1_700_000_000_000;
+
+/** Deterministic clock: `wait` records each sleep and advances time by it. */
+const fakeClock = () => {
+  let now = START;
+  const waits = [];
+  return {
+    now: () => now,
+    wait: async (ms) => {
+      waits.push(ms);
+      now += ms;
+    },
+    waits,
+    elapsed: () => now - START,
+  };
+};
+
+const throttled = (headers) => ({ statusCode: 429, responseHeaders: headers });
+
+test('parseRetryAfter reads delay-seconds and HTTP dates, ignores garbage', () => {
+  assert.equal(parseRetryAfter(undefined, START), null);
+  assert.equal(parseRetryAfter('', START), null);
+  assert.equal(parseRetryAfter('soon', START), null);
+  assert.equal(parseRetryAfter('2', START), 2000);
+  assert.equal(parseRetryAfter(' 7 ', START), 7000);
+  assert.equal(
+    parseRetryAfter(new Date(START + 4000).toUTCString(), START),
+    4000,
+  );
+  assert.equal(parseRetryAfter(new Date(START - 4000).toUTCString(), START), 0);
+});
+
+test('only throttled and gateway replies retry; everything else fails fast', async () => {
+  const clock = fakeClock();
+  const policy = createTerrainRetryPolicy({ ...clock, random: () => 0 });
+  for (const statusCode of [400, 403, 404, 500, undefined]) {
+    assert.equal(await policy.retryCallback({}, { statusCode }), false);
+  }
+  assert.equal(await policy.retryCallback({}, undefined), false);
+  assert.deepEqual(clock.waits, []);
+  for (const statusCode of [429, 502, 503, 504]) {
+    assert.equal(await policy.retryCallback({}, { statusCode }), true);
+  }
+  assert.equal(policy.retryAttempts, TERRAIN_RETRY_ATTEMPTS);
+});
+
+test('a throttled tile waits out an escalating backoff before Cesium re-requests it', async () => {
+  const clock = fakeClock();
+  const policy = createTerrainRetryPolicy({ ...clock, random: () => 0 });
+  const tile = {};
+  const expected = [
+    TERRAIN_RETRY_BASE_MS,
+    TERRAIN_RETRY_BASE_MS * 2,
+    TERRAIN_RETRY_BASE_MS * 4,
+  ];
+  for (const delay of expected) {
+    const before = clock.elapsed();
+    assert.equal(await policy.retryCallback(tile, throttled()), true);
+    assert.equal(clock.elapsed() - before, delay);
+  }
+  // Another tile starts its own backoff ladder from the first rung.
+  const before = clock.elapsed();
+  assert.equal(await policy.retryCallback({}, throttled()), true);
+  assert.equal(clock.elapsed() - before, TERRAIN_RETRY_BASE_MS);
+});
+
+test('Retry-After extends the wait, case-insensitively and capped', async () => {
+  const clock = fakeClock();
+  const policy = createTerrainRetryPolicy({ ...clock, random: () => 0 });
+  let before = clock.elapsed();
+  await policy.retryCallback({}, throttled({ 'Retry-After': '3' }));
+  assert.equal(clock.elapsed() - before, 3000);
+  before = clock.elapsed();
+  await policy.retryCallback(
+    {},
+    throttled({ 'retry-after': new Date(clock.now() + 5000).toUTCString() }),
+  );
+  assert.equal(clock.elapsed() - before, 5000);
+  before = clock.elapsed();
+  await policy.retryCallback({}, throttled({ 'retry-after': '9999' }));
+  assert.equal(clock.elapsed() - before, TERRAIN_RETRY_MAX_MS);
+  // A Retry-After shorter than the backoff never shortens the wait.
+  before = clock.elapsed();
+  await policy.retryCallback({}, throttled({ 'retry-after': '0' }));
+  assert.equal(clock.elapsed() - before, TERRAIN_RETRY_BASE_MS);
+});
+
+test('a burst shares one cooldown and trickles out with jitter instead of re-bursting', async () => {
+  let now = START;
+  /** @type {Array<{at: number, resolve: () => void}>} */
+  const sleepers = [];
+  const wait = (ms) =>
+    new Promise((resolve) => sleepers.push({ at: now + ms, resolve }));
+  const release = async () => {
+    // Advance to the earliest sleeper, wake everything due, let it settle.
+    sleepers.sort((a, b) => a.at - b.at);
+    now = Math.max(now, sleepers[0].at);
+    while (sleepers.length && sleepers[0].at <= now) sleepers.shift().resolve();
+    for (let i = 0; i < 4; i += 1) await Promise.resolve();
+  };
+  const jitters = [0, 0.5, 1 - Number.EPSILON];
+  let draw = 0;
+  const events = [];
+  const policy = createTerrainRetryPolicy({
+    now: () => now,
+    wait,
+    random: () => jitters[draw++ % jitters.length],
+    onThrottle: (info) => events.push(info),
+  });
+
+  const releasedAt = [];
+  const pending = Array.from({ length: 30 }, () =>
+    policy.retryCallback({}, throttled()).then((ok) => {
+      releasedAt.push(now);
+      return ok;
+    }),
+  );
+  await Promise.resolve();
+  // One window opened at the burst, not thirty stacked ones.
+  assert.equal(policy.cooldownUntil(), START + TERRAIN_RETRY_BASE_MS);
+  assert.deepEqual(
+    events.map((event) => event.inWindow),
+    Array.from({ length: 30 }, (_, i) => i + 1),
+  );
+  assert.ok(events.every((event) => event.delayMs === TERRAIN_RETRY_BASE_MS));
+
+  // A straggler throttled mid-window pushes the window out for everyone.
+  now = START + 400;
+  const straggler = policy.retryCallback({}, throttled());
+  await Promise.resolve();
+  assert.equal(policy.cooldownUntil(), START + 400 + TERRAIN_RETRY_BASE_MS);
+
+  while (sleepers.length) await release();
+  assert.ok((await Promise.all([...pending, straggler])).every(Boolean));
+  const earliest = Math.min(...releasedAt);
+  const latest = Math.max(...releasedAt);
+  assert.ok(
+    earliest >= START + 400 + TERRAIN_RETRY_BASE_MS,
+    'nothing retried inside the window',
+  );
+  assert.ok(
+    latest <= START + 400 + TERRAIN_RETRY_BASE_MS + TERRAIN_RETRY_SPREAD_MS,
+    'retries spread across the jitter window',
+  );
+  assert.ok(
+    latest - earliest >= TERRAIN_RETRY_SPREAD_MS / 2,
+    'retries do not re-burst',
+  );
+});
+
+test('Cesium copies the policy onto every derived tile resource and honours its verdicts', async () => {
+  const clock = fakeClock();
+  const calls = [];
+  const policy = createTerrainRetryPolicy({
+    ...clock,
+    random: () => 0,
+    onThrottle: (info) => calls.push(info.attempt),
+  });
+  const resource = createKeylessTerrainResource(policy);
+  assert.equal(resource.url, KEYLESS_TERRAIN_URL);
+  resource.appendForwardSlash(); // what CesiumTerrainProvider.fromUrl does
+  const tile = resource.getDerivedResource({ url: '4/7/13.terrain' });
+  assert.equal(tile.url, `${KEYLESS_TERRAIN_URL}/4/7/13.terrain`);
+  assert.equal(tile.retryCallback, policy.retryCallback);
+  assert.equal(tile.retryAttempts, TERRAIN_RETRY_ATTEMPTS);
+
+  // Cesium charges every consulted failure against the budget, declined ones
+  // included, so the 404 probe uses its own tile.
+  const missing = resource.getDerivedResource({ url: '4/7/14.terrain' });
+  assert.equal(
+    await missing.retryOnError(new Cesium.RequestErrorEvent(404)),
+    false,
+  );
+  assert.deepEqual(clock.waits, []);
+  const headers = 'content-type: text/html\r\nretry-after: 2\r\n';
+  for (let attempt = 1; attempt <= TERRAIN_RETRY_ATTEMPTS; attempt += 1) {
+    const before = clock.elapsed();
+    assert.equal(
+      await tile.retryOnError(
+        new Cesium.RequestErrorEvent(429, undefined, headers),
+      ),
+      true,
+    );
+    assert.equal(
+      clock.elapsed() - before,
+      Math.max(2000, TERRAIN_RETRY_BASE_MS * 2 ** (attempt - 1)),
+    );
+  }
+  // The attempt budget is Cesium's: a fourth throttle fails without consulting the policy.
+  assert.equal(
+    await tile.retryOnError(new Cesium.RequestErrorEvent(429)),
+    false,
+  );
+  assert.deepEqual(calls, [1, 2, 3]);
+  // Budgets are per tile: a sibling starts fresh.
+  const sibling = resource.getDerivedResource({ url: '4/8/13.terrain' });
+  assert.equal(
+    await sibling.retryOnError(new Cesium.RequestErrorEvent(503)),
+    true,
+  );
+});
+
+test('the default resource carries the policy without any injected clock', () => {
+  const resource = createKeylessTerrainResource();
+  assert.equal(resource.retryAttempts, TERRAIN_RETRY_ATTEMPTS);
+  assert.equal(typeof resource.retryCallback, 'function');
+});
+
+/** A sleeper the test resolves by hand, so later replies can land mid-wait. */
+const manualClock = () => {
+  const clock = { now: START, sleeps: [] };
+  clock.wait = (ms) =>
+    new Promise((resolve) => clock.sleeps.push({ ms, resolve }));
+  return clock;
+};
+const flush = () => new Promise((resolve) => setImmediate(resolve));
+
+test('one attempt never waits past the maximum backoff plus spread', async () => {
+  const clock = manualClock();
+  const policy = createTerrainRetryPolicy({
+    now: () => clock.now,
+    wait: clock.wait,
+    random: () => 0,
+  });
+  let verdict;
+  policy.retryCallback({}, throttled()).then((value) => (verdict = value));
+  let waited = 0;
+  for (let i = 0; i < 10 && verdict === undefined; i += 1) {
+    const sleep = clock.sleeps.at(-1);
+    clock.now += sleep.ms;
+    waited += sleep.ms;
+    // A fresh 15 s throttle lands while this tile sleeps and pushes the
+    // shared window out again; its own sleep is never resolved here.
+    policy.retryCallback({}, throttled({ 'Retry-After': '15' }));
+    sleep.resolve();
+    await flush();
+  }
+  assert.equal(verdict, true);
+  assert.equal(waited, TERRAIN_RETRY_MAX_MS + TERRAIN_RETRY_SPREAD_MS);
+  assert.ok(policy.cooldownUntil() - START > waited, 'window moved further');
+});
+
+test('a tile cancelled before or during its wait is not re-requested', async () => {
+  assert.equal(Cesium.RequestState.CANCELLED, 4);
+  const clock = manualClock();
+  const policy = createTerrainRetryPolicy({
+    now: () => clock.now,
+    wait: clock.wait,
+    random: () => 0,
+  });
+  const gone = { request: { state: Cesium.RequestState.CANCELLED } };
+  assert.equal(await policy.retryCallback(gone, throttled()), false);
+  assert.equal(clock.sleeps.length, 0, 'no wait for a cancelled tile');
+
+  const request = new Cesium.Request({ url: KEYLESS_TERRAIN_URL });
+  const pending = policy.retryCallback({ request }, throttled());
+  request.cancel();
+  clock.now += clock.sleeps.at(-1).ms;
+  clock.sleeps.at(-1).resolve();
+  assert.equal(await pending, false);
+});