Merge pull request #1111 from Rohilalala/fix/map-fallback-retry

fix(maps): retry the real imagery and terrain provider after a fallback

changelog.d/1111-map-fallback-retry.md

@@ -0,0 +1,5 @@
+- A failed Esri Satellite or keyless terrain load still shows OSM or flat terrain
+  at once, but no longer lasts the whole session: while the fallback is on
+  screen, the real provider is retried in the background (30 seconds, doubling
+  to a 10-minute cap) and replaces it live once it loads. Map switches never
+  wait for a retry (Aditya, @Rohilalala, #1111).

docs/CURRENT-STATE.md

@@ -384,7 +384,12 @@ doubling to a 15 s cap, with every retry waiting behind one shared cooldown
 plus up to 1.5 s of jitter, and `Retry-After` extending the cooldown when the
 upstream exposes it. One console warning is logged per cooldown window. Other
 failures keep Cesium's handling and the flat `EllipsoidTerrainProvider`
-fallback. `scripts/qa-terrain-429.mjs` reproduces the burst against a keyless
+fallback; while that flat terrain is on a shown globe the map controller
+retries Re:Earth in the background (30 s, doubling to a 10-minute cap, each
+attempt abandoned after 60 s) and installs the real terrain without waiting
+for a map switch or an imagery retry. Behind a 3D tileset it neither retries
+nor installs; the next globe activation does.
+`scripts/qa-terrain-429.mjs` reproduces the burst against a keyless
 dev server, counts real throttles with their headers as evidence, synthesises
 throttles with `--inject N` when the upstream is not throttling, and fails if
 a throttled tile never loads.
@@ -4787,7 +4792,7 @@ are omitted rather than framing the wrong part of the globe.
 - The bottom Visual Presets tray presents a **five-tile MAP SOURCE row** (`#map-stack-chips`, `src/mapStackChips.js`): Google 3D, Esri Satellite, Bing Aerial, Bing Labels, and OSM. The duplicate left `#stack-panel` is retired. The five tiles share one row on desktop and two rows on narrow screens, carry `aria-pressed` on the active source, and remain keyboard-reachable with a visible focus outline.
 - The lit tile follows controller state, not the click: a rejected switch (no ion token) or a superseded one (rapid A→B) leaves the genuinely active source lit, and the tray heading keeps its short-label status readout (`...` while switching, amber on `lastError`).
 - Ion stacks remain visible and keyboard-focusable when no ion token is configured, but expose `aria-disabled="true"` and do not switch. Their accessible label and tooltip quote `getStacks().unavailableReason` — the same string `setStack()` puts in the toast. OSM works keyless. The `ION` badge is gated on the stack's own `requiresIon`, so a `photoreal` chip unavailable because the Google tileset failed says so instead of falsely demanding an ion token.
-- Stack choice participates in share links (`src/sharelink.js`) and falls back to the best available stack when the requested one is unavailable (keyless boots land on Esri; OSM takes over automatically if Esri is unreachable). Share-link restore, the `set_map_stack` voice tool, and the chip row all land on the same `_setMapStack()` path.
+- Stack choice participates in share links (`src/sharelink.js`) and falls back to the best available stack when the requested one is unavailable (keyless boots land on Esri; OSM takes over automatically if Esri is unreachable; while that construction fallback is on screen Esri is retried in the background, 30 s doubling to a 10-minute cap with each attempt abandoned after 60 s, and replaces OSM once it loads. The switch to OSM after repeated Esri tile failures is not retried; selecting Esri again tries it). Share-link restore, the `set_map_stack` voice tool, and the chip row all land on the same `_setMapStack()` path.
 
 ### Voice Map Whiteboard / Annotations (June 2026)
 

src/maps/controller.js

@@ -9,6 +9,18 @@ import { acquireImageryComparison } from './imageryComparison.js';
  */
 const AUTOMATIC = Symbol('automatic switch');
 
+/** Background retries of a fallback in use back off from 30 s to 10 min. */
+const FALLBACK_RETRY_MS = 30_000;
+const FALLBACK_RETRY_MAX_MS = 600_000;
+/** A retry attempt still unsettled after a minute counts as failed. */
+const FALLBACK_ATTEMPT_MS = 60_000;
+
+const resolved = (provider, effectiveStackId) => ({
+  provider,
+  effectiveStackId,
+  fallbackMessage: null,
+});
+
 /** Coordinate source lifetimes and scene changes; the registry owns provider choices. */
 export class MapSourceController {
   constructor(
@@ -46,8 +58,18 @@ export class MapSourceController {
     this._lastError = null;
     this._imageryLayer = null;
     this._activeImageryProvider = null;
+    this._activeResolution = null;
     this._removeImageryErrorListener = null;
     this._terrainMode = null;
+    this._terrainFallback = null;
+    this._retryTimer = null;
+    this._retrying = false;
+    this._retryRequested = false;
+    this._retryDelay = FALLBACK_RETRY_MS;
+    /** Expire callbacks of retry attempts in flight; destroy() runs them. */
+    this._retryDeadlines = new Set();
+    /** A fallback terrain a retry replaced while it stayed installed. */
+    this._replacedTerrain = null;
     this._subscribers = new Set();
     this._destroyed = false;
   }
@@ -285,21 +307,171 @@ export class MapSourceController {
     this._showTileset(null);
     this.viewer.scene.globe.show = true;
     this._activeId = resolution.effectiveStackId;
+    this._activeResolution = resolution;
     if (source?.terrain && source.terrain.id !== this._terrainMode) {
       const terrain = source.terrain;
-      const result = await this._cached(
-        this._terrainProviders,
-        terrain.id,
-        () => terrain.create({ signal: this._abort.signal }),
+      const pending = this._cached(this._terrainProviders, terrain.id, () =>
+        terrain.create({ signal: this._abort.signal }),
       );
+      let result = await pending;
       if (gen !== this._switchGen) return;
-      if (result.terrain) this.viewer.scene.setTerrain(result.terrain);
-      else this.viewer.terrainProvider = result.provider;
+      // A background retry may have replaced a fallback while this waited.
+      const latest = this._terrainProviders.get(terrain.id);
+      if (latest !== pending) result = await latest;
+      if (gen !== this._switchGen) return;
+      this._applyTerrain(result);
       this._terrainMode = terrain.id;
     }
+    if (resolution.fallbackFor || this._terrainFallback) this._scheduleRetry();
     return resolution;
   }
 
+  _applyTerrain(result) {
+    if (result.terrain) this.viewer.scene.setTerrain(result.terrain);
+    else this.viewer.terrainProvider = result.provider;
+    this._terrainFallback = result.fallback ? result : null;
+    if (this._replacedTerrain && this._replacedTerrain !== result) {
+      this._dispose(this._replacedTerrain.provider);
+      this._replacedTerrain = null;
+    }
+  }
+
+  /**
+   * One timer retries the real provider behind whichever fallback is on
+   * screen: Esri's OSM stand-in, or flat terrain on a shown globe. Fallbacks
+   * stay cached and shown meanwhile, so no switch waits on a retry; a
+   * fallback out of use is left alone until an activation shows it again.
+   * One retry runs at a time; a request made meanwhile re-arms once it ends.
+   */
+  _scheduleRetry() {
+    if (this._retryTimer || this._destroyed) return;
+    if (this._retrying) {
+      this._retryRequested = true;
+      return;
+    }
+    this._retryTimer = setTimeout(() => {
+      this._retryTimer = null;
+      void this._retryFallbacks();
+    }, this._retryDelay);
+  }
+
+  async _retryFallbacks() {
+    const resolution = this._activeResolution;
+    const flat = this.viewer.scene.globe.show ? this._terrainFallback : null;
+    const id = resolution?.fallbackFor;
+    let terrain = flat && this._sources.get(this._activeId)?.terrain;
+    if (terrain?.id !== this._terrainMode) terrain = null;
+    if (!id && !terrain) return;
+    this._retrying = true;
+    this._retryRequested = false;
+    // An attempt that outlives its deadline counts as a failure. Its signal is
+    // aborted, but the Esri and keyless factories ignore it and Cesium's
+    // requests have no timeout, so a late result is released, never used.
+    const attempt = async (create, release) => {
+      const expiry = new AbortController();
+      let expire;
+      const deadline = new Promise((resolve) => {
+        expire = () => {
+          clearTimeout(timer);
+          resolve(expiry);
+        };
+      });
+      const timer = setTimeout(expire, FALLBACK_ATTEMPT_MS);
+      this._retryDeadlines.add(expire);
+      const pending = Promise.resolve().then(() =>
+        create?.({
+          signal: AbortSignal.any([this._abort.signal, expiry.signal]),
+        }),
+      );
+      try {
+        const result = await Promise.race([pending, deadline]);
+        if (result !== expiry) return result;
+        expiry.abort();
+        pending.then(release, () => {});
+        return null;
+      } catch {
+        return null; // Still unavailable: keep the fallback.
+      } finally {
+        clearTimeout(timer);
+        this._retryDeadlines.delete(expire);
+      }
+    };
+    // Each recovers as soon as it loads, never waiting on the other.
+    const retryImagery = async () => {
+      const cached = this._imageryProviders.get(id);
+      const provider = await attempt(this._sources.get(id).imagery, (late) =>
+        this._dispose(late),
+      );
+      // Obsolete: destroyed, or the cache moved on while this was in flight.
+      if (this._destroyed || this._imageryProviders.get(id) !== cached) {
+        this._dispose(provider);
+        return true;
+      }
+      if (!provider) return false;
+      this._imageryProviders.set(id, Promise.resolve(resolved(provider, id)));
+      // Still showing the fallback once any switch settles: use the real one.
+      const recover = () => {
+        if (this._activeResolution === resolution)
+          void this.setStack(id, { [AUTOMATIC]: true });
+      };
+      if (!this._isSwitching) recover();
+      else {
+        const stop = this.subscribe(() => {
+          stop();
+          recover();
+        });
+      }
+      return true;
+    };
+    const retryTerrain = async () => {
+      const cached = this._terrainProviders.get(terrain.id);
+      const real = await attempt(terrain.create, (late) =>
+        this._dispose(late?.provider),
+      );
+      if (
+        this._destroyed ||
+        this._terrainProviders.get(terrain.id) !== cached ||
+        !real ||
+        real.fallback
+      ) {
+        this._dispose(real?.provider);
+        return Boolean(real && !real.fallback);
+      }
+      this._terrainProviders.set(terrain.id, Promise.resolve(real));
+      if (this._terrainFallback === flat && this.viewer.scene.globe.show) {
+        this._applyTerrain(real);
+        this._dispose(flat.provider);
+        this._requestRender('map-stack');
+      } else {
+        this._terrainMode = null; // The next globe activation installs it.
+        // Release the replaced fallback once nothing shows it.
+        if (this._terrainFallback === flat) this._replacedTerrain = flat;
+        else this._dispose(flat.provider);
+      }
+      return true;
+    };
+    const settled = (retry) =>
+      retry().catch((error) => {
+        console.warn('[MapSourceController] fallback recovery failed:', error);
+        return false;
+      });
+    let failed = false;
+    try {
+      const results = await Promise.all([
+        id && settled(retryImagery),
+        terrain && settled(retryTerrain),
+      ]);
+      failed = results.includes(false);
+    } finally {
+      this._retrying = false;
+    }
+    if (this._destroyed) return;
+    this._retryDelay = failed
+      ? Math.min(this._retryDelay * 2, FALLBACK_RETRY_MAX_MS)
+      : FALLBACK_RETRY_MS;
+    if (failed || this._retryRequested) this._scheduleRetry();
+  }
+
   _cached(cache, id, create) {
     if (cache.has(id)) return cache.get(id);
     const promise = Promise.resolve()
@@ -323,7 +495,7 @@ export class MapSourceController {
       try {
         const provider = await source.imagery({ signal: this._abort.signal });
         if (this._destroyed) this._dispose(provider);
-        return { provider, effectiveStackId: stack.id, fallbackMessage: null };
+        return resolved(provider, stack.id);
       } catch (error) {
         const fallback = source.constructionFallback;
         if (this._destroyed || !fallback || !this.isStackAvailable(fallback.id))
@@ -333,7 +505,11 @@ export class MapSourceController {
           this.getStack(fallback.id),
           next,
         );
-        return { ...resolution, fallbackMessage: fallback.message };
+        return {
+          ...resolution,
+          fallbackMessage: fallback.message,
+          fallbackFor: stack.id,
+        };
       }
     });
   }
@@ -388,6 +564,7 @@ export class MapSourceController {
       this.viewer.imageryLayers.remove(this._imageryLayer, true);
     this._imageryLayer = null;
     this._activeImageryProvider = null;
+    this._activeResolution = null;
   }
   _dispose(value) {
     if (!value || this._disposed.has(value)) return;
@@ -401,6 +578,12 @@ export class MapSourceController {
     this._subscribers.clear();
     this._switchGen++;
     this._abort.abort();
+    clearTimeout(this._retryTimer);
+    for (const expire of this._retryDeadlines) expire();
+    this._retryDeadlines.clear();
+    this._retrying = false;
+    this._dispose(this._replacedTerrain?.provider);
+    this._replacedTerrain = null;
     this._isSwitching = false;
     this._removeImageryLayer();
     this._credits.destroy();

src/maps/controller.test.mjs

@@ -70,7 +70,7 @@ const settle = async () => {
   for (let i = 0; i < 20; i++) await Promise.resolve();
 };
 const descriptor = (id) => ({ id, label: id, kind: 'imagery' });
-function publicFixture() {
+function publicFixture(options) {
   const tileset = { show: true };
   const registry = createDefaultMapSources({ googleTileset: tileset });
   const providers = new Map();
@@ -84,7 +84,7 @@ function publicFixture() {
       create: async () => ({ provider: { id: 'terrain' } }),
     };
   }
-  return { ...fixture(registry), registry, providers, tileset };
+  return { ...fixture(registry, options), registry, providers, tileset };
 }
 
 test('an additional imagery source needs no controller branch and owns its cached resources', async () => {
@@ -839,3 +839,546 @@ test('each switch generation reports its origin: an outside setStack is manual,
   ]);
   recovering.controller.destroy();
 });
+
+function failingEsri(env) {
+  const esri = { failing: true, attempts: 0, next: null };
+  env.registry.sources.find(
+    (source) => source.descriptor.id === 'esri-imagery',
+  ).imagery = async () => {
+    esri.attempts++;
+    if (esri.next) await esri.next;
+    if (esri.failing) throw new Error('Esri unreachable');
+    return env.providers.get('esri-imagery');
+  };
+  return esri;
+}
+const deferred = () => {
+  let resolve;
+  const promise = new Promise((done) => (resolve = done));
+  return { promise, resolve };
+};
+
+test('a failed Esri construction shows OSM at once, backs off while retries fail, and recovers in place', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  const env = publicFixture();
+  const esri = failingEsri(env);
+  await env.controller.setStack('esri-imagery');
+  assert.equal(env.controller.getActiveId(), 'osm');
+  const osmLayer = env.imagery.at(-1);
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(esri.attempts, 2, 'one background retry');
+  assert.equal(env.imagery.at(-1), osmLayer, 'a failed retry keeps the layer');
+  await env.controller.setStack('osm');
+  await env.controller.setStack('esri-imagery');
+  assert.equal(esri.attempts, 2, 'a reselection does not retry at once');
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(esri.attempts, 2, 'the second retry waits twice as long');
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(esri.attempts, 3);
+  esri.failing = false;
+  t.mock.timers.tick(120_000);
+  await settle();
+  assert.equal(esri.attempts, 4);
+  assert.equal(env.controller.getActiveId(), 'esri-imagery');
+  assert.equal(env.controller.getSwitchOrigin(), 'automatic');
+  assert.equal(env.controller.getState().lastError, null);
+  assert.equal(env.imagery.at(-1).provider, env.providers.get('esri-imagery'));
+  t.mock.timers.tick(600_000);
+  await settle();
+  assert.equal(esri.attempts, 4, 'the real provider ends the retries');
+  env.controller.destroy();
+});
+
+test('retries of a fallback that keeps failing back off to a 10 minute cap', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  const env = publicFixture();
+  const esri = failingEsri(env);
+  await env.controller.setStack('esri-imagery');
+  for (const delay of [30, 60, 120, 240, 480, 600, 600]) {
+    t.mock.timers.tick(delay * 1000);
+    await settle();
+  }
+  assert.equal(esri.attempts, 8);
+  env.controller.destroy();
+});
+
+test('Esri is retried only while its fallback is shown', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  const env = publicFixture();
+  const esri = failingEsri(env);
+  await env.controller.setStack('esri-imagery');
+  await env.controller.setStack('osm');
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(esri.attempts, 1, 'no retry while OSM was chosen');
+  await env.controller.setStack('esri-imagery');
+  assert.equal(esri.attempts, 1, 'the cached fallback shows at once');
+  assert.equal(env.controller.getActiveId(), 'osm');
+  esri.failing = false;
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(esri.attempts, 2);
+  assert.equal(env.controller.getActiveId(), 'esri-imagery');
+  env.controller.destroy();
+});
+
+test('an Esri recovery that lands mid-switch applies once the switch settles', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  const env = publicFixture();
+  const esri = failingEsri(env);
+  await env.controller.setStack('esri-imagery');
+  const retry = deferred();
+  esri.next = retry.promise;
+  esri.failing = false;
+  t.mock.timers.tick(30_000);
+  await settle();
+  // The fallback activates OSM's terrain; make that switch slow.
+  const slow = deferred();
+  env.registry.sources.find(
+    (source) => source.descriptor.id === 'osm',
+  ).terrain = { id: 'slow', create: () => slow.promise };
+  const switching = env.controller.setStack('esri-imagery');
+  retry.resolve();
+  await settle();
+  assert.equal(env.controller.getActiveId(), 'osm', 'the switch is still open');
+  slow.resolve({ provider: { id: 'slow' } });
+  await switching;
+  await settle();
+  assert.equal(env.controller.getActiveId(), 'esri-imagery');
+  assert.equal(env.imagery.at(-1).provider, env.providers.get('esri-imagery'));
+  env.controller.destroy();
+});
+
+function terrainFixture(t, create) {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  let renders = 0;
+  const env = publicFixture({ requestRender: () => renders++ });
+  const installed = [];
+  Object.defineProperty(env.viewer, 'terrainProvider', {
+    get: () => installed.at(-1),
+    set: (provider) => installed.push(provider),
+  });
+  const terrain = { id: 'keyless', calls: 0 };
+  terrain.create = () => create(++terrain.calls);
+  for (const source of env.registry.sources)
+    if (source.imagery) source.terrain = terrain;
+  return { env, installed, terrain, renders: () => renders };
+}
+const flatTerrain = (n) => ({ provider: { id: `flat-${n}` }, fallback: true });
+
+test('flat fallback terrain stays installed while retries run, and real terrain replaces it live', async (t) => {
+  const real = { id: 'real' };
+  let working = false;
+  const { env, installed, terrain, renders } = terrainFixture(t, async (n) =>
+    working ? { provider: real } : flatTerrain(n),
+  );
+  await env.controller.setStack('esri-imagery');
+  const [flat] = installed;
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(terrain.calls, 2, 'one background retry');
+  await env.controller.setStack('osm');
+  assert.deepEqual(installed, [flat], 'failed retries and switches keep it');
+  working = true;
+  const before = renders();
+  t.mock.timers.tick(60_000);
+  await settle();
+  assert.equal(terrain.calls, 3);
+  assert.deepEqual(installed, [flat, real]);
+  assert.equal(renders(), before + 1, 'the live swap requests a render');
+  t.mock.timers.tick(600_000);
+  await settle();
+  assert.equal(terrain.calls, 3, 'the real terrain ends the retries');
+  env.controller.destroy();
+});
+
+test('terrain is not retried or installed while a 3D tileset hides the globe', async (t) => {
+  const real = { id: 'real' };
+  let working = false;
+  const { env, installed, terrain } = terrainFixture(t, async (n) =>
+    working ? { provider: real } : flatTerrain(n),
+  );
+  await env.controller.setStack('esri-imagery');
+  await env.controller.setStack('photoreal');
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(terrain.calls, 1, 'no retry behind a hidden globe');
+  await env.controller.setStack('esri-imagery');
+  const hidden = deferred();
+  terrain.create = async () => {
+    terrain.calls++;
+    await hidden.promise;
+    return { provider: real };
+  };
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(terrain.calls, 2, 'the globe activation re-arms the retry');
+  await env.controller.setStack('photoreal');
+  hidden.resolve();
+  await settle();
+  assert.equal(installed.length, 1, 'nothing is installed behind a tileset');
+  await env.controller.setStack('osm');
+  assert.equal(installed.at(-1), real, 'the next globe activation installs it');
+  assert.equal(terrain.calls, 2);
+  env.controller.destroy();
+});
+
+test('a switch that read the flat fallback installs the terrain a retry just cached', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  // Sweep the microtask distance between the retry landing and the switch.
+  for (let hops = 0; hops < 12; hops++) {
+    const real = { id: 'real' },
+      gate = deferred();
+    const keyless = {
+      id: 'keyless',
+      create: async () => {
+        if (!keyless.retried) return flatTerrain(1);
+        await gate.promise;
+        return { provider: real };
+      },
+    };
+    const other = { id: 'other', create: async () => ({ provider: {} }) };
+    const source = (id, terrain) => ({
+      descriptor: descriptor(id),
+      imagery: async () => ({ id }),
+      terrain,
+    });
+    const env = fixture({
+      defaultId: 'a',
+      sources: [source('a', keyless), source('c', other)],
+    });
+    const installed = [];
+    Object.defineProperty(env.viewer, 'terrainProvider', {
+      get: () => installed.at(-1),
+      set: (provider) => installed.push(provider),
+    });
+    await env.controller.setStack('a');
+    keyless.retried = true;
+    t.mock.timers.tick(30_000);
+    await settle();
+    await env.controller.setStack('c');
+    gate.resolve();
+    for (let i = 0; i < hops; i++) await null;
+    await env.controller.setStack('a');
+    await settle();
+    assert.equal(installed.at(-1), real, `hops ${hops}`);
+    env.controller.destroy();
+  }
+});
+
+test('destroy cancels a pending retry and releases one in flight', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  const env = publicFixture();
+  const esri = failingEsri(env);
+  await env.controller.setStack('esri-imagery');
+  env.controller.destroy();
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(esri.attempts, 1, 'a pending retry is cancelled');
+
+  const late = publicFixture();
+  const lateEsri = failingEsri(late);
+  await late.controller.setStack('esri-imagery');
+  const retry = deferred();
+  lateEsri.next = retry.promise;
+  lateEsri.failing = false;
+  let destroyed = 0;
+  late.providers.get('esri-imagery').destroy = () => destroyed++;
+  t.mock.timers.tick(30_000);
+  await settle();
+  late.controller.destroy();
+  retry.resolve();
+  await settle();
+  assert.equal(destroyed, 1, 'a late provider is released');
+  assert.equal(late.imagery.length, 0);
+  t.mock.timers.tick(600_000);
+  await settle();
+  assert.equal(lateEsri.attempts, 2);
+});
+
+test('retries stay single-flight when an activation re-shows the fallback mid-retry', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  const env = publicFixture();
+  const created = [],
+    pending = [];
+  let attempts = 0;
+  env.registry.sources.find(
+    (source) => source.descriptor.id === 'esri-imagery',
+  ).imagery = async () => {
+    if (++attempts === 1) throw new Error('Esri unreachable');
+    const gate = deferred();
+    pending.push(gate);
+    await gate.promise;
+    const provider = { id: `esri-${attempts}`, errorEvent: event() };
+    provider.destroy = () => (provider.destroyed = true);
+    created.push(provider);
+    return provider;
+  };
+  await env.controller.setStack('esri-imagery');
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(attempts, 2, 'retry #1 is in flight');
+  await env.controller.setStack('osm');
+  await env.controller.setStack('esri-imagery');
+  assert.equal(env.controller.getActiveId(), 'osm', 'the fallback is shown');
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(attempts, 2, 'no second retry while one is in flight');
+  for (const gate of pending.reverse()) {
+    gate.resolve();
+    await settle();
+  }
+  const shown = env.imagery.at(-1).provider;
+  assert.equal(env.controller.getActiveId(), 'esri-imagery');
+  await env.controller.setStack('osm');
+  await env.controller.setStack('esri-imagery');
+  assert.equal(env.imagery.at(-1).provider, shown, 'the cached one is shown');
+  env.controller.destroy();
+  await settle();
+  assert.ok(created.length > 0);
+  for (const provider of created)
+    assert.equal(provider.destroyed, true, `${provider.id} is disposed`);
+});
+
+test('a stalled terrain retry does not hold back recovered imagery', async (t) => {
+  const stalled = deferred();
+  const { env, terrain } = terrainFixture(t, async (n) => {
+    if (n > 1) await stalled.promise;
+    return flatTerrain(n);
+  });
+  const esri = failingEsri(env);
+  await env.controller.setStack('esri-imagery');
+  assert.equal(env.controller.getActiveId(), 'osm');
+  esri.failing = false;
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(terrain.calls, 2, 'the terrain retry is in flight');
+  assert.equal(env.controller.getActiveId(), 'esri-imagery');
+  assert.equal(env.imagery.at(-1).provider, env.providers.get('esri-imagery'));
+  stalled.resolve();
+  await settle();
+  env.controller.destroy();
+});
+
+test('a stalled imagery retry does not hold back recovered terrain', async (t) => {
+  const real = { id: 'real' };
+  const { env, installed } = terrainFixture(t, async (n) =>
+    n > 1 ? { provider: real } : flatTerrain(n),
+  );
+  const esri = failingEsri(env);
+  await env.controller.setStack('esri-imagery');
+  const stalled = deferred();
+  esri.next = stalled.promise;
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(esri.attempts, 2, 'the imagery retry is in flight');
+  assert.equal(installed.at(-1), real, 'real terrain is installed');
+  stalled.resolve();
+  await settle();
+  env.controller.destroy();
+});
+
+test('a retry attempt that never settles is abandoned at its deadline and released if it lands late', async (t) => {
+  const hung = deferred();
+  const late = { id: 'late', destroy: () => (late.destroyed = true) };
+  const { env, installed, terrain } = terrainFixture(t, async (n) => {
+    if (n === 2) await hung.promise;
+    return n === 2 ? { provider: late } : flatTerrain(n);
+  });
+  const esri = failingEsri(env);
+  await env.controller.setStack('esri-imagery');
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(esri.attempts, 2, 'imagery failed');
+  assert.equal(terrain.calls, 2, 'terrain hangs');
+  t.mock.timers.tick(60_000); // The hung attempt's deadline.
+  await settle();
+  t.mock.timers.tick(60_000); // The doubled backoff.
+  await settle();
+  assert.equal(esri.attempts, 3, 'imagery is retried again');
+  assert.equal(terrain.calls, 3, 'terrain is retried again');
+  hung.resolve();
+  await settle();
+  assert.equal(late.destroyed, true, 'the late provider is released');
+  assert.ok(!installed.includes(late), 'and never installed');
+  env.controller.destroy();
+});
+
+test('a failed retry after a mid-retry re-show re-arms once, after the doubled delay', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  const env = publicFixture();
+  const esri = failingEsri(env);
+  await env.controller.setStack('esri-imagery');
+  const gate = deferred();
+  esri.next = gate.promise;
+  t.mock.timers.tick(30_000);
+  await settle();
+  await env.controller.setStack('osm');
+  await env.controller.setStack('esri-imagery');
+  gate.resolve();
+  await settle();
+  assert.equal(esri.attempts, 2);
+  t.mock.timers.tick(59_999);
+  await settle();
+  assert.equal(esri.attempts, 2, 'not before the doubled delay');
+  t.mock.timers.tick(1);
+  await settle();
+  assert.equal(esri.attempts, 3, 'one retry after it');
+  t.mock.timers.tick(119_999);
+  await settle();
+  assert.equal(esri.attempts, 3, 'no second timer was armed');
+  env.controller.destroy();
+});
+
+test('destroy releases recovered imagery and terrain that lands afterwards', async (t) => {
+  const stalled = deferred();
+  const real = { id: 'real', destroy: () => (real.destroyed = true) };
+  const { env, installed } = terrainFixture(t, async (n) => {
+    if (n === 1) return flatTerrain(n);
+    await stalled.promise;
+    return { provider: real };
+  });
+  const esri = failingEsri(env);
+  const imagery = env.providers.get('esri-imagery');
+  imagery.destroy = () => (imagery.destroyed = true);
+  await env.controller.setStack('esri-imagery');
+  esri.failing = false;
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(env.imagery.at(-1).provider, imagery, 'imagery has landed');
+  env.controller.destroy();
+  stalled.resolve();
+  await settle();
+  assert.equal(imagery.destroyed, true);
+  assert.equal(real.destroyed, true);
+  assert.ok(!installed.includes(real));
+});
+
+function liveTimers(t) {
+  const live = new Set();
+  const { setTimeout: set, clearTimeout: clear } = globalThis;
+  globalThis.setTimeout = (fn, ms, ...args) => {
+    const id = set(
+      (...values) => {
+        live.delete(id);
+        fn(...values);
+      },
+      ms,
+      ...args,
+    );
+    live.add(id);
+    return id;
+  };
+  globalThis.clearTimeout = (id) => {
+    live.delete(id);
+    clear(id);
+  };
+  t.after(() =>
+    Object.assign(globalThis, { setTimeout: set, clearTimeout: clear }),
+  );
+  return live;
+}
+
+test('destroy during a hung retry attempt clears its deadline at once', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  const live = liveTimers(t);
+  const env = publicFixture();
+  const esri = failingEsri(env);
+  await env.controller.setStack('esri-imagery');
+  esri.next = new Promise(() => {});
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(esri.attempts, 2, 'the attempt hangs');
+  assert.equal(live.size, 1, 'only its deadline is armed');
+  env.controller.destroy();
+  assert.equal(live.size, 0, 'destroy clears the deadline');
+  assert.equal(env.controller._retrying, false);
+  await settle();
+  assert.equal(live.size, 0, 'and nothing re-arms');
+});
+
+function destroyableFlat(n) {
+  const provider = { id: `flat-${n}` };
+  provider.destroy = () => (provider.destroyed = true);
+  return { provider, fallback: true };
+}
+
+test('terrain recovered on screen releases the fallback it replaced', async (t) => {
+  const real = { id: 'real' };
+  let flat;
+  const { env, installed } = terrainFixture(t, async (n) =>
+    n === 1 ? (flat = destroyableFlat(n)) : { provider: real },
+  );
+  await env.controller.setStack('esri-imagery');
+  t.mock.timers.tick(30_000);
+  await settle();
+  assert.equal(installed.at(-1), real);
+  assert.equal(flat.provider.destroyed, true);
+  env.controller.destroy();
+});
+
+test('terrain recovered behind a tileset releases the fallback once a globe replaces it', async (t) => {
+  const real = { id: 'real' };
+  const gate = deferred();
+  let flat;
+  const { env, installed } = terrainFixture(t, async (n) => {
+    if (n === 1) return (flat = destroyableFlat(n));
+    await gate.promise;
+    return { provider: real };
+  });
+  await env.controller.setStack('esri-imagery');
+  t.mock.timers.tick(30_000);
+  await settle();
+  await env.controller.setStack('photoreal');
+  gate.resolve();
+  await settle();
+  assert.equal(installed.at(-1), flat.provider);
+  assert.ok(!flat.provider.destroyed, 'still installed behind the tileset');
+  await env.controller.setStack('osm');
+  assert.equal(installed.at(-1), real);
+  assert.equal(flat.provider.destroyed, true);
+  env.controller.destroy();
+});
+
+test('a hung imagery attempt that lands after its deadline is released, never cached or shown', async (t) => {
+  t.mock.timers.enable({ apis: ['setTimeout'] });
+  const env = publicFixture();
+  const hung = deferred();
+  const late = { id: 'late', errorEvent: event() };
+  const good = { id: 'good', errorEvent: event() };
+  for (const provider of [late, good])
+    provider.destroy = () => (provider.destroyed = true);
+  let attempts = 0;
+  env.registry.sources.find(
+    (source) => source.descriptor.id === 'esri-imagery',
+  ).imagery = async () => {
+    if (++attempts === 1) throw new Error('Esri unreachable');
+    if (attempts === 2) {
+      await hung.promise;
+      return late;
+    }
+    return good;
+  };
+  await env.controller.setStack('esri-imagery');
+  t.mock.timers.tick(30_000);
+  await settle();
+  t.mock.timers.tick(60_000); // The deadline.
+  await settle();
+  hung.resolve();
+  await settle();
+  assert.equal(late.destroyed, true, 'the late provider is released');
+  assert.equal(env.controller.getActiveId(), 'osm', 'and not shown');
+  t.mock.timers.tick(60_000);
+  await settle();
+  assert.equal(attempts, 3);
+  assert.equal(env.imagery.at(-1).provider, good, 'the next retry installs');
+  await env.controller.setStack('osm');
+  await env.controller.setStack('esri-imagery');
+  assert.equal(env.imagery.at(-1).provider, good, 'only its own is cached');
+  env.controller.destroy();
+  await settle();
+  assert.equal(good.destroyed, true);
+});

src/maps/sourceFactories.test.mjs

@@ -317,3 +317,13 @@ test('both Google 3D routes keep tiles drawing while draped imagery loads', asyn
   assert.equal(calls[1][1].asynchronouslyLoadImagery, true);
   assert.equal(calls[1][1].enableCollision, true);
 });
+
+test('unreachable keyless terrain falls back to flat terrain marked as a fallback', async (t) => {
+  t.mock.method(Cesium.CesiumTerrainProvider, 'fromUrl', async () => {
+    throw new Error('layer.json unreachable');
+  });
+  t.mock.method(console, 'warn', () => {});
+  const result = await createKeylessTerrain();
+  assert.ok(result.provider instanceof Cesium.EllipsoidTerrainProvider);
+  assert.equal(result.fallback, true);
+});

src/maps/terrain.js

@@ -59,6 +59,6 @@ export async function createKeylessTerrain() {
       '[MapStack] Re:Earth terrain unavailable, falling back to flat ellipsoid terrain:',
       error,
     );
-    return { provider: new Cesium.EllipsoidTerrainProvider() };
+    return { provider: new Cesium.EllipsoidTerrainProvider(), fallback: true };
   }
 }