diff --git a/changelog.d/1111-map-fallback-retry.md b/changelog.d/1111-map-fallback-retry.md new file mode 100644 index 000000000..45086eab1 --- /dev/null +++ b/changelog.d/1111-map-fallback-retry.md @@ -0,0 +1,5 @@ +- A failed Esri Satellite or keyless terrain load still shows OSM or flat terrain + at once, but no longer lasts the whole session: while the fallback is on + screen, the real provider is retried in the background (30 seconds, doubling + to a 10-minute cap) and replaces it live once it loads. Map switches never + wait for a retry (Aditya, @Rohilalala, #1111). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 41fe806ea..8cb7b6d17 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -384,7 +384,12 @@ doubling to a 15 s cap, with every retry waiting behind one shared cooldown plus up to 1.5 s of jitter, and `Retry-After` extending the cooldown when the upstream exposes it. One console warning is logged per cooldown window. Other failures keep Cesium's handling and the flat `EllipsoidTerrainProvider` -fallback. `scripts/qa-terrain-429.mjs` reproduces the burst against a keyless +fallback; while that flat terrain is on a shown globe the map controller +retries Re:Earth in the background (30 s, doubling to a 10-minute cap, each +attempt abandoned after 60 s) and installs the real terrain without waiting +for a map switch or an imagery retry. Behind a 3D tileset it neither retries +nor installs; the next globe activation does. +`scripts/qa-terrain-429.mjs` reproduces the burst against a keyless dev server, counts real throttles with their headers as evidence, synthesises throttles with `--inject N` when the upstream is not throttling, and fails if a throttled tile never loads. @@ -4787,7 +4792,7 @@ are omitted rather than framing the wrong part of the globe. - The bottom Visual Presets tray presents a **five-tile MAP SOURCE row** (`#map-stack-chips`, `src/mapStackChips.js`): Google 3D, Esri Satellite, Bing Aerial, Bing Labels, and OSM. The duplicate left `#stack-panel` is retired. The five tiles share one row on desktop and two rows on narrow screens, carry `aria-pressed` on the active source, and remain keyboard-reachable with a visible focus outline. - The lit tile follows controller state, not the click: a rejected switch (no ion token) or a superseded one (rapid A→B) leaves the genuinely active source lit, and the tray heading keeps its short-label status readout (`...` while switching, amber on `lastError`). - Ion stacks remain visible and keyboard-focusable when no ion token is configured, but expose `aria-disabled="true"` and do not switch. Their accessible label and tooltip quote `getStacks().unavailableReason` — the same string `setStack()` puts in the toast. OSM works keyless. The `ION` badge is gated on the stack's own `requiresIon`, so a `photoreal` chip unavailable because the Google tileset failed says so instead of falsely demanding an ion token. -- Stack choice participates in share links (`src/sharelink.js`) and falls back to the best available stack when the requested one is unavailable (keyless boots land on Esri; OSM takes over automatically if Esri is unreachable). Share-link restore, the `set_map_stack` voice tool, and the chip row all land on the same `_setMapStack()` path. +- Stack choice participates in share links (`src/sharelink.js`) and falls back to the best available stack when the requested one is unavailable (keyless boots land on Esri; OSM takes over automatically if Esri is unreachable; while that construction fallback is on screen Esri is retried in the background, 30 s doubling to a 10-minute cap with each attempt abandoned after 60 s, and replaces OSM once it loads. The switch to OSM after repeated Esri tile failures is not retried; selecting Esri again tries it). Share-link restore, the `set_map_stack` voice tool, and the chip row all land on the same `_setMapStack()` path. ### Voice Map Whiteboard / Annotations (June 2026) diff --git a/src/maps/controller.js b/src/maps/controller.js index 2b2ea462f..bfdd82033 100644 --- a/src/maps/controller.js +++ b/src/maps/controller.js @@ -9,6 +9,18 @@ import { acquireImageryComparison } from './imageryComparison.js'; */ const AUTOMATIC = Symbol('automatic switch'); +/** Background retries of a fallback in use back off from 30 s to 10 min. */ +const FALLBACK_RETRY_MS = 30_000; +const FALLBACK_RETRY_MAX_MS = 600_000; +/** A retry attempt still unsettled after a minute counts as failed. */ +const FALLBACK_ATTEMPT_MS = 60_000; + +const resolved = (provider, effectiveStackId) => ({ + provider, + effectiveStackId, + fallbackMessage: null, +}); + /** Coordinate source lifetimes and scene changes; the registry owns provider choices. */ export class MapSourceController { constructor( @@ -46,8 +58,18 @@ export class MapSourceController { this._lastError = null; this._imageryLayer = null; this._activeImageryProvider = null; + this._activeResolution = null; this._removeImageryErrorListener = null; this._terrainMode = null; + this._terrainFallback = null; + this._retryTimer = null; + this._retrying = false; + this._retryRequested = false; + this._retryDelay = FALLBACK_RETRY_MS; + /** Expire callbacks of retry attempts in flight; destroy() runs them. */ + this._retryDeadlines = new Set(); + /** A fallback terrain a retry replaced while it stayed installed. */ + this._replacedTerrain = null; this._subscribers = new Set(); this._destroyed = false; } @@ -285,21 +307,171 @@ export class MapSourceController { this._showTileset(null); this.viewer.scene.globe.show = true; this._activeId = resolution.effectiveStackId; + this._activeResolution = resolution; if (source?.terrain && source.terrain.id !== this._terrainMode) { const terrain = source.terrain; - const result = await this._cached( - this._terrainProviders, - terrain.id, - () => terrain.create({ signal: this._abort.signal }), + const pending = this._cached(this._terrainProviders, terrain.id, () => + terrain.create({ signal: this._abort.signal }), ); + let result = await pending; if (gen !== this._switchGen) return; - if (result.terrain) this.viewer.scene.setTerrain(result.terrain); - else this.viewer.terrainProvider = result.provider; + // A background retry may have replaced a fallback while this waited. + const latest = this._terrainProviders.get(terrain.id); + if (latest !== pending) result = await latest; + if (gen !== this._switchGen) return; + this._applyTerrain(result); this._terrainMode = terrain.id; } + if (resolution.fallbackFor || this._terrainFallback) this._scheduleRetry(); return resolution; } + _applyTerrain(result) { + if (result.terrain) this.viewer.scene.setTerrain(result.terrain); + else this.viewer.terrainProvider = result.provider; + this._terrainFallback = result.fallback ? result : null; + if (this._replacedTerrain && this._replacedTerrain !== result) { + this._dispose(this._replacedTerrain.provider); + this._replacedTerrain = null; + } + } + + /** + * One timer retries the real provider behind whichever fallback is on + * screen: Esri's OSM stand-in, or flat terrain on a shown globe. Fallbacks + * stay cached and shown meanwhile, so no switch waits on a retry; a + * fallback out of use is left alone until an activation shows it again. + * One retry runs at a time; a request made meanwhile re-arms once it ends. + */ + _scheduleRetry() { + if (this._retryTimer || this._destroyed) return; + if (this._retrying) { + this._retryRequested = true; + return; + } + this._retryTimer = setTimeout(() => { + this._retryTimer = null; + void this._retryFallbacks(); + }, this._retryDelay); + } + + async _retryFallbacks() { + const resolution = this._activeResolution; + const flat = this.viewer.scene.globe.show ? this._terrainFallback : null; + const id = resolution?.fallbackFor; + let terrain = flat && this._sources.get(this._activeId)?.terrain; + if (terrain?.id !== this._terrainMode) terrain = null; + if (!id && !terrain) return; + this._retrying = true; + this._retryRequested = false; + // An attempt that outlives its deadline counts as a failure. Its signal is + // aborted, but the Esri and keyless factories ignore it and Cesium's + // requests have no timeout, so a late result is released, never used. + const attempt = async (create, release) => { + const expiry = new AbortController(); + let expire; + const deadline = new Promise((resolve) => { + expire = () => { + clearTimeout(timer); + resolve(expiry); + }; + }); + const timer = setTimeout(expire, FALLBACK_ATTEMPT_MS); + this._retryDeadlines.add(expire); + const pending = Promise.resolve().then(() => + create?.({ + signal: AbortSignal.any([this._abort.signal, expiry.signal]), + }), + ); + try { + const result = await Promise.race([pending, deadline]); + if (result !== expiry) return result; + expiry.abort(); + pending.then(release, () => {}); + return null; + } catch { + return null; // Still unavailable: keep the fallback. + } finally { + clearTimeout(timer); + this._retryDeadlines.delete(expire); + } + }; + // Each recovers as soon as it loads, never waiting on the other. + const retryImagery = async () => { + const cached = this._imageryProviders.get(id); + const provider = await attempt(this._sources.get(id).imagery, (late) => + this._dispose(late), + ); + // Obsolete: destroyed, or the cache moved on while this was in flight. + if (this._destroyed || this._imageryProviders.get(id) !== cached) { + this._dispose(provider); + return true; + } + if (!provider) return false; + this._imageryProviders.set(id, Promise.resolve(resolved(provider, id))); + // Still showing the fallback once any switch settles: use the real one. + const recover = () => { + if (this._activeResolution === resolution) + void this.setStack(id, { [AUTOMATIC]: true }); + }; + if (!this._isSwitching) recover(); + else { + const stop = this.subscribe(() => { + stop(); + recover(); + }); + } + return true; + }; + const retryTerrain = async () => { + const cached = this._terrainProviders.get(terrain.id); + const real = await attempt(terrain.create, (late) => + this._dispose(late?.provider), + ); + if ( + this._destroyed || + this._terrainProviders.get(terrain.id) !== cached || + !real || + real.fallback + ) { + this._dispose(real?.provider); + return Boolean(real && !real.fallback); + } + this._terrainProviders.set(terrain.id, Promise.resolve(real)); + if (this._terrainFallback === flat && this.viewer.scene.globe.show) { + this._applyTerrain(real); + this._dispose(flat.provider); + this._requestRender('map-stack'); + } else { + this._terrainMode = null; // The next globe activation installs it. + // Release the replaced fallback once nothing shows it. + if (this._terrainFallback === flat) this._replacedTerrain = flat; + else this._dispose(flat.provider); + } + return true; + }; + const settled = (retry) => + retry().catch((error) => { + console.warn('[MapSourceController] fallback recovery failed:', error); + return false; + }); + let failed = false; + try { + const results = await Promise.all([ + id && settled(retryImagery), + terrain && settled(retryTerrain), + ]); + failed = results.includes(false); + } finally { + this._retrying = false; + } + if (this._destroyed) return; + this._retryDelay = failed + ? Math.min(this._retryDelay * 2, FALLBACK_RETRY_MAX_MS) + : FALLBACK_RETRY_MS; + if (failed || this._retryRequested) this._scheduleRetry(); + } + _cached(cache, id, create) { if (cache.has(id)) return cache.get(id); const promise = Promise.resolve() @@ -323,7 +495,7 @@ export class MapSourceController { try { const provider = await source.imagery({ signal: this._abort.signal }); if (this._destroyed) this._dispose(provider); - return { provider, effectiveStackId: stack.id, fallbackMessage: null }; + return resolved(provider, stack.id); } catch (error) { const fallback = source.constructionFallback; if (this._destroyed || !fallback || !this.isStackAvailable(fallback.id)) @@ -333,7 +505,11 @@ export class MapSourceController { this.getStack(fallback.id), next, ); - return { ...resolution, fallbackMessage: fallback.message }; + return { + ...resolution, + fallbackMessage: fallback.message, + fallbackFor: stack.id, + }; } }); } @@ -388,6 +564,7 @@ export class MapSourceController { this.viewer.imageryLayers.remove(this._imageryLayer, true); this._imageryLayer = null; this._activeImageryProvider = null; + this._activeResolution = null; } _dispose(value) { if (!value || this._disposed.has(value)) return; @@ -401,6 +578,12 @@ export class MapSourceController { this._subscribers.clear(); this._switchGen++; this._abort.abort(); + clearTimeout(this._retryTimer); + for (const expire of this._retryDeadlines) expire(); + this._retryDeadlines.clear(); + this._retrying = false; + this._dispose(this._replacedTerrain?.provider); + this._replacedTerrain = null; this._isSwitching = false; this._removeImageryLayer(); this._credits.destroy(); diff --git a/src/maps/controller.test.mjs b/src/maps/controller.test.mjs index 1e6c1413f..f7e764754 100644 --- a/src/maps/controller.test.mjs +++ b/src/maps/controller.test.mjs @@ -70,7 +70,7 @@ const settle = async () => { for (let i = 0; i < 20; i++) await Promise.resolve(); }; const descriptor = (id) => ({ id, label: id, kind: 'imagery' }); -function publicFixture() { +function publicFixture(options) { const tileset = { show: true }; const registry = createDefaultMapSources({ googleTileset: tileset }); const providers = new Map(); @@ -84,7 +84,7 @@ function publicFixture() { create: async () => ({ provider: { id: 'terrain' } }), }; } - return { ...fixture(registry), registry, providers, tileset }; + return { ...fixture(registry, options), registry, providers, tileset }; } test('an additional imagery source needs no controller branch and owns its cached resources', async () => { @@ -839,3 +839,546 @@ test('each switch generation reports its origin: an outside setStack is manual, ]); recovering.controller.destroy(); }); + +function failingEsri(env) { + const esri = { failing: true, attempts: 0, next: null }; + env.registry.sources.find( + (source) => source.descriptor.id === 'esri-imagery', + ).imagery = async () => { + esri.attempts++; + if (esri.next) await esri.next; + if (esri.failing) throw new Error('Esri unreachable'); + return env.providers.get('esri-imagery'); + }; + return esri; +} +const deferred = () => { + let resolve; + const promise = new Promise((done) => (resolve = done)); + return { promise, resolve }; +}; + +test('a failed Esri construction shows OSM at once, backs off while retries fail, and recovers in place', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + assert.equal(env.controller.getActiveId(), 'osm'); + const osmLayer = env.imagery.at(-1); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 2, 'one background retry'); + assert.equal(env.imagery.at(-1), osmLayer, 'a failed retry keeps the layer'); + await env.controller.setStack('osm'); + await env.controller.setStack('esri-imagery'); + assert.equal(esri.attempts, 2, 'a reselection does not retry at once'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 2, 'the second retry waits twice as long'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 3); + esri.failing = false; + t.mock.timers.tick(120_000); + await settle(); + assert.equal(esri.attempts, 4); + assert.equal(env.controller.getActiveId(), 'esri-imagery'); + assert.equal(env.controller.getSwitchOrigin(), 'automatic'); + assert.equal(env.controller.getState().lastError, null); + assert.equal(env.imagery.at(-1).provider, env.providers.get('esri-imagery')); + t.mock.timers.tick(600_000); + await settle(); + assert.equal(esri.attempts, 4, 'the real provider ends the retries'); + env.controller.destroy(); +}); + +test('retries of a fallback that keeps failing back off to a 10 minute cap', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + for (const delay of [30, 60, 120, 240, 480, 600, 600]) { + t.mock.timers.tick(delay * 1000); + await settle(); + } + assert.equal(esri.attempts, 8); + env.controller.destroy(); +}); + +test('Esri is retried only while its fallback is shown', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + await env.controller.setStack('osm'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 1, 'no retry while OSM was chosen'); + await env.controller.setStack('esri-imagery'); + assert.equal(esri.attempts, 1, 'the cached fallback shows at once'); + assert.equal(env.controller.getActiveId(), 'osm'); + esri.failing = false; + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 2); + assert.equal(env.controller.getActiveId(), 'esri-imagery'); + env.controller.destroy(); +}); + +test('an Esri recovery that lands mid-switch applies once the switch settles', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + const retry = deferred(); + esri.next = retry.promise; + esri.failing = false; + t.mock.timers.tick(30_000); + await settle(); + // The fallback activates OSM's terrain; make that switch slow. + const slow = deferred(); + env.registry.sources.find( + (source) => source.descriptor.id === 'osm', + ).terrain = { id: 'slow', create: () => slow.promise }; + const switching = env.controller.setStack('esri-imagery'); + retry.resolve(); + await settle(); + assert.equal(env.controller.getActiveId(), 'osm', 'the switch is still open'); + slow.resolve({ provider: { id: 'slow' } }); + await switching; + await settle(); + assert.equal(env.controller.getActiveId(), 'esri-imagery'); + assert.equal(env.imagery.at(-1).provider, env.providers.get('esri-imagery')); + env.controller.destroy(); +}); + +function terrainFixture(t, create) { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let renders = 0; + const env = publicFixture({ requestRender: () => renders++ }); + const installed = []; + Object.defineProperty(env.viewer, 'terrainProvider', { + get: () => installed.at(-1), + set: (provider) => installed.push(provider), + }); + const terrain = { id: 'keyless', calls: 0 }; + terrain.create = () => create(++terrain.calls); + for (const source of env.registry.sources) + if (source.imagery) source.terrain = terrain; + return { env, installed, terrain, renders: () => renders }; +} +const flatTerrain = (n) => ({ provider: { id: `flat-${n}` }, fallback: true }); + +test('flat fallback terrain stays installed while retries run, and real terrain replaces it live', async (t) => { + const real = { id: 'real' }; + let working = false; + const { env, installed, terrain, renders } = terrainFixture(t, async (n) => + working ? { provider: real } : flatTerrain(n), + ); + await env.controller.setStack('esri-imagery'); + const [flat] = installed; + t.mock.timers.tick(30_000); + await settle(); + assert.equal(terrain.calls, 2, 'one background retry'); + await env.controller.setStack('osm'); + assert.deepEqual(installed, [flat], 'failed retries and switches keep it'); + working = true; + const before = renders(); + t.mock.timers.tick(60_000); + await settle(); + assert.equal(terrain.calls, 3); + assert.deepEqual(installed, [flat, real]); + assert.equal(renders(), before + 1, 'the live swap requests a render'); + t.mock.timers.tick(600_000); + await settle(); + assert.equal(terrain.calls, 3, 'the real terrain ends the retries'); + env.controller.destroy(); +}); + +test('terrain is not retried or installed while a 3D tileset hides the globe', async (t) => { + const real = { id: 'real' }; + let working = false; + const { env, installed, terrain } = terrainFixture(t, async (n) => + working ? { provider: real } : flatTerrain(n), + ); + await env.controller.setStack('esri-imagery'); + await env.controller.setStack('photoreal'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(terrain.calls, 1, 'no retry behind a hidden globe'); + await env.controller.setStack('esri-imagery'); + const hidden = deferred(); + terrain.create = async () => { + terrain.calls++; + await hidden.promise; + return { provider: real }; + }; + t.mock.timers.tick(30_000); + await settle(); + assert.equal(terrain.calls, 2, 'the globe activation re-arms the retry'); + await env.controller.setStack('photoreal'); + hidden.resolve(); + await settle(); + assert.equal(installed.length, 1, 'nothing is installed behind a tileset'); + await env.controller.setStack('osm'); + assert.equal(installed.at(-1), real, 'the next globe activation installs it'); + assert.equal(terrain.calls, 2); + env.controller.destroy(); +}); + +test('a switch that read the flat fallback installs the terrain a retry just cached', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + // Sweep the microtask distance between the retry landing and the switch. + for (let hops = 0; hops < 12; hops++) { + const real = { id: 'real' }, + gate = deferred(); + const keyless = { + id: 'keyless', + create: async () => { + if (!keyless.retried) return flatTerrain(1); + await gate.promise; + return { provider: real }; + }, + }; + const other = { id: 'other', create: async () => ({ provider: {} }) }; + const source = (id, terrain) => ({ + descriptor: descriptor(id), + imagery: async () => ({ id }), + terrain, + }); + const env = fixture({ + defaultId: 'a', + sources: [source('a', keyless), source('c', other)], + }); + const installed = []; + Object.defineProperty(env.viewer, 'terrainProvider', { + get: () => installed.at(-1), + set: (provider) => installed.push(provider), + }); + await env.controller.setStack('a'); + keyless.retried = true; + t.mock.timers.tick(30_000); + await settle(); + await env.controller.setStack('c'); + gate.resolve(); + for (let i = 0; i < hops; i++) await null; + await env.controller.setStack('a'); + await settle(); + assert.equal(installed.at(-1), real, `hops ${hops}`); + env.controller.destroy(); + } +}); + +test('destroy cancels a pending retry and releases one in flight', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + env.controller.destroy(); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 1, 'a pending retry is cancelled'); + + const late = publicFixture(); + const lateEsri = failingEsri(late); + await late.controller.setStack('esri-imagery'); + const retry = deferred(); + lateEsri.next = retry.promise; + lateEsri.failing = false; + let destroyed = 0; + late.providers.get('esri-imagery').destroy = () => destroyed++; + t.mock.timers.tick(30_000); + await settle(); + late.controller.destroy(); + retry.resolve(); + await settle(); + assert.equal(destroyed, 1, 'a late provider is released'); + assert.equal(late.imagery.length, 0); + t.mock.timers.tick(600_000); + await settle(); + assert.equal(lateEsri.attempts, 2); +}); + +test('retries stay single-flight when an activation re-shows the fallback mid-retry', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const created = [], + pending = []; + let attempts = 0; + env.registry.sources.find( + (source) => source.descriptor.id === 'esri-imagery', + ).imagery = async () => { + if (++attempts === 1) throw new Error('Esri unreachable'); + const gate = deferred(); + pending.push(gate); + await gate.promise; + const provider = { id: `esri-${attempts}`, errorEvent: event() }; + provider.destroy = () => (provider.destroyed = true); + created.push(provider); + return provider; + }; + await env.controller.setStack('esri-imagery'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(attempts, 2, 'retry #1 is in flight'); + await env.controller.setStack('osm'); + await env.controller.setStack('esri-imagery'); + assert.equal(env.controller.getActiveId(), 'osm', 'the fallback is shown'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(attempts, 2, 'no second retry while one is in flight'); + for (const gate of pending.reverse()) { + gate.resolve(); + await settle(); + } + const shown = env.imagery.at(-1).provider; + assert.equal(env.controller.getActiveId(), 'esri-imagery'); + await env.controller.setStack('osm'); + await env.controller.setStack('esri-imagery'); + assert.equal(env.imagery.at(-1).provider, shown, 'the cached one is shown'); + env.controller.destroy(); + await settle(); + assert.ok(created.length > 0); + for (const provider of created) + assert.equal(provider.destroyed, true, `${provider.id} is disposed`); +}); + +test('a stalled terrain retry does not hold back recovered imagery', async (t) => { + const stalled = deferred(); + const { env, terrain } = terrainFixture(t, async (n) => { + if (n > 1) await stalled.promise; + return flatTerrain(n); + }); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + assert.equal(env.controller.getActiveId(), 'osm'); + esri.failing = false; + t.mock.timers.tick(30_000); + await settle(); + assert.equal(terrain.calls, 2, 'the terrain retry is in flight'); + assert.equal(env.controller.getActiveId(), 'esri-imagery'); + assert.equal(env.imagery.at(-1).provider, env.providers.get('esri-imagery')); + stalled.resolve(); + await settle(); + env.controller.destroy(); +}); + +test('a stalled imagery retry does not hold back recovered terrain', async (t) => { + const real = { id: 'real' }; + const { env, installed } = terrainFixture(t, async (n) => + n > 1 ? { provider: real } : flatTerrain(n), + ); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + const stalled = deferred(); + esri.next = stalled.promise; + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 2, 'the imagery retry is in flight'); + assert.equal(installed.at(-1), real, 'real terrain is installed'); + stalled.resolve(); + await settle(); + env.controller.destroy(); +}); + +test('a retry attempt that never settles is abandoned at its deadline and released if it lands late', async (t) => { + const hung = deferred(); + const late = { id: 'late', destroy: () => (late.destroyed = true) }; + const { env, installed, terrain } = terrainFixture(t, async (n) => { + if (n === 2) await hung.promise; + return n === 2 ? { provider: late } : flatTerrain(n); + }); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 2, 'imagery failed'); + assert.equal(terrain.calls, 2, 'terrain hangs'); + t.mock.timers.tick(60_000); // The hung attempt's deadline. + await settle(); + t.mock.timers.tick(60_000); // The doubled backoff. + await settle(); + assert.equal(esri.attempts, 3, 'imagery is retried again'); + assert.equal(terrain.calls, 3, 'terrain is retried again'); + hung.resolve(); + await settle(); + assert.equal(late.destroyed, true, 'the late provider is released'); + assert.ok(!installed.includes(late), 'and never installed'); + env.controller.destroy(); +}); + +test('a failed retry after a mid-retry re-show re-arms once, after the doubled delay', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + const gate = deferred(); + esri.next = gate.promise; + t.mock.timers.tick(30_000); + await settle(); + await env.controller.setStack('osm'); + await env.controller.setStack('esri-imagery'); + gate.resolve(); + await settle(); + assert.equal(esri.attempts, 2); + t.mock.timers.tick(59_999); + await settle(); + assert.equal(esri.attempts, 2, 'not before the doubled delay'); + t.mock.timers.tick(1); + await settle(); + assert.equal(esri.attempts, 3, 'one retry after it'); + t.mock.timers.tick(119_999); + await settle(); + assert.equal(esri.attempts, 3, 'no second timer was armed'); + env.controller.destroy(); +}); + +test('destroy releases recovered imagery and terrain that lands afterwards', async (t) => { + const stalled = deferred(); + const real = { id: 'real', destroy: () => (real.destroyed = true) }; + const { env, installed } = terrainFixture(t, async (n) => { + if (n === 1) return flatTerrain(n); + await stalled.promise; + return { provider: real }; + }); + const esri = failingEsri(env); + const imagery = env.providers.get('esri-imagery'); + imagery.destroy = () => (imagery.destroyed = true); + await env.controller.setStack('esri-imagery'); + esri.failing = false; + t.mock.timers.tick(30_000); + await settle(); + assert.equal(env.imagery.at(-1).provider, imagery, 'imagery has landed'); + env.controller.destroy(); + stalled.resolve(); + await settle(); + assert.equal(imagery.destroyed, true); + assert.equal(real.destroyed, true); + assert.ok(!installed.includes(real)); +}); + +function liveTimers(t) { + const live = new Set(); + const { setTimeout: set, clearTimeout: clear } = globalThis; + globalThis.setTimeout = (fn, ms, ...args) => { + const id = set( + (...values) => { + live.delete(id); + fn(...values); + }, + ms, + ...args, + ); + live.add(id); + return id; + }; + globalThis.clearTimeout = (id) => { + live.delete(id); + clear(id); + }; + t.after(() => + Object.assign(globalThis, { setTimeout: set, clearTimeout: clear }), + ); + return live; +} + +test('destroy during a hung retry attempt clears its deadline at once', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const live = liveTimers(t); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + esri.next = new Promise(() => {}); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 2, 'the attempt hangs'); + assert.equal(live.size, 1, 'only its deadline is armed'); + env.controller.destroy(); + assert.equal(live.size, 0, 'destroy clears the deadline'); + assert.equal(env.controller._retrying, false); + await settle(); + assert.equal(live.size, 0, 'and nothing re-arms'); +}); + +function destroyableFlat(n) { + const provider = { id: `flat-${n}` }; + provider.destroy = () => (provider.destroyed = true); + return { provider, fallback: true }; +} + +test('terrain recovered on screen releases the fallback it replaced', async (t) => { + const real = { id: 'real' }; + let flat; + const { env, installed } = terrainFixture(t, async (n) => + n === 1 ? (flat = destroyableFlat(n)) : { provider: real }, + ); + await env.controller.setStack('esri-imagery'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(installed.at(-1), real); + assert.equal(flat.provider.destroyed, true); + env.controller.destroy(); +}); + +test('terrain recovered behind a tileset releases the fallback once a globe replaces it', async (t) => { + const real = { id: 'real' }; + const gate = deferred(); + let flat; + const { env, installed } = terrainFixture(t, async (n) => { + if (n === 1) return (flat = destroyableFlat(n)); + await gate.promise; + return { provider: real }; + }); + await env.controller.setStack('esri-imagery'); + t.mock.timers.tick(30_000); + await settle(); + await env.controller.setStack('photoreal'); + gate.resolve(); + await settle(); + assert.equal(installed.at(-1), flat.provider); + assert.ok(!flat.provider.destroyed, 'still installed behind the tileset'); + await env.controller.setStack('osm'); + assert.equal(installed.at(-1), real); + assert.equal(flat.provider.destroyed, true); + env.controller.destroy(); +}); + +test('a hung imagery attempt that lands after its deadline is released, never cached or shown', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const hung = deferred(); + const late = { id: 'late', errorEvent: event() }; + const good = { id: 'good', errorEvent: event() }; + for (const provider of [late, good]) + provider.destroy = () => (provider.destroyed = true); + let attempts = 0; + env.registry.sources.find( + (source) => source.descriptor.id === 'esri-imagery', + ).imagery = async () => { + if (++attempts === 1) throw new Error('Esri unreachable'); + if (attempts === 2) { + await hung.promise; + return late; + } + return good; + }; + await env.controller.setStack('esri-imagery'); + t.mock.timers.tick(30_000); + await settle(); + t.mock.timers.tick(60_000); // The deadline. + await settle(); + hung.resolve(); + await settle(); + assert.equal(late.destroyed, true, 'the late provider is released'); + assert.equal(env.controller.getActiveId(), 'osm', 'and not shown'); + t.mock.timers.tick(60_000); + await settle(); + assert.equal(attempts, 3); + assert.equal(env.imagery.at(-1).provider, good, 'the next retry installs'); + await env.controller.setStack('osm'); + await env.controller.setStack('esri-imagery'); + assert.equal(env.imagery.at(-1).provider, good, 'only its own is cached'); + env.controller.destroy(); + await settle(); + assert.equal(good.destroyed, true); +}); diff --git a/src/maps/sourceFactories.test.mjs b/src/maps/sourceFactories.test.mjs index d3aaa5068..3f1edc7ba 100644 --- a/src/maps/sourceFactories.test.mjs +++ b/src/maps/sourceFactories.test.mjs @@ -317,3 +317,13 @@ test('both Google 3D routes keep tiles drawing while draped imagery loads', asyn assert.equal(calls[1][1].asynchronouslyLoadImagery, true); assert.equal(calls[1][1].enableCollision, true); }); + +test('unreachable keyless terrain falls back to flat terrain marked as a fallback', async (t) => { + t.mock.method(Cesium.CesiumTerrainProvider, 'fromUrl', async () => { + throw new Error('layer.json unreachable'); + }); + t.mock.method(console, 'warn', () => {}); + const result = await createKeylessTerrain(); + assert.ok(result.provider instanceof Cesium.EllipsoidTerrainProvider); + assert.equal(result.fallback, true); +}); diff --git a/src/maps/terrain.js b/src/maps/terrain.js index f58c1c755..d2f214700 100644 --- a/src/maps/terrain.js +++ b/src/maps/terrain.js @@ -59,6 +59,6 @@ export async function createKeylessTerrain() { '[MapStack] Re:Earth terrain unavailable, falling back to flat ellipsoid terrain:', error, ); - return { provider: new Cesium.EllipsoidTerrainProvider() }; + return { provider: new Cesium.EllipsoidTerrainProvider(), fallback: true }; } }