diff --git a/CHANGELOG.md b/CHANGELOG.md index dd76d7a25..16bdcf39e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +- A failed Esri Satellite or keyless terrain load still shows OSM or flat terrain + at once, but no longer lasts the whole session: while the fallback is on + screen, the real provider is retried in the background (30 seconds, doubling + to a 10-minute cap) and replaces it live once it loads. Map switches never + wait for a retry. + - Add Barcelona's bike share to the GBFS registry: Bicing (543 stations, the city's PBSC system) and AMBici (237 e-bike stations across the metropolitan area, run by nextbike). Both are keyless GBFS 2.3 feeds listed in diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 4c06dd3cb..b8699b4c8 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -369,7 +369,11 @@ doubling to a 15 s cap, with every retry waiting behind one shared cooldown plus up to 1.5 s of jitter, and `Retry-After` extending the cooldown when the upstream exposes it. One console warning is logged per cooldown window. Other failures keep Cesium's handling and the flat `EllipsoidTerrainProvider` -fallback. `scripts/qa-terrain-429.mjs` reproduces the burst against a keyless +fallback; while that flat terrain is on a shown globe the map controller +retries Re:Earth in the background (30 s, doubling to a 10-minute cap) and +installs the real terrain without waiting for a map switch. Behind a 3D tileset +it neither retries nor installs; the next globe activation does. +`scripts/qa-terrain-429.mjs` reproduces the burst against a keyless dev server, counts real throttles with their headers as evidence, synthesises throttles with `--inject N` when the upstream is not throttling, and fails if a throttled tile never loads. @@ -4692,7 +4696,7 @@ are omitted rather than framing the wrong part of the globe. - The bottom Visual Presets tray presents a **five-tile MAP SOURCE row** (`#map-stack-chips`, `src/mapStackChips.js`): Google 3D, Esri Satellite, Bing Aerial, Bing Labels, and OSM. The duplicate left `#stack-panel` is retired. The five tiles share one row on desktop and two rows on narrow screens, carry `aria-pressed` on the active source, and remain keyboard-reachable with a visible focus outline. - The lit tile follows controller state, not the click: a rejected switch (no ion token) or a superseded one (rapid A→B) leaves the genuinely active source lit, and the tray heading keeps its short-label status readout (`...` while switching, amber on `lastError`). - Ion stacks remain visible and keyboard-focusable when no ion token is configured, but expose `aria-disabled="true"` and do not switch. Their accessible label and tooltip quote `getStacks().unavailableReason` — the same string `setStack()` puts in the toast. OSM works keyless. The `ION` badge is gated on the stack's own `requiresIon`, so a `photoreal` chip unavailable because the Google tileset failed says so instead of falsely demanding an ion token. -- Stack choice participates in share links (`src/sharelink.js`) and falls back to the best available stack when the requested one is unavailable (keyless boots land on Esri; OSM takes over automatically if Esri is unreachable). Share-link restore, the `set_map_stack` voice tool, and the chip row all land on the same `_setMapStack()` path. +- Stack choice participates in share links (`src/sharelink.js`) and falls back to the best available stack when the requested one is unavailable (keyless boots land on Esri; OSM takes over automatically if Esri is unreachable; while that fallback is on screen Esri is retried in the background, 30 s doubling to a 10-minute cap, and replaces OSM once it loads). Share-link restore, the `set_map_stack` voice tool, and the chip row all land on the same `_setMapStack()` path. ### Voice Map Whiteboard / Annotations (June 2026) diff --git a/src/maps/controller.js b/src/maps/controller.js index 2b2ea462f..445e689f2 100644 --- a/src/maps/controller.js +++ b/src/maps/controller.js @@ -9,6 +9,16 @@ import { acquireImageryComparison } from './imageryComparison.js'; */ const AUTOMATIC = Symbol('automatic switch'); +/** Background retries of a fallback in use back off from 30 s to 10 min. */ +const FALLBACK_RETRY_MS = 30_000; +const FALLBACK_RETRY_MAX_MS = 600_000; + +const resolved = (provider, effectiveStackId) => ({ + provider, + effectiveStackId, + fallbackMessage: null, +}); + /** Coordinate source lifetimes and scene changes; the registry owns provider choices. */ export class MapSourceController { constructor( @@ -46,8 +56,12 @@ export class MapSourceController { this._lastError = null; this._imageryLayer = null; this._activeImageryProvider = null; + this._activeResolution = null; this._removeImageryErrorListener = null; this._terrainMode = null; + this._terrainFallback = null; + this._retryTimer = null; + this._retryDelay = FALLBACK_RETRY_MS; this._subscribers = new Set(); this._destroyed = false; } @@ -285,21 +299,102 @@ export class MapSourceController { this._showTileset(null); this.viewer.scene.globe.show = true; this._activeId = resolution.effectiveStackId; + this._activeResolution = resolution; if (source?.terrain && source.terrain.id !== this._terrainMode) { const terrain = source.terrain; - const result = await this._cached( - this._terrainProviders, - terrain.id, - () => terrain.create({ signal: this._abort.signal }), + const pending = this._cached(this._terrainProviders, terrain.id, () => + terrain.create({ signal: this._abort.signal }), ); + let result = await pending; if (gen !== this._switchGen) return; - if (result.terrain) this.viewer.scene.setTerrain(result.terrain); - else this.viewer.terrainProvider = result.provider; + // A background retry may have replaced a fallback while this waited. + const latest = this._terrainProviders.get(terrain.id); + if (latest !== pending) result = await latest; + if (gen !== this._switchGen) return; + this._applyTerrain(result); this._terrainMode = terrain.id; } + if (resolution.fallbackFor || this._terrainFallback) this._scheduleRetry(); return resolution; } + _applyTerrain(result) { + if (result.terrain) this.viewer.scene.setTerrain(result.terrain); + else this.viewer.terrainProvider = result.provider; + this._terrainFallback = result.fallback ? result : null; + } + + /** + * One timer retries the real provider behind whichever fallback is on + * screen: Esri's OSM stand-in, or flat terrain on a shown globe. Fallbacks + * stay cached and shown meanwhile, so no switch waits on a retry; a + * fallback out of use is left alone until an activation shows it again. + */ + _scheduleRetry() { + if (this._retryTimer) return; + this._retryTimer = setTimeout(() => { + this._retryTimer = null; + void this._retryFallbacks(); + }, this._retryDelay); + } + + async _retryFallbacks() { + const resolution = this._activeResolution; + const flat = this.viewer.scene.globe.show ? this._terrainFallback : null; + const id = resolution?.fallbackFor; + let terrain = flat && this._sources.get(this._activeId)?.terrain; + if (terrain?.id !== this._terrainMode) terrain = null; + if (!id && !terrain) return; + const attempt = async (create) => { + try { + return await create?.({ signal: this._abort.signal }); + } catch { + return null; // Still unavailable: keep the fallback. + } + }; + const [provider, real] = await Promise.all([ + id && attempt(this._sources.get(id).imagery), + terrain && attempt(terrain.create), + ]); + if (this._destroyed) { + this._dispose(provider); + this._dispose(real?.provider); + return; + } + let failed = (id && !provider) || (terrain && (!real || real.fallback)); + try { + if (provider) { + this._imageryProviders.set(id, Promise.resolve(resolved(provider, id))); + // Still showing the fallback once any switch settles: use the real one. + const recover = () => { + if (this._activeResolution === resolution) + void this.setStack(id, { [AUTOMATIC]: true }); + }; + if (!this._isSwitching) recover(); + else { + const stop = this.subscribe(() => { + stop(); + recover(); + }); + } + } + if (terrain && real && !real.fallback) { + this._terrainProviders.set(terrain.id, Promise.resolve(real)); + if (this._terrainFallback === flat && this.viewer.scene.globe.show) { + this._applyTerrain(real); + this._requestRender('map-stack'); + } else this._terrainMode = null; // The next globe activation installs it. + } + } catch (error) { + console.warn('[MapSourceController] fallback recovery failed:', error); + failed = true; + } + this._retryDelay = failed + ? Math.min(this._retryDelay * 2, FALLBACK_RETRY_MAX_MS) + : FALLBACK_RETRY_MS; + if (failed) this._scheduleRetry(); + } + _cached(cache, id, create) { if (cache.has(id)) return cache.get(id); const promise = Promise.resolve() @@ -323,7 +418,7 @@ export class MapSourceController { try { const provider = await source.imagery({ signal: this._abort.signal }); if (this._destroyed) this._dispose(provider); - return { provider, effectiveStackId: stack.id, fallbackMessage: null }; + return resolved(provider, stack.id); } catch (error) { const fallback = source.constructionFallback; if (this._destroyed || !fallback || !this.isStackAvailable(fallback.id)) @@ -333,7 +428,11 @@ export class MapSourceController { this.getStack(fallback.id), next, ); - return { ...resolution, fallbackMessage: fallback.message }; + return { + ...resolution, + fallbackMessage: fallback.message, + fallbackFor: stack.id, + }; } }); } @@ -388,6 +487,7 @@ export class MapSourceController { this.viewer.imageryLayers.remove(this._imageryLayer, true); this._imageryLayer = null; this._activeImageryProvider = null; + this._activeResolution = null; } _dispose(value) { if (!value || this._disposed.has(value)) return; @@ -401,6 +501,7 @@ export class MapSourceController { this._subscribers.clear(); this._switchGen++; this._abort.abort(); + clearTimeout(this._retryTimer); this._isSwitching = false; this._removeImageryLayer(); this._credits.destroy(); diff --git a/src/maps/controller.test.mjs b/src/maps/controller.test.mjs index 1e6c1413f..8dcead487 100644 --- a/src/maps/controller.test.mjs +++ b/src/maps/controller.test.mjs @@ -70,7 +70,7 @@ const settle = async () => { for (let i = 0; i < 20; i++) await Promise.resolve(); }; const descriptor = (id) => ({ id, label: id, kind: 'imagery' }); -function publicFixture() { +function publicFixture(options) { const tileset = { show: true }; const registry = createDefaultMapSources({ googleTileset: tileset }); const providers = new Map(); @@ -84,7 +84,7 @@ function publicFixture() { create: async () => ({ provider: { id: 'terrain' } }), }; } - return { ...fixture(registry), registry, providers, tileset }; + return { ...fixture(registry, options), registry, providers, tileset }; } test('an additional imagery source needs no controller branch and owns its cached resources', async () => { @@ -839,3 +839,262 @@ test('each switch generation reports its origin: an outside setStack is manual, ]); recovering.controller.destroy(); }); + +function failingEsri(env) { + const esri = { failing: true, attempts: 0, next: null }; + env.registry.sources.find( + (source) => source.descriptor.id === 'esri-imagery', + ).imagery = async () => { + esri.attempts++; + if (esri.next) await esri.next; + if (esri.failing) throw new Error('Esri unreachable'); + return env.providers.get('esri-imagery'); + }; + return esri; +} +const deferred = () => { + let resolve; + const promise = new Promise((done) => (resolve = done)); + return { promise, resolve }; +}; + +test('a failed Esri construction shows OSM at once, backs off while retries fail, and recovers in place', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + assert.equal(env.controller.getActiveId(), 'osm'); + const osmLayer = env.imagery.at(-1); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 2, 'one background retry'); + assert.equal(env.imagery.at(-1), osmLayer, 'a failed retry keeps the layer'); + await env.controller.setStack('osm'); + await env.controller.setStack('esri-imagery'); + assert.equal(esri.attempts, 2, 'a reselection does not retry at once'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 2, 'the second retry waits twice as long'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 3); + esri.failing = false; + t.mock.timers.tick(120_000); + await settle(); + assert.equal(esri.attempts, 4); + assert.equal(env.controller.getActiveId(), 'esri-imagery'); + assert.equal(env.controller.getSwitchOrigin(), 'automatic'); + assert.equal(env.controller.getState().lastError, null); + assert.equal(env.imagery.at(-1).provider, env.providers.get('esri-imagery')); + t.mock.timers.tick(600_000); + await settle(); + assert.equal(esri.attempts, 4, 'the real provider ends the retries'); + env.controller.destroy(); +}); + +test('retries of a fallback that keeps failing back off to a 10 minute cap', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + for (const delay of [30, 60, 120, 240, 480, 600, 600]) { + t.mock.timers.tick(delay * 1000); + await settle(); + } + assert.equal(esri.attempts, 8); + env.controller.destroy(); +}); + +test('Esri is retried only while its fallback is shown', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + await env.controller.setStack('osm'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 1, 'no retry while OSM was chosen'); + await env.controller.setStack('esri-imagery'); + assert.equal(esri.attempts, 1, 'the cached fallback shows at once'); + assert.equal(env.controller.getActiveId(), 'osm'); + esri.failing = false; + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 2); + assert.equal(env.controller.getActiveId(), 'esri-imagery'); + env.controller.destroy(); +}); + +test('an Esri recovery that lands mid-switch applies once the switch settles', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + const retry = deferred(); + esri.next = retry.promise; + esri.failing = false; + t.mock.timers.tick(30_000); + await settle(); + // The fallback activates OSM's terrain; make that switch slow. + const slow = deferred(); + env.registry.sources.find( + (source) => source.descriptor.id === 'osm', + ).terrain = { id: 'slow', create: () => slow.promise }; + const switching = env.controller.setStack('esri-imagery'); + retry.resolve(); + await settle(); + assert.equal(env.controller.getActiveId(), 'osm', 'the switch is still open'); + slow.resolve({ provider: { id: 'slow' } }); + await switching; + await settle(); + assert.equal(env.controller.getActiveId(), 'esri-imagery'); + assert.equal(env.imagery.at(-1).provider, env.providers.get('esri-imagery')); + env.controller.destroy(); +}); + +function terrainFixture(t, create) { + t.mock.timers.enable({ apis: ['setTimeout'] }); + let renders = 0; + const env = publicFixture({ requestRender: () => renders++ }); + const installed = []; + Object.defineProperty(env.viewer, 'terrainProvider', { + get: () => installed.at(-1), + set: (provider) => installed.push(provider), + }); + const terrain = { id: 'keyless', calls: 0 }; + terrain.create = () => create(++terrain.calls); + for (const source of env.registry.sources) + if (source.imagery) source.terrain = terrain; + return { env, installed, terrain, renders: () => renders }; +} +const flatTerrain = (n) => ({ provider: { id: `flat-${n}` }, fallback: true }); + +test('flat fallback terrain stays installed while retries run, and real terrain replaces it live', async (t) => { + const real = { id: 'real' }; + let working = false; + const { env, installed, terrain, renders } = terrainFixture(t, async (n) => + working ? { provider: real } : flatTerrain(n), + ); + await env.controller.setStack('esri-imagery'); + const [flat] = installed; + t.mock.timers.tick(30_000); + await settle(); + assert.equal(terrain.calls, 2, 'one background retry'); + await env.controller.setStack('osm'); + assert.deepEqual(installed, [flat], 'failed retries and switches keep it'); + working = true; + const before = renders(); + t.mock.timers.tick(60_000); + await settle(); + assert.equal(terrain.calls, 3); + assert.deepEqual(installed, [flat, real]); + assert.equal(renders(), before + 1, 'the live swap requests a render'); + t.mock.timers.tick(600_000); + await settle(); + assert.equal(terrain.calls, 3, 'the real terrain ends the retries'); + env.controller.destroy(); +}); + +test('terrain is not retried or installed while a 3D tileset hides the globe', async (t) => { + const real = { id: 'real' }; + let working = false; + const { env, installed, terrain } = terrainFixture(t, async (n) => + working ? { provider: real } : flatTerrain(n), + ); + await env.controller.setStack('esri-imagery'); + await env.controller.setStack('photoreal'); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(terrain.calls, 1, 'no retry behind a hidden globe'); + await env.controller.setStack('esri-imagery'); + const hidden = deferred(); + terrain.create = async () => { + terrain.calls++; + await hidden.promise; + return { provider: real }; + }; + t.mock.timers.tick(30_000); + await settle(); + assert.equal(terrain.calls, 2, 'the globe activation re-arms the retry'); + await env.controller.setStack('photoreal'); + hidden.resolve(); + await settle(); + assert.equal(installed.length, 1, 'nothing is installed behind a tileset'); + await env.controller.setStack('osm'); + assert.equal(installed.at(-1), real, 'the next globe activation installs it'); + assert.equal(terrain.calls, 2); + env.controller.destroy(); +}); + +test('a switch that read the flat fallback installs the terrain a retry just cached', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + // Sweep the microtask distance between the retry landing and the switch. + for (let hops = 0; hops < 12; hops++) { + const real = { id: 'real' }, + gate = deferred(); + const keyless = { + id: 'keyless', + create: async () => { + if (!keyless.retried) return flatTerrain(1); + await gate.promise; + return { provider: real }; + }, + }; + const other = { id: 'other', create: async () => ({ provider: {} }) }; + const source = (id, terrain) => ({ + descriptor: descriptor(id), + imagery: async () => ({ id }), + terrain, + }); + const env = fixture({ + defaultId: 'a', + sources: [source('a', keyless), source('c', other)], + }); + const installed = []; + Object.defineProperty(env.viewer, 'terrainProvider', { + get: () => installed.at(-1), + set: (provider) => installed.push(provider), + }); + await env.controller.setStack('a'); + keyless.retried = true; + t.mock.timers.tick(30_000); + await settle(); + await env.controller.setStack('c'); + gate.resolve(); + for (let i = 0; i < hops; i++) await null; + await env.controller.setStack('a'); + await settle(); + assert.equal(installed.at(-1), real, `hops ${hops}`); + env.controller.destroy(); + } +}); + +test('destroy cancels a pending retry and releases one in flight', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const env = publicFixture(); + const esri = failingEsri(env); + await env.controller.setStack('esri-imagery'); + env.controller.destroy(); + t.mock.timers.tick(30_000); + await settle(); + assert.equal(esri.attempts, 1, 'a pending retry is cancelled'); + + const late = publicFixture(); + const lateEsri = failingEsri(late); + await late.controller.setStack('esri-imagery'); + const retry = deferred(); + lateEsri.next = retry.promise; + lateEsri.failing = false; + let destroyed = 0; + late.providers.get('esri-imagery').destroy = () => destroyed++; + t.mock.timers.tick(30_000); + await settle(); + late.controller.destroy(); + retry.resolve(); + await settle(); + assert.equal(destroyed, 1, 'a late provider is released'); + assert.equal(late.imagery.length, 0); + t.mock.timers.tick(600_000); + await settle(); + assert.equal(lateEsri.attempts, 2); +}); diff --git a/src/maps/sourceFactories.test.mjs b/src/maps/sourceFactories.test.mjs index d3aaa5068..3f1edc7ba 100644 --- a/src/maps/sourceFactories.test.mjs +++ b/src/maps/sourceFactories.test.mjs @@ -317,3 +317,13 @@ test('both Google 3D routes keep tiles drawing while draped imagery loads', asyn assert.equal(calls[1][1].asynchronouslyLoadImagery, true); assert.equal(calls[1][1].enableCollision, true); }); + +test('unreachable keyless terrain falls back to flat terrain marked as a fallback', async (t) => { + t.mock.method(Cesium.CesiumTerrainProvider, 'fromUrl', async () => { + throw new Error('layer.json unreachable'); + }); + t.mock.method(console, 'warn', () => {}); + const result = await createKeylessTerrain(); + assert.ok(result.provider instanceof Cesium.EllipsoidTerrainProvider); + assert.equal(result.fallback, true); +}); diff --git a/src/maps/terrain.js b/src/maps/terrain.js index f58c1c755..d2f214700 100644 --- a/src/maps/terrain.js +++ b/src/maps/terrain.js @@ -59,6 +59,6 @@ export async function createKeylessTerrain() { '[MapStack] Re:Earth terrain unavailable, falling back to flat ellipsoid terrain:', error, ); - return { provider: new Cesium.EllipsoidTerrainProvider() }; + return { provider: new Cesium.EllipsoidTerrainProvider(), fallback: true }; } }