From a07d9f5d858fc02e615f1919ef54d56c5ea1c18c Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Tue, 10 Nov 2026 16:04:01 +0400 Subject: [PATCH] Retry a failed 3D model inside the engine instead of in the layers A mover set whose glTF or atlas fails to load retries by itself after retryAfterMs (1,500 ms by default), at most three attempts, on Cesium, Navara and the fake engine alike: failed() is true while the latest attempt failed and stays true after the last, ready() turns true when a retry loads, and an instance given another pickId loads again. Healthy models of the same class no longer load again when one fails. The layers drop what they did instead: moverBank's rebuilds, the loops that dropped failed handles in flights, military and local ADS-B, and the tracked-model backoff in the aircraft display. The contract suite checks the one behaviour on both engines, with a mutant that never retries. --- scripts/contract-mutants.test.mjs | 21 +- src/globe/cesium/createCesiumEngine.test.mjs | 18 +- src/globe/cesium/movers.js | 155 +++++----- src/globe/cesium/movers.test.mjs | 133 +++++++- src/globe/contract.js | 17 +- src/globe/movers.js | 45 ++- src/globe/movers.test.mjs | 86 ++++++ src/globe/navara/movers/index.js | 14 +- src/globe/navara/movers/modelMover.js | 232 ++++++++------ src/globe/navara/movers/modelMover.test.mjs | 102 ++++++- src/globe/navara/movers/moverMesh.js | 34 ++- src/globe/navara/movers/moverMesh.test.mjs | 48 +++ src/globe/testing/contractSuite/movers.mjs | 119 +++++++- src/globe/testing/fakeEngine.js | 60 +++- src/layers/aircraft/render/contract.js | 4 +- src/layers/aircraft/render/display.js | 21 +- .../aircraft/trackedModelFailure.test.mjs | 289 +++++++++--------- src/layers/flights/rendering.js | 15 - src/layers/localAdsb/models.js | 4 - src/layers/localAdsb/models.test.mjs | 64 ++-- src/layers/military/rendering.js | 15 - src/layers/shared/render/moverBank.js | 13 +- 22 files changed, 1039 insertions(+), 470 deletions(-) diff --git a/scripts/contract-mutants.test.mjs b/scripts/contract-mutants.test.mjs index 96252b91a..30a197698 100644 --- a/scripts/contract-mutants.test.mjs +++ b/scripts/contract-mutants.test.mjs @@ -207,7 +207,7 @@ const mutants = [ ], [ 'failed model reports no failure', - 'MoverSet.failed:', + 'movers.createSet: failed model retries', (e) => { handle(e, 'movers', 'createSet', (set) => { set.failed = () => false; @@ -216,13 +216,30 @@ const mutants = [ ], [ 'failed model reports ready', - 'MoverSet.failed:', + 'movers.createSet: failed model retries', (e) => { handle(e, 'movers', 'createSet', (set) => { set.ready = () => true; }); }, ], + [ + 'failed model never retries', + 'movers.createSet: failed model retries', + (e) => { + handle(e, 'movers', 'createSet', (set) => { + replace(set, 'commit', (commit) => () => { + const schedule = globalThis.setTimeout; + globalThis.setTimeout = () => null; + try { + commit(); + } finally { + globalThis.setTimeout = schedule; + } + }); + }); + }, + ], [ 'hidden mover reports shown', 'movers.list: shown follows', diff --git a/src/globe/cesium/createCesiumEngine.test.mjs b/src/globe/cesium/createCesiumEngine.test.mjs index 2b3e999a5..87eef5aa3 100644 --- a/src/globe/cesium/createCesiumEngine.test.mjs +++ b/src/globe/cesium/createCesiumEngine.test.mjs @@ -216,20 +216,32 @@ test('the contract suite movers and lines checks pass on the Cesium engine (no p test('the camera, clock and failed model suite checks pass on the Cesium adapter', async (t) => { const { createAppClock } = await import('../../time/clock.js'); let wall = 1_000_000; + // The failed-model check turns the model's load failure on and off. + let modelFails = false; const { viewer, camera, engine, frame } = createCameraViewer({ engineOptions: { createEventHandler: createFakeEventHandlers().create, appClock: createAppClock({ now: () => wall }), moverFactories: { loadModel: async () => { - throw new Error('Model unavailable'); + if (modelFails) throw new Error('Model unavailable'); + return { + ready: true, + modelMatrix: new Cesium.Matrix4(), + isDestroyed: () => false, + destroy() {}, + }; }, }, }, }); viewer.entities = new Cesium.EntityCollection(); const checks = createContractSuite(engine, { - fixtures: { failModel() {} }, + fixtures: { + failModel(id, fails) { + modelFails = fails; + }, + }, waitFrames: async (count) => { for (let i = 0; i < count; i++) { wall += 16; @@ -239,7 +251,7 @@ test('the camera, clock and failed model suite checks pass on the Cesium adapter } }, }).filter(({ name }) => - /^(camera.changed: a pure|camera.following:|clock.frameTime:|MoverSet.failed:)/.test( + /^(camera.changed: a pure|camera.following:|clock.frameTime:|movers.createSet: failed model)/.test( name, ), ); diff --git a/src/globe/cesium/movers.js b/src/globe/cesium/movers.js index 39a6a058b..48d90e19b 100644 --- a/src/globe/cesium/movers.js +++ b/src/globe/cesium/movers.js @@ -11,7 +11,7 @@ import * as Cesium from 'cesium'; import { horizonVisible } from '../cameraMath.js'; -import { createMoverRegistry } from '../movers.js'; +import { createMoverRegistry, createMoverRetry } from '../movers.js'; let scratchPosition; let scratchColor; @@ -150,30 +150,35 @@ function spritePart(scene, options, store, ports) { } } }; - let atlasFailed = false; // Instances wait for the atlas; then everything written so far is applied. - factories - .cropCells(sprite.atlas, store.cellKeys) - .then((ready) => { - if (destroyed) return; - cells = ready; - apply(0, Math.max(store.count, billboards.length)); - requestRender('movers'); - }) - .catch((error) => { - atlasFailed = true; - console.warn(`[movers] ${id}: atlas failed`, error); - }); + const retry = createMoverRetry(options.retryAfterMs, () => { + factories + .cropCells(sprite.atlas, store.cellKeys) + .then((ready) => { + if (destroyed) return; + cells = ready; + retry.succeed(); + apply(0, Math.max(store.count, billboards.length)); + requestRender('movers'); + }) + .catch((error) => { + if (destroyed) return; + retry.fail(); + console.warn(`[movers] ${id}: atlas failed`, error); + }); + }); + retry.start(); return { primitive: collection, ready: () => cells !== null, - failed: () => atlasFailed, + failed: retry.failed, commit(range) { if (cells && range) apply(range.from, range.to); requestRender('movers'); }, destroy() { destroyed = true; + retry.destroy(); billboards.length = 0; shownCell.length = 0; cells = null; @@ -230,14 +235,9 @@ function modelPart(options, store, { factories, requestRender, owners }) { const collection = factories.models(); const { model, pickable } = options; const models = []; - const loading = []; - // Instances whose model failed: not tried again (MoverSet.failed). An - // index holds the pickId it failed for: a layer moves another instance - // into a freed index, and that one loads its own model. - const failed = []; - const failedFor = []; - const failedNow = (index) => - failed[index] === true && failedFor[index] === store.pickIds[index]; + const loads = []; + const current = (index) => + loads[index]?.pickId === store.pickIds[index] ? loads[index] : null; let destroyed = false; const place = (item, index) => { const { orientations } = store; @@ -258,69 +258,77 @@ function modelPart(options, store, { factories, requestRender, owners }) { item.color = readColor(store, index); }; const ensure = (index) => { - if (models[index] || loading[index]) return; - loading[index] = true; - factories - .loadModel({ - url: model.url, - // Resources built at once, not spread over frames behind tile jobs. - asynchronous: false, - // The instance orientation turns the glTF's own axes (the contract's - // model frame), as Navara draws them. - upAxis: Cesium.Axis.Z, - forwardAxis: Cesium.Axis.X, - // Navara has no environment map; Cesium's renders the sky around - // each model and reads it back from the GPU. - environmentMapOptions: { enabled: false }, - minimumPixelSize: model.minimumPixelSize, - ...modelBlend(model.colorBlend), - allowPicking: pickable, - ...(model.unlit - ? { - customShader: new Cesium.CustomShader({ - lightingModel: Cesium.LightingModel.UNLIT, - }), - } - : {}), - }) - .then((item) => { - if (destroyed) { - item.destroy?.(); - return; - } - models[index] = item; - owners.set(item, { owner: options.id, instance: index }); - place(item, index); - collection.add(item); - requestRender('movers'); - }) - .catch((error) => { - if (destroyed) return; - failed[index] = true; - failedFor[index] = store.pickIds[index]; - console.warn(`[movers] ${options.id}: model failed`, error); - }); + if (current(index)) return; + loads[index]?.retry.destroy(); + if (models[index]) collection.remove(models[index]); + models[index] = null; + const entry = { pickId: store.pickIds[index], retry: null }; + loads[index] = entry; + const live = () => !destroyed && current(index) === entry; + entry.retry = createMoverRetry(options.retryAfterMs, () => { + if (!live()) return; + factories + .loadModel({ + url: model.url, + // Resources built at once, not spread over frames behind tile jobs. + asynchronous: false, + // The instance orientation turns the glTF's own axes (the contract's + // model frame), as Navara draws them. + upAxis: Cesium.Axis.Z, + forwardAxis: Cesium.Axis.X, + // Navara has no environment map; Cesium's renders the sky around + // each model and reads it back from the GPU. + environmentMapOptions: { enabled: false }, + minimumPixelSize: model.minimumPixelSize, + ...modelBlend(model.colorBlend), + allowPicking: pickable, + ...(model.unlit + ? { + customShader: new Cesium.CustomShader({ + lightingModel: Cesium.LightingModel.UNLIT, + }), + } + : {}), + }) + .then((item) => { + if (!live()) { + item.destroy?.(); + return; + } + models[index] = item; + entry.retry.succeed(); + owners.set(item, { owner: options.id, instance: index }); + place(item, index); + collection.add(item); + requestRender('movers'); + }) + .catch((error) => { + if (!live()) return; + entry.retry.fail(); + console.warn(`[movers] ${options.id}: model failed`, error); + }); + }); + entry.retry.start(); }; return { primitive: collection, // Every drawn instance's model has loaded (with an index, that one's). ready(index) { if (index !== undefined) - return index < store.count && models[index]?.ready === true; + return Boolean(current(index)) && models[index]?.ready === true; for (let at = 0; at < store.count; at++) - if (!models[at]?.ready) return false; + if (!current(at) || !models[at]?.ready) return false; return true; }, failed(index) { - if (index !== undefined) return failedNow(index); - for (let at = 0; at < store.count; at++) if (failedNow(at)) return true; + if (index !== undefined) return current(index)?.retry.failed() === true; + for (let at = 0; at < store.count; at++) + if (current(at)?.retry.failed()) return true; return false; }, commit(range) { if (range) for (let index = range.from; index < range.to; index++) { - if (failed[index] && !failedNow(index)) - failed[index] = loading[index] = false; if (index < store.count) ensure(index); if (models[index]) place(models[index], index); } @@ -328,10 +336,9 @@ function modelPart(options, store, { factories, requestRender, owners }) { }, destroy() { destroyed = true; + for (const entry of loads) entry?.retry.destroy(); models.length = 0; - loading.length = 0; - failed.length = 0; - failedFor.length = 0; + loads.length = 0; }, }; } diff --git a/src/globe/cesium/movers.test.mjs b/src/globe/cesium/movers.test.mjs index 640bb8ef8..f48f9f515 100644 --- a/src/globe/cesium/movers.test.mjs +++ b/src/globe/cesium/movers.test.mjs @@ -8,17 +8,19 @@ import { createFakeViewer } from './testing/fakeViewer.mjs'; // Cesium's own collections, billboards and points (they draw nothing // without a renderer, but hold what the engine writes). What needs the // network or a DOM is faked: the glTF load and the atlas crop. -function setup({ loadModel } = {}) { +function setup({ loadModel, cropCells } = {}) { const viewer = createFakeViewer(); viewer.scene.frameState.mode = Cesium.SceneMode.SCENE3D; const requests = []; const crops = []; const loads = []; const factories = { - cropCells: async (atlas, keys) => { - crops.push({ atlas, keys }); - return new Map(keys.map((key) => [key, { width: 8, height: 8, key }])); - }, + cropCells: + cropCells ?? + (async (atlas, keys) => { + crops.push({ atlas, keys }); + return new Map(keys.map((key) => [key, { width: 8, height: 8, key }])); + }), loadModel: loadModel ?? (async (options) => { @@ -412,13 +414,14 @@ test('a model that loads after its set is destroyed is released', async () => { assert.equal(root().length, 0); }); -test('a model that fails to load marks its instance failed, once', async (t) => { +test('failed model attempts stop at three and a replacement gets a fresh budget', async (t) => { t.mock.method(console, 'warn', () => {}); + t.mock.timers.enable({ apis: ['setTimeout'] }); let loads = 0; const { movers } = setup({ loadModel: async () => { loads++; - throw new Error('404'); + throw new Error('503'); }, }); const set = movers.createSet({ @@ -427,19 +430,123 @@ test('a model that fails to load marks its instance failed, once', async (t) => capacity: 1, model: { url: '/a.glb' }, }); - set.set(0, { position: at(0, 0) }); + set.set(0, { position: at(0, 0), pickId: 'a' }); set.setCount(1); set.commit(); - assert.equal(set.failed(), false, 'still loading'); - await settle(); + assert.equal(set.failed(), false); + await new Promise(setImmediate); assert.equal(set.failed(0), true); + t.mock.timers.tick(1499); + assert.equal(loads, 1); + for (const delay of [1, 1500]) { + t.mock.timers.tick(delay); + await new Promise(setImmediate); + } + t.mock.timers.tick(10000); + assert.equal(loads, 3); assert.equal(set.failed(), true); assert.equal(set.ready(), false); + set.set(0, { pickId: 'b' }); set.commit(); - await settle(); - assert.equal(loads, 1, 'not tried again'); + await new Promise(setImmediate); + assert.equal(loads, 4); set.destroy(); - assert.equal(set.failed(), false, 'a destroyed set reports nothing'); + t.mock.timers.tick(10000); + assert.equal(loads, 4); + assert.equal(set.failed(), false); +}); + +test('a failed instance retries without reloading its healthy neighbour', async (t) => { + t.mock.method(console, 'warn', () => {}); + t.mock.timers.enable({ apis: ['setTimeout'] }); + let loads = 0; + let finish; + const { movers, root } = setup({ + loadModel: async (options) => { + loads++; + if (loads === 1) throw new Error('503'); + if (loads === 3) await new Promise((resolve) => (finish = resolve)); + return { ...fakeModel(options), ready: true }; + }, + }); + const set = movers.createSet({ + id: 'fleet', + kind: 'model', + capacity: 2, + retryAfterMs: 20, + model: { url: '/a.glb' }, + }); + t.after(() => set.destroy()); + set.set(0, { position: at(0, 0), pickId: 'a' }); + set.set(1, { position: at(1, 0), pickId: 'b' }); + set.setCount(2); + set.commit(); + await new Promise(setImmediate); + const [collection] = items(root()); + const [healthy] = items(collection); + assert.equal(set.failed(0), true); + assert.equal(set.ready(1), true); + t.mock.timers.tick(20); + assert.equal( + set.failed(0), + true, + 'failure stays visible while retry is pending', + ); + set.set(0, { position: at(2, 0), show: false }); + set.commit(); + finish(); + await new Promise(setImmediate); + assert.equal(loads, 3); + assert.equal(set.ready(), true); + assert.equal(set.failed(), false); + assert.equal(collection.get(0), healthy); + assert.equal(collection.get(1).show, false); + assert.equal(collection.get(1).modelMatrix[12], at(2, 0).x); + set.set(0, { pickId: 'c' }); + set.commit(); + await new Promise(setImmediate); + assert.equal(loads, 4, 'a healthy index reloads for its new identity'); +}); + +test('atlas retries apply the latest instances and cancel on destruction', async (t) => { + t.mock.method(console, 'warn', () => {}); + t.mock.timers.enable({ apis: ['setTimeout'] }); + let loads = 0; + const { movers, root } = setup({ + cropCells: async (atlas, keys) => { + if (++loads < 3) throw new Error('503'); + return new Map(keys.map((key) => [key, { width: 8, height: 8 }])); + }, + }); + const set = movers.createSet({ + id: 'sprites', + kind: 'sprite', + capacity: 1, + retryAfterMs: 20, + sprite: { atlas, sizePx: 8 }, + }); + set.setCount(1); + set.commit(); + await new Promise(setImmediate); + assert.equal(set.failed(), true); + for (let i = 0; i < 2; i++) { + t.mock.timers.tick(20); + await new Promise(setImmediate); + } + assert.equal(set.ready(), true); + assert.equal(set.failed(), false); + assert.equal(root().get(0).length, 1); + set.destroy(); + const cancelled = movers.createSet({ + id: 'cancelled', + kind: 'sprite', + capacity: 1, + sprite: { atlas, sizePx: 8 }, + }); + cancelled.destroy(); + await new Promise(setImmediate); + t.mock.timers.tick(10000); + assert.equal(loads, 4); }); // A layer removes an instance by moving its last one into the freed index diff --git a/src/globe/contract.js b/src/globe/contract.js index 08dfc95c8..9db45cea6 100644 --- a/src/globe/contract.js +++ b/src/globe/contract.js @@ -419,6 +419,8 @@ * @property {string} id Unique among the engine's live sets. * @property {'sprite'|'point'|'model'} kind * @property {number} capacity The most instances the set holds. + * @property {number} [retryAfterMs=1500] Delay after a model or atlas load + * failure before retrying. At most three attempts, including the first. * @property {number} [order] Draw order among sets (higher draws later, on * top); 0 by default. * @property {'normal'|'always'|{ disableWithinM: number }} [depthTest] @@ -474,14 +476,15 @@ * @property {() => void} commit Uploads what changed (one buffer write) and * requests a render; nothing else draws a frame. * @property {(index?: number) => boolean} ready Whether what the set draws - * has loaded: a model set's glTF (Cesium: every drawn instance's model), a - * sprite set's atlas. With an index, what that instance draws (Cesium: its - * own model; the other engines load a set's glTF once). - * A layer keeps its fallback visual until then. + * has loaded: every active model or the sprite atlas. With an index, what + * that active instance draws; false outside [0, count). A successful retry + * makes it ready. A layer keeps its fallback visual until then. * @property {(index?: number) => boolean} failed Whether what the set draws - * failed to load (with an index, what that instance draws): it stays not - * ready and is not tried again; a layer that retries makes a new set. - * Cesium loads per instance: an index given another pickId loads again. + * failed on its latest attempt (with an index, that active instance; + * false outside [0, count)). It stays true during the retry and clears on + * success; after three failed attempts it stays true. The engine retries + * after retryAfterMs. A model index given another pickId loads again with + * a fresh attempt budget. Destroy cancels retries and both queries return false. * @property {() => void} destroy * * @typedef {object} GlobeMovers diff --git a/src/globe/movers.js b/src/globe/movers.js index 88d325615..8fb122840 100644 --- a/src/globe/movers.js +++ b/src/globe/movers.js @@ -68,7 +68,7 @@ function atlasOf(atlas) { * @returns {{ id: string, kind: string, capacity: number, order: number, * depthTest: 'normal'|'always'|{ disableWithinM: number }, pickable: boolean, sharpen: boolean, * cellKeys: string[], sprite: object | null, point: object | null, - * model: object | null }} + * model: object | null, retryAfterMs: number }} */ export function normalizeMoverSetOptions(options) { if (!isObject(options)) fail('options must be an object'); @@ -89,6 +89,7 @@ export function normalizeMoverSetOptions(options) { kind, capacity, order, + retryAfterMs: nonNegative(options.retryAfterMs, 'retryAfterMs', 1500), depthTest: depthTestOf(options.depthTest), pickable: options.pickable === true, sharpen: options.sharpen === true, @@ -353,6 +354,9 @@ export function createMoverSet( const alive = () => { if (destroyed) throw new Error('Mover set was destroyed'); }; + const active = (index) => + index === undefined || + (Number.isInteger(index) && index >= 0 && index < store.count); return { // The set's id (createSet's `id`): surface exclusion and picks name it. id, @@ -372,9 +376,9 @@ export function createMoverSet( }, // Whether what it draws has loaded (a model's glTF, an atlas image); with // an index, what that instance draws. - ready: (index) => !destroyed && ready(index) === true, - // Whether that load failed for good (the set is not tried again). - failed: (index) => !destroyed && failed(index) === true, + ready: (index) => !destroyed && active(index) && ready(index) === true, + // The latest attempt failed; the engine owns the bounded retry. + failed: (index) => !destroyed && active(index) && failed(index) === true, destroy() { if (destroyed) return; destroyed = true; @@ -385,6 +389,39 @@ export function createMoverSet( }; } +/** A load gets three attempts, including the first, until cancelled. */ +export function createMoverRetry(retryAfterMs, load) { + let attempts = 0; + let failed = false; + let timer = null; + let destroyed = false; + const attempt = () => { + timer = null; + if (!destroyed) load(++attempts); + }; + return { + start() { + clearTimeout(timer); + attempts = 0; + failed = false; + attempt(); + }, + fail() { + if (destroyed) return; + failed = true; + if (attempts < 3) timer = setTimeout(attempt, retryAfterMs); + }, + succeed() { + failed = false; + }, + failed: () => failed, + destroy() { + destroyed = true; + clearTimeout(timer); + }, + }; +} + /** * engine.movers over an engine's own drawing: `build(options, store)` makes * the engine part of one set ({ commit, destroy }, both called with the diff --git a/src/globe/movers.test.mjs b/src/globe/movers.test.mjs index 40cd8f147..c07988fd2 100644 --- a/src/globe/movers.test.mjs +++ b/src/globe/movers.test.mjs @@ -1,5 +1,6 @@ import assert from 'node:assert/strict'; import test from 'node:test'; +import { createFakeEngine } from './testing/fakeEngine.js'; import { MOVER_KINDS, createMoverRegistry, @@ -23,6 +24,7 @@ test('set options are normalised with the contract defaults', () => { }); assert.equal(sprite.id, 'flights'); assert.equal(sprite.order, 0); + assert.equal(sprite.retryAfterMs, 1500); assert.equal(sprite.depthTest, 'normal'); assert.equal(sprite.pickable, false); assert.deepEqual(sprite.cellKeys, ['jet', 'heli']); @@ -292,3 +294,87 @@ test('native-stage sprite fields opt into sharpening explicitly and reject model TypeError, ); }); + +test('retry delay rejects invalid values and accepts zero', () => { + const options = { + id: 'retry', + kind: 'model', + capacity: 1, + model: { url: '/a.glb' }, + }; + for (const retryAfterMs of [-1, NaN, Infinity, '1500']) + assert.throws( + () => normalizeMoverSetOptions({ ...options, retryAfterMs }), + /retryAfterMs/, + ); + assert.equal( + normalizeMoverSetOptions({ ...options, retryAfterMs: 0 }).retryAfterMs, + 0, + ); +}); + +for (const kind of ['cesium', 'navara']) + test(`${kind} fake models retry twice, keep failure until success and reset for a new identity`, (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const engine = createFakeEngine({ kind }); + t.after(() => engine.destroy()); + const attempts = []; + engine.fake.script('moverFailed', (id, index, attempt) => { + attempts.push(attempt); + return attempt < 3; + }); + const set = engine.movers.createSet({ + id: 'model', + kind: 'model', + capacity: 2, + model: { url: '/a.glb' }, + }); + set.set(0, { pickId: 'a' }); + set.setCount(1); + set.commit(); + assert.equal(set.failed(), true); + assert.equal(set.failed(1), false); + assert.equal(set.ready(1), false); + t.mock.timers.tick(1499); + assert.deepEqual(attempts, [1]); + t.mock.timers.tick(1); + assert.equal(set.failed(0), true); + t.mock.timers.tick(1500); + assert.equal(set.failed(), false); + assert.equal(set.ready(0), true); + assert.deepEqual(attempts, [1, 2, 3]); + set.set(0, { pickId: 'b' }); + set.commit(); + assert.equal(set.failed(0), true); + set.destroy(); + t.mock.timers.tick(10000); + assert.deepEqual(attempts, [1, 2, 3, 1]); + assert.equal(set.ready(), false); + assert.equal(set.failed(), false); + }); + +test('fake model permanent failure stops after three attempts', (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const engine = createFakeEngine(); + t.after(() => engine.destroy()); + let attempts = 0; + engine.fake.script('moverFailed', () => { + attempts++; + return true; + }); + const set = engine.movers.createSet({ + id: 'model', + kind: 'model', + capacity: 1, + retryAfterMs: 10, + model: { url: '/a.glb' }, + }); + set.setCount(1); + set.commit(); + t.mock.timers.tick(10); + t.mock.timers.tick(10); + t.mock.timers.tick(1000); + assert.equal(attempts, 3); + assert.equal(set.failed(), true); + assert.equal(set.ready(), false); +}); diff --git a/src/globe/navara/movers/index.js b/src/globe/navara/movers/index.js index daba85972..c8a153db0 100644 --- a/src/globe/navara/movers/index.js +++ b/src/globe/navara/movers/index.js @@ -51,14 +51,22 @@ class GevMoverSetDesc extends withFrameUpdates(MeshDesc) { const group = new Group(); group.frustumCulled = false; // Meshes arrive with the glTF; they join the group (and the pick pass). - mover.group.ready.then(() => { + mover.group.onLoad = () => { if (this.destroyedByGev) return; + for (const { key, wrapper } of this.wrappers) { + this.ctx.unregisterPickableMesh(key); + batchIds.release(wrapper); + } + this.wrappers.length = 0; + this.gev.clearPickable(); + group.clear(); mover.group.meshes.forEach((mesh, index) => { group.add(mesh); register(mover.group.pickMeshes[index], `${this.id}#${index}`); }); this.view.forceUpdate(); - }); + }; + if (mover.group.meshes.length) mover.group.onLoad(); return group; } @@ -68,6 +76,7 @@ class GevMoverSetDesc extends withFrameUpdates(MeshDesc) { onDestroy() { this.destroyedByGev = true; + this.gev.mover.group.onLoad = null; for (const { key } of this.wrappers) this.ctx.unregisterPickableMesh(key); this.gev.mover.dispose(); for (const { wrapper } of this.wrappers) this.gev.batchIds.release(wrapper); @@ -163,6 +172,7 @@ export function createNavaraMovers(view, { overlay, aces = false }) { kind: options.kind, pickable: options.pickable, batchIds, + clearPickable: () => pickIndex.remove(options.id), onPickable: (wrapper) => pickIndex.add(options.id, wrapper, (index) => store.pickIds[index]), }, diff --git a/src/globe/navara/movers/modelMover.js b/src/globe/navara/movers/modelMover.js index 7473b2a81..f83eb1276 100644 --- a/src/globe/navara/movers/modelMover.js +++ b/src/globe/navara/movers/modelMover.js @@ -1,4 +1,4 @@ -// The three side of a Navara model set: the glTF loaded once, one +// The three side of a Navara model set: the glTF shared by the set, one // InstancedMesh per mesh in it (all instances share the model's meshes), // each instance placed by position, orientation and scale relative to an // origin near the camera. Cesium's minimumPixelSize is kept on the CPU once @@ -17,6 +17,7 @@ import { Sphere, Vector3, } from 'three'; +import { createMoverRetry } from '../../movers.js'; import { footprintPerDepth, originFor } from './pack.js'; import { createModelTint } from './modelTint.js'; @@ -188,94 +189,133 @@ export function createModelMover( for (const { mesh, tint } of parts) { if (tint) { tint.count = store.count; - tint.visible = store.count > 0; + tint.visible = loaded && store.count > 0; } mesh.count = store.count; - mesh.visible = store.count > 0; + mesh.visible = loaded && store.count > 0; mesh.instanceMatrix.needsUpdate = true; } for (const attribute of colorAttributes) attribute.needsUpdate = true; syncTransparency(); }; - const group = { meshes: [], pickMeshes: [], ready: null }; - let failed = false; - const asset = load(model.url).catch(() => { - // A missing asset keeps the layer's glyph fallback. Do not leak the - // rejected loader promise (or its URL) through descriptor creation. - if (!disposed) { - failed = true; - console.warn('[globe] Model asset unavailable'); + const group = { meshes: [], pickMeshes: [], ready: null, onLoad: null }; + const pickIds = new Map(); + const current = (index) => pickIds.get(index) === store.pickIds[index]; + let generation = 0; + let finish; + group.ready = new Promise((resolve) => (finish = resolve)); + const releaseParts = () => { + for (const { mesh, tint } of parts) { + if (tint) { + overlay.remove(tint); + tint.material.dispose(); + tint.dispose(); + } + mesh.removeFromParent(); + mesh.geometry.dispose(); + mesh.material.dispose(); + mesh.dispose?.(); } - return null; - }); - group.ready = asset.then((gltf) => { - if (!gltf) return; - const root = gltf.scene; - root.traverse((child) => { + parts.length = materials.length = colorAttributes.length = 0; + group.meshes.length = group.pickMeshes.length = 0; + disposeTextures(); + }; + const releaseAsset = (gltf, releaseTexture) => { + gltf.scene.traverse((child) => { + child.geometry?.dispose(); for (const material of Array.isArray(child.material) ? child.material - : [child.material]) - if (material) + : [child.material]) { + if (!material) continue; + if (releaseTexture) for (const value of Object.values(material)) - if (value?.isTexture) textures.add(value); + if (value?.isTexture) value.dispose(); + material.dispose(); + } }); - if (disposed) { - disposeTextures(); - return; - } - root.updateMatrixWorld(true); - const sphere = new Box3() - .setFromObject(root) - .getBoundingSphere(new Sphere()); - radius = Math.max(sphere.radius, 1e-3); - root.traverse((child) => { - if (!child.isMesh) return; - let material = child.material; - if (model.unlit) - material = new MeshBasicMaterial({ - map: material.map ?? null, - color: material.color, + }; + const retry = createMoverRetry(options.retryAfterMs, () => { + const version = generation; + load(model.url) + .then((gltf) => { + if (disposed || version !== generation) { + releaseAsset(gltf, true); + return; + } + releaseParts(); + const root = gltf.scene; + root.traverse((child) => { + for (const material of Array.isArray(child.material) + ? child.material + : [child.material]) + if (material) + for (const value of Object.values(material)) + if (value?.isTexture) textures.add(value); }); - else material = material.clone(); - blendInstanceColor(material, uniforms, Boolean(overlay)); - const geometry = child.geometry.clone(); - const color = new InstancedBufferAttribute( - new Float32Array(capacity * 4).fill(1), - 4, - ); - color.setUsage(DynamicDrawUsage); - geometry.setAttribute('gevColor', color); - colorAttributes.push(color); - const mesh = new InstancedMesh(geometry, material, capacity); - mesh.instanceMatrix.setUsage(DynamicDrawUsage); - mesh.frustumCulled = false; - mesh.renderOrder = options.order; - mesh.count = 0; - mesh.visible = false; - const tint = overlay - ? createModelTint(mesh, uniforms, overlay.uniforms) - : null; - if (tint) { - // Still present in the scene depth and GPU pick registry. Only colour - // moves to the display pass, where it is toned before raster resolve. - material.colorWrite = false; - overlay.add(tint); - } - materials.push(material); - parts.push({ mesh, tint, local: child.matrixWorld.clone() }); - group.meshes.push(mesh); - group.pickMeshes.push(tint ?? mesh); - }); - loaded = true; - if (origin) - for (const { mesh, tint } of parts) { - mesh.position.set(origin.x, origin.y, origin.z); - tint?.position.copy(mesh.position); - } - write(0, store.count); - onLoad(group.meshes); + root.updateMatrixWorld(true); + const sphere = new Box3() + .setFromObject(root) + .getBoundingSphere(new Sphere()); + radius = Math.max(sphere.radius, 1e-3); + root.traverse((child) => { + if (!child.isMesh) return; + let material = child.material; + if (model.unlit) + material = new MeshBasicMaterial({ + map: material.map ?? null, + color: material.color, + }); + else material = material.clone(); + blendInstanceColor(material, uniforms, Boolean(overlay)); + const geometry = child.geometry.clone(); + const color = new InstancedBufferAttribute( + new Float32Array(capacity * 4).fill(1), + 4, + ); + color.setUsage(DynamicDrawUsage); + geometry.setAttribute('gevColor', color); + colorAttributes.push(color); + const mesh = new InstancedMesh(geometry, material, capacity); + mesh.instanceMatrix.setUsage(DynamicDrawUsage); + mesh.frustumCulled = false; + mesh.renderOrder = options.order; + mesh.count = 0; + mesh.visible = false; + const tint = overlay + ? createModelTint(mesh, uniforms, overlay.uniforms) + : null; + if (tint) { + // Still present in the scene depth and GPU pick registry. Only colour + // moves to the display pass, where it is toned before raster resolve. + material.colorWrite = false; + overlay.add(tint); + } + materials.push(material); + parts.push({ mesh, tint, local: child.matrixWorld.clone() }); + group.meshes.push(mesh); + group.pickMeshes.push(tint ?? mesh); + }); + loaded = true; + retry.succeed(); + releaseAsset(gltf, false); + if (origin) + for (const { mesh, tint } of parts) { + mesh.position.set(origin.x, origin.y, origin.z); + tint?.position.copy(mesh.position); + } + write(0, store.count); + group.onLoad?.(); + finish(); + onLoad(group.meshes); + }) + .catch(() => { + if (disposed || version !== generation) return; + retry.fail(); + console.warn('[globe] Model asset unavailable'); + }); }); + retry.start(); const placeAt = (next) => { origin = { x: next.x, y: next.y, z: next.z }; @@ -290,8 +330,19 @@ export function createModelMover( return { group, uniforms, - ready: () => loaded, - failed: () => failed, + ready(index) { + if (disposed) return false; + if (index !== undefined) return loaded && current(index); + for (let at = 0; at < store.count; at++) + if (!loaded || !current(at)) return false; + return true; + }, + failed(index) { + if (disposed || !retry.failed()) return false; + if (index !== undefined) return current(index); + for (let at = 0; at < store.count; at++) if (current(at)) return true; + return false; + }, origin: () => origin, /** * The drawn radius (px) of an instance at its last write, or null when @@ -303,6 +354,17 @@ export function createModelMover( ? drawnPx[index] / 2 : null, commit(range) { + let changed = false; + for (let index = 0; index < store.count; index++) { + if (pickIds.has(index) ? !current(index) : retry.failed()) + changed = true; + pickIds.set(index, store.pickIds[index]); + } + if (changed) { + generation++; + loaded = false; + retry.start(); + } if (!origin && store.count > 0) placeAt({ x: store.positions[0], @@ -313,11 +375,11 @@ export function createModelMover( for (const { mesh, tint } of parts) { if (tint) { tint.count = store.count; - tint.visible = store.count > 0; + tint.visible = loaded && store.count > 0; } mesh.count = store.count; // Retained empty banks must also skip three's material/uniform work. - mesh.visible = store.count > 0; + mesh.visible = loaded && store.count > 0; } }, /** Each drawn frame: pixel limits need every instance rewritten. */ @@ -340,17 +402,9 @@ export function createModelMover( dispose() { if (disposed) return; disposed = true; - for (const { mesh, tint } of parts) { - if (tint) { - overlay.remove(tint); - tint.material.dispose(); - tint.dispose(); - } - mesh.geometry.dispose(); - mesh.material.dispose(); - mesh.dispose?.(); - } - disposeTextures(); + retry.destroy(); + releaseParts(); + finish(); }, }; } diff --git a/src/globe/navara/movers/modelMover.test.mjs b/src/globe/navara/movers/modelMover.test.mjs index 3a837b715..249d46c76 100644 --- a/src/globe/navara/movers/modelMover.test.mjs +++ b/src/globe/navara/movers/modelMover.test.mjs @@ -197,12 +197,14 @@ test('a model removed before its load completes releases the arriving textures', test('a failed model load stays not-ready without an unhandled rejection or leaking the URL', async (t) => { const warnings = []; t.mock.method(console, 'warn', (message) => warnings.push(message)); - const mover = createModelMover(options, createMoverStore(options), { + const store = createMoverStore(options); + store.setCount(1); + const mover = createModelMover(options, store, { load: async () => { throw new Error('unavailable asset URL'); }, }); - await mover.group.ready; + await new Promise(setImmediate); assert.equal(mover.ready(), false); assert.equal(mover.failed(), true); assert.equal(mover.group.meshes.length, 0); @@ -333,3 +335,99 @@ test('the display pass preserves unlit model colour without a lighting tone curv ); mover.dispose(); }); + +for (const failures of [1, 2, 3]) + test(`model retries ${failures} failures with one budget and releases its timer`, async (t) => { + t.mock.method(console, 'warn', () => {}); + t.mock.timers.enable({ apis: ['setTimeout'] }); + const store = createMoverStore(options); + let loads = 0; + let added = 0; + const mover = createModelMover({ ...options, retryAfterMs: 20 }, store, { + load: async () => { + if (++loads <= failures) throw new Error('503'); + return fakeGltf(); + }, + }); + mover.group.onLoad = () => added++; + t.after(() => mover.dispose()); + store.set(0, { position: { x: 6378137, y: 0, z: 0 }, pickId: 'a' }); + store.setCount(1); + mover.commit(store.dirty()); + await new Promise(setImmediate); + assert.equal(mover.failed(0), true); + for (let attempt = 0; attempt < 2; attempt++) { + t.mock.timers.tick(19); + assert.equal(loads, attempt + 1); + t.mock.timers.tick(1); + await new Promise(setImmediate); + if (mover.ready()) break; + } + assert.equal(loads, Math.min(failures + 1, 3)); + assert.equal(mover.ready(), failures < 3); + assert.equal(mover.failed(), failures === 3); + t.mock.timers.tick(10000); + assert.equal(loads, Math.min(failures + 1, 3)); + store.set(0, { pickId: 'b' }); + assert.equal( + mover.failed(), + false, + 'the replacement has no failed attempt', + ); + mover.commit(store.dirty()); + assert.equal(mover.failed(0), false); + await new Promise(setImmediate); + assert.equal(mover.ready(0), true); + assert.equal(added, failures < 3 ? 2 : 1); + assert.equal( + mover.group.meshes.length, + 1, + 'replacement does not accumulate meshes', + ); + mover.dispose(); + t.mock.timers.tick(10000); + assert.equal(loads, Math.min(failures + 1, 3) + 1); + }); + +test('a pending obsolete identity releases its asset without overwriting the replacement', async (t) => { + const store = createMoverStore(options); + const pending = []; + const mover = createModelMover(options, store, { + load: () => new Promise((resolve) => pending.push(resolve)), + }); + t.after(() => mover.dispose()); + store.set(0, { pickId: 'a' }); + store.setCount(1); + mover.commit(store.dirty()); + store.set(0, { pickId: 'b' }); + mover.commit(store.dirty()); + const stale = fakeGltf(); + let released = 0; + stale.scene.children[0].geometry.addEventListener( + 'dispose', + () => released++, + ); + pending[0](stale); + await new Promise(setImmediate); + assert.equal(released, 1); + assert.equal(mover.ready(), false); + pending[1](fakeGltf()); + await mover.group.ready; + assert.equal(mover.ready(), true); +}); + +test('destroying a failed model cancels its pending retry', async (t) => { + t.mock.method(console, 'warn', () => {}); + t.mock.timers.enable({ apis: ['setTimeout'] }); + let loads = 0; + const mover = createModelMover(options, createMoverStore(options), { + load: async () => { + loads++; + throw new Error('503'); + }, + }); + await new Promise(setImmediate); + mover.dispose(); + t.mock.timers.tick(10000); + assert.equal(loads, 1); +}); diff --git a/src/globe/navara/movers/moverMesh.js b/src/globe/navara/movers/moverMesh.js index cb32f21fb..9abc3a654 100644 --- a/src/globe/navara/movers/moverMesh.js +++ b/src/globe/navara/movers/moverMesh.js @@ -31,6 +31,7 @@ import { OVERLAY_DEPTH_VERTEX, } from '../overlayDepth.js'; import { DETACHED_SONAR, SONAR_VERTEX } from '../sonar.js'; +import { createMoverRetry } from '../../movers.js'; const VERTEX_HEAD = /* glsl */ ` attribute vec4 gevStyle; @@ -317,22 +318,9 @@ export function createMoverMesh( }; // A canvas atlas is there at once; a URL one once its image loads. let textureReady = point || Boolean(options.sprite.atlas.canvas); - let textureFailed = false; const encodedAtlas = !point && options.sharpen === true; const material = new MeshBasicMaterial({ - map: point - ? null - : atlasTexture( - options.sprite.atlas, - encodedAtlas, - () => { - textureReady = true; - onTextureLoad(); - }, - () => { - textureFailed = true; - }, - ), + map: null, transparent: true, alphaTest: point || encodedAtlas ? 0.005 : 0.02, // Drawn after tone mapping, with no depth buffer: the shader tests the @@ -341,6 +329,21 @@ export function createMoverMesh( depthWrite: false, side: FrontSide, }); + const retry = createMoverRetry(options.retryAfterMs, () => { + material.map?.dispose(); + material.map = atlasTexture( + options.sprite.atlas, + encodedAtlas, + () => { + textureReady = true; + retry.succeed(); + onTextureLoad(); + }, + () => retry.fail(), + ); + material.needsUpdate = true; + }); + if (!point) retry.start(); material.onBeforeCompile = (shader) => patchMoverShader(shader, uniforms, { point, @@ -394,7 +397,7 @@ export function createMoverMesh( mesh, uniforms, ready: () => textureReady, - failed: () => textureFailed, + failed: retry.failed, origin: () => origin, /** Writes the changed instances; the origin starts at the first one. */ commit(range) { @@ -455,6 +458,7 @@ export function createMoverMesh( return true; }, dispose() { + retry.destroy(); geometry.dispose(); material.map?.dispose(); material.dispose(); diff --git a/src/globe/navara/movers/moverMesh.test.mjs b/src/globe/navara/movers/moverMesh.test.mjs index 46251f42d..7df6148c4 100644 --- a/src/globe/navara/movers/moverMesh.test.mjs +++ b/src/globe/navara/movers/moverMesh.test.mjs @@ -348,3 +348,51 @@ test('native-stage glow atlas filters encoded colors and keeps the native faint- glow.dispose(); ordinary.dispose(); }); + +test('a URL atlas retries on fresh images, stops after three failures and cancels on dispose', (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const previous = globalThis.Image; + const images = []; + globalThis.Image = class { + constructor() { + images.push(this); + } + }; + t.after(() => { + if (previous) globalThis.Image = previous; + else delete globalThis.Image; + }); + const options = normalizeMoverSetOptions({ + ...spriteOptions, + retryAfterMs: 20, + sprite: { + sizePx: 8, + atlas: { url: '/atlas.png', cells: { a: [0, 0, 1, 1] } }, + }, + }); + const mover = createMoverMesh(options, createMoverStore(options)); + images[0].onerror(); + assert.equal(mover.failed(), true); + t.mock.timers.tick(19); + assert.equal(images.length, 1); + t.mock.timers.tick(1); + assert.equal(mover.failed(), true); + images[1].onload(); + assert.equal(mover.ready(), true); + assert.equal(mover.failed(), false); + assert.equal(images[0].onload, null); + mover.dispose(); + const permanent = createMoverMesh(options, createMoverStore(options)); + for (let i = 2; i < 5; i++) { + images[i].onerror(); + t.mock.timers.tick(20); + } + assert.equal(images.length, 5); + assert.equal(permanent.failed(), true); + permanent.dispose(); + const cancelled = createMoverMesh(options, createMoverStore(options)); + images[5].onerror(); + cancelled.dispose(); + t.mock.timers.tick(10000); + assert.equal(images.length, 6); +}); diff --git a/src/globe/testing/contractSuite/movers.mjs b/src/globe/testing/contractSuite/movers.mjs index 6048d9887..f460a735f 100644 --- a/src/globe/testing/contractSuite/movers.mjs +++ b/src/globe/testing/contractSuite/movers.mjs @@ -498,43 +498,132 @@ export function moverChecks( ]; } -export function failedModelChecks(engine, { settle, until, fixtures }) { +export function failedModelChecks( + engine, + { until, fixtures, nadirView, readRegion }, +) { return [ [ - 'MoverSet.failed: a failed model stays unready and reports its failure scope', + 'movers.createSet: failed model retries, recovers and stops after three attempts', async () => { + await nadirView(1e6); const id = 'contract-failed-model'; + const url = '/models/airplane.glb?contract-retry'; + let failing = true; + let attempts = 0; + const fetch = globalThis.fetch; + const open = globalThis.XMLHttpRequest?.prototype.open; + // Route this model alone to the runner's missing asset while failing. + // The next attempt uses the real GLB without replacing the mover set. + const route = (value) => { + if (!String(value).endsWith(url)) return value; + attempts++; + return failing ? '/__contract_missing_model.glb' : value; + }; + if (fixtures?.failModel) fixtures.failModel(id, true); + else { + globalThis.fetch = (input, init) => { + const value = route(input?.url ?? input); + return fetch(value === (input?.url ?? input) ? input : value, init); + }; + globalThis.XMLHttpRequest.prototype.open = function ( + method, + value, + ...args + ) { + return open.call(this, method, route(value), ...args); + }; + } const set = engine.movers.createSet({ id, kind: 'model', capacity: 2, - model: { url: '/__contract_missing_model.glb' }, + retryAfterMs: 100, + model: { + url, + minimumPixelSize: 80, + colorBlend: { mode: 'replace', amount: 1 }, + }, }); + const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + const fail = (value) => { + failing = value; + fixtures?.failModel?.(id, value); + }; try { - set.set(0, { - position: cartesian3.fromDegrees(20, 10, 1000), - pickId: 'failed', - }); + ensure( + set.ready() && !set.failed(), + 'empty model set reports a load', + ); + const position = cartesian3.fromDegrees(20, 10, 50_000); + set.set(0, { position, color: [1, 0, 1, 1], pickId: 'first' }); set.setCount(1); set.commit(); - fixtures?.failModel?.(id, true); await until(() => set.failed(0), 'failed model'); - ensure(set.failed() === true, 'set did not report the failure'); ensure( - set.ready(0) === false && set.ready() === false, + set.failed() && !set.ready() && !set.ready(0), 'failed model is ready', ); ensure( - set.failed(1) === (engine.kind === 'navara'), - 'failure scope changed', + !set.failed(1) && !set.ready(1), + 'inactive instance reports a load', ); - await settle(3); + fail(false); + await wait(120); + await until(() => set.ready(0), 'model ready after engine retry'); ensure( - set.failed(0) && !set.ready(0), - 'failed model silently recovered', + set.ready() && !set.failed(), + 'retry did not clear the failure', ); + if (readRegion) { + const screen = engine.projector.worldToWindow(position); + await until(() => { + const data = readRegion( + Math.round(screen.x) - 50, + Math.round(screen.y) - 50, + 100, + 100, + ); + let pixels = 0; + for (let i = 0; i < data.length; i += 4) + if ( + Math.min(data[i], data[i + 2]) - data[i + 1] > 40 && + data[i] > 60 + ) + pixels++; + return pixels > 150; + }, 'retried model drawn'); + } + // A new identity reloads even a previously healthy index. + fail(true); + attempts = 0; + set.set(0, { pickId: 'second' }); + set.commit(); + await until(() => set.failed(0), 'replacement model failure'); + await wait(350); + ensure(set.failed() && !set.ready(), 'permanent failure disappeared'); + if (!fixtures?.failModel) + ensure(attempts === 3, `expected 3 attempts, got ${attempts}`); + fail(false); + await wait(150); + ensure(set.failed(0) && !set.ready(0), 'exhausted model retried'); + set.set(0, { pickId: 'third' }); + ensure( + !set.failed(0) && !set.failed(), + 'new identity inherited failure', + ); + set.commit(); + await until( + () => set.ready(0), + 'replacement after exhausted attempts', + ); + ensure(!set.failed(), 'replacement inherited exhausted failure'); } finally { fixtures?.failModel?.(id, false); + if (!fixtures?.failModel) { + globalThis.fetch = fetch; + globalThis.XMLHttpRequest.prototype.open = open; + } set.destroy(); } ensure( diff --git a/src/globe/testing/fakeEngine.js b/src/globe/testing/fakeEngine.js index 5cafbb937..78badf667 100644 --- a/src/globe/testing/fakeEngine.js +++ b/src/globe/testing/fakeEngine.js @@ -54,7 +54,7 @@ import { createGroundPicks, createGroundRegistry } from '../ground.js'; import { createMeshRegistry, meshRayDistance } from '../meshes.js'; import { createImageryRegistry } from '../imagery.js'; import { createLineRegistry } from '../lines.js'; -import { createMoverRegistry } from '../movers.js'; +import { createMoverRegistry, createMoverRetry } from '../movers.js'; import { createCpuPicker } from '../cpuPicks.js'; import { createFollowController } from '../follow.js'; import { normalizeSurfaceOptions } from '../surfaceDetail.js'; @@ -257,19 +257,63 @@ export function createFakeEngine(options = {}) { }; }); // Movers keep their store for tests; commit counts as a render request. - const moverRegistry = createMoverRegistry((options) => { + const moverRegistry = createMoverRegistry((options, store) => { + const loads = new Map(); + const ensure = (index) => { + const pickId = store.pickIds[index]; + let entry = loads.get(index); + if (entry && entry.pickId === pickId) return entry; + entry?.retry.destroy(); + entry = { pickId, ready: false, retry: null }; + entry.retry = createMoverRetry(options.retryAfterMs, (attempt) => { + // A script can fail the first N attempts: (id, index, attempt) => attempt <= N. + if (scripted('moverFailed', [options.id, index, attempt], () => false)) + entry.retry.fail(); + else { + entry.ready = true; + entry.retry.succeed(); + } + fake.renderRequests++; + }); + loads.set(index, entry); + entry.retry.start(); + return entry; + }; + if (options.kind === 'sprite') ensure(undefined); + const current = (index) => { + const entry = loads.get(options.kind === 'sprite' ? undefined : index); + return options.kind === 'sprite' || entry?.pickId === store.pickIds[index] + ? entry + : null; + }; const part = { commits: 0, commit() { + if (options.kind === 'model') + for (let index = 0; index < store.count; index++) ensure(index); part.commits++; fake.renderRequests++; }, - ready: (index) => - !scripted('moverFailed', [options.id, index], () => false), - // Scripted as 'moverFailed' (set id, index) for a load that fails. - failed: (index) => - scripted('moverFailed', [options.id, index], () => false), - destroy() {}, + ready(index) { + if (options.kind === 'point') return true; + if (options.kind === 'sprite') return current(index).ready; + if (index !== undefined) return current(index)?.ready === true; + for (let at = 0; at < store.count; at++) + if (!current(at)?.ready) return false; + return true; + }, + failed(index) { + if (options.kind === 'point') return false; + if (options.kind === 'sprite') return current(index).retry.failed(); + if (index !== undefined) return current(index)?.retry.failed() === true; + for (let at = 0; at < store.count; at++) + if (current(at)?.retry.failed()) return true; + return false; + }, + destroy() { + for (const entry of loads.values()) entry.retry.destroy(); + loads.clear(); + }, }; return part; }); diff --git a/src/layers/aircraft/render/contract.js b/src/layers/aircraft/render/contract.js index 7532d902e..7f2de8f25 100644 --- a/src/layers/aircraft/render/contract.js +++ b/src/layers/aircraft/render/contract.js @@ -19,8 +19,8 @@ // bookkeeping fields on a handle (prefixed _gev). // ModelHandle (models.load*) — one glTF instance: // show, ready (read-only: placed, and its model loaded), failed -// (read-only: its model would not load; a tracked one is loaded again -// after models.remove), scale, +// (read-only: the latest load of its model failed; the engine retries +// it, and ready turns true once a retry loads), scale, // computedScale (read-only: scale after minimumPixelSize), color, id, // modelMatrix (16 numbers, column-major, the model frame: written by // models.setPose and read for anchors). diff --git a/src/layers/aircraft/render/display.js b/src/layers/aircraft/render/display.js index 8c3c644fd..d03eb56ae 100644 --- a/src/layers/aircraft/render/display.js +++ b/src/layers/aircraft/render/display.js @@ -29,10 +29,6 @@ import { SPRITE_LAYER_ORDER } from '../../../data/spriteOrder.js'; import { createMoverBank, writeFields } from '../../shared/render/moverBank.js'; const BASE_SIZE_PX = 20; -// A failed glTF is fetched again no sooner than this (the layers' tracked -// model backoff), not on the next fleet tick. -const MODEL_RETRY_BACKOFF_MS = 1500; - /** * The model frame → the glTF's own axes (the movers' orientation): Cesium's * glTF axis correction, Y_UP_TO_Z_UP × Z_UP_TO_X_UP (data/modelVisualAnchor.js), @@ -460,7 +456,6 @@ export function createAircraftDisplay( // --- models ---------------------------------------------------------------- let modelsVisible = true; const modelBanks = new Map(); - const modelRetryAt = new Map(); const scratchMatrix3 = geo.matrix3.create(); const scratchCorrected = geo.matrix4.create(); const scratchQuaternion = geo.quaternion.create(); @@ -603,16 +598,7 @@ export function createAircraftDisplay( } } - // A failed glTF needs a fresh set, created only when a retry is admitted. - const dropModel = (model) => { - const bank = model?.gevBank; - const retry = model?.failed; - bank?.remove(model); - if (retry) { - modelRetryAt.set(model.spec.url, Date.now() + MODEL_RETRY_BACKOFF_MS); - bank.rebuild(); - } - }; + const dropModel = (model) => model?.gevBank?.remove(model); const models = { live: () => attached && !destroyed, @@ -620,10 +606,7 @@ export function createAircraftDisplay( modelsVisible = visible; for (const bank of modelBanks.values()) bank.setVisible(visible); }, - loadFleet: (spec) => - Date.now() < (modelRetryAt.get(spec.url) ?? 0) - ? Promise.reject(new Error('model retry backoff')) - : Promise.resolve(new ModelHandle(spec)), + loadFleet: (spec) => Promise.resolve(new ModelHandle(spec)), loadTracked: (spec) => Promise.resolve(new ModelHandle(spec)), add(model) { modelBankFor(model).add(model); diff --git a/src/layers/aircraft/trackedModelFailure.test.mjs b/src/layers/aircraft/trackedModelFailure.test.mjs index 1038f0b94..c4bf570c5 100644 --- a/src/layers/aircraft/trackedModelFailure.test.mjs +++ b/src/layers/aircraft/trackedModelFailure.test.mjs @@ -1,8 +1,3 @@ -// An aircraft glTF that the engine fails to load: the glyph stays the -// visual, the load is tried again after the backoff in a new mover set (a set -// does not load a failed glTF again), and a tracked contact gives up after its -// failure budget. A failed fleet model gives its slot back at once, and a -// later aircraft of its class loads the glTF again after the backoff. import test from 'node:test'; import assert from 'node:assert/strict'; import * as Cesium from 'cesium'; @@ -23,6 +18,7 @@ import militaryFlightsLayer, { _trackedBillboardColorForTest as militaryTrackedBillboardColor, } from '../../data/militaryFlights.js'; import { clearTr3bRegistry } from '../../data/tr3bRegistry.js'; +import { applyAircraftModelTreatment } from '../../data/aircraftRecession.js'; import { recordingAircraftRenderer } from '../../testSupport/aircraftRecordingRenderer.mjs'; import { fleetTickFixture } from '../../testSupport/fleetTickFixture.mjs'; import { createFlightState as civilState } from '../flights/state.js'; @@ -42,38 +38,60 @@ const drawAtlas = async () => ({ cells: layout.cells, }); -test('a failed tracked model loads again in a new set once it is removed', async (t) => { +const settle = () => new Promise(setImmediate); + +async function displayFixture(t, failures = 1) { + t.mock.timers.enable({ apis: ['setTimeout'] }); const engine = createFakeEngine(); - t.after(() => engine.destroy()); + const attempts = []; + engine.fake.script('moverFailed', (id, index, attempt) => { + if (!id.includes('-models-')) return false; + attempts.push({ index, attempt }); + return index === 0 && attempt <= failures; + }); const display = createAircraftDisplay(engine, { layerId: 'flights', headingOffsetDeg: 180, drawAtlas, }); - t.after(() => display.destroy()); + t.after(() => { + display.destroy(); + engine.destroy(); + }); await display.whenReady(); display.fleet.attach(); - const failing = new Set(); - engine.fake.script('moverFailed', (id) => failing.has(id)); - const spec = { url: 'plane.glb', minimumPixelSize: 64, scale: 1, id: 'abc' }; + return { display, engine, attempts }; +} +const spec = (id) => ({ + url: 'airliner.glb', + minimumPixelSize: 64, + scale: 1, + id, +}); +const position = geo.cartesian3.fromDegrees(1, 0, 1000); - const first = await display.models.loadTracked(spec); - display.models.add(first); - const failedSet = first.gevBank.set.id; - assert.equal(first.failed, false); - failing.add(failedSet); - assert.equal(first.failed, true, 'the engine reports the failed load'); - display.models.remove(first); - assert.equal( - engine.movers.list().filter((set) => set.kind === 'model').length, - 0, - 'removal does not start another load', +async function addModel(display, id, tracked = false) { + const model = await display.models[tracked ? 'loadTracked' : 'loadFleet']( + spec(id), ); + display.models.add(model); + display.models.setPose(model, position, 0); + await settle(); + return model; +} - const retry = await display.models.loadTracked(spec); - display.models.add(retry); - assert.notEqual(retry.gevBank.set.id, failedSet, 'a new set loads it again'); - assert.equal(retry.failed, false); +test('a tracked model becomes ready in its original set after the engine retries', async (t) => { + const { display, attempts } = await displayFixture(t); + const model = await addModel(display, 'tracked', true); + const set = model.gevBank.set; + assert.equal(model.failed, true); + assert.equal(model.ready, false); + t.mock.timers.tick(1499); + assert.equal(attempts.length, 1); + t.mock.timers.tick(1); + assert.equal(model.ready, true); + assert.equal(model.failed, false); + assert.equal(model.gevBank.set, set); }); /** A viewer stub inside the tracked model regime. */ @@ -119,47 +137,37 @@ const LAYERS = [ }, ]; -for (const fixture of LAYERS) { - test(`${fixture.name}: a tracked model the engine fails to load is retried, then given up`, async () => { - clearTr3bRegistry(); - fixture.layer.setParams({ models3d: false }); - const realNow = Date.now; - const realWarn = console.warn; - let now = 1_000_000; - Date.now = () => now; - const warnings = []; - console.warn = (...args) => warnings.push(args.join(' ')); - let loads = 0; - const removed = []; - const viewer = viewerAt(30_000); - try { +for (const fixture of LAYERS) + for (const failures of [1, 3]) + test(`${fixture.name}: tracked fallback survives ${failures} failed attempts without replacing the handle`, async (t) => { + clearTr3bRegistry(); + fixture.layer.setParams({ models3d: false }); + const { engine, display, attempts } = await displayFixture(t, failures); + const viewer = viewerAt(30_000); + viewer.engine = engine; + let handles = 0; + const removed = []; const billboard = { - position: Cesium.Cartesian3.fromDegrees(-97.71, 30.21, 10_668), + position: geo.cartesian3.fromDegrees(-97.71, 30.21, 10_668), color: null, show: true, rotation: 0, }; fixture.seed({ renderer: () => - recordingAircraftRenderer(viewer.engine, { + recordingAircraftRenderer(engine, { view: { camera: viewer.camera, scene: viewer.scene }, fleet: { shown: true }, models: { - live: () => true, - loadTracked: async (spec) => { - loads++; - return { - spec, - failed: true, - ready: false, - show: false, - modelMatrix: geo.matrix4.create(), - }; + ...display.models, + loadTracked: (spec) => { + handles++; + return display.models.loadTracked(spec); + }, + remove: (model) => { + removed.push(model); + display.models.remove(model); }, - add() {}, - remove: (model) => removed.push(model), - destroy() {}, - setPose: () => geo.matrix4.create(), }, }), icao24: 'abc123', @@ -170,63 +178,49 @@ for (const fixture of LAYERS) { tracked: true, meta: { ...fixture.meta, onGround: false }, }); - for (let frame = 0; frame < 30; frame++) { + try { fixture.drive(); - await Promise.resolve(); - await Promise.resolve(); - now += 2000; // past the retry backoff + await settle(); + fixture.drive(); + assert.equal(fixture.billboardColor().alpha, 1); + for (let frame = 0; frame < 5; frame++) { + t.mock.timers.tick(1500); + fixture.drive(); + await settle(); + } + assert.equal(handles, 1); + assert.equal(removed.length, 0); + assert.equal(attempts.length, failures === 3 ? 3 : 2); + if (failures === 3) assert.equal(fixture.billboardColor().alpha, 1); + else + assert.equal( + fixture.billboardColor().alpha, + 0, + 'the recovered model owns the visual', + ); + } finally { + fixture.layer.stopTracking(); } - assert.ok(loads > 1, `the failed load is tried again (${loads})`); - assert.ok(loads <= 3, `within the failure budget (${loads})`); - assert.equal(removed.length, loads, 'each failed model leaves the scene'); - assert.equal(warnings.length, 1, 'the give-up is announced once'); - assert.equal( - fixture.billboardColor().alpha, - 1, - 'the tracked glyph stays the visual', - ); - } finally { - Date.now = realNow; - console.warn = realWarn; - fixture.layer.stopTracking(); - } - }); -} + }); -test('a failed fleet model does not condemn later aircraft of its class', async (t) => { - t.mock.timers.enable({ apis: ['Date'], now: 1000 }); - const engine = createFakeEngine(); - t.after(() => engine.destroy()); - const display = createAircraftDisplay(engine, { - layerId: 'flights', - headingOffsetDeg: 180, - drawAtlas, - }); - t.after(() => display.destroy()); - await display.whenReady(); - display.fleet.attach(); - const failing = new Set(); - engine.fake.script('moverFailed', (id) => failing.has(id)); - const spec = (id) => ({ - url: 'airliner.glb', - minimumPixelSize: 64, - scale: 1, - id, - }); - const first = await display.models.loadFleet(spec('a1')); - display.models.add(first); - failing.add(first.gevBank.set.id); // a 5xx or a network blip on the GLB +test('a failed fleet model does not reload healthy aircraft when another joins its class', async (t) => { + const { display, attempts } = await displayFixture(t); + const first = await addModel(display, 'a'); + const healthy = await addModel(display, 'b'); + const set = first.gevBank.set; + const next = await addModel(display, 'c'); assert.equal(first.failed, true); - display.models.remove(first); - await assert.rejects(display.models.loadFleet(spec('a2')), /retry/); + assert.equal(healthy.ready, true); + assert.equal(next.ready, true); + assert.equal(next.gevBank.set, set); t.mock.timers.tick(1500); - const next = await display.models.loadFleet(spec('a2')); - display.models.add(next); - assert.equal( - next.failed, - false, - 'the next airliner joined the failed set and never draws its model', - ); + assert.equal(first.ready, true); + assert.deepEqual(attempts, [ + { index: 0, attempt: 1 }, + { index: 1, attempt: 1 }, + { index: 2, attempt: 1 }, + { index: 0, attempt: 2 }, + ]); }); for (const [name, createFlightState, createRendering] of [ @@ -234,51 +228,52 @@ for (const [name, createFlightState, createRendering] of [ ['military', militaryState, militaryRendering], ]) for (const inView of [true, false]) - test(`${name}: the fleet tick frees a failed model's slot (${inView ? 'in view' : 'beyond the horizon'})`, () => { - const fleet = fleetTickFixture({ createFlightState, createRendering }); + test(`${name}: the fleet retains a failed model through recovery (${inView ? 'in view' : 'beyond the horizon'})`, async (t) => { + const { display } = await displayFixture(t); + const fleet = fleetTickFixture({ + createFlightState, + createRendering(options) { + options.services.recession.applyAircraftModelTreatment = + applyAircraftModelTreatment; + return createRendering(options); + }, + }); + Object.assign(fleet.flightState._renderer.models, display.models); if (inView) - fleet.camera.positionWC = Cesium.Cartesian3.fromDegrees(1, 0, 100); + fleet.camera.positionWC = geo.cartesian3.fromDegrees(1, 0, 100); fleet.flightState._models3dEnabled = true; - const model = { failed: true, ready: false, show: false }; + const model = await addModel(display, fleet.id); fleet.flightState._models.set(fleet.id, model); fleet.tick(); - assert.equal(fleet.flightState._models.has(fleet.id), false); - assert.deepEqual(fleet.removed, [model]); + assert.equal(fleet.flightState._models.get(fleet.id), model); + assert.equal(fleet.billboard.show, inView); + assert.equal(model.ready, false); + t.mock.timers.tick(1500); + fleet.tick(); + assert.equal(fleet.flightState._models.get(fleet.id), model); + assert.equal(model.ready, true); + assert.equal(model.show, inView); + if (inView) assert.equal(fleet.billboard.show, false); + assert.deepEqual(fleet.removed, []); }); -test('removing failures from a shared bank does not reload until the backoff ends', async (t) => { - t.mock.timers.enable({ apis: ['Date'], now: 1000 }); - const engine = createFakeEngine(); - t.after(() => engine.destroy()); - let builds = 0; - const createSet = engine.movers.createSet; - engine.movers.createSet = (options) => { - if (options.kind === 'model') builds++; - return createSet(options); - }; - const display = createAircraftDisplay(engine, { - layerId: 'flights', - drawAtlas, - }); - t.after(() => display.destroy()); - const spec = { url: 'airliner.glb', minimumPixelSize: 64 }; - const first = await display.models.loadFleet({ ...spec, id: 'a' }); - const second = await display.models.loadFleet({ ...spec, id: 'b' }); - display.models.add(first); - display.models.add(second); - const failedId = first.gevBank.set.id; - engine.fake.script('moverFailed', (id) => id === failedId); +test('removing failed handles retains the shared bank and retries a replacement identity', async (t) => { + const { display, engine } = await displayFixture(t); + const first = await addModel(display, 'a'); + const second = await addModel(display, 'b'); + const set = first.gevBank.set; display.models.remove(first); - assert.equal(second.failed, true); - assert.equal(builds, 1, 'another handle in the bank does not trigger a load'); display.models.remove(second); - assert.equal(builds, 1); - t.mock.timers.tick(1499); - await assert.rejects(display.models.loadFleet(spec), /retry/); - t.mock.timers.tick(1); - const retry = await display.models.loadFleet(spec); - assert.equal(builds, 1, 'a load starts only when the handle is admitted'); - display.models.add(retry); - assert.equal(builds, 2); - assert.equal(retry.failed, false); + await settle(); + assert.equal(set.count(), 0); + assert.equal(set.isDestroyed(), false); + const next = await addModel(display, 'c'); + assert.equal(next.gevBank.set, set); + assert.equal(next.failed, true); + t.mock.timers.tick(1500); + assert.equal(next.ready, true); + assert.equal( + engine.movers.list().filter((row) => row.kind === 'model').length, + 1, + ); }); diff --git a/src/layers/flights/rendering.js b/src/layers/flights/rendering.js index a4a98b375..2265c27e2 100644 --- a/src/layers/flights/rendering.js +++ b/src/layers/flights/rendering.js @@ -727,17 +727,6 @@ export function createRendering({ }); return; // billboard carries the visual until the model is ready } - if (flightState._trackedModel?.failed) { - // The engine could not load the glTF: the billboard stays the visual - // and the load is tried again after the backoff. - const { url } = flightState._trackedModel.spec ?? {}; - _releaseTrackedModel(); - parts.tracking._noteTrackedModelLoadFailure( - url, - new Error('model failed to load'), - ); - return; - } if (flightState._trackedModel) { // Keep the transform current while GPU resources are still loading. Cesium // can flip ready during scene update after this callback; waiting for ready @@ -824,10 +813,6 @@ export function createRendering({ const occluder = view.horizon(); const focusTarget = getFocusTarget(); - // A model the engine failed to load gives its slot back, in view or not; - // the renderer loads its class again after a backoff. - for (const [icao24, model] of flightState._models) - if (model.failed) _releaseModel(icao24); // 3D model regime: only when enabled AND the camera is zoomed in past the altitude ceiling. // Drop all models the moment we leave it (toggled off / zoomed out) so billboards resume. const useModels = _modelRegimeActive(); diff --git a/src/layers/localAdsb/models.js b/src/layers/localAdsb/models.js index 3d649fc01..94d43d6c9 100644 --- a/src/layers/localAdsb/models.js +++ b/src/layers/localAdsb/models.js @@ -208,10 +208,6 @@ export function createLocalAdsbModels({ for (const [id, marker] of markers) { const model = models.get(id); marker.modelOwnsVisual = false; - if (model?.failed) { - release(id); - continue; - } if (!model || !marker.position) continue; const position = placement(id, marker); if (!position || !occluder.isPointVisible(marker.position)) { diff --git a/src/layers/localAdsb/models.test.mjs b/src/layers/localAdsb/models.test.mjs index e5b2f5516..dd1b1dc96 100644 --- a/src/layers/localAdsb/models.test.mjs +++ b/src/layers/localAdsb/models.test.mjs @@ -2,6 +2,11 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import * as geo from '../../geo/index.js'; import { createLocalAdsbModels } from './models.js'; +import { createFakeEngine } from '../../globe/testing/fakeEngine.js'; +import { + createAircraftDisplay, + aircraftAtlasLayout, +} from '../aircraft/render/display.js'; test('ground snapping excludes the aircraft by its engine pick id', async () => { const position = geo.cartesian3.fromDegrees(0, 0, 0); @@ -49,26 +54,35 @@ test('ground snapping excludes the aircraft by its engine pick id', async () => models.destroy(); }); -test('a failed model frees its slot and leaves the marker visible', async () => { +test('a failed model retains its slot and the marker until the engine retry succeeds', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const engine = createFakeEngine(); + const attempts = []; + engine.fake.script('moverFailed', (id, index, attempt) => { + if (!id.includes('-models-')) return false; + attempts.push(attempt); + return attempt === 1; + }); const position = geo.cartesian3.fromDegrees(0, 0, 100); - const removed = []; - let loads = 0; - const renderer = { - models: { - loadFleet: async () => ({ failed: loads++ === 0, ready: loads > 1 }), - add() {}, - remove: (model) => removed.push(model), - destroy() {}, - setPose() {}, - }, - view: { - camera: { positionWC: position, positionCartographic: { height: 100 } }, - frustumTester: () => () => true, - horizon: () => ({ isPointVisible: () => true }), - }, - }; + const layout = aircraftAtlasLayout(); + const display = createAircraftDisplay(engine, { + layerId: 'local-adsb', + drawAtlas: async () => ({ + canvas: { width: layout.width, height: layout.height }, + cells: layout.cells, + }), + }); + await display.whenReady(); + display.fleet.attach(); const models = createLocalAdsbModels({ - renderer, + renderer: { + models: display.models, + view: { + camera: { positionWC: position, positionCartographic: { height: 100 } }, + frustumTester: () => () => true, + horizon: () => ({ isPointVisible: () => true }), + }, + }, color: {}, resolveAsset: (url) => url, }); @@ -78,15 +92,21 @@ test('a failed model frees its slot and leaves the marker visible', async () => models.frame(markers, { models3d: true }, 1000); await new Promise(setImmediate); models.frame(markers, { models3d: true }, 1100); - assert.equal(models.size, 0); - assert.equal(removed.length, 1); + const set = engine.movers.list().find((row) => row.kind === 'model'); + assert.equal(models.size, 1); assert.equal(marker.modelOwnsVisual, false); - models.frame(markers, { models3d: true }, 2700); - await new Promise(setImmediate); + t.mock.timers.tick(1500); models.frame(markers, { models3d: true }, 2800); assert.equal(models.size, 1); assert.equal(marker.modelOwnsVisual, true); + assert.deepEqual(attempts, [1, 2]); + assert.equal( + engine.movers.list().find((row) => row.kind === 'model').id, + set.id, + ); } finally { models.destroy(); + display.destroy(); + engine.destroy(); } }); diff --git a/src/layers/military/rendering.js b/src/layers/military/rendering.js index 08635485e..66ec15ce3 100644 --- a/src/layers/military/rendering.js +++ b/src/layers/military/rendering.js @@ -705,17 +705,6 @@ export function createRendering({ }); return; } - if (flightState._trackedModel?.failed) { - // The engine could not load the glTF: the billboard stays the visual - // and the load is tried again after the backoff. - const { url } = flightState._trackedModel.spec ?? {}; - _releaseTrackedModel(); - parts.tracking._noteTrackedModelLoadFailure( - url, - new Error('model failed to load'), - ); - return; - } if (flightState._trackedModel) { // Keep the transform current while GPU resources are still loading. Cesium // can flip ready during scene update after this callback; waiting for ready @@ -805,10 +794,6 @@ export function createRendering({ const occluder = view.horizon(); const focusTarget = getFocusTarget(); - // A model the engine failed to load gives its slot back, in view or not; - // the renderer loads its class again after a backoff. - for (const [icao24, model] of flightState._models) - if (model.failed) _releaseModel(icao24); // 3D model regime: only when enabled AND the camera is zoomed in past the altitude ceiling. // Drop all models the moment we leave it (toggled off / zoomed out) so billboards resume. const useModels = _modelRegimeActive(); diff --git a/src/layers/shared/render/moverBank.js b/src/layers/shared/render/moverBank.js index f23968b6a..2b6378e39 100644 --- a/src/layers/shared/render/moverBank.js +++ b/src/layers/shared/render/moverBank.js @@ -35,13 +35,11 @@ export function createMoverBank(engine, { options, write, onCommit }) { let capacity = 0; let visible = true; let dirty = false; - let rebuildPending = false; const build = (wanted) => { const next = engine.movers.createSet(options(wanted, ++generation)); set?.destroy(); set = next; capacity = wanted; - rebuildPending = false; handles.forEach((handle, index) => write(handle, set, index)); set.setCount(visible ? handles.length : 0); }; @@ -57,7 +55,7 @@ export function createMoverBank(engine, { options, write, onCommit }) { handle.gevBank = bank; handle.gevIndex = handles.length; handles.push(handle); - if (!set || rebuildPending || handles.length > capacity) + if (!set || handles.length > capacity) build( Math.max( FIRST_CAPACITY, @@ -83,7 +81,6 @@ export function createMoverBank(engine, { options, write, onCommit }) { handle.gevBank = null; handle.gevIndex = -1; set?.setCount(visible ? handles.length : 0); - if (!handles.length && rebuildPending) bank.rebuild(); bank.touch(); }, /** Writes `fields` of one handle (when it is in this bank). */ @@ -98,14 +95,6 @@ export function createMoverBank(engine, { options, write, onCommit }) { bank.touch(); }, visible: () => visible, - /** Reload assets on the next add, after the caller's retry backoff. */ - rebuild() { - rebuildPending = true; - if (!handles.length) { - set?.destroy(); - set = null; - } - }, touch() { if (!dirty) { dirty = true;