benmichael/gods-eye-view
1
0 Fork 0
mirror of https://github.com/bilawalsidhu/gods-eye-view.git synced 2026-11-11 13:52:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'german/refactor-e4-model-retry' into german/navara-refactor

Browse Source
This commit is contained in:
German Khokhlov committed 2026-11-10 16:04:06 +04:00
2 parents 1ac9f28c4a a07d9f5d85
commit 75cf0aeaf9
22 files changed
+1039 -470
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/contract-mutants.test.mjs
+19 -2
View File
@@ -207,7 +207,7 @@ const mutants = [
],
[
'failed model reports no failure',
'MoverSet.failed:',
'movers.createSet: failed model retries',
(e) => {
handle(e, 'movers', 'createSet', (set) => {
set.failed = () => false;
@@ -216,13 +216,30 @@ const mutants = [
],
[
'failed model reports ready',
'MoverSet.failed:',
'movers.createSet: failed model retries',
(e) => {
handle(e, 'movers', 'createSet', (set) => {
set.ready = () => true;
});
},
],
[
'failed model never retries',
'movers.createSet: failed model retries',
(e) => {
handle(e, 'movers', 'createSet', (set) => {
replace(set, 'commit', (commit) => () => {
const schedule = globalThis.setTimeout;
globalThis.setTimeout = () => null;
try {
commit();
} finally {
globalThis.setTimeout = schedule;
}
});
});
},
],
[
'hidden mover reports shown',
'movers.list: shown follows',
src/globe/cesium/createCesiumEngine.test.mjs
+15 -3
View File
@@ -216,20 +216,32 @@ test('the contract suite movers and lines checks pass on the Cesium engine (no p
test('the camera, clock and failed model suite checks pass on the Cesium adapter', async (t) => {
const { createAppClock } = await import('../../time/clock.js');
let wall = 1_000_000;
// The failed-model check turns the model's load failure on and off.
let modelFails = false;
const { viewer, camera, engine, frame } = createCameraViewer({
engineOptions: {
createEventHandler: createFakeEventHandlers().create,
appClock: createAppClock({ now: () => wall }),
moverFactories: {
loadModel: async () => {
throw new Error('Model unavailable');
if (modelFails) throw new Error('Model unavailable');
return {
ready: true,
modelMatrix: new Cesium.Matrix4(),
isDestroyed: () => false,
destroy() {},
};
},
},
},
});
viewer.entities = new Cesium.EntityCollection();
const checks = createContractSuite(engine, {
fixtures: { failModel() {} },
fixtures: {
failModel(id, fails) {
modelFails = fails;
},
},
waitFrames: async (count) => {
for (let i = 0; i < count; i++) {
wall += 16;
@@ -239,7 +251,7 @@ test('the camera, clock and failed model suite checks pass on the Cesium adapter
}
},
}).filter(({ name }) =>
/^(camera.changed: a pure|camera.following:|clock.frameTime:|MoverSet.failed:)/.test(
/^(camera.changed: a pure|camera.following:|clock.frameTime:|movers.createSet: failed model)/.test(
name,
),
);
src/globe/cesium/movers.js
+81 -74
View File
@@ -11,7 +11,7 @@
import * as Cesium from 'cesium';
import { horizonVisible } from '../cameraMath.js';
import { createMoverRegistry } from '../movers.js';
import { createMoverRegistry, createMoverRetry } from '../movers.js';
let scratchPosition;
let scratchColor;
@@ -150,30 +150,35 @@ function spritePart(scene, options, store, ports) {
}
}
};
let atlasFailed = false;
// Instances wait for the atlas; then everything written so far is applied.
factories
.cropCells(sprite.atlas, store.cellKeys)
.then((ready) => {
if (destroyed) return;
cells = ready;
apply(0, Math.max(store.count, billboards.length));
requestRender('movers');
})
.catch((error) => {
atlasFailed = true;
console.warn(`[movers] ${id}: atlas failed`, error);
});
const retry = createMoverRetry(options.retryAfterMs, () => {
factories
.cropCells(sprite.atlas, store.cellKeys)
.then((ready) => {
if (destroyed) return;
cells = ready;
retry.succeed();
apply(0, Math.max(store.count, billboards.length));
requestRender('movers');
})
.catch((error) => {
if (destroyed) return;
retry.fail();
console.warn(`[movers] ${id}: atlas failed`, error);
});
});
retry.start();
return {
primitive: collection,
ready: () => cells !== null,
failed: () => atlasFailed,
failed: retry.failed,
commit(range) {
if (cells && range) apply(range.from, range.to);
requestRender('movers');
},
destroy() {
destroyed = true;
retry.destroy();
billboards.length = 0;
shownCell.length = 0;
cells = null;
@@ -230,14 +235,9 @@ function modelPart(options, store, { factories, requestRender, owners }) {
const collection = factories.models();
const { model, pickable } = options;
const models = [];
const loading = [];
// Instances whose model failed: not tried again (MoverSet.failed). An
// index holds the pickId it failed for: a layer moves another instance
// into a freed index, and that one loads its own model.
const failed = [];
const failedFor = [];
const failedNow = (index) =>
failed[index] === true && failedFor[index] === store.pickIds[index];
const loads = [];
const current = (index) =>
loads[index]?.pickId === store.pickIds[index] ? loads[index] : null;
let destroyed = false;
const place = (item, index) => {
const { orientations } = store;
@@ -258,69 +258,77 @@ function modelPart(options, store, { factories, requestRender, owners }) {
item.color = readColor(store, index);
};
const ensure = (index) => {
if (models[index] || loading[index]) return;
loading[index] = true;
factories
.loadModel({
url: model.url,
// Resources built at once, not spread over frames behind tile jobs.
asynchronous: false,
// The instance orientation turns the glTF's own axes (the contract's
// model frame), as Navara draws them.
upAxis: Cesium.Axis.Z,
forwardAxis: Cesium.Axis.X,
// Navara has no environment map; Cesium's renders the sky around
// each model and reads it back from the GPU.
environmentMapOptions: { enabled: false },
minimumPixelSize: model.minimumPixelSize,
...modelBlend(model.colorBlend),
allowPicking: pickable,
...(model.unlit
? {
customShader: new Cesium.CustomShader({
lightingModel: Cesium.LightingModel.UNLIT,
}),
}
: {}),
})
.then((item) => {
if (destroyed) {
item.destroy?.();
return;
}
models[index] = item;
owners.set(item, { owner: options.id, instance: index });
place(item, index);
collection.add(item);
requestRender('movers');
})
.catch((error) => {
if (destroyed) return;
failed[index] = true;
failedFor[index] = store.pickIds[index];
console.warn(`[movers] ${options.id}: model failed`, error);
});
if (current(index)) return;
loads[index]?.retry.destroy();
if (models[index]) collection.remove(models[index]);
models[index] = null;
const entry = { pickId: store.pickIds[index], retry: null };
loads[index] = entry;
const live = () => !destroyed && current(index) === entry;
entry.retry = createMoverRetry(options.retryAfterMs, () => {
if (!live()) return;
factories
.loadModel({
url: model.url,
// Resources built at once, not spread over frames behind tile jobs.
asynchronous: false,
// The instance orientation turns the glTF's own axes (the contract's
// model frame), as Navara draws them.
upAxis: Cesium.Axis.Z,
forwardAxis: Cesium.Axis.X,
// Navara has no environment map; Cesium's renders the sky around
// each model and reads it back from the GPU.
environmentMapOptions: { enabled: false },
minimumPixelSize: model.minimumPixelSize,
...modelBlend(model.colorBlend),
allowPicking: pickable,
...(model.unlit
? {
customShader: new Cesium.CustomShader({
lightingModel: Cesium.LightingModel.UNLIT,
}),
}
: {}),
})
.then((item) => {
if (!live()) {
item.destroy?.();
return;
}
models[index] = item;
entry.retry.succeed();
owners.set(item, { owner: options.id, instance: index });
place(item, index);
collection.add(item);
requestRender('movers');
})
.catch((error) => {
if (!live()) return;
entry.retry.fail();
console.warn(`[movers] ${options.id}: model failed`, error);
});
});
entry.retry.start();
};
return {
primitive: collection,
// Every drawn instance's model has loaded (with an index, that one's).
ready(index) {
if (index !== undefined)
return index < store.count && models[index]?.ready === true;
return Boolean(current(index)) && models[index]?.ready === true;
for (let at = 0; at < store.count; at++)
if (!models[at]?.ready) return false;
if (!current(at) || !models[at]?.ready) return false;
return true;
},
failed(index) {
if (index !== undefined) return failedNow(index);
for (let at = 0; at < store.count; at++) if (failedNow(at)) return true;
if (index !== undefined) return current(index)?.retry.failed() === true;
for (let at = 0; at < store.count; at++)
if (current(at)?.retry.failed()) return true;
return false;
},
commit(range) {
if (range)
for (let index = range.from; index < range.to; index++) {
if (failed[index] && !failedNow(index))
failed[index] = loading[index] = false;
if (index < store.count) ensure(index);
if (models[index]) place(models[index], index);
}
@@ -328,10 +336,9 @@ function modelPart(options, store, { factories, requestRender, owners }) {
},
destroy() {
destroyed = true;
for (const entry of loads) entry?.retry.destroy();
models.length = 0;
loading.length = 0;
failed.length = 0;
failedFor.length = 0;
loads.length = 0;
},
};
}
src/globe/cesium/movers.test.mjs
+120 -13
View File
@@ -8,17 +8,19 @@ import { createFakeViewer } from './testing/fakeViewer.mjs';
// Cesium's own collections, billboards and points (they draw nothing
// without a renderer, but hold what the engine writes). What needs the
// network or a DOM is faked: the glTF load and the atlas crop.
function setup({ loadModel } = {}) {
function setup({ loadModel, cropCells } = {}) {
const viewer = createFakeViewer();
viewer.scene.frameState.mode = Cesium.SceneMode.SCENE3D;
const requests = [];
const crops = [];
const loads = [];
const factories = {
cropCells: async (atlas, keys) => {
crops.push({ atlas, keys });
return new Map(keys.map((key) => [key, { width: 8, height: 8, key }]));
},
cropCells:
cropCells ??
(async (atlas, keys) => {
crops.push({ atlas, keys });
return new Map(keys.map((key) => [key, { width: 8, height: 8, key }]));
}),
loadModel:
loadModel ??
(async (options) => {
@@ -412,13 +414,14 @@ test('a model that loads after its set is destroyed is released', async () => {
assert.equal(root().length, 0);
});
test('a model that fails to load marks its instance failed, once', async (t) => {
test('failed model attempts stop at three and a replacement gets a fresh budget', async (t) => {
t.mock.method(console, 'warn', () => {});
t.mock.timers.enable({ apis: ['setTimeout'] });
let loads = 0;
const { movers } = setup({
loadModel: async () => {
loads++;
throw new Error('404');
throw new Error('503');
},
});
const set = movers.createSet({
@@ -427,19 +430,123 @@ test('a model that fails to load marks its instance failed, once', async (t) =>
capacity: 1,
model: { url: '/a.glb' },
});
set.set(0, { position: at(0, 0) });
set.set(0, { position: at(0, 0), pickId: 'a' });
set.setCount(1);
set.commit();
assert.equal(set.failed(), false, 'still loading');
await settle();
assert.equal(set.failed(), false);
await new Promise(setImmediate);
assert.equal(set.failed(0), true);
t.mock.timers.tick(1499);
assert.equal(loads, 1);
for (const delay of [1, 1500]) {
t.mock.timers.tick(delay);
await new Promise(setImmediate);
}
t.mock.timers.tick(10000);
assert.equal(loads, 3);
assert.equal(set.failed(), true);
assert.equal(set.ready(), false);
set.set(0, { pickId: 'b' });
set.commit();
await settle();
assert.equal(loads, 1, 'not tried again');
await new Promise(setImmediate);
assert.equal(loads, 4);
set.destroy();
assert.equal(set.failed(), false, 'a destroyed set reports nothing');
t.mock.timers.tick(10000);
assert.equal(loads, 4);
assert.equal(set.failed(), false);
});
test('a failed instance retries without reloading its healthy neighbour', async (t) => {
t.mock.method(console, 'warn', () => {});
t.mock.timers.enable({ apis: ['setTimeout'] });
let loads = 0;
let finish;
const { movers, root } = setup({
loadModel: async (options) => {
loads++;
if (loads === 1) throw new Error('503');
if (loads === 3) await new Promise((resolve) => (finish = resolve));
return { ...fakeModel(options), ready: true };
},
});
const set = movers.createSet({
id: 'fleet',
kind: 'model',
capacity: 2,
retryAfterMs: 20,
model: { url: '/a.glb' },
});
t.after(() => set.destroy());
set.set(0, { position: at(0, 0), pickId: 'a' });
set.set(1, { position: at(1, 0), pickId: 'b' });
set.setCount(2);
set.commit();
await new Promise(setImmediate);
const [collection] = items(root());
const [healthy] = items(collection);
assert.equal(set.failed(0), true);
assert.equal(set.ready(1), true);
t.mock.timers.tick(20);
assert.equal(
set.failed(0),
true,
'failure stays visible while retry is pending',
);
set.set(0, { position: at(2, 0), show: false });
set.commit();
finish();
await new Promise(setImmediate);
assert.equal(loads, 3);
assert.equal(set.ready(), true);
assert.equal(set.failed(), false);
assert.equal(collection.get(0), healthy);
assert.equal(collection.get(1).show, false);
assert.equal(collection.get(1).modelMatrix[12], at(2, 0).x);
set.set(0, { pickId: 'c' });
set.commit();
await new Promise(setImmediate);
assert.equal(loads, 4, 'a healthy index reloads for its new identity');
});
test('atlas retries apply the latest instances and cancel on destruction', async (t) => {
t.mock.method(console, 'warn', () => {});
t.mock.timers.enable({ apis: ['setTimeout'] });
let loads = 0;
const { movers, root } = setup({
cropCells: async (atlas, keys) => {
if (++loads < 3) throw new Error('503');
return new Map(keys.map((key) => [key, { width: 8, height: 8 }]));
},
});
const set = movers.createSet({
id: 'sprites',
kind: 'sprite',
capacity: 1,
retryAfterMs: 20,
sprite: { atlas, sizePx: 8 },
});
set.setCount(1);
set.commit();
await new Promise(setImmediate);
assert.equal(set.failed(), true);
for (let i = 0; i < 2; i++) {
t.mock.timers.tick(20);
await new Promise(setImmediate);
}
assert.equal(set.ready(), true);
assert.equal(set.failed(), false);
assert.equal(root().get(0).length, 1);
set.destroy();
const cancelled = movers.createSet({
id: 'cancelled',
kind: 'sprite',
capacity: 1,
sprite: { atlas, sizePx: 8 },
});
cancelled.destroy();
await new Promise(setImmediate);
t.mock.timers.tick(10000);
assert.equal(loads, 4);
});
// A layer removes an instance by moving its last one into the freed index
src/globe/contract.js
+10 -7
View File
@@ -419,6 +419,8 @@
* @property {string} id Unique among the engine's live sets.
* @property {'sprite'|'point'|'model'} kind
* @property {number} capacity The most instances the set holds.
* @property {number} [retryAfterMs=1500] Delay after a model or atlas load
* failure before retrying. At most three attempts, including the first.
* @property {number} [order] Draw order among sets (higher draws later, on
* top); 0 by default.
* @property {'normal'|'always'|{ disableWithinM: number }} [depthTest]
@@ -474,14 +476,15 @@
* @property {() => void} commit Uploads what changed (one buffer write) and
* requests a render; nothing else draws a frame.
* @property {(index?: number) => boolean} ready Whether what the set draws
* has loaded: a model set's glTF (Cesium: every drawn instance's model), a
* sprite set's atlas. With an index, what that instance draws (Cesium: its
* own model; the other engines load a set's glTF once).
* A layer keeps its fallback visual until then.
* has loaded: every active model or the sprite atlas. With an index, what
* that active instance draws; false outside [0, count). A successful retry
* makes it ready. A layer keeps its fallback visual until then.
* @property {(index?: number) => boolean} failed Whether what the set draws
* failed to load (with an index, what that instance draws): it stays not
* ready and is not tried again; a layer that retries makes a new set.
* Cesium loads per instance: an index given another pickId loads again.
* failed on its latest attempt (with an index, that active instance;
* false outside [0, count)). It stays true during the retry and clears on
* success; after three failed attempts it stays true. The engine retries
* after retryAfterMs. A model index given another pickId loads again with
* a fresh attempt budget. Destroy cancels retries and both queries return false.
* @property {() => void} destroy
*
* @typedef {object} GlobeMovers
src/globe/movers.js
+41 -4
View File
@@ -68,7 +68,7 @@ function atlasOf(atlas) {
* @returns {{ id: string, kind: string, capacity: number, order: number,
* depthTest: 'normal'|'always'|{ disableWithinM: number }, pickable: boolean, sharpen: boolean,
* cellKeys: string[], sprite: object | null, point: object | null,
* model: object | null }}
* model: object | null, retryAfterMs: number }}
*/
export function normalizeMoverSetOptions(options) {
if (!isObject(options)) fail('options must be an object');
@@ -89,6 +89,7 @@ export function normalizeMoverSetOptions(options) {
kind,
capacity,
order,
retryAfterMs: nonNegative(options.retryAfterMs, 'retryAfterMs', 1500),
depthTest: depthTestOf(options.depthTest),
pickable: options.pickable === true,
sharpen: options.sharpen === true,
@@ -353,6 +354,9 @@ export function createMoverSet(
const alive = () => {
if (destroyed) throw new Error('Mover set was destroyed');
};
const active = (index) =>
index === undefined ||
(Number.isInteger(index) && index >= 0 && index < store.count);
return {
// The set's id (createSet's `id`): surface exclusion and picks name it.
id,
@@ -372,9 +376,9 @@ export function createMoverSet(
},
// Whether what it draws has loaded (a model's glTF, an atlas image); with
// an index, what that instance draws.
ready: (index) => !destroyed && ready(index) === true,
// Whether that load failed for good (the set is not tried again).
failed: (index) => !destroyed && failed(index) === true,
ready: (index) => !destroyed && active(index) && ready(index) === true,
// The latest attempt failed; the engine owns the bounded retry.
failed: (index) => !destroyed && active(index) && failed(index) === true,
destroy() {
if (destroyed) return;
destroyed = true;
@@ -385,6 +389,39 @@ export function createMoverSet(
};
}
/** A load gets three attempts, including the first, until cancelled. */
export function createMoverRetry(retryAfterMs, load) {
let attempts = 0;
let failed = false;
let timer = null;
let destroyed = false;
const attempt = () => {
timer = null;
if (!destroyed) load(++attempts);
};
return {
start() {
clearTimeout(timer);
attempts = 0;
failed = false;
attempt();
},
fail() {
if (destroyed) return;
failed = true;
if (attempts < 3) timer = setTimeout(attempt, retryAfterMs);
},
succeed() {
failed = false;
},
failed: () => failed,
destroy() {
destroyed = true;
clearTimeout(timer);
},
};
}
/**
* engine.movers over an engine's own drawing: `build(options, store)` makes
* the engine part of one set ({ commit, destroy }, both called with the
src/globe/movers.test.mjs
+86
View File
@@ -1,5 +1,6 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { createFakeEngine } from './testing/fakeEngine.js';
import {
MOVER_KINDS,
createMoverRegistry,
@@ -23,6 +24,7 @@ test('set options are normalised with the contract defaults', () => {
});
assert.equal(sprite.id, 'flights');
assert.equal(sprite.order, 0);
assert.equal(sprite.retryAfterMs, 1500);
assert.equal(sprite.depthTest, 'normal');
assert.equal(sprite.pickable, false);
assert.deepEqual(sprite.cellKeys, ['jet', 'heli']);
@@ -292,3 +294,87 @@ test('native-stage sprite fields opt into sharpening explicitly and reject model
TypeError,
);
});
test('retry delay rejects invalid values and accepts zero', () => {
const options = {
id: 'retry',
kind: 'model',
capacity: 1,
model: { url: '/a.glb' },
};
for (const retryAfterMs of [-1, NaN, Infinity, '1500'])
assert.throws(
() => normalizeMoverSetOptions({ ...options, retryAfterMs }),
/retryAfterMs/,
);
assert.equal(
normalizeMoverSetOptions({ ...options, retryAfterMs: 0 }).retryAfterMs,
0,
);
});
for (const kind of ['cesium', 'navara'])
test(`${kind} fake models retry twice, keep failure until success and reset for a new identity`, (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
const engine = createFakeEngine({ kind });
t.after(() => engine.destroy());
const attempts = [];
engine.fake.script('moverFailed', (id, index, attempt) => {
attempts.push(attempt);
return attempt < 3;
});
const set = engine.movers.createSet({
id: 'model',
kind: 'model',
capacity: 2,
model: { url: '/a.glb' },
});
set.set(0, { pickId: 'a' });
set.setCount(1);
set.commit();
assert.equal(set.failed(), true);
assert.equal(set.failed(1), false);
assert.equal(set.ready(1), false);
t.mock.timers.tick(1499);
assert.deepEqual(attempts, [1]);
t.mock.timers.tick(1);
assert.equal(set.failed(0), true);
t.mock.timers.tick(1500);
assert.equal(set.failed(), false);
assert.equal(set.ready(0), true);
assert.deepEqual(attempts, [1, 2, 3]);
set.set(0, { pickId: 'b' });
set.commit();
assert.equal(set.failed(0), true);
set.destroy();
t.mock.timers.tick(10000);
assert.deepEqual(attempts, [1, 2, 3, 1]);
assert.equal(set.ready(), false);
assert.equal(set.failed(), false);
});
test('fake model permanent failure stops after three attempts', (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
const engine = createFakeEngine();
t.after(() => engine.destroy());
let attempts = 0;
engine.fake.script('moverFailed', () => {
attempts++;
return true;
});
const set = engine.movers.createSet({
id: 'model',
kind: 'model',
capacity: 1,
retryAfterMs: 10,
model: { url: '/a.glb' },
});
set.setCount(1);
set.commit();
t.mock.timers.tick(10);
t.mock.timers.tick(10);
t.mock.timers.tick(1000);
assert.equal(attempts, 3);
assert.equal(set.failed(), true);
assert.equal(set.ready(), false);
});
src/globe/navara/movers/index.js
+12 -2
View File
@@ -51,14 +51,22 @@ class GevMoverSetDesc extends withFrameUpdates(MeshDesc) {
const group = new Group();
group.frustumCulled = false;
// Meshes arrive with the glTF; they join the group (and the pick pass).
mover.group.ready.then(() => {
mover.group.onLoad = () => {
if (this.destroyedByGev) return;
for (const { key, wrapper } of this.wrappers) {
this.ctx.unregisterPickableMesh(key);
batchIds.release(wrapper);
}
this.wrappers.length = 0;
this.gev.clearPickable();
group.clear();
mover.group.meshes.forEach((mesh, index) => {
group.add(mesh);
register(mover.group.pickMeshes[index], `${this.id}#${index}`);
});
this.view.forceUpdate();
});
};
if (mover.group.meshes.length) mover.group.onLoad();
return group;
}
@@ -68,6 +76,7 @@ class GevMoverSetDesc extends withFrameUpdates(MeshDesc) {
onDestroy() {
this.destroyedByGev = true;
this.gev.mover.group.onLoad = null;
for (const { key } of this.wrappers) this.ctx.unregisterPickableMesh(key);
this.gev.mover.dispose();
for (const { wrapper } of this.wrappers) this.gev.batchIds.release(wrapper);
@@ -163,6 +172,7 @@ export function createNavaraMovers(view, { overlay, aces = false }) {
kind: options.kind,
pickable: options.pickable,
batchIds,
clearPickable: () => pickIndex.remove(options.id),
onPickable: (wrapper) =>
pickIndex.add(options.id, wrapper, (index) => store.pickIds[index]),
},
src/globe/navara/movers/modelMover.js
+143 -89
View File
@@ -1,4 +1,4 @@
// The three side of a Navara model set: the glTF loaded once, one
// The three side of a Navara model set: the glTF shared by the set, one
// InstancedMesh per mesh in it (all instances share the model's meshes),
// each instance placed by position, orientation and scale relative to an
// origin near the camera. Cesium's minimumPixelSize is kept on the CPU once
@@ -17,6 +17,7 @@ import {
Sphere,
Vector3,
} from 'three';
import { createMoverRetry } from '../../movers.js';
import { footprintPerDepth, originFor } from './pack.js';
import { createModelTint } from './modelTint.js';
@@ -188,94 +189,133 @@ export function createModelMover(
for (const { mesh, tint } of parts) {
if (tint) {
tint.count = store.count;
tint.visible = store.count > 0;
tint.visible = loaded && store.count > 0;
}
mesh.count = store.count;
mesh.visible = store.count > 0;
mesh.visible = loaded && store.count > 0;
mesh.instanceMatrix.needsUpdate = true;
}
for (const attribute of colorAttributes) attribute.needsUpdate = true;
syncTransparency();
};
const group = { meshes: [], pickMeshes: [], ready: null };
let failed = false;
const asset = load(model.url).catch(() => {
// A missing asset keeps the layer's glyph fallback. Do not leak the
// rejected loader promise (or its URL) through descriptor creation.
if (!disposed) {
failed = true;
console.warn('[globe] Model asset unavailable');
const group = { meshes: [], pickMeshes: [], ready: null, onLoad: null };
const pickIds = new Map();
const current = (index) => pickIds.get(index) === store.pickIds[index];
let generation = 0;
let finish;
group.ready = new Promise((resolve) => (finish = resolve));
const releaseParts = () => {
for (const { mesh, tint } of parts) {
if (tint) {
overlay.remove(tint);
tint.material.dispose();
tint.dispose();
}
mesh.removeFromParent();
mesh.geometry.dispose();
mesh.material.dispose();
mesh.dispose?.();
}
return null;
});
group.ready = asset.then((gltf) => {
if (!gltf) return;
const root = gltf.scene;
root.traverse((child) => {
parts.length = materials.length = colorAttributes.length = 0;
group.meshes.length = group.pickMeshes.length = 0;
disposeTextures();
};
const releaseAsset = (gltf, releaseTexture) => {
gltf.scene.traverse((child) => {
child.geometry?.dispose();
for (const material of Array.isArray(child.material)
? child.material
: [child.material])
if (material)
: [child.material]) {
if (!material) continue;
if (releaseTexture)
for (const value of Object.values(material))
if (value?.isTexture) textures.add(value);
if (value?.isTexture) value.dispose();
material.dispose();
}
});
if (disposed) {
disposeTextures();
return;
}
root.updateMatrixWorld(true);
const sphere = new Box3()
.setFromObject(root)
.getBoundingSphere(new Sphere());
radius = Math.max(sphere.radius, 1e-3);
root.traverse((child) => {
if (!child.isMesh) return;
let material = child.material;
if (model.unlit)
material = new MeshBasicMaterial({
map: material.map ?? null,
color: material.color,
};
const retry = createMoverRetry(options.retryAfterMs, () => {
const version = generation;
load(model.url)
.then((gltf) => {
if (disposed || version !== generation) {
releaseAsset(gltf, true);
return;
}
releaseParts();
const root = gltf.scene;
root.traverse((child) => {
for (const material of Array.isArray(child.material)
? child.material
: [child.material])
if (material)
for (const value of Object.values(material))
if (value?.isTexture) textures.add(value);
});
else material = material.clone();
blendInstanceColor(material, uniforms, Boolean(overlay));
const geometry = child.geometry.clone();
const color = new InstancedBufferAttribute(
new Float32Array(capacity * 4).fill(1),
4,
);
color.setUsage(DynamicDrawUsage);
geometry.setAttribute('gevColor', color);
colorAttributes.push(color);
const mesh = new InstancedMesh(geometry, material, capacity);
mesh.instanceMatrix.setUsage(DynamicDrawUsage);
mesh.frustumCulled = false;
mesh.renderOrder = options.order;
mesh.count = 0;
mesh.visible = false;
const tint = overlay
? createModelTint(mesh, uniforms, overlay.uniforms)
: null;
if (tint) {
// Still present in the scene depth and GPU pick registry. Only colour
// moves to the display pass, where it is toned before raster resolve.
material.colorWrite = false;
overlay.add(tint);
}
materials.push(material);
parts.push({ mesh, tint, local: child.matrixWorld.clone() });
group.meshes.push(mesh);
group.pickMeshes.push(tint ?? mesh);
});
loaded = true;
if (origin)
for (const { mesh, tint } of parts) {
mesh.position.set(origin.x, origin.y, origin.z);
tint?.position.copy(mesh.position);
}
write(0, store.count);
onLoad(group.meshes);
root.updateMatrixWorld(true);
const sphere = new Box3()
.setFromObject(root)
.getBoundingSphere(new Sphere());
radius = Math.max(sphere.radius, 1e-3);
root.traverse((child) => {
if (!child.isMesh) return;
let material = child.material;
if (model.unlit)
material = new MeshBasicMaterial({
map: material.map ?? null,
color: material.color,
});
else material = material.clone();
blendInstanceColor(material, uniforms, Boolean(overlay));
const geometry = child.geometry.clone();
const color = new InstancedBufferAttribute(
new Float32Array(capacity * 4).fill(1),
4,
);
color.setUsage(DynamicDrawUsage);
geometry.setAttribute('gevColor', color);
colorAttributes.push(color);
const mesh = new InstancedMesh(geometry, material, capacity);
mesh.instanceMatrix.setUsage(DynamicDrawUsage);
mesh.frustumCulled = false;
mesh.renderOrder = options.order;
mesh.count = 0;
mesh.visible = false;
const tint = overlay
? createModelTint(mesh, uniforms, overlay.uniforms)
: null;
if (tint) {
// Still present in the scene depth and GPU pick registry. Only colour
// moves to the display pass, where it is toned before raster resolve.
material.colorWrite = false;
overlay.add(tint);
}
materials.push(material);
parts.push({ mesh, tint, local: child.matrixWorld.clone() });
group.meshes.push(mesh);
group.pickMeshes.push(tint ?? mesh);
});
loaded = true;
retry.succeed();
releaseAsset(gltf, false);
if (origin)
for (const { mesh, tint } of parts) {
mesh.position.set(origin.x, origin.y, origin.z);
tint?.position.copy(mesh.position);
}
write(0, store.count);
group.onLoad?.();
finish();
onLoad(group.meshes);
})
.catch(() => {
if (disposed || version !== generation) return;
retry.fail();
console.warn('[globe] Model asset unavailable');
});
});
retry.start();
const placeAt = (next) => {
origin = { x: next.x, y: next.y, z: next.z };
@@ -290,8 +330,19 @@ export function createModelMover(
return {
group,
uniforms,
ready: () => loaded,
failed: () => failed,
ready(index) {
if (disposed) return false;
if (index !== undefined) return loaded && current(index);
for (let at = 0; at < store.count; at++)
if (!loaded || !current(at)) return false;
return true;
},
failed(index) {
if (disposed || !retry.failed()) return false;
if (index !== undefined) return current(index);
for (let at = 0; at < store.count; at++) if (current(at)) return true;
return false;
},
origin: () => origin,
/**
* The drawn radius (px) of an instance at its last write, or null when
@@ -303,6 +354,17 @@ export function createModelMover(
? drawnPx[index] / 2
: null,
commit(range) {
let changed = false;
for (let index = 0; index < store.count; index++) {
if (pickIds.has(index) ? !current(index) : retry.failed())
changed = true;
pickIds.set(index, store.pickIds[index]);
}
if (changed) {
generation++;
loaded = false;
retry.start();
}
if (!origin && store.count > 0)
placeAt({
x: store.positions[0],
@@ -313,11 +375,11 @@ export function createModelMover(
for (const { mesh, tint } of parts) {
if (tint) {
tint.count = store.count;
tint.visible = store.count > 0;
tint.visible = loaded && store.count > 0;
}
mesh.count = store.count;
// Retained empty banks must also skip three's material/uniform work.
mesh.visible = store.count > 0;
mesh.visible = loaded && store.count > 0;
}
},
/** Each drawn frame: pixel limits need every instance rewritten. */
@@ -340,17 +402,9 @@ export function createModelMover(
dispose() {
if (disposed) return;
disposed = true;
for (const { mesh, tint } of parts) {
if (tint) {
overlay.remove(tint);
tint.material.dispose();
tint.dispose();
}
mesh.geometry.dispose();
mesh.material.dispose();
mesh.dispose?.();
}
disposeTextures();
retry.destroy();
releaseParts();
finish();
},
};
}
src/globe/navara/movers/modelMover.test.mjs
+100 -2
View File
@@ -197,12 +197,14 @@ test('a model removed before its load completes releases the arriving textures',
test('a failed model load stays not-ready without an unhandled rejection or leaking the URL', async (t) => {
const warnings = [];
t.mock.method(console, 'warn', (message) => warnings.push(message));
const mover = createModelMover(options, createMoverStore(options), {
const store = createMoverStore(options);
store.setCount(1);
const mover = createModelMover(options, store, {
load: async () => {
throw new Error('unavailable asset URL');
},
});
await mover.group.ready;
await new Promise(setImmediate);
assert.equal(mover.ready(), false);
assert.equal(mover.failed(), true);
assert.equal(mover.group.meshes.length, 0);
@@ -333,3 +335,99 @@ test('the display pass preserves unlit model colour without a lighting tone curv
);
mover.dispose();
});
for (const failures of [1, 2, 3])
test(`model retries ${failures} failures with one budget and releases its timer`, async (t) => {
t.mock.method(console, 'warn', () => {});
t.mock.timers.enable({ apis: ['setTimeout'] });
const store = createMoverStore(options);
let loads = 0;
let added = 0;
const mover = createModelMover({ ...options, retryAfterMs: 20 }, store, {
load: async () => {
if (++loads <= failures) throw new Error('503');
return fakeGltf();
},
});
mover.group.onLoad = () => added++;
t.after(() => mover.dispose());
store.set(0, { position: { x: 6378137, y: 0, z: 0 }, pickId: 'a' });
store.setCount(1);
mover.commit(store.dirty());
await new Promise(setImmediate);
assert.equal(mover.failed(0), true);
for (let attempt = 0; attempt < 2; attempt++) {
t.mock.timers.tick(19);
assert.equal(loads, attempt + 1);
t.mock.timers.tick(1);
await new Promise(setImmediate);
if (mover.ready()) break;
}
assert.equal(loads, Math.min(failures + 1, 3));
assert.equal(mover.ready(), failures < 3);
assert.equal(mover.failed(), failures === 3);
t.mock.timers.tick(10000);
assert.equal(loads, Math.min(failures + 1, 3));
store.set(0, { pickId: 'b' });
assert.equal(
mover.failed(),
false,
'the replacement has no failed attempt',
);
mover.commit(store.dirty());
assert.equal(mover.failed(0), false);
await new Promise(setImmediate);
assert.equal(mover.ready(0), true);
assert.equal(added, failures < 3 ? 2 : 1);
assert.equal(
mover.group.meshes.length,
1,
'replacement does not accumulate meshes',
);
mover.dispose();
t.mock.timers.tick(10000);
assert.equal(loads, Math.min(failures + 1, 3) + 1);
});
test('a pending obsolete identity releases its asset without overwriting the replacement', async (t) => {
const store = createMoverStore(options);
const pending = [];
const mover = createModelMover(options, store, {
load: () => new Promise((resolve) => pending.push(resolve)),
});
t.after(() => mover.dispose());
store.set(0, { pickId: 'a' });
store.setCount(1);
mover.commit(store.dirty());
store.set(0, { pickId: 'b' });
mover.commit(store.dirty());
const stale = fakeGltf();
let released = 0;
stale.scene.children[0].geometry.addEventListener(
'dispose',
() => released++,
);
pending[0](stale);
await new Promise(setImmediate);
assert.equal(released, 1);
assert.equal(mover.ready(), false);
pending[1](fakeGltf());
await mover.group.ready;
assert.equal(mover.ready(), true);
});
test('destroying a failed model cancels its pending retry', async (t) => {
t.mock.method(console, 'warn', () => {});
t.mock.timers.enable({ apis: ['setTimeout'] });
let loads = 0;
const mover = createModelMover(options, createMoverStore(options), {
load: async () => {
loads++;
throw new Error('503');
},
});
await new Promise(setImmediate);
mover.dispose();
t.mock.timers.tick(10000);
assert.equal(loads, 1);
});
src/globe/navara/movers/moverMesh.js
+19 -15
View File
@@ -31,6 +31,7 @@ import {
OVERLAY_DEPTH_VERTEX,
} from '../overlayDepth.js';
import { DETACHED_SONAR, SONAR_VERTEX } from '../sonar.js';
import { createMoverRetry } from '../../movers.js';
const VERTEX_HEAD = /* glsl */ `
attribute vec4 gevStyle;
@@ -317,22 +318,9 @@ export function createMoverMesh(
};
// A canvas atlas is there at once; a URL one once its image loads.
let textureReady = point || Boolean(options.sprite.atlas.canvas);
let textureFailed = false;
const encodedAtlas = !point && options.sharpen === true;
const material = new MeshBasicMaterial({
map: point
? null
: atlasTexture(
options.sprite.atlas,
encodedAtlas,
() => {
textureReady = true;
onTextureLoad();
},
() => {
textureFailed = true;
},
),
map: null,
transparent: true,
alphaTest: point || encodedAtlas ? 0.005 : 0.02,
// Drawn after tone mapping, with no depth buffer: the shader tests the
@@ -341,6 +329,21 @@ export function createMoverMesh(
depthWrite: false,
side: FrontSide,
});
const retry = createMoverRetry(options.retryAfterMs, () => {
material.map?.dispose();
material.map = atlasTexture(
options.sprite.atlas,
encodedAtlas,
() => {
textureReady = true;
retry.succeed();
onTextureLoad();
},
() => retry.fail(),
);
material.needsUpdate = true;
});
if (!point) retry.start();
material.onBeforeCompile = (shader) =>
patchMoverShader(shader, uniforms, {
point,
@@ -394,7 +397,7 @@ export function createMoverMesh(
mesh,
uniforms,
ready: () => textureReady,
failed: () => textureFailed,
failed: retry.failed,
origin: () => origin,
/** Writes the changed instances; the origin starts at the first one. */
commit(range) {
@@ -455,6 +458,7 @@ export function createMoverMesh(
return true;
},
dispose() {
retry.destroy();
geometry.dispose();
material.map?.dispose();
material.dispose();
src/globe/navara/movers/moverMesh.test.mjs
+48
View File
@@ -348,3 +348,51 @@ test('native-stage glow atlas filters encoded colors and keeps the native faint-
glow.dispose();
ordinary.dispose();
});
test('a URL atlas retries on fresh images, stops after three failures and cancels on dispose', (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
const previous = globalThis.Image;
const images = [];
globalThis.Image = class {
constructor() {
images.push(this);
}
};
t.after(() => {
if (previous) globalThis.Image = previous;
else delete globalThis.Image;
});
const options = normalizeMoverSetOptions({
...spriteOptions,
retryAfterMs: 20,
sprite: {
sizePx: 8,
atlas: { url: '/atlas.png', cells: { a: [0, 0, 1, 1] } },
},
});
const mover = createMoverMesh(options, createMoverStore(options));
images[0].onerror();
assert.equal(mover.failed(), true);
t.mock.timers.tick(19);
assert.equal(images.length, 1);
t.mock.timers.tick(1);
assert.equal(mover.failed(), true);
images[1].onload();
assert.equal(mover.ready(), true);
assert.equal(mover.failed(), false);
assert.equal(images[0].onload, null);
mover.dispose();
const permanent = createMoverMesh(options, createMoverStore(options));
for (let i = 2; i < 5; i++) {
images[i].onerror();
t.mock.timers.tick(20);
}
assert.equal(images.length, 5);
assert.equal(permanent.failed(), true);
permanent.dispose();
const cancelled = createMoverMesh(options, createMoverStore(options));
images[5].onerror();
cancelled.dispose();
t.mock.timers.tick(10000);
assert.equal(images.length, 6);
});
src/globe/testing/contractSuite/movers.mjs
+104 -15
View File
@@ -498,43 +498,132 @@ export function moverChecks(
];
}
export function failedModelChecks(engine, { settle, until, fixtures }) {
export function failedModelChecks(
engine,
{ until, fixtures, nadirView, readRegion },
) {
return [
[
'MoverSet.failed: a failed model stays unready and reports its failure scope',
'movers.createSet: failed model retries, recovers and stops after three attempts',
async () => {
await nadirView(1e6);
const id = 'contract-failed-model';
const url = '/models/airplane.glb?contract-retry';
let failing = true;
let attempts = 0;
const fetch = globalThis.fetch;
const open = globalThis.XMLHttpRequest?.prototype.open;
// Route this model alone to the runner's missing asset while failing.
// The next attempt uses the real GLB without replacing the mover set.
const route = (value) => {
if (!String(value).endsWith(url)) return value;
attempts++;
return failing ? '/__contract_missing_model.glb' : value;
};
if (fixtures?.failModel) fixtures.failModel(id, true);
else {
globalThis.fetch = (input, init) => {
const value = route(input?.url ?? input);
return fetch(value === (input?.url ?? input) ? input : value, init);
};
globalThis.XMLHttpRequest.prototype.open = function (
method,
value,
...args
) {
return open.call(this, method, route(value), ...args);
};
}
const set = engine.movers.createSet({
id,
kind: 'model',
capacity: 2,
model: { url: '/__contract_missing_model.glb' },
retryAfterMs: 100,
model: {
url,
minimumPixelSize: 80,
colorBlend: { mode: 'replace', amount: 1 },
},
});
const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const fail = (value) => {
failing = value;
fixtures?.failModel?.(id, value);
};
try {
set.set(0, {
position: cartesian3.fromDegrees(20, 10, 1000),
pickId: 'failed',
});
ensure(
set.ready() && !set.failed(),
'empty model set reports a load',
);
const position = cartesian3.fromDegrees(20, 10, 50_000);
set.set(0, { position, color: [1, 0, 1, 1], pickId: 'first' });
set.setCount(1);
set.commit();
fixtures?.failModel?.(id, true);
await until(() => set.failed(0), 'failed model');
ensure(set.failed() === true, 'set did not report the failure');
ensure(
set.ready(0) === false && set.ready() === false,
set.failed() && !set.ready() && !set.ready(0),
'failed model is ready',
);
ensure(
set.failed(1) === (engine.kind === 'navara'),
'failure scope changed',
!set.failed(1) && !set.ready(1),
'inactive instance reports a load',
);
await settle(3);
fail(false);
await wait(120);
await until(() => set.ready(0), 'model ready after engine retry');
ensure(
set.failed(0) && !set.ready(0),
'failed model silently recovered',
set.ready() && !set.failed(),
'retry did not clear the failure',
);
if (readRegion) {
const screen = engine.projector.worldToWindow(position);
await until(() => {
const data = readRegion(
Math.round(screen.x) - 50,
Math.round(screen.y) - 50,
100,
100,
);
let pixels = 0;
for (let i = 0; i < data.length; i += 4)
if (
Math.min(data[i], data[i + 2]) - data[i + 1] > 40 &&
data[i] > 60
)
pixels++;
return pixels > 150;
}, 'retried model drawn');
}
// A new identity reloads even a previously healthy index.
fail(true);
attempts = 0;
set.set(0, { pickId: 'second' });
set.commit();
await until(() => set.failed(0), 'replacement model failure');
await wait(350);
ensure(set.failed() && !set.ready(), 'permanent failure disappeared');
if (!fixtures?.failModel)
ensure(attempts === 3, `expected 3 attempts, got ${attempts}`);
fail(false);
await wait(150);
ensure(set.failed(0) && !set.ready(0), 'exhausted model retried');
set.set(0, { pickId: 'third' });
ensure(
!set.failed(0) && !set.failed(),
'new identity inherited failure',
);
set.commit();
await until(
() => set.ready(0),
'replacement after exhausted attempts',
);
ensure(!set.failed(), 'replacement inherited exhausted failure');
} finally {
fixtures?.failModel?.(id, false);
if (!fixtures?.failModel) {
globalThis.fetch = fetch;
globalThis.XMLHttpRequest.prototype.open = open;
}
set.destroy();
}
ensure(
src/globe/testing/fakeEngine.js
+52 -8
View File
@@ -54,7 +54,7 @@ import { createGroundPicks, createGroundRegistry } from '../ground.js';
import { createMeshRegistry, meshRayDistance } from '../meshes.js';
import { createImageryRegistry } from '../imagery.js';
import { createLineRegistry } from '../lines.js';
import { createMoverRegistry } from '../movers.js';
import { createMoverRegistry, createMoverRetry } from '../movers.js';
import { createCpuPicker } from '../cpuPicks.js';
import { createFollowController } from '../follow.js';
import { normalizeSurfaceOptions } from '../surfaceDetail.js';
@@ -257,19 +257,63 @@ export function createFakeEngine(options = {}) {
};
});
// Movers keep their store for tests; commit counts as a render request.
const moverRegistry = createMoverRegistry((options) => {
const moverRegistry = createMoverRegistry((options, store) => {
const loads = new Map();
const ensure = (index) => {
const pickId = store.pickIds[index];
let entry = loads.get(index);
if (entry && entry.pickId === pickId) return entry;
entry?.retry.destroy();
entry = { pickId, ready: false, retry: null };
entry.retry = createMoverRetry(options.retryAfterMs, (attempt) => {
// A script can fail the first N attempts: (id, index, attempt) => attempt <= N.
if (scripted('moverFailed', [options.id, index, attempt], () => false))
entry.retry.fail();
else {
entry.ready = true;
entry.retry.succeed();
}
fake.renderRequests++;
});
loads.set(index, entry);
entry.retry.start();
return entry;
};
if (options.kind === 'sprite') ensure(undefined);
const current = (index) => {
const entry = loads.get(options.kind === 'sprite' ? undefined : index);
return options.kind === 'sprite' || entry?.pickId === store.pickIds[index]
? entry
: null;
};
const part = {
commits: 0,
commit() {
if (options.kind === 'model')
for (let index = 0; index < store.count; index++) ensure(index);
part.commits++;
fake.renderRequests++;
},
ready: (index) =>
!scripted('moverFailed', [options.id, index], () => false),
// Scripted as 'moverFailed' (set id, index) for a load that fails.
failed: (index) =>
scripted('moverFailed', [options.id, index], () => false),
destroy() {},
ready(index) {
if (options.kind === 'point') return true;
if (options.kind === 'sprite') return current(index).ready;
if (index !== undefined) return current(index)?.ready === true;
for (let at = 0; at < store.count; at++)
if (!current(at)?.ready) return false;
return true;
},
failed(index) {
if (options.kind === 'point') return false;
if (options.kind === 'sprite') return current(index).retry.failed();
if (index !== undefined) return current(index)?.retry.failed() === true;
for (let at = 0; at < store.count; at++)
if (current(at)?.retry.failed()) return true;
return false;
},
destroy() {
for (const entry of loads.values()) entry.retry.destroy();
loads.clear();
},
};
return part;
});
src/layers/aircraft/render/contract.js
+2 -2
View File
@@ -19,8 +19,8 @@
// bookkeeping fields on a handle (prefixed _gev).
// ModelHandle (models.load*) — one glTF instance:
// show, ready (read-only: placed, and its model loaded), failed
// (read-only: its model would not load; a tracked one is loaded again
// after models.remove), scale,
// (read-only: the latest load of its model failed; the engine retries
// it, and ready turns true once a retry loads), scale,
// computedScale (read-only: scale after minimumPixelSize), color, id,
// modelMatrix (16 numbers, column-major, the model frame: written by
// models.setPose and read for anchors).
src/layers/aircraft/render/display.js
+2 -19
View File
@@ -29,10 +29,6 @@ import { SPRITE_LAYER_ORDER } from '../../../data/spriteOrder.js';
import { createMoverBank, writeFields } from '../../shared/render/moverBank.js';
const BASE_SIZE_PX = 20;
// A failed glTF is fetched again no sooner than this (the layers' tracked
// model backoff), not on the next fleet tick.
const MODEL_RETRY_BACKOFF_MS = 1500;
/**
* The model frame → the glTF's own axes (the movers' orientation): Cesium's
* glTF axis correction, Y_UP_TO_Z_UP × Z_UP_TO_X_UP (data/modelVisualAnchor.js),
@@ -460,7 +456,6 @@ export function createAircraftDisplay(
// --- models ----------------------------------------------------------------
let modelsVisible = true;
const modelBanks = new Map();
const modelRetryAt = new Map();
const scratchMatrix3 = geo.matrix3.create();
const scratchCorrected = geo.matrix4.create();
const scratchQuaternion = geo.quaternion.create();
@@ -603,16 +598,7 @@ export function createAircraftDisplay(
}
}
// A failed glTF needs a fresh set, created only when a retry is admitted.
const dropModel = (model) => {
const bank = model?.gevBank;
const retry = model?.failed;
bank?.remove(model);
if (retry) {
modelRetryAt.set(model.spec.url, Date.now() + MODEL_RETRY_BACKOFF_MS);
bank.rebuild();
}
};
const dropModel = (model) => model?.gevBank?.remove(model);
const models = {
live: () => attached && !destroyed,
@@ -620,10 +606,7 @@ export function createAircraftDisplay(
modelsVisible = visible;
for (const bank of modelBanks.values()) bank.setVisible(visible);
},
loadFleet: (spec) =>
Date.now() < (modelRetryAt.get(spec.url) ?? 0)
? Promise.reject(new Error('model retry backoff'))
: Promise.resolve(new ModelHandle(spec)),
loadFleet: (spec) => Promise.resolve(new ModelHandle(spec)),
loadTracked: (spec) => Promise.resolve(new ModelHandle(spec)),
add(model) {
modelBankFor(model).add(model);
src/layers/aircraft/trackedModelFailure.test.mjs
+142 -147
View File
@@ -1,8 +1,3 @@
// An aircraft glTF that the engine fails to load: the glyph stays the
// visual, the load is tried again after the backoff in a new mover set (a set
// does not load a failed glTF again), and a tracked contact gives up after its
// failure budget. A failed fleet model gives its slot back at once, and a
// later aircraft of its class loads the glTF again after the backoff.
import test from 'node:test';
import assert from 'node:assert/strict';
import * as Cesium from 'cesium';
@@ -23,6 +18,7 @@ import militaryFlightsLayer, {
_trackedBillboardColorForTest as militaryTrackedBillboardColor,
} from '../../data/militaryFlights.js';
import { clearTr3bRegistry } from '../../data/tr3bRegistry.js';
import { applyAircraftModelTreatment } from '../../data/aircraftRecession.js';
import { recordingAircraftRenderer } from '../../testSupport/aircraftRecordingRenderer.mjs';
import { fleetTickFixture } from '../../testSupport/fleetTickFixture.mjs';
import { createFlightState as civilState } from '../flights/state.js';
@@ -42,38 +38,60 @@ const drawAtlas = async () => ({
cells: layout.cells,
});
test('a failed tracked model loads again in a new set once it is removed', async (t) => {
const settle = () => new Promise(setImmediate);
async function displayFixture(t, failures = 1) {
t.mock.timers.enable({ apis: ['setTimeout'] });
const engine = createFakeEngine();
t.after(() => engine.destroy());
const attempts = [];
engine.fake.script('moverFailed', (id, index, attempt) => {
if (!id.includes('-models-')) return false;
attempts.push({ index, attempt });
return index === 0 && attempt <= failures;
});
const display = createAircraftDisplay(engine, {
layerId: 'flights',
headingOffsetDeg: 180,
drawAtlas,
});
t.after(() => display.destroy());
t.after(() => {
display.destroy();
engine.destroy();
});
await display.whenReady();
display.fleet.attach();
const failing = new Set();
engine.fake.script('moverFailed', (id) => failing.has(id));
const spec = { url: 'plane.glb', minimumPixelSize: 64, scale: 1, id: 'abc' };
return { display, engine, attempts };
}
const spec = (id) => ({
url: 'airliner.glb',
minimumPixelSize: 64,
scale: 1,
id,
});
const position = geo.cartesian3.fromDegrees(1, 0, 1000);
const first = await display.models.loadTracked(spec);
display.models.add(first);
const failedSet = first.gevBank.set.id;
assert.equal(first.failed, false);
failing.add(failedSet);
assert.equal(first.failed, true, 'the engine reports the failed load');
display.models.remove(first);
assert.equal(
engine.movers.list().filter((set) => set.kind === 'model').length,
0,
'removal does not start another load',
async function addModel(display, id, tracked = false) {
const model = await display.models[tracked ? 'loadTracked' : 'loadFleet'](
spec(id),
);
display.models.add(model);
display.models.setPose(model, position, 0);
await settle();
return model;
}
const retry = await display.models.loadTracked(spec);
display.models.add(retry);
assert.notEqual(retry.gevBank.set.id, failedSet, 'a new set loads it again');
assert.equal(retry.failed, false);
test('a tracked model becomes ready in its original set after the engine retries', async (t) => {
const { display, attempts } = await displayFixture(t);
const model = await addModel(display, 'tracked', true);
const set = model.gevBank.set;
assert.equal(model.failed, true);
assert.equal(model.ready, false);
t.mock.timers.tick(1499);
assert.equal(attempts.length, 1);
t.mock.timers.tick(1);
assert.equal(model.ready, true);
assert.equal(model.failed, false);
assert.equal(model.gevBank.set, set);
});
/** A viewer stub inside the tracked model regime. */
@@ -119,47 +137,37 @@ const LAYERS = [
},
];
for (const fixture of LAYERS) {
test(`${fixture.name}: a tracked model the engine fails to load is retried, then given up`, async () => {
clearTr3bRegistry();
fixture.layer.setParams({ models3d: false });
const realNow = Date.now;
const realWarn = console.warn;
let now = 1_000_000;
Date.now = () => now;
const warnings = [];
console.warn = (...args) => warnings.push(args.join(' '));
let loads = 0;
const removed = [];
const viewer = viewerAt(30_000);
try {
for (const fixture of LAYERS)
for (const failures of [1, 3])
test(`${fixture.name}: tracked fallback survives ${failures} failed attempts without replacing the handle`, async (t) => {
clearTr3bRegistry();
fixture.layer.setParams({ models3d: false });
const { engine, display, attempts } = await displayFixture(t, failures);
const viewer = viewerAt(30_000);
viewer.engine = engine;
let handles = 0;
const removed = [];
const billboard = {
position: Cesium.Cartesian3.fromDegrees(-97.71, 30.21, 10_668),
position: geo.cartesian3.fromDegrees(-97.71, 30.21, 10_668),
color: null,
show: true,
rotation: 0,
};
fixture.seed({
renderer: () =>
recordingAircraftRenderer(viewer.engine, {
recordingAircraftRenderer(engine, {
view: { camera: viewer.camera, scene: viewer.scene },
fleet: { shown: true },
models: {
live: () => true,
loadTracked: async (spec) => {
loads++;
return {
spec,
failed: true,
ready: false,
show: false,
modelMatrix: geo.matrix4.create(),
};
...display.models,
loadTracked: (spec) => {
handles++;
return display.models.loadTracked(spec);
},
remove: (model) => {
removed.push(model);
display.models.remove(model);
},
add() {},
remove: (model) => removed.push(model),
destroy() {},
setPose: () => geo.matrix4.create(),
},
}),
icao24: 'abc123',
@@ -170,63 +178,49 @@ for (const fixture of LAYERS) {
tracked: true,
meta: { ...fixture.meta, onGround: false },
});
for (let frame = 0; frame < 30; frame++) {
try {
fixture.drive();
await Promise.resolve();
await Promise.resolve();
now += 2000; // past the retry backoff
await settle();
fixture.drive();
assert.equal(fixture.billboardColor().alpha, 1);
for (let frame = 0; frame < 5; frame++) {
t.mock.timers.tick(1500);
fixture.drive();
await settle();
}
assert.equal(handles, 1);
assert.equal(removed.length, 0);
assert.equal(attempts.length, failures === 3 ? 3 : 2);
if (failures === 3) assert.equal(fixture.billboardColor().alpha, 1);
else
assert.equal(
fixture.billboardColor().alpha,
0,
'the recovered model owns the visual',
);
} finally {
fixture.layer.stopTracking();
}
assert.ok(loads > 1, `the failed load is tried again (${loads})`);
assert.ok(loads <= 3, `within the failure budget (${loads})`);
assert.equal(removed.length, loads, 'each failed model leaves the scene');
assert.equal(warnings.length, 1, 'the give-up is announced once');
assert.equal(
fixture.billboardColor().alpha,
1,
'the tracked glyph stays the visual',
);
} finally {
Date.now = realNow;
console.warn = realWarn;
fixture.layer.stopTracking();
}
});
}
});
test('a failed fleet model does not condemn later aircraft of its class', async (t) => {
t.mock.timers.enable({ apis: ['Date'], now: 1000 });
const engine = createFakeEngine();
t.after(() => engine.destroy());
const display = createAircraftDisplay(engine, {
layerId: 'flights',
headingOffsetDeg: 180,
drawAtlas,
});
t.after(() => display.destroy());
await display.whenReady();
display.fleet.attach();
const failing = new Set();
engine.fake.script('moverFailed', (id) => failing.has(id));
const spec = (id) => ({
url: 'airliner.glb',
minimumPixelSize: 64,
scale: 1,
id,
});
const first = await display.models.loadFleet(spec('a1'));
display.models.add(first);
failing.add(first.gevBank.set.id); // a 5xx or a network blip on the GLB
test('a failed fleet model does not reload healthy aircraft when another joins its class', async (t) => {
const { display, attempts } = await displayFixture(t);
const first = await addModel(display, 'a');
const healthy = await addModel(display, 'b');
const set = first.gevBank.set;
const next = await addModel(display, 'c');
assert.equal(first.failed, true);
display.models.remove(first);
await assert.rejects(display.models.loadFleet(spec('a2')), /retry/);
assert.equal(healthy.ready, true);
assert.equal(next.ready, true);
assert.equal(next.gevBank.set, set);
t.mock.timers.tick(1500);
const next = await display.models.loadFleet(spec('a2'));
display.models.add(next);
assert.equal(
next.failed,
false,
'the next airliner joined the failed set and never draws its model',
);
assert.equal(first.ready, true);
assert.deepEqual(attempts, [
{ index: 0, attempt: 1 },
{ index: 1, attempt: 1 },
{ index: 2, attempt: 1 },
{ index: 0, attempt: 2 },
]);
});
for (const [name, createFlightState, createRendering] of [
@@ -234,51 +228,52 @@ for (const [name, createFlightState, createRendering] of [
['military', militaryState, militaryRendering],
])
for (const inView of [true, false])
test(`${name}: the fleet tick frees a failed model's slot (${inView ? 'in view' : 'beyond the horizon'})`, () => {
const fleet = fleetTickFixture({ createFlightState, createRendering });
test(`${name}: the fleet retains a failed model through recovery (${inView ? 'in view' : 'beyond the horizon'})`, async (t) => {
const { display } = await displayFixture(t);
const fleet = fleetTickFixture({
createFlightState,
createRendering(options) {
options.services.recession.applyAircraftModelTreatment =
applyAircraftModelTreatment;
return createRendering(options);
},
});
Object.assign(fleet.flightState._renderer.models, display.models);
if (inView)
fleet.camera.positionWC = Cesium.Cartesian3.fromDegrees(1, 0, 100);
fleet.camera.positionWC = geo.cartesian3.fromDegrees(1, 0, 100);
fleet.flightState._models3dEnabled = true;
const model = { failed: true, ready: false, show: false };
const model = await addModel(display, fleet.id);
fleet.flightState._models.set(fleet.id, model);
fleet.tick();
assert.equal(fleet.flightState._models.has(fleet.id), false);
assert.deepEqual(fleet.removed, [model]);
assert.equal(fleet.flightState._models.get(fleet.id), model);
assert.equal(fleet.billboard.show, inView);
assert.equal(model.ready, false);
t.mock.timers.tick(1500);
fleet.tick();
assert.equal(fleet.flightState._models.get(fleet.id), model);
assert.equal(model.ready, true);
assert.equal(model.show, inView);
if (inView) assert.equal(fleet.billboard.show, false);
assert.deepEqual(fleet.removed, []);
});
test('removing failures from a shared bank does not reload until the backoff ends', async (t) => {
t.mock.timers.enable({ apis: ['Date'], now: 1000 });
const engine = createFakeEngine();
t.after(() => engine.destroy());
let builds = 0;
const createSet = engine.movers.createSet;
engine.movers.createSet = (options) => {
if (options.kind === 'model') builds++;
return createSet(options);
};
const display = createAircraftDisplay(engine, {
layerId: 'flights',
drawAtlas,
});
t.after(() => display.destroy());
const spec = { url: 'airliner.glb', minimumPixelSize: 64 };
const first = await display.models.loadFleet({ ...spec, id: 'a' });
const second = await display.models.loadFleet({ ...spec, id: 'b' });
display.models.add(first);
display.models.add(second);
const failedId = first.gevBank.set.id;
engine.fake.script('moverFailed', (id) => id === failedId);
test('removing failed handles retains the shared bank and retries a replacement identity', async (t) => {
const { display, engine } = await displayFixture(t);
const first = await addModel(display, 'a');
const second = await addModel(display, 'b');
const set = first.gevBank.set;
display.models.remove(first);
assert.equal(second.failed, true);
assert.equal(builds, 1, 'another handle in the bank does not trigger a load');
display.models.remove(second);
assert.equal(builds, 1);
t.mock.timers.tick(1499);
await assert.rejects(display.models.loadFleet(spec), /retry/);
t.mock.timers.tick(1);
const retry = await display.models.loadFleet(spec);
assert.equal(builds, 1, 'a load starts only when the handle is admitted');
display.models.add(retry);
assert.equal(builds, 2);
assert.equal(retry.failed, false);
await settle();
assert.equal(set.count(), 0);
assert.equal(set.isDestroyed(), false);
const next = await addModel(display, 'c');
assert.equal(next.gevBank.set, set);
assert.equal(next.failed, true);
t.mock.timers.tick(1500);
assert.equal(next.ready, true);
assert.equal(
engine.movers.list().filter((row) => row.kind === 'model').length,
1,
);
});
src/layers/flights/rendering.js
-15
View File
@@ -727,17 +727,6 @@ export function createRendering({
});
return; // billboard carries the visual until the model is ready
}
if (flightState._trackedModel?.failed) {
// The engine could not load the glTF: the billboard stays the visual
// and the load is tried again after the backoff.
const { url } = flightState._trackedModel.spec ?? {};
_releaseTrackedModel();
parts.tracking._noteTrackedModelLoadFailure(
url,
new Error('model failed to load'),
);
return;
}
if (flightState._trackedModel) {
// Keep the transform current while GPU resources are still loading. Cesium
// can flip ready during scene update after this callback; waiting for ready
@@ -824,10 +813,6 @@ export function createRendering({
const occluder = view.horizon();
const focusTarget = getFocusTarget();
// A model the engine failed to load gives its slot back, in view or not;
// the renderer loads its class again after a backoff.
for (const [icao24, model] of flightState._models)
if (model.failed) _releaseModel(icao24);
// 3D model regime: only when enabled AND the camera is zoomed in past the altitude ceiling.
// Drop all models the moment we leave it (toggled off / zoomed out) so billboards resume.
const useModels = _modelRegimeActive();
src/layers/localAdsb/models.js
-4
View File
@@ -208,10 +208,6 @@ export function createLocalAdsbModels({
for (const [id, marker] of markers) {
const model = models.get(id);
marker.modelOwnsVisual = false;
if (model?.failed) {
release(id);
continue;
}
if (!model || !marker.position) continue;
const position = placement(id, marker);
if (!position || !occluder.isPointVisible(marker.position)) {
src/layers/localAdsb/models.test.mjs
+42 -22
View File
@@ -2,6 +2,11 @@ import test from 'node:test';
import assert from 'node:assert/strict';
import * as geo from '../../geo/index.js';
import { createLocalAdsbModels } from './models.js';
import { createFakeEngine } from '../../globe/testing/fakeEngine.js';
import {
createAircraftDisplay,
aircraftAtlasLayout,
} from '../aircraft/render/display.js';
test('ground snapping excludes the aircraft by its engine pick id', async () => {
const position = geo.cartesian3.fromDegrees(0, 0, 0);
@@ -49,26 +54,35 @@ test('ground snapping excludes the aircraft by its engine pick id', async () =>
models.destroy();
});
test('a failed model frees its slot and leaves the marker visible', async () => {
test('a failed model retains its slot and the marker until the engine retry succeeds', async (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
const engine = createFakeEngine();
const attempts = [];
engine.fake.script('moverFailed', (id, index, attempt) => {
if (!id.includes('-models-')) return false;
attempts.push(attempt);
return attempt === 1;
});
const position = geo.cartesian3.fromDegrees(0, 0, 100);
const removed = [];
let loads = 0;
const renderer = {
models: {
loadFleet: async () => ({ failed: loads++ === 0, ready: loads > 1 }),
add() {},
remove: (model) => removed.push(model),
destroy() {},
setPose() {},
},
view: {
camera: { positionWC: position, positionCartographic: { height: 100 } },
frustumTester: () => () => true,
horizon: () => ({ isPointVisible: () => true }),
},
};
const layout = aircraftAtlasLayout();
const display = createAircraftDisplay(engine, {
layerId: 'local-adsb',
drawAtlas: async () => ({
canvas: { width: layout.width, height: layout.height },
cells: layout.cells,
}),
});
await display.whenReady();
display.fleet.attach();
const models = createLocalAdsbModels({
renderer,
renderer: {
models: display.models,
view: {
camera: { positionWC: position, positionCartographic: { height: 100 } },
frustumTester: () => () => true,
horizon: () => ({ isPointVisible: () => true }),
},
},
color: {},
resolveAsset: (url) => url,
});
@@ -78,15 +92,21 @@ test('a failed model frees its slot and leaves the marker visible', async () =>
models.frame(markers, { models3d: true }, 1000);
await new Promise(setImmediate);
models.frame(markers, { models3d: true }, 1100);
assert.equal(models.size, 0);
assert.equal(removed.length, 1);
const set = engine.movers.list().find((row) => row.kind === 'model');
assert.equal(models.size, 1);
assert.equal(marker.modelOwnsVisual, false);
models.frame(markers, { models3d: true }, 2700);
await new Promise(setImmediate);
t.mock.timers.tick(1500);
models.frame(markers, { models3d: true }, 2800);
assert.equal(models.size, 1);
assert.equal(marker.modelOwnsVisual, true);
assert.deepEqual(attempts, [1, 2]);
assert.equal(
engine.movers.list().find((row) => row.kind === 'model').id,
set.id,
);
} finally {
models.destroy();
display.destroy();
engine.destroy();
}
});
src/layers/military/rendering.js
-15
View File
@@ -705,17 +705,6 @@ export function createRendering({
});
return;
}
if (flightState._trackedModel?.failed) {
// The engine could not load the glTF: the billboard stays the visual
// and the load is tried again after the backoff.
const { url } = flightState._trackedModel.spec ?? {};
_releaseTrackedModel();
parts.tracking._noteTrackedModelLoadFailure(
url,
new Error('model failed to load'),
);
return;
}
if (flightState._trackedModel) {
// Keep the transform current while GPU resources are still loading. Cesium
// can flip ready during scene update after this callback; waiting for ready
@@ -805,10 +794,6 @@ export function createRendering({
const occluder = view.horizon();
const focusTarget = getFocusTarget();
// A model the engine failed to load gives its slot back, in view or not;
// the renderer loads its class again after a backoff.
for (const [icao24, model] of flightState._models)
if (model.failed) _releaseModel(icao24);
// 3D model regime: only when enabled AND the camera is zoomed in past the altitude ceiling.
// Drop all models the moment we leave it (toggled off / zoomed out) so billboards resume.
const useModels = _modelRegimeActive();
src/layers/shared/render/moverBank.js
+1 -12
View File
@@ -35,13 +35,11 @@ export function createMoverBank(engine, { options, write, onCommit }) {
let capacity = 0;
let visible = true;
let dirty = false;
let rebuildPending = false;
const build = (wanted) => {
const next = engine.movers.createSet(options(wanted, ++generation));
set?.destroy();
set = next;
capacity = wanted;
rebuildPending = false;
handles.forEach((handle, index) => write(handle, set, index));
set.setCount(visible ? handles.length : 0);
};
@@ -57,7 +55,7 @@ export function createMoverBank(engine, { options, write, onCommit }) {
handle.gevBank = bank;
handle.gevIndex = handles.length;
handles.push(handle);
if (!set || rebuildPending || handles.length > capacity)
if (!set || handles.length > capacity)
build(
Math.max(
FIRST_CAPACITY,
@@ -83,7 +81,6 @@ export function createMoverBank(engine, { options, write, onCommit }) {
handle.gevBank = null;
handle.gevIndex = -1;
set?.setCount(visible ? handles.length : 0);
if (!handles.length && rebuildPending) bank.rebuild();
bank.touch();
},
/** Writes `fields` of one handle (when it is in this bank). */
@@ -98,14 +95,6 @@ export function createMoverBank(engine, { options, write, onCommit }) {
bank.touch();
},
visible: () => visible,
/** Reload assets on the next add, after the caller's retry backoff. */
rebuild() {
rebuildPending = true;
if (!handles.length) {
set?.destroy();
set = null;
}
},
touch() {
if (!dirty) {
dirty = true;