Report model load failures to model sets, and bound the tracked model's retries again

The renderer contract gains an optional event on model draw sets,
`on('loadFailed', (id, message) => ...)`, beside `ready`: a model's url
failed to load, once per failed load. The item stays in the set drawing
what it drew before; removing it and adding it again tries at once. A
renderer that cannot tell a failure may never fire it.

The Cesium draw port fires it once Cesium is done with the frame it
raised the error in, and not for a load that was superseded or whose item
went. The stub fails urls under `stub-fail:`, telling each load once. A
conformance check (`draw.models.loadFailed`) runs on both, with a model
url the dev server does not serve for Cesium.

The flights and military layers use it in place of rewriting every
unloaded model every 5 s. A fleet model whose load failed is removed and
added again 5 s later. The tracked model gets back its per-selection
budget: a failed load is let go and tried again 1.5 s later, and after
three failures the contact stays 2D until another is selected, as
before the models moved onto the draw port. A good load retires the
budget, and a model that loads after all is kept.
This commit is contained in:
Sameh Khamis committed 2026-11-07 19:44:15 -08:00
1 parent 396e7f0deb
commit 5370feb825
23 files changed
+536 -58

No files matched your search

+4 -1
View File
@@ -267,7 +267,10 @@ export function createCesiumDraw({
function makeSet(kind, spec, build) {
if (sets.has(spec.id)) throw new Error(`a set with id ${spec.id} is live`);
const items = new Map();
const listeners = { ready: listenerList(report) };
const listeners = {
ready: listenerList(report),
loadFailed: listenerList(report),
};
const record = {
kind,
spec,
+53
View File
@@ -4052,6 +4052,59 @@ test('a model load superseded or failed settles once', async () => {
}
});
test('a failed model load fires loadFailed once, after the frame, with why', async () => {
const { loads, restore } = fakeModelLoads();
try {
const scene = fakeScene();
const part = createCesiumDraw({
scene,
report: (error) => assert.fail(error),
});
const set = part.port.models({ id: 'm' });
const failed = [];
set.on('loadFailed', (id, message) => failed.push([id, message]));
const item = (id, url) => ({
id,
pos: { lon: 0, lat: 0 },
url,
orientation: { heading: 0, pitch: 0, roll: 0 },
});
// A load that rejects (the glTF did not fetch).
set.set([item('a', 'a.glb'), item('b', 'b.glb')]);
loads[0].reject(new Error('404 for a.glb'));
await settle();
assert.deepEqual(failed, [['a', '404 for a.glb']]);
// One that fails in its update: told after that update, not inside it.
loads[1].resolve();
await settle();
loads[1].model.errorEvent.raiseEvent(new Error('bad texture'));
assert.equal(failed.length, 1, 'not inside the model update');
await settle();
assert.deepEqual(failed[1], ['b', 'bad texture']);
// A listener may change the set: removing and adding again loads at once.
set.remove(['a']);
set.upsert([item('a', 'a.glb')]);
assert.equal(loads.length, 3, 'added again, loaded again');
// A load superseded before it fails is not told.
set.upsert([item('a', 'c.glb')]);
loads[2].reject(new Error('late'));
await settle();
assert.equal(failed.length, 2, 'a superseded load does not fire');
// Nor one whose item went before the failure was told.
loads[3].reject('plain reason');
set.remove(['a']);
await settle();
assert.equal(failed.length, 2, 'an item gone does not fire');
set.upsert([item('d', 'd.glb')]);
loads[4].reject({});
await settle();
assert.deepEqual(failed[2], ['d', 'the model did not load']);
set.dispose();
} finally {
restore();
}
});
test('a failed model load is tried again on a later update, after a wait', async () => {
const { loads, restore } = fakeModelLoads();
const realNow = Date.now;
+24 -9
View File
@@ -15,8 +15,9 @@
* measures `minPixelSize`), `scaleByDistance`, `fadeByDistance` (the tint's
* alpha) and the set's `horizonCull`.
*
* A failed load is tried again when its item is next updated, once
* `MODEL_RETRY_MS` has passed.
* A failed load fires `loadFailed` (after the frame Cesium reported it in,
* so a listener may change the set), and is tried again when its item is
* next updated, once `MODEL_RETRY_MS` has passed.
*
* `shading: 'thermal'` draws the model unlit and white-hot (its texture
* left out), then tinted as asked: a hot target, as the thermal views draw
@@ -116,6 +117,13 @@ const tintKey = (tint) =>
? tint
: tint.join(',');
/** Why a load failed, as `loadFailed` reports it. */
function failureMessage(error) {
if (error instanceof Error && error.message) return error.message;
const text = error === undefined || error === null ? '' : String(error);
return text && text !== '[object Object]' ? text : 'the model did not load';
}
/** Whether an item needs the per-frame pass. */
const needsFrame = (item, spec) =>
item.maxPixelSize > 0 ||
@@ -183,8 +191,11 @@ export function createModelSets({
// A load counts until it is ready, fails or is superseded; once.
host.loading += 1;
let settled = false;
const item = entry.item;
const current = () =>
entry.token === token && record.host === host && !host.isDestroyed();
const settleLoad = () => settle(false);
const settle = (failed) => {
const settle = (failed, error) => {
if (settled) return;
settled = true;
if (entry.settleLoad === settleLoad) entry.settleLoad = null;
@@ -193,13 +204,17 @@ export function createModelSets({
entry.failed = true;
entry.failedAt = Date.now();
host.failed += 1;
// Told once Cesium is done with the frame it raised this in; not
// when the item has moved on to another load meanwhile.
const message = failureMessage(error);
queueMicrotask(() => {
if (current() && entry.failed)
record.listeners.loadFailed.fire(item.id, message);
});
}
requestRender();
};
entry.settleLoad = settleLoad;
const item = entry.item;
const current = () =>
entry.token === token && record.host === host && !host.isDestroyed();
Cesium.Model.fromGltfAsync({
url: item.url,
modelMatrix: matrixOf(entry, new Cesium.Matrix4()),
@@ -241,7 +256,7 @@ export function createModelSets({
});
// A failed load is let go (Cesium would keep drawing frames for
// it); the failure stays for `whenSettled` until the item changes.
model.errorEvent.addEventListener(() => {
model.errorEvent.addEventListener((error) => {
if (current() && entry.next === model) {
entry.next = null;
// Cesium raises this inside the model's update: removing it
@@ -251,11 +266,11 @@ export function createModelSets({
host.remove(model);
});
}
settle(true);
settle(true, error);
});
requestRender();
},
() => settle(true),
(error) => settle(true, error),
);
}
+10 -4
View File
@@ -45,6 +45,7 @@ export const CESIUM_CHECKS = new Set([
'draw.lines.segments',
'draw.ids',
'draw.models',
'draw.models.loadFailed',
'draw.models.tint',
'draw.models.update',
'draw.motion',
@@ -113,11 +114,16 @@ function nextFrame() {
}
/**
* @param {{name?: string, model?: string}} [options] `model`: a glTF URL
* for the model checks.
* @param {{name?: string, model?: string, missingModel?: string}} [options]
* `model`: a glTF URL for the model checks; `missingModel`: one that
* fails to load, for the failure check.
* @returns {import('../conformance/index.js').ConformanceHarness}
*/
export function createCesiumHarness({ name = 'cesium', model } = {}) {
export function createCesiumHarness({
name = 'cesium',
model,
missingModel,
} = {}) {
return {
name,
async create({ viewport, capabilities }) {
@@ -157,6 +163,6 @@ export function createCesiumHarness({ name = 'cesium', model } = {}) {
toEcef,
tolerance: DEFAULT_TOLERANCE,
flightPaths: 'endpoints',
...(model ? { assets: { model } } : {}),
...(model ? { assets: { model, missingModel } } : {}),
};
}
+54
View File
@@ -1754,6 +1754,60 @@ export function portChecks() {
},
{ capability: 'models' },
),
check(
'draw.models.loadFailed',
'a model set says when a model fails to load, and loads it when added again',
async (h, r) => {
if (!h.assets?.model || !h.assets?.missingModel)
return { skip: 'no missing model asset in this harness' };
const set = r.draw.models({ id: 'conf:models.loadFailed' });
const ready = [];
const failed = [];
set.on('ready', (id) => ready.push(id));
set.on('loadFailed', (id, message) => failed.push([id, message]));
const model = (id, url) => ({
id,
pos: { ...AUSTIN, height: 0 },
url,
orientation: { heading: 0, pitch: 0, roll: 0 },
});
const until = async (done) => {
for (let i = 0; i < 600 && !done(); i++) await drawn(h, r, 1);
};
set.set([
model('bad', h.assets.missingModel),
model('good', h.assets.model),
]);
await until(() => failed.length >= 1 && ready.length >= 1);
deepEqual(ready, ['good'], 'the good one is ready');
deepEqual(
failed.map(([id]) => id),
['bad'],
'the missing one failed',
);
ok(
typeof failed[0]?.[1] === 'string' && failed[0][1].length > 0,
'with a message',
);
equal(set.size(), 2, 'the failed item stays in the set');
// Removed and added again, it is tried at once, and fails again.
set.remove(['bad']);
set.upsert([model('bad', h.assets.missingModel)]);
await until(() => failed.length >= 2);
deepEqual(
failed.map(([id]) => id),
['bad', 'bad'],
'tried again when added again',
);
// Pointed at a url that loads, it is ready.
set.upsert([model('bad', h.assets.model)]);
await until(() => ready.length >= 2);
deepEqual(ready, ['good', 'bad'], 'a good url loads');
equal(failed.length, 2, 'no failure for a good url');
set.dispose();
},
{ capability: 'models' },
),
check(
'draw.models.update',
'a model moves, turns and restyles in place; only a new url loads it again',
+31
View File
@@ -143,6 +143,37 @@ test('the capability and frame-step lists are the types’ own', () => {
);
});
test('a draw set’s events: ready with the id, loadFailed with the id and why', () => {
const on = checker
.getTypeAtLocation(declared('DrawSet').name)
.getProperty('on');
const signatures = checker
.getTypeOfSymbolAtLocation(on, declared('DrawSet'))
.getCallSignatures();
const events = Object.fromEntries(
signatures.map((signature) => {
const [event, cb] = signature.getParameters();
const callback = checker
.getTypeOfSymbolAtLocation(cb, cb.valueDeclaration)
.getCallSignatures()[0];
return [
checker.getTypeOfSymbolAtLocation(event, event.valueDeclaration).value,
callback
.getParameters()
.map((p) =>
checker.typeToString(
checker.getTypeOfSymbolAtLocation(p, p.valueDeclaration),
),
),
];
}),
);
assert.deepEqual(events, {
ready: ['string'],
loadFailed: ['string', 'string'],
});
});
test('the shape check names what is missing and what a capability does not allow', () => {
assert.deepEqual(checkRendererShape(null), ['renderer is not an object']);
assert.deepEqual(checkRendererShape({}), [
+9
View File
@@ -507,6 +507,15 @@ export interface DrawSet<T> {
show: boolean;
/** `models`: a model has loaded and is posed. */
on(event: 'ready', cb: (id: string) => void): Off;
/**
* `models`: a model's url failed to load (`message` says why), once per
* failed load. The item stays in the set, drawing what it drew before (a
* new item draws nothing); a renderer may try the url again when the item
* is next updated. Removing the item and adding it again tries at once.
* Optional: where the renderer cannot tell a failure it may never fire it, so
* a fallback must not wait on it.
*/
on(event: 'loadFailed', cb: (id: string, message: string) => void): Off;
dispose(): void;
}
+7 -3
View File
@@ -4,7 +4,7 @@
* the Node test and the browser page alike.
*/
import { DEFAULT_TOLERANCE } from '../conformance/index.js';
import { createStubRenderer } from './index.js';
import { createStubRenderer, STUB_FAILING_MODEL_SCHEME } from './index.js';
import { toEcef } from './math.js';
/** @returns {import('../conformance/index.js').ConformanceHarness} */
@@ -23,7 +23,11 @@ export function createStubHarness(name = 'stub') {
pointer: (r, type, x, y, extra) => r.testing.pointer(type, x, y, extra),
scriptHeights: (r, fn) => r.testing.setHeights(fn),
wallTime: (r) => r.testing.wallTime(),
// The stub loads nothing: any url is a model it draws.
assets: { model: 'stub:model.glb' },
// The stub loads nothing: any url is a model it draws, but one under
// its failing scheme.
assets: {
model: 'stub:model.glb',
missingModel: `${STUB_FAILING_MODEL_SCHEME}model.glb`,
},
};
}
+34 -6
View File
@@ -34,6 +34,12 @@ import {
const FRAME_MS = 1000 / 60;
/**
* Model urls under this scheme fail to load (`loadFailed`), so tests can
* see a failure; the stub loads nothing, and every other url is a model.
*/
export const STUB_FAILING_MODEL_SCHEME = 'stub-fail:';
/** Every setting the stub accepts, with its descriptor. */
const KNOBS = Object.freeze([
{ key: 'pixelRatio', kind: 'number', min: 0, max: 4, default: 0, cost: 1 },
@@ -314,21 +320,37 @@ export function createStubRenderer(options = {}) {
items: new Map(),
show: spec.show ?? true,
readyListeners: [],
loadFailedListeners: [],
};
sets.set(spec.id, record);
log('draw', kind, [spec]);
// A model is ready once per load: when it is new or its url changes,
// not when it is moved or restyled.
// not when it is moved or restyled. One under the failing scheme fails
// its load instead.
// Each load is told once: a listener that starts another load of the
// same item (or removes it) supersedes the one queued here.
const loadTokens = new Map();
const loads = (items) =>
kind === 'models'
? items
.filter(
(item) => record.items.get(String(item.id))?.url !== item.url,
)
.map((item) => String(item.id))
.map((item) => {
const token = {};
loadTokens.set(String(item.id), token);
return [String(item.id), token];
})
: [];
const ready = (ids) => {
for (const id of ids) deliver(record.readyListeners, id);
const ready = (loading) => {
for (const [id, token] of loading) {
if (loadTokens.get(id) !== token || !record.items.has(id)) continue;
loadTokens.delete(id);
const url = String(record.items.get(id).url ?? '');
if (url.startsWith(STUB_FAILING_MODEL_SCHEME))
deliver(record.loadFailedListeners, id, `${url} does not load`);
else deliver(record.readyListeners, id);
}
};
const handle = {
id: spec.id,
@@ -369,9 +391,15 @@ export function createStubRenderer(options = {}) {
requestRender('draw');
},
on(event, cb) {
if (event !== 'ready') return () => {};
const list =
event === 'ready'
? record.readyListeners
: event === 'loadFailed'
? record.loadFailedListeners
: null;
if (!list) return () => {};
const entry = { cb, live: true };
record.readyListeners.push(entry);
list.push(entry);
return () => {
entry.live = false;
};
+23
View File
@@ -74,3 +74,26 @@ test('a throwing listener with no error listener is reported and the frame goes
globalThis.reportError = saved;
}
});
test('a model load is told once, even when a listener loads it again', () => {
const r = createStubRenderer({ capabilities: ['models'] });
const set = r.draw.models({ id: 'm' });
const told = [];
const model = (id, url) => ({
id,
pos: { lon: 0, lat: 0 },
url,
orientation: { heading: 0, pitch: 0, roll: 0 },
});
set.on('ready', (id) => told.push(`ready ${id}`));
set.on('loadFailed', (id) => {
told.push(`failed ${id}`);
// Pointing b at another failing url is a new load, told by its upsert.
if (id === 'a') set.upsert([model('b', 'stub-fail:other.glb')]);
});
set.set([model('a', 'stub-fail:a.glb'), model('b', 'stub-fail:b.glb')]);
assert.deepEqual(told, ['failed a', 'failed b']);
set.remove(['a']);
set.upsert([model('a', 'good.glb')]);
assert.deepEqual(told.slice(2), ['ready a']);
});