Report model load failures to model sets, and bound the tracked model's retries again

The renderer contract gains an optional event on model draw sets,
`on('loadFailed', (id, message) => ...)`, beside `ready`: a model's url
failed to load, once per failed load. The item stays in the set drawing
what it drew before; removing it and adding it again tries at once. A
renderer that cannot tell a failure may never fire it.

The Cesium draw port fires it once Cesium is done with the frame it
raised the error in, and not for a load that was superseded or whose item
went. The stub fails urls under `stub-fail:`, telling each load once. A
conformance check (`draw.models.loadFailed`) runs on both, with a model
url the dev server does not serve for Cesium.

The flights and military layers use it in place of rewriting every
unloaded model every 5 s. A fleet model whose load failed is removed and
added again 5 s later. The tracked model gets back its per-selection
budget: a failed load is let go and tried again 1.5 s later, and after
three failures the contact stays 2D until another is selected, as
before the models moved onto the draw port. A good load retires the
budget, and a model that loads after all is kept.
This commit is contained in:
Sameh Khamis committed 2026-11-07 19:44:15 -08:00
1 parent 396e7f0deb
commit 5370feb825
23 files changed
+536 -58

No files matched your search

+86 -15
View File
@@ -30,10 +30,9 @@ const PRELOAD_ID = '~preload';
/** Where the preload item waits, unshown. */
const NOWHERE = Object.freeze({ x: 0, y: 0, z: 0 });
/** How often (ms) a model that has not loaded is written again, so a
* renderer that retries failed loads on write (as the Cesium one does)
* gets the chance even for a parked contact whose item never changes. */
const UNREADY_REWRITE_MS = 5000;
/** How long (ms) a fleet model whose load failed (`loadFailed`) waits
* before it is removed and added again, which loads it again. */
const FLEET_MODEL_RETRY_MS = 5000;
/** A colour as the draw contract's `[r, g, b, a]`, from `{red, green, blue, alpha}`. */
function rgba(color) {
@@ -68,8 +67,13 @@ function inView(pose, point) {
* layer lifetime (`_createModelSets`): the fleet's, one item per modelled
* contact, and the tracked contact's (with a hidden item that keeps the
* tracked asset loaded). Each item is re-posed and restyled in place; the
* layer keeps its own record of each (`modelRecord.js`), and a set's `ready`
* says when a model has loaded. Picks name the set and the contact's id.
* layer keeps its own record of each (`modelRecord.js`), a set's `ready`
* says when a model has loaded and its `loadFailed` when one failed to. A
* failed fleet model is loaded again every `FLEET_MODEL_RETRY_MS`; the
* tracked model has a per-selection budget (`TRACKED_MODEL_MAX_LOAD_FAILS`
* failures, `TRACKED_MODEL_RETRY_BACKOFF_MS` apart), after which the contact
* stays an icon until another is selected. Picks name the set and the
* contact's id.
* @param {object} options
* @param {object} options.state The layer state.
* @param {object} options.services
@@ -79,7 +83,8 @@ function inView(pose, point) {
* @param {string} options.logLabel Console prefix, e.g. `Data:Flights`.
* @param {string} options.setPrefix The draw sets' id prefix, e.g. `flights`.
* @param {object} options.policy The layer's model budgets, radii, pixel
* bounds, heading offset and fleet tick interval.
* bounds, heading offset, fleet tick interval and tracked-model load
* budget.
* @param {(klass: string) => object} options.modelSpec The class's GLB, scale,
* geometry and tint blend; `headingOffsetDeg` overrides the policy's.
* @param {(icao24: string) => object} options.modelColor Model tint
@@ -161,11 +166,32 @@ export function createAircraftModels({
const offs = [
fleet.on('ready', (id) => {
const record = state._models.get(id);
if (record) record.ready = true;
if (!record) return;
record.ready = true;
// Loaded after all (a renderer may retry on its own): no retry due.
record.failedAt = 0;
}),
fleet.on('loadFailed', (id) => {
const record = state._models.get(id);
if (record) record.failedAt = Date.now();
}),
tracked.on('ready', (id) => {
if (id === state._trackedIcao && state._trackedModel)
state._trackedModel.ready = true;
if (id !== state._trackedIcao || !state._trackedModel) return;
state._trackedModel.ready = true;
// Loaded after all (a renderer may retry on its own): kept.
state._trackedModel.failedAt = 0;
// A good load retires this selection's failure budget: a contact
// that recovers after a transient blip is not one attempt from
// giving up.
state._trackedModelFailIcao = null;
state._trackedModelFailCount = 0;
state._trackedModelRetryAtMs = 0;
}),
tracked.on('loadFailed', (id, message) => {
const record = state._trackedModel;
if (id !== state._trackedIcao || !record || record.id !== id) return;
record.failedAt = Date.now();
_noteTrackedModelLoadFailure(record.url, message);
}),
];
state._modelSets = { fleet, tracked, offs };
@@ -351,11 +377,7 @@ export function createAircraftModels({
if (!set) return;
const p = record.position;
const t = record.tint ?? WHITE;
// A model still loading is rewritten now and then, for the renderer's retry.
const loading = record.ready
? ''
: Math.floor(Date.now() / UNREADY_REWRITE_MS);
const key = `${record.url}|${p.x},${p.y},${p.z}|${record.heading}|${record.scale}|${record.minPixelSize}|${t[0]},${t[1]},${t[2]},${t[3]}|${state._irBoost === true}|${spec.blendAmount}|${record.show}|${loading}`;
const key = `${record.url}|${p.x},${p.y},${p.z}|${record.heading}|${record.scale}|${record.minPixelSize}|${t[0]},${t[1]},${t[2]},${t[3]}|${state._irBoost === true}|${spec.blendAmount}|${record.show}`;
if (key === record.sent) return;
record.sent = key;
const item = _itemOf(id, record, spec);
@@ -363,6 +385,23 @@ export function createAircraftModels({
else set.upsert([item]);
}
/** A fleet model whose load failed, removed from its set once
* `FLEET_MODEL_RETRY_MS` has passed; the write that follows adds it
* again, and the set loads it again. */
function _retryFailedFleetModel(set, icao24, record) {
if (
!set ||
!record.failedAt ||
Date.now() - record.failedAt < FLEET_MODEL_RETRY_MS
)
return;
record.failedAt = 0;
record.sent = null;
// A fresh load: not ready until the set says so again.
record.ready = false;
set.remove([icao24]);
}
/** Atomically hand one fleet contact from its icon to a safely placed,
* ready 3D model. Missing/loading models and unresolved terrain always leave
* the icon owning the visual, so no render frame can hide both.
@@ -386,6 +425,7 @@ export function createAircraftModels({
const klass = state.records.data.get(icao24)?.klass;
const spec = _modelSpec(klass);
const set = state._modelSets?.fleet ?? null;
_retryFailedFleetModel(set, icao24, record);
const displayPos = _modelDisplayPosition(icao24, pos);
if (!displayPos) {
showModel(record, false); // no ground evidence → nothing safe to place a depth-tested model at
@@ -587,6 +627,33 @@ export function createAircraftModels({
icons.setShown(icao24, true);
}
/** Whether the driver may load the tracked model again: an untried
* selection, or one under its failure budget past its backoff. */
function _trackedModelLoadAllowed(nowMs = Date.now()) {
if (state._trackedModelFailIcao !== state._trackedIcao) return true;
if (state._trackedModelFailCount >= policy.TRACKED_MODEL_MAX_LOAD_FAILS)
return false;
return nowMs >= state._trackedModelRetryAtMs;
}
/** Count a failed tracked-model load against the selection's budget and
* arm the backoff, or the give-up latch. */
function _noteTrackedModelLoadFailure(url, message) {
if (state._trackedModelFailIcao !== state._trackedIcao) {
state._trackedModelFailIcao = state._trackedIcao;
state._trackedModelFailCount = 0;
}
state._trackedModelFailCount += 1;
state._trackedModelRetryAtMs =
Date.now() + policy.TRACKED_MODEL_RETRY_BACKOFF_MS;
if (state._trackedModelFailCount >= policy.TRACKED_MODEL_MAX_LOAD_FAILS)
console.warn(
`[${logLabel}] tracked 3D model gave up after ${state._trackedModelFailCount} failed loads of ${url}; ` +
'this contact stays 2D until another is selected',
message,
);
}
/** Drop the standalone tracked-aircraft model. */
function _releaseTrackedModel() {
@@ -615,6 +682,9 @@ export function createAircraftModels({
* in, can never stall or freeze the centering (the old model-graphic-on-entity failure mode). */
function _updateTrackedModel() {
// A failed load is let go (the icon carries the contact); the budget
// decides when, if ever, it is loaded again.
if (state._trackedModel?.failedAt) _releaseTrackedModel();
const active =
state._trackedIcao &&
parts.tracking._trackedModelRegimeActive() &&
@@ -640,6 +710,7 @@ export function createAircraftModels({
if (state._trackedModel && state._trackedModel.id !== icao24)
_releaseTrackedModel();
if (!state._trackedModel) {
if (!_trackedModelLoadAllowed()) return; // the icon carries it
const record = createModelRecord({
specKey: _specKeyFor(klass), // class-change sync compares against this
bellyM: spec.bellyM,
+29 -6
View File
@@ -19,6 +19,8 @@ const POLICY = {
TRACKED_MODEL_MAX_PX: 600,
MODEL_HEADING_OFFSET_DEG: 0,
FLEET_DR_INTERVAL_MS: 80,
TRACKED_MODEL_MAX_LOAD_FAILS: 3,
TRACKED_MODEL_RETRY_BACKOFF_MS: 1500,
};
const SPEC = {
@@ -247,7 +249,7 @@ test('a failed fleet write leaves the icons in charge and is sent again', () =>
assert.equal(f.shown.get('a1'), false);
});
test('a model that has not loaded is written again for the renderer to retry', () => {
test('a fleet model whose load failed is loaded again after a wait, not before', () => {
const positions = new Map([['a1', near(1)]]);
const f = fleetFixture(positions);
const realNow = Date.now;
@@ -256,18 +258,39 @@ test('a model that has not loaded is written again for the renderer to retry', (
try {
f.tick();
const writes = f.fleet().writes;
clock += 60_000;
f.tick();
assert.equal(
f.fleet().writes,
writes,
'an unchanged loading model is not rewritten every tick',
'a model still loading is not rewritten, however long it takes',
);
clock += 5_000;
f.draw.loadFailed(f.fleetId, 'a1');
f.removes.length = 0;
clock += 4_999;
f.tick();
assert.deepEqual(f.removes, [], 'not before the wait');
assert.equal(f.shown.get('a1'), true, 'the icon carries the contact');
clock += 1;
f.tick();
assert.deepEqual(f.removes, [['a1']], 'removed once the wait is over');
assert.ok(f.fleet().items.has('a1'), 'and added again in the same tick');
assert.equal(f.state._models.get('a1').failedAt, 0);
assert.equal(f.state._models.get('a1').ready, false, 'a fresh load');
f.removes.length = 0;
clock += 60_000;
f.tick();
assert.deepEqual(f.removes, [], 'tried again only after another failure');
// A renderer's own retry that loads before the layer's: nothing is due.
f.draw.loadFailed(f.fleetId, 'a1');
f.draw.ready(f.fleetId, 'a1');
clock += 60_000;
f.tick();
assert.deepEqual(f.removes, [], 'a model that loaded is not tried again');
assert.equal(
f.fleet().writes,
writes + 1,
'but it is rewritten every few seconds',
f.shown.get('a1'),
false,
'the model takes over once it loads',
);
} finally {
Date.now = realNow;
+5 -1
View File
@@ -5,7 +5,8 @@
* layer reads what it pushed (and what the set told it) back from itself.
*
* Write `show` and the placement with {@link showModel} and
* {@link placeModel}; the set's `ready` event sets `ready`. The placement is
* {@link placeModel}; the set's `ready` event sets `ready`, and its
* `loadFailed` event `failedAt`. The placement is
* kept as the column-major frame the model is posed in, the same frame a
* renderer builds from the item's position and heading, so anchors on the
* model can be read from it.
@@ -19,6 +20,8 @@ import { eastNorthUp } from '../geo/frames.js';
* @property {string|null} url The model asset the layer sends.
* @property {boolean} ready Whether the set reported the model loaded and
* posed (`ready`).
* @property {number} failedAt When (epoch ms) the set last reported the
* model's load failed (`loadFailed`); 0 when it has not.
* @property {boolean} show Whether the layer last showed the model.
* @property {boolean} placed Whether the layer has placed it yet.
* @property {Float64Array} matrix Where the layer last placed it: the
@@ -55,6 +58,7 @@ export function createModelRecord({
id,
url,
ready: false,
failedAt: 0,
show: false,
placed: false,
matrix,
+78
View File
@@ -345,6 +345,84 @@ for (const fixture of LAYERS) {
}
});
test(`${fixture.name}: a tracked model that fails to load is retried within a per-selection budget`, () => {
clearTr3bRegistry();
fixture.layer.stopTracking();
fixture.layer.setParams({ models3d: false });
const realNow = Date.now;
const realWarn = console.warn;
const warnings = [];
let now = 7_000_000;
Date.now = () => now;
console.warn = (...args) => warnings.push(args.join(' '));
const viewer = viewerAtHeight(30_000);
const icao24 = fixture.name === 'flights' ? 'a0f401' : 'b0f401';
try {
fixture.seed({ viewer, icao24 });
fixture.driveTrackedModel();
let tracked = modelSet(fixture, viewer, 'tracked');
const fail = () => viewer.draw.loadFailed(tracked.spec.id, icao24, '404');
assert.ok(tracked.items.has(icao24), 'the tracked model loads');
for (let attempt = 1; attempt <= 3; attempt++) {
fail();
fixture.driveTrackedModel();
assert.equal(tracked.items.has(icao24), false, `failure ${attempt}: let go`);
assert.equal(fixture.trackedModel(), null);
assert.equal(fixture.trackedBillboardColor()[3], 1, 'the icon carries the contact');
now += 1499;
fixture.driveTrackedModel();
assert.equal(tracked.items.has(icao24), false, 'not before the backoff');
now += 1;
fixture.driveTrackedModel();
assert.equal(tracked.items.has(icao24), attempt < 3,
attempt < 3 ? 'loaded again after the backoff' : 'given up after the third failure');
}
assert.equal(warnings.length, 1, 'giving up is said once');
now += 60_000;
fixture.driveTrackedModel();
assert.equal(tracked.items.has(icao24), false, 'the selection stays 2D');
// A new selection of the same contact starts with a full budget.
fixture.layer.stopTracking();
fixture.seed({ viewer, icao24 });
fixture.driveTrackedModel();
tracked = modelSet(fixture, viewer, 'tracked');
assert.ok(tracked.items.has(icao24), 'reselected, it loads again');
// A failure the renderer recovers from before the driver runs keeps it.
const recovered = fixture.trackedModel();
fail();
viewer.draw.ready(tracked.spec.id, icao24);
fixture.driveTrackedModel();
assert.equal(fixture.trackedModel(), recovered, 'a model that loaded after all is kept');
assert.ok(tracked.items.has(icao24));
fixture.layer.stopTracking();
fixture.seed({ viewer, icao24 });
fixture.driveTrackedModel();
tracked = modelSet(fixture, viewer, 'tracked');
// And a good load retires the failures counted so far.
fail();
fixture.driveTrackedModel();
now += 1500;
fixture.driveTrackedModel();
viewer.draw.ready(tracked.spec.id, icao24);
fixture.driveTrackedModel();
assert.equal(fixture.trackedModel()?.ready, true);
fail();
fixture.driveTrackedModel();
now += 1500;
fixture.driveTrackedModel();
fail();
fixture.driveTrackedModel();
now += 1500;
fixture.driveTrackedModel();
assert.ok(tracked.items.has(icao24), 'two failures after a good load are under the budget');
} finally {
Date.now = realNow;
console.warn = realWarn;
fixture.layer.stopTracking();
}
});
test(`${fixture.name}: the IR boost restyles a drawn model in place, flat white and unlit`, () => {
clearTr3bRegistry();
fixture.layer.stopTracking();