mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 08:12:24 +00:00
Report model load failures to model sets, and bound the tracked model's retries again
The renderer contract gains an optional event on model draw sets,
`on('loadFailed', (id, message) => ...)`, beside `ready`: a model's url
failed to load, once per failed load. The item stays in the set drawing
what it drew before; removing it and adding it again tries at once. A
renderer that cannot tell a failure may never fire it.
The Cesium draw port fires it once Cesium is done with the frame it
raised the error in, and not for a load that was superseded or whose item
went. The stub fails urls under `stub-fail:`, telling each load once. A
conformance check (`draw.models.loadFailed`) runs on both, with a model
url the dev server does not serve for Cesium.
The flights and military layers use it in place of rewriting every
unloaded model every 5 s. A fleet model whose load failed is removed and
added again 5 s later. The tracked model gets back its per-selection
budget: a failed load is let go and tried again 1.5 s later, and after
three failures the contact stays 2D until another is selected, as
before the models moved onto the draw port. A good load retires the
budget, and a model that loads after all is kept.
This commit is contained in:
1 parent
396e7f0deb
commit
5370feb825
23 files changed
+536
-58
No files matched your search
@@ -30,10 +30,9 @@ const PRELOAD_ID = '~preload';
|
||||
/** Where the preload item waits, unshown. */
|
||||
const NOWHERE = Object.freeze({ x: 0, y: 0, z: 0 });
|
||||
|
||||
/** How often (ms) a model that has not loaded is written again, so a
|
||||
* renderer that retries failed loads on write (as the Cesium one does)
|
||||
* gets the chance even for a parked contact whose item never changes. */
|
||||
const UNREADY_REWRITE_MS = 5000;
|
||||
/** How long (ms) a fleet model whose load failed (`loadFailed`) waits
|
||||
* before it is removed and added again, which loads it again. */
|
||||
const FLEET_MODEL_RETRY_MS = 5000;
|
||||
|
||||
/** A colour as the draw contract's `[r, g, b, a]`, from `{red, green, blue, alpha}`. */
|
||||
function rgba(color) {
|
||||
@@ -68,8 +67,13 @@ function inView(pose, point) {
|
||||
* layer lifetime (`_createModelSets`): the fleet's, one item per modelled
|
||||
* contact, and the tracked contact's (with a hidden item that keeps the
|
||||
* tracked asset loaded). Each item is re-posed and restyled in place; the
|
||||
* layer keeps its own record of each (`modelRecord.js`), and a set's `ready`
|
||||
* says when a model has loaded. Picks name the set and the contact's id.
|
||||
* layer keeps its own record of each (`modelRecord.js`), a set's `ready`
|
||||
* says when a model has loaded and its `loadFailed` when one failed to. A
|
||||
* failed fleet model is loaded again every `FLEET_MODEL_RETRY_MS`; the
|
||||
* tracked model has a per-selection budget (`TRACKED_MODEL_MAX_LOAD_FAILS`
|
||||
* failures, `TRACKED_MODEL_RETRY_BACKOFF_MS` apart), after which the contact
|
||||
* stays an icon until another is selected. Picks name the set and the
|
||||
* contact's id.
|
||||
* @param {object} options
|
||||
* @param {object} options.state The layer state.
|
||||
* @param {object} options.services
|
||||
@@ -79,7 +83,8 @@ function inView(pose, point) {
|
||||
* @param {string} options.logLabel Console prefix, e.g. `Data:Flights`.
|
||||
* @param {string} options.setPrefix The draw sets' id prefix, e.g. `flights`.
|
||||
* @param {object} options.policy The layer's model budgets, radii, pixel
|
||||
* bounds, heading offset and fleet tick interval.
|
||||
* bounds, heading offset, fleet tick interval and tracked-model load
|
||||
* budget.
|
||||
* @param {(klass: string) => object} options.modelSpec The class's GLB, scale,
|
||||
* geometry and tint blend; `headingOffsetDeg` overrides the policy's.
|
||||
* @param {(icao24: string) => object} options.modelColor Model tint
|
||||
@@ -161,11 +166,32 @@ export function createAircraftModels({
|
||||
const offs = [
|
||||
fleet.on('ready', (id) => {
|
||||
const record = state._models.get(id);
|
||||
if (record) record.ready = true;
|
||||
if (!record) return;
|
||||
record.ready = true;
|
||||
// Loaded after all (a renderer may retry on its own): no retry due.
|
||||
record.failedAt = 0;
|
||||
}),
|
||||
fleet.on('loadFailed', (id) => {
|
||||
const record = state._models.get(id);
|
||||
if (record) record.failedAt = Date.now();
|
||||
}),
|
||||
tracked.on('ready', (id) => {
|
||||
if (id === state._trackedIcao && state._trackedModel)
|
||||
state._trackedModel.ready = true;
|
||||
if (id !== state._trackedIcao || !state._trackedModel) return;
|
||||
state._trackedModel.ready = true;
|
||||
// Loaded after all (a renderer may retry on its own): kept.
|
||||
state._trackedModel.failedAt = 0;
|
||||
// A good load retires this selection's failure budget: a contact
|
||||
// that recovers after a transient blip is not one attempt from
|
||||
// giving up.
|
||||
state._trackedModelFailIcao = null;
|
||||
state._trackedModelFailCount = 0;
|
||||
state._trackedModelRetryAtMs = 0;
|
||||
}),
|
||||
tracked.on('loadFailed', (id, message) => {
|
||||
const record = state._trackedModel;
|
||||
if (id !== state._trackedIcao || !record || record.id !== id) return;
|
||||
record.failedAt = Date.now();
|
||||
_noteTrackedModelLoadFailure(record.url, message);
|
||||
}),
|
||||
];
|
||||
state._modelSets = { fleet, tracked, offs };
|
||||
@@ -351,11 +377,7 @@ export function createAircraftModels({
|
||||
if (!set) return;
|
||||
const p = record.position;
|
||||
const t = record.tint ?? WHITE;
|
||||
// A model still loading is rewritten now and then, for the renderer's retry.
|
||||
const loading = record.ready
|
||||
? ''
|
||||
: Math.floor(Date.now() / UNREADY_REWRITE_MS);
|
||||
const key = `${record.url}|${p.x},${p.y},${p.z}|${record.heading}|${record.scale}|${record.minPixelSize}|${t[0]},${t[1]},${t[2]},${t[3]}|${state._irBoost === true}|${spec.blendAmount}|${record.show}|${loading}`;
|
||||
const key = `${record.url}|${p.x},${p.y},${p.z}|${record.heading}|${record.scale}|${record.minPixelSize}|${t[0]},${t[1]},${t[2]},${t[3]}|${state._irBoost === true}|${spec.blendAmount}|${record.show}`;
|
||||
if (key === record.sent) return;
|
||||
record.sent = key;
|
||||
const item = _itemOf(id, record, spec);
|
||||
@@ -363,6 +385,23 @@ export function createAircraftModels({
|
||||
else set.upsert([item]);
|
||||
}
|
||||
|
||||
/** A fleet model whose load failed, removed from its set once
|
||||
* `FLEET_MODEL_RETRY_MS` has passed; the write that follows adds it
|
||||
* again, and the set loads it again. */
|
||||
function _retryFailedFleetModel(set, icao24, record) {
|
||||
if (
|
||||
!set ||
|
||||
!record.failedAt ||
|
||||
Date.now() - record.failedAt < FLEET_MODEL_RETRY_MS
|
||||
)
|
||||
return;
|
||||
record.failedAt = 0;
|
||||
record.sent = null;
|
||||
// A fresh load: not ready until the set says so again.
|
||||
record.ready = false;
|
||||
set.remove([icao24]);
|
||||
}
|
||||
|
||||
/** Atomically hand one fleet contact from its icon to a safely placed,
|
||||
* ready 3D model. Missing/loading models and unresolved terrain always leave
|
||||
* the icon owning the visual, so no render frame can hide both.
|
||||
@@ -386,6 +425,7 @@ export function createAircraftModels({
|
||||
const klass = state.records.data.get(icao24)?.klass;
|
||||
const spec = _modelSpec(klass);
|
||||
const set = state._modelSets?.fleet ?? null;
|
||||
_retryFailedFleetModel(set, icao24, record);
|
||||
const displayPos = _modelDisplayPosition(icao24, pos);
|
||||
if (!displayPos) {
|
||||
showModel(record, false); // no ground evidence → nothing safe to place a depth-tested model at
|
||||
@@ -587,6 +627,33 @@ export function createAircraftModels({
|
||||
icons.setShown(icao24, true);
|
||||
}
|
||||
|
||||
/** Whether the driver may load the tracked model again: an untried
|
||||
* selection, or one under its failure budget past its backoff. */
|
||||
function _trackedModelLoadAllowed(nowMs = Date.now()) {
|
||||
if (state._trackedModelFailIcao !== state._trackedIcao) return true;
|
||||
if (state._trackedModelFailCount >= policy.TRACKED_MODEL_MAX_LOAD_FAILS)
|
||||
return false;
|
||||
return nowMs >= state._trackedModelRetryAtMs;
|
||||
}
|
||||
|
||||
/** Count a failed tracked-model load against the selection's budget and
|
||||
* arm the backoff, or the give-up latch. */
|
||||
function _noteTrackedModelLoadFailure(url, message) {
|
||||
if (state._trackedModelFailIcao !== state._trackedIcao) {
|
||||
state._trackedModelFailIcao = state._trackedIcao;
|
||||
state._trackedModelFailCount = 0;
|
||||
}
|
||||
state._trackedModelFailCount += 1;
|
||||
state._trackedModelRetryAtMs =
|
||||
Date.now() + policy.TRACKED_MODEL_RETRY_BACKOFF_MS;
|
||||
if (state._trackedModelFailCount >= policy.TRACKED_MODEL_MAX_LOAD_FAILS)
|
||||
console.warn(
|
||||
`[${logLabel}] tracked 3D model gave up after ${state._trackedModelFailCount} failed loads of ${url}; ` +
|
||||
'this contact stays 2D until another is selected',
|
||||
message,
|
||||
);
|
||||
}
|
||||
|
||||
/** Drop the standalone tracked-aircraft model. */
|
||||
|
||||
function _releaseTrackedModel() {
|
||||
@@ -615,6 +682,9 @@ export function createAircraftModels({
|
||||
* in, can never stall or freeze the centering (the old model-graphic-on-entity failure mode). */
|
||||
|
||||
function _updateTrackedModel() {
|
||||
// A failed load is let go (the icon carries the contact); the budget
|
||||
// decides when, if ever, it is loaded again.
|
||||
if (state._trackedModel?.failedAt) _releaseTrackedModel();
|
||||
const active =
|
||||
state._trackedIcao &&
|
||||
parts.tracking._trackedModelRegimeActive() &&
|
||||
@@ -640,6 +710,7 @@ export function createAircraftModels({
|
||||
if (state._trackedModel && state._trackedModel.id !== icao24)
|
||||
_releaseTrackedModel();
|
||||
if (!state._trackedModel) {
|
||||
if (!_trackedModelLoadAllowed()) return; // the icon carries it
|
||||
const record = createModelRecord({
|
||||
specKey: _specKeyFor(klass), // class-change sync compares against this
|
||||
bellyM: spec.bellyM,
|
||||
|
||||
@@ -19,6 +19,8 @@ const POLICY = {
|
||||
TRACKED_MODEL_MAX_PX: 600,
|
||||
MODEL_HEADING_OFFSET_DEG: 0,
|
||||
FLEET_DR_INTERVAL_MS: 80,
|
||||
TRACKED_MODEL_MAX_LOAD_FAILS: 3,
|
||||
TRACKED_MODEL_RETRY_BACKOFF_MS: 1500,
|
||||
};
|
||||
|
||||
const SPEC = {
|
||||
@@ -247,7 +249,7 @@ test('a failed fleet write leaves the icons in charge and is sent again', () =>
|
||||
assert.equal(f.shown.get('a1'), false);
|
||||
});
|
||||
|
||||
test('a model that has not loaded is written again for the renderer to retry', () => {
|
||||
test('a fleet model whose load failed is loaded again after a wait, not before', () => {
|
||||
const positions = new Map([['a1', near(1)]]);
|
||||
const f = fleetFixture(positions);
|
||||
const realNow = Date.now;
|
||||
@@ -256,18 +258,39 @@ test('a model that has not loaded is written again for the renderer to retry', (
|
||||
try {
|
||||
f.tick();
|
||||
const writes = f.fleet().writes;
|
||||
clock += 60_000;
|
||||
f.tick();
|
||||
assert.equal(
|
||||
f.fleet().writes,
|
||||
writes,
|
||||
'an unchanged loading model is not rewritten every tick',
|
||||
'a model still loading is not rewritten, however long it takes',
|
||||
);
|
||||
clock += 5_000;
|
||||
f.draw.loadFailed(f.fleetId, 'a1');
|
||||
f.removes.length = 0;
|
||||
clock += 4_999;
|
||||
f.tick();
|
||||
assert.deepEqual(f.removes, [], 'not before the wait');
|
||||
assert.equal(f.shown.get('a1'), true, 'the icon carries the contact');
|
||||
clock += 1;
|
||||
f.tick();
|
||||
assert.deepEqual(f.removes, [['a1']], 'removed once the wait is over');
|
||||
assert.ok(f.fleet().items.has('a1'), 'and added again in the same tick');
|
||||
assert.equal(f.state._models.get('a1').failedAt, 0);
|
||||
assert.equal(f.state._models.get('a1').ready, false, 'a fresh load');
|
||||
f.removes.length = 0;
|
||||
clock += 60_000;
|
||||
f.tick();
|
||||
assert.deepEqual(f.removes, [], 'tried again only after another failure');
|
||||
// A renderer's own retry that loads before the layer's: nothing is due.
|
||||
f.draw.loadFailed(f.fleetId, 'a1');
|
||||
f.draw.ready(f.fleetId, 'a1');
|
||||
clock += 60_000;
|
||||
f.tick();
|
||||
assert.deepEqual(f.removes, [], 'a model that loaded is not tried again');
|
||||
assert.equal(
|
||||
f.fleet().writes,
|
||||
writes + 1,
|
||||
'but it is rewritten every few seconds',
|
||||
f.shown.get('a1'),
|
||||
false,
|
||||
'the model takes over once it loads',
|
||||
);
|
||||
} finally {
|
||||
Date.now = realNow;
|
||||
|
||||
@@ -5,7 +5,8 @@
|
||||
* layer reads what it pushed (and what the set told it) back from itself.
|
||||
*
|
||||
* Write `show` and the placement with {@link showModel} and
|
||||
* {@link placeModel}; the set's `ready` event sets `ready`. The placement is
|
||||
* {@link placeModel}; the set's `ready` event sets `ready`, and its
|
||||
* `loadFailed` event `failedAt`. The placement is
|
||||
* kept as the column-major frame the model is posed in, the same frame a
|
||||
* renderer builds from the item's position and heading, so anchors on the
|
||||
* model can be read from it.
|
||||
@@ -19,6 +20,8 @@ import { eastNorthUp } from '../geo/frames.js';
|
||||
* @property {string|null} url The model asset the layer sends.
|
||||
* @property {boolean} ready Whether the set reported the model loaded and
|
||||
* posed (`ready`).
|
||||
* @property {number} failedAt When (epoch ms) the set last reported the
|
||||
* model's load failed (`loadFailed`); 0 when it has not.
|
||||
* @property {boolean} show Whether the layer last showed the model.
|
||||
* @property {boolean} placed Whether the layer has placed it yet.
|
||||
* @property {Float64Array} matrix Where the layer last placed it: the
|
||||
@@ -55,6 +58,7 @@ export function createModelRecord({
|
||||
id,
|
||||
url,
|
||||
ready: false,
|
||||
failedAt: 0,
|
||||
show: false,
|
||||
placed: false,
|
||||
matrix,
|
||||
|
||||
@@ -345,6 +345,84 @@ for (const fixture of LAYERS) {
|
||||
}
|
||||
});
|
||||
|
||||
test(`${fixture.name}: a tracked model that fails to load is retried within a per-selection budget`, () => {
|
||||
clearTr3bRegistry();
|
||||
fixture.layer.stopTracking();
|
||||
fixture.layer.setParams({ models3d: false });
|
||||
const realNow = Date.now;
|
||||
const realWarn = console.warn;
|
||||
const warnings = [];
|
||||
let now = 7_000_000;
|
||||
Date.now = () => now;
|
||||
console.warn = (...args) => warnings.push(args.join(' '));
|
||||
const viewer = viewerAtHeight(30_000);
|
||||
const icao24 = fixture.name === 'flights' ? 'a0f401' : 'b0f401';
|
||||
try {
|
||||
fixture.seed({ viewer, icao24 });
|
||||
fixture.driveTrackedModel();
|
||||
let tracked = modelSet(fixture, viewer, 'tracked');
|
||||
const fail = () => viewer.draw.loadFailed(tracked.spec.id, icao24, '404');
|
||||
assert.ok(tracked.items.has(icao24), 'the tracked model loads');
|
||||
for (let attempt = 1; attempt <= 3; attempt++) {
|
||||
fail();
|
||||
fixture.driveTrackedModel();
|
||||
assert.equal(tracked.items.has(icao24), false, `failure ${attempt}: let go`);
|
||||
assert.equal(fixture.trackedModel(), null);
|
||||
assert.equal(fixture.trackedBillboardColor()[3], 1, 'the icon carries the contact');
|
||||
now += 1499;
|
||||
fixture.driveTrackedModel();
|
||||
assert.equal(tracked.items.has(icao24), false, 'not before the backoff');
|
||||
now += 1;
|
||||
fixture.driveTrackedModel();
|
||||
assert.equal(tracked.items.has(icao24), attempt < 3,
|
||||
attempt < 3 ? 'loaded again after the backoff' : 'given up after the third failure');
|
||||
}
|
||||
assert.equal(warnings.length, 1, 'giving up is said once');
|
||||
now += 60_000;
|
||||
fixture.driveTrackedModel();
|
||||
assert.equal(tracked.items.has(icao24), false, 'the selection stays 2D');
|
||||
|
||||
// A new selection of the same contact starts with a full budget.
|
||||
fixture.layer.stopTracking();
|
||||
fixture.seed({ viewer, icao24 });
|
||||
fixture.driveTrackedModel();
|
||||
tracked = modelSet(fixture, viewer, 'tracked');
|
||||
assert.ok(tracked.items.has(icao24), 'reselected, it loads again');
|
||||
// A failure the renderer recovers from before the driver runs keeps it.
|
||||
const recovered = fixture.trackedModel();
|
||||
fail();
|
||||
viewer.draw.ready(tracked.spec.id, icao24);
|
||||
fixture.driveTrackedModel();
|
||||
assert.equal(fixture.trackedModel(), recovered, 'a model that loaded after all is kept');
|
||||
assert.ok(tracked.items.has(icao24));
|
||||
fixture.layer.stopTracking();
|
||||
fixture.seed({ viewer, icao24 });
|
||||
fixture.driveTrackedModel();
|
||||
tracked = modelSet(fixture, viewer, 'tracked');
|
||||
// And a good load retires the failures counted so far.
|
||||
fail();
|
||||
fixture.driveTrackedModel();
|
||||
now += 1500;
|
||||
fixture.driveTrackedModel();
|
||||
viewer.draw.ready(tracked.spec.id, icao24);
|
||||
fixture.driveTrackedModel();
|
||||
assert.equal(fixture.trackedModel()?.ready, true);
|
||||
fail();
|
||||
fixture.driveTrackedModel();
|
||||
now += 1500;
|
||||
fixture.driveTrackedModel();
|
||||
fail();
|
||||
fixture.driveTrackedModel();
|
||||
now += 1500;
|
||||
fixture.driveTrackedModel();
|
||||
assert.ok(tracked.items.has(icao24), 'two failures after a good load are under the budget');
|
||||
} finally {
|
||||
Date.now = realNow;
|
||||
console.warn = realWarn;
|
||||
fixture.layer.stopTracking();
|
||||
}
|
||||
});
|
||||
|
||||
test(`${fixture.name}: the IR boost restyles a drawn model in place, flat white and unlit`, () => {
|
||||
clearTr3bRegistry();
|
||||
fixture.layer.stopTracking();
|
||||
|
||||
Reference in new issue
Block a user