diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index cfc18f7dd..87da00f3e 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -207,6 +207,7 @@ "modules": [ "src/app/viewer.js", "src/app/atmosphereCompat.js", + "src/app/renderRecovery.js", "src/globeInView.js", "src/sky/starrySky.js", "src/sky/stars.js", @@ -2610,6 +2611,7 @@ "src/sky/starrySky.js", "src/sky/stars.js", "src/app/atmosphereCompat.js", + "src/app/renderRecovery.js", "src/bloom.js", "src/camera.js", "src/cameraGroundGuard.js", diff --git a/src/app/renderRecovery.js b/src/app/renderRecovery.js new file mode 100644 index 000000000..d4388b05f --- /dev/null +++ b/src/app/renderRecovery.js @@ -0,0 +1,155 @@ +/** + * Keeps one bad frame from stopping the globe for good. + * + * Cesium's render loop stops itself on the first error a frame throws and + * leaves an error panel over the canvas; nothing restarts it short of a + * reload. Two things here: + * + * 1. `guardDegenerateRotation` removes one such error at its cause. Zooming + * with the wheel above 1,000 km, Cesium turns the globe from the point under + * the screen centre towards the point under the pointer + * (`ScreenSpaceCameraController` `handleZoom`): it normalizes both points, + * takes their dot product, and rotates about their cross product when the + * dot product is below 1. With the pointer on the centre pixel the two + * points are the same, but a unit vector's dot product with itself often + * rounds to 0.9999999999999999, so the check passes and the cross product is + * the zero vector. `Camera.rotate` then normalizes it and throws + * "normalized result is not a number" from `Scene.initializeFrame`, which + * runs outside the scene's own error handling. Builds without Cesium's + * debug checks would write NaN into the camera instead. + * + * 2. `installRenderRecovery` restarts the loop after any error that stops it, + * from the last pose a frame was drawn with, a bounded number of times. + */ +import * as Cesium from 'cesium'; +import { governorRequestRender } from '../renderGovernor.js'; + +/** Restarts allowed within RECOVERY_WINDOW_MS before the error is left to stand. */ +export const RECOVERY_LIMIT = 3; +/** An error that keeps coming back this fast is not one a restart gets past. */ +export const RECOVERY_WINDOW_MS = 10_000; + +/** + * Make `camera.rotate` ignore an axis it cannot normalize (zero or not finite) + * and an angle that is not finite. There is nothing to rotate about, so the + * camera stays where it is and the caller carries on; Cesium's wheel zoom + * still moves the camera in after the skipped turn. + * @param {Cesium.Camera} camera + * @returns {() => void} Restore the camera's own rotate. + */ +export function guardDegenerateRotation(camera) { + const own = Object.hasOwn(camera, 'rotate'); + const rotate = camera.rotate; + const guarded = function (axis, angle) { + if (axis) { + const magnitudeSquared = Cesium.Cartesian3.magnitudeSquared(axis); + if ( + !(magnitudeSquared > 0) || + !Number.isFinite(magnitudeSquared) || + (angle !== undefined && !Number.isFinite(angle)) + ) + return; + } + return rotate.call(this, axis, angle); + }; + camera.rotate = guarded; + return () => { + if (camera.rotate !== guarded) return; + if (own) camera.rotate = rotate; + else delete camera.rotate; + }; +} + +function finite(cartesian) { + return ( + Number.isFinite(cartesian?.x) && + Number.isFinite(cartesian?.y) && + Number.isFinite(cartesian?.z) + ); +} + +/** + * Restart Cesium's render loop when an error stops it. + * + * Cesium reports a stopped loop only through `showErrorPanel`: errors thrown + * while a frame is prepared (camera input, in `Scene.initializeFrame`) never + * reach `scene.renderError`, so that event alone would miss them. The panel + * call is the one place both kinds of stop pass through, after the loop is + * already off. + * + * On a stop this puts the camera back on the last pose a frame was drawn + * with, discards the input that was being applied (Cesium skips its own reset + * when the frame throws, so the same gesture would otherwise run again), and + * turns the loop back on. After RECOVERY_LIMIT restarts inside + * RECOVERY_WINDOW_MS the error is shown as before and rendering stays stopped. + * @param {Cesium.Viewer} viewer + * @param {object} [options] + * @param {() => number} [options.now] + * @param {(reason: string) => void} [options.requestRender] + * @returns {() => void} Idempotent disposer. + */ +export function installRenderRecovery( + viewer, + { now = () => performance.now(), requestRender = governorRequestRender } = {}, +) { + const widget = viewer?.cesiumWidget; + const scene = viewer?.scene; + const camera = viewer?.camera; + if (!widget?.showErrorPanel || !scene?.postRender || !camera) + throw new TypeError('A complete Cesium viewer is required'); + + // The pose of the last frame that drew: three vector copies a frame. + const position = new Cesium.Cartesian3(); + const direction = new Cesium.Cartesian3(); + const up = new Cesium.Cartesian3(); + let hasPose = false; + const removePostRender = scene.postRender.addEventListener(() => { + if (!finite(camera.position) || !finite(camera.direction)) return; + Cesium.Cartesian3.clone(camera.position, position); + Cesium.Cartesian3.clone(camera.direction, direction); + Cesium.Cartesian3.clone(camera.up, up); + hasPose = true; + }); + + const recoveredAt = []; + const own = Object.hasOwn(widget, 'showErrorPanel'); + const showErrorPanel = widget.showErrorPanel; + const recovering = function (title, message, error) { + const at = now(); + while (recoveredAt.length && at - recoveredAt[0] > RECOVERY_WINDOW_MS) + recoveredAt.shift(); + // Only a stopped loop is restarted; other panels (a failed start-up, a + // caller's own message) are shown as they are. + if ( + viewer.useDefaultRenderLoop !== false || + recoveredAt.length >= RECOVERY_LIMIT + ) + return showErrorPanel.call(this, title, message, error); + recoveredAt.push(at); + console.error( + `[viewer] render loop stopped by an error; restarting (${recoveredAt.length} of ${RECOVERY_LIMIT})`, + error, + ); + if (hasPose) { + Cesium.Cartesian3.clone(position, camera.position); + Cesium.Cartesian3.clone(direction, camera.direction); + Cesium.Cartesian3.clone(up, camera.up); + Cesium.Cartesian3.cross(camera.direction, camera.up, camera.right); + } + // Cesium has no public way to drop the input a failed frame left queued. + scene.screenSpaceCameraController?._aggregator?.reset?.(); + viewer.useDefaultRenderLoop = true; + requestRender('render-recovery'); + }; + widget.showErrorPanel = recovering; + + let active = true; + return () => { + if (!active) return; + active = false; + removePostRender?.(); + if (widget.showErrorPanel !== recovering) return; + if (own) widget.showErrorPanel = showErrorPanel; + else delete widget.showErrorPanel; + }; +} diff --git a/src/app/renderRecovery.test.mjs b/src/app/renderRecovery.test.mjs new file mode 100644 index 000000000..47961893e --- /dev/null +++ b/src/app/renderRecovery.test.mjs @@ -0,0 +1,220 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import * as Cesium from 'cesium'; +import { + guardDegenerateRotation, + installRenderRecovery, + RECOVERY_LIMIT, + RECOVERY_WINDOW_MS, +} from './renderRecovery.js'; + +/** Cesium's own Camera.rotate on a camera-shaped object, with no scene behind it. */ +function cameraFixture() { + const camera = Object.create(Cesium.Camera.prototype); + camera.position = new Cesium.Cartesian3(28_378_137, 0, 0); + camera.direction = new Cesium.Cartesian3(-1, 0, 0); + camera.up = new Cesium.Cartesian3(0, 0, 1); + camera.right = new Cesium.Cartesian3(0, 1, 0); + camera._adjustOrthographicFrustum = () => {}; + return camera; +} + +const pose = (camera) => + [camera.position, camera.direction, camera.up, camera.right].map((v) => + Cesium.Cartesian3.clone(v), + ); + +test("the wheel zoom's own arithmetic hands Camera.rotate a zero axis", () => { + // What handleZoom computes with the pointer on the centre pixel: the same + // picked point twice. The dot product rounds below 1, the cross product is 0. + const picked = Cesium.Cartesian3.fromDegrees(-41.11, -13.37); + const pickedNormal = Cesium.Cartesian3.normalize( + picked, + new Cesium.Cartesian3(), + ); + const centreNormal = Cesium.Cartesian3.normalize( + picked, + new Cesium.Cartesian3(), + ); + const axis = Cesium.Cartesian3.cross( + pickedNormal, + centreNormal, + new Cesium.Cartesian3(), + ); + assert.deepEqual([axis.x, axis.y, axis.z], [0, 0, 0]); + + const unguarded = cameraFixture(); + assert.throws(() => unguarded.rotate(axis, 0.01), /not a number/); + + const camera = cameraFixture(); + const before = pose(camera); + guardDegenerateRotation(camera); + assert.doesNotThrow(() => camera.rotate(axis, 0.01)); + assert.deepEqual(pose(camera), before); +}); + +test('an axis or angle that is not finite leaves the camera where it is', () => { + const camera = cameraFixture(); + const before = pose(camera); + guardDegenerateRotation(camera); + camera.rotate(new Cesium.Cartesian3(NaN, 0, 1), 0.01); + camera.rotate(new Cesium.Cartesian3(Infinity, 0, 1), 0.01); + camera.rotate(Cesium.Cartesian3.UNIT_Z, NaN); + camera.rotate(Cesium.Cartesian3.UNIT_Z, Infinity); + assert.deepEqual(pose(camera), before); +}); + +test('an ordinary rotation is exactly what Cesium does unguarded', () => { + const axis = new Cesium.Cartesian3(0.2, -0.5, 0.84); + const plain = cameraFixture(); + plain.rotate(axis, 0.37); + plain.rotate(Cesium.Cartesian3.UNIT_Z); + + const camera = cameraFixture(); + const restore = guardDegenerateRotation(camera); + camera.rotate(axis, 0.37); + // No angle: Cesium's default amount still applies. + camera.defaultRotateAmount = plain.defaultRotateAmount; + camera.rotate(Cesium.Cartesian3.UNIT_Z); + assert.deepEqual(pose(camera), pose(plain)); + + restore(); + assert.equal(Object.hasOwn(camera, 'rotate'), false); + assert.equal(camera.rotate, Cesium.Camera.prototype.rotate); +}); + +/** Vector equality as Cesium has it: a recomputed right vector may hold -0. */ +function assertPose(camera, expected) { + pose(camera).forEach((vector, i) => + assert.ok(Cesium.Cartesian3.equals(vector, expected[i]), `vector ${i}`), + ); +} + +function viewerFixture() { + const postRender = new Cesium.Event(); + const panels = []; + const resets = []; + const camera = cameraFixture(); + const viewer = { + camera, + useDefaultRenderLoop: true, + cesiumWidget: { + showErrorPanel(title, message, error) { + panels.push({ title, message, error }); + }, + }, + scene: { + postRender, + screenSpaceCameraController: { + _aggregator: { reset: () => resets.push(true) }, + }, + }, + }; + // What Cesium's loop does when a frame throws. + const fail = (error = new Error('frame failed')) => { + viewer.useDefaultRenderLoop = false; + viewer.cesiumWidget.showErrorPanel('Rendering has stopped.', undefined, error); + }; + return { viewer, camera, postRender, panels, resets, fail }; +} + +function quietly(run) { + const original = console.error; + const logged = []; + console.error = (...args) => logged.push(args); + try { + run(); + } finally { + console.error = original; + } + return logged; +} + +test('a stopped render loop restarts from the last drawn pose', () => { + const { viewer, camera, postRender, panels, resets, fail } = viewerFixture(); + const requests = []; + installRenderRecovery(viewer, { + now: () => 0, + requestRender: (reason) => requests.push(reason), + }); + postRender.raiseEvent(); + const drawn = pose(camera); + + // The failed frame left the camera unusable. + camera.position.x = NaN; + camera.direction.y = NaN; + const error = new Error('normalized result is not a number'); + const logged = quietly(() => fail(error)); + + assert.equal(viewer.useDefaultRenderLoop, true); + assert.deepEqual(panels, []); + assertPose(camera, drawn); + assert.deepEqual(resets, [true]); + assert.deepEqual(requests, ['render-recovery']); + assert.equal(logged.length, 1); + assert.equal(logged[0][1], error); +}); + +test('a frame drawn with a broken camera is not kept as the pose to return to', () => { + const { viewer, camera, postRender, fail } = viewerFixture(); + installRenderRecovery(viewer, { now: () => 0, requestRender: () => {} }); + postRender.raiseEvent(); + const drawn = pose(camera); + camera.position.x = NaN; + postRender.raiseEvent(); + quietly(() => fail()); + assertPose(camera, drawn); +}); + +test('an error that keeps returning is shown and rendering stays stopped', () => { + const { viewer, panels, fail } = viewerFixture(); + let at = 0; + installRenderRecovery(viewer, { now: () => at, requestRender: () => {} }); + quietly(() => { + for (let i = 0; i < RECOVERY_LIMIT; i++) { + at += 100; + fail(); + assert.equal(viewer.useDefaultRenderLoop, true, `restart ${i + 1}`); + } + at += 100; + fail(); + }); + assert.equal(viewer.useDefaultRenderLoop, false); + assert.equal(panels.length, 1); + assert.equal(panels[0].title, 'Rendering has stopped.'); + + // Once the window has passed, a later, unrelated error is recovered again. + at += RECOVERY_WINDOW_MS + 1; + quietly(() => fail()); + assert.equal(viewer.useDefaultRenderLoop, true); + assert.equal(panels.length, 1); +}); + +test('a panel shown while the loop is running is left alone', () => { + const { viewer, panels, resets } = viewerFixture(); + installRenderRecovery(viewer, { now: () => 0, requestRender: () => {} }); + viewer.cesiumWidget.showErrorPanel('Error constructing CesiumWidget.', 'x'); + assert.equal(panels.length, 1); + assert.deepEqual(resets, []); +}); + +test('disposing restores the widget and stops recording poses', () => { + const { viewer, postRender, panels, fail } = viewerFixture(); + const original = viewer.cesiumWidget.showErrorPanel; + const dispose = installRenderRecovery(viewer, { + now: () => 0, + requestRender: () => {}, + }); + assert.equal(postRender.numberOfListeners, 1); + dispose(); + dispose(); + assert.equal(postRender.numberOfListeners, 0); + assert.equal(viewer.cesiumWidget.showErrorPanel, original); + fail(); + assert.equal(viewer.useDefaultRenderLoop, false); + assert.equal(panels.length, 1); +}); + +test('an incomplete viewer is refused', () => { + assert.throws(() => installRenderRecovery({}), TypeError); +}); diff --git a/src/app/viewer.js b/src/app/viewer.js index e4288e8a3..2e8820c4e 100644 --- a/src/app/viewer.js +++ b/src/app/viewer.js @@ -1,5 +1,9 @@ import * as Cesium from 'cesium'; import { applyModelAtmosphereWorkaround } from './atmosphereCompat.js'; +import { + guardDegenerateRotation, + installRenderRecovery, +} from './renderRecovery.js'; import { StarrySky } from '../sky/starrySky.js'; import { FREE_LOOK_MAX_HEIGHT_METERS, @@ -415,6 +419,11 @@ export function createApplicationViewer({ container, creditContainer }) { if (globalThis.matchMedia?.('(pointer: coarse)').matches) viewer.scene.screenSpaceCameraController.zoomFactor *= 3; limitZoomOut(viewer); + // Cesium's wheel zoom can hand the camera a zero rotation axis and stop + // the render loop for good; and whatever else stops it, restart it rather + // than leave the globe dead until a reload. See app/renderRecovery.js. + guardDegenerateRotation(viewer.camera); + installRenderRecovery(viewer); // Before any tile builds a draw command: Cesium's per-vertex model // atmosphere fails to LINK on Apple's Metal backend and kills the // render loop. See app/atmosphereCompat.js.