Report thrown render loop errors and keep one loop after an inline Retry

An error thrown outside Scene's own error handling (a postRender listener,
the clock's onTick, a resize) stopped Cesium's loop without a renderError,
and with Cesium's panel turned off the globe froze silently. The notice now
raises such an error as a render error.

In an inline panel whose host stopped animation frames, Retry left the
panel's timer latched off, and once frames resumed a second render loop
drew every frame twice. The timer now follows Cesium's loop, and keeps the
loop marked running when its next frame is still queued.

docs/CURRENT-STATE.md

@@ -1590,7 +1590,11 @@ the error and publishes a notice to the status store, which
 application scene creates its viewer with `showRenderLoopErrors: false`, so
 Cesium's own "Rendering has stopped" panel, which offers no way back, no longer
 appears; `applicationViewerOptions()` and `createApplicationViewer()` keep
-Cesium's panel by default for other callers.
+Cesium's panel by default for other callers. An error thrown outside Scene's
+own error handling (a `preRender` or `postRender` listener, the clock's
+`onTick`, a resize) also stops the loop without a `renderError`, so the notice
+wraps the widget's `render()` and `resize()` and raises such an error as a
+render error before Cesium's loop handles it as before.
 
 - **Retry** after a render error restarts the render loop. The notice stays
   until a frame starts and ends without a render error (Cesium raises
@@ -1603,8 +1607,15 @@ Cesium's panel by default for other callers.
 
 An inline conversation panel never reloads or navigates its host's page, so it
 offers Retry after a render error only, and nothing after a lost context, where
-the panel runtime's own notice applies. The listener does not
-`preventDefault()` the context loss or restore the context; it only reports it.
+the panel runtime's own notice applies. The panel's timer
+(`keepPanelRendering()`) draws only while Cesium's render loop is on, so it
+stops after a render error and draws again after Retry. An error in a frame
+the timer draws leaves the loop's next animation frame queued, and the timer
+keeps Cesium's loop marked running, so Retry does not start a second loop that
+would draw every frame twice once the host resumes frames.
+
+The `webglcontextlost` listener does not `preventDefault()` the context loss or
+restore the context; it only reports it.
 
 ## Map Source control ownership
 

scripts/format-scope.json

@@ -547,6 +547,8 @@
   "src/ui/frameRateView.test.mjs",
   "src/ui/renderFailure.test.mjs",
   "src/ui/renderFailureView.test.mjs",
+  "src/ui/renderFailureLoopErrors.test.mjs",
+  "src/app/renderFailureInlineRetry.test.mjs",
   "src/layers/cctv/lifecycle.js",
   "src/layers/cctv/controls.js",
   "src/layers/cctv/geometry.js",

src/app/embed.js

@@ -100,13 +100,18 @@ export function keepPanelRendering(
     frameRequest = windowRef.requestAnimationFrame(onFrame);
   };
   frameRequest = windowRef.requestAnimationFrame(onFrame);
-  // Cesium stops drawing after a render error, and so does the timer.
-  // Report the error in full: workers report plain objects, which Cesium
-  // prints as [object Object].
-  let failed = false;
+  // Cesium stops its render loop after a render error, and the timer stops
+  // with it; Retry restarts both. Report the error in full: workers report
+  // plain objects, which Cesium prints as [object Object].
+  let drawing = false;
   const removeErrorListener = viewer.scene?.renderError?.addEventListener(
     (_scene, error) => {
-      failed = true;
+      // Cesium marks its loop stopped, but in a frame the timer draws, the
+      // loop's next animation frame is still queued and resumes when frames
+      // do. Restarting the loop now would start a second one beside it, and
+      // the globe would draw twice a frame.
+      if (drawing && viewer.cesiumWidget)
+        viewer.cesiumWidget._renderLoopRunning = true;
       console.error(
         "[God's Eye View panel] render error:",
         describeError(error),
@@ -115,13 +120,18 @@ export function keepPanelRendering(
   );
   const timer = windowRef.setInterval(() => {
     if (
-      failed ||
+      viewer.useDefaultRenderLoop === false ||
       now() - lastFrame < MISSED_FRAMES_MS ||
       viewer.isDestroyed?.()
     )
       return;
-    viewer.resize();
-    viewer.render();
+    drawing = true;
+    try {
+      viewer.resize();
+      viewer.render();
+    } finally {
+      drawing = false;
+    }
   }, PANEL_FRAME_MS);
   return () => {
     windowRef.clearInterval(timer);

src/app/embed.test.mjs

@@ -355,6 +355,8 @@ test('a panel reports a render error in full and stops drawing', () => {
   try {
     let clock = 0;
     const stop = keepPanelRendering(viewer, { windowRef, now: () => clock });
+    // Cesium stops its render loop after a render error.
+    viewer.useDefaultRenderLoop = false;
     listener(viewer.scene, { name: 'RuntimeError', message: 'worker failed' });
     assert.match(logged[0], /render error: .*"message":"worker failed"/);
     clock = 1000;

src/app/renderFailureInlineRetry.test.mjs

@@ -0,0 +1,185 @@
+// Retry in an inline panel whose host stopped animation frames: the panel
+// draws from keepPanelRendering's timer, so Retry must make that timer draw
+// again, or the notice offers a Retry that does nothing.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import * as Cesium from 'cesium';
+import { keepPanelRendering } from './embed.js';
+import { installRenderFailureNotice } from '../ui/renderFailure.js';
+
+test('Retry resumes the inline panel timer after a render error', () => {
+  let tick = null;
+  let clock = 0;
+  // The host reports the panel hidden: animation frames never arrive.
+  const windowRef = {
+    requestAnimationFrame: () => 1,
+    cancelAnimationFrame() {},
+    setInterval: (fn) => ((tick = fn), 1),
+    clearInterval() {},
+  };
+  const previousRaf = globalThis.requestAnimationFrame;
+  globalThis.requestAnimationFrame = () => 1;
+  const previousError = console.error;
+  console.error = () => {};
+  try {
+    let broken = true;
+    let drawn = 0;
+    const scene = {
+      renderError: new Cesium.Event(),
+      preUpdate: new Cesium.Event(),
+      postRender: new Cesium.Event(),
+      requestRender() {},
+    };
+    const viewer = {
+      scene,
+      canvas: { addEventListener() {}, removeEventListener() {} },
+      useDefaultRenderLoop: true,
+      isDestroyed: () => false,
+      resize() {},
+      // Scene.render: a failing primitive update raises renderError
+      // (tryAndCatchError), and postRender is still raised.
+      render() {
+        drawn += 1;
+        scene.preUpdate.raiseEvent(scene);
+        if (broken) scene.renderError.raiseEvent(scene, new Error('boom'));
+        scene.postRender.raiseEvent(scene);
+      },
+    };
+    // CesiumWidget._onRenderError stops the default loop.
+    scene.renderError.addEventListener(() => {
+      viewer.useDefaultRenderLoop = false;
+    });
+    const store = {
+      renderFailure: null,
+      publishRenderFailure(next) {
+        this.renderFailure = next;
+      },
+    };
+    const uninstall = installRenderFailureNotice(viewer, {
+      store,
+      allowReload: false,
+      locationRef: { href: 'https://gev.example/', reload() {}, assign() {} },
+      logError() {},
+    });
+    const stop = keepPanelRendering(viewer, { windowRef, now: () => clock });
+    clock = 400;
+    tick();
+    assert.equal(store.renderFailure?.reason, 'render-error');
+    assert.equal(typeof store.renderFailure.retry, 'function');
+    // The cause goes away and the user presses Retry.
+    broken = false;
+    store.renderFailure.retry();
+    const before = drawn;
+    for (let i = 0; i < 5; i += 1) {
+      clock += 33;
+      tick();
+    }
+    assert.ok(drawn > before, 'the panel draws again after Retry');
+    assert.equal(store.renderFailure, null, 'a clean frame clears the notice');
+    stop();
+    uninstall();
+  } finally {
+    globalThis.requestAnimationFrame = previousRaf;
+    console.error = previousError;
+  }
+});
+
+test('Retry in a panel whose host stopped frames keeps one render loop', () => {
+  const frames = [];
+  let tick = null;
+  let clock = 0;
+  const windowRef = {
+    requestAnimationFrame: () => 1,
+    cancelAnimationFrame() {},
+    setInterval: (fn) => ((tick = fn), 1),
+    clearInterval() {},
+  };
+  const previousRaf = globalThis.requestAnimationFrame;
+  globalThis.requestAnimationFrame = (fn) => frames.push(fn);
+  const previousError = console.error;
+  console.error = () => {};
+  try {
+    let broken = false;
+    let drawn = 0;
+    const scene = {
+      renderError: new Cesium.Event(),
+      preUpdate: new Cesium.Event(),
+      postRender: new Cesium.Event(),
+      requestRender() {},
+      initializeFrame() {},
+      render() {
+        drawn += 1;
+        this.preUpdate.raiseEvent(this);
+        if (broken) this.renderError.raiseEvent(this, new Error('boom'));
+        this.postRender.raiseEvent(this);
+      },
+    };
+    const widget = Object.create(Cesium.CesiumWidget.prototype);
+    Object.assign(widget, {
+      _scene: scene,
+      _clock: { tick() {} },
+      _canRender: true,
+      _useDefaultRenderLoop: false,
+      _renderLoopRunning: false,
+      _showRenderLoopErrors: false,
+      _targetFrameRate: undefined,
+      isDestroyed: () => false,
+      resize() {},
+    });
+    // CesiumWidget's own renderError handler, which marks the loop stopped.
+    scene.renderError.addEventListener(() => {
+      widget._useDefaultRenderLoop = false;
+      widget._renderLoopRunning = false;
+    });
+    const viewer = {
+      scene,
+      cesiumWidget: widget,
+      canvas: { addEventListener() {}, removeEventListener() {} },
+      get useDefaultRenderLoop() {
+        return widget.useDefaultRenderLoop;
+      },
+      set useDefaultRenderLoop(value) {
+        widget.useDefaultRenderLoop = value;
+      },
+      isDestroyed: () => false,
+      render: () => widget.render(),
+      resize: () => widget.resize(),
+    };
+    const store = {
+      renderFailure: null,
+      publishRenderFailure(next) {
+        this.renderFailure = next;
+      },
+    };
+    const uninstall = installRenderFailureNotice(viewer, {
+      store,
+      allowReload: false,
+      locationRef: { href: 'https://gev.example/', reload() {}, assign() {} },
+      logError() {},
+    });
+    const stop = keepPanelRendering(viewer, { windowRef, now: () => clock });
+    viewer.useDefaultRenderLoop = true;
+    // The host stops animation frames with the loop's next one queued.
+    assert.equal(frames.length, 1);
+    broken = true;
+    clock = 400;
+    tick();
+    assert.equal(store.renderFailure?.reason, 'render-error');
+    broken = false;
+    store.renderFailure.retry();
+    clock += 33;
+    tick();
+    assert.equal(store.renderFailure, null);
+    assert.equal(frames.length, 1, 'Retry starts no second loop');
+    // The host resumes frames: the globe draws once a frame.
+    drawn = 0;
+    for (let frame = 1; frame <= 3; frame += 1)
+      for (const render of frames.splice(0)) render(frame * 16);
+    assert.equal(drawn, 3);
+    stop();
+    uninstall();
+  } finally {
+    globalThis.requestAnimationFrame = previousRaf;
+    console.error = previousError;
+  }
+});

src/ui/renderFailure.js

@@ -105,10 +105,45 @@ export function installRenderFailureNotice(
   };
 
   const removeRenderError = scene.renderError.addEventListener(onRenderError);
+  // An error thrown outside Scene's own error handling (a preRender or
+  // postRender listener, the clock's onTick, a resize) also stops Cesium's
+  // render loop, but raises no renderError, and Cesium reports it only in the
+  // panel the app turns off. Raise it as a render error, so Cesium's handler,
+  // an inline panel's timer and this notice all see it.
+  const unguard = ['resize', 'render'].map((name) =>
+    reportThrown(viewer.cesiumWidget, name, (error) =>
+      scene.renderError.raiseEvent(scene, error),
+    ),
+  );
   canvas?.addEventListener?.('webglcontextlost', onContextLost);
   return () => {
     removeRenderError();
+    unguard.forEach((restore) => restore());
     canvas?.removeEventListener?.('webglcontextlost', onContextLost);
     clear();
   };
 }
+
+/**
+ * Wrap `target[name]` so an error it throws is reported and thrown on, for
+ * the caller to handle as before. Returns a function that unwraps it.
+ */
+function reportThrown(target, name, report) {
+  const original = target?.[name];
+  if (typeof original !== 'function') return () => {};
+  const own = Object.hasOwn(target, name);
+  const guarded = function (...args) {
+    try {
+      return original.apply(this, args);
+    } catch (error) {
+      report(error);
+      throw error;
+    }
+  };
+  target[name] = guarded;
+  return () => {
+    if (target[name] !== guarded) return;
+    if (own) target[name] = original;
+    else delete target[name];
+  };
+}

src/ui/renderFailureLoopErrors.test.mjs

@@ -0,0 +1,83 @@
+// An error thrown in the render loop outside Scene's tryAndCatchError
+// (a preRender/postRender listener, clock.onTick, resize) must still reach
+// the render failure notice. Drives Cesium's real startRenderLoop through the
+// CesiumWidget.prototype.useDefaultRenderLoop setter.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import * as Cesium from 'cesium';
+import { installRenderFailureNotice } from './renderFailure.js';
+
+// What src/app/scene.js passes to the viewer (showRenderLoopErrors).
+const SHOW_RENDER_LOOP_ERRORS = false;
+
+test('an error thrown outside tryAndCatchError still shows the notice', () => {
+  const frames = [];
+  const previousRaf = globalThis.requestAnimationFrame;
+  globalThis.requestAnimationFrame = (fn) => frames.push(fn);
+  try {
+    const scene = {
+      renderError: new Cesium.Event(),
+      preUpdate: new Cesium.Event(),
+      postRender: new Cesium.Event(),
+      requestRender() {},
+      initializeFrame() {},
+      // Scene.render raises preRender outside tryAndCatchError
+      // (Scene.js:4439), so a listener's exception propagates.
+      render() {
+        this.preUpdate.raiseEvent(this);
+        throw new TypeError(
+          "Cannot read properties of undefined (reading 'x')",
+        );
+      },
+    };
+    const panels = [];
+    const widget = Object.create(Cesium.CesiumWidget.prototype);
+    Object.assign(widget, {
+      _scene: scene,
+      _clock: { tick() {} },
+      _canRender: true,
+      _useDefaultRenderLoop: false,
+      _renderLoopRunning: false,
+      _showRenderLoopErrors: SHOW_RENDER_LOOP_ERRORS,
+      _targetFrameRate: undefined,
+      isDestroyed: () => false,
+      resize() {},
+      showErrorPanel: (...args) => panels.push(args),
+    });
+    const viewer = {
+      scene,
+      cesiumWidget: widget,
+      canvas: { addEventListener() {}, removeEventListener() {} },
+      get useDefaultRenderLoop() {
+        return widget.useDefaultRenderLoop;
+      },
+      set useDefaultRenderLoop(value) {
+        widget.useDefaultRenderLoop = value;
+      },
+      render: () => widget.render(),
+      resize: () => widget.resize(),
+    };
+    const store = {
+      renderFailure: null,
+      publishRenderFailure(next) {
+        this.renderFailure = next;
+      },
+    };
+    const logged = [];
+    const uninstall = installRenderFailureNotice(viewer, {
+      store,
+      locationRef: { href: 'https://gev.example/', reload() {}, assign() {} },
+      logError: (error) => logged.push(error),
+    });
+    viewer.useDefaultRenderLoop = true; // starts Cesium's loop
+    while (frames.length) frames.shift()(16);
+    // Cesium's catch stopped the loop.
+    assert.equal(widget.useDefaultRenderLoop, false);
+    // The user is told, and the error is logged.
+    assert.equal(store.renderFailure?.reason, 'render-error');
+    assert.equal(logged.length, 1);
+    uninstall();
+  } finally {
+    globalThis.requestAnimationFrame = previousRaf;
+  }
+});