Read pixels and release unused caches through the Cesium renderer

The shared Cesium ports gain readPixels and releaseUnused, on both the
conformance renderer and the application's.

readPixels reads the drawn frame off the canvas at once, cropped in CSS
pixels and scaled from the canvas's CSS size, through one reused 2D canvas.
A canvas with no size reads blank.

releaseUnused trims every 3D tileset's cache, and after the next drawn frame
frees the globe tiles that frame did not draw. Cesium trims the globe's
cache only while tiles load, so a settled view would otherwise free nothing;
the trim goes through internals.js. The reader canvas is dropped too.

Two new checks, renderer.readPixels and renderer.releaseUnused, run on
Cesium and the stub.

src/render/cesium/appRenderer.js

@@ -28,7 +28,8 @@ import { cesiumCapabilities, createCesiumPorts } from './ports.js';
  * @typedef {Pick<import('../contract/renderer').Renderer,
  *   'id' | 'contractVersion' | 'capabilities' | 'canvas' | 'frame' | 'draw'
  *   | 'camera' | 'query' | 'input' | 'effects' | 'quality' | 'capture'
- *   | 'credits' | 'whenSettled' | 'on' | 'resize' | 'destroy'>}
+ *   | 'credits' | 'readPixels' | 'releaseUnused' | 'whenSettled' | 'on'
+ *   | 'resize' | 'destroy'>}
  *   TransitionalRenderer
  */
 
@@ -117,6 +118,8 @@ export function createCesiumRendererPlugin({
         effects: ports.effects,
         quality: ports.quality,
         capture: ports.capture,
+        readPixels: ports.readPixels,
+        releaseUnused: ports.releaseUnused,
         credits: ports.credits,
         whenSettled: ports.whenSettled,
         on: ports.on,

src/render/cesium/appRenderer.test.mjs

@@ -317,8 +317,6 @@ test('the application refuses a renderer it cannot run on, and destroys it', asy
   assert.deepEqual(destroyed, ['other']);
 });
 
-const EXPECTED_MISSING = [
-  'renderer.globe',
-  'renderer.readPixels',
-  'renderer.releaseUnused',
-].map((member) => `${member} is missing`);
+const EXPECTED_MISSING = ['renderer.globe'].map(
+  (member) => `${member} is missing`,
+);

src/render/cesium/harness.js

@@ -64,6 +64,8 @@ export const CESIUM_CHECKS = new Set([
   'quality',
   'quality.signals',
   'renderer.listeners',
+  'renderer.readPixels',
+  'renderer.releaseUnused',
   'renderer.whenSettled',
 ]);
 

src/render/cesium/internals.js

@@ -180,6 +180,17 @@ export function billboardLoadError(billboard) {
   return billboard.loadError;
 }
 
+/**
+ * Frees the globe tiles the frame just drawn did not use. Cesium trims its
+ * tile cache only while tiles are loading, so a settled view never would;
+ * the renderer's `releaseUnused` trims it after a drawn frame, while the
+ * queue still marks what that frame drew.
+ * @param {object} globe Cesium Globe.
+ */
+export function trimGlobeTiles(globe) {
+  globe?._surface?._tileReplacementQueue?.trimTiles(0);
+}
+
 const NO_CREDITS = Object.freeze([]);
 
 /**

src/render/cesium/internals.test.mjs

@@ -19,6 +19,7 @@ import {
   onEngineWorkCompleted,
   pickOffscreenView,
   renderPending,
+  trimGlobeTiles,
   updateCameraChanged,
   webglContext,
 } from './internals.js';
@@ -55,6 +56,8 @@ test('every private field Core reaches is still declared by the installed Cesium
     [ENGINE, 'Scene/Scene.js', '_view'],
     [ENGINE, 'Scene/View.js', '_cameraClone'],
     [ENGINE, 'Scene/Scene.js', '_frameState'],
+    [ENGINE, 'Scene/Globe.js', '_surface'],
+    [ENGINE, 'Scene/QuadtreePrimitive.js', '_tileReplacementQueue'],
     [ENGINE, 'Scene/CreditDisplay.js', '_staticCredits'],
     [ENGINE, 'Scene/CreditDisplay.js', '_currentFrameCredits'],
     [ENGINE, 'Scene/CreditDisplay.js', '_currentCesiumCredit'],
@@ -259,3 +262,12 @@ test('private Cesium fields are reached only through internals.js', () => {
   }
   assert.deepEqual([...new Set(offenders)], []);
 });
+
+test('trimGlobeTiles empties the globe cache of what the frame did not draw', () => {
+  const trims = [];
+  trimGlobeTiles({
+    _surface: { _tileReplacementQueue: { trimTiles: (n) => trims.push(n) } },
+  });
+  trimGlobeTiles(undefined);
+  assert.deepEqual(trims, [0]);
+});

src/render/cesium/ports.js

@@ -26,6 +26,7 @@ import {
   lastSeenCamera,
   onEngineWorkCompleted,
   renderPending,
+  trimGlobeTiles,
 } from './internals.js';
 
 export { listenerList, toCartesian };
@@ -225,6 +226,7 @@ export function createCesiumPorts(widget, { capabilities }) {
     scene.postRender.addEventListener(drew),
     scene.postRender.addEventListener(onPhase('postRender')),
     scene.postRender.addEventListener(() => watchClamp()),
+    scene.postRender.addEventListener(() => trimGlobeAfterDraw()),
   ];
 
   /**
@@ -1148,6 +1150,74 @@ export function createCesiumPorts(widget, { capabilities }) {
     );
   }
 
+  // The 2D canvas frames are read through, made on the first read.
+  let reader = null;
+
+  /**
+   * The drawn frame's pixels, `crop` in CSS pixels and `scale` relative to
+   * the canvas's CSS size. Read straight off the canvas: the application
+   * and conformance widgets keep their drawing buffers, and a read right
+   * after a draw sees it either way.
+   */
+  function readPixels({ crop, scale = 1 } = {}) {
+    const canvas = scene.canvas;
+    const cssWidth = canvas.clientWidth || canvas.width;
+    const cssHeight = canvas.clientHeight || canvas.height;
+    const rect = crop ?? { x: 0, y: 0, width: cssWidth, height: cssHeight };
+    const width = Math.max(1, Math.round(rect.width * scale));
+    const height = Math.max(1, Math.round(rect.height * scale));
+    const made =
+      reader ??
+      document
+        .createElement('canvas')
+        .getContext('2d', { willReadFrequently: true });
+    // Disposed ports keep nothing to free.
+    if (!disposed) reader = made;
+    made.canvas.width = width;
+    made.canvas.height = height;
+    // A canvas with no size has drawn nothing: the read is blank.
+    if (!canvas.width || !canvas.height)
+      return made.getImageData(0, 0, width, height);
+    const sx = canvas.width / cssWidth;
+    const sy = canvas.height / cssHeight;
+    made.drawImage(
+      canvas,
+      rect.x * sx,
+      rect.y * sy,
+      rect.width * sx,
+      rect.height * sy,
+      0,
+      0,
+      width,
+      height,
+    );
+    return made.getImageData(0, 0, width, height);
+  }
+
+  // A release asked for whose globe trim waits for the next drawn frame.
+  let trimGlobe = false;
+
+  /**
+   * Frees the tiles the last frame did not draw: each 3D tileset trims its
+   * cache, and a shown globe its own after the next drawn frame. The reader
+   * pixels are read through goes too.
+   */
+  function releaseUnused() {
+    if (disposed) return;
+    for (const tileset of tilesets()) tileset.trimLoadedTiles();
+    if (scene.globe?.show) trimGlobe = true;
+    if (reader) {
+      reader.canvas.width = reader.canvas.height = 0;
+      reader = null;
+    }
+    frame.requestRender('releaseUnused');
+  }
+  function trimGlobeAfterDraw() {
+    if (!trimGlobe) return;
+    trimGlobe = false;
+    if (scene.globe?.show) trimGlobeTiles(scene.globe);
+  }
+
   /** The live model sets, for `whenSettled`. */
   const modelSets = () =>
     [...sets.values()].filter((r) => r.kind === 'models' && r.host);
@@ -1327,11 +1397,16 @@ export function createCesiumPorts(widget, { capabilities }) {
     effects: effectsPart.port,
     quality: qualityPart.port,
     capture,
+    readPixels,
+    releaseUnused,
     whenSettled,
     /** Detach from the scene; the widget's owner destroys the widget. */
     dispose() {
       disposed = true;
       for (const finish of [...watching]) finish('ended');
+      trimGlobe = false;
+      if (reader) reader.canvas.width = reader.canvas.height = 0;
+      reader = null;
       // The tick under way is no longer seen through: it is not claimed.
       settleTick(false);
       for (const off of removers) off();

src/render/cesium/ports.test.mjs

@@ -770,3 +770,38 @@ test('whenSettled settles once shown tiles have loaded under a hidden globe', as
     ports.dispose();
   }
 });
+
+test('releaseUnused trims the tiles caches, the globe after the next drawn frame', () => {
+  const { widget, scene, raise } = tickingWidget();
+  const list = [];
+  scene.primitives = listedPrimitives(list);
+  const trimmed = [];
+  scene.globe = {
+    show: true,
+    _surface: {
+      _tileReplacementQueue: { trimTiles: (n) => trimmed.push(`globe ${n}`) },
+    },
+  };
+  const tileset = Object.create(Cesium.Cesium3DTileset.prototype);
+  tileset.trimLoadedTiles = () => trimmed.push('tileset');
+  list.push(tileset);
+  const ports = createCesiumPorts(widget, { capabilities: all });
+  ports.releaseUnused();
+  ports.releaseUnused();
+  assert.deepEqual(trimmed, ['tileset', 'tileset'], 'the globe waits');
+  assert.ok(ports.frame.diagnostics().pendingReasons.includes('releaseUnused'));
+  raise('postRender');
+  raise('postRender');
+  assert.deepEqual(trimmed, ['tileset', 'tileset', 'globe 0'], 'once');
+  scene.globe.show = false;
+  ports.releaseUnused();
+  raise('postRender');
+  assert.equal(trimmed.at(-1), 'tileset', 'a hidden globe is left alone');
+  scene.globe.show = true;
+  ports.releaseUnused();
+  ports.dispose();
+  raise('postRender');
+  assert.equal(trimmed.at(-1), 'tileset', 'nothing once disposed');
+  ports.releaseUnused();
+  assert.equal(trimmed.length, 5, 'nor a release after');
+});

src/render/cesium/sceneRenderer.js

@@ -3,7 +3,8 @@
  * @description The part of the Cesium renderer the reference scenes need,
  * through the contract's own members: a bare globe and sky, icons, points,
  * lines, shapes and models, `setView`, picks and projection, the frame
- * phases, pointer input, credits, effects, settling and capture. The
+ * phases, pointer input, credits, effects, settling, capture, pixel reads
+ * and cache release. The
  * `/dev/conformance/` page draws the scenes with it to make and check the
  * visual references.
  *
@@ -143,6 +144,8 @@ export async function createCesiumSceneRenderer({
     effects: ports.effects,
     quality: ports.quality,
     capture: ports.capture,
+    readPixels: ports.readPixels,
+    releaseUnused: ports.releaseUnused,
     credits: ports.credits,
     whenSettled: ports.whenSettled,
     resize: () => widget.resize(),

src/render/conformance/checks/ports.js

@@ -2260,6 +2260,44 @@ fn fs_main(@location(0) uv: vec2f) -> @location(0) vec4f {
         );
       },
     ),
+    check(
+      'renderer.readPixels',
+      'readPixels reads the drawn frame, cropped and scaled, at once',
+      async (h, r) => {
+        await overAustin(h, r);
+        r.frame.renderNow();
+        const full = r.readPixels();
+        equal(full.width, SCRIPT_VIEWPORT.width, 'the canvas’s CSS width');
+        equal(full.height, SCRIPT_VIEWPORT.height, 'the canvas’s CSS height');
+        equal(full.data.length, full.width * full.height * 4, 'RGBA bytes');
+        const part = r.readPixels({
+          crop: { x: 10, y: 20, width: 40, height: 30 },
+          scale: 0.5,
+        });
+        deepEqual(
+          [part.width, part.height, part.data.length],
+          [20, 15, 20 * 15 * 4],
+          'a crop, scaled',
+        );
+      },
+    ),
+    check(
+      'renderer.releaseUnused',
+      'releaseUnused keeps what is in view drawn and picked',
+      async (h, r) => {
+        await overAustin(h, r);
+        const set = r.draw.points({ id: 'conf:release' });
+        const pos = h.toEcef({ ...AUSTIN, height: 0 });
+        set.set([{ id: 'kept', pos, pixelSize: 20, color: '#f00' }]);
+        await drawn(h, r);
+        r.releaseUnused();
+        await drawn(h, r);
+        const s = r.query.project(pos);
+        ok(s, 'the point projects');
+        equal(r.query.pick(s.x, s.y)?.id, 'kept', 'still picked');
+        set.dispose();
+      },
+    ),
     check(
       'quality.signals',
       'frame signals arrive for drawn frames',