diff --git a/changelog.d/1834-decoded-imagery.md b/changelog.d/1834-decoded-imagery.md new file mode 100644 index 0000000000..5b3900d027 --- /dev/null +++ b/changelog.d/1834-decoded-imagery.md @@ -0,0 +1,5 @@ +- Renderer contract: a timed imagery source can be `decoded` (the + `imageryDecoded` capability), tiles the application makes itself and hands + over as finished images, slot by slot; the Cesium renderer draws each slot + as an imagery layer of its own, and the check `globe.imagery.decoded` covers + it (Sameh Khamis, #1834). diff --git a/docs/RENDERER-CONTRACT.md b/docs/RENDERER-CONTRACT.md index 76d547cb09..99f41b8c3c 100644 --- a/docs/RENDERER-CONTRACT.md +++ b/docs/RENDERER-CONTRACT.md @@ -361,6 +361,18 @@ names the nearest frame's time once every globe tile drawn shows that frame's own imagery (`src/render/cesium/imageryFrames.js`). The checks `globe.imagery.frames` and `globe.imagery.video` cover both. +It also draws `decoded` sources (`imageryDecoded`): tiles the application +makes itself and hands over as finished images. Their slots are times: the +source's `slotAt` names the slot a time shows and `prefetch` the slots held +ahead, of which the first `prefetch` (the option, 1 by default) load unseen. +Each slot is a Cesium imagery layer in the layer's tiling whose tiles are the +images the source's `tile` answers, asked as `shown` or `prefetch`; a +deferred tile is asked again on a later frame, a missing one draws its +parent's, and a slot let go aborts the tiles it still waits on and closes an +image that arrives after. A time with no slot draws nothing and announces +nothing; `frameReady` names the slot shown once it is drawn. The check +`globe.imagery.decoded` covers it. + Every Cesium draw set takes `setColumns` (`columns`): icon sets on the GPU, any other as the rows the columns make over the items it holds, each row the item of that id with the columns' fields over it, replacing every item as diff --git a/src/render/cesium/harness.js b/src/render/cesium/harness.js index 7e5704910a..85efe4569a 100644 --- a/src/render/cesium/harness.js +++ b/src/render/cesium/harness.js @@ -117,6 +117,7 @@ export const CESIUM_CHECKS = new Set([ 'frame.time', 'globe.base.vector', 'globe.imagery', + 'globe.imagery.decoded', 'globe.imagery.frames', 'globe.imagery.video', 'globe.sky', diff --git a/src/render/cesium/imagery.js b/src/render/cesium/imagery.js index c70baa8d43..183dc736ad 100644 --- a/src/render/cesium/imagery.js +++ b/src/render/cesium/imagery.js @@ -18,13 +18,21 @@ * `prefetch` frames loading unseen at alpha 0. A frame is ready * (`frameReady`) once the globe's tiles drawn are drawn from its own * imagery (`imageryFrames.js`). + * + * With `imageryDecoded`, a `decoded` source is drawn the same way, a layer + * a slot: the slot its clock shows, and the next `prefetch` of the slots the + * source holds ahead, each tile the image the source answers for it. */ import * as Cesium from 'cesium'; import { vectorBaseProvider } from './vectorBase.js'; import { creditMarkup } from './credits.js'; import { listenerList } from './common.js'; import { + MissingTile, createTileVideos, + decodedHeld, + decodedProvider, + decodedShownAt, framesOf, heldFrames, layerDrawn, @@ -41,14 +49,6 @@ export const TILE_RETRY_DELAY_MS = 250; /** Ion's world imagery assets, by contract style. */ const WORLD_ASSETS = { aerial: 2, aerialLabels: 3 }; -/** A tile the source says is not there: Cesium draws the parent's. */ -class MissingTile extends Error { - constructor() { - super('No tile here'); - this.name = 'MissingTile'; - } -} - const rectangleOf = (rect) => rect && Cesium.Rectangle.fromDegrees(rect.west, rect.south, rect.east, rect.north); @@ -676,11 +676,21 @@ export function createCesiumImagery( }; if (capabilities.has('imageryFrames')) { handle.setFrames = (source, options = {}) => { + const decoded = source?.kind === 'decoded'; + if (decoded && !capabilities.has('imageryDecoded')) + throw new TypeError('decoded imagery needs imageryDecoded'); + if ( + decoded && + (typeof source.slotAt !== 'function' || + typeof source.tile !== 'function') + ) + throw new TypeError('a decoded source needs slotAt and tile'); const frames = framesOf(source); if (record.removed) return; dropFrames(record); record.timed = { video: source.kind === 'video', + decoded: decoded ? source : null, frames, mode: options.mode === 'blend' ? 'blend' : 'nearest', prefetch: Math.max(0, Math.floor(options.prefetch ?? 1)), @@ -723,12 +733,24 @@ export function createCesiumImagery( function frameLayer(record, frame) { const timed = record.timed; const spec = { ...record.spec, url: frame.url }; - const entry = { layer: null, host: null, offError: null }; + const entry = { + layer: null, + host: null, + offError: null, + signals: new Set(), + }; let made; try { - made = timed.video - ? videoProvider(record.spec, frame, timed.videos) - : providerFor(spec, { worldDataToken: worldDataToken() }); + made = timed.decoded + ? decodedProvider(record.spec, frame.slot, timed.decoded, { + laterFrame, + signals: entry.signals, + urgency: () => + timed.at?.nearest === frame.slot ? 'shown' : 'prefetch', + }) + : timed.video + ? videoProvider(record.spec, frame, timed.videos) + : providerFor(spec, { worldDataToken: worldDataToken() }); } catch (error) { report(error); made = Promise.reject(error); @@ -746,7 +768,9 @@ export function createCesiumImagery( } else { entry.offError = record.watchErrors(made); entry.layer = new Cesium.ImageryLayer( - timed.video ? made : hookProvider(made, spec, record.ctx), + timed.video || timed.decoded + ? made + : hookProvider(made, spec, record.ctx), layerOptions(spec), ); } @@ -756,6 +780,9 @@ export function createCesiumImagery( function dropFrameLayer(entry) { entry.removed = true; + // A decoded slot's tiles still asked for are not wanted any more. + for (const controller of entry.signals) controller.abort(); + entry.signals.clear(); entry.offError?.(); if (usable(entry.host) && entry.host.contains(entry.layer)) entry.host.remove(entry.layer, true); @@ -785,9 +812,15 @@ export function createCesiumImagery( const timed = record.timed; if (!timed || record.removed) continue; const ms = timed.clock === 'own' ? (timed.own ?? NaN) : sceneMs; - const at = shownAt(timed.frames, ms, timed.mode); + const at = timed.decoded + ? decodedShownAt(timed.decoded, ms) + : shownAt(timed.frames, ms, timed.mode); timed.at = at; - const held = at ? heldFrames(timed.frames, at, timed.prefetch) : []; + const held = !at + ? [] + : timed.decoded + ? decodedHeld(timed.decoded, at, timed.prefetch, ms) + : heldFrames(timed.frames, at, timed.prefetch); for (const [index, entry] of timed.layers) if (!held.includes(index) || entry.layer.isDestroyed()) { dropFrameLayer(entry); @@ -802,7 +835,10 @@ export function createCesiumImagery( for (const index of held.sort((a, b) => a - b)) { let entry = timed.layers.get(index); if (!entry) { - entry = frameLayer(record, timed.frames[index]); + entry = frameLayer( + record, + timed.decoded ? { slot: index, url: '' } : timed.frames[index], + ); timed.layers.set(index, entry); } const layer = entry.layer; @@ -846,7 +882,9 @@ export function createCesiumImagery( const timed = record.timed; // A layer removed by another's listener says nothing. if (record.removed || !timed?.at || !record.show) continue; - const time = timed.frames[timed.at.nearest].time; + const time = timed.decoded + ? timed.at.nearest + : timed.frames[timed.at.nearest].time; if (time === timed.announced) continue; if (!timed.at.shown.every((s) => frameDrawn(timed.layers.get(s.index)))) continue; diff --git a/src/render/cesium/imageryFrames.js b/src/render/cesium/imageryFrames.js index 735649814b..d3dba47a24 100644 --- a/src/render/cesium/imageryFrames.js +++ b/src/render/cesium/imageryFrames.js @@ -11,10 +11,22 @@ * steps a segment): a time is shown by the step nearest it, its offset into * the segment's videos in proportion; outside every segment, the nearest * segment end is the nearest frame. + * + * A `decoded` source (`imageryDecoded`) has no list: the slot a time shows + * is the source's `slotAt`, the slots held ahead its `prefetch`, and each + * slot's tiles are the images its `tile` answers. */ import * as Cesium from 'cesium'; import { globeTilesDrawn } from './internals.js'; +/** A tile the source says is not there: Cesium draws the parent's. */ +export class MissingTile extends Error { + constructor() { + super('No tile here'); + this.name = 'MissingTile'; + } +} + /** How far apart the times a video segment is seen at are, ms. */ export const VIDEO_STEP_MS = 250; /** The most times one video segment is seen at. */ @@ -61,6 +73,7 @@ export function framesOf(source) { } return frames.sort((a, b) => a.time - b.time); } + if (source?.kind === 'decoded') return []; if (source?.kind === 'frames') return (source.frames ?? []) .filter((f) => Number.isFinite(f?.time) && typeof f?.url === 'string') @@ -408,3 +421,74 @@ export function layerDrawn(scene, layer) { } return true; } + +/** + * What `ms` shows of a `decoded` source: its slot, as `shownAt` answers for + * a list (the slot standing for the index), or null with none. + * @returns {{nearest: number, shown: Array<{index: number, weight: number}>}|null} + */ +export function decodedShownAt(source, ms) { + if (!Number.isFinite(ms)) return null; + const slot = source.slotAt(ms); + if (!Number.isFinite(slot)) return null; + return { nearest: slot, shown: [{ index: slot, weight: 1 }] }; +} + +/** + * The slots of a `decoded` source to hold for `ms`: the one shown, and the + * first `prefetch` of those the source holds ahead (finite, not shown, each + * once). + * @returns {number[]} + */ +export function decodedHeld(source, at, prefetch, ms) { + const held = [at.nearest]; + if (prefetch <= 0 || typeof source.prefetch !== 'function') return held; + for (const slot of source.prefetch(ms) ?? []) { + if (held.length > prefetch) break; + if (Number.isFinite(slot) && !held.includes(slot)) held.push(slot); + } + return held; +} + +/** + * A provider drawing one slot of a `decoded` source: each tile the image + * the source answers for it, in the spec's tiling (an `xyz` spec), or one + * image over the layer's rectangle (any other). A deferred tile is asked + * again on a later frame (`laterFrame`); a missing one draws the parent's. + * @param {import('../contract/globe').ImagerySpec} spec + * @param {number} slot + * @param {Extract} source + * @param {{laterFrame: () => Promise, signals: Set, + * urgency: () => 'shown'|'prefetch'}} ctx + */ +export function decodedProvider(spec, slot, source, ctx) { + // The same tiling and bounds a video frame's provider draws in. + const provider = videoProvider(spec, { url: '', fraction: 0 }, null); + provider.requestImage = async (x, y, level) => { + const controller = new AbortController(); + ctx.signals.add(controller); + try { + for (;;) { + const answer = await source.tile({ + tile: { x, y, level }, + slot, + signal: controller.signal, + urgency: ctx.urgency(), + }); + // An image no longer wanted is let go at once. + if (controller.signal.aborted) answer?.image?.close?.(); + controller.signal.throwIfAborted(); + if (answer?.defer) { + await ctx.laterFrame(); + controller.signal.throwIfAborted(); + continue; + } + if (!answer?.image) throw new MissingTile(); + return answer.image; + } + } finally { + ctx.signals.delete(controller); + } + }; + return provider; +} diff --git a/src/render/cesium/imageryFrames.test.mjs b/src/render/cesium/imageryFrames.test.mjs index c810ab485b..c6a8fe3f94 100644 --- a/src/render/cesium/imageryFrames.test.mjs +++ b/src/render/cesium/imageryFrames.test.mjs @@ -2,7 +2,11 @@ import assert from 'node:assert/strict'; import * as Cesium from 'cesium'; import test, { mock } from 'node:test'; import { + MissingTile, createTileVideos, + decodedHeld, + decodedProvider, + decodedShownAt, framesOf, heldFrames, layerDrawn, @@ -336,3 +340,98 @@ test('a video’s length is learnt with a longer wait, and a failed one is not l restore(); } }); + +/** A decoded source over slots 0, 100, 200, with what its tiles answer. */ +function decodedSource(answers = () => ({ missing: true })) { + const asked = []; + return { + asked, + kind: 'decoded', + slotAt: (ms) => (ms < 0 ? null : Math.min(200, Math.round(ms / 100) * 100)), + prefetch: (ms) => [Math.round(ms / 100) * 100, 300, NaN, 300, 400, 500], + tile: async (request) => { + asked.push(request); + return answers(request); + }, + }; +} + +test('a decoded source shows the slot its clock names and holds the next it prefetches', () => { + const source = decodedSource(); + assert.equal(framesOf(source).length, 0, 'no list to step through'); + const at = decodedShownAt(source, 140); + assert.deepEqual(at, { nearest: 100, shown: [{ index: 100, weight: 1 }] }); + assert.equal(decodedShownAt(source, -5), null, 'no slot: nothing shown'); + assert.equal(decodedShownAt(source, NaN), null); + assert.deepEqual(decodedHeld(source, at, 2, 140), [100, 300, 400]); + assert.deepEqual(decodedHeld(source, at, 0, 140), [100]); + assert.deepEqual( + decodedHeld({ slotAt: () => 0 }, at, 3, 140), + [100], + 'a source with no prefetch holds the shown slot only', + ); +}); + +test("a decoded slot's tiles are the source's images, asked again when deferred", async () => { + const image = { width: 256, height: 256 }; + let defers = 1; + const source = decodedSource(({ tile }) => { + if (tile.level === 9) return { missing: true }; + if (defers-- > 0) return { defer: true }; + return { image }; + }); + let frames = 0; + let shown = true; + const signals = new Set(); + const provider = decodedProvider( + { kind: 'xyz', url: '', maxLevel: 10, tiling: 'webmercator' }, + 100, + source, + { + laterFrame: async () => { + frames += 1; + }, + signals, + urgency: () => (shown ? 'shown' : 'prefetch'), + }, + ); + assert.ok(provider.tilingScheme instanceof Cesium.WebMercatorTilingScheme); + assert.equal(provider.maximumLevel, 10); + assert.equal(provider.hasAlphaChannel, true); + assert.equal(await provider.requestImage(1, 2, 3), image); + assert.equal(frames, 1, 'the deferred tile waited a frame'); + assert.deepEqual( + source.asked.map((r) => [r.tile, r.slot, r.urgency]), + [ + [{ x: 1, y: 2, level: 3 }, 100, 'shown'], + [{ x: 1, y: 2, level: 3 }, 100, 'shown'], + ], + ); + shown = false; + await assert.rejects(provider.requestImage(0, 0, 9), MissingTile); + assert.equal(source.asked.at(-1).urgency, 'prefetch'); + assert.equal(signals.size, 0, 'no tile left waiting'); +}); + +test('a decoded tile answered after its slot was let go is closed, not drawn', async () => { + let release; + const closed = []; + const image = { close: () => closed.push('image') }; + const source = decodedSource( + () => new Promise((resolve) => (release = () => resolve({ image }))), + ); + const signals = new Set(); + const provider = decodedProvider({ kind: 'single-image' }, 0, source, { + laterFrame: async () => {}, + signals, + urgency: () => 'shown', + }); + assert.equal(provider.maximumLevel, 0, 'one image over the rectangle'); + const pending = provider.requestImage(0, 0, 0); + await Promise.resolve(); + assert.ok(source.asked[0].signal instanceof AbortSignal); + for (const controller of signals) controller.abort(); + release(); + await assert.rejects(pending); + assert.deepEqual(closed, ['image']); +}); diff --git a/src/render/cesium/ports.js b/src/render/cesium/ports.js index 5f73daa482..09554e04c6 100644 --- a/src/render/cesium/ports.js +++ b/src/render/cesium/ports.js @@ -49,6 +49,8 @@ const GLOBE_CAPABILITIES = [ 'imageHooks', // Timed imagery: a layer a frame, prefetched unseen (`imagery.js`). 'imageryFrames', + // Timed imagery handed over as finished images (`imageryFrames.js`). + 'imageryDecoded', 'surfaceIdentity', 'surfaceShading', // A MapLibre style's vector tiles, painted as imagery (`vectorBase.js`). diff --git a/src/render/cesium/ports.test.mjs b/src/render/cesium/ports.test.mjs index fc0e40bff2..36602d89a9 100644 --- a/src/render/cesium/ports.test.mjs +++ b/src/render/cesium/ports.test.mjs @@ -28,6 +28,7 @@ test('capabilities follow what the scene supports', () => { 'splitSwipe', 'imageHooks', 'imageryFrames', + 'imageryDecoded', 'surfaceIdentity', 'surfaceShading', 'vectorBase', diff --git a/src/render/conformance/checks/ports.js b/src/render/conformance/checks/ports.js index 9d2409d9e2..10062c4f67 100644 --- a/src/render/conformance/checks/ports.js +++ b/src/render/conformance/checks/ports.js @@ -5425,6 +5425,125 @@ export function portChecks() { }, { capability: 'imageryFrames', bareSky: true }, ), + check( + 'globe.imagery.decoded', + 'decoded imagery draws the images its source hands over for the slot shown', + async (h, r) => { + await overAustin(h, r, 200000); + const t0 = r.frame.time(); + const slots = [t0, t0 + 60000, t0 + 120000]; + const colours = ['#ff0000', '#0000ff', '#00ff00']; + const pixels = h.pixels !== false; + /** A solid tile of a slot's colour, or a canvas where none is made. */ + const imageOf = async (slot) => { + const canvas = document.createElement('canvas'); + canvas.width = 16; + canvas.height = 16; + const g = canvas.getContext('2d'); + g.fillStyle = colours[slots.indexOf(slot)]; + g.fillRect(0, 0, 16, 16); + return createImageBitmap(canvas); + }; + const asked = []; + const source = { + kind: 'decoded', + slotAt: (ms) => { + if (ms < t0 - 30000) return null; + let best = slots[0]; + for (const slot of slots) + if (Math.abs(slot - ms) < Math.abs(best - ms)) best = slot; + return best; + }, + prefetch: (ms) => slots.filter((slot) => slot > ms), + async tile({ tile, slot, signal, urgency }) { + asked.push({ slot, urgency }); + signal.throwIfAborted(); + // The first ask of each slot is put off: the renderer asks again. + if (asked.filter((a) => a.slot === slot).length === 1) + return { defer: true }; + if (tile.level > 0) return { missing: true }; + return { image: await imageOf(slot) }; + }, + }; + const layer = r.globe.imagery.add({ + kind: 'single-image', + url: PIXEL_PNG, + rectangle: { + west: AUSTIN.lon - 2, + south: AUSTIN.lat - 2, + east: AUSTIN.lon + 2, + north: AUSTIN.lat + 2, + }, + }); + const ready = []; + layer.on('frameReady', (time) => ready.push(time)); + const wait = (ms) => new Promise((done) => setTimeout(done, ms)); + const shows = async (ms, slot, colour, what) => { + const before = ready.length; + layer.setFrameTime(ms); + for (let i = 0; i < 500 && ready.length === before; i++) { + await drawn(h, r, 1); + await wait(10); + } + equal(ready.at(-1), slot, `${what}: its slot is ready`); + if (!pixels) return; + r.frame.renderNow(); + const s = r.query.project(h.toEcef({ ...AUSTIN, height: 0 })); + const image = r.readPixels({ + crop: { + x: Math.round(s.x), + y: Math.round(s.y), + width: 1, + height: 1, + }, + }); + const [red, green, blue] = image.data; + ok(colour(red, green, blue), `${what}: (${red}, ${green}, ${blue})`); + }; + try { + throws( + () => layer.setFrames({ kind: 'decoded', slotAt: () => null }), + 'a decoded source with no tile', + ); + layer.setFrames(source, { clock: 'own', prefetch: 1 }); + await shows( + t0 + 10000, + t0, + (rr, g, b) => rr > 180 && g < 70 && b < 70, + 'the first slot', + ); + if (pixels) { + ok( + asked.some((a) => a.slot === t0 && a.urgency === 'shown'), + 'the slot shown is asked for as shown', + ); + ok( + asked.some( + (a) => a.slot === t0 + 60000 && a.urgency === 'prefetch', + ), + 'the next slot is asked for ahead, as a prefetch', + ); + ok( + !asked.some((a) => a.slot === t0 + 120000), + 'no more slots are held ahead than prefetch allows', + ); + } + await shows( + t0 + 65000, + t0 + 60000, + (rr, g, b) => b > 180 && rr < 70 && g < 70, + 'the second slot', + ); + ready.length = 0; + layer.setFrameTime(t0 - 60000); + for (let i = 0; i < 20; i++) await drawn(h, r, 1); + deepEqual(ready, [], 'a time with no slot announces nothing'); + } finally { + layer.remove(); + } + }, + { capability: ['imageryFrames', 'imageryDecoded'], bareSky: true }, + ), check( 'globe.withoutImageHooks', 'without imageHooks, a spec with main-thread tile hooks throws', diff --git a/src/render/contract/common.d.ts b/src/render/contract/common.d.ts index df585c2484..1ac6413542 100644 --- a/src/render/contract/common.d.ts +++ b/src/render/contract/common.d.ts @@ -165,6 +165,8 @@ export type Capability = | 'splitSwipe' /** Timed imagery frames with hidden prefetch. */ | 'imageryFrames' + /** Timed imagery the application hands over as finished tile images (`decoded`). */ + | 'imageryDecoded' /** Wind-relief and similar shading of the globe surface. */ | 'surfaceShading' /** A version of what surface heights answer against (`globe.surfaceVersion`). */ diff --git a/src/render/contract/globe.d.ts b/src/render/contract/globe.d.ts index abf63537e9..38516e66e4 100644 --- a/src/render/contract/globe.d.ts +++ b/src/render/contract/globe.d.ts @@ -130,17 +130,51 @@ export interface TimedUrl { url: string; } +/** + * A tile of a `decoded` source at one slot: its finished image, `missing` + * (no tile there; draw the parent's), or `defer` (ask again on a later + * frame). The renderer owns the image once answered: it draws it as it is + * (its alpha too) and may close it once drawn. + */ +export type DecodedTile = + { image: ImageBitmap } | { missing: true } | { defer: true }; + +/** What a `decoded` source is asked for: one tile at one slot. */ +export interface DecodedTileRequest { + tile: TileRequest; + /** The slot, as `slotAt` or `prefetch` named it. */ + slot: number; + /** Aborted once the renderer no longer wants the tile. */ + signal: AbortSignal; + /** `shown`: the slot the clock shows; `prefetch`: one held ahead. */ + urgency: 'shown' | 'prefetch'; +} + /** * A timed source: frames as URLs (one template per time), or video tiles * in segments, each a template whose tiles are videos spanning `start` to * `end` (epoch ms), a time within shown by its offset into the segment's * videos. Outside every segment, the nearest segment end is shown. + * + * `decoded` (`imageryDecoded`): tiles the application makes itself and hands + * over as finished images. Its slots are times (epoch ms): `slotAt` names + * the slot a time shows (`null`: none, and nothing is drawn), `prefetch` the + * slots to hold ahead of it, nearest first. The renderer asks `tile` for the + * tiles each held slot draws, in the layer's tiling, and only draws what it + * is given. It shows the nearest slot only (`mode` is not read); the + * `prefetch` option caps how many of the source's slots it holds ahead. */ export type TimedSource = | { kind: 'frames'; frames: TimedUrl[] } | { kind: 'video'; segments: Array<{ url: string; start: number; end: number }>; + } + | { + kind: 'decoded'; + slotAt(time: number): number | null; + prefetch?(time: number): number[]; + tile(request: DecodedTileRequest): Promise; }; export interface ImageryLayer { @@ -176,8 +210,8 @@ export interface ImageryLayer { /** `imageryFrames`: which time to show, on the `own` clock. */ setFrameTime?(ms: number): void; /** - * `imageryFrames`: the frame the layer's clock now shows (by its time) - * has been drawn. Fires once each time the shown frame changes, never + * `imageryFrames`: the frame the layer's clock now shows (by its time; + * a `decoded` source's by its slot) has been drawn. Fires once each time the shown frame changes, never * for frames only prefetched. */ on(event: 'frameReady', cb: (time: number) => void): Off; diff --git a/src/render/contract/index.js b/src/render/contract/index.js index 2198b6226c..3accf55f71 100644 --- a/src/render/contract/index.js +++ b/src/render/contract/index.js @@ -23,6 +23,7 @@ export const CAPABILITIES = Object.freeze([ 'imageryOnTiles', 'splitSwipe', 'imageryFrames', + 'imageryDecoded', 'surfaceShading', 'surfaceIdentity', 'imageHooks', diff --git a/src/render/stub/index.js b/src/render/stub/index.js index 9eb21cfc58..7ef3bcf7c4 100644 --- a/src/render/stub/index.js +++ b/src/render/stub/index.js @@ -1574,8 +1574,12 @@ export function createStubRenderer(options = {}) { if (capabilities.has('imageryFrames')) { let source = null; let clockKind = 'scene'; - /** The time the source shows for `ms`: the nearest frame, or within the videos. */ + /** The time the source shows for `ms`: the nearest frame, within the videos, or a decoded source's slot. */ const shownTime = (ms) => { + if (source?.kind === 'decoded') { + const slot = Number.isFinite(ms) ? source.slotAt(ms) : null; + return Number.isFinite(slot) ? slot : null; + } if (source?.kind === 'video') { let best = null; let bestOff = Infinity; @@ -1609,10 +1613,22 @@ export function createStubRenderer(options = {}) { deliver(listeners.frameReady, drawnTime); }); layer.setFrames = (next, opts = {}) => { + if (next.kind === 'decoded' && !capabilities.has('imageryDecoded')) + throw new TypeError('decoded imagery needs imageryDecoded'); + if ( + next.kind === 'decoded' && + (typeof next.slotAt !== 'function' || typeof next.tile !== 'function') + ) + throw new TypeError('a decoded source needs slotAt and tile'); source = - next.kind === 'video' - ? { kind: 'video', segments: next.segments.map((s) => ({ ...s })) } - : { kind: 'frames', frames: next.frames.slice() }; + next.kind === 'decoded' + ? next + : next.kind === 'video' + ? { + kind: 'video', + segments: next.segments.map((s) => ({ ...s })), + } + : { kind: 'frames', frames: next.frames.slice() }; wanted = null; drawnTime = null; clockKind = opts.clock ?? 'scene';