From a356d094d540617e56008af16dfd5605358fc168 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Wed, 11 Nov 2026 17:06:33 -0800 Subject: [PATCH] Timed imagery on the Cesium renderer (imageryFrames) A layer given a timed source hides its own imagery and draws the frames its clock shows (the scene's time, or setFrameTime on its own clock) as Cesium layers just above it: the nearest frame, or in blend mode the later over the earlier at its share of the way, with the next prefetch frames loading unseen at alpha 0. Video segments are seen every 250 ms, each tile its video seeked to the time's share of its length. frameReady names the nearest frame once every globe tile drawn shows its own imagery. The check globe.imagery.frames now runs on Cesium, and globe.imagery.video draws a recorded video's frames. --- docs/RENDERER-CONTRACT.md | 12 + scripts/package-boundaries.json | 7 + src/render/cesium/harness.js | 2 + src/render/cesium/imagery.js | 199 +++++++++++++- src/render/cesium/imageryFrames.js | 318 +++++++++++++++++++++++ src/render/cesium/imageryFrames.test.mjs | 158 +++++++++++ src/render/cesium/ports.js | 2 + src/render/cesium/ports.test.mjs | 1 + src/render/conformance/checks/ports.js | 89 +++++++ src/render/stub/stub.test.mjs | 3 +- 10 files changed, 783 insertions(+), 8 deletions(-) create mode 100644 src/render/cesium/imageryFrames.js create mode 100644 src/render/cesium/imageryFrames.test.mjs diff --git a/docs/RENDERER-CONTRACT.md b/docs/RENDERER-CONTRACT.md index 22a29abc4..ddb3c0a97 100644 --- a/docs/RENDERER-CONTRACT.md +++ b/docs/RENDERER-CONTRACT.md @@ -192,6 +192,18 @@ order-independently and would undo the addition; ground commands keep their pass and stencil and only blend differently. The checks `draw.sets.blend` (points, icons, lines, shapes) and `draw.models.blend` read the pixels. +The Cesium port draws timed imagery (`imageryFrames`): a layer given a timed +source hides its own imagery and draws each frame its clock shows as a Cesium +imagery layer just above it (in `blend` mode the later frame over the earlier +at its share of the way, as a layer's `alpha` blends), with the next +`prefetch` frames (1 by default) loading unseen at alpha 0. A video source is +seen every 250 ms of each segment (at most 1,200 times a segment), each tile +the video its URL names for it (or one video over the layer's rectangle for a +`single-image` layer), seeked to the time's share of its length. `frameReady` +names the nearest frame's time once every globe tile drawn shows that +frame's own imagery (`src/render/cesium/imageryFrames.js`). The checks +`globe.imagery.frames` and `globe.imagery.video` cover both. + The Cesium port also draws `pattern` line styles (the image repeated `repeat` times along the line, in the translucent pass; draped too), lines in the `inertial` frame (turned each frame by the set's motion clock, a depth-fail diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 8b4a2adcd..43683eaff 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1208,6 +1208,7 @@ "src/render/cesium/globe.js", "src/render/baseTilesets.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/tileRequests.js", "src/render/cesium/google3d.js", "src/maps/startupRoute.js", @@ -2116,6 +2117,7 @@ "src/render/baseTilesets.js", "src/render/cesium/google3d.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/tileRequests.js", "src/render/cesium/iconSetPrimitive.js", "src/data/aircraftRecession.js", @@ -2763,6 +2765,7 @@ "src/render/baseTilesets.js", "src/render/cesium/google3d.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/tileRequests.js", "src/layers/traffic/model.js", "src/geo/ellipsoid.js", @@ -2916,6 +2919,7 @@ "src/render/cesium/google3d.js", "src/render/cesium/globe.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/tileRequests.js", "src/data/headingDegrees.js", "src/data/cctvCards.js", @@ -3513,6 +3517,7 @@ "src/render/creditMarkup.js", "src/maps/defaultSources.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/google3d.js", "src/render/cesium/mapImagery.js", "src/maps/registry.js", @@ -4190,6 +4195,7 @@ "src/maps/credits.js", "src/maps/defaultSources.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/google3d.js", "src/render/cesium/mapImagery.js", "src/maps/registry.js", @@ -4915,6 +4921,7 @@ "src/render/cesium/gpuIcons.js", "src/render/motion/iconMotion.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/google3d.js", "src/render/cesium/cameraControls.js", "src/render/cesium/follow.js", diff --git a/src/render/cesium/harness.js b/src/render/cesium/harness.js index 401222831..3b3df6db8 100644 --- a/src/render/cesium/harness.js +++ b/src/render/cesium/harness.js @@ -89,6 +89,8 @@ export const CESIUM_CHECKS = new Set([ 'frame.suspend', 'frame.time', 'globe.imagery', + 'globe.imagery.frames', + 'globe.imagery.video', 'globe.sky', 'globe.split', 'globe.surfaceShading', diff --git a/src/render/cesium/imagery.js b/src/render/cesium/imagery.js index 1aefadd18..5aed7998f 100644 --- a/src/render/cesium/imagery.js +++ b/src/render/cesium/imagery.js @@ -10,10 +10,26 @@ * is shown, else on the base tileset. Port layers keep their order among * themselves in every collection; layers others added to a collection stay * where they are. + * + * With `imageryFrames`, a handle given a timed source (`setFrames`) hides + * its own layer and draws the frames its clock shows instead, each a Cesium + * layer of its own just above it: the nearest frame (in `blend` mode the + * later frame over the earlier one, at its share of the way), and the next + * `prefetch` frames loading unseen at alpha 0. A frame is ready + * (`frameReady`) once the globe's tiles drawn are drawn from its own + * imagery (`imageryFrames.js`). */ import * as Cesium from 'cesium'; import { creditMarkup } from './credits.js'; import { listenerList } from './common.js'; +import { + createTileVideos, + framesOf, + heldFrames, + layerDrawn, + shownAt, + videoProvider, +} from './imageryFrames.js'; import { withTileRequestHook } from './tileRequests.js'; const SPLITS = { none: 'NONE', left: 'LEFT', right: 'RIGHT' }; @@ -422,7 +438,8 @@ export function createCesiumImagery( ? new Cesium.ImageryLayer(record.provider) : Cesium.ImageryLayer.fromProviderAsync(record.providerPromise); layer.alpha = record.alpha; - layer.show = record.show; + // A layer showing timed frames hides its own imagery. + layer.show = record.show && !record.timed; layer.splitDirection = Cesium.SplitDirection[SPLITS[record.split]]; return layer; } @@ -505,8 +522,8 @@ export function createCesiumImagery( signals, offError: null, }; - const watchErrors = (provider) => { - record.offError = provider.errorEvent?.addEventListener?.((error) => { + const watchErrors = (provider) => + provider.errorEvent?.addEventListener?.((error) => { if (error?.error instanceof MissingTile) return; if (listeners.tileError.size === 0) return; const status = error?.error?.statusCode; @@ -515,12 +532,14 @@ export function createCesiumImagery( ...(Number.isFinite(status) ? { status } : {}), }); }); - }; + record.listeners = listeners; + record.ctx = ctx; + record.watchErrors = watchErrors; const made = providerFor(spec, { worldDataToken: worldDataToken() }); if (typeof made?.then === 'function') { record.providerPromise = Promise.resolve(made).then((provider) => { const hooked = hookProvider(provider, spec, ctx); - if (!record.removed) watchErrors(provider); + if (!record.removed) record.offError = watchErrors(provider); record.provider = hooked; return hooked; }); @@ -536,7 +555,7 @@ export function createCesiumImagery( }); } else { record.provider = hookProvider(made, spec, ctx); - watchErrors(made); + record.offError = watchErrors(made); } record.cesium = build(record); @@ -554,7 +573,7 @@ export function createCesiumImagery( }, set show(value) { record.show = Boolean(value); - record.cesium.show = record.show; + record.cesium.show = record.show && !record.timed; requestRender('imagery'); }, get split() { @@ -607,6 +626,7 @@ export function createCesiumImagery( const i = layers.indexOf(record); if (i >= 0) layers.splice(i, 1); record.offError?.(); + dropFrames(record); for (const controller of signals) controller.abort(); // Cesium leaves a removed layer's requests running: cancelled, they // are aborted and free their per-server slots on the next frame. @@ -621,6 +641,33 @@ export function createCesiumImagery( requestRender('imagery'); }, }; + if (capabilities.has('imageryFrames')) { + handle.setFrames = (source, options = {}) => { + const frames = framesOf(source); + if (record.removed) return; + dropFrames(record); + record.timed = { + video: source.kind === 'video', + frames, + mode: options.mode === 'blend' ? 'blend' : 'nearest', + prefetch: Math.max(0, Math.floor(options.prefetch ?? 1)), + clock: options.clock === 'own' ? 'own' : 'scene', + own: null, + at: null, + announced: null, + layers: new Map(), + videos: source.kind === 'video' ? createTileVideos() : null, + }; + record.cesium.show = false; + requestRender('imagery'); + }; + handle.setFrameTime = (ms) => { + // On the scene clock the frame port's time decides. + if (record.timed?.clock !== 'own') return; + record.timed.own = Number(ms); + requestRender('imagery'); + }; + } record.handle = handle; let at = index; @@ -637,6 +684,143 @@ export function createCesiumImagery( return handle; } + // --- Timed frames (`imageryFrames`) --------------------------------------- + + /** A Cesium layer drawing one frame of a record's timed source. */ + function frameLayer(record, frame) { + const timed = record.timed; + const spec = { ...record.spec, url: frame.url }; + const entry = { layer: null, host: null, offError: null }; + let made; + try { + made = timed.video + ? videoProvider(record.spec, frame, timed.videos) + : providerFor(spec, { worldDataToken: worldDataToken() }); + } catch (error) { + report(error); + made = Promise.reject(error); + } + if (typeof made?.then === 'function') { + const provider = Promise.resolve(made).then((provider) => { + if (!entry.removed) entry.offError = record.watchErrors(provider); + return hookProvider(provider, spec, record.ctx); + }); + provider.catch(() => {}); + entry.layer = Cesium.ImageryLayer.fromProviderAsync(provider); + } else { + entry.offError = record.watchErrors(made); + entry.layer = new Cesium.ImageryLayer( + timed.video ? made : hookProvider(made, spec, record.ctx), + ); + } + entry.layer.alpha = 0; + return entry; + } + + function dropFrameLayer(entry) { + entry.removed = true; + entry.offError?.(); + if (usable(entry.host) && entry.host.contains(entry.layer)) + entry.host.remove(entry.layer, true); + else if (!entry.layer.isDestroyed()) entry.layer.destroy(); + entry.host = null; + } + + /** Let go of a record's timed source and its frames' layers. */ + function dropFrames(record) { + const timed = record.timed; + if (!timed) return; + for (const entry of timed.layers.values()) dropFrameLayer(entry); + timed.layers.clear(); + timed.videos?.clear(); + record.timed = null; + if (!record.cesium.isDestroyed()) record.cesium.show = record.show; + } + + /** + * Before each frame: the frames each timed layer's clock shows now, and + * those it prefetches, as layers just above its own, in time order. + */ + function stepFrames(time) { + if (disposed) return; + const sceneMs = time ? Cesium.JulianDate.toDate(time).getTime() : NaN; + for (const record of layers) { + const timed = record.timed; + if (!timed || record.removed) continue; + const ms = timed.clock === 'own' ? (timed.own ?? NaN) : sceneMs; + const at = shownAt(timed.frames, ms, timed.mode); + timed.at = at; + const held = at ? heldFrames(timed.frames, at, timed.prefetch) : []; + for (const [index, entry] of timed.layers) + if (!held.includes(index) || entry.layer.isDestroyed()) { + dropFrameLayer(entry); + timed.layers.delete(index); + } + const host = + usable(record.host) && record.host.contains(record.cesium) + ? record.host + : null; + const weights = new Map(at?.shown.map((s) => [s.index, s.weight])); + let place = host ? host.indexOf(record.cesium) + 1 : -1; + for (const index of held.sort((a, b) => a - b)) { + let entry = timed.layers.get(index); + if (!entry) { + entry = frameLayer(record, timed.frames[index]); + timed.layers.set(index, entry); + } + const layer = entry.layer; + const weight = weights.get(index); + // Prefetched frames load unseen: shown, at alpha 0. + layer.show = record.show; + layer.alpha = weight === undefined ? 0 : record.alpha * weight; + layer.splitDirection = Cesium.SplitDirection[SPLITS[record.split]]; + if (entry.host !== host) { + if (usable(entry.host) && entry.host.contains(layer)) + entry.host.remove(layer, false); + entry.host = null; + if (host) { + host.add(layer, Math.min(place, host.length)); + entry.host = host; + } + } else if (host && host.indexOf(layer) !== place) { + host.remove(layer, false); + host.add(layer, Math.min(place, host.length)); + } + if (host) place = host.indexOf(layer) + 1; + } + } + } + + /** Whether a frame's layer is drawn where it drapes. */ + function frameDrawn(entry) { + if (!entry?.host || !entry.layer.ready) return false; + if (entry.host === globeCollection()) return layerDrawn(scene, entry.layer); + return objects()?.tilesLoaded === true; + } + + /** After each frame: each timed layer's shown frame, once it is drawn. */ + function announceFrames() { + if (disposed) return; + for (const record of layers.slice()) { + const timed = record.timed; + // A layer removed by another's listener says nothing. + if (record.removed || !timed?.at || !record.show) continue; + const time = timed.frames[timed.at.nearest].time; + if (time === timed.announced) continue; + if (!timed.at.shown.every((s) => frameDrawn(timed.layers.get(s.index)))) + continue; + timed.announced = time; + record.listeners.frameReady.fire(time); + } + } + + const frameOffs = capabilities.has('imageryFrames') + ? [ + scene.preRender?.addEventListener?.((_, time) => stepFrames(time)), + scene.postRender?.addEventListener?.(() => announceFrames()), + ] + : []; + return { add, list: () => layers.map((record) => record.handle), @@ -656,6 +840,7 @@ export function createCesiumImagery( */ dispose() { disposed = true; + for (const off of frameOffs.splice(0)) off?.(); clearTimeout(retryTimer); retryTimer = null; for (const resolve of waiters.splice(0)) resolve(); diff --git a/src/render/cesium/imageryFrames.js b/src/render/cesium/imageryFrames.js new file mode 100644 index 000000000..851d69dd0 --- /dev/null +++ b/src/render/cesium/imageryFrames.js @@ -0,0 +1,318 @@ +/** + * @module render/cesium/imageryFrames + * @description Timed imagery (`imageryFrames`) on the Cesium globe: which + * frames of a `TimedSource` a time shows and prefetches, video tiles drawn + * at a time within their segment, and whether a layer's tiles in view are + * drawn from its own imagery. + * + * A source is a list of frames, in time order. A `frames` source is its own + * list. A `video` source is each segment's videos seen every + * `VIDEO_STEP_MS` from its start to its end (at most `VIDEO_STEPS_MAX` + * steps a segment): a time is shown by the step nearest it, its offset into + * the segment's videos in proportion; outside every segment, the nearest + * segment end is the nearest frame. + */ +import * as Cesium from 'cesium'; + +/** How far apart the times a video segment is seen at are, ms. */ +export const VIDEO_STEP_MS = 250; +/** The most times one video segment is seen at. */ +export const VIDEO_STEPS_MAX = 1200; +/** Tile videos kept loaded a layer, the least recently seen let go first. */ +export const VIDEO_CACHE = 48; + +/** + * The frames of a timed source, in time order: `{time, url}`, and for + * video `fraction`, how far into the segment's videos the time is. + * @param {import('../contract/globe').TimedSource} source + * @returns {Array<{time: number, url: string, fraction?: number}>} + */ +export function framesOf(source) { + if (source?.kind === 'video') { + const frames = []; + for (const segment of source.segments ?? []) { + const { url, start, end } = segment; + if (typeof url !== 'string' || !(end >= start)) continue; + const span = end - start; + const steps = Math.min( + VIDEO_STEPS_MAX, + Math.max(1, Math.ceil(span / VIDEO_STEP_MS)), + ); + for (let k = 0; k <= steps; k++) + frames.push({ + time: start + (span * k) / steps, + url, + fraction: span > 0 ? k / steps : 0, + }); + } + return frames.sort((a, b) => a.time - b.time); + } + if (source?.kind === 'frames') + return (source.frames ?? []) + .filter((f) => Number.isFinite(f?.time) && typeof f?.url === 'string') + .map((f) => ({ time: f.time, url: f.url })) + .sort((a, b) => a.time - b.time); + throw new TypeError(`Unknown timed source: ${source?.kind}`); +} + +/** The index of the frame nearest `ms` (the earlier one on a tie). */ +function nearestIndex(frames, ms) { + let lo = 0; + let hi = frames.length - 1; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (frames[mid].time < ms) lo = mid + 1; + else hi = mid; + } + if (lo > 0 && ms - frames[lo - 1].time <= frames[lo].time - ms) return lo - 1; + return lo; +} + +/** + * What `ms` shows: the nearest frame, and in `blend` mode the frames either + * side of it, each with its weight (the later one's share of the way). + * `nearest` is the frame the clock shows, which `frameReady` names. + * @returns {{nearest: number, shown: Array<{index: number, weight: number}>}|null} + */ +export function shownAt(frames, ms, mode = 'nearest') { + if (!frames.length || !Number.isFinite(ms)) return null; + const nearest = nearestIndex(frames, ms); + if (mode !== 'blend') + return { nearest, shown: [{ index: nearest, weight: 1 }] }; + let before = nearest; + if (frames[before].time > ms && before > 0) before -= 1; + const after = Math.min(frames.length - 1, before + 1); + const t0 = frames[before].time; + const t1 = frames[after].time; + if (after === before || ms <= t0 || t1 <= t0) + return { nearest, shown: [{ index: before, weight: 1 }] }; + if (ms >= t1) return { nearest, shown: [{ index: after, weight: 1 }] }; + const f = (ms - t0) / (t1 - t0); + return { + nearest, + shown: [ + { index: before, weight: 1 }, + { index: after, weight: f }, + ], + }; +} + +/** + * The frames to hold loaded: those shown, and the `prefetch` after the + * last of them (playback runs forward). + * @returns {number[]} Indices. + */ +export function heldFrames(frames, shown, prefetch) { + const held = new Set(shown.shown.map((s) => s.index)); + const last = Math.max(...held); + for (let k = 1; k <= prefetch && last + k < frames.length; k++) + held.add(last + k); + return [...held]; +} + +/** The URL of a tile from a template, as `xyz` imagery fills it. */ +function tileUrlOf(template, tilingScheme, x, y, level) { + const rows = tilingScheme.getNumberOfYTilesAtLevel(level); + return template + .replaceAll('{x}', String(x)) + .replaceAll('{y}', String(y)) + .replaceAll('{z}', String(level)) + .replaceAll('{reverseY}', String(rows - 1 - y)); +} + +/** + * Seek a video, resolving once it has: on a `seeked` that leaves it no + * longer seeking (one a seek before it sends late does not count), and its + * frame there presented where the browser says so. + */ +function seekTo(video, time, url) { + return new Promise((resolve, reject) => { + const failed = () => { + video.removeEventListener('seeked', seeked); + reject(new Error(`The video tile did not seek: ${url}`)); + }; + const seeked = () => { + if (video.seeking) return; + video.removeEventListener('seeked', seeked); + video.removeEventListener('error', failed); + if (typeof video.requestVideoFrameCallback !== 'function') resolve(); + else { + // Its frame is drawable once presented; never wait on it for long. + const timer = setTimeout(resolve, 100); + video.requestVideoFrameCallback(() => { + clearTimeout(timer); + resolve(); + }); + } + }; + video.addEventListener('seeked', seeked); + video.addEventListener('error', failed, { once: true }); + video.currentTime = time; + }); +} + +/** + * A layer's tile videos, loaded once each and seeked for each frame drawn + * from them, one seek at a time a video. + */ +export function createTileVideos(limit = VIDEO_CACHE) { + /** url → {video, ready: Promise, queue: Promise} */ + const videos = new Map(); + function load(url) { + let entry = videos.get(url); + if (entry) { + videos.delete(url); + videos.set(url, entry); + return entry; + } + const video = document.createElement('video'); + video.muted = true; + video.playsInline = true; + video.preload = 'auto'; + video.crossOrigin = 'anonymous'; + const loaded = new Promise((resolve, reject) => { + video.addEventListener('loadeddata', () => resolve(video), { + once: true, + }); + video.addEventListener( + 'error', + () => reject(new Error(`The video tile did not load: ${url}`)), + { once: true }, + ); + }); + // A video that does not say how long it is (a recorded WebM, say) is + // seeked past its end once to learn it. + const ready = loaded.then(async () => { + if (!Number.isFinite(video.duration)) + await seekTo(video, Number.MAX_SAFE_INTEGER, url); + return video; + }); + ready.catch(() => {}); + video.src = url; + entry = { video, ready, queue: Promise.resolve() }; + videos.set(url, entry); + while (videos.size > limit) { + const [oldest, old] = videos.entries().next().value; + videos.delete(oldest); + old.video.removeAttribute('src'); + old.video.load(); + } + return entry; + } + /** The video at `fraction` of its length, drawn on a canvas. */ + function frame(url, fraction) { + const entry = load(url); + const run = entry.queue.then(async () => { + const video = await entry.ready; + const duration = Number.isFinite(video.duration) ? video.duration : 0; + const at = Math.max(0, Math.min(duration - 1e-3, fraction * duration)); + if (Math.abs(video.currentTime - at) > 1e-4) await seekTo(video, at, url); + const canvas = document.createElement('canvas'); + canvas.width = video.videoWidth || 1; + canvas.height = video.videoHeight || 1; + canvas.getContext('2d').drawImage(video, 0, 0); + return canvas; + }); + entry.queue = run.catch(() => {}); + return run; + } + return { + frame, + clear() { + for (const { video } of videos.values()) { + video.removeAttribute('src'); + video.load(); + } + videos.clear(); + }, + }; +} + +/** + * A provider drawing a video frame's tiles: each tile the video its URL + * template names for it (an `xyz` spec), or the one video over the layer's + * rectangle (any other), at the frame's place in it. + * @param {import('../contract/globe').ImagerySpec} spec + * @param {{url: string, fraction: number}} frame + * @param {ReturnType} videos + */ +export function videoProvider(spec, frame, videos) { + const tiled = spec.kind === 'xyz'; + const rectangle = spec.rectangle + ? Cesium.Rectangle.fromDegrees( + spec.rectangle.west, + spec.rectangle.south, + spec.rectangle.east, + spec.rectangle.north, + ) + : undefined; + const tilingScheme = tiled + ? spec.tiling === 'geographic' + ? new Cesium.GeographicTilingScheme() + : new Cesium.WebMercatorTilingScheme() + : new Cesium.GeographicTilingScheme({ + rectangle: rectangle ?? Cesium.Rectangle.MAX_VALUE, + numberOfLevelZeroTilesX: 1, + numberOfLevelZeroTilesY: 1, + }); + const size = spec.tileSize ?? 256; + return { + tilingScheme, + tileWidth: size, + tileHeight: size, + minimumLevel: tiled ? (spec.minLevel ?? 0) : 0, + maximumLevel: tiled ? spec.maxLevel : 0, + rectangle: rectangle ?? tilingScheme.rectangle, + credit: undefined, + errorEvent: new Cesium.Event(), + hasAlphaChannel: true, + tileDiscardPolicy: undefined, + proxy: undefined, + getTileCredits: () => undefined, + pickFeatures: () => undefined, + requestImage(x, y, level) { + const url = tiled + ? tileUrlOf(frame.url, tilingScheme, x, y, level) + : frame.url; + return videos.frame(url, frame.fraction ?? 0); + }, + }; +} + +const { ImageryState } = Cesium; +/** States a tile's imagery settles in: drawn, or nothing better to come. */ +const SETTLED = new Set([ + ImageryState.READY, + ImageryState.FAILED, + ImageryState.INVALID, +]); + +/** + * Whether every globe tile drawn last frame draws `layer` from its own + * imagery (not an ancestor's while its own loads). Tiles outside the + * layer's rectangle have none to draw. + */ +export function layerDrawn(scene, layer) { + if (!layer || layer.isDestroyed?.() || !layer.ready) return false; + const tiles = scene.globe?._surface?._tilesToRender; + if (!tiles?.length) return false; + const bounds = layer.imageryProvider?.rectangle ?? layer.rectangle; + for (const tile of tiles) { + let found = false; + for (const tileImagery of tile.data?.imagery ?? []) { + const loading = tileImagery.loadingImagery; + const ready = tileImagery.readyImagery; + if ((loading ?? ready)?.imageryLayer !== layer) continue; + found = true; + if (loading && !SETTLED.has(loading.state)) return false; + if (loading?.state === ImageryState.READY && ready !== loading) + return false; + } + if ( + !found && + (!bounds || Cesium.Rectangle.intersection(tile.rectangle, bounds)) + ) + return false; + } + return true; +} diff --git a/src/render/cesium/imageryFrames.test.mjs b/src/render/cesium/imageryFrames.test.mjs new file mode 100644 index 000000000..2bc61253e --- /dev/null +++ b/src/render/cesium/imageryFrames.test.mjs @@ -0,0 +1,158 @@ +import assert from 'node:assert/strict'; +import * as Cesium from 'cesium'; +import test from 'node:test'; +import { + framesOf, + heldFrames, + layerDrawn, + shownAt, + VIDEO_STEP_MS, + videoProvider, +} from './imageryFrames.js'; + +test('frames come in time order; video segments are seen at steps from start to end', () => { + assert.deepEqual( + framesOf({ + kind: 'frames', + frames: [ + { time: 20, url: 'b' }, + { time: 10, url: 'a' }, + { time: NaN, url: 'x' }, + ], + }), + [ + { time: 10, url: 'a' }, + { time: 20, url: 'b' }, + ], + ); + const video = framesOf({ + kind: 'video', + segments: [ + { url: 'late', start: 5000, end: 5000 + 2 * VIDEO_STEP_MS }, + { url: 'early', start: 0, end: VIDEO_STEP_MS }, + ], + }); + assert.deepEqual( + video.map((f) => [f.time, f.url, f.fraction]), + [ + [0, 'early', 0], + [VIDEO_STEP_MS, 'early', 1], + [5000, 'late', 0], + [5000 + VIDEO_STEP_MS, 'late', 0.5], + [5000 + 2 * VIDEO_STEP_MS, 'late', 1], + ], + ); + assert.throws(() => framesOf({ kind: 'nope' }), /Unknown timed source/); +}); + +test('a time shows its nearest frame, or in blend mode the two either side', () => { + const frames = [0, 100, 200].map((time) => ({ time, url: String(time) })); + assert.deepEqual(shownAt(frames, 40), { + nearest: 0, + shown: [{ index: 0, weight: 1 }], + }); + assert.equal(shownAt(frames, 160).nearest, 2); + assert.equal(shownAt(frames, -50).nearest, 0, 'before the first'); + assert.equal(shownAt(frames, 900).nearest, 2, 'after the last'); + assert.deepEqual(shownAt(frames, 140, 'blend'), { + nearest: 1, + shown: [ + { index: 1, weight: 1 }, + { index: 2, weight: 0.4 }, + ], + }); + assert.deepEqual(shownAt(frames, 60, 'blend').shown, [ + { index: 0, weight: 1 }, + { index: 1, weight: 0.6 }, + ]); + assert.deepEqual(shownAt(frames, 100, 'blend').shown, [ + { index: 1, weight: 1 }, + ]); + assert.deepEqual(shownAt(frames, 900, 'blend').shown, [ + { index: 2, weight: 1 }, + ]); + assert.equal(shownAt([], 0), null); + assert.equal(shownAt(frames, NaN), null, 'no time yet'); +}); + +test('the frames held are those shown and the next ones to prefetch', () => { + const frames = [0, 1, 2, 3, 4].map((time) => ({ time, url: '' })); + assert.deepEqual(heldFrames(frames, shownAt(frames, 1), 2), [1, 2, 3]); + assert.deepEqual( + heldFrames(frames, shownAt(frames, 1.5, 'blend'), 1), + [1, 2, 3], + ); + assert.deepEqual(heldFrames(frames, shownAt(frames, 4), 3), [4]); +}); + +test('a video provider asks for each tile’s video at the frame’s place in it', async () => { + const asked = []; + const videos = { + frame: (url, fraction) => (asked.push([url, fraction]), 'canvas'), + }; + const tiled = videoProvider( + { kind: 'xyz', url: '', tiling: 'geographic' }, + { url: 'v/{z}/{x}/{y}.webm', fraction: 0.25 }, + videos, + ); + assert.equal(await tiled.requestImage(3, 1, 2), 'canvas'); + const single = videoProvider( + { + kind: 'single-image', + url: '', + rectangle: { west: -10, south: -5, east: 10, north: 5 }, + }, + { url: 'one.webm', fraction: 0.5 }, + videos, + ); + await single.requestImage(0, 0, 0); + assert.deepEqual(asked, [ + ['v/2/3/1.webm', 0.25], + ['one.webm', 0.5], + ]); + assert.equal(single.maximumLevel, 0, 'one tile'); + assert.ok(Math.abs(Cesium.Math.toDegrees(single.rectangle.east) - 10) < 1e-9); +}); + +test('a layer is drawn once every tile drawn shows its own imagery', () => { + const layer = { ready: true, isDestroyed: () => false, rectangle: undefined }; + const other = { ready: true }; + const { READY, TRANSITIONING, FAILED } = Cesium.ImageryState; + const world = Cesium.Rectangle.MAX_VALUE; + const tile = (...imagery) => ({ rectangle: world, data: { imagery } }); + const own = (state, ready = true) => { + const imagery = { imageryLayer: layer, state }; + return { + loadingImagery: imagery, + readyImagery: ready ? imagery : { imageryLayer: layer, state: READY }, + }; + }; + const scene = (tiles) => ({ globe: { _surface: { _tilesToRender: tiles } } }); + const theirs = { loadingImagery: { imageryLayer: other, state: READY } }; + assert.equal(layerDrawn(scene([tile(own(READY), theirs)]), layer), true); + assert.equal( + layerDrawn(scene([tile(own(READY)), tile(own(TRANSITIONING))]), layer), + false, + 'one still loading', + ); + assert.equal( + layerDrawn(scene([tile(own(READY, false))]), layer), + false, + 'drawn from an ancestor still', + ); + assert.equal( + layerDrawn(scene([tile(own(FAILED, false))]), layer), + true, + 'failed: nothing better to come', + ); + assert.equal( + layerDrawn(scene([tile(theirs)]), layer), + false, + 'its imagery not made yet', + ); + assert.equal(layerDrawn(scene([]), layer), false, 'nothing drawn'); + assert.equal( + layerDrawn(scene([tile(own(READY))]), { ...layer, ready: false }), + false, + ); +}); diff --git a/src/render/cesium/ports.js b/src/render/cesium/ports.js index 8e5e4ef0a..2d3a35d0e 100644 --- a/src/render/cesium/ports.js +++ b/src/render/cesium/ports.js @@ -46,6 +46,8 @@ const GLOBE_CAPABILITIES = [ 'imageryOnTiles', 'splitSwipe', 'imageHooks', + // Timed imagery: a layer a frame, prefetched unseen (`imagery.js`). + 'imageryFrames', 'surfaceIdentity', 'surfaceShading', 'globeVisibility', diff --git a/src/render/cesium/ports.test.mjs b/src/render/cesium/ports.test.mjs index f559fd1f9..4737b1121 100644 --- a/src/render/cesium/ports.test.mjs +++ b/src/render/cesium/ports.test.mjs @@ -27,6 +27,7 @@ test('capabilities follow what the scene supports', () => { 'imageryOnTiles', 'splitSwipe', 'imageHooks', + 'imageryFrames', 'surfaceIdentity', 'surfaceShading', ]; diff --git a/src/render/conformance/checks/ports.js b/src/render/conformance/checks/ports.js index 19c02da95..14a5cce84 100644 --- a/src/render/conformance/checks/ports.js +++ b/src/render/conformance/checks/ports.js @@ -3964,6 +3964,95 @@ export function portChecks() { }, { capability: 'imageryFrames' }, ), + check( + 'globe.imagery.video', + 'video tiles show the time within their segment, nearest segment end outside it', + async (h, r) => { + if (h.pixels === false) + return { skip: 'this harness reads no drawn pixels' }; + if (typeof MediaRecorder !== 'function') + return { skip: 'no MediaRecorder to make a video with' }; + await overAustin(h, r, 200000); + // Two seconds of video: red, then blue. + const canvas = document.createElement('canvas'); + canvas.width = 32; + canvas.height = 32; + const g = canvas.getContext('2d'); + const recorder = new MediaRecorder(canvas.captureStream(30), { + mimeType: 'video/webm', + }); + const chunks = []; + recorder.ondataavailable = (e) => chunks.push(e.data); + let colour = '#ff0000'; + const paint = setInterval(() => { + g.fillStyle = colour; + g.fillRect(0, 0, 32, 32); + }, 20); + const wait = (ms) => new Promise((done) => setTimeout(done, ms)); + recorder.start(); + await wait(1000); + colour = '#0000ff'; + await wait(1000); + await new Promise((done) => { + recorder.onstop = done; + recorder.stop(); + }); + clearInterval(paint); + const url = URL.createObjectURL( + new Blob(chunks, { type: 'video/webm' }), + ); + const t0 = r.frame.time(); + const layer = r.globe.imagery.add({ + kind: 'single-image', + url: PIXEL_PNG, + rectangle: { + west: AUSTIN.lon - 2, + south: AUSTIN.lat - 2, + east: AUSTIN.lon + 2, + north: AUSTIN.lat + 2, + }, + }); + const ready = []; + layer.on('frameReady', (time) => ready.push(time)); + const shows = async (ms, like, what) => { + const before = ready.length; + layer.setFrameTime(ms); + for (let i = 0; i < 500 && ready.length === before; i++) { + await drawn(h, r, 1); + await wait(10); + } + ok(ready.length > before, `${what}: ready`); + r.frame.renderNow(); + const s = r.query.project(h.toEcef({ ...AUSTIN, height: 0 })); + const image = r.readPixels({ + crop: { + x: Math.round(s.x), + y: Math.round(s.y), + width: 1, + height: 1, + }, + }); + const [red, , blue] = image.data; + ok(like(red, blue), `${what}: (${red}, ${blue})`); + }; + try { + layer.setFrames( + { kind: 'video', segments: [{ url, start: t0, end: t0 + 2000 }] }, + { clock: 'own' }, + ); + const red = (rr, b) => rr > 180 && b < 70; + const blue = (rr, b) => b > 180 && rr < 70; + await shows(t0 + 300, red, 'early in the segment'); + await shows(t0 + 1700, blue, 'late in the segment'); + await shows(t0 - 60000, red, 'before it: its start'); + await shows(t0 + 60000, blue, 'after it: its end'); + } finally { + layer.remove(); + URL.revokeObjectURL(url); + } + }, + { capability: 'imageryFrames' }, + ), check( 'globe.withoutImageHooks', 'without imageHooks, a spec with main-thread tile hooks throws', diff --git a/src/render/stub/stub.test.mjs b/src/render/stub/stub.test.mjs index 164a820df..f23a489da 100644 --- a/src/render/stub/stub.test.mjs +++ b/src/render/stub/stub.test.mjs @@ -20,13 +20,14 @@ test('the stub passes the conformance suite', async () => { .filter((r) => r.status === 'skip') .map((r) => r.id); // It draws only models into pixels, so the sweep's, a line outline's and - // the blends' pixels (over a drawn shape) go unread; its camera takes no user gestures: the drag and wheel scripts are for a + // the blends' (over a drawn shape) and video tiles' pixels go unread; its camera takes no user gestures: the drag and wheel scripts are for a // camera that does (a controller over it, say). assert.deepEqual(skipped, [ 'effects.screenModulation.pixels', 'draw.lines.outline', 'draw.sets.blend', 'draw.models.blend', + 'globe.imagery.video', ...CAMERA_SCRIPTS.filter(usesInput).map((s) => `camera.trace.${s.name}`), ]); });