diff --git a/docs/RENDERER-CONTRACT.md b/docs/RENDERER-CONTRACT.md index 22a29abc4..ddb3c0a97 100644 --- a/docs/RENDERER-CONTRACT.md +++ b/docs/RENDERER-CONTRACT.md @@ -192,6 +192,18 @@ order-independently and would undo the addition; ground commands keep their pass and stencil and only blend differently. The checks `draw.sets.blend` (points, icons, lines, shapes) and `draw.models.blend` read the pixels. +The Cesium port draws timed imagery (`imageryFrames`): a layer given a timed +source hides its own imagery and draws each frame its clock shows as a Cesium +imagery layer just above it (in `blend` mode the later frame over the earlier +at its share of the way, as a layer's `alpha` blends), with the next +`prefetch` frames (1 by default) loading unseen at alpha 0. A video source is +seen every 250 ms of each segment (at most 1,200 times a segment), each tile +the video its URL names for it (or one video over the layer's rectangle for a +`single-image` layer), seeked to the time's share of its length. `frameReady` +names the nearest frame's time once every globe tile drawn shows that +frame's own imagery (`src/render/cesium/imageryFrames.js`). The checks +`globe.imagery.frames` and `globe.imagery.video` cover both. + The Cesium port also draws `pattern` line styles (the image repeated `repeat` times along the line, in the translucent pass; draped too), lines in the `inertial` frame (turned each frame by the set's motion clock, a depth-fail diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 8b4a2adcd..43683eaff 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1208,6 +1208,7 @@ "src/render/cesium/globe.js", "src/render/baseTilesets.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/tileRequests.js", "src/render/cesium/google3d.js", "src/maps/startupRoute.js", @@ -2116,6 +2117,7 @@ "src/render/baseTilesets.js", "src/render/cesium/google3d.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/tileRequests.js", "src/render/cesium/iconSetPrimitive.js", "src/data/aircraftRecession.js", @@ -2763,6 +2765,7 @@ "src/render/baseTilesets.js", "src/render/cesium/google3d.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/tileRequests.js", "src/layers/traffic/model.js", "src/geo/ellipsoid.js", @@ -2916,6 +2919,7 @@ "src/render/cesium/google3d.js", "src/render/cesium/globe.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/tileRequests.js", "src/data/headingDegrees.js", "src/data/cctvCards.js", @@ -3513,6 +3517,7 @@ "src/render/creditMarkup.js", "src/maps/defaultSources.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/google3d.js", "src/render/cesium/mapImagery.js", "src/maps/registry.js", @@ -4190,6 +4195,7 @@ "src/maps/credits.js", "src/maps/defaultSources.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/google3d.js", "src/render/cesium/mapImagery.js", "src/maps/registry.js", @@ -4915,6 +4921,7 @@ "src/render/cesium/gpuIcons.js", "src/render/motion/iconMotion.js", "src/render/cesium/imagery.js", + "src/render/cesium/imageryFrames.js", "src/render/cesium/google3d.js", "src/render/cesium/cameraControls.js", "src/render/cesium/follow.js", diff --git a/src/render/cesium/harness.js b/src/render/cesium/harness.js index 401222831..3b3df6db8 100644 --- a/src/render/cesium/harness.js +++ b/src/render/cesium/harness.js @@ -89,6 +89,8 @@ export const CESIUM_CHECKS = new Set([ 'frame.suspend', 'frame.time', 'globe.imagery', + 'globe.imagery.frames', + 'globe.imagery.video', 'globe.sky', 'globe.split', 'globe.surfaceShading', diff --git a/src/render/cesium/imagery.js b/src/render/cesium/imagery.js index 1aefadd18..5aed7998f 100644 --- a/src/render/cesium/imagery.js +++ b/src/render/cesium/imagery.js @@ -10,10 +10,26 @@ * is shown, else on the base tileset. Port layers keep their order among * themselves in every collection; layers others added to a collection stay * where they are. + * + * With `imageryFrames`, a handle given a timed source (`setFrames`) hides + * its own layer and draws the frames its clock shows instead, each a Cesium + * layer of its own just above it: the nearest frame (in `blend` mode the + * later frame over the earlier one, at its share of the way), and the next + * `prefetch` frames loading unseen at alpha 0. A frame is ready + * (`frameReady`) once the globe's tiles drawn are drawn from its own + * imagery (`imageryFrames.js`). */ import * as Cesium from 'cesium'; import { creditMarkup } from './credits.js'; import { listenerList } from './common.js'; +import { + createTileVideos, + framesOf, + heldFrames, + layerDrawn, + shownAt, + videoProvider, +} from './imageryFrames.js'; import { withTileRequestHook } from './tileRequests.js'; const SPLITS = { none: 'NONE', left: 'LEFT', right: 'RIGHT' }; @@ -422,7 +438,8 @@ export function createCesiumImagery( ? new Cesium.ImageryLayer(record.provider) : Cesium.ImageryLayer.fromProviderAsync(record.providerPromise); layer.alpha = record.alpha; - layer.show = record.show; + // A layer showing timed frames hides its own imagery. + layer.show = record.show && !record.timed; layer.splitDirection = Cesium.SplitDirection[SPLITS[record.split]]; return layer; } @@ -505,8 +522,8 @@ export function createCesiumImagery( signals, offError: null, }; - const watchErrors = (provider) => { - record.offError = provider.errorEvent?.addEventListener?.((error) => { + const watchErrors = (provider) => + provider.errorEvent?.addEventListener?.((error) => { if (error?.error instanceof MissingTile) return; if (listeners.tileError.size === 0) return; const status = error?.error?.statusCode; @@ -515,12 +532,14 @@ export function createCesiumImagery( ...(Number.isFinite(status) ? { status } : {}), }); }); - }; + record.listeners = listeners; + record.ctx = ctx; + record.watchErrors = watchErrors; const made = providerFor(spec, { worldDataToken: worldDataToken() }); if (typeof made?.then === 'function') { record.providerPromise = Promise.resolve(made).then((provider) => { const hooked = hookProvider(provider, spec, ctx); - if (!record.removed) watchErrors(provider); + if (!record.removed) record.offError = watchErrors(provider); record.provider = hooked; return hooked; }); @@ -536,7 +555,7 @@ export function createCesiumImagery( }); } else { record.provider = hookProvider(made, spec, ctx); - watchErrors(made); + record.offError = watchErrors(made); } record.cesium = build(record); @@ -554,7 +573,7 @@ export function createCesiumImagery( }, set show(value) { record.show = Boolean(value); - record.cesium.show = record.show; + record.cesium.show = record.show && !record.timed; requestRender('imagery'); }, get split() { @@ -607,6 +626,7 @@ export function createCesiumImagery( const i = layers.indexOf(record); if (i >= 0) layers.splice(i, 1); record.offError?.(); + dropFrames(record); for (const controller of signals) controller.abort(); // Cesium leaves a removed layer's requests running: cancelled, they // are aborted and free their per-server slots on the next frame. @@ -621,6 +641,33 @@ export function createCesiumImagery( requestRender('imagery'); }, }; + if (capabilities.has('imageryFrames')) { + handle.setFrames = (source, options = {}) => { + const frames = framesOf(source); + if (record.removed) return; + dropFrames(record); + record.timed = { + video: source.kind === 'video', + frames, + mode: options.mode === 'blend' ? 'blend' : 'nearest', + prefetch: Math.max(0, Math.floor(options.prefetch ?? 1)), + clock: options.clock === 'own' ? 'own' : 'scene', + own: null, + at: null, + announced: null, + layers: new Map(), + videos: source.kind === 'video' ? createTileVideos() : null, + }; + record.cesium.show = false; + requestRender('imagery'); + }; + handle.setFrameTime = (ms) => { + // On the scene clock the frame port's time decides. + if (record.timed?.clock !== 'own') return; + record.timed.own = Number(ms); + requestRender('imagery'); + }; + } record.handle = handle; let at = index; @@ -637,6 +684,143 @@ export function createCesiumImagery( return handle; } + // --- Timed frames (`imageryFrames`) --------------------------------------- + + /** A Cesium layer drawing one frame of a record's timed source. */ + function frameLayer(record, frame) { + const timed = record.timed; + const spec = { ...record.spec, url: frame.url }; + const entry = { layer: null, host: null, offError: null }; + let made; + try { + made = timed.video + ? videoProvider(record.spec, frame, timed.videos) + : providerFor(spec, { worldDataToken: worldDataToken() }); + } catch (error) { + report(error); + made = Promise.reject(error); + } + if (typeof made?.then === 'function') { + const provider = Promise.resolve(made).then((provider) => { + if (!entry.removed) entry.offError = record.watchErrors(provider); + return hookProvider(provider, spec, record.ctx); + }); + provider.catch(() => {}); + entry.layer = Cesium.ImageryLayer.fromProviderAsync(provider); + } else { + entry.offError = record.watchErrors(made); + entry.layer = new Cesium.ImageryLayer( + timed.video ? made : hookProvider(made, spec, record.ctx), + ); + } + entry.layer.alpha = 0; + return entry; + } + + function dropFrameLayer(entry) { + entry.removed = true; + entry.offError?.(); + if (usable(entry.host) && entry.host.contains(entry.layer)) + entry.host.remove(entry.layer, true); + else if (!entry.layer.isDestroyed()) entry.layer.destroy(); + entry.host = null; + } + + /** Let go of a record's timed source and its frames' layers. */ + function dropFrames(record) { + const timed = record.timed; + if (!timed) return; + for (const entry of timed.layers.values()) dropFrameLayer(entry); + timed.layers.clear(); + timed.videos?.clear(); + record.timed = null; + if (!record.cesium.isDestroyed()) record.cesium.show = record.show; + } + + /** + * Before each frame: the frames each timed layer's clock shows now, and + * those it prefetches, as layers just above its own, in time order. + */ + function stepFrames(time) { + if (disposed) return; + const sceneMs = time ? Cesium.JulianDate.toDate(time).getTime() : NaN; + for (const record of layers) { + const timed = record.timed; + if (!timed || record.removed) continue; + const ms = timed.clock === 'own' ? (timed.own ?? NaN) : sceneMs; + const at = shownAt(timed.frames, ms, timed.mode); + timed.at = at; + const held = at ? heldFrames(timed.frames, at, timed.prefetch) : []; + for (const [index, entry] of timed.layers) + if (!held.includes(index) || entry.layer.isDestroyed()) { + dropFrameLayer(entry); + timed.layers.delete(index); + } + const host = + usable(record.host) && record.host.contains(record.cesium) + ? record.host + : null; + const weights = new Map(at?.shown.map((s) => [s.index, s.weight])); + let place = host ? host.indexOf(record.cesium) + 1 : -1; + for (const index of held.sort((a, b) => a - b)) { + let entry = timed.layers.get(index); + if (!entry) { + entry = frameLayer(record, timed.frames[index]); + timed.layers.set(index, entry); + } + const layer = entry.layer; + const weight = weights.get(index); + // Prefetched frames load unseen: shown, at alpha 0. + layer.show = record.show; + layer.alpha = weight === undefined ? 0 : record.alpha * weight; + layer.splitDirection = Cesium.SplitDirection[SPLITS[record.split]]; + if (entry.host !== host) { + if (usable(entry.host) && entry.host.contains(layer)) + entry.host.remove(layer, false); + entry.host = null; + if (host) { + host.add(layer, Math.min(place, host.length)); + entry.host = host; + } + } else if (host && host.indexOf(layer) !== place) { + host.remove(layer, false); + host.add(layer, Math.min(place, host.length)); + } + if (host) place = host.indexOf(layer) + 1; + } + } + } + + /** Whether a frame's layer is drawn where it drapes. */ + function frameDrawn(entry) { + if (!entry?.host || !entry.layer.ready) return false; + if (entry.host === globeCollection()) return layerDrawn(scene, entry.layer); + return objects()?.tilesLoaded === true; + } + + /** After each frame: each timed layer's shown frame, once it is drawn. */ + function announceFrames() { + if (disposed) return; + for (const record of layers.slice()) { + const timed = record.timed; + // A layer removed by another's listener says nothing. + if (record.removed || !timed?.at || !record.show) continue; + const time = timed.frames[timed.at.nearest].time; + if (time === timed.announced) continue; + if (!timed.at.shown.every((s) => frameDrawn(timed.layers.get(s.index)))) + continue; + timed.announced = time; + record.listeners.frameReady.fire(time); + } + } + + const frameOffs = capabilities.has('imageryFrames') + ? [ + scene.preRender?.addEventListener?.((_, time) => stepFrames(time)), + scene.postRender?.addEventListener?.(() => announceFrames()), + ] + : []; + return { add, list: () => layers.map((record) => record.handle), @@ -656,6 +840,7 @@ export function createCesiumImagery( */ dispose() { disposed = true; + for (const off of frameOffs.splice(0)) off?.(); clearTimeout(retryTimer); retryTimer = null; for (const resolve of waiters.splice(0)) resolve(); diff --git a/src/render/cesium/imageryFrames.js b/src/render/cesium/imageryFrames.js new file mode 100644 index 000000000..851d69dd0 --- /dev/null +++ b/src/render/cesium/imageryFrames.js @@ -0,0 +1,318 @@ +/** + * @module render/cesium/imageryFrames + * @description Timed imagery (`imageryFrames`) on the Cesium globe: which + * frames of a `TimedSource` a time shows and prefetches, video tiles drawn + * at a time within their segment, and whether a layer's tiles in view are + * drawn from its own imagery. + * + * A source is a list of frames, in time order. A `frames` source is its own + * list. A `video` source is each segment's videos seen every + * `VIDEO_STEP_MS` from its start to its end (at most `VIDEO_STEPS_MAX` + * steps a segment): a time is shown by the step nearest it, its offset into + * the segment's videos in proportion; outside every segment, the nearest + * segment end is the nearest frame. + */ +import * as Cesium from 'cesium'; + +/** How far apart the times a video segment is seen at are, ms. */ +export const VIDEO_STEP_MS = 250; +/** The most times one video segment is seen at. */ +export const VIDEO_STEPS_MAX = 1200; +/** Tile videos kept loaded a layer, the least recently seen let go first. */ +export const VIDEO_CACHE = 48; + +/** + * The frames of a timed source, in time order: `{time, url}`, and for + * video `fraction`, how far into the segment's videos the time is. + * @param {import('../contract/globe').TimedSource} source + * @returns {Array<{time: number, url: string, fraction?: number}>} + */ +export function framesOf(source) { + if (source?.kind === 'video') { + const frames = []; + for (const segment of source.segments ?? []) { + const { url, start, end } = segment; + if (typeof url !== 'string' || !(end >= start)) continue; + const span = end - start; + const steps = Math.min( + VIDEO_STEPS_MAX, + Math.max(1, Math.ceil(span / VIDEO_STEP_MS)), + ); + for (let k = 0; k <= steps; k++) + frames.push({ + time: start + (span * k) / steps, + url, + fraction: span > 0 ? k / steps : 0, + }); + } + return frames.sort((a, b) => a.time - b.time); + } + if (source?.kind === 'frames') + return (source.frames ?? []) + .filter((f) => Number.isFinite(f?.time) && typeof f?.url === 'string') + .map((f) => ({ time: f.time, url: f.url })) + .sort((a, b) => a.time - b.time); + throw new TypeError(`Unknown timed source: ${source?.kind}`); +} + +/** The index of the frame nearest `ms` (the earlier one on a tie). */ +function nearestIndex(frames, ms) { + let lo = 0; + let hi = frames.length - 1; + while (lo < hi) { + const mid = (lo + hi) >> 1; + if (frames[mid].time < ms) lo = mid + 1; + else hi = mid; + } + if (lo > 0 && ms - frames[lo - 1].time <= frames[lo].time - ms) return lo - 1; + return lo; +} + +/** + * What `ms` shows: the nearest frame, and in `blend` mode the frames either + * side of it, each with its weight (the later one's share of the way). + * `nearest` is the frame the clock shows, which `frameReady` names. + * @returns {{nearest: number, shown: Array<{index: number, weight: number}>}|null} + */ +export function shownAt(frames, ms, mode = 'nearest') { + if (!frames.length || !Number.isFinite(ms)) return null; + const nearest = nearestIndex(frames, ms); + if (mode !== 'blend') + return { nearest, shown: [{ index: nearest, weight: 1 }] }; + let before = nearest; + if (frames[before].time > ms && before > 0) before -= 1; + const after = Math.min(frames.length - 1, before + 1); + const t0 = frames[before].time; + const t1 = frames[after].time; + if (after === before || ms <= t0 || t1 <= t0) + return { nearest, shown: [{ index: before, weight: 1 }] }; + if (ms >= t1) return { nearest, shown: [{ index: after, weight: 1 }] }; + const f = (ms - t0) / (t1 - t0); + return { + nearest, + shown: [ + { index: before, weight: 1 }, + { index: after, weight: f }, + ], + }; +} + +/** + * The frames to hold loaded: those shown, and the `prefetch` after the + * last of them (playback runs forward). + * @returns {number[]} Indices. + */ +export function heldFrames(frames, shown, prefetch) { + const held = new Set(shown.shown.map((s) => s.index)); + const last = Math.max(...held); + for (let k = 1; k <= prefetch && last + k < frames.length; k++) + held.add(last + k); + return [...held]; +} + +/** The URL of a tile from a template, as `xyz` imagery fills it. */ +function tileUrlOf(template, tilingScheme, x, y, level) { + const rows = tilingScheme.getNumberOfYTilesAtLevel(level); + return template + .replaceAll('{x}', String(x)) + .replaceAll('{y}', String(y)) + .replaceAll('{z}', String(level)) + .replaceAll('{reverseY}', String(rows - 1 - y)); +} + +/** + * Seek a video, resolving once it has: on a `seeked` that leaves it no + * longer seeking (one a seek before it sends late does not count), and its + * frame there presented where the browser says so. + */ +function seekTo(video, time, url) { + return new Promise((resolve, reject) => { + const failed = () => { + video.removeEventListener('seeked', seeked); + reject(new Error(`The video tile did not seek: ${url}`)); + }; + const seeked = () => { + if (video.seeking) return; + video.removeEventListener('seeked', seeked); + video.removeEventListener('error', failed); + if (typeof video.requestVideoFrameCallback !== 'function') resolve(); + else { + // Its frame is drawable once presented; never wait on it for long. + const timer = setTimeout(resolve, 100); + video.requestVideoFrameCallback(() => { + clearTimeout(timer); + resolve(); + }); + } + }; + video.addEventListener('seeked', seeked); + video.addEventListener('error', failed, { once: true }); + video.currentTime = time; + }); +} + +/** + * A layer's tile videos, loaded once each and seeked for each frame drawn + * from them, one seek at a time a video. + */ +export function createTileVideos(limit = VIDEO_CACHE) { + /** url → {video, ready: Promise, queue: Promise} */ + const videos = new Map(); + function load(url) { + let entry = videos.get(url); + if (entry) { + videos.delete(url); + videos.set(url, entry); + return entry; + } + const video = document.createElement('video'); + video.muted = true; + video.playsInline = true; + video.preload = 'auto'; + video.crossOrigin = 'anonymous'; + const loaded = new Promise((resolve, reject) => { + video.addEventListener('loadeddata', () => resolve(video), { + once: true, + }); + video.addEventListener( + 'error', + () => reject(new Error(`The video tile did not load: ${url}`)), + { once: true }, + ); + }); + // A video that does not say how long it is (a recorded WebM, say) is + // seeked past its end once to learn it. + const ready = loaded.then(async () => { + if (!Number.isFinite(video.duration)) + await seekTo(video, Number.MAX_SAFE_INTEGER, url); + return video; + }); + ready.catch(() => {}); + video.src = url; + entry = { video, ready, queue: Promise.resolve() }; + videos.set(url, entry); + while (videos.size > limit) { + const [oldest, old] = videos.entries().next().value; + videos.delete(oldest); + old.video.removeAttribute('src'); + old.video.load(); + } + return entry; + } + /** The video at `fraction` of its length, drawn on a canvas. */ + function frame(url, fraction) { + const entry = load(url); + const run = entry.queue.then(async () => { + const video = await entry.ready; + const duration = Number.isFinite(video.duration) ? video.duration : 0; + const at = Math.max(0, Math.min(duration - 1e-3, fraction * duration)); + if (Math.abs(video.currentTime - at) > 1e-4) await seekTo(video, at, url); + const canvas = document.createElement('canvas'); + canvas.width = video.videoWidth || 1; + canvas.height = video.videoHeight || 1; + canvas.getContext('2d').drawImage(video, 0, 0); + return canvas; + }); + entry.queue = run.catch(() => {}); + return run; + } + return { + frame, + clear() { + for (const { video } of videos.values()) { + video.removeAttribute('src'); + video.load(); + } + videos.clear(); + }, + }; +} + +/** + * A provider drawing a video frame's tiles: each tile the video its URL + * template names for it (an `xyz` spec), or the one video over the layer's + * rectangle (any other), at the frame's place in it. + * @param {import('../contract/globe').ImagerySpec} spec + * @param {{url: string, fraction: number}} frame + * @param {ReturnType} videos + */ +export function videoProvider(spec, frame, videos) { + const tiled = spec.kind === 'xyz'; + const rectangle = spec.rectangle + ? Cesium.Rectangle.fromDegrees( + spec.rectangle.west, + spec.rectangle.south, + spec.rectangle.east, + spec.rectangle.north, + ) + : undefined; + const tilingScheme = tiled + ? spec.tiling === 'geographic' + ? new Cesium.GeographicTilingScheme() + : new Cesium.WebMercatorTilingScheme() + : new Cesium.GeographicTilingScheme({ + rectangle: rectangle ?? Cesium.Rectangle.MAX_VALUE, + numberOfLevelZeroTilesX: 1, + numberOfLevelZeroTilesY: 1, + }); + const size = spec.tileSize ?? 256; + return { + tilingScheme, + tileWidth: size, + tileHeight: size, + minimumLevel: tiled ? (spec.minLevel ?? 0) : 0, + maximumLevel: tiled ? spec.maxLevel : 0, + rectangle: rectangle ?? tilingScheme.rectangle, + credit: undefined, + errorEvent: new Cesium.Event(), + hasAlphaChannel: true, + tileDiscardPolicy: undefined, + proxy: undefined, + getTileCredits: () => undefined, + pickFeatures: () => undefined, + requestImage(x, y, level) { + const url = tiled + ? tileUrlOf(frame.url, tilingScheme, x, y, level) + : frame.url; + return videos.frame(url, frame.fraction ?? 0); + }, + }; +} + +const { ImageryState } = Cesium; +/** States a tile's imagery settles in: drawn, or nothing better to come. */ +const SETTLED = new Set([ + ImageryState.READY, + ImageryState.FAILED, + ImageryState.INVALID, +]); + +/** + * Whether every globe tile drawn last frame draws `layer` from its own + * imagery (not an ancestor's while its own loads). Tiles outside the + * layer's rectangle have none to draw. + */ +export function layerDrawn(scene, layer) { + if (!layer || layer.isDestroyed?.() || !layer.ready) return false; + const tiles = scene.globe?._surface?._tilesToRender; + if (!tiles?.length) return false; + const bounds = layer.imageryProvider?.rectangle ?? layer.rectangle; + for (const tile of tiles) { + let found = false; + for (const tileImagery of tile.data?.imagery ?? []) { + const loading = tileImagery.loadingImagery; + const ready = tileImagery.readyImagery; + if ((loading ?? ready)?.imageryLayer !== layer) continue; + found = true; + if (loading && !SETTLED.has(loading.state)) return false; + if (loading?.state === ImageryState.READY && ready !== loading) + return false; + } + if ( + !found && + (!bounds || Cesium.Rectangle.intersection(tile.rectangle, bounds)) + ) + return false; + } + return true; +} diff --git a/src/render/cesium/imageryFrames.test.mjs b/src/render/cesium/imageryFrames.test.mjs new file mode 100644 index 000000000..2bc61253e --- /dev/null +++ b/src/render/cesium/imageryFrames.test.mjs @@ -0,0 +1,158 @@ +import assert from 'node:assert/strict'; +import * as Cesium from 'cesium'; +import test from 'node:test'; +import { + framesOf, + heldFrames, + layerDrawn, + shownAt, + VIDEO_STEP_MS, + videoProvider, +} from './imageryFrames.js'; + +test('frames come in time order; video segments are seen at steps from start to end', () => { + assert.deepEqual( + framesOf({ + kind: 'frames', + frames: [ + { time: 20, url: 'b' }, + { time: 10, url: 'a' }, + { time: NaN, url: 'x' }, + ], + }), + [ + { time: 10, url: 'a' }, + { time: 20, url: 'b' }, + ], + ); + const video = framesOf({ + kind: 'video', + segments: [ + { url: 'late', start: 5000, end: 5000 + 2 * VIDEO_STEP_MS }, + { url: 'early', start: 0, end: VIDEO_STEP_MS }, + ], + }); + assert.deepEqual( + video.map((f) => [f.time, f.url, f.fraction]), + [ + [0, 'early', 0], + [VIDEO_STEP_MS, 'early', 1], + [5000, 'late', 0], + [5000 + VIDEO_STEP_MS, 'late', 0.5], + [5000 + 2 * VIDEO_STEP_MS, 'late', 1], + ], + ); + assert.throws(() => framesOf({ kind: 'nope' }), /Unknown timed source/); +}); + +test('a time shows its nearest frame, or in blend mode the two either side', () => { + const frames = [0, 100, 200].map((time) => ({ time, url: String(time) })); + assert.deepEqual(shownAt(frames, 40), { + nearest: 0, + shown: [{ index: 0, weight: 1 }], + }); + assert.equal(shownAt(frames, 160).nearest, 2); + assert.equal(shownAt(frames, -50).nearest, 0, 'before the first'); + assert.equal(shownAt(frames, 900).nearest, 2, 'after the last'); + assert.deepEqual(shownAt(frames, 140, 'blend'), { + nearest: 1, + shown: [ + { index: 1, weight: 1 }, + { index: 2, weight: 0.4 }, + ], + }); + assert.deepEqual(shownAt(frames, 60, 'blend').shown, [ + { index: 0, weight: 1 }, + { index: 1, weight: 0.6 }, + ]); + assert.deepEqual(shownAt(frames, 100, 'blend').shown, [ + { index: 1, weight: 1 }, + ]); + assert.deepEqual(shownAt(frames, 900, 'blend').shown, [ + { index: 2, weight: 1 }, + ]); + assert.equal(shownAt([], 0), null); + assert.equal(shownAt(frames, NaN), null, 'no time yet'); +}); + +test('the frames held are those shown and the next ones to prefetch', () => { + const frames = [0, 1, 2, 3, 4].map((time) => ({ time, url: '' })); + assert.deepEqual(heldFrames(frames, shownAt(frames, 1), 2), [1, 2, 3]); + assert.deepEqual( + heldFrames(frames, shownAt(frames, 1.5, 'blend'), 1), + [1, 2, 3], + ); + assert.deepEqual(heldFrames(frames, shownAt(frames, 4), 3), [4]); +}); + +test('a video provider asks for each tile’s video at the frame’s place in it', async () => { + const asked = []; + const videos = { + frame: (url, fraction) => (asked.push([url, fraction]), 'canvas'), + }; + const tiled = videoProvider( + { kind: 'xyz', url: '', tiling: 'geographic' }, + { url: 'v/{z}/{x}/{y}.webm', fraction: 0.25 }, + videos, + ); + assert.equal(await tiled.requestImage(3, 1, 2), 'canvas'); + const single = videoProvider( + { + kind: 'single-image', + url: '', + rectangle: { west: -10, south: -5, east: 10, north: 5 }, + }, + { url: 'one.webm', fraction: 0.5 }, + videos, + ); + await single.requestImage(0, 0, 0); + assert.deepEqual(asked, [ + ['v/2/3/1.webm', 0.25], + ['one.webm', 0.5], + ]); + assert.equal(single.maximumLevel, 0, 'one tile'); + assert.ok(Math.abs(Cesium.Math.toDegrees(single.rectangle.east) - 10) < 1e-9); +}); + +test('a layer is drawn once every tile drawn shows its own imagery', () => { + const layer = { ready: true, isDestroyed: () => false, rectangle: undefined }; + const other = { ready: true }; + const { READY, TRANSITIONING, FAILED } = Cesium.ImageryState; + const world = Cesium.Rectangle.MAX_VALUE; + const tile = (...imagery) => ({ rectangle: world, data: { imagery } }); + const own = (state, ready = true) => { + const imagery = { imageryLayer: layer, state }; + return { + loadingImagery: imagery, + readyImagery: ready ? imagery : { imageryLayer: layer, state: READY }, + }; + }; + const scene = (tiles) => ({ globe: { _surface: { _tilesToRender: tiles } } }); + const theirs = { loadingImagery: { imageryLayer: other, state: READY } }; + assert.equal(layerDrawn(scene([tile(own(READY), theirs)]), layer), true); + assert.equal( + layerDrawn(scene([tile(own(READY)), tile(own(TRANSITIONING))]), layer), + false, + 'one still loading', + ); + assert.equal( + layerDrawn(scene([tile(own(READY, false))]), layer), + false, + 'drawn from an ancestor still', + ); + assert.equal( + layerDrawn(scene([tile(own(FAILED, false))]), layer), + true, + 'failed: nothing better to come', + ); + assert.equal( + layerDrawn(scene([tile(theirs)]), layer), + false, + 'its imagery not made yet', + ); + assert.equal(layerDrawn(scene([]), layer), false, 'nothing drawn'); + assert.equal( + layerDrawn(scene([tile(own(READY))]), { ...layer, ready: false }), + false, + ); +}); diff --git a/src/render/cesium/ports.js b/src/render/cesium/ports.js index 8e5e4ef0a..2d3a35d0e 100644 --- a/src/render/cesium/ports.js +++ b/src/render/cesium/ports.js @@ -46,6 +46,8 @@ const GLOBE_CAPABILITIES = [ 'imageryOnTiles', 'splitSwipe', 'imageHooks', + // Timed imagery: a layer a frame, prefetched unseen (`imagery.js`). + 'imageryFrames', 'surfaceIdentity', 'surfaceShading', 'globeVisibility', diff --git a/src/render/cesium/ports.test.mjs b/src/render/cesium/ports.test.mjs index f559fd1f9..4737b1121 100644 --- a/src/render/cesium/ports.test.mjs +++ b/src/render/cesium/ports.test.mjs @@ -27,6 +27,7 @@ test('capabilities follow what the scene supports', () => { 'imageryOnTiles', 'splitSwipe', 'imageHooks', + 'imageryFrames', 'surfaceIdentity', 'surfaceShading', ]; diff --git a/src/render/conformance/checks/ports.js b/src/render/conformance/checks/ports.js index 19c02da95..14a5cce84 100644 --- a/src/render/conformance/checks/ports.js +++ b/src/render/conformance/checks/ports.js @@ -3964,6 +3964,95 @@ export function portChecks() { }, { capability: 'imageryFrames' }, ), + check( + 'globe.imagery.video', + 'video tiles show the time within their segment, nearest segment end outside it', + async (h, r) => { + if (h.pixels === false) + return { skip: 'this harness reads no drawn pixels' }; + if (typeof MediaRecorder !== 'function') + return { skip: 'no MediaRecorder to make a video with' }; + await overAustin(h, r, 200000); + // Two seconds of video: red, then blue. + const canvas = document.createElement('canvas'); + canvas.width = 32; + canvas.height = 32; + const g = canvas.getContext('2d'); + const recorder = new MediaRecorder(canvas.captureStream(30), { + mimeType: 'video/webm', + }); + const chunks = []; + recorder.ondataavailable = (e) => chunks.push(e.data); + let colour = '#ff0000'; + const paint = setInterval(() => { + g.fillStyle = colour; + g.fillRect(0, 0, 32, 32); + }, 20); + const wait = (ms) => new Promise((done) => setTimeout(done, ms)); + recorder.start(); + await wait(1000); + colour = '#0000ff'; + await wait(1000); + await new Promise((done) => { + recorder.onstop = done; + recorder.stop(); + }); + clearInterval(paint); + const url = URL.createObjectURL( + new Blob(chunks, { type: 'video/webm' }), + ); + const t0 = r.frame.time(); + const layer = r.globe.imagery.add({ + kind: 'single-image', + url: PIXEL_PNG, + rectangle: { + west: AUSTIN.lon - 2, + south: AUSTIN.lat - 2, + east: AUSTIN.lon + 2, + north: AUSTIN.lat + 2, + }, + }); + const ready = []; + layer.on('frameReady', (time) => ready.push(time)); + const shows = async (ms, like, what) => { + const before = ready.length; + layer.setFrameTime(ms); + for (let i = 0; i < 500 && ready.length === before; i++) { + await drawn(h, r, 1); + await wait(10); + } + ok(ready.length > before, `${what}: ready`); + r.frame.renderNow(); + const s = r.query.project(h.toEcef({ ...AUSTIN, height: 0 })); + const image = r.readPixels({ + crop: { + x: Math.round(s.x), + y: Math.round(s.y), + width: 1, + height: 1, + }, + }); + const [red, , blue] = image.data; + ok(like(red, blue), `${what}: (${red}, ${blue})`); + }; + try { + layer.setFrames( + { kind: 'video', segments: [{ url, start: t0, end: t0 + 2000 }] }, + { clock: 'own' }, + ); + const red = (rr, b) => rr > 180 && b < 70; + const blue = (rr, b) => b > 180 && rr < 70; + await shows(t0 + 300, red, 'early in the segment'); + await shows(t0 + 1700, blue, 'late in the segment'); + await shows(t0 - 60000, red, 'before it: its start'); + await shows(t0 + 60000, blue, 'after it: its end'); + } finally { + layer.remove(); + URL.revokeObjectURL(url); + } + }, + { capability: 'imageryFrames' }, + ), check( 'globe.withoutImageHooks', 'without imageHooks, a spec with main-thread tile hooks throws', diff --git a/src/render/stub/stub.test.mjs b/src/render/stub/stub.test.mjs index 164a820df..f23a489da 100644 --- a/src/render/stub/stub.test.mjs +++ b/src/render/stub/stub.test.mjs @@ -20,13 +20,14 @@ test('the stub passes the conformance suite', async () => { .filter((r) => r.status === 'skip') .map((r) => r.id); // It draws only models into pixels, so the sweep's, a line outline's and - // the blends' pixels (over a drawn shape) go unread; its camera takes no user gestures: the drag and wheel scripts are for a + // the blends' (over a drawn shape) and video tiles' pixels go unread; its camera takes no user gestures: the drag and wheel scripts are for a // camera that does (a controller over it, say). assert.deepEqual(skipped, [ 'effects.screenModulation.pixels', 'draw.lines.outline', 'draw.sets.blend', 'draw.models.blend', + 'globe.imagery.video', ...CAMERA_SCRIPTS.filter(usesInput).map((s) => `camera.trace.${s.name}`), ]); });