Renderer contract: decoded imagery, tiles the application hands over as finished images

A timed source of kind 'decoded' (capability imageryDecoded) names the slot a
time shows (slotAt), the slots to hold ahead (prefetch) and answers each tile
of a slot with an image, missing or defer. The Cesium renderer draws each held
slot as an imagery layer of its own, as it draws frames; the stub shows the
slot. The check globe.imagery.decoded covers it.
This commit is contained in:
Sameh Khamis committed 2026-11-18 18:37:23 -08:00
1 parent cd144ecad5
commit 4ee0aba82e
12 files changed
+432 -23

No files matched your search

+12
View File
@@ -361,6 +361,18 @@ names the nearest frame's time once every globe tile drawn shows that
frame's own imagery (`src/render/cesium/imageryFrames.js`). The checks
`globe.imagery.frames` and `globe.imagery.video` cover both.
It also draws `decoded` sources (`imageryDecoded`): tiles the application
makes itself and hands over as finished images. Their slots are times: the
source's `slotAt` names the slot a time shows and `prefetch` the slots held
ahead, of which the first `prefetch` (the option, 1 by default) load unseen.
Each slot is a Cesium imagery layer in the layer's tiling whose tiles are the
images the source's `tile` answers, asked as `shown` or `prefetch`; a
deferred tile is asked again on a later frame, a missing one draws its
parent's, and a slot let go aborts the tiles it still waits on and closes an
image that arrives after. A time with no slot draws nothing and announces
nothing; `frameReady` names the slot shown once it is drawn. The check
`globe.imagery.decoded` covers it.
Every Cesium draw set takes `setColumns` (`columns`): icon sets on the GPU,
any other as the rows the columns make over the items it holds, each row the
item of that id with the columns' fields over it, replacing every item as
+1
View File
@@ -117,6 +117,7 @@ export const CESIUM_CHECKS = new Set([
'frame.time',
'globe.base.vector',
'globe.imagery',
'globe.imagery.decoded',
'globe.imagery.frames',
'globe.imagery.video',
'globe.sky',
+55 -17
View File
@@ -18,13 +18,21 @@
* `prefetch` frames loading unseen at alpha 0. A frame is ready
* (`frameReady`) once the globe's tiles drawn are drawn from its own
* imagery (`imageryFrames.js`).
*
* With `imageryDecoded`, a `decoded` source is drawn the same way, a layer
* a slot: the slot its clock shows, and the next `prefetch` of the slots the
* source holds ahead, each tile the image the source answers for it.
*/
import * as Cesium from 'cesium';
import { vectorBaseProvider } from './vectorBase.js';
import { creditMarkup } from './credits.js';
import { listenerList } from './common.js';
import {
MissingTile,
createTileVideos,
decodedHeld,
decodedProvider,
decodedShownAt,
framesOf,
heldFrames,
layerDrawn,
@@ -41,14 +49,6 @@ export const TILE_RETRY_DELAY_MS = 250;
/** Ion's world imagery assets, by contract style. */
const WORLD_ASSETS = { aerial: 2, aerialLabels: 3 };
/** A tile the source says is not there: Cesium draws the parent's. */
class MissingTile extends Error {
constructor() {
super('No tile here');
this.name = 'MissingTile';
}
}
const rectangleOf = (rect) =>
rect &&
Cesium.Rectangle.fromDegrees(rect.west, rect.south, rect.east, rect.north);
@@ -676,11 +676,21 @@ export function createCesiumImagery(
};
if (capabilities.has('imageryFrames')) {
handle.setFrames = (source, options = {}) => {
const decoded = source?.kind === 'decoded';
if (decoded && !capabilities.has('imageryDecoded'))
throw new TypeError('decoded imagery needs imageryDecoded');
if (
decoded &&
(typeof source.slotAt !== 'function' ||
typeof source.tile !== 'function')
)
throw new TypeError('a decoded source needs slotAt and tile');
const frames = framesOf(source);
if (record.removed) return;
dropFrames(record);
record.timed = {
video: source.kind === 'video',
decoded: decoded ? source : null,
frames,
mode: options.mode === 'blend' ? 'blend' : 'nearest',
prefetch: Math.max(0, Math.floor(options.prefetch ?? 1)),
@@ -723,12 +733,24 @@ export function createCesiumImagery(
function frameLayer(record, frame) {
const timed = record.timed;
const spec = { ...record.spec, url: frame.url };
const entry = { layer: null, host: null, offError: null };
const entry = {
layer: null,
host: null,
offError: null,
signals: new Set(),
};
let made;
try {
made = timed.video
? videoProvider(record.spec, frame, timed.videos)
: providerFor(spec, { worldDataToken: worldDataToken() });
made = timed.decoded
? decodedProvider(record.spec, frame.slot, timed.decoded, {
laterFrame,
signals: entry.signals,
urgency: () =>
timed.at?.nearest === frame.slot ? 'shown' : 'prefetch',
})
: timed.video
? videoProvider(record.spec, frame, timed.videos)
: providerFor(spec, { worldDataToken: worldDataToken() });
} catch (error) {
report(error);
made = Promise.reject(error);
@@ -746,7 +768,9 @@ export function createCesiumImagery(
} else {
entry.offError = record.watchErrors(made);
entry.layer = new Cesium.ImageryLayer(
timed.video ? made : hookProvider(made, spec, record.ctx),
timed.video || timed.decoded
? made
: hookProvider(made, spec, record.ctx),
layerOptions(spec),
);
}
@@ -756,6 +780,9 @@ export function createCesiumImagery(
function dropFrameLayer(entry) {
entry.removed = true;
// A decoded slot's tiles still asked for are not wanted any more.
for (const controller of entry.signals) controller.abort();
entry.signals.clear();
entry.offError?.();
if (usable(entry.host) && entry.host.contains(entry.layer))
entry.host.remove(entry.layer, true);
@@ -785,9 +812,15 @@ export function createCesiumImagery(
const timed = record.timed;
if (!timed || record.removed) continue;
const ms = timed.clock === 'own' ? (timed.own ?? NaN) : sceneMs;
const at = shownAt(timed.frames, ms, timed.mode);
const at = timed.decoded
? decodedShownAt(timed.decoded, ms)
: shownAt(timed.frames, ms, timed.mode);
timed.at = at;
const held = at ? heldFrames(timed.frames, at, timed.prefetch) : [];
const held = !at
? []
: timed.decoded
? decodedHeld(timed.decoded, at, timed.prefetch, ms)
: heldFrames(timed.frames, at, timed.prefetch);
for (const [index, entry] of timed.layers)
if (!held.includes(index) || entry.layer.isDestroyed()) {
dropFrameLayer(entry);
@@ -802,7 +835,10 @@ export function createCesiumImagery(
for (const index of held.sort((a, b) => a - b)) {
let entry = timed.layers.get(index);
if (!entry) {
entry = frameLayer(record, timed.frames[index]);
entry = frameLayer(
record,
timed.decoded ? { slot: index, url: '' } : timed.frames[index],
);
timed.layers.set(index, entry);
}
const layer = entry.layer;
@@ -846,7 +882,9 @@ export function createCesiumImagery(
const timed = record.timed;
// A layer removed by another's listener says nothing.
if (record.removed || !timed?.at || !record.show) continue;
const time = timed.frames[timed.at.nearest].time;
const time = timed.decoded
? timed.at.nearest
: timed.frames[timed.at.nearest].time;
if (time === timed.announced) continue;
if (!timed.at.shown.every((s) => frameDrawn(timed.layers.get(s.index))))
continue;
+84
View File
@@ -11,10 +11,22 @@
* steps a segment): a time is shown by the step nearest it, its offset into
* the segment's videos in proportion; outside every segment, the nearest
* segment end is the nearest frame.
*
* A `decoded` source (`imageryDecoded`) has no list: the slot a time shows
* is the source's `slotAt`, the slots held ahead its `prefetch`, and each
* slot's tiles are the images its `tile` answers.
*/
import * as Cesium from 'cesium';
import { globeTilesDrawn } from './internals.js';
/** A tile the source says is not there: Cesium draws the parent's. */
export class MissingTile extends Error {
constructor() {
super('No tile here');
this.name = 'MissingTile';
}
}
/** How far apart the times a video segment is seen at are, ms. */
export const VIDEO_STEP_MS = 250;
/** The most times one video segment is seen at. */
@@ -61,6 +73,7 @@ export function framesOf(source) {
}
return frames.sort((a, b) => a.time - b.time);
}
if (source?.kind === 'decoded') return [];
if (source?.kind === 'frames')
return (source.frames ?? [])
.filter((f) => Number.isFinite(f?.time) && typeof f?.url === 'string')
@@ -408,3 +421,74 @@ export function layerDrawn(scene, layer) {
}
return true;
}
/**
* What `ms` shows of a `decoded` source: its slot, as `shownAt` answers for
* a list (the slot standing for the index), or null with none.
* @returns {{nearest: number, shown: Array<{index: number, weight: number}>}|null}
*/
export function decodedShownAt(source, ms) {
if (!Number.isFinite(ms)) return null;
const slot = source.slotAt(ms);
if (!Number.isFinite(slot)) return null;
return { nearest: slot, shown: [{ index: slot, weight: 1 }] };
}
/**
* The slots of a `decoded` source to hold for `ms`: the one shown, and the
* first `prefetch` of those the source holds ahead (finite, not shown, each
* once).
* @returns {number[]}
*/
export function decodedHeld(source, at, prefetch, ms) {
const held = [at.nearest];
if (prefetch <= 0 || typeof source.prefetch !== 'function') return held;
for (const slot of source.prefetch(ms) ?? []) {
if (held.length > prefetch) break;
if (Number.isFinite(slot) && !held.includes(slot)) held.push(slot);
}
return held;
}
/**
* A provider drawing one slot of a `decoded` source: each tile the image
* the source answers for it, in the spec's tiling (an `xyz` spec), or one
* image over the layer's rectangle (any other). A deferred tile is asked
* again on a later frame (`laterFrame`); a missing one draws the parent's.
* @param {import('../contract/globe').ImagerySpec} spec
* @param {number} slot
* @param {Extract<import('../contract/globe').TimedSource, {kind: 'decoded'}>} source
* @param {{laterFrame: () => Promise<void>, signals: Set<AbortController>,
* urgency: () => 'shown'|'prefetch'}} ctx
*/
export function decodedProvider(spec, slot, source, ctx) {
// The same tiling and bounds a video frame's provider draws in.
const provider = videoProvider(spec, { url: '', fraction: 0 }, null);
provider.requestImage = async (x, y, level) => {
const controller = new AbortController();
ctx.signals.add(controller);
try {
for (;;) {
const answer = await source.tile({
tile: { x, y, level },
slot,
signal: controller.signal,
urgency: ctx.urgency(),
});
// An image no longer wanted is let go at once.
if (controller.signal.aborted) answer?.image?.close?.();
controller.signal.throwIfAborted();
if (answer?.defer) {
await ctx.laterFrame();
controller.signal.throwIfAborted();
continue;
}
if (!answer?.image) throw new MissingTile();
return answer.image;
}
} finally {
ctx.signals.delete(controller);
}
};
return provider;
}
+99
View File
@@ -2,7 +2,11 @@ import assert from 'node:assert/strict';
import * as Cesium from 'cesium';
import test, { mock } from 'node:test';
import {
MissingTile,
createTileVideos,
decodedHeld,
decodedProvider,
decodedShownAt,
framesOf,
heldFrames,
layerDrawn,
@@ -336,3 +340,98 @@ test('a video’s length is learnt with a longer wait, and a failed one is not l
restore();
}
});
/** A decoded source over slots 0, 100, 200, with what its tiles answer. */
function decodedSource(answers = () => ({ missing: true })) {
const asked = [];
return {
asked,
kind: 'decoded',
slotAt: (ms) => (ms < 0 ? null : Math.min(200, Math.round(ms / 100) * 100)),
prefetch: (ms) => [Math.round(ms / 100) * 100, 300, NaN, 300, 400, 500],
tile: async (request) => {
asked.push(request);
return answers(request);
},
};
}
test('a decoded source shows the slot its clock names and holds the next it prefetches', () => {
const source = decodedSource();
assert.equal(framesOf(source).length, 0, 'no list to step through');
const at = decodedShownAt(source, 140);
assert.deepEqual(at, { nearest: 100, shown: [{ index: 100, weight: 1 }] });
assert.equal(decodedShownAt(source, -5), null, 'no slot: nothing shown');
assert.equal(decodedShownAt(source, NaN), null);
assert.deepEqual(decodedHeld(source, at, 2, 140), [100, 300, 400]);
assert.deepEqual(decodedHeld(source, at, 0, 140), [100]);
assert.deepEqual(
decodedHeld({ slotAt: () => 0 }, at, 3, 140),
[100],
'a source with no prefetch holds the shown slot only',
);
});
test("a decoded slot's tiles are the source's images, asked again when deferred", async () => {
const image = { width: 256, height: 256 };
let defers = 1;
const source = decodedSource(({ tile }) => {
if (tile.level === 9) return { missing: true };
if (defers-- > 0) return { defer: true };
return { image };
});
let frames = 0;
let shown = true;
const signals = new Set();
const provider = decodedProvider(
{ kind: 'xyz', url: '', maxLevel: 10, tiling: 'webmercator' },
100,
source,
{
laterFrame: async () => {
frames += 1;
},
signals,
urgency: () => (shown ? 'shown' : 'prefetch'),
},
);
assert.ok(provider.tilingScheme instanceof Cesium.WebMercatorTilingScheme);
assert.equal(provider.maximumLevel, 10);
assert.equal(provider.hasAlphaChannel, true);
assert.equal(await provider.requestImage(1, 2, 3), image);
assert.equal(frames, 1, 'the deferred tile waited a frame');
assert.deepEqual(
source.asked.map((r) => [r.tile, r.slot, r.urgency]),
[
[{ x: 1, y: 2, level: 3 }, 100, 'shown'],
[{ x: 1, y: 2, level: 3 }, 100, 'shown'],
],
);
shown = false;
await assert.rejects(provider.requestImage(0, 0, 9), MissingTile);
assert.equal(source.asked.at(-1).urgency, 'prefetch');
assert.equal(signals.size, 0, 'no tile left waiting');
});
test('a decoded tile answered after its slot was let go is closed, not drawn', async () => {
let release;
const closed = [];
const image = { close: () => closed.push('image') };
const source = decodedSource(
() => new Promise((resolve) => (release = () => resolve({ image }))),
);
const signals = new Set();
const provider = decodedProvider({ kind: 'single-image' }, 0, source, {
laterFrame: async () => {},
signals,
urgency: () => 'shown',
});
assert.equal(provider.maximumLevel, 0, 'one image over the rectangle');
const pending = provider.requestImage(0, 0, 0);
await Promise.resolve();
assert.ok(source.asked[0].signal instanceof AbortSignal);
for (const controller of signals) controller.abort();
release();
await assert.rejects(pending);
assert.deepEqual(closed, ['image']);
});
+2
View File
@@ -49,6 +49,8 @@ const GLOBE_CAPABILITIES = [
'imageHooks',
// Timed imagery: a layer a frame, prefetched unseen (`imagery.js`).
'imageryFrames',
// Timed imagery handed over as finished images (`imageryFrames.js`).
'imageryDecoded',
'surfaceIdentity',
'surfaceShading',
// A MapLibre style's vector tiles, painted as imagery (`vectorBase.js`).
+1
View File
@@ -28,6 +28,7 @@ test('capabilities follow what the scene supports', () => {
'splitSwipe',
'imageHooks',
'imageryFrames',
'imageryDecoded',
'surfaceIdentity',
'surfaceShading',
'vectorBase',
+119
View File
@@ -5425,6 +5425,125 @@ export function portChecks() {
},
{ capability: 'imageryFrames', bareSky: true },
),
check(
'globe.imagery.decoded',
'decoded imagery draws the images its source hands over for the slot shown',
async (h, r) => {
await overAustin(h, r, 200000);
const t0 = r.frame.time();
const slots = [t0, t0 + 60000, t0 + 120000];
const colours = ['#ff0000', '#0000ff', '#00ff00'];
const pixels = h.pixels !== false;
/** A solid tile of a slot's colour, or a canvas where none is made. */
const imageOf = async (slot) => {
const canvas = document.createElement('canvas');
canvas.width = 16;
canvas.height = 16;
const g = canvas.getContext('2d');
g.fillStyle = colours[slots.indexOf(slot)];
g.fillRect(0, 0, 16, 16);
return createImageBitmap(canvas);
};
const asked = [];
const source = {
kind: 'decoded',
slotAt: (ms) => {
if (ms < t0 - 30000) return null;
let best = slots[0];
for (const slot of slots)
if (Math.abs(slot - ms) < Math.abs(best - ms)) best = slot;
return best;
},
prefetch: (ms) => slots.filter((slot) => slot > ms),
async tile({ tile, slot, signal, urgency }) {
asked.push({ slot, urgency });
signal.throwIfAborted();
// The first ask of each slot is put off: the renderer asks again.
if (asked.filter((a) => a.slot === slot).length === 1)
return { defer: true };
if (tile.level > 0) return { missing: true };
return { image: await imageOf(slot) };
},
};
const layer = r.globe.imagery.add({
kind: 'single-image',
url: PIXEL_PNG,
rectangle: {
west: AUSTIN.lon - 2,
south: AUSTIN.lat - 2,
east: AUSTIN.lon + 2,
north: AUSTIN.lat + 2,
},
});
const ready = [];
layer.on('frameReady', (time) => ready.push(time));
const wait = (ms) => new Promise((done) => setTimeout(done, ms));
const shows = async (ms, slot, colour, what) => {
const before = ready.length;
layer.setFrameTime(ms);
for (let i = 0; i < 500 && ready.length === before; i++) {
await drawn(h, r, 1);
await wait(10);
}
equal(ready.at(-1), slot, `${what}: its slot is ready`);
if (!pixels) return;
r.frame.renderNow();
const s = r.query.project(h.toEcef({ ...AUSTIN, height: 0 }));
const image = r.readPixels({
crop: {
x: Math.round(s.x),
y: Math.round(s.y),
width: 1,
height: 1,
},
});
const [red, green, blue] = image.data;
ok(colour(red, green, blue), `${what}: (${red}, ${green}, ${blue})`);
};
try {
throws(
() => layer.setFrames({ kind: 'decoded', slotAt: () => null }),
'a decoded source with no tile',
);
layer.setFrames(source, { clock: 'own', prefetch: 1 });
await shows(
t0 + 10000,
t0,
(rr, g, b) => rr > 180 && g < 70 && b < 70,
'the first slot',
);
if (pixels) {
ok(
asked.some((a) => a.slot === t0 && a.urgency === 'shown'),
'the slot shown is asked for as shown',
);
ok(
asked.some(
(a) => a.slot === t0 + 60000 && a.urgency === 'prefetch',
),
'the next slot is asked for ahead, as a prefetch',
);
ok(
!asked.some((a) => a.slot === t0 + 120000),
'no more slots are held ahead than prefetch allows',
);
}
await shows(
t0 + 65000,
t0 + 60000,
(rr, g, b) => b > 180 && rr < 70 && g < 70,
'the second slot',
);
ready.length = 0;
layer.setFrameTime(t0 - 60000);
for (let i = 0; i < 20; i++) await drawn(h, r, 1);
deepEqual(ready, [], 'a time with no slot announces nothing');
} finally {
layer.remove();
}
},
{ capability: ['imageryFrames', 'imageryDecoded'], bareSky: true },
),
check(
'globe.withoutImageHooks',
'without imageHooks, a spec with main-thread tile hooks throws',
+2
View File
@@ -165,6 +165,8 @@ export type Capability =
| 'splitSwipe'
/** Timed imagery frames with hidden prefetch. */
| 'imageryFrames'
/** Timed imagery the application hands over as finished tile images (`decoded`). */
| 'imageryDecoded'
/** Wind-relief and similar shading of the globe surface. */
| 'surfaceShading'
/** A version of what surface heights answer against (`globe.surfaceVersion`). */
+36 -2
View File
@@ -130,17 +130,51 @@ export interface TimedUrl {
url: string;
}
/**
* A tile of a `decoded` source at one slot: its finished image, `missing`
* (no tile there; draw the parent's), or `defer` (ask again on a later
* frame). The renderer owns the image once answered: it draws it as it is
* (its alpha too) and may close it once drawn.
*/
export type DecodedTile =
{ image: ImageBitmap } | { missing: true } | { defer: true };
/** What a `decoded` source is asked for: one tile at one slot. */
export interface DecodedTileRequest {
tile: TileRequest;
/** The slot, as `slotAt` or `prefetch` named it. */
slot: number;
/** Aborted once the renderer no longer wants the tile. */
signal: AbortSignal;
/** `shown`: the slot the clock shows; `prefetch`: one held ahead. */
urgency: 'shown' | 'prefetch';
}
/**
* A timed source: frames as URLs (one template per time), or video tiles
* in segments, each a template whose tiles are videos spanning `start` to
* `end` (epoch ms), a time within shown by its offset into the segment's
* videos. Outside every segment, the nearest segment end is shown.
*
* `decoded` (`imageryDecoded`): tiles the application makes itself and hands
* over as finished images. Its slots are times (epoch ms): `slotAt` names
* the slot a time shows (`null`: none, and nothing is drawn), `prefetch` the
* slots to hold ahead of it, nearest first. The renderer asks `tile` for the
* tiles each held slot draws, in the layer's tiling, and only draws what it
* is given. It shows the nearest slot only (`mode` is not read); the
* `prefetch` option caps how many of the source's slots it holds ahead.
*/
export type TimedSource =
| { kind: 'frames'; frames: TimedUrl[] }
| {
kind: 'video';
segments: Array<{ url: string; start: number; end: number }>;
}
| {
kind: 'decoded';
slotAt(time: number): number | null;
prefetch?(time: number): number[];
tile(request: DecodedTileRequest): Promise<DecodedTile>;
};
export interface ImageryLayer {
@@ -176,8 +210,8 @@ export interface ImageryLayer {
/** `imageryFrames`: which time to show, on the `own` clock. */
setFrameTime?(ms: number): void;
/**
* `imageryFrames`: the frame the layer's clock now shows (by its time)
* has been drawn. Fires once each time the shown frame changes, never
* `imageryFrames`: the frame the layer's clock now shows (by its time;
* a `decoded` source's by its slot) has been drawn. Fires once each time the shown frame changes, never
* for frames only prefetched.
*/
on(event: 'frameReady', cb: (time: number) => void): Off;
+1
View File
@@ -23,6 +23,7 @@ export const CAPABILITIES = Object.freeze([
'imageryOnTiles',
'splitSwipe',
'imageryFrames',
'imageryDecoded',
'surfaceShading',
'surfaceIdentity',
'imageHooks',
+20 -4
View File
@@ -1574,8 +1574,12 @@ export function createStubRenderer(options = {}) {
if (capabilities.has('imageryFrames')) {
let source = null;
let clockKind = 'scene';
/** The time the source shows for `ms`: the nearest frame, or within the videos. */
/** The time the source shows for `ms`: the nearest frame, within the videos, or a decoded source's slot. */
const shownTime = (ms) => {
if (source?.kind === 'decoded') {
const slot = Number.isFinite(ms) ? source.slotAt(ms) : null;
return Number.isFinite(slot) ? slot : null;
}
if (source?.kind === 'video') {
let best = null;
let bestOff = Infinity;
@@ -1609,10 +1613,22 @@ export function createStubRenderer(options = {}) {
deliver(listeners.frameReady, drawnTime);
});
layer.setFrames = (next, opts = {}) => {
if (next.kind === 'decoded' && !capabilities.has('imageryDecoded'))
throw new TypeError('decoded imagery needs imageryDecoded');
if (
next.kind === 'decoded' &&
(typeof next.slotAt !== 'function' || typeof next.tile !== 'function')
)
throw new TypeError('a decoded source needs slotAt and tile');
source =
next.kind === 'video'
? { kind: 'video', segments: next.segments.map((s) => ({ ...s })) }
: { kind: 'frames', frames: next.frames.slice() };
next.kind === 'decoded'
? next
: next.kind === 'video'
? {
kind: 'video',
segments: next.segments.map((s) => ({ ...s })),
}
: { kind: 'frames', frames: next.frames.slice() };
wanted = null;
drawnTime = null;
clockKind = opts.clock ?? 'scene';