From c330eb801a4a51c0bba3cbe472c92f822107bc4f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 04:03:02 +0000 Subject: [PATCH] feat(director): recorded camera tracks and shot lens (scene document v7) Scene documents gain a second kind of explicit move: `move.track`, timed samples of position, orientation and an optional 35 mm-equivalent focal length. Playback, Load, Replay and seek share one sampler, which stretches the recording to the shot's duration (so editing durationSec retimes a take), takes the short arc for longitude/heading/roll and returns recorded samples exactly. Tracks require the ellipsoid altitude reference and are bounded to 10,000 samples. Shots may also carry a `lens`: focal length and a thin-lens depth of field whose distances are divided by a miniature scale, so a 1:1000 rig renders the city like a tabletop model. A new camera-lens owner applies it to the frustum and a depth-of-field post-process stage; shots without a lens restore the default view. SceneDirector accepts the lens as an injected dependency, captures a changed lens with a shot and gains addRecordedShot() for takes. Versions 1-6 import unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015H1iLusTzWpVKitGic2AW8 --- docs/DIRECTOR-CAMERA.md | 60 +++++++++ docs/SCENE-DOCUMENT.md | 5 + src/director/authoring.js | 2 + src/director/camera.js | 85 ++++++++++++ src/director/camera.test.mjs | 2 +- src/director/cameraDocument.js | 61 +++++++++ src/director/document.js | 8 +- src/director/document.test.mjs | 2 +- src/director/lens.js | 41 ++++++ src/director/packs/packs.test.mjs | 2 +- src/director/track.test.mjs | 118 +++++++++++++++++ src/scenes/cameraLens.js | 207 ++++++++++++++++++++++++++++++ src/scenes/cameraLens.test.mjs | 66 ++++++++++ src/scenes/director.js | 85 ++++++++++++ src/scenes/director.test.mjs | 48 ++++++- src/scenes/playbackAdapter.js | 9 +- src/scenes/project.js | 1 + src/scenes/sharing.js | 1 + 18 files changed, 793 insertions(+), 10 deletions(-) create mode 100644 src/director/lens.js create mode 100644 src/director/track.test.mjs create mode 100644 src/scenes/cameraLens.js create mode 100644 src/scenes/cameraLens.test.mjs diff --git a/docs/DIRECTOR-CAMERA.md b/docs/DIRECTOR-CAMERA.md index 2481ba296..5490b90f5 100644 --- a/docs/DIRECTOR-CAMERA.md +++ b/docs/DIRECTOR-CAMERA.md @@ -108,3 +108,63 @@ Version-1/2/3 projects migrate to version 4 without converting their ordinary shots into explicit moves. IDs, positions, durations, visual settings, packs and source attribution survive. The bloom scale remains independent of the document version; version-3 bloom is never interpreted as the older inverted scale. + +## Recorded tracks + +Version 7 adds a second kind of explicit move: a recorded track, which is what +[VCAM](VCAM.md) takes write. Instead of `from` and `easing`, `move.track` holds +timed samples: + +```json +{ + "id": "take-1", + "title": "Take 1", + "durationSec": 6.4, + "holdSec": 0, + "camera": { "lat": 30.27, "lon": -97.74, "alt": 950, "heading": 20, + "pitch": -25, "roll": 0, "altitudeReference": "ellipsoid" }, + "move": { + "track": { + "altitudeReference": "ellipsoid", + "samples": [ + [0, -97.7431, 30.2672, 900, 10, -30, 0, 24], + [6.4, -97.7400, 30.2700, 950, 20, -25, 0, 50] + ] + } + }, + "lens": { "focalMm": 50, "dof": { "enabled": true, "focusM": 300, "fStop": 2, "scale": 100 } } +} +``` + +Each sample is `[t, lon, lat, alt, heading, pitch, roll, focalMm?]`: seconds from +the start of the take, then the same coordinate, height and orientation +conventions as a pose, then an optional 35 mm-equivalent focal length (4 to +2,000 mm). The first sample is at 0 seconds and times strictly increase. A track +has 2 to 10,000 samples and requires the ellipsoid altitude reference. The shot +`camera` stays the destination: the final sample, which Load, Capture and the +ordinary flight from the previous shot use. + +Playback stretches the recorded time to `durationSec`, so changing the duration +retimes the take without touching its samples. Between samples, latitude, height, +pitch and focal length interpolate linearly; longitude, heading and roll take +the short arc. A time that lands on a sample returns that sample exactly, +including both ends. Playback, Load, Replay and seek all use +`sampleCameraMove`, which dispatches to `sampleCameraTrack` for tracks. + +## Lens + +Version 7 shots may carry an optional `lens`: + +- `focalMm`: 35 mm-equivalent focal length across the frame's longer side + (4 to 2,000 mm), applied to the camera's field of view. Cesium's default + 60-degree view is about 31 mm. +- `dof`: depth of field, with `enabled`, `focusM` (focus distance in metres, + 0.1 to 10⁸), `fStop` (0.5 to 64) and `scale` (0.001 to 10⁷). The blur is a + thin-lens circle of confusion computed after dividing distances by `scale`, + so a scale of 1,000 renders the scene as a 1:1000 miniature. + +A shot's lens is applied after its visual state. A shot without one restores +the application's field of view and turns depth of field off, so older shots +look as they always did. Capture and Update Shot store the live lens only when +it differs from the default. A track's per-sample focal length overrides +`focalMm` while the track plays. diff --git a/docs/SCENE-DOCUMENT.md b/docs/SCENE-DOCUMENT.md index f6dc66636..42d9004da 100644 --- a/docs/SCENE-DOCUMENT.md +++ b/docs/SCENE-DOCUMENT.md @@ -103,3 +103,8 @@ change with this document boundary. Version 5 additionally accepts scene `dataPacks` and shot `dataPackIds`, as defined in [data packs](DIRECTOR-DATA-PACKS.md). Version-4 camera documents continue to import without adding any packs. + +Version 7 is the write format. It additionally accepts recorded camera tracks +(`move.track`) and an optional shot `lens` (focal length and depth of field), as +defined in [camera directions](DIRECTOR-CAMERA.md#recorded-tracks). Versions 1 +to 6 import unchanged; their shots receive no lens and keep the default view. diff --git a/src/director/authoring.js b/src/director/authoring.js index 22a082971..ef2b1e026 100644 --- a/src/director/authoring.js +++ b/src/director/authoring.js @@ -24,6 +24,7 @@ export function editSceneDetails( 'holdSec', 'dataPackIds', 'interactions', + 'lens', ].includes(key) ) throw new Error('Unsupported shot detail'); @@ -38,6 +39,7 @@ export function editSceneDetails( 'holdSec', 'dataPackIds', 'interactions', + 'lens', ]) { delete shot[key]; if (Object.hasOwn(shotDetails, key)) shot[key] = shotDetails[key]; diff --git a/src/director/camera.js b/src/director/camera.js index e009c2cba..995465289 100644 --- a/src/director/camera.js +++ b/src/director/camera.js @@ -17,9 +17,45 @@ export function resolveCameraPose(scene, camera) { }; } +/** Columns of one recorded track sample; focal length is optional. */ +export const CAMERA_TRACK_COLUMNS = Object.freeze([ + 't', + 'lon', + 'lat', + 'alt', + 'heading', + 'pitch', + 'roll', + 'focalMm', +]); + +/** Expand one compact track sample into a pose; `t` stays in recorded seconds. */ +export function cameraTrackSample(sample) { + const pose = { + t: sample[0], + lon: sample[1], + lat: sample[2], + alt: sample[3], + heading: sample[4], + pitch: sample[5], + roll: sample[6], + }; + if (Number.isFinite(sample[7])) pose.focalMm = sample[7]; + return pose; +} + /** Resolve both authored endpoints. An explicit move never depends on the live viewport. */ export function resolveCameraMove(scene, shot) { if (!shot?.move) return null; + if (shot.move.track) { + const samples = shot.move.track.samples; + return { + from: sampleCameraTrack(samples, 0), + to: sampleCameraTrack(samples, 1), + track: samples, + durationSec: shot.durationSec, + }; + } return { from: resolveCameraPose(scene, shot.move.from), to: resolveCameraPose(scene, shot.camera), @@ -33,8 +69,57 @@ export function normalizeCameraLongitude(longitude) { return ((((longitude + 180) % 360) + 360) % 360) - 180; } +/** + * Sample a recorded track at normalized progress. The recording is stretched + * to the shot duration, so editing `durationSec` retimes a take without + * touching its samples. Angles and longitude take the short arc between + * neighbouring samples. + */ +export function sampleCameraTrack(samples, progress) { + const t = Math.max(0, Math.min(1, Number(progress) || 0)); + const last = samples.length - 1; + const exact = (sample) => { + const { t: _recorded, ...pose } = cameraTrackSample(sample); + return pose; + }; + if (t === 0) return exact(samples[0]); + if (t === 1) return exact(samples[last]); + const target = t * samples[last][0]; + let low = 0; + let high = last; + while (high - low > 1) { + const middle = (low + high) >> 1; + if (samples[middle][0] <= target) low = middle; + else high = middle; + } + const a = cameraTrackSample(samples[low]); + const b = cameraTrackSample(samples[high]); + const span = b.t - a.t; + const k = span > 0 ? Math.max(0, Math.min(1, (target - a.t) / span)) : 1; + // A time that lands on a recorded sample returns it unchanged. + if (k === 0) return exact(samples[low]); + if (k === 1) return exact(samples[high]); + const lerp = (from, to) => from + (to - from) * k; + const angle = (from, to) => + from + (((((to - from + 540) % 360) + 360) % 360) - 180) * k; + const pose = { + lat: lerp(a.lat, b.lat), + lon: normalizeCameraLongitude(angle(a.lon, b.lon)), + alt: lerp(a.alt, b.alt), + heading: angle(a.heading, b.heading), + pitch: lerp(a.pitch, b.pitch), + roll: angle(a.roll, b.roll), + }; + if (Number.isFinite(a.focalMm) && Number.isFinite(b.focalMm)) + pose.focalMm = lerp(a.focalMm, b.focalMm); + else if (Number.isFinite(a.focalMm ?? b.focalMm)) + pose.focalMm = a.focalMm ?? b.focalMm; + return pose; +} + /** Sample the same authored curve for playback and seeking, taking the short longitude/angle arc. */ export function sampleCameraMove(move, progress) { + if (move.track) return sampleCameraTrack(move.track, progress); const t = Math.max(0, Math.min(1, Number(progress) || 0)); if (t === 0) return { ...move.from }; if (t === 1) return { ...move.to }; diff --git a/src/director/camera.test.mjs b/src/director/camera.test.mjs index cf7c43bb0..365a98509 100644 --- a/src/director/camera.test.mjs +++ b/src/director/camera.test.mjs @@ -73,7 +73,7 @@ test('v3 unversioned bloom already uses scale 2 and is not migrated again in lat }), ), ); - assert.equal(project.version, 6); + assert.equal(project.version, 7); assert.equal(project.scenes[0].shots[0].visual.bloom.intensity, 50); }); test('move endpoints, easing, shortest arcs and hold agree with scene seeking', () => { diff --git a/src/director/cameraDocument.js b/src/director/cameraDocument.js index 1579d54a8..c4f858cab 100644 --- a/src/director/cameraDocument.js +++ b/src/director/cameraDocument.js @@ -50,6 +50,59 @@ function pose(value, path, version, anchorIds, explicit = false) { } } +// Recorded focal lengths are 35 mm-equivalent; the range spans fisheye to supertele. +export const FOCAL_LENGTH_MM = Object.freeze([4, 2000]); +const TRACK_COLUMNS = [ + [-1, 86400], + POSITION.lon, + POSITION.lat, + POSITION.alt, + ORIENTATION.heading, + ORIENTATION.pitch, + ORIENTATION.roll, + FOCAL_LENGTH_MM, +]; +function track(value, path) { + fields(value, path, ['altitudeReference', 'samples']); + reference(value.altitudeReference, `${path}.altitudeReference`); + array(value.samples, `${path}.samples`); + if (value.samples.length < 2) + fail(`${path}.samples`, 'expected at least two samples'); + let previous = -1; + value.samples.forEach((sample, i) => { + const at = `${path}.samples[${i}]`; + if (!Array.isArray(sample) || sample.length < 7 || sample.length > 8) + fail(at, 'expected [t, lon, lat, alt, heading, pitch, roll, focalMm?]'); + sample.forEach((item, column) => + number(item, `${at}[${column}]`, ...TRACK_COLUMNS[column], false), + ); + if (i === 0 ? sample[0] !== 0 : sample[0] <= previous) + fail( + `${at}[0]`, + i === 0 + ? 'expected the first sample at 0 seconds' + : 'expected increasing sample times', + ); + previous = sample[0]; + }); +} + +/** Validate a shot lens: focal length and an optional depth-of-field focus. */ +export function validateShotLens(lens, path) { + fields(lens, path, ['focalMm', 'dof']); + optional(lens, 'focalMm', path, (v, p) => + number(v, p, ...FOCAL_LENGTH_MM, false), + ); + optional(lens, 'dof', path, (dof, field) => { + fields(dof, field, ['enabled', 'focusM', 'fStop', 'scale']); + if (typeof dof.enabled !== 'boolean') + fail(`${field}.enabled`, 'expected a boolean'); + optional(dof, 'focusM', field, (v, p) => number(v, p, 0.1, 1e8, false)); + optional(dof, 'fStop', field, (v, p) => number(v, p, 0.5, 64, false)); + optional(dof, 'scale', field, (v, p) => number(v, p, 1e-3, 1e7, false)); + }); +} + /** Validate scene-local anchor IDs, camera references and explicit motion without reading terrain or applying state. */ export function validateSceneCameras(scene, path, version) { const anchorIds = new Set(); @@ -73,6 +126,14 @@ export function validateSceneCameras(scene, path, version) { scene.shots.forEach((shot, index) => { const at = `${path}.shots[${index}]`; optional(shot, 'move', at, (move, field) => { + if (version >= 7 && Object.hasOwn(move || {}, 'track')) { + fields(move, field, ['track']); + track(move.track, `${field}.track`); + pose(shot.camera, `${at}.camera`, version, anchorIds, true); + number(shot.durationSec, `${at}.durationSec`, 0.2, 86400, false); + number(shot.holdSec, `${at}.holdSec`, 0, 86400, false); + return; + } fields(move, field, ['from', 'easing']); if (!['linear', 'cubic-in-out'].includes(move.easing)) fail(`${field}.easing`, 'expected linear or cubic-in-out'); diff --git a/src/director/document.js b/src/director/document.js index 48804d83b..1f0f5e2ce 100644 --- a/src/director/document.js +++ b/src/director/document.js @@ -13,10 +13,10 @@ import { jsonTree, } from './documentFields.js'; import { validateSceneDataPacks } from './packs/manifest.js'; -import { validateSceneCameras } from './cameraDocument.js'; +import { validateSceneCameras, validateShotLens } from './cameraDocument.js'; export { SCENE_DOCUMENT_LIMITS, SceneDocumentError } from './documentFields.js'; import { validateSceneInteractions } from './interactions/document.js'; -export const SCENE_DOCUMENT_VERSION = 6; +export const SCENE_DOCUMENT_VERSION = 7; function visual(value, path, legacy) { fields(value, path, [ @@ -67,7 +67,7 @@ export function validateSceneDocument(project) { 'scenes', ]); const version = Object.hasOwn(project, 'version') ? project.version : 1; - if (![1, 2, 3, 4, 5, 6].includes(version)) + if (![1, 2, 3, 4, 5, 6, 7].includes(version)) fail('$.version', 'unsupported scene project version'); const legacy = version < 3; for (const key of ['createdAt', 'updatedAt']) @@ -128,6 +128,7 @@ export function validateSceneDocument(project) { ...(version >= 4 ? ['move'] : []), ...(version >= 5 ? ['dataPackIds'] : []), ...(version >= 6 ? ['interactions'] : []), + ...(version >= 7 ? ['lens'] : []), ]); uniqueId(shot, at, shotIds); optional(shot, 'title', at, (v, p) => string(v, p, 4096)); @@ -140,6 +141,7 @@ export function validateSceneDocument(project) { ); optional(shot, 'holdSec', at, (v, p) => number(v, p, 0, 86400, legacy)); optional(shot, 'visual', at, (v, p) => visual(v, p, legacy)); + optional(shot, 'lens', at, validateShotLens); optional(shot, 'layers', at, (layers, field) => { object(layers, field); for (const [id, entry] of Object.entries(layers)) { diff --git a/src/director/document.test.mjs b/src/director/document.test.mjs index c0b8b0935..bc52b4804 100644 --- a/src/director/document.test.mjs +++ b/src/director/document.test.mjs @@ -59,7 +59,7 @@ test('v1/v2 bloom migrates once; IDs, edits, pack bindings and zero holds surviv const migrated = normalizeProject( parseSceneDocument(JSON.stringify(project)), ); - assert.equal(migrated.version, 6); + assert.equal(migrated.version, 7); assert.equal(migrated.scenes[0].shots[0].visual.bloom.intensity, 150); assert.equal(migrated.scenes[0].shots[0].id, 'shot'); assert.deepEqual( diff --git a/src/director/lens.js b/src/director/lens.js new file mode 100644 index 000000000..8a9067768 --- /dev/null +++ b/src/director/lens.js @@ -0,0 +1,41 @@ +/** + * Renderer-independent lens math. Focal lengths are 35 mm full-frame + * equivalents measured across the frame's longer side (36 mm), which is the + * dimension Cesium's `PerspectiveFrustum.fov` describes. + */ +export const SENSOR_LONG_SIDE_MM = 36; + +/** Cesium's default 60-degree field of view expressed as a focal length. */ +export const DEFAULT_FOCAL_MM = SENSOR_LONG_SIDE_MM / 2 / Math.tan(Math.PI / 6); + +/** Field of view in radians across the longer side for a focal length. */ +export function focalLengthToFov(focalMm) { + return 2 * Math.atan(SENSOR_LONG_SIDE_MM / (2 * focalMm)); +} + +/** Focal length in millimetres for a field of view across the longer side. */ +export function fovToFocalLength(fovRad) { + return SENSOR_LONG_SIDE_MM / (2 * Math.tan(fovRad / 2)); +} + +/** + * Thin-lens circle-of-confusion radius in pixels. `sceneScale` divides world + * distances first, so a 1:1000 rig focuses on the city as if it were a + * tabletop model and produces the matching shallow depth of field. The + * depth-of-field shader evaluates the same expression per pixel. + */ +export function circleOfConfusionPx({ + focalMm, + fStop, + focusM, + distanceM, + sceneScale = 1, + viewportPx, +}) { + const f = focalMm / 1000; + const focus = Math.max(focusM / sceneScale, f * 1.01); + const distance = Math.max(distanceM / sceneScale, 1e-4); + const diameterM = + ((f * f) / (fStop * (focus - f))) * (Math.abs(distance - focus) / distance); + return (0.5 * diameterM * 1000 * viewportPx) / SENSOR_LONG_SIDE_MM; +} diff --git a/src/director/packs/packs.test.mjs b/src/director/packs/packs.test.mjs index ed5d42500..76545219e 100644 --- a/src/director/packs/packs.test.mjs +++ b/src/director/packs/packs.test.mjs @@ -51,7 +51,7 @@ test('v5 packs and shot references round trip through project migration without JSON.stringify({ version, scenes: [{ id: 'old', shots: [] }] }), ), ); - assert.equal(old.version, 6); + assert.equal(old.version, 7); assert.equal(old.scenes[0].dataPacks, undefined); } }); diff --git a/src/director/track.test.mjs b/src/director/track.test.mjs new file mode 100644 index 000000000..2df17f342 --- /dev/null +++ b/src/director/track.test.mjs @@ -0,0 +1,118 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { parseSceneDocument, stringifySceneDocument } from './document.js'; +import { normalizeProject } from '../scenes/project.js'; +import { + resolveCameraMove, + sampleCameraMove, + sampleCameraTrack, +} from './camera.js'; +import { sceneSeekState } from './timeline.js'; +import { + DEFAULT_FOCAL_MM, + circleOfConfusionPx, + focalLengthToFov, + fovToFocalLength, +} from './lens.js'; + +const samples = () => [ + [0, 179.9, 10, 500, 350, -30, 0, 24], + [1, -179.9, 10.5, 600, 10, -20, 2, 50], + [3, -179.7, 11, 400, 30, -10, 4, 50], +]; +const fixture = (move = { track: { altitudeReference: 'ellipsoid', samples: samples() } }) => ({ + version: 7, + scenes: [ + { + id: 'scene', + shots: [ + { + id: 'take', + title: 'Take 1', + durationSec: 3, + holdSec: 1, + camera: { lat: 11, lon: -179.7, alt: 400, heading: 30, pitch: -10, roll: 4, altitudeReference: 'ellipsoid' }, + move, + lens: { focalMm: 50, dof: { enabled: true, focusM: 250, fStop: 1.8, scale: 1000 } }, + }, + ], + }, + ], +}); + +test('a recorded track round trips through the document and project migration', () => { + const project = normalizeProject(parseSceneDocument(JSON.stringify(fixture()))); + const shot = project.scenes[0].shots[0]; + assert.deepEqual(shot.move.track.samples, samples()); + assert.deepEqual(shot.lens, fixture().scenes[0].shots[0].lens); + assert.deepEqual(parseSceneDocument(stringifySceneDocument(project)).scenes[0].shots[0].move, fixture().scenes[0].shots[0].move); +}); + +test('track samples interpolate by recorded time, take short arcs and keep exact endpoints', () => { + const rows = samples(); + assert.deepEqual(sampleCameraTrack(rows, 0), { lon: 179.9, lat: 10, alt: 500, heading: 350, pitch: -30, roll: 0, focalMm: 24 }); + assert.deepEqual(sampleCameraTrack(rows, 1), { lon: -179.7, lat: 11, alt: 400, heading: 30, pitch: -10, roll: 4, focalMm: 50 }); + const mid = sampleCameraTrack(rows, 1 / 6); // 0.5 s, halfway through the first segment + assert.ok(Math.abs(Math.abs(mid.lon) - 180) < 1e-9, 'longitude crosses the dateline, not the globe'); + assert.ok(Math.abs(mid.heading - 360) < 1e-9 || Math.abs(mid.heading) < 1e-9); + assert.equal(mid.alt, 550); + assert.equal(mid.focalMm, 37); + const late = sampleCameraTrack(rows, 2 / 3); // 2 s, halfway through the second segment + assert.equal(late.alt, 500); + assert.equal(late.heading, 20); + const noFocal = sampleCameraTrack(rows.map((row) => row.slice(0, 7)), 0.5); + assert.equal(Object.hasOwn(noFocal, 'focalMm'), false); +}); + +test('a track stretches to its shot duration and drives scene seeking', () => { + const scene = normalizeProject(parseSceneDocument(JSON.stringify(fixture()))).scenes[0]; + const shot = scene.shots[0]; + const move = resolveCameraMove(scene, shot); + assert.equal(move.from.alt, 500); + assert.equal(move.to.alt, 400); + assert.deepEqual(sampleCameraMove(move, 2 / 3), sampleCameraTrack(shot.move.track.samples, 2 / 3)); + // Doubling the duration plays the same path at half speed. + shot.durationSec = 6; + const seek = sceneSeekState(scene, 2 / 7, (_, item) => item.durationSec + item.holdSec, (_, item) => item.holdSec); + assert.equal(seek.cameraProgress, 1 / 3); + assert.equal(seek.camera.alt, 600); +}); + +test('malformed tracks and lenses are rejected with a field path', () => { + const bad = (mutate, pattern) => { + const doc = fixture(); + mutate(doc.scenes[0].shots[0]); + assert.throws(() => parseSceneDocument(JSON.stringify(doc)), pattern); + }; + bad((shot) => { shot.move.track.samples = [shot.move.track.samples[0]]; }, /at least two samples/); + bad((shot) => { shot.move.track.samples[0][0] = 0.1; }, /first sample at 0/); + bad((shot) => { shot.move.track.samples[2][0] = 1; }, /increasing sample times/); + bad((shot) => { shot.move.track.samples[1] = [1, 0, 0]; }, /samples\[1\]: expected \[t, lon/); + bad((shot) => { shot.move.track.samples[1][2] = 91; }, /samples\[1\]\[2\]/); + bad((shot) => { shot.move.track.samples[1][7] = 1; }, /samples\[1\]\[7\]/); + bad((shot) => { delete shot.move.track.altitudeReference; }, /altitudeReference/); + bad((shot) => { shot.move.easing = 'linear'; }, /move\.easing: unsupported field/); + bad((shot) => { shot.lens.focalMm = 1; }, /lens\.focalMm/); + bad((shot) => { shot.lens.dof.enabled = 'yes'; }, /lens\.dof\.enabled/); + bad((shot) => { shot.lens.aperture = 2; }, /lens\.aperture: unsupported field/); + const v6 = fixture(); + v6.version = 6; + assert.throws(() => parseSceneDocument(JSON.stringify(v6)), /lens: unsupported field/); + delete v6.scenes[0].shots[0].lens; + assert.throws(() => parseSceneDocument(JSON.stringify(v6)), /move\.track: unsupported field/); +}); + +test('focal lengths convert to Cesium field of view and back', () => { + assert.ok(Math.abs(focalLengthToFov(DEFAULT_FOCAL_MM) - Math.PI / 3) < 1e-12); + assert.ok(Math.abs(fovToFocalLength(focalLengthToFov(85)) - 85) < 1e-9); + assert.ok(focalLengthToFov(14) > focalLengthToFov(200)); +}); + +test('the miniature scale turns invisible real-world blur into tabletop depth of field', () => { + const lens = { focalMm: 35, fStop: 2.8, focusM: 500, distanceM: 1000, viewportPx: 1920 }; + assert.ok(circleOfConfusionPx(lens) < 0.05, 'a real 35 mm lens barely blurs a city'); + const miniature = circleOfConfusionPx({ ...lens, sceneScale: 1000 }); + assert.ok(miniature > 10 && miniature < 20, `1:1000 blurs like a model (${miniature})`); + assert.equal(circleOfConfusionPx({ ...lens, distanceM: 500, sceneScale: 1000 }), 0); + assert.ok(circleOfConfusionPx({ ...lens, sceneScale: 1000, fStop: 16 }) < miniature / 4); +}); diff --git a/src/scenes/cameraLens.js b/src/scenes/cameraLens.js new file mode 100644 index 000000000..1605529a4 --- /dev/null +++ b/src/scenes/cameraLens.js @@ -0,0 +1,207 @@ +import * as Cesium from 'cesium'; +import { + DEFAULT_FOCAL_MM, + focalLengthToFov, + fovToFocalLength, +} from '../director/lens.js'; +import { FOCAL_LENGTH_MM } from '../director/cameraDocument.js'; + +/** + * Thin-lens depth of field. Distances are divided by `sceneScale` before the + * circle of confusion is computed (see circleOfConfusionPx), and each tap is + * weighted by whether its own blur reaches the centre pixel, so a sharp + * foreground does not smear into a blurred background. + */ +const DEPTH_OF_FIELD_SHADER = ` +uniform sampler2D colorTexture; +uniform sampler2D depthTexture; +uniform float focusDistance; +uniform float focalLength; +uniform float fStop; +uniform float sceneScale; +uniform float maxRadius; +in vec2 v_textureCoordinates; + +float eyeDistance(vec2 uv) { + float depth = czm_readDepth(depthTexture, uv); + vec2 xy = vec2(uv.x * 2.0 - 1.0, uv.y * 2.0 - 1.0); + vec4 eye = czm_inverseProjection * vec4(xy, depth, 1.0); + return -eye.z / eye.w; +} + +float cocRadius(float distance) { + float f = focalLength; + float s1 = max(focusDistance / sceneScale, f * 1.01); + float s2 = max(distance / sceneScale, 1.0e-4); + float c = (f * f) / (fStop * (s1 - f)) * abs(s2 - s1) / s2; + return min(maxRadius, 0.5 * c / 0.036 * czm_viewport.z); +} + +void main(void) { + vec4 base = texture(colorTexture, v_textureCoordinates); + float radius = cocRadius(eyeDistance(v_textureCoordinates)); + if (radius < 0.5) { + out_FragColor = base; + return; + } + vec2 texel = 1.0 / czm_viewport.zw; + vec3 sum = base.rgb; + float weight = 1.0; + for (int i = 1; i < 32; i++) { + float fi = float(i); + float r = sqrt(fi / 32.0) * radius; + float theta = fi * 2.39996323; + vec2 uv = v_textureCoordinates + vec2(cos(theta), sin(theta)) * r * texel; + float reach = cocRadius(eyeDistance(uv)); + float w = clamp(reach - r + 1.0, 0.0, 1.0); + sum += texture(colorTexture, uv).rgb * w; + weight += w; + } + out_FragColor = vec4(sum / weight, base.a); +} +`; + +const clampFocal = (mm) => + Math.max(FOCAL_LENGTH_MM[0], Math.min(FOCAL_LENGTH_MM[1], Number(mm))); + +/** + * Own the viewer camera's focal length and the depth-of-field stage. The + * lens records the frustum's field of view when it first changes it and + * restores that on reset, so shots without a lens keep the application's + * ordinary view. + * + * @param {Cesium.Viewer} viewer + * @param {object} [options] + * @param {(reason: string) => void} [options.requestRender] + * @param {(options: object) => object} [options.createStage] + */ +export function createCameraLens( + viewer, + { + requestRender = () => viewer.scene?.requestRender?.(), + createStage = (options) => new Cesium.PostProcessStage(options), + } = {}, +) { + let baseFov = null; + let focalMm = null; + let stage = null; + let destroyed = false; + const dof = { enabled: false, focusM: 500, fStop: 2.8, scale: 1 }; + const frustum = () => viewer.camera?.frustum; + const listeners = new Set(); + const notify = () => { + const state = getState(); + for (const listener of listeners) listener(state); + }; + + function ensureStage() { + if (stage || destroyed) return stage; + stage = createStage({ + name: 'godsEyeView_depthOfField', + fragmentShader: DEPTH_OF_FIELD_SHADER, + uniforms: { + focusDistance: dof.focusM, + focalLength: (focalMm ?? DEFAULT_FOCAL_MM) / 1000, + fStop: dof.fStop, + sceneScale: dof.scale, + maxRadius: 18, + }, + }); + stage.enabled = false; + viewer.scene.postProcessStages.add(stage); + return stage; + } + + function syncStage() { + if (!stage && !dof.enabled) return; + const target = ensureStage(); + if (!target) return; + Object.assign(target.uniforms, { + focusDistance: dof.focusM, + focalLength: currentFocalMm() / 1000, + fStop: dof.fStop, + sceneScale: dof.scale, + }); + target.enabled = dof.enabled; + } + + function currentFocalMm() { + if (focalMm !== null) return focalMm; + const fov = frustum()?.fov; + return Number.isFinite(fov) ? fovToFocalLength(fov) : DEFAULT_FOCAL_MM; + } + + /** Set the focal length; `null` restores the application's field of view. */ + function setFocalLength(mm, { silent = false } = {}) { + if (destroyed) return; + const target = frustum(); + if (!target || !('fov' in target)) return; + if (mm === null || mm === undefined) { + if (baseFov !== null) target.fov = baseFov; + baseFov = null; + focalMm = null; + } else { + if (!Number.isFinite(Number(mm))) return; + baseFov ??= target.fov; + focalMm = clampFocal(mm); + target.fov = focalLengthToFov(focalMm); + } + syncStage(); + requestRender('camera-lens'); + if (!silent) notify(); + } + + /** Merge depth-of-field settings; unspecified fields keep their values. */ + function setDepthOfField(next = {}) { + if (destroyed) return; + if (typeof next.enabled === 'boolean') dof.enabled = next.enabled; + if (Number.isFinite(next.focusM) && next.focusM > 0) + dof.focusM = Math.min(1e8, Math.max(0.1, next.focusM)); + if (Number.isFinite(next.fStop)) + dof.fStop = Math.min(64, Math.max(0.5, next.fStop)); + if (Number.isFinite(next.scale) && next.scale > 0) + dof.scale = Math.min(1e7, Math.max(1e-3, next.scale)); + syncStage(); + requestRender('camera-lens'); + notify(); + } + + /** Serializable shot lens, or null when the lens is at its defaults. */ + function getState() { + if (focalMm === null && !dof.enabled) return null; + return { + ...(focalMm !== null ? { focalMm: Math.round(focalMm * 100) / 100 } : {}), + ...(dof.enabled ? { dof: { ...dof } } : {}), + }; + } + + /** Apply a shot lens exactly; missing fields return to the defaults. */ + function apply(lens) { + if (destroyed) return; + setFocalLength(lens?.focalMm ?? null, { silent: true }); + setDepthOfField({ enabled: false, ...lens?.dof }); + } + + return { + setFocalLength, + setDepthOfField, + getState, + apply, + reset: () => apply(null), + getFocalLength: currentFocalMm, + getDepthOfField: () => ({ ...dof }), + subscribe(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + }, + destroy() { + if (destroyed) return; + apply(null); + destroyed = true; + listeners.clear(); + if (stage && !viewer.isDestroyed?.()) + viewer.scene.postProcessStages.remove(stage); + stage = null; + }, + }; +} diff --git a/src/scenes/cameraLens.test.mjs b/src/scenes/cameraLens.test.mjs new file mode 100644 index 000000000..fa614b632 --- /dev/null +++ b/src/scenes/cameraLens.test.mjs @@ -0,0 +1,66 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createCameraLens } from './cameraLens.js'; +import { focalLengthToFov } from '../director/lens.js'; + +function harness() { + const stages = []; + const renders = []; + const viewer = { + camera: { frustum: { fov: Math.PI / 3 } }, + scene: { + postProcessStages: { + add: (stage) => stages.push(stage), + remove: (stage) => stages.splice(stages.indexOf(stage), 1), + }, + }, + }; + const lens = createCameraLens(viewer, { + requestRender: (reason) => renders.push(reason), + createStage: (options) => ({ ...options, uniforms: { ...options.uniforms } }), + }); + return { viewer, lens, stages, renders }; +} + +test('focal length owns the frustum and restores the application view', () => { + const { viewer, lens, stages, renders } = harness(); + assert.equal(lens.getState(), null); + lens.setFocalLength(85); + assert.equal(viewer.camera.frustum.fov, focalLengthToFov(85)); + assert.deepEqual(lens.getState(), { focalMm: 85 }); + lens.setFocalLength(1); + assert.equal(lens.getFocalLength(), 4, 'focal lengths clamp to the document range'); + lens.setFocalLength(null); + assert.equal(viewer.camera.frustum.fov, Math.PI / 3); + assert.equal(lens.getState(), null); + assert.equal(stages.length, 0, 'no depth-of-field stage exists until it is enabled'); + assert.ok(renders.length >= 3); +}); + +test('depth of field creates one stage, tracks the lens and resets with the shot', () => { + const { viewer, lens, stages } = harness(); + const seen = []; + const unsubscribe = lens.subscribe((state) => seen.push(state)); + lens.apply({ focalMm: 50, dof: { enabled: true, focusM: 300, fStop: 2, scale: 500 } }); + assert.equal(stages.length, 1); + const [stage] = stages; + assert.equal(stage.enabled, true); + assert.deepEqual( + [stage.uniforms.focusDistance, stage.uniforms.focalLength, stage.uniforms.fStop, stage.uniforms.sceneScale], + [300, 0.05, 2, 500], + ); + assert.deepEqual(lens.getState(), { focalMm: 50, dof: { enabled: true, focusM: 300, fStop: 2, scale: 500 } }); + lens.setDepthOfField({ focusM: -1, fStop: 100 }); + assert.equal(stage.uniforms.focusDistance, 300, 'invalid focus is ignored'); + assert.equal(stage.uniforms.fStop, 64); + lens.apply(null); + assert.equal(stage.enabled, false); + assert.equal(viewer.camera.frustum.fov, Math.PI / 3); + assert.equal(lens.getState(), null); + assert.ok(seen.length >= 3); + unsubscribe(); + lens.setFocalLength(35); + lens.destroy(); + assert.equal(stages.length, 0); + assert.equal(viewer.camera.frustum.fov, Math.PI / 3, 'destroy restores the view'); +}); diff --git a/src/scenes/director.js b/src/scenes/director.js index 000c02681..48c59395c 100644 --- a/src/scenes/director.js +++ b/src/scenes/director.js @@ -90,6 +90,7 @@ export class SceneDirector { isMapStackAvailable = () => false, scenePacks = createDefaultScenePacks(), dataPacks = {}, + lens = null, } = {}, ) { this._destroyed = false; @@ -99,6 +100,8 @@ export class SceneDirector { this.dataManager = dataManager; this._isMapStackAvailable = isMapStackAvailable; this._scenePacks = scenePacks; + /** Focal length and depth of field; shots without a lens restore the default view. */ + this._lens = lens; this._bundleAssets = createBundleAssets(); this._dataPacks = createSceneDataPacks(viewer, { ...dataPacks, @@ -982,6 +985,7 @@ export class SceneDirector { camera, visual: this.styleManager.getVisualState(), layers: this._captureLayerStates(), + ...this._capturedLens(), }, scene.shots.length, ); @@ -1000,6 +1004,82 @@ export class SceneDirector { } } + /** The live lens as a shot field; an unchanged lens adds nothing. */ + _capturedLens() { + const lens = this._lens?.getState(); + return lens ? { lens } : {}; + } + + /** Apply a shot's lens after its visual state; shots without one restore the default. */ + _applyShotLens(shot) { + this._lens?.apply(shot?.lens ?? null); + } + + /** + * Append a recorded camera take as an explicit track shot. `samples` are + * compact `[t, lon, lat, alt, heading, pitch, roll, focalMm?]` rows at + * ellipsoidal heights, the first at 0 seconds. The take keeps the live + * visual and layer state so it replays over the same map it was shot on. + * + * @param {object} take + * @param {Array} take.samples + * @param {string} [take.title] + * @param {string} [take.sceneId] - Defaults to the selected scene. + * @param {object|null} [take.lens] - Depth of field at the end of the take. + * @returns {{ sceneId: string, shotId: string, saved: boolean }|null} + */ + addRecordedShot({ samples, title, sceneId, lens = null } = {}) { + if (this._destroyed) return null; + const scene = sceneId + ? this._project.scenes.find((item) => item.id === sceneId) + : this._getSelectedScene(); + if (!scene || !Array.isArray(samples) || samples.length < 2) return null; + const last = samples.at(-1); + const shot = normalizeShot( + { + id: uid('shot'), + // i18n-ignore: stored English; displayShotTitle translates it when shown. + title: title || `Take ${scene.shots.length + 1}`, + durationSec: Math.max(0.2, last[0]), + holdSec: 0, + camera: { + lon: last[1], + lat: last[2], + alt: last[3], + heading: last[4], + pitch: last[5], + roll: last[6], + altitudeReference: 'ellipsoid', + }, + move: { + track: { + altitudeReference: 'ellipsoid', + samples: samples.map((row) => [...row]), + }, + }, + visual: this.styleManager.getVisualState(), + layers: this._captureLayerStates(), + ...(lens ? { lens } : {}), + }, + scene.shots.length, + ); + scene.shots.push(shot); + this._selectedSceneId = scene.id; + this._selectedShotId = shot.id; + // An unsaved take stays in the session like an unsaved capture; the + // storage toast tells the person to export it. + const saved = this._saveProject(); + this._shotOutcome('shot-captured', scene, shot); + if (saved) + this._updateStatus( + t('Recorded: {scene} / {shot}', { + scene: displaySceneTitle(scene), + shot: displayShotTitle(shot.title, shot, scene), + }), + ); + return { sceneId: scene.id, shotId: shot.id, saved }; + } + /** * Overwrite the currently selected shot's camera, visual, and layer states * with the live viewport state. Useful for fine-tuning a shot in-place. @@ -1022,6 +1102,8 @@ export class SceneDirector { : camera; shot.visual = this.styleManager.getVisualState(); shot.layers = this._captureLayerStates(); + delete shot.lens; + Object.assign(shot, this._capturedLens()); const saved = this._saveProject(); this._shotOutcome('shot-updated', scene, shot); @@ -1157,6 +1239,7 @@ export class SceneDirector { isCurrent: () => !token.cancelled, }); if (token.cancelled) return; + this._applyShotLens(shot); const seekState = sceneSeek && typeof sceneSeek === 'object' ? sceneSeek : null; // Previous-scene disable revokes media ownership. Grant the target only @@ -1665,6 +1748,8 @@ export class SceneDirector { roll: Cesium.Math.toRadians(cameraState.roll || 0), }, }); + if (Number.isFinite(cameraState.focalMm)) + this._lens?.setFocalLength(cameraState.focalMm, { silent: true }); return true; } diff --git a/src/scenes/director.test.mjs b/src/scenes/director.test.mjs index f72b43dc7..c890efdcf 100644 --- a/src/scenes/director.test.mjs +++ b/src/scenes/director.test.mjs @@ -736,6 +736,7 @@ function makeDirector(options = {}) { const dataManager = fakeDataManager(options.data); const director = new SceneDirector(viewer, styleManager, dataManager, { isMapStackAvailable: options.isMapStackAvailable, + lens: options.lens, }); // Telemetry is only accumulated during a run; observable-failure assertions // need the accumulator without driving a whole run. @@ -1522,7 +1523,7 @@ test('unsupported stored documents cannot be overwritten by fallback edits', asy assert.equal(localStorage.getItem('godsEyeView.sceneProject.v2'), saved); await director.importProjectFile({ name: 'valid.json', text: async () => JSON.stringify(PROJECT_FIXTURE) }); assert.equal(director._storageReadError, null); - assert.equal(JSON.parse(localStorage.getItem('godsEyeView.sceneProject.v2')).version, 6); + assert.equal(JSON.parse(localStorage.getItem('godsEyeView.sceneProject.v2')).version, 7); } finally { restore(); } }); @@ -1658,3 +1659,48 @@ test('a caught run failure resolves as run-failed, not success (#971)', async () ); } finally { console.warn = warn; await director.destroy(); restore(); } }); + +test('a recorded take becomes a validated track shot that replays its samples and lens', async () => { + const applied = []; + const lens = { + state: { focalMm: 85 }, + getState() { return this.state; }, + apply(value) { applied.push(['apply', value]); }, + setFocalLength(mm) { applied.push(['focal', mm]); }, + }; + const { director, viewer, restore } = makeDirector({ lens }); + try { + const samples = [ + [0, -97.74, 30.26, 900, 10, -30, 0, 24], + [1.5, -97.73, 30.27, 950, 20, -25, 0, 50], + ]; + const result = director.addRecordedShot({ samples, title: 'Take A', lens: { dof: { enabled: true, focusM: 300, fStop: 2, scale: 100 } } }); + assert.equal(result.saved, true); + const scene = director._project.scenes.find(({ id }) => id === result.sceneId); + const shot = scene.shots.find(({ id }) => id === result.shotId); + assert.equal(shot.durationSec, 1.5); + assert.equal(shot.holdSec, 0); + assert.deepEqual(shot.move.track.samples, samples); + assert.notEqual(shot.move.track.samples[0], samples[0], 'the take owns copies of its rows'); + assert.deepEqual(shot.camera, { lat: 30.27, lon: -97.73, alt: 950, heading: 20, pitch: -25, roll: 0, altitudeReference: 'ellipsoid' }); + const stored = JSON.parse(localStorage.getItem('godsEyeView.sceneProject.v2')); + assert.equal(stored.version, 7); + assert.ok(stored.scenes.some((item) => item.shots.some(({ id }) => id === result.shotId))); + + const views = []; + viewer.camera.setView = (options) => views.push(options); + scene.shots = [shot]; + applied.length = 0; + await director.seekScene(result.sceneId, 0.5); + assert.deepEqual(applied[0], ['apply', shot.lens], 'the shot lens is applied after its visual state'); + assert.deepEqual(applied.at(-1), ['focal', 37], 'seeking samples the recorded focal length'); + assert.equal(views.length, 1); + + director.captureShot(); + assert.deepEqual(scene.shots.at(-1).lens, { focalMm: 85 }, 'capture keeps a changed lens'); + lens.state = null; + director.captureShot(); + assert.equal(Object.hasOwn(scene.shots.at(-1), 'lens'), false, 'a default lens adds nothing'); + assert.equal(director.addRecordedShot({ samples: [samples[0]] }), null, 'one sample is not a take'); + } finally { await director.destroy(); restore(); } +}); diff --git a/src/scenes/playbackAdapter.js b/src/scenes/playbackAdapter.js index e44286988..aaac33280 100644 --- a/src/scenes/playbackAdapter.js +++ b/src/scenes/playbackAdapter.js @@ -31,11 +31,14 @@ export function createScenePlaybackAdapter(director, defaultShotDurationSec) { index, }); }, - applyVisual({ shot, token }) { - return director.styleManager.applyVisualState( + async applyVisual({ shot, token }) { + const isCurrent = () => !token.cancelled && !token.signal?.aborted; + const result = await director.styleManager.applyVisualState( director._visualStateForShot(shot), - { isCurrent: () => !token.cancelled && !token.signal?.aborted }, + { isCurrent }, ); + if (isCurrent()) director._applyShotLens(shot); + return result; }, async applyLayers({ scene, shot, token }) { director._setSceneMediaPlayback(scene, shot, token); diff --git a/src/scenes/project.js b/src/scenes/project.js index eb3084962..de6b6b456 100644 --- a/src/scenes/project.js +++ b/src/scenes/project.js @@ -261,6 +261,7 @@ export function normalizeShot( : {}), ...(rawShot?.dataPackIds ? { dataPackIds: [...rawShot.dataPackIds] } : {}), ...(rawShot?.move ? { move: deepClone(rawShot.move) } : {}), + ...(rawShot?.lens ? { lens: deepClone(rawShot.lens) } : {}), visual: { style: visual.style || 'normal', bloom: normalizeBloomState(bloom, { diff --git a/src/scenes/sharing.js b/src/scenes/sharing.js index b3adde27a..f6bd626d0 100644 --- a/src/scenes/sharing.js +++ b/src/scenes/sharing.js @@ -43,6 +43,7 @@ const shotKeys = [ 'holdSec', 'dataPackIds', 'interactions', + 'lens', ]; const json = (value) => JSON.stringify(value, null, 2); const mimeFor = (file) =>