/** * Desktop side of the virtual camera: pairs a phone through the local relay, * drives the Cesium camera from its pose, streams a preview back and records * takes into the Scene Director. * * Camera ownership follows the application's rules. The rig claims the camera * through the shared navigation seam (so Cockpit can refuse it) and lets go * — holds — whenever anything else takes it: a search, a scene playing, or * the mouse on the globe. Releasing the hold re-anchors the rig wherever the * camera now is, so nothing jumps. */ import * as Cesium from 'cesium'; import { createRig, SCALE_RANGE } from './rig.js'; import { msg, t } from '../i18n/index.js'; import { holdContinuousRender, releaseContinuousRender, } from '../renderGovernor.js'; const RECORD_INTERVAL_MS = 1000 / 30; // The scene document's per-array limit. export const MAX_TAKE_SAMPLES = 10000; const PREVIEW_FPS = 12; const PREVIEW_WIDTH = 720; const PREVIEW_QUALITY = 0.62; const PREVIEW_BACKLOG_BYTES = 512 * 1024; // Sticks stop when the phone stops talking (a dropped link, a locked screen). const STALE_POSE_MS = 400; const STATE_HEARTBEAT_MS = 1000; const MIN_GROUND_CLEARANCE_M = 0.5; const MAX_STEP_SEC = 0.5; const MAX_RECONNECT_ATTEMPTS = 5; const RENDER_HOLD = 'vcam'; const finite = (value) => typeof value === 'number' && Number.isFinite(value); const vectorOf = (value, length) => Array.isArray(value) && value.length === length && value.every(finite) ? value : null; const round = (value, places) => { const k = 10 ** places; return Math.round(value * k) / k; }; /** * @param {object} options * @param {Cesium.Viewer} options.viewer * @param {object} options.navigation - Shell navigation: `runImmediateNavigation`, `subscribeCameraHandoff`. * @param {object} options.director - SceneDirector (`addRecordedShot`). * @param {object} options.lens - Camera lens (`setFocalLength`, `setDepthOfField`, ...). */ export function createVcamController({ viewer, navigation, director, lens, fetchImpl = (...args) => fetch(...args), WebSocketImpl = globalThis.WebSocket, pageLocation = globalThis.location, now = () => performance.now(), }) { const camera = viewer.camera; const scene = viewer.scene; const listeners = new Set(); const rig = createRig(); const state = { status: 'idle', error: null, code: null, port: null, addresses: [], address: null, caFingerprint: null, phoneConnected: false, mode: null, driving: false, held: false, holdReason: null, recording: false, recordStartedAt: 0, takes: 0, lastTake: null, }; let session = null; let socket = null; let reconnectAttempts = 0; let latestPose = null; let lastStep = null; let anchor = null; let claiming = false; let rendering = false; let samples = []; let lastSampleAt = -Infinity; let previewBusy = false; let lastPreviewAt = -Infinity; let heartbeat = null; let destroyed = false; const previewCanvas = document.createElement('canvas'); const removers = []; // ---------------------------------------------------------------- state const snapshot = () => ({ ...state, pairingUrl: state.address && state.port && state.code ? `https://${state.address}:${state.port}/#${state.code}` : null, recordingSec: state.recording ? (now() - state.recordStartedAt) / 1000 : 0, settings: settings(), }); function settings() { const dof = lens?.getDepthOfField?.() || {}; return { ...rig.settings, focalMm: round(lens?.getFocalLength?.() ?? 31.18, 2), fStop: dof.fStop ?? 2.8, dof: !!dof.enabled, }; } function publish() { const current = snapshot(); for (const listener of listeners) { try { listener(current); } catch (error) { console.error('[vcam] listener failed', error); } } sendPhoneState(current); } function sendPhoneState(current = snapshot()) { send({ type: 'state', state: { recording: current.recording, elapsedSec: round(current.recordingSec, 2), held: current.held, takes: current.takes, ...current.settings, }, }); } function send(message) { if (socket?.readyState === 1) socket.send(JSON.stringify(message)); } function toastPhone(text) { send({ type: 'toast', text }); } // ------------------------------------------------------------- session /** Create a pairing session and connect the desktop end of the relay. */ async function start() { if (destroyed || session) return snapshot(); Object.assign(state, { status: 'starting', error: null }); publish(); let response; try { response = await fetchImpl('/api/vcam/session', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}', }); } catch { return fail(msg('The phone camera service is unavailable')); } const body = await response.json().catch(() => ({})); if (!response.ok) return fail( body.code === 'port-in-use' ? msg('The phone camera port is in use (set VCAM_PORT)') : body.code === 'no-network' ? msg('No network connection for a phone to reach') : msg('The phone camera service is unavailable'), ); if (destroyed) return snapshot(); session = body; reconnectAttempts = 0; Object.assign(state, { status: 'waiting', code: body.code, port: body.port, addresses: body.addresses || [], address: body.addresses?.[0] || null, caFingerprint: body.caFingerprint || null, }); openSocket(); heartbeat = setInterval(() => { if (state.phoneConnected) sendPhoneState(); }, STATE_HEARTBEAT_MS); publish(); return snapshot(); } function fail(error) { Object.assign(state, { status: 'error', error }); publish(); return snapshot(); } function openSocket() { if (!session || destroyed) return; const protocol = pageLocation.protocol === 'https:' ? 'wss:' : 'ws:'; const ws = new WebSocketImpl( `${protocol}//${pageLocation.host}/api/vcam/host?key=${encodeURIComponent(session.hostKey)}`, ); ws.binaryType = 'arraybuffer'; socket = ws; ws.onopen = () => { if (socket === ws) reconnectAttempts = 0; }; ws.onmessage = (event) => { if (socket !== ws || typeof event.data !== 'string') return; let message; try { message = JSON.parse(event.data); } catch { return; } handleMessage(message); }; ws.onclose = (event) => { if (socket !== ws) return; socket = null; setPhoneConnected(false); // 4000: the server ended the session; anything else may be transient. // A restarted server refuses the old key, which looks like a network // failure from here; give up rather than retry forever. if ( event.code === 4000 || event.code === 4003 || ++reconnectAttempts > MAX_RECONNECT_ATTEMPTS ) { endLocally(msg('Phone camera session ended')); return; } setTimeout(() => { if (session && !socket) openSocket(); }, 1500); }; } /** End the pairing session and release the camera. */ async function stop() { const ending = session; endLocally(null); if (ending) await fetchImpl('/api/vcam/session', { method: 'DELETE', headers: { 'X-Vcam-Host-Key': ending.hostKey }, }).catch(() => {}); return snapshot(); } function endLocally(error) { if (state.recording) stopRecording(); session = null; const ws = socket; socket = null; ws?.close(); clearInterval(heartbeat); heartbeat = null; setPhoneConnected(false); Object.assign(state, { status: error ? 'error' : 'idle', error, code: null, port: null, addresses: [], address: null, mode: null, }); stopDriving(); publish(); } function setPhoneConnected(connected) { if (state.phoneConnected === connected) return; state.phoneConnected = connected; if (!connected) { latestPose = null; stopDriving(); if (state.recording) stopRecording(); } if (connected && state.status === 'waiting') state.status = 'connected'; if (!connected && state.status === 'connected') state.status = 'waiting'; syncRenderHold(); publish(); } // ------------------------------------------------------------ messages function handleMessage(message) { switch (message?.type) { case 'code': if (typeof message.code === 'string') { state.code = message.code.slice(0, 16); publish(); } return; case 'peer': if (message.role === 'phone') setPhoneConnected(!!message.connected); return; case 'hello': state.mode = message.mode === '6dof' ? '6dof' : '3dof'; setPhoneConnected(true); publish(); return; case 'pose': return receivePose(message); case 'cmd': return command(message.cmd); case 'set': return set(message.key, message.value); case 'focus': if (finite(message.x) && finite(message.y)) focusAt(message.x, message.y); return; } } function receivePose(message) { const q = vectorOf(message.q, 4); if (!q) return; latestPose = { q, p: message.p === null ? null : vectorOf(message.p, 3), joy: (vectorOf(message.j, 4) || [0, 0, 0, 0]).map((v) => Math.max(-1, Math.min(1, v)), ), at: now(), }; const mode = latestPose.p ? '6dof' : '3dof'; if (state.mode !== mode) { state.mode = mode; publish(); } if (!state.phoneConnected) setPhoneConnected(true); if (!state.driving && !state.held) beginDriving(); } function command(name) { if (name === 'record') startRecording(); else if (name === 'stop') stopRecording(); else if (name === 'hold') hold('phone'); else if (name === 'release') release(); else if (name === 'recenter') recenter(); } /** Change one setting from either the phone or the desktop panel. */ function set(key, value) { if (key === 'scale' && finite(value)) { const scale = Math.max(SCALE_RANGE[0], Math.min(SCALE_RANGE[1], value)); shiftAnchor(rig.configure({ scale })); // Depth of field treats the world at the rig's scale: a miniature. lens?.setDepthOfField?.({ scale }); } else if (key === 'smoothing' && finite(value)) rig.configure({ smoothing: value }); else if ( (key === 'heightLock' || key === 'horizonLock') && typeof value === 'boolean' ) shiftAnchor(rig.configure({ [key]: value })); else if (key === 'focalMm' && finite(value)) lens?.setFocalLength?.(value); else if (key === 'fStop' && finite(value)) lens?.setDepthOfField?.({ fStop: value }); else if (key === 'dof' && typeof value === 'boolean') lens?.setDepthOfField?.({ enabled: value, scale: rig.settings.scale }); else return; publish(); } // -------------------------------------------------------------- camera function claimCamera() { if (typeof navigation?.runImmediateNavigation !== 'function') return true; claiming = true; try { return navigation.runImmediateNavigation('vcam', () => true) !== false; } finally { claiming = false; } } function beginDriving() { if (!latestPose || state.driving) return; if (!claimCamera()) { hold('refused'); toastPhone(t('Camera unavailable — exit cockpit first')); return; } camera.cancelFlight?.(); reanchor(); state.driving = true; state.held = false; state.holdReason = null; syncRenderHold(); publish(); } function stopDriving() { state.driving = false; lastStep = null; syncRenderHold(); } /** Start the phone's motion from the camera's current pose. */ function reanchor() { anchor = Cesium.Cartesian3.clone(camera.positionWC); if (latestPose) { rig.step({ ...latestPose, joy: [0, 0, 0, 0], dt: 0 }); rig.recenter(camera.heading); } lastStep = null; } function hold(reason = 'phone') { if (state.held && state.holdReason === reason) return; state.held = true; state.holdReason = reason; stopDriving(); publish(); } function release() { if (!state.held) return; state.held = false; state.holdReason = null; if (latestPose) beginDriving(); publish(); } function recenter() { if (!state.driving) return release(); reanchor(); publish(); } function shiftAnchor(shift) { if (!anchor || !shift || (!shift[0] && !shift[1] && !shift[2])) return; const frame = Cesium.Transforms.eastNorthUpToFixedFrame(anchor); const delta = Cesium.Matrix4.multiplyByPointAsVector( frame, new Cesium.Cartesian3(shift[0], shift[1], shift[2]), new Cesium.Cartesian3(), ); anchor = Cesium.Cartesian3.add(anchor, delta, anchor); } function placeCamera() { if (!state.driving || state.held || !latestPose || !anchor) return; const at = now(); // Smoothing and sticks follow wall time, so a slow frame rate does not // slow the camera; only a long stall (a hidden tab) is cut short. const dt = lastStep === null ? 0 : Math.min(MAX_STEP_SEC, (at - lastStep) / 1000); lastStep = at; const stale = at - latestPose.at > STALE_POSE_MS; const out = rig.step({ ...latestPose, joy: stale ? [0, 0, 0, 0] : latestPose.joy, dt, }); shiftAnchor(out.anchorShift); const frame = Cesium.Transforms.eastNorthUpToFixedFrame(anchor); const destination = Cesium.Matrix4.multiplyByPoint( frame, Cesium.Cartesian3.fromArray(out.offset), new Cesium.Cartesian3(), ); const direction = Cesium.Cartesian3.normalize( Cesium.Matrix4.multiplyByPointAsVector( frame, Cesium.Cartesian3.fromArray(out.forward), new Cesium.Cartesian3(), ), new Cesium.Cartesian3(), ); const up = Cesium.Cartesian3.normalize( Cesium.Matrix4.multiplyByPointAsVector( frame, Cesium.Cartesian3.fromArray(out.up), new Cesium.Cartesian3(), ), new Cesium.Cartesian3(), ); keepAboveGround(destination); camera.setView({ destination, orientation: { direction, up } }); } /** Stop at the terrain surface rather than passing under it. */ function keepAboveGround(position) { const cartographic = Cesium.Cartographic.fromCartesian(position); if (!cartographic) return; const ground = scene.globe?.show ? scene.globe.getHeight(cartographic) : undefined; if (!finite(ground)) return; const floor = ground + MIN_GROUND_CLEARANCE_M; if (cartographic.height >= floor) return; cartographic.height = floor; Cesium.Cartographic.toCartesian(cartographic, undefined, position); } /** Tap-to-focus: focus the lens on what lies under a normalized point. */ function focusAt(x, y) { const canvas = scene.canvas; const point = new Cesium.Cartesian2( Math.max(0, Math.min(1, x)) * canvas.clientWidth, Math.max(0, Math.min(1, y)) * canvas.clientHeight, ); let target; try { if (scene.pickPositionSupported) target = scene.pickPosition(point); } catch { target = undefined; } if (!target) { const ray = camera.getPickRay(point); target = ray && scene.globe?.pick(ray, scene); } if (!target) { toastPhone(t('Nothing to focus on there')); return; } const focusM = Cesium.Cartesian3.distance(camera.positionWC, target); lens?.setDepthOfField?.({ enabled: true, focusM, scale: rig.settings.scale, }); publish(); } // ----------------------------------------------------------- recording function startRecording() { if (state.recording || !session) return; samples = []; lastSampleAt = -Infinity; state.recording = true; state.recordStartedAt = now(); syncRenderHold(); publish(); } function recordSample() { if (!state.recording) return; const at = now(); if (at - lastSampleAt < RECORD_INTERVAL_MS - 1) return; const carto = camera.positionCartographic; if (!carto) return; lastSampleAt = at; const t0 = samples.length ? state.recordStartedAt : (state.recordStartedAt = at); samples.push([ round((at - t0) / 1000, 3), round(Cesium.Math.toDegrees(carto.longitude), 7), round(Cesium.Math.toDegrees(carto.latitude), 7), round(carto.height, 2), round(Cesium.Math.toDegrees(camera.heading), 2), round(Cesium.Math.clamp(Cesium.Math.toDegrees(camera.pitch), -90, 90), 2), round(Cesium.Math.toDegrees(camera.roll), 2), round(lens?.getFocalLength?.() ?? 31.18, 2), ]); if (samples.length >= MAX_TAKE_SAMPLES) { stopRecording(); toastPhone(t('Take reached its length limit and was saved')); } } /** Stop recording and add the take to the selected scene. */ function stopRecording() { if (!state.recording) return null; state.recording = false; syncRenderHold(); const take = samples; samples = []; let result = null; if (take.length >= 2) { // Playback takes the short arc between samples, so headings may wrap; // rounding can still reach 360, which stays inside the ±360 bound. for (const row of take) for (const column of [4, 6]) row[column] = round(((row[column] % 360) + 360) % 360, 2); const dof = lens?.getDepthOfField?.(); result = director?.addRecordedShot?.({ samples: take, lens: { focalMm: take.at(-1)[7], ...(dof?.enabled ? { dof: { ...dof } } : {}), }, }); if (result) { state.takes += 1; state.lastTake = { ...result, durationSec: take.at(-1)[0] }; toastPhone(t('Take saved to the scene')); } } else toastPhone(t('Take too short to keep')); publish(); return result; } // ------------------------------------------------------------- preview function sendPreview() { if (!state.phoneConnected || socket?.readyState !== 1 || previewBusy) return; const at = now(); if (at - lastPreviewAt < 1000 / PREVIEW_FPS) return; if (socket.bufferedAmount > PREVIEW_BACKLOG_BYTES) return; const source = scene.canvas; if (!source?.width || !source?.height) return; lastPreviewAt = at; previewBusy = true; const width = Math.min(PREVIEW_WIDTH, source.width); const height = Math.round((width * source.height) / source.width); if (previewCanvas.width !== width) previewCanvas.width = width; if (previewCanvas.height !== height) previewCanvas.height = height; // Copy inside postRender, while the drawing buffer still holds this frame. previewCanvas.getContext('2d').drawImage(source, 0, 0, width, height); previewCanvas.toBlob( async (blob) => { try { if (blob && socket?.readyState === 1) socket.send(await blob.arrayBuffer()); } finally { previewBusy = false; } }, 'image/jpeg', PREVIEW_QUALITY, ); } function syncRenderHold() { const wanted = !destroyed && (state.driving || state.phoneConnected || state.recording); if (wanted === rendering) return; rendering = wanted; if (wanted) holdContinuousRender(RENDER_HOLD); else releaseContinuousRender(RENDER_HOLD); } // ------------------------------------------------------------ wiring removers.push(scene.preUpdate.addEventListener(() => placeCamera())); removers.push( scene.postRender.addEventListener(() => { recordSample(); sendPreview(); }), ); const yieldToNavigation = () => { if (!claiming && state.driving) hold('navigation'); }; const unsubscribeHandoff = navigation?.subscribeCameraHandoff?.(yieldToNavigation); if (unsubscribeHandoff) removers.push(unsubscribeHandoff); const canvas = scene.canvas; const yieldToMouse = () => { if (state.driving) hold('desktop'); }; for (const type of ['pointerdown', 'wheel']) { canvas?.addEventListener(type, yieldToMouse, { passive: true }); removers.push(() => canvas?.removeEventListener(type, yieldToMouse)); } return { start, stop, hold: () => hold('desktop-button'), release, recenter, set, startRecording, stopRecording, toggleRecording: () => state.recording ? stopRecording() : startRecording(), setAddress(address) { if (state.addresses.includes(address)) { state.address = address; publish(); } }, getState: snapshot, subscribe(listener) { listeners.add(listener); listener(snapshot()); return () => listeners.delete(listener); }, get active() { return !!session; }, destroy() { if (destroyed) return; const ending = session; endLocally(null); if (ending) fetchImpl('/api/vcam/session', { method: 'DELETE', headers: { 'X-Vcam-Host-Key': ending.hostKey }, keepalive: true, }).catch(() => {}); destroyed = true; syncRenderHold(); for (const remove of removers.splice(0)) remove(); listeners.clear(); }, }; }