From 56586a234bb5b85902bdfe7c87146b2dad91cca8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 19 Oct 2026 04:03:02 +0000 Subject: [PATCH] feat(vcam): use a phone on the network as a virtual camera SCENES > VCAM pairs a phone through a QR code and turns it into a handheld camera, after Unreal's Live Link VCAM and Blender's VirtuCamera: - Android Chrome tracks 6DoF through WebXR immersive-ar; iPhone Safari, which has no WebXR AR, tracks rotation through DeviceOrientation, and thumb sticks dolly, truck, crane and pan. - A motion scale (1:1 walk to 1:10000 orbit), smoothing, height and horizon locks, hold/clutch and recenter, none of which make the camera jump. Focal length and scale-aware depth of field with tap-to-focus. - The phone shows the app's render as a monitor (newest-frame JPEG pull). - REC records a take as a version-7 track shot in the selected scene. - The rig claims the camera through the navigation seam (Cockpit refuses it) and holds on any other navigation, scene run or globe input. Pairing never exposes the app: a loopback-only session request starts a separate HTTPS listener (VCAM_PORT, default 4174) that serves only the static phone page and a token-paired relay, and stops after the last session. Its certificate comes from a local CA built with node:crypto, which an iPhone can trust once; without it the phone falls back from WebSocket to HTTPS requests, which iOS allows past a certificate warning. scripts/qa-vcam.mjs drives it end to end: the real phone page over the HTTPS fallback, a scripted 6DoF phone, a recorded and replayed take, depth of field and camera handoff. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015H1iLusTzWpVKitGic2AW8 --- .env.example | 6 + README.md | 13 + SECURITY.md | 1 + docs/CURRENT-STATE.md | 14 + docs/VCAM.md | 165 ++++++ scripts/package-boundaries.json | 8 +- scripts/qa-vcam.mjs | 365 ++++++++++++ server/standalone/vite.config.js | 2 + server/vcam/phone/icon.svg | 1 + server/vcam/phone/index.html | 154 +++++ server/vcam/phone/orientation.js | 45 ++ server/vcam/phone/orientation.test.mjs | 39 ++ server/vcam/phone/phone.css | 371 ++++++++++++ server/vcam/phone/phone.js | 770 +++++++++++++++++++++++++ server/vcam/plugin.js | 522 +++++++++++++++++ server/vcam/plugin.test.mjs | 274 +++++++++ server/vcam/relay.js | 319 ++++++++++ server/vcam/relay.test.mjs | 166 ++++++ server/vcam/tls.js | 237 ++++++++ src/app/tools.js | 25 + src/i18n/locales/es.json | 50 +- src/i18n/locales/ru.json | 51 +- src/i18n/same-as-english.json | 2 + src/navigationPolicy.js | 2 + src/tooling/viteBuild.test.mjs | 8 +- src/ui/styles/scenes.css | 154 +++++ src/ui/templates/layer-panels.html | 42 ++ src/ui/vcamPanel.js | 195 +++++++ src/vcam/controller.js | 706 +++++++++++++++++++++++ src/vcam/qr.js | 325 +++++++++++ src/vcam/qr.test.mjs | 53 ++ src/vcam/rig.js | 272 +++++++++ src/vcam/rig.test.mjs | 121 ++++ 33 files changed, 5472 insertions(+), 6 deletions(-) create mode 100644 docs/VCAM.md create mode 100644 scripts/qa-vcam.mjs create mode 100644 server/vcam/phone/icon.svg create mode 100644 server/vcam/phone/index.html create mode 100644 server/vcam/phone/orientation.js create mode 100644 server/vcam/phone/orientation.test.mjs create mode 100644 server/vcam/phone/phone.css create mode 100644 server/vcam/phone/phone.js create mode 100644 server/vcam/plugin.js create mode 100644 server/vcam/plugin.test.mjs create mode 100644 server/vcam/relay.js create mode 100644 server/vcam/relay.test.mjs create mode 100644 server/vcam/tls.js create mode 100644 src/ui/vcamPanel.js create mode 100644 src/vcam/controller.js create mode 100644 src/vcam/qr.js create mode 100644 src/vcam/qr.test.mjs create mode 100644 src/vcam/rig.js create mode 100644 src/vcam/rig.test.mjs diff --git a/.env.example b/.env.example index a2f8afa24..32046e585 100644 --- a/.env.example +++ b/.env.example @@ -141,6 +141,12 @@ PORT=4173 # wildcard entries are ignored. # GEV_ALLOWED_HOSTS= +# SCENES β€Ί VCAM (phone virtual camera, docs/VCAM.md): the HTTPS listener a +# paired phone connects to. It starts only when you open a pairing session and +# serves only the phone page and relay, not the app or its keys. +# VCAM_PORT=4174 +# VCAM_HOST=0.0.0.0 + # Optional: NASA FIRMS live active fires. SERVER-SIDE ONLY (never reaches the # browser β€” the client polls the /api/firms proxy). Free key: # https://firms.modaps.eosdis.nasa.gov/api/map_key/ diff --git a/README.md b/README.md index c49c53ece..4e57ae27c 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ Start with the included data sources, then add your own. Each layer is a separat - **πŸŽ–οΈ Military HUD:** Tactical heads-up display with intelligence-style telemetry. - **🌐 Global Context:** Stage the full situational picture with one switch β€” and get your exact view back when you leave. - **πŸŽ₯ Scene director:** Capture cinematic camera tours for clips and demos. +- **πŸ“± Phone virtual camera:** Scan a QR code and your phone becomes the camera β€” walk the room at 1:1, or set 1:1000 and one step crosses the city. Focal length, depth of field and recorded takes that land in the scene editor ([docs/VCAM.md](docs/VCAM.md)). - **πŸ”— Share Links:** Camera, style, layers, and even one tracked target serialize into a URL β€” a live target is a handoff, not a bookmark. - **🏠 Reset Globe:** One control β€” or one sentence β€” back to the full Earth. - **🌦️ Weather:** Animate GFS or ECMWF forecast wind, replay observed radar, satellite clouds and lightning on one timeline, and follow NHC/CPHC cyclone tracks. No key needed. @@ -307,6 +308,18 @@ _Why cockpit mode exists: you're riding a real aircraft over real terrain β€” an --- +## πŸ“± Phone Virtual Camera + +> Frame the shot with your hands, like a virtual-production VCAM. + +Open **SCENES β€Ί VCAM** and scan the code with a phone on the same Wi-Fi. The phone shows the globe as a monitor and becomes the camera: on Android Chrome it tracks position and rotation through WebXR AR, and on iPhone Safari it tracks rotation while two thumb sticks dolly, truck, crane and pan. The motion scale sets the world under you, from 1:1 street level through 1:100 drone to 1:10000 orbit, and changing it never makes the camera jump. + +Dial in a focal length from 12 to 400 mm, turn on depth of field and tap the picture to focus. Distances are scaled with the rig, so at 1:1000 the city falls out of focus like a tabletop miniature. Press **REC** and the move becomes a take: a shot in the selected scene, with every sample's position, orientation and focal length, which replays, seeks and retimes like any other. + +Nothing extra is exposed by default: pairing starts a separate HTTPS listener that serves only the phone page and its relay, never the app or its keys. See [docs/VCAM.md](docs/VCAM.md) for trusting the local certificate on iPhone and for the protocol. + +--- + ## πŸŽ™οΈ Talk to It > Voice can use an **OpenAI API key** or, on the same machine, an opt-in signed-in **ChatGPT/Codex OAuth** session. The API-key path remains the default. Selecting ChatGPT OAuth in Provider Settings reuses an existing local sign-in or starts the Codex browser sign-in flow and selects OAuth when it completes. The OpenAI key still drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. diff --git a/SECURITY.md b/SECURITY.md index e53883548..c5086e5fa 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -86,6 +86,7 @@ The dev server is a **key broker**: every server-side key above is spendable by - **Local-only by default.** `./scripts/dev-fresh.sh` (and the Vite config itself) bind to `localhost`, so only your machine can reach the server β€” and only local names are accepted (`allowedHosts` stays restricted, which also blunts DNS-rebinding tricks). Binding to all interfaces keeps that restriction: IP addresses and `localhost` work, and LAN hostnames must be listed in `GEV_ALLOWED_HOSTS` (for example `GEV_ALLOWED_HOSTS=globe.lan`); suffix and wildcard entries are ignored. - **LAN exposure is an explicit opt-in**: `HOST=0.0.0.0 ./scripts/dev-fresh.sh`. The launcher prints a prominent warning plus your LAN URL. Understand what opting in means: **every device on that network can drive the proxies and spend your OpenAI / Google / OpenSky / AISStream / TomTom / FIRMS quota** for as long as the server runs. Do this only on networks you trust. +- **The phone camera (SCENES β€Ί VCAM) opens a separate, narrow LAN listener β€” only on request.** Creating a pairing session (a loopback, same-origin request from the app, behind the same gate as `/mcp`) starts an HTTPS listener on `VCAM_PORT` (default `4174`, all interfaces unless `VCAM_HOST` narrows it); it stops two minutes after the last session ends. It serves only the static phone page, a local CA certificate download and the phone side of a relay: never the app, its provider proxies or any key, so pairing a phone does not amount to the `HOST=0.0.0.0` opt-in. A phone joins with the six-character code shown on the desktop (about 30 bits; eight wrong codes rotate it) and receives a per-phone key; the relay forwards bounded JSON (4 KiB, 120/s) and one newest JPEG preview frame, and WebSockets on both ends must come from their own page's origin. The listener answers only IP addresses, `localhost` and `GEV_ALLOWED_HOSTS` names. Its TLS certificate is issued by a CA created in `.gev-cache/vcam/` (private keys owner-only), constrained to server certificates (path length 0); a phone that installs and trusts it trusts certificates from this machine's CA, so remove the profile when you no longer need it. **Limit:** anyone on the network who reads the code off your screen can drive your camera and see its preview until you press END. - **App-level throttles (on by default):** `GEV_RATELIMIT_OPENAI_PER_MIN` (default `30`) and `GEV_RATELIMIT_GOOGLE_PER_MIN` (default `120`) β€” the values the Pinokio build already ships β€” cap the cost-bearing endpoints per client IP per minute (over-limit requests receive a sanitized `429`). Both defaults sit well above what the app itself generates; set either to exactly `0` to run unthrottled. They are **per-IP, process-local, in-memory guards** β€” they reset on restart, a caller with several addresses gets a bucket per address, and they are **not billing caps**. - **Docker trusts one address, and only because the port is on loopback.** Docker delivers the host's own browser to the container from the container diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9562dc9f5..cd333071c 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -429,6 +429,20 @@ or unknown freshness and transport failures retain their warnings. Partial snapshots still retain missing contacts within the existing age and row limits. A complete successful snapshot clears the partial indicator. +SCENES β€Ί VCAM pairs a phone on the same network as a virtual camera. Opening +the card creates a loopback-only session and starts a separate HTTPS phone +listener (`VCAM_PORT`, default 4174) that serves only the phone page and relay; +END, or two minutes after the last session, stops it. Android Chrome tracks +6DoF through WebXR `immersive-ar`; iPhone Safari tracks rotation through +DeviceOrientation with on-screen sticks, falling back from WebSocket to HTTPS +requests under an untrusted certificate. The rig claims the camera through the +navigation seam (Cockpit refuses it) and holds on any other navigation, scene +run or globe pointer input. Motion scale, smoothing, height and horizon locks, +focal length and scale-aware depth of field are shared by phone and card. REC +writes version-7 track shots (`move.track`, up to 10,000 samples with focal +length) into the selected scene; shots may also carry a `lens`. See +[VCAM](VCAM.md); gate: `node scripts/qa-vcam.mjs`. + Director imports now open a non-mutating preview before Apply. EDIT DETAILS authors validated anchor, camera, pack and interaction drafts; SHARE SCENE exports a selected scene or a bounded bundle of explicitly chosen pack files. Bundled diff --git a/docs/VCAM.md b/docs/VCAM.md new file mode 100644 index 000000000..d0386caae --- /dev/null +++ b/docs/VCAM.md @@ -0,0 +1,165 @@ +# VCAM: a phone as the virtual camera + +VCAM turns a phone on the same network into a handheld camera for the globe, +like Unreal's Live Link VCAM or Blender's VirtuCamera. Walk around the room +holding the phone and the app's camera moves with it. A motion scale decides +how big a step is: 1:1 walks the street, 1:100 is a drone, 1:1000 flies over +the city. The phone shows the app's render as a monitor, and **REC** records the +move as a take: a shot in the selected scene that replays, seeks and retimes +like any other shot. + +## Using it + +1. Open **SCENES β€Ί VCAM**. The card shows a QR code, a six-character code and + the phone URL (`https://:4174/`). +2. Scan the code with the phone's camera app, or open the URL and type the + code. The phone and the computer must be on the same network. +3. The page warns about the certificate the first time (see [Trust](#trust)); + continue past it, or install the local certificate once. +4. Choose **START AR CAMERA** (6DoF) or **START GYRO CAMERA** (3DoF). +5. Hold the phone like a camera. The rig starts from wherever the globe camera + is, so nothing jumps. +6. **REC** on the phone or in the card records a take. Takes are added to the + scene selected in the Scenes panel. + +| Control | Phone | Card | What it does | +| --- | --- | --- | --- | +| Motion scale | SCALE drawer | SCALE | World metres per phone metre, 0.01 to 100,000. Presets: 1:1 walk, 1:10 crane, 1:100 drone, 1:1000 flyover, 1:10000 orbit. Changing it never moves the camera. | +| Move stick | left | | Dolly and truck along the camera's heading, at 2.5 m/s Γ— scale. | +| Crane Β· pan stick | right | | Raise or lower the rig; pan about the camera itself. | +| Hold | HOLD | HOLD / RESUME | Freeze the camera to reposition yourself (a clutch). Release re-anchors where the camera is. | +| Recenter | RECENTER | RECENTER | Keep the camera where it is and point the phone's current heading along it. | +| Smoothing | SCALE drawer | | Damps hand shake (0 raw to 100 heavy). | +| Height lock | SCALE drawer | | Ignore the phone's vertical motion. | +| Horizon lock | SCALE drawer | | Keep the horizon level (no roll). | +| Focal length | LENS drawer | LENS | 35 mm-equivalent, 12 to 400 mm. Recorded per sample. | +| Depth of field | LENS drawer | DEPTH OF FIELD | Thin-lens blur; tap the phone's picture to focus. | +| Aperture | LENS drawer | APERTURE | f/1.2 to f/22. | +| Record | ● | ● REC | Start and stop a take. | + +The rig yields the camera whenever something else takes it, as scene playback +does: a search or route, a scene starting, Cockpit, or a mouse press or wheel on +the globe. It then reads **Held**. Tap HOLD on the phone (or RESUME) to continue +from wherever the camera now is. Cockpit refuses the camera, as it does for +scenes. + +### Tracking: what each phone gets + +| Phone and browser | Mode | Position | Rotation | +| --- | --- | --- | --- | +| Android Chrome with Google Play Services for AR | AR (WebXR `immersive-ar`) | tracked (6DoF) | tracked | +| iPhone Safari | Gyro (DeviceOrientation) | sticks only | tracked (3DoF) | +| Other browsers without WebXR AR | Gyro | sticks only | tracked | + +iPhone Safari does not offer WebXR AR, so a web page cannot read ARKit's +position. Gyro mode tracks the phone's rotation and the sticks move it. WebXR +browsers that do offer `immersive-ar` on iOS get the AR mode automatically. + +### Depth of field and the miniature look + +The blur follows a thin-lens circle of confusion. Distances are divided by the +motion scale first, so the lens treats the world at the rig's size. At 1:1 a +city a kilometre away is effectively sharp, as it is through a real lens. At +1:1000 the same city focuses like a tabletop model and falls off like +tilt-shift miniature photography. Tap the phone's picture to set the focus +distance at that point. + +## Takes in the Director + +A take is a version-7 scene-document shot with a recorded track +([camera directions](DIRECTOR-CAMERA.md#recorded-tracks)). Each sample stores +time, position, heading, pitch, roll and focal length, about 30 per second. A +take holds at most 10,000 samples (about 5Β½ minutes); recording stops and +saves at that limit. Editing the shot's `durationSec` retimes the take: a longer +duration plays the same path in slow motion. Depth of field is stored in the +shot's `lens`. Takes export, import and share like any other shot. + +Takes live in the browser's scene storage like other shots. A browser that +refuses to save a long take keeps it for the session and shows the usual +storage warning; use EXPORT PRESETS to keep it. + +## Trust + +Phones expose motion tracking only to secure (HTTPS) pages, so the phone +listener serves HTTPS with a certificate from a local certificate authority it +creates on first use (`.gev-cache/vcam/`, git-ignored, private keys owner-only). +The server certificate names this computer's current LAN addresses and is +reissued when they change; the CA stays the same. + +- **Android Chrome:** continue past the warning once. AR and the live + connection work. +- **iPhone Safari:** the page loads past the warning, but Safari refuses its + WebSocket, so the phone falls back to HTTPS requests (an amber dot rather + than green). It works with a little more latency. To trust the computer: + open **Phone certificate** in the card (or the link on the phone's pairing + page), allow the profile, install it under **Settings β€Ί General β€Ί VPN & + Device Management**, then enable it under **Settings β€Ί General β€Ί About β€Ί + Certificate Trust Settings**. + +The CA is constrained to sign server certificates only (path length 0). Remove +the profile from the phone, or delete `.gev-cache/vcam/`, to revoke it. + +## How it is built + +``` +phone (HTTPS :4174) desktop app (http://localhost:4173) + server/vcam/phone/* ── pose JSON ──▢ relay ──▢ /api/vcam/host (WebSocket) + ◀── state ───── ◀── src/vcam/controller.js + ◀── JPEG pull ── newest frame only +``` + +- **`server/vcam/plugin.js`**: on the app's server, loopback-only routes behind + the `/mcp` gate (`POST`/`DELETE /api/vcam/session`, `/api/vcam/ca.crt`, the + host WebSocket). It starts a separate HTTPS listener (`VCAM_PORT`, default + 4174, on `VCAM_HOST`, default all interfaces) only when a session is created, + and stops it two minutes after the last one ends. That listener serves the + static phone page and the phone side of the relay, never the app or its + provider proxies. +- **`server/vcam/relay.js`**: pairing and forwarding, independent of transport. + The phone exchanges the six-character code (31 unambiguous characters, about + 30 bits, case and separators ignored) for its own key; eight wrong codes + rotate every session's code and the card updates. Messages are JSON objects of + at most 4 KiB, at most 120 a second with a burst of 240. The host's JPEG + preview is held as the newest frame only; the phone pulls it, so a slow link + drops frames instead of queueing them. +- **`server/vcam/tls.js`**: the local CA and server certificate, DER-encoded with + `node:crypto`; there is no OpenSSL or dependency. +- **`server/vcam/phone/`**: the phone page. It uses a WebSocket when it can, + otherwise POSTs poses (newest-wins) and listens on server-sent events. Poses + are a quaternion and position in WebXR's Y-up frame (+x right, +y up, βˆ’z + forward); gyro mode converts DeviceOrientation into the same frame. +- **`src/vcam/rig.js`**: the renderer-independent mapping from phone pose to an + east-north-up camera offset and basis around a geographic anchor. Recenter, + scale changes and stick moves shift the anchor rather than the phone's origin, + so the camera stays continuous. +- **`src/vcam/controller.js`**: session lifecycle, camera ownership, per-frame + placement (`scene.preUpdate`), recording and preview capture + (`scene.postRender`, 720 px JPEG at 12 frames a second). +- **`src/scenes/cameraLens.js`**: focal length (frustum field of view) and the + depth-of-field post-process stage, shared with scene playback. +- **`src/vcam/qr.js`**: a QR encoder (byte mode, level M, versions 1–10). + +`node scripts/qa-vcam.mjs` drives all of it end to end against a running dev +server: the real phone page in gyro mode over the HTTPS fallback, a scripted +6DoF phone over a WebSocket, a take recorded and replayed, and camera handoff. + +## Configuration + +| Variable | Default | | +| --- | --- | --- | +| `VCAM_PORT` | `4174` | Phone listener port. | +| `VCAM_HOST` | `0.0.0.0` | Phone listener interface. Set an address to restrict it. | + +The Docker image publishes only the app's port on loopback, so VCAM needs the +phone port published as well (see [SECURITY.md](../SECURITY.md)). + +## Next + +- Preview over WebRTC (`canvas.captureStream`) for lower latency and higher + resolution, with the JPEG pull kept as the fallback. +- Bookmarks and a take browser in the phone UI; slate and take naming. +- Focus pulls and animated aperture recorded per sample. +- A full-quality offline render of a take (fixed-step, every frame). +- iPhone 6DoF through an ARKit companion (an App Clip or a small native app + speaking the same relay protocol). +- Phone page translation through the app's catalogs. diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 43bc3d4de..0f09b4238 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -2490,7 +2490,13 @@ "src/layers/airQuality/index.js", "src/layers/airQuality/model.js", "src/layers/airQuality/source.js", - "src/ui/statusText.js" + "src/ui/statusText.js", + "src/scenes/cameraLens.js", + "src/director/lens.js", + "src/vcam/controller.js", + "src/vcam/rig.js", + "src/vcam/qr.js", + "src/ui/vcamPanel.js" ], "external": [ "hls.js", diff --git a/scripts/qa-vcam.mjs b/scripts/qa-vcam.mjs new file mode 100644 index 000000000..d970332ea --- /dev/null +++ b/scripts/qa-vcam.mjs @@ -0,0 +1,365 @@ +#!/usr/bin/env node +/** + * qa-vcam.mjs β€” phone virtual camera, end to end. + * + * Drives the real desktop app and the real phone page (served by the HTTPS + * phone listener) in one browser, plus a scripted 6DoF phone over the same + * relay: + * + * 1. SCENES β–Έ VCAM starts a session and draws a QR code for the pairing URL. + * 2. The phone page pairs from the URL fragment, falls back to HTTPS requests + * when WebSockets are refused (as iOS does under a certificate warning), + * starts the gyro camera and turns the desktop camera with device + * orientation; the desktop's render arrives on the phone's monitor. + * 3. The phone's lens preset and REC button reach the desktop; the recorded + * take becomes a validated track shot in the selected scene and replays. + * 4. A scripted 6DoF phone moves one metre at 1:100 and the camera moves + * about a hundred metres; desktop navigation holds the rig. + * + * Needs the dev server (default http://localhost:4173, or QA_BASE_URL) and + * its phone listener on VCAM_PORT (default 4174), reachable on loopback. + * Screenshots go to qa-shots/vcam/. + * + * npm run dev # in another terminal + * node scripts/qa-vcam.mjs + */ +import fs from 'node:fs'; +import https from 'node:https'; +import path from 'node:path'; +import puppeteer from 'puppeteer'; +import WebSocket from 'ws'; + +const BASE = process.env.QA_BASE_URL || 'http://localhost:4173'; +const output = path.resolve('qa-shots/vcam'); +fs.mkdirSync(output, { recursive: true }); +let failed = 0; +function check(name, passed, detail = '') { + console.log(`[${passed ? 'PASS' : 'FAIL'}] ${name}${detail ? ` β€” ${detail}` : ''}`); + if (!passed) failed++; +} +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +// Separate browsers, like separate devices: a background tab would hide the +// desktop page, which suspends its render loop. +const launch = () => + puppeteer.launch({ + headless: true, + acceptInsecureCerts: true, + executablePath: process.env.PUPPETEER_EXECUTABLE_PATH, + args: ['--no-sandbox', '--disable-dev-shm-usage', '--ignore-certificate-errors'], + }); +const browser = await launch(); +const phoneBrowser = await launch(); +const errors = []; +try { + const desk = await browser.newPage(); + desk.on('pageerror', (error) => errors.push(`desktop: ${error.message}`)); + if (process.env.QA_VCAM_DEBUG) desk.on('pageerror', (error) => console.log('DESKTOP ERROR', error.stack || error.message)); + await desk.setViewport({ width: 1280, height: 800 }); + await desk.goto(`${BASE}/?welcome=0`, { waitUntil: 'domcontentloaded' }); + await desk.waitForFunction( + () => + window.__godsEyeView?.vcam && + document.getElementById('loading-screen')?.classList.contains('hidden'), + { timeout: 90000 }, + ); + // A shader that fails to compile stops Cesium's render loop. + await desk.evaluate(() => { + window.__vcamRenderErrors = []; + window.__godsEyeView.viewer.scene.renderError.addEventListener((_scene, error) => + window.__vcamRenderErrors.push(String(error?.message || error)), + ); + }); + // Start from a known city view. + await desk.evaluate(() => + window.__godsEyeView.styleManager.applyCameraState( + { lat: 30.2672, lon: -97.7431, alt: 1200, heading: 0, pitch: -30, roll: 0 }, + 0.2, + ), + ); + await sleep(800); + + // 1. Pairing UI. + await desk.evaluate(() => { + const panel = document.getElementById('scene-panel'); + if (panel.classList.contains('collapsed')) + panel.querySelector('.panel-collapse-btn')?.click(); + }); + await desk.click('#vcam-toggle-btn'); + if (process.env.QA_VCAM_DEBUG) + await desk.evaluate(() => { + window.__vcamSeen = []; + window.__godsEyeView.vcam.subscribe((s) => + window.__vcamSeen.push([s.status, s.phoneConnected, s.driving, s.held, s.holdReason].join(',')), + ); + }); + await desk.waitForFunction(() => window.__godsEyeView.vcam.getState().code, { timeout: 15000 }); + const pairing = await desk.evaluate(() => { + const state = window.__godsEyeView.vcam.getState(); + const qr = document.getElementById('vcam-qr'); + return { + state, + shownCode: document.getElementById('vcam-code').textContent, + qrDrawn: !qr.hidden && qr.width > 100, + link: document.getElementById('vcam-url').textContent, + }; + }); + check('VCAM opens a pairing session with a code and QR', pairing.qrDrawn && pairing.shownCode === pairing.state.code, pairing.link); + const phoneUrl = `https://127.0.0.1:${pairing.state.port}/#${pairing.state.code}`; + + // 2. The real phone page, with WebSockets refused as on iOS. + const phone = await phoneBrowser.newPage(); + phone.on('pageerror', (error) => errors.push(`phone: ${error.message}`)); + if (process.env.QA_VCAM_DEBUG) + phone.on('request', (request) => { + if (request.url().includes('/api/') && !request.url().includes('/api/frame')) + console.log('PHONE', request.method(), request.url().split('?')[0], request.postData()?.slice(0, 90) ?? ''); + }); + await phone.emulate({ + viewport: { width: 390, height: 844, isMobile: true, hasTouch: true, deviceScaleFactor: 2 }, + userAgent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Mobile/15E148 Safari/604.1', + }); + await phone.evaluateOnNewDocument(() => { + window.WebSocket = class { + static OPEN = 1; + readyState = 0; + constructor() { + setTimeout(() => { + this.onerror?.(new Event('error')); + this.onclose?.({ code: 1006 }); + }, 50); + } + close() {} + }; + }); + await phone.goto(phoneUrl, { waitUntil: 'domcontentloaded' }); + await phone.waitForSelector('#start:not([hidden])', { timeout: 10000 }); + check('the phone pairs from the QR link and clears the code from the address bar', + !(await phone.evaluate(() => location.hash))); + await phone.click('#start-gyro'); + await phone.waitForSelector('#camera:not([hidden])', { timeout: 5000 }); + const orient = (alpha, beta = 90, gamma = 0) => + phone.evaluate((a, b, g) => { + window.dispatchEvent(new DeviceOrientationEvent('deviceorientation', { alpha: a, beta: b, gamma: g })); + }, alpha, beta, gamma); + await orient(0); + await desk + .waitForFunction(() => window.__godsEyeView.vcam.getState().driving, { timeout: 10000 }) + .catch(async (error) => { + const { status, phoneConnected, held, holdReason, mode } = await desk.evaluate(() => + window.__godsEyeView.vcam.getState(), + ); + if (process.env.QA_VCAM_DEBUG) + console.log('SEEN', await desk.evaluate(() => JSON.stringify([...new Set(window.__vcamSeen)]))); + throw new Error(`rig never drove: ${JSON.stringify({ status, phoneConnected, held, holdReason, mode })}`, { + cause: error, + }); + }); + const linkState = await phone.evaluate(() => document.getElementById('link-dot').className); + check('the phone falls back to HTTPS requests when WebSockets fail', linkState.includes('slow'), linkState); + const before = await desk.evaluate(() => window.__godsEyeView.styleManager.getCameraState()); + for (let alpha = 0; alpha <= 45; alpha += 3) { + await orient(alpha); + await sleep(40); + } + // Headless WebGL renders a few frames a second; wait for the rig to settle. + const headingAfter = (target) => + desk + .waitForFunction( + (from, wanted) => { + const { heading } = window.__godsEyeView.styleManager.getCameraState(); + return Math.abs((((from - heading + 540) % 360) - 180) - wanted) < 1; + }, + { timeout: 15000, polling: 100 }, + before.heading, + target, + ) + .catch(() => {}); + await headingAfter(45); + const turned = await desk.evaluate(() => window.__godsEyeView.styleManager.getCameraState()); + const headingDelta = ((before.heading - turned.heading + 540) % 360) - 180; + check('turning the phone 45Β° left turns the camera 45Β° left', Math.abs(headingDelta - 45) < 1, + `heading ${before.heading.toFixed(1)} β†’ ${turned.heading.toFixed(1)}`); + check('an upright phone looks at the horizon', Math.abs(turned.pitch) < 5, `pitch ${turned.pitch.toFixed(1)}`); + const vcamMode = await desk.evaluate(() => window.__godsEyeView.vcam.getState().mode); + check('the desktop reports gyro (3DoF) tracking', vcamMode === '3dof'); + await phone.waitForFunction(() => document.getElementById('monitor').naturalWidth > 0, { timeout: 10000 }); + check('the desktop render reaches the phone monitor', true); + + // 3. Lens and a take from the phone. + await phone.click('#lens-btn'); + await phone.click('#focal-presets button[data-focal="85"]'); + await desk.waitForFunction(() => Math.abs(window.__godsEyeView.lens.getFocalLength() - 85) < 0.01, { timeout: 5000 }); + const fov = await desk.evaluate(() => window.__godsEyeView.viewer.camera.frustum.fov); + check('the phone lens preset sets the desktop focal length', Math.abs(fov - 2 * Math.atan(18 / 85)) < 1e-6); + await phone.click('#lens-btn'); + const shotsBefore = await desk.evaluate(() => { + const d = window.__godsEyeView.sceneDirector; + return d._getSelectedScene().shots.length; + }); + await phone.click('#record-btn'); + await desk.waitForFunction(() => window.__godsEyeView.vcam.getState().recording, { timeout: 5000 }); + for (let alpha = 45; alpha >= -20; alpha -= 2) { + await orient(alpha, 80); + await sleep(50); + } + await phone.click('#record-btn'); + await desk.waitForFunction(() => !window.__godsEyeView.vcam.getState().recording, { timeout: 5000 }); + const take = await desk.evaluate(() => { + const d = window.__godsEyeView.sceneDirector; + const scene = d._getSelectedScene(); + const shot = scene.shots.at(-1); + return { + count: scene.shots.length, + title: shot.title, + samples: shot.move?.track?.samples?.length || 0, + duration: shot.durationSec, + focal: shot.move?.track?.samples?.[0]?.[7], + listed: document.getElementById('scene-shot-list').textContent.includes(shot.title), + saved: JSON.parse(localStorage.getItem('godsEyeView.sceneProject.v2')).scenes.some((s) => s.shots.some((x) => x.id === shot.id)), + sceneId: scene.id, + shotId: shot.id, + last: shot.move?.track?.samples?.at(-1), + }; + }); + // One sample per rendered frame, at most 30 a second. + check('REC on the phone adds a track shot to the selected scene', + take.count === shotsBefore + 1 && take.samples >= 2 && take.duration > 2 && + take.samples <= take.duration * 30 + 2 && take.listed && take.saved, + `${take.title}: ${take.samples} samples over ${take.duration}s`); + check('the take records the focal length per sample', take.focal === 85); + await desk.screenshot({ path: path.join(output, 'desktop-take.png') }); + await phone.screenshot({ path: path.join(output, 'phone-camera.png') }); + + // Replaying the take ends exactly on its last sample. + await desk.evaluate(() => window.__godsEyeView.vcam.hold()); + await desk.evaluate(({ sceneId, shotId }) => + window.__godsEyeView.sceneDirector.loadShot(sceneId, shotId), take); + await desk.waitForFunction(({ last }) => { + const c = window.__godsEyeView.styleManager.getCameraState(); + return Math.abs(c.lat - last[2]) < 1e-5 && Math.abs(c.lon - last[1]) < 1e-5; + }, { timeout: 20000 }, take).catch(() => {}); + const replayed = await desk.evaluate(() => window.__godsEyeView.styleManager.getCameraState()); + check('loading the take plays it to its final recorded pose', + Math.abs(replayed.lat - take.last[2]) < 1e-5 && Math.abs(replayed.alt - take.last[3]) < 0.5, + `alt ${replayed.alt.toFixed(1)} vs ${take.last[3]}`); + await phone.close(); + + // 4. A scripted 6DoF phone: translation follows the motion scale. + const join = await new Promise((resolve) => { + const request = https.request( + `https://127.0.0.1:${pairing.state.port}/api/join`, + { method: 'POST', rejectUnauthorized: false, headers: { 'Content-Type': 'application/json' } }, + (response) => { + let body = ''; + response.on('data', (chunk) => (body += chunk)); + response.on('end', () => resolve({ ok: response.statusCode === 200, body })); + }, + ); + request.on('error', () => resolve({ ok: false })); + request.end(JSON.stringify({ code: pairing.state.code })); + }); + let sixDof = null; + if (join.ok) { + const { phoneKey } = JSON.parse(join.body); + sixDof = new WebSocket(`wss://127.0.0.1:${pairing.state.port}/ws?key=${phoneKey}`, { + origin: `https://127.0.0.1:${pairing.state.port}`, + rejectUnauthorized: false, + }); + await new Promise((resolve, reject) => { + sixDof.once('open', resolve); + sixDof.once('error', reject); + }); + } + check('a second phone can pair over a WebSocket', !!sixDof); + if (sixDof) { + const pose = (z, extra = {}) => + sixDof.send(JSON.stringify({ type: 'pose', t: Date.now(), q: [0, 0, 0, 1], p: [0, 1.5, z], j: [0, 0, 0, 0], ...extra })); + sixDof.send(JSON.stringify({ type: 'set', key: 'scale', value: 100 })); + sixDof.send(JSON.stringify({ type: 'set', key: 'smoothing', value: 0 })); + sixDof.send(JSON.stringify({ type: 'cmd', cmd: 'release' })); + for (let i = 0; i < 10; i++) { + pose(0); + await sleep(30); + } + await desk.waitForFunction(() => window.__godsEyeView.vcam.getState().driving && window.__godsEyeView.vcam.getState().mode === '6dof', { timeout: 5000 }); + const position = () => + desk.evaluate(() => { + const { x, y, z } = window.__godsEyeView.viewer.camera.positionWC; + return [x, y, z]; + }); + const start = await position(); + for (let i = 1; i <= 20; i++) { + pose(-i / 20); + await sleep(30); + } + await desk + .waitForFunction( + (from) => { + const { x, y, z } = window.__godsEyeView.viewer.camera.positionWC; + return Math.abs(Math.hypot(x - from[0], y - from[1], z - from[2]) - 100) < 1; + }, + { timeout: 15000, polling: 100 }, + start, + ) + .catch(() => {}); + const end = await position(); + const moved = Math.hypot(end[0] - start[0], end[1] - start[1], end[2] - start[2]); + check('one metre of phone motion at 1:100 moves the camera 100 m', Math.abs(moved - 100) < 1, `${moved.toFixed(1)} m`); + + // Depth of field at a miniature scale, focused by a tap at the centre. + sixDof.send(JSON.stringify({ type: 'set', key: 'scale', value: 1000 })); + sixDof.send(JSON.stringify({ type: 'set', key: 'dof', value: true })); + sixDof.send(JSON.stringify({ type: 'set', key: 'fStop', value: 4 })); + // Pitch down 30 degrees, let the rig settle, then focus on the ground ahead. + for (let i = 0; i < 20; i++) { + pose(-1, { q: [-0.2588, 0, 0, 0.9659] }); + await sleep(50); + } + await desk + .waitForFunction(() => window.__godsEyeView.styleManager.getCameraState().pitch < -29, { + timeout: 15000, + polling: 100, + }) + .catch(() => {}); + sixDof.send(JSON.stringify({ type: 'focus', x: 0.5, y: 0.5 })); + await sleep(2000); + const dof = await desk.evaluate(() => { + const { lens, viewer } = window.__godsEyeView; + const stage = viewer.scene.postProcessStages; + let found = null; + for (let i = 0; i < stage.length; i++) + if (stage.get(i).name === 'godsEyeView_depthOfField') found = stage.get(i); + return { + state: lens.getDepthOfField(), + enabled: !!found?.enabled, + ready: !!found?.ready, + errors: window.__vcamRenderErrors, + }; + }); + check('tap-to-focus enables depth of field at the rig scale', + dof.enabled && dof.state.scale === 1000 && dof.state.focusM > 1000 && dof.state.focusM < 5000 && dof.state.fStop === 4, + `focus ${dof.state.focusM?.toFixed?.(0)} m at 1:${dof.state.scale}`); + check('the depth-of-field shader compiles and renders', dof.ready && dof.errors.length === 0, dof.errors.join(' | ')); + await desk.screenshot({ path: path.join(output, 'desktop-dof.png') }); + + // Mouse navigation on the globe holds the rig instead of fighting it. + await desk.mouse.click(640, 400); + const held = await desk.evaluate(() => window.__godsEyeView.vcam.getState()); + check('desktop mouse input holds the rig', held.held && !held.driving, held.holdReason); + sixDof.close(); + } + await desk.click('#vcam-end-btn'); + await desk.waitForFunction(() => window.__godsEyeView.vcam.getState().status === 'idle', { timeout: 5000 }); + check('END closes the session', true); +} catch (error) { + console.error(error); + failed++; +} finally { + await browser.close(); + await phoneBrowser.close(); +} +const relevant = errors.filter((message) => !/Failed to fetch|NetworkError|tile|imagery/i.test(message)); +check('no page errors', relevant.length === 0, relevant.slice(0, 3).join(' | ')); +console.log(failed ? `\n${failed} check(s) failed` : '\nAll VCAM checks passed'); +process.exit(failed ? 1 : 0); diff --git a/server/standalone/vite.config.js b/server/standalone/vite.config.js index 03a935092..04e714f70 100644 --- a/server/standalone/vite.config.js +++ b/server/standalone/vite.config.js @@ -7,6 +7,7 @@ import { localMcpPlugin } from '../mcp/plugin.js'; import { apiNotFoundPlugin } from './api-not-found.js'; import { standaloneVoiceTools } from './voiceTools.js'; import { unwatchedRuntimeDirsPlugin } from './unwatched-cache.js'; +import { vcamPlugin } from '../vcam/plugin.js'; const root = fileURLToPath(new URL('../../', import.meta.url)); @@ -20,6 +21,7 @@ export default defineConfig(({ command, mode }) => { plugins: [ ...localProviderPlugins({ realtime: { tools: standaloneVoiceTools() } }), localMcpPlugin(), + vcamPlugin(), apiNotFoundPlugin(), unwatchedRuntimeDirsPlugin(), ], diff --git a/server/vcam/phone/icon.svg b/server/vcam/phone/icon.svg new file mode 100644 index 000000000..ade4b6693 --- /dev/null +++ b/server/vcam/phone/icon.svg @@ -0,0 +1 @@ + diff --git a/server/vcam/phone/index.html b/server/vcam/phone/index.html new file mode 100644 index 000000000..74fb0f09f --- /dev/null +++ b/server/vcam/phone/index.html @@ -0,0 +1,154 @@ + + + + + + + + GEV Virtual Camera + + + + +
+
+ +

VIRTUAL CAMERA

+

+ Enter the code shown in God's Eye View under SCENES β€Ί VCAM. +

+
+ + +
+

+
+ Connection drops on iPhone? +

+ Safari can load this page past a certificate warning but may still + refuse its live connection, so the camera falls back to slower + requests. To fix that once: +

+
    +
  1. + Download the local certificate + and allow the profile. +
    2. +
  2. Settings β€Ί General β€Ί VPN & Device Management: install it.
    3. +
  3. + Settings β€Ί General β€Ί About β€Ί Certificate Trust Settings: turn on + full trust for "God's Eye View Local VCam CA". +
    4. +
+

The certificate only lets your phone trust this computer.

+
+
+ + + + +
+ + + diff --git a/server/vcam/phone/orientation.js b/server/vcam/phone/orientation.js new file mode 100644 index 000000000..4490e41a6 --- /dev/null +++ b/server/vcam/phone/orientation.js @@ -0,0 +1,45 @@ +/** + * Device orientation to the virtual camera's Y-up pose frame (+x right, + * +y up, -z forward), shared by the phone page and its tests. + */ +export const quat = { + multiply(a, b) { + const [ax, ay, az, aw] = a; + const [bx, by, bz, bw] = b; + return [ + aw * bx + ax * bw + ay * bz - az * by, + aw * by - ax * bz + ay * bw + az * bx, + aw * bz + ax * by - ay * bx + az * bw, + aw * bw - ax * bx - ay * by - az * bz, + ]; + }, + axisAngle(axis, angle) { + const s = Math.sin(angle / 2); + return [axis[0] * s, axis[1] * s, axis[2] * s, Math.cos(angle / 2)]; + }, +}; +const DEG = Math.PI / 180; +// Device orientation is deviceβ†’earth (x east, y north, z up). Rotating -90Β° +// about x maps that earth frame to the Y-up frame (north becomes -z). +const EARTH_TO_Y_UP = quat.axisAngle([1, 0, 0], -Math.PI / 2); + +/** W3C DeviceOrientation (Z-X'-Y'') to a Y-up quaternion for the current screen orientation. */ +export function orientationQuaternion(alpha, beta, gamma, screenAngle) { + const x = (beta || 0) * DEG; + const y = (gamma || 0) * DEG; + const z = (alpha || 0) * DEG; + const cX = Math.cos(x / 2); + const cY = Math.cos(y / 2); + const cZ = Math.cos(z / 2); + const sX = Math.sin(x / 2); + const sY = Math.sin(y / 2); + const sZ = Math.sin(z / 2); + const device = [ + sX * cY * cZ - cX * sY * sZ, + cX * sY * cZ + sX * cY * sZ, + cX * cY * sZ + sX * sY * cZ, + cX * cY * cZ - sX * sY * sZ, + ]; + const screen = quat.axisAngle([0, 0, 1], -(screenAngle || 0) * DEG); + return quat.multiply(EARTH_TO_Y_UP, quat.multiply(device, screen)); +} diff --git a/server/vcam/phone/orientation.test.mjs b/server/vcam/phone/orientation.test.mjs new file mode 100644 index 000000000..f69ac1370 --- /dev/null +++ b/server/vcam/phone/orientation.test.mjs @@ -0,0 +1,39 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { orientationQuaternion } from './orientation.js'; +import { rotateByQuaternion } from '../../../src/vcam/rig.js'; + +const close = (actual, expected, message) => + actual.forEach((value, i) => + assert.ok(Math.abs(value - expected[i]) < 1e-9, `${message}: ${actual} vs ${expected}`), + ); +const axes = (q) => ({ + forward: rotateByQuaternion(q, [0, 0, -1]), + up: rotateByQuaternion(q, [0, 1, 0]), +}); + +test('an upright portrait phone looks out of its back along the Y-up forward axis', () => { + // beta 90: stood up from flat, screen toward the user, back toward north. + const { forward, up } = axes(orientationQuaternion(0, 90, 0, 0)); + close(forward, [0, 0, -1], 'faces north (-z)'); + close(up, [0, 1, 0], 'top edge up'); +}); + +test('alpha turns the phone counterclockwise seen from above', () => { + const { forward } = axes(orientationQuaternion(90, 90, 0, 0)); + close(forward, [-1, 0, 0], 'faces west'); +}); + +test('a landscape screen reports the picture\'s up, not the device\'s top edge', () => { + // Rotated a quarter turn counterclockwise: the top edge points west and the + // right edge up, the back still faces north, and the screen angle is 90. + const { forward, up } = axes(orientationQuaternion(90, 0, -90, 90)); + close(forward, [0, 0, -1], 'still faces north'); + close(up, [0, 1, 0], 'picture up is world up'); +}); + +test('a phone lying flat, screen up, looks straight down', () => { + const { forward, up } = axes(orientationQuaternion(0, 0, 0, 0)); + close(forward, [0, -1, 0], 'down'); + close(up, [0, 0, -1], 'top edge toward north'); +}); diff --git a/server/vcam/phone/phone.css b/server/vcam/phone/phone.css new file mode 100644 index 000000000..cce0381ce --- /dev/null +++ b/server/vcam/phone/phone.css @@ -0,0 +1,371 @@ +:root { + --bg: #0a0a0f; + --glass: rgba(12, 12, 20, 0.72); + --border: rgba(255, 255, 255, 0.12); + --accent: #00d4ff; + --accent-dim: rgba(0, 212, 255, 0.18); + --text: #e8eaed; + --muted: rgba(232, 234, 237, 0.6); + --rec: #ff3b4e; + --mono: 'SF Mono', ui-monospace, 'JetBrains Mono', Menlo, monospace; + --sans: -apple-system, BlinkMacSystemFont, 'Inter', system-ui, sans-serif; + color-scheme: dark; +} +* { + box-sizing: border-box; + -webkit-tap-highlight-color: transparent; +} +html, +body { + margin: 0; + height: 100%; + background: var(--bg); + color: var(--text); + font-family: var(--sans); + overscroll-behavior: none; +} +[hidden] { + display: none !important; +} +button { + font: inherit; + color: inherit; + cursor: pointer; +} +.screen { + min-height: 100%; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 14px; + padding: max(24px, env(safe-area-inset-top)) 24px + max(24px, env(safe-area-inset-bottom)); + text-align: center; +} +.logo { + width: 64px; + height: 64px; +} +h1 { + margin: 0; + font-family: var(--mono); + font-size: 20px; + letter-spacing: 0.18em; +} +.lede, +.hint, +.status { + margin: 0; + max-width: 34ch; + color: var(--muted); + font-size: 14px; + line-height: 1.45; +} +.status:empty { + display: none; +} +.status.error { + color: #ff8a95; +} +#pair-form { + display: flex; + gap: 8px; +} +#code { + width: 9.5ch; + padding: 12px; + border: 1px solid var(--border); + border-radius: 10px; + background: var(--glass); + color: var(--text); + font: 600 22px var(--mono); + letter-spacing: 0.2em; + text-align: center; + text-transform: uppercase; +} +.primary { + min-width: 220px; + padding: 14px 20px; + border: 1px solid var(--accent); + border-radius: 10px; + background: var(--accent-dim); + font: 600 15px var(--mono); + letter-spacing: 0.12em; +} +#pair-form .primary { + min-width: 0; +} +.help { + max-width: 36ch; + color: var(--muted); + font-size: 13px; + text-align: left; +} +.help summary { + cursor: pointer; + text-align: center; +} +.help a { + color: var(--accent); +} + +/* Camera monitor: the desktop's render fills the screen. */ +.camera { + position: fixed; + inset: 0; + padding: 0; + overflow: hidden; + touch-action: none; + user-select: none; + -webkit-user-select: none; + background: #000; +} +#monitor { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + object-fit: contain; + background: #000; +} +.topbar, +.bottombar { + position: absolute; + left: max(10px, env(safe-area-inset-left)); + right: max(10px, env(safe-area-inset-right)); + display: flex; + align-items: center; + gap: 8px; +} +.topbar { + top: max(10px, env(safe-area-inset-top)); +} +.bottombar { + bottom: max(12px, env(safe-area-inset-bottom)); + justify-content: center; + gap: 10px; +} +.spacer { + flex: 1; +} +.chip { + padding: 5px 9px; + border: 1px solid var(--border); + border-radius: 999px; + background: var(--glass); + font: 600 12px var(--mono); + letter-spacing: 0.06em; +} +.chip.rec { + border-color: var(--rec); + color: var(--rec); +} +.dot { + width: 10px; + height: 10px; + border-radius: 50%; + background: #777; + box-shadow: 0 0 0 3px rgba(0, 0, 0, 0.4); +} +.dot.live { + background: #3ddc84; +} +.dot.slow { + background: #ffb020; +} +.toast { + position: absolute; + top: calc(max(10px, env(safe-area-inset-top)) + 40px); + left: 50%; + transform: translateX(-50%); + margin: 0; + padding: 7px 12px; + border-radius: 10px; + background: var(--glass); + font-size: 13px; + white-space: nowrap; +} +.tool { + min-width: 64px; + padding: 10px 10px; + border: 1px solid var(--border); + border-radius: 10px; + background: var(--glass); + font: 600 11px var(--mono); + letter-spacing: 0.08em; +} +.tool[aria-pressed='true'], +.tool[aria-expanded='true'] { + border-color: var(--accent); + background: var(--accent-dim); +} +.record { + display: grid; + place-items: center; + width: 64px; + height: 64px; + border: 3px solid #fff; + border-radius: 50%; + background: rgba(0, 0, 0, 0.35); +} +.record-dot { + width: 44px; + height: 44px; + border-radius: 50%; + background: var(--rec); + transition: + border-radius 150ms, + width 150ms, + height 150ms; +} +.record[aria-pressed='true'] .record-dot { + width: 24px; + height: 24px; + border-radius: 5px; +} +.sr { + position: absolute; + width: 1px; + height: 1px; + overflow: hidden; + clip: rect(0 0 0 0); +} + +/* Virtual sticks sit low on each side, under the thumbs. */ +.stick { + position: absolute; + bottom: calc(max(12px, env(safe-area-inset-bottom)) + 84px); + width: 128px; + height: 128px; + border: 1px solid var(--border); + border-radius: 50%; + background: rgba(12, 12, 20, 0.38); + touch-action: none; +} +.stick.left { + left: max(18px, env(safe-area-inset-left)); +} +.stick.right { + right: max(18px, env(safe-area-inset-right)); +} +.stick span { + position: absolute; + bottom: -20px; + left: 0; + right: 0; + color: var(--muted); + font: 600 10px var(--mono); + letter-spacing: 0.1em; + text-align: center; +} +.knob { + position: absolute; + top: 50%; + left: 50%; + width: 52px; + height: 52px; + margin: -26px 0 0 -26px; + border: 1px solid var(--accent); + border-radius: 50%; + background: var(--accent-dim); +} + +.drawer { + position: absolute; + left: 50%; + bottom: calc(max(12px, env(safe-area-inset-bottom)) + 84px); + transform: translateX(-50%); + width: min(360px, calc(100% - 24px)); + max-height: 60%; + overflow: auto; + padding: 14px; + border: 1px solid var(--border); + border-radius: 16px; + background: rgba(12, 12, 20, 0.9); + display: flex; + flex-direction: column; + gap: 12px; + font: 600 11px var(--mono); + letter-spacing: 0.08em; + z-index: 2; +} +.drawer label { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + gap: 6px; +} +.drawer input[type='range'] { + width: 100%; + accent-color: var(--accent); +} +.drawer output { + color: var(--accent); +} +.presets { + display: flex; + gap: 6px; +} +.presets button { + flex: 1; + padding: 8px 2px; + border: 1px solid var(--border); + border-radius: 8px; + background: transparent; + font: 600 12px var(--mono); +} +.presets button.active { + border-color: var(--accent); + background: var(--accent-dim); +} +.presets small { + display: block; + margin-top: 2px; + color: var(--muted); + font-size: 8px; +} +.toggles { + display: flex; + flex-wrap: wrap; + gap: 14px; +} +.toggles label { + justify-content: flex-start; + align-items: center; +} +.drawer .hint { + font: 12px var(--sans); + letter-spacing: 0; +} +#focus-ring { + position: absolute; + width: 64px; + height: 64px; + margin: -32px 0 0 -32px; + border: 2px solid #ffd60a; + border-radius: 6px; + pointer-events: none; + animation: focus 700ms ease-out forwards; +} +@keyframes focus { + from { + transform: scale(1.5); + opacity: 1; + } + to { + transform: scale(1); + opacity: 0; + } +} +@media (orientation: landscape) and (max-height: 500px) { + .stick { + width: 112px; + height: 112px; + bottom: calc(max(12px, env(safe-area-inset-bottom)) + 24px); + } + .bottombar { + left: 160px; + right: 160px; + } +} diff --git a/server/vcam/phone/phone.js b/server/vcam/phone/phone.js new file mode 100644 index 000000000..b9e79d542 --- /dev/null +++ b/server/vcam/phone/phone.js @@ -0,0 +1,770 @@ +/** + * God's Eye View virtual camera β€” phone side. + * + * Streams this phone's pose to the desktop app and shows the app's render as + * a monitor. Tracking uses WebXR `immersive-ar` (6DoF, Android Chrome with + * ARCore) when available, otherwise device orientation (3DoF, iPhone Safari) + * with on-screen sticks for movement. Poses are sent in one Y-up frame for + * both: +x right, +y up, -z forward (WebXR's convention); yaw is arbitrary + * and the desktop aligns it on recenter. + * + * Transport: a WebSocket when the connection allows it, otherwise HTTPS + * requests (iOS refuses WebSockets under a tapped-through certificate). The + * monitor always pulls the newest frame, so a slow network drops frames + * instead of building a backlog. + */ + +import { orientationQuaternion } from './orientation.js'; + +const $ = (id) => document.getElementById(id); +const KEY_STORAGE = 'gevVcam.phoneKey'; +const CODE_STORAGE = 'gevVcam.code'; +const SCALE_PRESETS = [1, 10, 100, 1000, 10000]; +const FOCAL_RANGE = [12, 400]; +const FSTOP_RANGE = [1.2, 22]; +const STICK_DEADZONE = 0.12; + +const store = { + get(key) { + try { + return sessionStorage.getItem(key); + } catch { + return null; + } + }, + set(key, value) { + try { + if (value === null) sessionStorage.removeItem(key); + else sessionStorage.setItem(key, value); + } catch { + // Private browsing: pairing still works for this page load. + } + }, +}; + +const logMap = ([low, high], v) => low * (high / low) ** v; +const logUnmap = ([low, high], value) => + Math.log(value / low) / Math.log(high / low); +const round = (value, places = 5) => { + const k = 10 ** places; + return Math.round(value * k) / k; +}; +const formatScale = (scale) => + scale >= 1000 + ? `1:${+(scale / 1000).toFixed(scale >= 10000 ? 0 : 1)}K` + : `1:${+scale.toFixed(scale < 10 ? 1 : 0)}`; +const formatTime = (sec) => { + const s = Math.max(0, Math.floor(sec)); + return `${String(Math.floor(s / 60)).padStart(2, '0')}:${String(s % 60).padStart(2, '0')}`; +}; + +const screenAngle = () => + screen.orientation?.angle ?? (Number(window.orientation) || 0); + +// ------------------------------------------------------------------ transport +class Link { + constructor(key, { onMessage, onStatus, onUnpaired }) { + this.key = key; + this.onMessage = onMessage; + this.onStatus = onStatus; + this.onUnpaired = onUnpaired; + this.ws = null; + this.events = null; + this.mode = 'connecting'; + this.queue = []; + this.pendingPose = null; + this.inFlight = false; + this.closed = false; + this.webSocketFailed = false; + this.retryTimer = null; + } + + async start() { + if (this.closed) return; + this.onStatus('connecting'); + if (!this.webSocketFailed && (await this.openWebSocket())) return; + this.webSocketFailed = true; + this.openHttp(); + } + + openWebSocket() { + return new Promise((resolve) => { + let settled = false; + let ws; + try { + ws = new WebSocket( + `wss://${location.host}/ws?key=${encodeURIComponent(this.key)}`, + ); + } catch { + resolve(false); + return; + } + const done = (ok) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(ok); + }; + const timer = setTimeout(() => { + ws.close(); + done(false); + }, 3500); + ws.onopen = () => { + this.ws = ws; + this.mode = 'ws'; + this.onStatus('live'); + done(true); + }; + ws.onerror = () => done(false); + ws.onmessage = (event) => this.receive(event.data); + ws.onclose = (event) => { + done(false); + if (this.ws !== ws) return; + this.ws = null; + this.mode = 'connecting'; + this.handleClose(event.code); + }; + }); + } + + openHttp() { + if (this.closed) return; + this.mode = 'http'; + const events = new EventSource( + `/api/events?key=${encodeURIComponent(this.key)}`, + ); + this.events = events; + events.onopen = () => this.onStatus('slow'); + events.onmessage = (event) => this.receive(event.data); + events.onerror = async () => { + if (events.readyState !== EventSource.CLOSED || this.closed) return; + this.events = null; + // A closed stream is either a lost network or an ended session. + try { + const probe = await fetch('/api/send', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Vcam-Key': this.key, + }, + body: '[]', + cache: 'no-store', + }); + if (probe.status === 403) return this.handleClose(4003); + } catch { + // Offline: retry below. + } + this.handleClose(1006); + }; + this.flush(); + } + + handleClose(code) { + if (this.closed) return; + if (code === 4003 || code === 4000) { + this.onUnpaired(code === 4000 ? 'ended' : 'unpaired'); + return; + } + if (code === 4001) { + this.onStatus('replaced'); + this.close(); + return; + } + this.onStatus('connecting'); + clearTimeout(this.retryTimer); + this.retryTimer = setTimeout(() => this.start(), 1000); + } + + receive(text) { + try { + this.onMessage(JSON.parse(text)); + } catch { + // Ignore malformed relay input. + } + } + + /** Poses are latest-wins; every other message is delivered in order. */ + send(message) { + if (this.closed) return; + // 1 is WebSocket.OPEN. + if (this.ws && this.ws.readyState === 1) { + this.ws.send(JSON.stringify(message)); + return; + } + if (message.type === 'pose') this.pendingPose = message; + else this.queue.push(message); + if (this.mode === 'http') this.flush(); + } + + async flush() { + if (this.inFlight || this.mode !== 'http' || this.closed) return; + const batch = this.queue.splice(0, 32); + if (this.pendingPose) batch.push(this.pendingPose); + this.pendingPose = null; + if (!batch.length) return; + this.inFlight = true; + try { + const response = await fetch('/api/send', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Vcam-Key': this.key, + }, + body: JSON.stringify(batch), + cache: 'no-store', + }); + if (response.status === 403) this.handleClose(4003); + } catch { + // The event stream's error handling owns reconnection. + } finally { + this.inFlight = false; + } + if (this.queue.length || this.pendingPose) this.flush(); + } + + close() { + this.closed = true; + clearTimeout(this.retryTimer); + this.ws?.close(); + this.events?.close(); + this.ws = null; + this.events = null; + } +} + +// --------------------------------------------------------------------- state +const app = { + key: store.get(KEY_STORAGE), + link: null, + tracking: null, // { mode: '6dof'|'3dof', stop() } + orientation: null, + sticks: { left: [0, 0], right: [0, 0] }, + host: { + connected: false, + recording: false, + elapsedSec: 0, + scale: 1, + focalMm: 31.18, + fStop: 2.8, + dof: false, + held: false, + smoothing: 0.3, + heightLock: false, + horizonLock: false, + }, + recordingSince: 0, + monitorRunning: false, + wakeLock: null, +}; + +function show(screenId) { + for (const id of ['pair', 'start', 'camera']) $(id).hidden = id !== screenId; +} + +function setStatus(element, text, error = false) { + element.textContent = text || ''; + element.classList.toggle('error', !!error); +} + +let toastTimer = null; +function toast(text) { + const element = $('camera-status'); + element.textContent = text; + element.hidden = false; + clearTimeout(toastTimer); + toastTimer = setTimeout(() => (element.hidden = true), 2600); +} + +// ------------------------------------------------------------------- pairing +async function join(code) { + setStatus($('pair-status'), 'Pairing…'); + try { + const response = await fetch('/api/join', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ code }), + cache: 'no-store', + }); + if (response.status === 403) { + setStatus( + $('pair-status'), + 'That code is not active. Check the code on the desktop.', + true, + ); + return false; + } + if (!response.ok) throw new Error(String(response.status)); + const { phoneKey } = await response.json(); + app.key = phoneKey; + store.set(KEY_STORAGE, phoneKey); + store.set(CODE_STORAGE, code); + setStatus($('pair-status'), ''); + connect(); + showStart(); + return true; + } catch { + setStatus( + $('pair-status'), + 'Could not reach the desktop. Is it on the same network?', + true, + ); + return false; + } +} + +function connect() { + app.link?.close(); + app.link = new Link(app.key, { + onMessage: handleMessage, + onStatus: presentLink, + onUnpaired: async (reason) => { + app.link?.close(); + app.link = null; + store.set(KEY_STORAGE, null); + stopTracking(); + const code = store.get(CODE_STORAGE); + if (reason === 'unpaired' && code && (await join(code))) return; + show('pair'); + setStatus( + $('pair-status'), + reason === 'ended' + ? 'The desktop ended this session. Enter a new code.' + : 'Pairing expired. Enter the code shown on the desktop.', + true, + ); + }, + }); + app.link.start(); +} + +function presentLink(status) { + const dot = $('link-dot'); + dot.classList.toggle('live', status === 'live'); + dot.classList.toggle('slow', status === 'slow'); + dot.title = + status === 'live' + ? 'Live connection' + : status === 'slow' + ? 'Fallback connection (install the certificate for lower latency)' + : 'Connecting…'; + if (status === 'replaced') toast('Another phone took over this session'); +} + +async function showStart() { + show('start'); + const xr = await xrSupported(); + $('start-xr').hidden = !xr; + $('support').textContent = xr + ? 'This phone supports AR tracking. Point it like a camera.' + : 'AR tracking is not available in this browser, so the gyro camera tracks rotation and the sticks move you.'; +} + +async function xrSupported() { + try { + return !!(await navigator.xr?.isSessionSupported('immersive-ar')); + } catch { + return false; + } +} + +// ------------------------------------------------------------------ tracking +function sendPose(q, p) { + const { left, right } = app.sticks; + app.link?.send({ + type: 'pose', + t: Math.round(performance.now()), + mode: app.tracking?.mode, + q: q.map((v) => round(v)), + p: p ? p.map((v) => round(v, 4)) : null, + j: [left[0], left[1], right[0], right[1]].map((v) => round(v, 3)), + }); +} + +async function startXr() { + setStatus($('start-status'), 'Starting AR…'); + try { + const root = $('camera'); + const session = await navigator.xr.requestSession('immersive-ar', { + requiredFeatures: ['local'], + optionalFeatures: ['dom-overlay'], + domOverlay: { root }, + }); + // Screen taps on controls must not become XR select input. + root.addEventListener('beforexrselect', (event) => event.preventDefault()); + const canvas = document.createElement('canvas'); + const gl = canvas.getContext('webgl', { xrCompatible: true, alpha: true }); + await gl.makeXRCompatible?.(); + session.updateRenderState({ baseLayer: new XRWebGLLayer(session, gl) }); + const space = await session.requestReferenceSpace('local'); + let stopped = false; + const onFrame = (_time, frame) => { + if (stopped) return; + session.requestAnimationFrame(onFrame); + const pose = frame.getViewerPose(space); + if (pose) { + const { position, orientation } = pose.transform; + sendPose( + [orientation.x, orientation.y, orientation.z, orientation.w], + pose.emulatedPosition ? null : [position.x, position.y, position.z], + ); + } + const layer = session.renderState.baseLayer; + gl.bindFramebuffer(gl.FRAMEBUFFER, layer.framebuffer); + gl.clearColor(0, 0, 0, 0); + gl.clear(gl.COLOR_BUFFER_BIT); + }; + session.requestAnimationFrame(onFrame); + session.addEventListener('end', () => { + stopped = true; + if (app.tracking?.session === session) { + app.tracking = null; + showStart(); + } + }); + app.tracking = { + mode: '6dof', + session, + stop: () => session.end().catch(() => {}), + }; + enterCamera(); + } catch (error) { + setStatus( + $('start-status'), + `AR could not start (${error?.message || error}). Try the gyro camera.`, + true, + ); + } +} + +async function startGyro() { + // iOS asks for motion access only from a tap, before anything else awaits. + try { + if (typeof DeviceOrientationEvent?.requestPermission === 'function') { + const answer = await DeviceOrientationEvent.requestPermission(); + if (answer !== 'granted') { + setStatus( + $('start-status'), + 'Motion access was denied. Allow it in Safari settings for this site, then reload.', + true, + ); + return; + } + } + } catch { + setStatus($('start-status'), 'Motion access is unavailable.', true); + return; + } + let stopped = false; + const onOrientation = (event) => { + if (event.alpha === null && event.beta === null) return; + app.orientation = orientationQuaternion( + event.alpha, + event.beta, + event.gamma, + screenAngle(), + ); + }; + window.addEventListener('deviceorientation', onOrientation); + const tick = () => { + if (stopped) return; + requestAnimationFrame(tick); + if (app.orientation) sendPose(app.orientation, null); + }; + requestAnimationFrame(tick); + app.tracking = { + mode: '3dof', + stop: () => { + stopped = true; + window.removeEventListener('deviceorientation', onOrientation); + }, + }; + document.documentElement.requestFullscreen?.().catch(() => {}); + enterCamera(); + setTimeout(() => { + if (!app.orientation && !stopped) + toast('No motion data yet β€” this browser may block the gyroscope'); + }, 1500); +} + +function stopTracking() { + app.tracking?.stop(); + app.tracking = null; + app.monitorRunning = false; +} + +async function enterCamera() { + show('camera'); + $('mode-readout').textContent = + app.tracking?.mode === '6dof' ? 'AR 6DOF' : 'GYRO 3DOF'; + app.link?.send({ + type: 'hello', + mode: app.tracking?.mode, + screen: [window.innerWidth, window.innerHeight], + }); + try { + app.wakeLock = await navigator.wakeLock?.request('screen'); + } catch { + // Optional: the screen may dim during long takes. + } + runMonitor(); +} + +// ------------------------------------------------------------------- monitor +async function runMonitor() { + if (app.monitorRunning) return; + app.monitorRunning = true; + const monitor = $('monitor'); + let seq = 0; + let previousUrl = null; + while (app.monitorRunning && app.key) { + if (document.hidden) { + await new Promise((resolve) => setTimeout(resolve, 500)); + continue; + } + try { + const response = await fetch( + `/api/frame?key=${encodeURIComponent(app.key)}&after=${seq}`, + { cache: 'no-store' }, + ); + if (response.status === 403) break; + if (response.status !== 200) continue; + seq = Number(response.headers.get('X-Frame-Seq')) || seq + 1; + const url = URL.createObjectURL(await response.blob()); + await new Promise((resolve) => { + monitor.onload = resolve; + monitor.onerror = resolve; + monitor.src = url; + }); + if (previousUrl) URL.revokeObjectURL(previousUrl); + previousUrl = url; + } catch { + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } + app.monitorRunning = false; +} + +// ------------------------------------------------------------------ controls +function stick(element, target) { + const knob = element.querySelector('.knob'); + let pointer = null; + const set = (x, y) => { + target[0] = x; + target[1] = y; + const radius = element.clientWidth / 2 - 26; + knob.style.transform = `translate(${x * radius}px, ${-y * radius}px)`; + }; + const move = (event) => { + const rect = element.getBoundingClientRect(); + const radius = rect.width / 2; + let x = (event.clientX - rect.left - radius) / radius; + let y = -(event.clientY - rect.top - radius) / radius; + const length = Math.hypot(x, y); + if (length > 1) { + x /= length; + y /= length; + } + // Rescale past the dead zone so small nudges stay precise. + const live = Math.min(1, Math.hypot(x, y)); + const scale = + live < STICK_DEADZONE + ? 0 + : (live - STICK_DEADZONE) / (1 - STICK_DEADZONE) / live; + set(x * scale, y * scale); + }; + element.addEventListener('pointerdown', (event) => { + if (pointer !== null) return; + pointer = event.pointerId; + element.setPointerCapture(pointer); + move(event); + }); + element.addEventListener('pointermove', (event) => { + if (event.pointerId === pointer) move(event); + }); + const release = (event) => { + if (event.pointerId !== pointer) return; + pointer = null; + set(0, 0); + }; + element.addEventListener('pointerup', release); + element.addEventListener('pointercancel', release); +} + +const sendSet = (key, value) => app.link?.send({ type: 'set', key, value }); +const sendCommand = (command) => app.link?.send({ type: 'cmd', cmd: command }); + +function presentHost() { + const h = app.host; + $('record-btn').setAttribute('aria-pressed', String(h.recording)); + $('rec-readout').hidden = !h.recording; + $('hold-btn').setAttribute('aria-pressed', String(h.held)); + $('scale-readout').textContent = formatScale(h.scale); + $('scale-value').textContent = formatScale(h.scale); + $('focal-readout').textContent = `${Math.round(h.focalMm)}mm`; + $('focal-value').textContent = `${Math.round(h.focalMm)}mm`; + $('fstop-value').textContent = `f/${h.fStop.toFixed(1)}`; + $('smooth-value').textContent = `${Math.round(h.smoothing * 100)}%`; + const active = (element) => + element === document.activeElement && element.matches(':active'); + if (!active($('scale-range'))) + $('scale-range').value = String(Math.log10(h.scale)); + if (!active($('focal-range'))) + $('focal-range').value = String(logUnmap(FOCAL_RANGE, h.focalMm)); + if (!active($('fstop-range'))) + $('fstop-range').value = String(logUnmap(FSTOP_RANGE, h.fStop)); + if (!active($('smooth-range'))) + $('smooth-range').value = String(Math.round(h.smoothing * 100)); + $('dof-toggle').checked = h.dof; + $('height-lock').checked = h.heightLock; + $('horizon-lock').checked = h.horizonLock; + for (const button of $('scale-presets').querySelectorAll('button')) + button.classList.toggle( + 'active', + Math.abs(Number(button.dataset.scale) - h.scale) < 1e-6, + ); + for (const button of $('focal-presets').querySelectorAll('button')) + button.classList.toggle( + 'active', + Math.abs(Number(button.dataset.focal) - h.focalMm) < 0.5, + ); +} + +function handleMessage(message) { + if (message.type === 'state') { + const wasRecording = app.host.recording; + Object.assign(app.host, message.state); + if (app.host.recording && !wasRecording) + app.recordingSince = performance.now() - app.host.elapsedSec * 1000; + presentHost(); + } else if (message.type === 'peer' && message.role === 'host') { + app.host.connected = message.connected; + if (!message.connected) toast('Desktop disconnected β€” waiting…'); + } else if (message.type === 'toast' && typeof message.text === 'string') { + toast(message.text.slice(0, 140)); + } +} + +function toggleDrawer(name) { + for (const other of ['scale', 'lens']) { + const open = other === name && $(`${other}-drawer`).hidden; + $(`${other}-drawer`).hidden = !open; + $(`${other}-btn`).setAttribute('aria-expanded', String(open)); + } +} + +function bindControls() { + $('pair-form').addEventListener('submit', (event) => { + event.preventDefault(); + const code = $('code').value.trim().toUpperCase(); + if (code) join(code); + }); + $('start-xr').addEventListener('click', startXr); + $('start-gyro').addEventListener('click', startGyro); + stick($('stick-left'), app.sticks.left); + stick($('stick-right'), app.sticks.right); + $('record-btn').addEventListener('click', () => + sendCommand(app.host.recording ? 'stop' : 'record'), + ); + $('hold-btn').addEventListener('click', () => + sendCommand(app.host.held ? 'release' : 'hold'), + ); + $('recenter-btn').addEventListener('click', () => sendCommand('recenter')); + $('scale-btn').addEventListener('click', () => toggleDrawer('scale')); + $('lens-btn').addEventListener('click', () => toggleDrawer('lens')); + $('scale-presets').addEventListener('click', (event) => { + const value = Number(event.target.closest('button')?.dataset.scale); + if (SCALE_PRESETS.includes(value)) sendSet('scale', value); + }); + $('scale-range').addEventListener('input', (event) => { + const scale = 10 ** Number(event.target.value); + $('scale-value').textContent = formatScale(scale); + sendSet('scale', round(scale, 3)); + }); + $('smooth-range').addEventListener('input', (event) => + sendSet('smoothing', Number(event.target.value) / 100), + ); + $('height-lock').addEventListener('change', (event) => + sendSet('heightLock', event.target.checked), + ); + $('horizon-lock').addEventListener('change', (event) => + sendSet('horizonLock', event.target.checked), + ); + $('focal-range').addEventListener('input', (event) => { + const focal = logMap(FOCAL_RANGE, Number(event.target.value)); + $('focal-value').textContent = `${Math.round(focal)}mm`; + sendSet('focalMm', round(focal, 2)); + }); + $('focal-presets').addEventListener('click', (event) => { + const value = Number(event.target.closest('button')?.dataset.focal); + if (value) sendSet('focalMm', value); + }); + $('dof-toggle').addEventListener('change', (event) => + sendSet('dof', event.target.checked), + ); + $('fstop-range').addEventListener('input', (event) => { + const fStop = logMap(FSTOP_RANGE, Number(event.target.value)); + $('fstop-value').textContent = `f/${fStop.toFixed(1)}`; + sendSet('fStop', round(fStop, 2)); + }); + // Tap the monitor to focus: coordinates are within the shown picture. + $('monitor').addEventListener('click', (event) => { + const monitor = $('monitor'); + if (!monitor.naturalWidth) return; + const rect = monitor.getBoundingClientRect(); + const fit = Math.min( + rect.width / monitor.naturalWidth, + rect.height / monitor.naturalHeight, + ); + const width = monitor.naturalWidth * fit; + const height = monitor.naturalHeight * fit; + const x = (event.clientX - rect.left - (rect.width - width) / 2) / width; + const y = (event.clientY - rect.top - (rect.height - height) / 2) / height; + if (x < 0 || x > 1 || y < 0 || y > 1) return; + app.link?.send({ type: 'focus', x: round(x, 4), y: round(y, 4) }); + const ring = $('focus-ring'); + ring.hidden = true; + ring.style.left = `${event.clientX}px`; + ring.style.top = `${event.clientY}px`; + void ring.offsetWidth; + ring.hidden = false; + }); + setInterval(() => { + if (app.host.recording) + $('rec-readout').textContent = + `● ${formatTime((performance.now() - app.recordingSince) / 1000)}`; + }, 250); + // The system releases a wake lock whenever the page is hidden. + document.addEventListener('visibilitychange', async () => { + if (document.hidden || !app.tracking) return; + if (app.wakeLock && !app.wakeLock.released) return; + try { + app.wakeLock = await navigator.wakeLock?.request('screen'); + } catch { + // Optional. + } + }); +} + +function boot() { + bindControls(); + presentHost(); + const fromLink = decodeURIComponent(location.hash.slice(1)) + .toUpperCase() + .replace(/[^0-9A-Z]/g, ''); + if (fromLink) { + $('code').value = fromLink; + store.set(CODE_STORAGE, fromLink); + // Keep the code out of the address bar once read. + history.replaceState(null, '', location.pathname); + } + if (app.key) { + connect(); + showStart(); + } else if (fromLink) { + join(fromLink); + } else { + show('pair'); + } +} + +boot(); diff --git a/server/vcam/plugin.js b/server/vcam/plugin.js new file mode 100644 index 000000000..261114eef --- /dev/null +++ b/server/vcam/plugin.js @@ -0,0 +1,522 @@ +/** + * Virtual camera: lets a phone on the same network act as the app's camera. + * + * Two surfaces, deliberately separate: + * + * - On the app's own server (loopback only, behind the same gate as /mcp): + * `POST /api/vcam/session` creates a pairing session, `DELETE` ends it, + * `GET /api/vcam/ca.crt` downloads the local CA, and the desktop page joins + * its session over the `/api/vcam/host` WebSocket. + * - A separate HTTPS listener on the LAN (VCAM_PORT, default 4174) that starts + * only when the desktop creates a session and stops after the last one + * ends. It serves the static phone page and the phone side of the relay + * and nothing else, so pairing a phone never exposes the app's provider + * proxies or API keys to the network. + * + * The phone transport prefers a WebSocket and falls back to plain HTTPS + * requests (POST for poses, server-sent events for state), which work under + * a tapped-through certificate warning where iOS refuses WebSockets. + */ +import fs from 'node:fs'; +import https from 'node:https'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { WebSocketServer } from 'ws'; +import { isAllowedHost } from '../../build/allowedHosts.js'; +import { isSharingEnabled } from '../../src/keySetupCore.mjs'; +import { isLocalMcpRequest } from '../mcp/plugin.js'; +import { MAX_FRAME_BYTES, MAX_TEXT_BYTES, createVcamRelay } from './relay.js'; +import { loadLocalCertificates } from './tls.js'; + +const PHONE_DIR = fileURLToPath(new URL('./phone/', import.meta.url)); +const PHONE_FILES = Object.freeze({ + '/': ['index.html', 'text/html; charset=utf-8'], + '/index.html': ['index.html', 'text/html; charset=utf-8'], + '/phone.js': ['phone.js', 'text/javascript; charset=utf-8'], + '/orientation.js': ['orientation.js', 'text/javascript; charset=utf-8'], + '/phone.css': ['phone.css', 'text/css; charset=utf-8'], + '/icon.svg': ['icon.svg', 'image/svg+xml'], +}); +const DEFAULT_PORT = 4174; +const MAX_BATCH_BYTES = 64 * 1024; +const BODY_TIMEOUT_MS = 10000; +const SSE_KEEPALIVE_MS = 15000; +// The listener lingers after the last session in case the desktop reloads. +const IDLE_SHUTDOWN_MS = 120000; +const CA_CONTENT_TYPE = 'application/x-x509-ca-cert'; + +/** Headers for every phone-listener response; the page is static. */ +export const PHONE_HEADERS = Object.freeze({ + 'Content-Security-Policy': [ + "default-src 'self'", + "script-src 'self'", + "style-src 'self'", + "img-src 'self' blob: data:", + "connect-src 'self' wss:", + "base-uri 'none'", + "form-action 'none'", + "frame-ancestors 'none'", + "object-src 'none'", + ].join('; '), + 'X-Frame-Options': 'DENY', + 'X-Content-Type-Options': 'nosniff', + 'Referrer-Policy': 'no-referrer', + 'Cache-Control': 'no-store', + 'Permissions-Policy': 'xr-spatial-tracking=(self), camera=(self)', +}); + +/** + * IPv4 addresses a phone on the same network could reach, private ranges + * first (home and office Wi-Fi), then others. Loopback and link-local are + * left out. + */ +export function lanAddresses(interfaces = os.networkInterfaces()) { + const rank = (address) => + address.startsWith('192.168.') + ? 0 + : address.startsWith('10.') + ? 1 + : /^172\.(1[6-9]|2\d|3[01])\./.test(address) + ? 2 + : 3; + const found = []; + for (const entries of Object.values(interfaces || {})) + for (const entry of entries || []) { + const family = entry.family === 4 ? 'IPv4' : entry.family; + if (family !== 'IPv4' || entry.internal) continue; + if (entry.address.startsWith('169.254.')) continue; + if (!found.includes(entry.address)) found.push(entry.address); + } + return found.sort((a, b) => rank(a) - rank(b)); +} + +const json = (res, status, body) => { + res.statusCode = status; + res.setHeader('Content-Type', 'application/json; charset=utf-8'); + res.setHeader('Cache-Control', 'no-store'); + res.end(JSON.stringify(body)); +}; + +function readBody(req, limit) { + return new Promise((resolve, reject) => { + const chunks = []; + let size = 0; + const timer = setTimeout(() => { + reject(Object.assign(new Error('timeout'), { status: 408 })); + req.destroy(); + }, BODY_TIMEOUT_MS); + req.on('data', (chunk) => { + size += chunk.length; + if (size > limit) { + clearTimeout(timer); + reject(Object.assign(new Error('too large'), { status: 413 })); + req.destroy(); + return; + } + chunks.push(chunk); + }); + req.on('end', () => { + clearTimeout(timer); + resolve(Buffer.concat(chunks).toString('utf8')); + }); + req.on('error', (error) => { + clearTimeout(timer); + reject(error); + }); + }); +} + +/** Whether a desktop request may control pairing (loopback, same-origin). */ +function isDesktopRequest(req, env, { requireOrigin = true } = {}) { + if (requireOrigin && !req.headers.origin) return false; + return isLocalMcpRequest({ + remoteAddress: req.socket?.remoteAddress, + localPort: req.socket?.localPort, + host: req.headers.host || '', + origin: req.headers.origin, + headers: req.headers, + env, + }); +} + +/** + * Create the virtual-camera service: the relay, the lazily started phone + * listener and the handlers both servers mount. + */ +export function createVcamService({ + env = process.env, + cacheDir = path.join(process.cwd(), '.gev-cache', 'vcam'), + port = Number.parseInt(env.VCAM_PORT, 10) || DEFAULT_PORT, + host = env.VCAM_HOST || '0.0.0.0', + addresses = () => lanAddresses(), + allowedHosts = [], + logger = console, +} = {}) { + let listener = null; + let listenerNames = ''; + let starting = null; + let idleTimer = null; + let certificates = null; + const relay = createVcamRelay({ + onSessionClosed: () => { + if (relay.size === 0) scheduleIdleShutdown(); + }, + }); + const sockets = new WebSocketServer({ + noServer: true, + maxPayload: MAX_FRAME_BYTES, + }); + + function scheduleIdleShutdown() { + clearTimeout(idleTimer); + idleTimer = setTimeout(() => { + if (relay.size === 0) stopListener(); + }, IDLE_SHUTDOWN_MS); + idleTimer.unref?.(); + } + + function stopListener() { + const current = listener; + listener = null; + listenerNames = ''; + current?.closeAllConnections?.(); + current?.close(); + } + + /** Start (or restart for a changed address) the HTTPS phone listener. */ + async function ensureListener(requested) { + clearTimeout(idleTimer); + const names = [...new Set(requested)]; + const key = names.join(','); + if (listener && listenerNames === key) return; + if (starting) await starting.catch(() => {}); + if (listener && listenerNames === key) return; + stopListener(); + certificates = loadLocalCertificates(cacheDir, names); + const server = https.createServer( + { cert: certificates.cert, key: certificates.key }, + (req, res) => handlePhoneRequest(req, res), + ); + server.on('upgrade', (req, socket, head) => + handlePhoneUpgrade(req, socket, head), + ); + server.on('clientError', (_error, socket) => socket.destroy()); + starting = new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, host, () => { + server.off('error', reject); + resolve(); + }); + }); + try { + await starting; + } finally { + starting = null; + } + listener = server; + listenerNames = key; + logger.info?.( + `[vcam] phone listener on https://${names[0]}:${port}/ (pairing session only)`, + ); + } + + const phoneHostAllowed = (req) => + isAllowedHost(req.headers.host, allowedHosts); + + function servePhoneFile(req, res, pathname) { + const entry = PHONE_FILES[pathname]; + if (!entry) return false; + const [file, type] = entry; + let body; + try { + body = fs.readFileSync(path.join(PHONE_DIR, file)); + } catch { + json(res, 500, { error: 'phone page unavailable' }); + return true; + } + res.statusCode = 200; + res.setHeader('Content-Type', type); + res.end(req.method === 'HEAD' ? undefined : body); + return true; + } + + function sendCa(res) { + res.statusCode = 200; + res.setHeader('Content-Type', CA_CONTENT_TYPE); + res.setHeader( + 'Content-Disposition', + 'attachment; filename="gods-eye-view-vcam-ca.crt"', + ); + res.end(certificates?.ca || ''); + } + + function phoneSession(req, url) { + const key = req.headers['x-vcam-key'] || url.searchParams.get('key'); + return relay.sessionForPhone(key); + } + + async function handlePhoneRequest(req, res) { + for (const [name, value] of Object.entries(PHONE_HEADERS)) + res.setHeader(name, value); + if (!phoneHostAllowed(req)) return json(res, 421, { error: 'bad host' }); + let url; + try { + url = new URL(req.url, 'https://phone.invalid'); + } catch { + return json(res, 400, { error: 'bad request' }); + } + const { pathname } = url; + try { + if (req.method === 'GET' || req.method === 'HEAD') { + if (servePhoneFile(req, res, pathname)) return; + if (pathname === '/ca.crt') return sendCa(res); + } + if (pathname === '/api/join' && req.method === 'POST') { + let code = ''; + try { + code = JSON.parse(await readBody(req, 1024))?.code; + } catch (error) { + if (error?.status) throw error; + } + const joined = relay.joinPhone(code); + if (!joined.ok) return json(res, 403, { error: joined.reason }); + return json(res, 200, { + phoneKey: joined.phoneKey, + sessionId: joined.sessionId, + limits: { text: MAX_TEXT_BYTES }, + }); + } + if (pathname === '/api/send' && req.method === 'POST') { + const session = phoneSession(req, url); + if (!session) return json(res, 403, { error: 'unpaired' }); + let messages; + try { + messages = JSON.parse(await readBody(req, MAX_BATCH_BYTES)); + } catch (error) { + if (error?.status) throw error; + return json(res, 400, { error: 'expected a JSON array' }); + } + if (!Array.isArray(messages) || messages.length > 64) + return json(res, 400, { error: 'expected a JSON array' }); + let accepted = 0; + for (const message of messages) + if (relay.relay(session, 'phone', JSON.stringify(message))) + accepted += 1; + return json(res, 200, { accepted }); + } + if (pathname === '/api/events' && req.method === 'GET') + return openEventStream(req, res, phoneSession(req, url)); + if (pathname === '/api/frame' && req.method === 'GET') { + const session = phoneSession(req, url); + if (!session) return json(res, 403, { error: 'unpaired' }); + const after = Number.parseInt(url.searchParams.get('after'), 10) || 0; + const frame = await relay.nextFrame(session, after, 2000); + if (!frame) { + res.statusCode = 204; + return res.end(); + } + res.statusCode = 200; + res.setHeader('Content-Type', 'image/jpeg'); + res.setHeader('X-Frame-Seq', String(frame.seq)); + return res.end(frame.bytes); + } + json(res, 404, { error: 'not found' }); + } catch (error) { + if (!res.headersSent) + json(res, error?.status || 500, { error: 'request failed' }); + else res.end(); + } + } + + function openEventStream(req, res, session) { + if (!session) return json(res, 403, { error: 'unpaired' }); + res.statusCode = 200; + res.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); + res.setHeader('Connection', 'keep-alive'); + res.flushHeaders?.(); + const keepAlive = setInterval( + () => res.write(': keep-alive\n\n'), + SSE_KEEPALIVE_MS, + ); + const peer = { + send: (text) => res.write(`data: ${text}\n\n`), + close: () => res.end(), + }; + const attached = relay.attachPhone( + req.headers['x-vcam-key'] || + new URL(req.url, 'https://phone.invalid').searchParams.get('key'), + peer, + ); + if (!attached) { + clearInterval(keepAlive); + return res.end(); + } + req.on('close', () => { + clearInterval(keepAlive); + relay.detach(attached, 'phone', peer); + }); + } + + function refuseUpgrade(socket, status) { + socket.end(`HTTP/1.1 ${status}\r\nConnection: close\r\n\r\n`); + } + + function handlePhoneUpgrade(req, socket, head) { + let url; + try { + url = new URL(req.url, 'https://phone.invalid'); + } catch { + return refuseUpgrade(socket, '400 Bad Request'); + } + if (url.pathname !== '/ws' || !phoneHostAllowed(req)) + return refuseUpgrade(socket, '404 Not Found'); + // A WebSocket is not subject to CORS: only this page's own origin may open one. + if (req.headers.origin !== `https://${req.headers.host}`) + return refuseUpgrade(socket, '403 Forbidden'); + const key = url.searchParams.get('key'); + if (!relay.sessionForPhone(key)) + return refuseUpgrade(socket, '403 Forbidden'); + sockets.handleUpgrade(req, socket, head, (ws) => { + const peer = { + send: (text) => ws.readyState === ws.OPEN && ws.send(text), + close: (code, reason) => ws.close(code, reason), + }; + const session = relay.attachPhone(key, peer); + if (!session) return ws.close(4003, 'unpaired'); + ws.on('message', (data, isBinary) => { + if (isBinary) return; + relay.relay(session, 'phone', data.toString('utf8')); + }); + ws.on('close', () => relay.detach(session, 'phone', peer)); + ws.on('error', () => ws.terminate()); + }); + } + + /** Mount the desktop routes on the app's own (dev or preview) server. */ + function install(server) { + server.middlewares.use('/api/vcam', async (req, res, next) => { + const pathname = (req.url || '/').split('?')[0]; + try { + if (pathname === '/ca.crt' && req.method === 'GET') { + if (!isDesktopRequest(req, env, { requireOrigin: false })) + return json(res, 403, { error: 'local only' }); + if (!certificates) + certificates = loadLocalCertificates(cacheDir, [ + ...addresses(), + 'localhost', + '127.0.0.1', + ]); + return sendCa(res); + } + if (pathname !== '/session') return next(); + if (!isDesktopRequest(req, env)) + return json(res, 403, { error: 'local only' }); + if (isSharingEnabled(env)) + return json(res, 403, { error: 'sharing is enabled' }); + if (req.method === 'POST') { + const lan = addresses(); + if (!lan.length) + return json(res, 503, { + error: 'no network address', + code: 'no-network', + }); + try { + await ensureListener([...lan, 'localhost', '127.0.0.1']); + } catch (error) { + return json(res, 503, { + error: 'phone listener unavailable', + code: error?.code === 'EADDRINUSE' ? 'port-in-use' : 'listen', + port, + }); + } + const session = relay.createSession(); + return json(res, 200, { + sessionId: session.id, + hostKey: session.hostKey, + code: session.code, + port: listener.address()?.port ?? port, + addresses: lan, + caFingerprint: certificates.caFingerprint, + }); + } + if (req.method === 'DELETE') { + relay.endSession(req.headers['x-vcam-host-key']); + return json(res, 200, { ended: true }); + } + return json(res, 405, { error: 'method not allowed' }); + } catch { + if (!res.headersSent) json(res, 500, { error: 'request failed' }); + } + }); + + server.httpServer?.on('upgrade', (req, socket, head) => { + let url; + try { + url = new URL(req.url, 'http://local.invalid'); + } catch { + return; + } + if (url.pathname !== '/api/vcam/host') return; + // Browsers always send Origin on a WebSocket; it must be this app. + if (!isDesktopRequest(req, env)) + return refuseUpgrade(socket, '403 Forbidden'); + const key = url.searchParams.get('key'); + if (!relay.sessionForHost(key)) + return refuseUpgrade(socket, '403 Forbidden'); + sockets.handleUpgrade(req, socket, head, (ws) => { + const peer = { + send: (text) => ws.readyState === ws.OPEN && ws.send(text), + close: (code, reason) => ws.close(code, reason), + }; + const session = relay.attachHost(key, peer); + if (!session) return ws.close(4003, 'unknown session'); + ws.on('message', (data, isBinary) => { + if (isBinary) relay.publishFrame(session, Buffer.from(data)); + else relay.relay(session, 'host', data.toString('utf8')); + }); + ws.on('close', () => relay.detach(session, 'host', peer)); + ws.on('error', () => ws.terminate()); + }); + }); + server.httpServer?.once('close', () => destroy()); + } + + function destroy() { + clearTimeout(idleTimer); + relay.closeAll(); + stopListener(); + sockets.close(); + } + + return { + install, + destroy, + relay, + ensureListener, + get listening() { + return !!listener; + }, + get port() { + return listener?.address()?.port ?? port; + }, + }; +} + +/** Vite plugin mounting the virtual-camera desktop routes and phone listener. */ +export function vcamPlugin(options = {}) { + let service = null; + const install = (server) => { + service ??= createVcamService({ + allowedHosts: server.config?.server?.allowedHosts || [], + logger: server.config?.logger || console, + ...options, + }); + service.install(server); + }; + return { + name: 'gev-vcam', + configureServer: install, + configurePreviewServer: install, + }; +} diff --git a/server/vcam/plugin.test.mjs b/server/vcam/plugin.test.mjs new file mode 100644 index 000000000..7ea71425a --- /dev/null +++ b/server/vcam/plugin.test.mjs @@ -0,0 +1,274 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import http from 'node:http'; +import https from 'node:https'; +import os from 'node:os'; +import path from 'node:path'; +import WebSocket from 'ws'; +import { createVcamService, lanAddresses, PHONE_HEADERS } from './plugin.js'; + +/** The slice of a Vite server the service mounts on. */ +function appServer() { + const stack = []; + const httpServer = http.createServer((req, res) => { + const layer = stack.find(({ prefix }) => req.url.startsWith(prefix)); + if (!layer) { + res.statusCode = 404; + return res.end(); + } + req.url = req.url.slice(layer.prefix.length) || '/'; + layer.handle(req, res, () => { + res.statusCode = 404; + res.end(); + }); + }); + return { + httpServer, + middlewares: { use: (prefix, handle) => stack.push({ prefix, handle }) }, + }; +} + +const listen = (server) => + new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + +function request(url, { method = 'GET', headers = {}, body, ca, checkServerIdentity } = {}) { + const target = new URL(url); + const client = target.protocol === 'https:' ? https : http; + return new Promise((resolve, reject) => { + const req = client.request( + target, + { method, headers, ca, ...(checkServerIdentity ? { checkServerIdentity } : {}) }, + (res) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => + resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) }), + ); + }, + ); + req.on('error', reject); + req.end(body); + }); +} + +/** Buffer a socket's messages from creation so back-to-back sends are never missed. */ +const inboxes = new WeakMap(); +function connect(url, options) { + const ws = new WebSocket(url, options); + const box = { queued: [], waiting: [] }; + inboxes.set(ws, box); + ws.on('message', (data) => { + const message = JSON.parse(data.toString()); + const waiter = box.waiting.shift(); + if (waiter) waiter(message); + else box.queued.push(message); + }); + return ws; +} +function nextMessage(ws) { + const box = inboxes.get(ws); + if (box.queued.length) return Promise.resolve(box.queued.shift()); + return new Promise((resolve) => box.waiting.push(resolve)); +} +const opened = (ws) => + new Promise((resolve, reject) => { + ws.once('open', resolve); + ws.once('error', reject); + ws.once('unexpected-response', (_req, res) => reject(new Error(`HTTP ${res.statusCode}`))); + }); + +async function harness(t) { + const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gev-vcam-')); + const service = createVcamService({ + env: {}, + cacheDir, + port: 0, + host: '127.0.0.1', + // The phone reaches the listener on loopback in this test. + addresses: () => ['127.0.0.1'], + logger: {}, + }); + const app = appServer(); + service.install(app); + await listen(app.httpServer); + const appOrigin = `http://127.0.0.1:${app.httpServer.address().port}`; + t.after(() => { + service.destroy(); + app.httpServer.close(); + fs.rmSync(cacheDir, { recursive: true, force: true }); + }); + const session = async () => { + const response = await request(`${appOrigin}/api/vcam/session`, { + method: 'POST', + headers: { Origin: appOrigin }, + }); + assert.equal(response.status, 200); + return JSON.parse(response.body); + }; + const ca = () => fs.readFileSync(path.join(cacheDir, 'ca.crt')); + return { service, appOrigin, session, ca }; +} + +test('the desktop pairs a phone over a trusted local CA and both legs relay', async (t) => { + const { appOrigin, session, ca } = await harness(t); + const created = await session(); + assert.equal(created.code.length, 6); + assert.deepEqual(created.addresses, ['127.0.0.1']); + const phoneOrigin = `https://127.0.0.1:${created.port}`; + + const page = await request(`${phoneOrigin}/`, { ca: ca() }); + assert.equal(page.status, 200); + assert.match(page.body.toString(), /VIRTUAL CAMERA/); + for (const [name, value] of Object.entries(PHONE_HEADERS)) + assert.equal(page.headers[name.toLowerCase()], value); + assert.equal((await request(`${phoneOrigin}/api/vcam/session`, { method: 'POST', ca: ca() })).status, 404, + 'the phone listener exposes none of the desktop routes'); + const certificate = await request(`${phoneOrigin}/ca.crt`, { ca: ca() }); + assert.equal(certificate.headers['content-type'], 'application/x-x509-ca-cert'); + assert.equal(certificate.body.toString(), ca().toString()); + + const host = connect( + `ws://127.0.0.1:${new URL(appOrigin).port}/api/vcam/host?key=${created.hostKey}`, + { origin: appOrigin }, + ); + const hostCode = nextMessage(host); + await opened(host); + assert.deepEqual(await hostCode, { type: 'code', code: created.code }); + await nextMessage(host); // phone not yet connected + + const join = await request(`${phoneOrigin}/api/join`, { + method: 'POST', ca: ca(), body: JSON.stringify({ code: created.code }), + }); + assert.equal(join.status, 200); + const { phoneKey } = JSON.parse(join.body); + const phone = connect(`wss://127.0.0.1:${created.port}/ws?key=${phoneKey}`, { + ca: ca(), origin: phoneOrigin, + }); + const hostSeesPhone = nextMessage(host); + await opened(phone); + assert.deepEqual(await hostSeesPhone, { type: 'peer', role: 'phone', connected: true }); + + const pose = { type: 'pose', t: 1, q: [0, 0, 0, 1], p: [0, 1.5, 0] }; + const received = nextMessage(host); + phone.send(JSON.stringify(pose)); + assert.deepEqual(await received, pose); + await nextMessage(phone); // host presence + const state = nextMessage(phone); + host.send(JSON.stringify({ type: 'state', state: { recording: true } })); + assert.deepEqual(await state, { type: 'state', state: { recording: true } }); + + // The host's binary messages are preview frames the phone pulls over HTTPS. + host.send(Buffer.from([0xff, 0xd8, 0xff, 0xe0, 7])); + const frame = await request(`${phoneOrigin}/api/frame?after=0`, { ca: ca(), headers: { 'X-Vcam-Key': phoneKey } }); + assert.equal(frame.status, 200); + assert.equal(frame.headers['content-type'], 'image/jpeg'); + assert.equal(frame.headers['x-frame-seq'], '1'); + assert.equal(frame.body.at(-1), 7); + + phone.close(); + host.close(); +}); + +test('the HTTPS fallback relays poses by POST and state by server-sent events', async (t) => { + const { appOrigin, session, ca } = await harness(t); + const created = await session(); + const phoneOrigin = `https://127.0.0.1:${created.port}`; + const host = connect( + `ws://127.0.0.1:${new URL(appOrigin).port}/api/vcam/host?key=${created.hostKey}`, + { origin: appOrigin }, + ); + await opened(host); + const { phoneKey } = JSON.parse( + (await request(`${phoneOrigin}/api/join`, { method: 'POST', ca: ca(), body: JSON.stringify({ code: created.code }) })).body, + ); + const events = await new Promise((resolve) => { + https.get(`${phoneOrigin}/api/events?key=${phoneKey}`, { ca: ca() }, resolve); + }); + assert.equal(events.headers['content-type'], 'text/event-stream; charset=utf-8'); + const firstEvent = new Promise((resolve) => events.once('data', (chunk) => resolve(chunk.toString()))); + assert.match(await firstEvent, /^data: \{"type":"peer","role":"host","connected":true\}/); + + const relayed = []; + const collect = async () => { + for (;;) relayed.push(await nextMessage(host)); + }; + collect(); + const sent = await request(`${phoneOrigin}/api/send`, { + method: 'POST', ca: ca(), headers: { 'X-Vcam-Key': phoneKey }, + body: JSON.stringify([{ type: 'cmd', cmd: 'record' }, { type: 'pose', t: 2 }, 'not an object']), + }); + assert.deepEqual(JSON.parse(sent.body), { accepted: 2 }); + await new Promise((resolve) => setTimeout(resolve, 50)); + assert.deepEqual(relayed.filter(({ type }) => !['peer', 'code'].includes(type)), [{ type: 'cmd', cmd: 'record' }, { type: 'pose', t: 2 }]); + + const downstream = new Promise((resolve) => events.once('data', (chunk) => resolve(chunk.toString()))); + host.send(JSON.stringify({ type: 'toast', text: 'hi' })); + assert.equal(await downstream, 'data: {"type":"toast","text":"hi"}\n\n'); + + const ended = new Promise((resolve) => events.once('end', resolve)); + await request(`${appOrigin}/api/vcam/session`, { method: 'DELETE', headers: { Origin: appOrigin, 'X-Vcam-Host-Key': created.hostKey } }); + await ended; + const after = await request(`${phoneOrigin}/api/send`, { + method: 'POST', ca: ca(), headers: { 'X-Vcam-Key': phoneKey }, body: '[]', + }); + assert.equal(after.status, 403, 'an ended session revokes the phone'); + host.close(); +}); + +test('pairing refuses foreign origins, bad codes, bad keys and cross-site sockets', async (t) => { + const { appOrigin, session, ca } = await harness(t); + const noOrigin = await request(`${appOrigin}/api/vcam/session`, { method: 'POST' }); + assert.equal(noOrigin.status, 403, 'a session needs the page origin'); + const foreign = await request(`${appOrigin}/api/vcam/session`, { method: 'POST', headers: { Origin: 'https://evil.example' } }); + assert.equal(foreign.status, 403); + const proxied = await request(`${appOrigin}/api/vcam/session`, { method: 'POST', headers: { Origin: appOrigin, 'X-Forwarded-For': '1.2.3.4' } }); + assert.equal(proxied.status, 403); + + const created = await session(); + const phoneOrigin = `https://127.0.0.1:${created.port}`; + const badJoin = await request(`${phoneOrigin}/api/join`, { method: 'POST', ca: ca(), body: '{"code":"222222"}' }); + assert.equal(badJoin.status, 403); + const hugeJoin = await request(`${phoneOrigin}/api/join`, { method: 'POST', ca: ca(), body: 'x'.repeat(5000) }).catch(() => ({ status: 413 })); + assert.equal(hugeJoin.status, 413); + const badHost = await request(`${phoneOrigin}/`, { + ca: ca(), headers: { Host: 'rebind.example' }, checkServerIdentity: () => undefined, + }); + assert.equal(badHost.status, 421, 'the phone listener answers IP addresses and allowed hosts only'); + + await assert.rejects( + opened(new WebSocket(`ws://127.0.0.1:${new URL(appOrigin).port}/api/vcam/host?key=${created.hostKey}`, { origin: 'https://evil.example' })), + /403/, + ); + await assert.rejects( + opened(new WebSocket(`ws://127.0.0.1:${new URL(appOrigin).port}/api/vcam/host?key=wrong`, { origin: appOrigin })), + /403/, + ); + const { phoneKey } = JSON.parse( + (await request(`${phoneOrigin}/api/join`, { method: 'POST', ca: ca(), body: JSON.stringify({ code: created.code }) })).body, + ); + await assert.rejects( + opened(new WebSocket(`wss://127.0.0.1:${created.port}/ws?key=${phoneKey}`, { ca: ca(), origin: 'https://evil.example' })), + /403/, + ); + await assert.rejects( + opened(new WebSocket(`wss://127.0.0.1:${created.port}/ws?key=wrong`, { ca: ca(), origin: phoneOrigin })), + /403/, + ); +}); + +test('LAN addresses prefer private ranges and skip loopback and link-local', () => { + assert.deepEqual( + lanAddresses({ + lo: [{ family: 'IPv4', address: '127.0.0.1', internal: true }], + docker0: [{ family: 'IPv4', address: '172.17.0.1', internal: false }], + en0: [ + { family: 'IPv6', address: 'fe80::1', internal: false }, + { family: 4, address: '192.168.1.23', internal: false }, + ], + tun0: [{ family: 'IPv4', address: '100.64.0.2', internal: false }], + bad: [{ family: 'IPv4', address: '169.254.3.3', internal: false }], + }), + ['192.168.1.23', '172.17.0.1', '100.64.0.2'], + ); +}); diff --git a/server/vcam/relay.js b/server/vcam/relay.js new file mode 100644 index 000000000..0e60a30a0 --- /dev/null +++ b/server/vcam/relay.js @@ -0,0 +1,319 @@ +/** + * Pairing and message relay between the desktop app (host) and one phone. + * + * Transport-agnostic: a peer is `{ send(text), close(code, reason) }`, so the + * WebSocket and HTTP-fallback transports share every rule here. The desktop + * creates a session from a loopback request and receives a host key. The + * phone proves possession of the short pairing code shown on the desktop + * (carried in the QR code's URL fragment) and receives its own key. Too many + * wrong codes rotate the code, so guessing cannot outlast the session. + * + * Messages are small JSON texts the relay forwards without interpreting, + * bounded in size and rate. The newest preview frame (a JPEG the host sends) + * is held for the phone to pull, so a slow phone drops frames instead of + * queueing them. + */ +import crypto from 'node:crypto'; + +// No 0/O, 1/I/L: the code is read off a screen. +const CODE_ALPHABET = '23456789ABCDEFGHJKMNPQRSTUVWXYZ'; +export const PAIRING_CODE_LENGTH = 6; +export const MAX_TEXT_BYTES = 4096; +export const MAX_FRAME_BYTES = 768 * 1024; +const MAX_CODE_FAILURES = 8; +const MESSAGE_BURST = 240; +const MESSAGES_PER_SECOND = 120; +// A host that disappears (a reload, a closed tab) may come back this soon. +const HOST_GRACE_MS = 60000; +const MAX_SESSIONS = 4; + +const randomKey = () => crypto.randomBytes(24).toString('base64url'); +function pairingCode(randomBytes = crypto.randomBytes) { + const bytes = randomBytes(PAIRING_CODE_LENGTH); + let code = ''; + for (const byte of bytes) code += CODE_ALPHABET[byte % CODE_ALPHABET.length]; + return code; +} +const sameSecret = (a, b) => { + const left = Buffer.from(String(a)); + const right = Buffer.from(String(b)); + return left.length === right.length && crypto.timingSafeEqual(left, right); +}; +const normalizeCode = (code) => + String(code ?? '') + .toUpperCase() + .replace(/[^0-9A-Z]/g, '') + .slice(0, 32); + +/** + * @param {object} [options] + * @param {() => number} [options.now] + * @param {(session: object) => void} [options.onSessionClosed] + */ +export function createVcamRelay({ + now = () => Date.now(), + onSessionClosed = () => {}, + setTimer = setTimeout, + clearTimer = clearTimeout, +} = {}) { + /** @type {Map} */ + const sessions = new Map(); + + const bucket = () => ({ tokens: MESSAGE_BURST, at: now() }); + function admit(rate) { + const at = now(); + rate.tokens = Math.min( + MESSAGE_BURST, + rate.tokens + ((at - rate.at) / 1000) * MESSAGES_PER_SECOND, + ); + rate.at = at; + if (rate.tokens < 1) return false; + rate.tokens -= 1; + return true; + } + + function notify(peer, message) { + try { + peer?.send(JSON.stringify(message)); + } catch { + // A failed send belongs to that transport's own close handling. + } + } + + function closeSession(session, reason = 'ended') { + if (!sessions.has(session.id)) return; + sessions.delete(session.id); + clearTimer(session.hostTimer); + for (const role of ['host', 'phone']) { + const peer = session[role]; + session[role] = null; + try { + peer?.close(4000, reason); + } catch { + // Already closed. + } + } + for (const waiter of session.frameWaiters.splice(0)) waiter(null); + onSessionClosed(session); + } + + function scheduleHostExpiry(session) { + clearTimer(session.hostTimer); + session.hostTimer = setTimer(() => { + if (!session.host) closeSession(session, 'host-gone'); + }, HOST_GRACE_MS); + session.hostTimer?.unref?.(); + } + + /** Create a session for the desktop; older sessions beyond the cap close. */ + function createSession() { + while (sessions.size >= MAX_SESSIONS) + closeSession(sessions.values().next().value, 'replaced'); + const session = { + id: crypto.randomUUID(), + hostKey: randomKey(), + code: pairingCode(), + phoneKey: null, + failures: 0, + host: null, + phone: null, + hostRate: bucket(), + phoneRate: bucket(), + frame: null, + frameSeq: 0, + frameWaiters: [], + createdAt: now(), + }; + sessions.set(session.id, session); + scheduleHostExpiry(session); + return session; + } + + const sessionForHost = (hostKey) => + [...sessions.values()].find((s) => sameSecret(s.hostKey, hostKey)) || null; + const sessionForPhone = (phoneKey) => + (phoneKey && + [...sessions.values()].find( + (s) => s.phoneKey && sameSecret(s.phoneKey, phoneKey), + )) || + null; + + /** + * Exchange a pairing code for a phone key. A new join replaces the previous + * phone. Wrong codes count against every session; enough of them rotate + * the codes and tell the desktop to show the new one. + * + * @returns {{ ok: true, phoneKey: string, sessionId: string } | { ok: false, reason: string }} + */ + function joinPhone(code) { + const wanted = normalizeCode(code); + const session = + wanted.length === PAIRING_CODE_LENGTH + ? [...sessions.values()].find((s) => sameSecret(s.code, wanted)) + : null; + if (!session) { + for (const candidate of sessions.values()) { + candidate.failures += 1; + if (candidate.failures >= MAX_CODE_FAILURES) { + candidate.failures = 0; + candidate.code = pairingCode(); + notify(candidate.host, { type: 'code', code: candidate.code }); + } + } + return { ok: false, reason: 'invalid-code' }; + } + session.failures = 0; + session.phoneKey = randomKey(); + if (session.phone) { + const previous = session.phone; + session.phone = null; + try { + previous.close(4001, 'replaced'); + } catch { + // Already closed. + } + } + return { ok: true, phoneKey: session.phoneKey, sessionId: session.id }; + } + + /** Attach a desktop transport; returns the session or null for a bad key. */ + function attachHost(hostKey, peer) { + const session = sessionForHost(hostKey); + if (!session) return null; + const previous = session.host; + session.host = peer; + clearTimer(session.hostTimer); + if (previous && previous !== peer) { + try { + previous.close(4001, 'replaced'); + } catch { + // Already closed. + } + } + notify(peer, { type: 'code', code: session.code }); + notify(peer, { type: 'peer', role: 'phone', connected: !!session.phone }); + return session; + } + + /** Attach a phone transport; returns the session or null for a bad key. */ + function attachPhone(phoneKey, peer) { + const session = sessionForPhone(phoneKey); + if (!session) return null; + const previous = session.phone; + session.phone = peer; + if (previous && previous !== peer) { + try { + previous.close(4001, 'replaced'); + } catch { + // Already closed. + } + } + notify(session.host, { type: 'peer', role: 'phone', connected: true }); + notify(peer, { type: 'peer', role: 'host', connected: !!session.host }); + return session; + } + + /** Detach a transport if it is still the session's current one. */ + function detach(session, role, peer) { + if (!session || session[role] !== peer) return; + session[role] = null; + const other = role === 'host' ? session.phone : session.host; + notify(other, { type: 'peer', role, connected: false }); + if (role === 'host') scheduleHostExpiry(session); + } + + /** + * Forward one text message to the other peer. Refuses oversize, non-JSON + * and over-rate input; the caller may close a peer that keeps sending it. + * + * @returns {boolean} Whether the message was admitted. + */ + function relay(session, role, text) { + if (!session || !sessions.has(session.id)) return false; + if (typeof text !== 'string' || Buffer.byteLength(text) > MAX_TEXT_BYTES) + return false; + if (!admit(role === 'host' ? session.hostRate : session.phoneRate)) + return false; + try { + const parsed = JSON.parse(text); + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) + return false; + } catch { + return false; + } + const target = role === 'host' ? session.phone : session.host; + if (target) { + try { + target.send(text); + } catch { + // The target transport owns its own close handling. + } + } + return true; + } + + /** Hold the host's newest JPEG preview frame and wake waiting phones. */ + function publishFrame(session, bytes) { + if (!session || !sessions.has(session.id)) return false; + if (!Buffer.isBuffer(bytes) || !bytes.length) return false; + if (bytes.length > MAX_FRAME_BYTES) return false; + // JPEG start-of-image marker: the phone page only ever shows JPEG. + if (bytes[0] !== 0xff || bytes[1] !== 0xd8) return false; + session.frame = bytes; + session.frameSeq += 1; + for (const waiter of session.frameWaiters.splice(0)) waiter(session); + return true; + } + + /** + * Resolve with the newest frame after `afterSeq`, waiting up to `waitMs`. + * @returns {Promise<{ seq: number, bytes: Buffer } | null>} + */ + function nextFrame(session, afterSeq, waitMs = 2000) { + const current = () => + session.frame && session.frameSeq > afterSeq + ? { seq: session.frameSeq, bytes: session.frame } + : null; + const ready = current(); + if (ready || waitMs <= 0) return Promise.resolve(ready); + return new Promise((resolve) => { + let timer = null; + const waiter = (live) => { + clearTimer(timer); + resolve(live ? current() : null); + }; + session.frameWaiters.push(waiter); + timer = setTimer(() => { + const index = session.frameWaiters.indexOf(waiter); + if (index >= 0) session.frameWaiters.splice(index, 1); + resolve(current()); + }, waitMs); + }); + } + + return { + createSession, + joinPhone, + attachHost, + attachPhone, + detach, + relay, + publishFrame, + nextFrame, + sessionForHost, + sessionForPhone, + closeSession, + endSession(hostKey) { + const session = sessionForHost(hostKey); + if (session) closeSession(session, 'ended'); + return !!session; + }, + get size() { + return sessions.size; + }, + closeAll() { + for (const session of [...sessions.values()]) + closeSession(session, 'shutdown'); + }, + }; +} diff --git a/server/vcam/relay.test.mjs b/server/vcam/relay.test.mjs new file mode 100644 index 000000000..db25bb18d --- /dev/null +++ b/server/vcam/relay.test.mjs @@ -0,0 +1,166 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + MAX_FRAME_BYTES, + MAX_TEXT_BYTES, + PAIRING_CODE_LENGTH, + createVcamRelay, +} from './relay.js'; + +function peer() { + const sent = []; + const closed = []; + return { + sent, + closed, + messages: () => sent.map((text) => JSON.parse(text)), + send: (text) => sent.push(text), + close: (code, reason) => closed.push([code, reason]), + }; +} + +function clock() { + let at = 0; + const timers = new Set(); + return { + now: () => at, + advance(ms) { + at += ms; + for (const timer of [...timers]) + if (timer.due <= at) { + timers.delete(timer); + timer.fn(); + } + }, + setTimer: (fn, ms) => { + const timer = { fn, due: at + ms }; + timers.add(timer); + return timer; + }, + clearTimer: (timer) => timers.delete(timer), + }; +} + +test('a phone pairs with the displayed code and relays both ways', () => { + const relay = createVcamRelay(); + const session = relay.createSession(); + assert.equal(session.code.length, PAIRING_CODE_LENGTH); + assert.match(session.code, /^[2-9A-HJKMNP-Z]+$/, 'no ambiguous characters'); + const host = peer(); + assert.equal(relay.attachHost(session.hostKey, host), session); + assert.deepEqual(host.messages(), [ + { type: 'code', code: session.code }, + { type: 'peer', role: 'phone', connected: false }, + ]); + const joined = relay.joinPhone(` ${session.code.toLowerCase().slice(0, 3)}-${session.code.slice(3)} `); + assert.equal(joined.ok, true, 'codes are case- and separator-insensitive'); + const phone = peer(); + assert.equal(relay.attachPhone(joined.phoneKey, phone), session); + assert.deepEqual(host.messages().at(-1), { type: 'peer', role: 'phone', connected: true }); + assert.ok(relay.relay(session, 'phone', '{"type":"pose","q":[0,0,0,1]}')); + assert.equal(host.sent.at(-1), '{"type":"pose","q":[0,0,0,1]}'); + assert.ok(relay.relay(session, 'host', '{"type":"state"}')); + assert.equal(phone.sent.at(-1), '{"type":"state"}'); + relay.detach(session, 'phone', phone); + assert.deepEqual(host.messages().at(-1), { type: 'peer', role: 'phone', connected: false }); +}); + +test('keys are per role and a new phone join replaces the previous phone', () => { + const relay = createVcamRelay(); + const session = relay.createSession(); + assert.equal(relay.attachPhone(session.hostKey, peer()), null); + assert.equal(relay.attachHost('nope', peer()), null); + const first = relay.joinPhone(session.code); + const phone = peer(); + relay.attachPhone(first.phoneKey, phone); + const second = relay.joinPhone(session.code); + assert.notEqual(first.phoneKey, second.phoneKey); + assert.deepEqual(phone.closed, [[4001, 'replaced']]); + assert.equal(relay.sessionForPhone(first.phoneKey), null, 'the old key is revoked'); +}); + +test('wrong codes rotate the pairing code and tell the desktop', () => { + const relay = createVcamRelay(); + const session = relay.createSession(); + const host = peer(); + relay.attachHost(session.hostKey, host); + const original = session.code; + for (let i = 0; i < 7; i++) assert.equal(relay.joinPhone('ZZZZZZ').ok, false); + assert.equal(session.code, original); + relay.joinPhone('ZZZZZZ'); + assert.notEqual(session.code, original); + assert.deepEqual(host.messages().at(-1), { type: 'code', code: session.code }); + assert.equal(relay.joinPhone(original).ok, false, 'the old code no longer pairs'); + assert.equal(relay.joinPhone(session.code).ok, true); +}); + +test('oversize, non-object and over-rate messages are refused', () => { + const time = clock(); + const relay = createVcamRelay(time); + const session = relay.createSession(); + const host = peer(); + relay.attachHost(session.hostKey, host); + const big = JSON.stringify({ type: 'x', pad: 'a'.repeat(MAX_TEXT_BYTES) }); + assert.equal(relay.relay(session, 'phone', big), false); + assert.equal(relay.relay(session, 'phone', 'not json'), false); + assert.equal(relay.relay(session, 'phone', '[1,2]'), false); + let admitted = 0; + for (let i = 0; i < 400; i++) if (relay.relay(session, 'phone', '{}')) admitted++; + // The two malformed messages above already spent their tokens. + assert.equal(admitted, 238, 'a burst is capped, malformed input included'); + time.advance(500); + assert.equal(relay.relay(session, 'phone', '{}'), true, 'tokens refill over time'); +}); + +test('the newest preview frame is pulled, not queued', async () => { + const relay = createVcamRelay(); + const session = relay.createSession(); + const jpeg = (n) => Buffer.from([0xff, 0xd8, n]); + assert.equal(relay.publishFrame(session, Buffer.from('GIF89a')), false, 'only JPEG'); + assert.equal(relay.publishFrame(session, Buffer.alloc(MAX_FRAME_BYTES + 1, 0xff)), false); + assert.equal(await relay.nextFrame(session, 0, 0), null); + const waiting = relay.nextFrame(session, 0, 5000); + relay.publishFrame(session, jpeg(1)); + relay.publishFrame(session, jpeg(2)); + const first = await waiting; + assert.equal(first.seq, 1); + const latest = await relay.nextFrame(session, first.seq, 0); + assert.equal(latest.seq, 2, 'a slow phone skips straight to the newest frame'); + assert.equal(latest.bytes[2], 2); + relay.closeSession(session); + assert.equal(relay.publishFrame(session, jpeg(3)), false); +}); + +test('a session outlives a desktop reload but not a departed desktop', () => { + const time = clock(); + const closed = []; + const relay = createVcamRelay({ ...time, onSessionClosed: (s) => closed.push(s.id) }); + const session = relay.createSession(); + const host = peer(); + relay.attachHost(session.hostKey, host); + const phone = peer(); + relay.attachPhone(relay.joinPhone(session.code).phoneKey, phone); + relay.detach(session, 'host', host); + assert.deepEqual(phone.messages().at(-1), { type: 'peer', role: 'host', connected: false }); + time.advance(30000); + relay.attachHost(session.hostKey, peer()); + time.advance(60000); + assert.equal(relay.size, 1, 'a reattached host keeps the session'); + relay.detach(session, 'host', relay.sessionForHost(session.hostKey).host); + time.advance(60001); + assert.equal(relay.size, 0); + assert.deepEqual(closed, [session.id]); + assert.deepEqual(phone.closed, [[4000, 'host-gone']]); +}); + +test('ending a session closes both peers and only the host key can end it', () => { + const relay = createVcamRelay(); + const session = relay.createSession(); + const phone = peer(); + relay.attachPhone(relay.joinPhone(session.code).phoneKey, phone); + assert.equal(relay.endSession('wrong'), false); + assert.equal(relay.endSession(session.hostKey), true); + assert.deepEqual(phone.closed, [[4000, 'ended']]); + for (let i = 0; i < 6; i++) relay.createSession(); + assert.equal(relay.size, 4, 'sessions are capped'); +}); diff --git a/server/vcam/tls.js b/server/vcam/tls.js new file mode 100644 index 000000000..ae00da5d8 --- /dev/null +++ b/server/vcam/tls.js @@ -0,0 +1,237 @@ +/** + * Local certificate authority for the virtual-camera phone listener. + * + * Phones only expose camera tracking (WebXR, DeviceOrientation) to secure + * contexts, so the LAN listener must speak HTTPS. A tapped-through + * self-signed certificate is not enough on iOS: Safari loads the page but + * refuses its WebSocket. Instead this module keeps one long-lived local CA, + * which a phone can install and trust once, and issues the listener a short + * server certificate for this machine's current LAN addresses. + * + * Certificates are encoded here with node:crypto (EC P-256, ECDSA-SHA256), so + * no OpenSSL binary or dependency is needed. Keys never leave the cache + * directory, which is git-ignored and outside the dev server's watcher. + */ +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { isIP } from 'node:net'; + +const DAY_MS = 86400000; +// Apple rejects user-trusted TLS server certificates valid for over 825 days. +const SERVER_VALIDITY_DAYS = 397; +const CA_VALIDITY_DAYS = 3650; +// Reissue the server certificate this long before it expires. +const RENEW_MARGIN_DAYS = 30; + +const length = (n) => + n < 0x80 + ? Buffer.from([n]) + : n < 0x100 + ? Buffer.from([0x81, n]) + : Buffer.from([0x82, n >> 8, n & 0xff]); +const tlv = (tag, ...parts) => { + const body = Buffer.concat(parts); + return Buffer.concat([Buffer.from([tag]), length(body.length), body]); +}; +const sequence = (...parts) => tlv(0x30, ...parts); +const set = (...parts) => tlv(0x31, ...parts); +const explicit = (n, ...parts) => tlv(0xa0 + n, ...parts); +const octets = (buffer) => tlv(0x04, buffer); +const bits = (buffer, unused = 0) => tlv(0x03, Buffer.from([unused]), buffer); +const boolean = (value) => tlv(0x01, Buffer.from([value ? 0xff : 0])); +const utf8 = (text) => tlv(0x0c, Buffer.from(text, 'utf8')); +function integer(buffer) { + let bytes = Buffer.from(buffer); + while (bytes.length > 1 && bytes[0] === 0 && !(bytes[1] & 0x80)) + bytes = bytes.subarray(1); + if (bytes[0] & 0x80) bytes = Buffer.concat([Buffer.from([0]), bytes]); + return tlv(0x02, bytes); +} +function oid(dotted) { + const [first, second, ...rest] = dotted.split('.').map(Number); + const bytes = [40 * first + second]; + for (const part of rest) { + const group = [part & 0x7f]; + for (let v = part >>> 7; v; v >>>= 7) group.unshift((v & 0x7f) | 0x80); + bytes.push(...group); + } + return tlv(0x06, Buffer.from(bytes)); +} +function time(date) { + const iso = date.toISOString().replace(/[-:T]/g, '').slice(0, 14); + // RFC 5280: UTCTime through 2049, GeneralizedTime from 2050. + return date.getUTCFullYear() < 2050 + ? tlv(0x17, Buffer.from(`${iso.slice(2)}Z`)) + : tlv(0x18, Buffer.from(`${iso}Z`)); +} +const name = (commonName) => + sequence( + set(sequence(oid('2.5.4.10'), utf8("God's Eye View"))), + set(sequence(oid('2.5.4.3'), utf8(commonName))), + ); +const extension = (id, critical, value) => + sequence(oid(id), ...(critical ? [boolean(true)] : []), octets(value)); +const ECDSA_SHA256 = sequence(oid('1.2.840.10045.4.3.2')); +const spkiOf = (publicKey) => publicKey.export({ type: 'spki', format: 'der' }); +// RFC 5280 method 1: SHA-1 of the subjectPublicKey bit string's contents. +const keyIdentifier = (spki) => + crypto.createHash('sha1').update(spki.subarray(-65)).digest(); + +function subjectAltNames(names) { + return sequence( + ...names.map((entry) => + isIP(entry) === 4 + ? tlv(0x87, Buffer.from(entry.split('.').map(Number))) + : tlv(0x82, Buffer.from(entry, 'ascii')), + ), + ); +} + +function sign({ subject, issuer, publicKey, signingKey, days, extensions }) { + const now = Date.now(); + const tbs = sequence( + explicit(0, integer(Buffer.from([2]))), + integer(crypto.randomBytes(16)), + ECDSA_SHA256, + issuer, + sequence(time(new Date(now - DAY_MS)), time(new Date(now + days * DAY_MS))), + subject, + spkiOf(publicKey), + explicit(3, sequence(...extensions)), + ); + const signature = crypto.sign('sha256', tbs, signingKey); + return sequence(tbs, ECDSA_SHA256, bits(signature)); +} + +const pem = (label, der) => + `-----BEGIN ${label}-----\n${der + .toString('base64') + .match(/.{1,64}/g) + .join('\n')}\n-----END ${label}-----\n`; +const keyPem = (key) => key.export({ type: 'pkcs8', format: 'pem' }); + +const CA_COMMON_NAME = "God's Eye View Local VCam CA"; + +/** Create a CA that may only sign server certificates (path length 0). */ +export function createCertificateAuthority(commonName = CA_COMMON_NAME) { + const { publicKey, privateKey } = crypto.generateKeyPairSync('ec', { + namedCurve: 'P-256', + }); + const subject = name(commonName); + const der = sign({ + subject, + issuer: subject, + publicKey, + signingKey: privateKey, + days: CA_VALIDITY_DAYS, + extensions: [ + extension( + '2.5.29.19', + true, + sequence(boolean(true), integer(Buffer.from([0]))), + ), + // keyCertSign and cRLSign. + extension('2.5.29.15', true, bits(Buffer.from([0x06]), 1)), + extension('2.5.29.14', false, octets(keyIdentifier(spkiOf(publicKey)))), + ], + }); + return { cert: pem('CERTIFICATE', der), key: keyPem(privateKey) }; +} + +/** Issue a TLS server certificate for the given IP addresses and DNS names. */ +export function issueServerCertificate(authority, names) { + const caKey = crypto.createPrivateKey(authority.key); + const caCert = new crypto.X509Certificate(authority.cert); + const { publicKey, privateKey } = crypto.generateKeyPairSync('ec', { + namedCurve: 'P-256', + }); + const caSpki = spkiOf(caCert.publicKey); + const der = sign({ + subject: name(names[0]), + issuer: name(caCert.subject.match(/CN=([^\n]+)/)?.[1] ?? CA_COMMON_NAME), + publicKey, + signingKey: caKey, + days: SERVER_VALIDITY_DAYS, + extensions: [ + extension('2.5.29.19', true, sequence()), + // digitalSignature. + extension('2.5.29.15', true, bits(Buffer.from([0x80]), 7)), + extension('2.5.29.37', false, sequence(oid('1.3.6.1.5.5.7.3.1'))), + extension('2.5.29.17', false, subjectAltNames(names)), + extension('2.5.29.35', false, sequence(tlv(0x80, keyIdentifier(caSpki)))), + ], + }); + return { cert: pem('CERTIFICATE', der), key: keyPem(privateKey) }; +} + +function readPair(directory, base) { + try { + return { + cert: fs.readFileSync(path.join(directory, `${base}.crt`), 'utf8'), + key: fs.readFileSync(path.join(directory, `${base}.key`), 'utf8'), + }; + } catch { + return null; + } +} + +function writePair(directory, base, pair) { + fs.writeFileSync(path.join(directory, `${base}.key`), pair.key, { + mode: 0o600, + }); + fs.writeFileSync(path.join(directory, `${base}.crt`), pair.cert); +} + +function serverCertificateCovers(pair, authority, names) { + try { + const cert = new crypto.X509Certificate(pair.cert); + if (!cert.verify(new crypto.X509Certificate(authority.cert).publicKey)) + return false; + if (Date.parse(cert.validTo) - Date.now() < RENEW_MARGIN_DAYS * DAY_MS) + return false; + const listed = new Set( + String(cert.subjectAltName || '') + .split(',') + .map((entry) => entry.trim().replace(/^(DNS|IP Address):/, '')), + ); + return names.every((entry) => listed.has(entry)); + } catch { + return false; + } +} + +/** + * Load or create the cached CA and a server certificate covering `names`. + * The CA survives restarts so a phone's one-time trust keeps working; the + * server certificate is reissued when an address changes or it nears expiry. + * + * @param {string} directory - Cache directory, created with owner-only access. + * @param {string[]} names - IPv4 addresses and DNS names; the first is the subject. + * @returns {{ ca: string, cert: string, key: string, caFingerprint: string }} + */ +export function loadLocalCertificates(directory, names) { + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + let authority = readPair(directory, 'ca'); + try { + if (authority && !new crypto.X509Certificate(authority.cert).ca) + authority = null; + } catch { + authority = null; + } + if (!authority) { + authority = createCertificateAuthority(); + writePair(directory, 'ca', authority); + } + let server = readPair(directory, 'server'); + if (!server || !serverCertificateCovers(server, authority, names)) { + server = issueServerCertificate(authority, names); + writePair(directory, 'server', server); + } + return { + ca: authority.cert, + cert: server.cert, + key: server.key, + caFingerprint: new crypto.X509Certificate(authority.cert).fingerprint256, + }; +} diff --git a/src/app/tools.js b/src/app/tools.js index 7b3ea83c9..124cd56ff 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -1,4 +1,7 @@ import { SceneDirector } from '../scenes/director.js'; +import { createCameraLens } from '../scenes/cameraLens.js'; +import { createVcamController } from '../vcam/controller.js'; +import { bindVcamPanel } from '../ui/vcamPanel.js'; import { initAnnotations } from '../annotations/index.js'; import { initDrawTool } from '../annotations/drawTool.js'; import { initGeoFileImport } from '../ui/geoFileImport.js'; @@ -32,16 +35,36 @@ export function createApplicationTools({ const { viewer, tileset, mapStackController, operations } = scene; const { styleManager, weatherEffects, cockpitCloudEffects } = controls; const { dataManager } = data; + // Focal length and depth of field, shared by scene playback and the VCAM. + const lens = createCameraLens(viewer, { + requestRender: (reason) => governorRequestRender(reason), + }); const sceneDirector = new SceneDirector(viewer, styleManager, dataManager, { dataPacks: sceneDataPacks, isMapStackAvailable: (id) => mapStackController?.isStackAvailable(id) === true, + lens, }); dataManager.layers .get('bhote-koshi-2026') ?.module.attachSceneController(sceneDirector); defer(() => sceneDirector.destroy()); onSceneDirector?.(sceneDirector); + // SCENES β–Έ VCAM: a phone on the same network becomes the camera; its + // takes are recorded into the selected scene. Torn down before the lens + // and director it drives. + const vcam = createVcamController({ + viewer, + navigation: styleManager, + director: sceneDirector, + lens, + }); + const unbindVcamPanel = bindVcamPanel(vcam); + defer(() => lens.destroy()); + defer(() => { + unbindVcamPanel(); + vcam.destroy(); + }); const annotations = initAnnotations({ viewer, tileset, @@ -166,6 +189,8 @@ export function createApplicationTools({ getRenderGovernorDiagnostics, surfaceServices: operations.surface, requestRender: governorRequestRender, + vcam, + lens, }; const debug = window.__godsEyeView; defer(() => { diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index b5b0a3abb..cb74132e9 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -2889,5 +2889,53 @@ "USB work abandoned after a stalled transfer": "Trabajo USB cancelado tras una transferencia bloqueada", "SDR decoder did not start; receiver stopped": "El decodificador SDR no arrancΓ³; receptor detenido", "SDR decoder stopped responding; receiver stopped": "El decodificador SDR dejΓ³ de responder; receptor detenido", - "SDR decoder fell behind; receiver stopped": "El decodificador SDR se quedΓ³ atrΓ‘s; receptor detenido" + "SDR decoder fell behind; receiver stopped": "El decodificador SDR se quedΓ³ atrΓ‘s; receptor detenido", + "Held from the phone": "En pausa desde el telΓ©fono", + "Held: you moved the globe β€” press RESUME or tap HOLD on the phone": "En pausa: moviste el globo β€” pulsa REANUDAR o toca PAUSA en el telΓ©fono", + "Held": "En pausa", + "Held: another view took the camera": "En pausa: otra vista tomΓ³ la cΓ‘mara", + "Starting…": "Iniciando…", + "Phone camera off": "CΓ‘mara del telΓ©fono apagada", + "Scan the code with a phone on the same Wi-Fi": "Escanea el cΓ³digo con un telΓ©fono en la misma red Wi-Fi", + "AR tracking (6DoF)": "Seguimiento AR (6DoF)", + "Gyro tracking (3DoF)": "Seguimiento por giroscopio (3DoF)", + "Recording {time} Β· {mode}": "Grabando {time} Β· {mode}", + "Phone connected Β· {mode} Β· {count} takes": { + "one": "TelΓ©fono conectado Β· {mode} Β· {count} toma", + "many": "TelΓ©fono conectado Β· {mode} Β· {count} tomas", + "other": "TelΓ©fono conectado Β· {mode} Β· {count} tomas" + }, + "Phone connected Β· {mode}": "TelΓ©fono conectado Β· {mode}", + "HOLD": "PAUSA", + "β–  STOP": "β–  DETENER", + "● REC": "● GRABAR", + "The phone camera service is unavailable": "El servicio de cΓ‘mara del telΓ©fono no estΓ‘ disponible", + "The phone camera port is in use (set VCAM_PORT)": "El puerto de la cΓ‘mara del telΓ©fono estΓ‘ en uso (configura VCAM_PORT)", + "No network connection for a phone to reach": "No hay conexiΓ³n de red a la que pueda llegar un telΓ©fono", + "Phone camera session ended": "SesiΓ³n de cΓ‘mara del telΓ©fono finalizada", + "Nothing to focus on there": "No hay nada que enfocar ahΓ­", + "Take reached its length limit and was saved": "La toma alcanzΓ³ su duraciΓ³n mΓ‘xima y se guardΓ³", + "Take saved to the scene": "Toma guardada en la escena", + "Take too short to keep": "Toma demasiado corta para guardarla", + "Recorded: {scene} / {shot}": "Grabado: {scene} / {shot}", + "Use a phone on this network as a virtual camera": "Usa un telΓ©fono de esta red como cΓ‘mara virtual", + "Phone virtual camera": "CΓ‘mara virtual del telΓ©fono", + "Pairing QR code": "CΓ³digo QR de vinculaciΓ³n", + "SCAN WITH YOUR PHONE": "ESCANEA CON TU TELΓ‰FONO", + "Network address": "DirecciΓ³n de red", + "SCALE": "ESCALA", + "1:1 WALK": "1:1 A PIE", + "1:10 CRANE": "1:10 GRÚA", + "1:100 DRONE": "1:100 DRON", + "1:1000 FLYOVER": "1:1000 SOBREVUELO", + "1:10000 ORBIT": "1:10000 Γ“RBITA", + "LENS": "LENTE", + "DEPTH OF FIELD": "PROFUNDIDAD DE CAMPO", + "APERTURE": "APERTURA", + "RECENTER": "RECENTRAR", + "END": "TERMINAR", + "Takes become shots in the selected scene.": "Las tomas se convierten en planos de la escena seleccionada.", + "Phone certificate": "Certificado del telΓ©fono", + "VCAM": "VCAM", + "Exit cockpit to use the phone camera": "Sal de la cabina para usar la cΓ‘mara del telΓ©fono" } diff --git a/src/i18n/locales/ru.json b/src/i18n/locales/ru.json index 019415690..57b9d3446 100644 --- a/src/i18n/locales/ru.json +++ b/src/i18n/locales/ru.json @@ -2956,5 +2956,54 @@ "USB work abandoned after a stalled transfer": "Π Π°Π±ΠΎΡ‚Π° с USB ΠΏΡ€Π΅Ρ€Π²Π°Π½Π°: ΠΏΠ΅Ρ€Π΅Π΄Π°Ρ‡Π° Π΄Π°Π½Π½Ρ‹Ρ… зависла", "SDR decoder did not start; receiver stopped": "Π”Π΅ΠΊΠΎΠ΄Π΅Ρ€ SDR Π½Π΅ запустился; ΠΏΡ€ΠΈΡ‘ΠΌΠ½ΠΈΠΊ остановлСн", "SDR decoder stopped responding; receiver stopped": "Π”Π΅ΠΊΠΎΠ΄Π΅Ρ€ SDR пСрСстал ΠΎΡ‚Π²Π΅Ρ‡Π°Ρ‚ΡŒ; ΠΏΡ€ΠΈΡ‘ΠΌΠ½ΠΈΠΊ остановлСн", - "SDR decoder fell behind; receiver stopped": "Π”Π΅ΠΊΠΎΠ΄Π΅Ρ€ SDR Π½Π΅ успСваСт; ΠΏΡ€ΠΈΡ‘ΠΌΠ½ΠΈΠΊ остановлСн" + "SDR decoder fell behind; receiver stopped": "Π”Π΅ΠΊΠΎΠ΄Π΅Ρ€ SDR Π½Π΅ успСваСт; ΠΏΡ€ΠΈΡ‘ΠΌΠ½ΠΈΠΊ остановлСн", + "Held from the phone": "ΠŸΠ°ΡƒΠ·Π° с Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½Π°", + "Held: you moved the globe β€” press RESUME or tap HOLD on the phone": "ΠŸΠ°ΡƒΠ·Π°: Π²Ρ‹ сдвинули глобус β€” Π½Π°ΠΆΠΌΠΈΡ‚Π΅ ΠŸΠ ΠžΠ”ΠžΠ›Π–Π˜Π’Π¬ ΠΈΠ»ΠΈ ΠŸΠΠ£Π—Π Π½Π° Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½Π΅", + "Held": "ΠŸΠ°ΡƒΠ·Π°", + "Held: another view took the camera": "ΠŸΠ°ΡƒΠ·Π°: ΠΊΠ°ΠΌΠ΅Ρ€Ρƒ занял Π΄Ρ€ΡƒΠ³ΠΎΠΉ Π²ΠΈΠ΄", + "Starting…": "Запуск…", + "Phone camera off": "ΠšΠ°ΠΌΠ΅Ρ€Π° Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½Π° Π²Ρ‹ΠΊΠ»ΡŽΡ‡Π΅Π½Π°", + "Scan the code with a phone on the same Wi-Fi": "ΠžΡ‚ΡΠΊΠ°Π½ΠΈΡ€ΡƒΠΉΡ‚Π΅ ΠΊΠΎΠ΄ Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½ΠΎΠΌ Π² Ρ‚ΠΎΠΉ ΠΆΠ΅ сСти Wi-Fi", + "AR tracking (6DoF)": "AR-Ρ‚Ρ€Π΅ΠΊΠΈΠ½Π³ (6DoF)", + "Gyro tracking (3DoF)": "Гироскоп (3DoF)", + "Recording {time} Β· {mode}": "Π—Π°ΠΏΠΈΡΡŒ {time} Β· {mode}", + "Phone connected Β· {mode} Β· {count} takes": { + "one": "Π’Π΅Π»Π΅Ρ„ΠΎΠ½ ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Ρ‘Π½ Β· {mode} Β· {count} Π΄ΡƒΠ±Π»ΡŒ", + "few": "Π’Π΅Π»Π΅Ρ„ΠΎΠ½ ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Ρ‘Π½ Β· {mode} Β· {count} дубля", + "many": "Π’Π΅Π»Π΅Ρ„ΠΎΠ½ ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Ρ‘Π½ Β· {mode} Β· {count} Π΄ΡƒΠ±Π»Π΅ΠΉ", + "other": "Π’Π΅Π»Π΅Ρ„ΠΎΠ½ ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Ρ‘Π½ Β· {mode} Β· {count} дубля" + }, + "Phone connected Β· {mode}": "Π’Π΅Π»Π΅Ρ„ΠΎΠ½ ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Ρ‘Π½ Β· {mode}", + "HOLD": "ΠŸΠΠ£Π—Π", + "β–  STOP": "β–  БВОП", + "● REC": "● Π—ΠΠŸΠ˜Π‘Π¬", + "The phone camera service is unavailable": "Π‘Π»ΡƒΠΆΠ±Π° ΠΊΠ°ΠΌΠ΅Ρ€Ρ‹ Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½Π° нСдоступна", + "The phone camera port is in use (set VCAM_PORT)": "ΠŸΠΎΡ€Ρ‚ ΠΊΠ°ΠΌΠ΅Ρ€Ρ‹ Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½Π° занят (Π·Π°Π΄Π°ΠΉΡ‚Π΅ VCAM_PORT)", + "No network connection for a phone to reach": "НСт сСтСвого ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΡ, доступного Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½Ρƒ", + "Phone camera session ended": "БСанс ΠΊΠ°ΠΌΠ΅Ρ€Ρ‹ Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½Π° Π·Π°Π²Π΅Ρ€ΡˆΡ‘Π½", + "Nothing to focus on there": "Π—Π΄Π΅ΡΡŒ Π½Π΅ Π½Π° Ρ‡Π΅ΠΌ ΡΡ„ΠΎΠΊΡƒΡΠΈΡ€ΠΎΠ²Π°Ρ‚ΡŒΡΡ", + "Take reached its length limit and was saved": "Π”ΡƒΠ±Π»ΡŒ достиг ΠΏΡ€Π΅Π΄Π΅Π»ΡŒΠ½ΠΎΠΉ Π΄Π»ΠΈΠ½Ρ‹ ΠΈ сохранён", + "Take saved to the scene": "Π”ΡƒΠ±Π»ΡŒ сохранён Π² сцСну", + "Take too short to keep": "Π”ΡƒΠ±Π»ΡŒ слишком ΠΊΠΎΡ€ΠΎΡ‚ΠΊΠΈΠΉ", + "Recorded: {scene} / {shot}": "Записано: {scene} / {shot}", + "Use a phone on this network as a virtual camera": "Π˜ΡΠΏΠΎΠ»ΡŒΠ·ΠΎΠ²Π°Ρ‚ΡŒ Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½ Π² этой сСти ΠΊΠ°ΠΊ Π²ΠΈΡ€Ρ‚ΡƒΠ°Π»ΡŒΠ½ΡƒΡŽ ΠΊΠ°ΠΌΠ΅Ρ€Ρƒ", + "Phone virtual camera": "Π’ΠΈΡ€Ρ‚ΡƒΠ°Π»ΡŒΠ½Π°Ρ ΠΊΠ°ΠΌΠ΅Ρ€Π° Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½Π°", + "Pairing QR code": "QR-ΠΊΠΎΠ΄ для ΠΏΠΎΠ΄ΠΊΠ»ΡŽΡ‡Π΅Π½ΠΈΡ", + "SCAN WITH YOUR PHONE": "ΠžΠ’Π‘ΠšΠΠΠ˜Π Π£Π™Π’Π• Π’Π•Π›Π•Π€ΠžΠΠžΠœ", + "Network address": "Π‘Π΅Ρ‚Π΅Π²ΠΎΠΉ адрСс", + "SCALE": "ΠœΠΠ‘Π¨Π’ΠΠ‘", + "1:1 WALK": "1:1 ΠŸΠ•Π¨ΠšΠžΠœ", + "1:10 CRANE": "1:10 КРАН", + "1:100 DRONE": "1:100 Π”Π ΠžΠ", + "1:1000 FLYOVER": "1:1000 ΠŸΠ ΠžΠ›ΠΠ’", + "1:10000 ORBIT": "1:10000 ΠžΠ Π‘Π˜Π’Π", + "LENS": "ΠžΠ‘ΠͺΠ•ΠšΠ’Π˜Π’", + "DEPTH OF FIELD": "Π“Π›Π£Π‘Π˜ΠΠ Π Π•Π—ΠšΠžΠ‘Π’Π˜", + "APERTURE": "Π”Π˜ΠΠ€Π ΠΠ“ΠœΠ", + "RECENTER": "Π¦Π•ΠΠ’Π Π˜Π ΠžΠ’ΠΠ’Π¬", + "END": "Π—ΠΠ’Π•Π Π¨Π˜Π’Π¬", + "Takes become shots in the selected scene.": "Π”ΡƒΠ±Π»ΠΈ становятся ΠΊΠ°Π΄Ρ€Π°ΠΌΠΈ Π²Ρ‹Π±Ρ€Π°Π½Π½ΠΎΠΉ сцСны.", + "Phone certificate": "Π‘Π΅Ρ€Ρ‚ΠΈΡ„ΠΈΠΊΠ°Ρ‚ для Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½Π°", + "VCAM": "VCAM", + "Exit cockpit to use the phone camera": "Π’Ρ‹ΠΉΠ΄ΠΈΡ‚Π΅ ΠΈΠ· ΠΊΠ°Π±ΠΈΠ½Ρ‹, Ρ‡Ρ‚ΠΎΠ±Ρ‹ ΠΈΡΠΏΠΎΠ»ΡŒΠ·ΠΎΠ²Π°Ρ‚ΡŒ ΠΊΠ°ΠΌΠ΅Ρ€Ρƒ Ρ‚Π΅Π»Π΅Ρ„ΠΎΠ½Π°" } diff --git a/src/i18n/same-as-english.json b/src/i18n/same-as-english.json index c159beaab..651eb0745 100644 --- a/src/i18n/same-as-english.json +++ b/src/i18n/same-as-english.json @@ -89,6 +89,7 @@ "Techno", "Trance", "USB {rate} msg/s", + "VCAM", "VISUAL", "VS", "WHOT/BHOT", @@ -131,6 +132,7 @@ "NVG", "R&B", "REC", + "VCAM", "VS", "WHOT/BHOT" ] diff --git a/src/navigationPolicy.js b/src/navigationPolicy.js index d2ed6ff39..f72d01709 100644 --- a/src/navigationPolicy.js +++ b/src/navigationPolicy.js @@ -35,6 +35,8 @@ const COCKPIT_REFUSALS = Object.freeze({ aircraft: msg('Exit cockpit to fly to an aircraft'), satellite: msg('Exit cockpit to fly to a satellite'), frame: msg('Exit cockpit to frame contacts'), + // SCENES β–Έ VCAM: a paired phone driving the camera. + vcam: msg('Exit cockpit to use the phone camera'), }); /** The destination nouns with their own Cockpit refusal (for tests). */ diff --git a/src/tooling/viteBuild.test.mjs b/src/tooling/viteBuild.test.mjs index 9e38a0295..c2ed550c9 100644 --- a/src/tooling/viteBuild.test.mjs +++ b/src/tooling/viteBuild.test.mjs @@ -89,12 +89,14 @@ test('root config retains existing named exports and standalone provider order', assert.equal(compatibility[name], value, name); const config = standaloneConfig({ mode: 'test' }); assert.deepEqual( - config.plugins.slice(3, -5).map((plugin) => plugin.name), + config.plugins.slice(3, -6).map((plugin) => plugin.name), providers.localProviderPlugins().map((plugin) => plugin.name), ); - assert.equal(config.plugins.at(-6).name, 'gev-key-setup'); + assert.equal(config.plugins.at(-7).name, 'gev-key-setup'); // The local MCP route follows every provider and precedes the API fallback. - assert.equal(config.plugins.at(-5).name, 'local-mcp'); + assert.equal(config.plugins.at(-6).name, 'local-mcp'); + // The phone camera's desktop routes also answer before the API fallback. + assert.equal(config.plugins.at(-5).name, 'gev-vcam'); assert.equal(config.plugins.at(-4).name, 'api-not-found'); // Config-only, so it sits after the provider slice rather than in it. assert.equal(config.plugins.at(-3).name, 'gev-unwatched-runtime-dirs'); diff --git a/src/ui/styles/scenes.css b/src/ui/styles/scenes.css index 287b8b842..b33f1d5e0 100644 --- a/src/ui/styles/scenes.css +++ b/src/ui/styles/scenes.css @@ -474,3 +474,157 @@ html:not([lang='en']) #scene-panel .scene-controls .scene-btn { .director-sharing-dialog::backdrop { background: rgba(0, 8, 14, 0.65); } + +/* ── VCAM: phone virtual camera ─────────────── */ +#vcam-toggle-btn.active, +#vcam-toggle-btn.vcam-live { + color: var(--accent); + border-color: var(--accent); +} + +#vcam-toggle-btn.vcam-live::before { + content: ''; + display: inline-block; + width: 6px; + height: 6px; + margin-right: 5px; + border-radius: 50%; + background: #3ddc84; + vertical-align: 1px; +} + +.vcam-card { + margin: 0 0 10px; + padding: 10px; + border: 1px solid var(--glass-border); + border-radius: 10px; + background: rgba(255, 255, 255, 0.03); + display: flex; + flex-direction: column; + gap: 8px; + font-family: var(--font-mono); + font-size: 9px; + letter-spacing: 1px; + color: var(--text-secondary); +} + +.vcam-pair { + display: flex; + gap: 10px; + align-items: center; +} + +.vcam-qr { + flex: 0 0 auto; + width: 112px; + height: 112px; + border-radius: 6px; + image-rendering: pixelated; +} + +.vcam-pair-text { + display: flex; + flex-direction: column; + gap: 5px; + min-width: 0; +} + +.vcam-code { + color: var(--text-primary); + font-size: 20px; + font-weight: 600; + letter-spacing: 4px; +} + +.vcam-url { + color: var(--accent); + overflow-wrap: anywhere; + text-decoration: none; +} + +.vcam-address, +#vcam-scale { + background: rgba(255, 255, 255, 0.05); + border: 1px solid rgba(255, 255, 255, 0.14); + border-radius: 6px; + color: var(--text-primary); + font: inherit; + padding: 3px 4px; +} + +.vcam-status { + color: var(--text-primary); + font-size: 10px; + letter-spacing: 0.5px; +} + +.vcam-status.error { + color: #ff8a8a; +} + +.vcam-row { + display: flex; + gap: 10px; + align-items: center; + flex-wrap: wrap; +} + +.vcam-field { + display: flex; + align-items: center; + gap: 6px; +} + +.vcam-grow { + flex: 1; + min-width: 140px; +} + +.vcam-grow input[type='range'] { + flex: 1; + min-width: 60px; + accent-color: var(--accent); +} + +.vcam-field output { + min-width: 38px; + color: var(--accent); + text-align: right; +} + +.vcam-check { + display: flex; + align-items: center; + gap: 4px; +} + +.vcam-actions { + margin-bottom: 0; +} + +.vcam-record[aria-pressed='true'], +.vcam-record:hover:not(:disabled) { + color: #ff5a6a; + border-color: #ff5a6a; + background: rgba(255, 70, 90, 0.14); +} + +.vcam-hint { + margin: 0; + font-family: var(--font-sans); + font-size: 10px; + letter-spacing: 0; +} + +.vcam-hint a { + color: var(--accent); +} + +#scene-panel.collapsed .vcam-card { + display: none !important; +} + +body.recording-mode .vcam-card, +body.scene-playback-mode .vcam-card { + display: none; +} diff --git a/src/ui/templates/layer-panels.html b/src/ui/templates/layer-panels.html index 363cfe0af..038ce9ad5 100644 --- a/src/ui/templates/layer-panels.html +++ b/src/ui/templates/layer-panels.html @@ -163,7 +163,49 @@
+
+
diff --git a/src/ui/vcamPanel.js b/src/ui/vcamPanel.js new file mode 100644 index 000000000..101881649 --- /dev/null +++ b/src/ui/vcamPanel.js @@ -0,0 +1,195 @@ +/** Scene panel VCAM card: pairing QR, live status, lens and take controls. */ +import { formatNumber, msg, t, tn } from '../i18n/index.js'; +import { drawQr, encodeQr } from '../vcam/qr.js'; + +const FOCAL_RANGE = [12, 400]; +const FSTOP_RANGE = [1.2, 22]; +const logMap = ([low, high], v) => low * (high / low) ** v; +const logUnmap = ([low, high], value) => + Math.log(value / low) / Math.log(high / low); + +const HOLD_REASONS = Object.freeze({ + phone: msg('Held from the phone'), + desktop: msg( + 'Held: you moved the globe β€” press RESUME or tap HOLD on the phone', + ), + 'desktop-button': msg('Held'), + navigation: msg('Held: another view took the camera'), + refused: msg('Camera unavailable β€” exit cockpit first'), +}); + +function vcamElements(root = document) { + const ids = { + toggle: 'vcam-toggle-btn', + card: 'vcam-card', + qr: 'vcam-qr', + code: 'vcam-code', + address: 'vcam-address', + url: 'vcam-url', + status: 'vcam-status', + scale: 'vcam-scale', + focal: 'vcam-focal', + focalValue: 'vcam-focal-value', + dof: 'vcam-dof', + fStop: 'vcam-fstop', + fStopValue: 'vcam-fstop-value', + hold: 'vcam-hold-btn', + recenter: 'vcam-recenter-btn', + record: 'vcam-record-btn', + end: 'vcam-end-btn', + }; + return Object.fromEntries( + Object.entries(ids).map(([key, id]) => [key, root.getElementById(id)]), + ); +} + +function statusText(state) { + if (state.status === 'error') + return t(state.error || 'Phone camera unavailable'); + if (state.status === 'starting') return t('Starting…'); + if (state.status === 'idle') return t('Phone camera off'); + if (!state.phoneConnected) + return t('Scan the code with a phone on the same Wi-Fi'); + const mode = + state.mode === '6dof' ? t('AR tracking (6DoF)') : t('Gyro tracking (3DoF)'); + if (state.recording) + return t('Recording {time} Β· {mode}', { + time: formatNumber(state.recordingSec, { maximumFractionDigits: 1 }), + mode, + }); + if (state.held) return t(HOLD_REASONS[state.holdReason] || 'Held'); + if (state.takes) + return tn( + state.takes, + 'Phone connected Β· {mode} Β· {count} take', + 'Phone connected Β· {mode} Β· {count} takes', + { mode }, + ); + return t('Phone connected Β· {mode}', { mode }); +} + +/** + * Bind the VCAM card to a controller. Opening the card starts a pairing + * session; END stops it. Returns a disposer. + */ +export function bindVcamPanel(controller, elements = vcamElements()) { + if (!elements.toggle || !elements.card) return () => {}; + const removers = []; + let lastUrl = null; + let ticker = null; + const listen = (target, type, handler) => { + target?.addEventListener(type, handler); + removers.push(() => target?.removeEventListener(type, handler)); + }; + + const setOpen = (open) => { + elements.card.hidden = !open; + elements.toggle.setAttribute('aria-expanded', String(open)); + elements.toggle.classList.toggle('active', open); + }; + + function present(state) { + const active = state.status !== 'idle'; + elements.toggle.classList.toggle('vcam-live', state.phoneConnected); + elements.code.textContent = state.code || 'Β·Β·Β·Β·Β·Β·'; + if (state.pairingUrl !== lastUrl) { + lastUrl = state.pairingUrl; + elements.url.textContent = state.pairingUrl || ''; + if (state.pairingUrl) elements.url.href = state.pairingUrl; + else elements.url.removeAttribute('href'); + const context = elements.qr.getContext?.('2d'); + if (state.pairingUrl) drawQr(elements.qr, encodeQr(state.pairingUrl)); + else if (context) + context.clearRect(0, 0, elements.qr.width, elements.qr.height); + elements.qr.hidden = !state.pairingUrl; + } + const options = state.addresses.map((address) => address); + if ( + elements.address.options.length !== options.length || + [...elements.address.options].some( + (option, i) => option.value !== options[i], + ) + ) { + elements.address.replaceChildren( + ...options.map((address) => new Option(address, address)), + ); + } + elements.address.value = state.address || ''; + elements.address.hidden = options.length < 2; + elements.status.textContent = statusText(state); + elements.status.classList.toggle('error', state.status === 'error'); + + const { settings } = state; + const scale = String(settings.scale); + if (![...elements.scale.options].some((option) => option.value === scale)) + elements.scale.append( + new Option(`1:${formatNumber(settings.scale)}`, scale), + ); + elements.scale.value = scale; + if (document.activeElement !== elements.focal) + elements.focal.value = String(logUnmap(FOCAL_RANGE, settings.focalMm)); + // Unit symbols stay as they are in every language. + elements.focalValue.textContent = `${formatNumber(Math.round(settings.focalMm))}mm`; + if (document.activeElement !== elements.fStop) + elements.fStop.value = String(logUnmap(FSTOP_RANGE, settings.fStop)); + elements.fStopValue.textContent = `f/${formatNumber(settings.fStop, { + maximumFractionDigits: 1, + minimumFractionDigits: 1, + })}`; + elements.dof.checked = settings.dof; + + for (const control of [elements.hold, elements.recenter, elements.record]) + control.disabled = !state.phoneConnected; + elements.end.disabled = !active; + elements.hold.setAttribute('aria-pressed', String(state.held)); + elements.hold.textContent = state.held ? t('RESUME') : t('HOLD'); + elements.record.setAttribute('aria-pressed', String(state.recording)); + elements.record.textContent = state.recording ? t('β–  STOP') : t('● REC'); + + clearInterval(ticker); + ticker = state.recording + ? setInterval(() => present(controller.getState()), 500) + : null; + } + + listen(elements.toggle, 'click', () => { + const open = elements.card.hidden; + setOpen(open); + if (open && !controller.active) controller.start(); + }); + listen(elements.end, 'click', () => { + controller.stop(); + setOpen(false); + }); + listen(elements.address, 'change', () => + controller.setAddress(elements.address.value), + ); + listen(elements.scale, 'change', () => + controller.set('scale', Number(elements.scale.value)), + ); + listen(elements.focal, 'input', () => + controller.set( + 'focalMm', + logMap(FOCAL_RANGE, Number(elements.focal.value)), + ), + ); + listen(elements.fStop, 'input', () => + controller.set('fStop', logMap(FSTOP_RANGE, Number(elements.fStop.value))), + ); + listen(elements.dof, 'change', () => + controller.set('dof', elements.dof.checked), + ); + listen(elements.hold, 'click', () => + controller.getState().held ? controller.release() : controller.hold(), + ); + listen(elements.recenter, 'click', () => controller.recenter()); + listen(elements.record, 'click', () => controller.toggleRecording()); + + const unsubscribe = controller.subscribe(present); + return () => { + clearInterval(ticker); + unsubscribe(); + for (const remove of removers.splice(0)) remove(); + setOpen(false); + }; +} diff --git a/src/vcam/controller.js b/src/vcam/controller.js new file mode 100644 index 000000000..418923789 --- /dev/null +++ b/src/vcam/controller.js @@ -0,0 +1,706 @@ +/** + * Desktop side of the virtual camera: pairs a phone through the local relay, + * drives the Cesium camera from its pose, streams a preview back and records + * takes into the Scene Director. + * + * Camera ownership follows the application's rules. The rig claims the camera + * through the shared navigation seam (so Cockpit can refuse it) and lets go + * β€” holds β€” whenever anything else takes it: a search, a scene playing, or + * the mouse on the globe. Releasing the hold re-anchors the rig wherever the + * camera now is, so nothing jumps. + */ +import * as Cesium from 'cesium'; +import { createRig, SCALE_RANGE } from './rig.js'; +import { msg, t } from '../i18n/index.js'; +import { + holdContinuousRender, + releaseContinuousRender, +} from '../renderGovernor.js'; + +const RECORD_INTERVAL_MS = 1000 / 30; +// The scene document's per-array limit. +export const MAX_TAKE_SAMPLES = 10000; +const PREVIEW_FPS = 12; +const PREVIEW_WIDTH = 720; +const PREVIEW_QUALITY = 0.62; +const PREVIEW_BACKLOG_BYTES = 512 * 1024; +// Sticks stop when the phone stops talking (a dropped link, a locked screen). +const STALE_POSE_MS = 400; +const STATE_HEARTBEAT_MS = 1000; +const MIN_GROUND_CLEARANCE_M = 0.5; +const MAX_STEP_SEC = 0.5; +const MAX_RECONNECT_ATTEMPTS = 5; +const RENDER_HOLD = 'vcam'; + +const finite = (value) => typeof value === 'number' && Number.isFinite(value); +const vectorOf = (value, length) => + Array.isArray(value) && value.length === length && value.every(finite) + ? value + : null; +const round = (value, places) => { + const k = 10 ** places; + return Math.round(value * k) / k; +}; + +/** + * @param {object} options + * @param {Cesium.Viewer} options.viewer + * @param {object} options.navigation - Shell navigation: `runImmediateNavigation`, `subscribeCameraHandoff`. + * @param {object} options.director - SceneDirector (`addRecordedShot`). + * @param {object} options.lens - Camera lens (`setFocalLength`, `setDepthOfField`, ...). + */ +export function createVcamController({ + viewer, + navigation, + director, + lens, + fetchImpl = (...args) => fetch(...args), + WebSocketImpl = globalThis.WebSocket, + pageLocation = globalThis.location, + now = () => performance.now(), +}) { + const camera = viewer.camera; + const scene = viewer.scene; + const listeners = new Set(); + const rig = createRig(); + const state = { + status: 'idle', + error: null, + code: null, + port: null, + addresses: [], + address: null, + caFingerprint: null, + phoneConnected: false, + mode: null, + driving: false, + held: false, + holdReason: null, + recording: false, + recordStartedAt: 0, + takes: 0, + lastTake: null, + }; + let session = null; + let socket = null; + let reconnectAttempts = 0; + let latestPose = null; + let lastStep = null; + let anchor = null; + let claiming = false; + let rendering = false; + let samples = []; + let lastSampleAt = -Infinity; + let previewBusy = false; + let lastPreviewAt = -Infinity; + let heartbeat = null; + let destroyed = false; + const previewCanvas = document.createElement('canvas'); + const removers = []; + + // ---------------------------------------------------------------- state + const snapshot = () => ({ + ...state, + pairingUrl: + state.address && state.port && state.code + ? `https://${state.address}:${state.port}/#${state.code}` + : null, + recordingSec: state.recording ? (now() - state.recordStartedAt) / 1000 : 0, + settings: settings(), + }); + function settings() { + const dof = lens?.getDepthOfField?.() || {}; + return { + ...rig.settings, + focalMm: round(lens?.getFocalLength?.() ?? 31.18, 2), + fStop: dof.fStop ?? 2.8, + dof: !!dof.enabled, + }; + } + function publish() { + const current = snapshot(); + for (const listener of listeners) { + try { + listener(current); + } catch (error) { + console.error('[vcam] listener failed', error); + } + } + sendPhoneState(current); + } + function sendPhoneState(current = snapshot()) { + send({ + type: 'state', + state: { + recording: current.recording, + elapsedSec: round(current.recordingSec, 2), + held: current.held, + takes: current.takes, + ...current.settings, + }, + }); + } + function send(message) { + if (socket?.readyState === 1) socket.send(JSON.stringify(message)); + } + function toastPhone(text) { + send({ type: 'toast', text }); + } + + // ------------------------------------------------------------- session + /** Create a pairing session and connect the desktop end of the relay. */ + async function start() { + if (destroyed || session) return snapshot(); + Object.assign(state, { status: 'starting', error: null }); + publish(); + let response; + try { + response = await fetchImpl('/api/vcam/session', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{}', + }); + } catch { + return fail(msg('The phone camera service is unavailable')); + } + const body = await response.json().catch(() => ({})); + if (!response.ok) + return fail( + body.code === 'port-in-use' + ? msg('The phone camera port is in use (set VCAM_PORT)') + : body.code === 'no-network' + ? msg('No network connection for a phone to reach') + : msg('The phone camera service is unavailable'), + ); + if (destroyed) return snapshot(); + session = body; + reconnectAttempts = 0; + Object.assign(state, { + status: 'waiting', + code: body.code, + port: body.port, + addresses: body.addresses || [], + address: body.addresses?.[0] || null, + caFingerprint: body.caFingerprint || null, + }); + openSocket(); + heartbeat = setInterval(() => { + if (state.phoneConnected) sendPhoneState(); + }, STATE_HEARTBEAT_MS); + publish(); + return snapshot(); + } + + function fail(error) { + Object.assign(state, { status: 'error', error }); + publish(); + return snapshot(); + } + + function openSocket() { + if (!session || destroyed) return; + const protocol = pageLocation.protocol === 'https:' ? 'wss:' : 'ws:'; + const ws = new WebSocketImpl( + `${protocol}//${pageLocation.host}/api/vcam/host?key=${encodeURIComponent(session.hostKey)}`, + ); + ws.binaryType = 'arraybuffer'; + socket = ws; + ws.onopen = () => { + if (socket === ws) reconnectAttempts = 0; + }; + ws.onmessage = (event) => { + if (socket !== ws || typeof event.data !== 'string') return; + let message; + try { + message = JSON.parse(event.data); + } catch { + return; + } + handleMessage(message); + }; + ws.onclose = (event) => { + if (socket !== ws) return; + socket = null; + setPhoneConnected(false); + // 4000: the server ended the session; anything else may be transient. + // A restarted server refuses the old key, which looks like a network + // failure from here; give up rather than retry forever. + if ( + event.code === 4000 || + event.code === 4003 || + ++reconnectAttempts > MAX_RECONNECT_ATTEMPTS + ) { + endLocally(msg('Phone camera session ended')); + return; + } + setTimeout(() => { + if (session && !socket) openSocket(); + }, 1500); + }; + } + + /** End the pairing session and release the camera. */ + async function stop() { + const ending = session; + endLocally(null); + if (ending) + await fetchImpl('/api/vcam/session', { + method: 'DELETE', + headers: { 'X-Vcam-Host-Key': ending.hostKey }, + }).catch(() => {}); + return snapshot(); + } + + function endLocally(error) { + if (state.recording) stopRecording(); + session = null; + const ws = socket; + socket = null; + ws?.close(); + clearInterval(heartbeat); + heartbeat = null; + setPhoneConnected(false); + Object.assign(state, { + status: error ? 'error' : 'idle', + error, + code: null, + port: null, + addresses: [], + address: null, + mode: null, + }); + stopDriving(); + publish(); + } + + function setPhoneConnected(connected) { + if (state.phoneConnected === connected) return; + state.phoneConnected = connected; + if (!connected) { + latestPose = null; + stopDriving(); + if (state.recording) stopRecording(); + } + if (connected && state.status === 'waiting') state.status = 'connected'; + if (!connected && state.status === 'connected') state.status = 'waiting'; + syncRenderHold(); + publish(); + } + + // ------------------------------------------------------------ messages + function handleMessage(message) { + switch (message?.type) { + case 'code': + if (typeof message.code === 'string') { + state.code = message.code.slice(0, 16); + publish(); + } + return; + case 'peer': + if (message.role === 'phone') setPhoneConnected(!!message.connected); + return; + case 'hello': + state.mode = message.mode === '6dof' ? '6dof' : '3dof'; + setPhoneConnected(true); + publish(); + return; + case 'pose': + return receivePose(message); + case 'cmd': + return command(message.cmd); + case 'set': + return set(message.key, message.value); + case 'focus': + if (finite(message.x) && finite(message.y)) + focusAt(message.x, message.y); + return; + } + } + + function receivePose(message) { + const q = vectorOf(message.q, 4); + if (!q) return; + latestPose = { + q, + p: message.p === null ? null : vectorOf(message.p, 3), + joy: (vectorOf(message.j, 4) || [0, 0, 0, 0]).map((v) => + Math.max(-1, Math.min(1, v)), + ), + at: now(), + }; + const mode = latestPose.p ? '6dof' : '3dof'; + if (state.mode !== mode) { + state.mode = mode; + publish(); + } + if (!state.phoneConnected) setPhoneConnected(true); + if (!state.driving && !state.held) beginDriving(); + } + + function command(name) { + if (name === 'record') startRecording(); + else if (name === 'stop') stopRecording(); + else if (name === 'hold') hold('phone'); + else if (name === 'release') release(); + else if (name === 'recenter') recenter(); + } + + /** Change one setting from either the phone or the desktop panel. */ + function set(key, value) { + if (key === 'scale' && finite(value)) { + const scale = Math.max(SCALE_RANGE[0], Math.min(SCALE_RANGE[1], value)); + shiftAnchor(rig.configure({ scale })); + // Depth of field treats the world at the rig's scale: a miniature. + lens?.setDepthOfField?.({ scale }); + } else if (key === 'smoothing' && finite(value)) + rig.configure({ smoothing: value }); + else if ( + (key === 'heightLock' || key === 'horizonLock') && + typeof value === 'boolean' + ) + shiftAnchor(rig.configure({ [key]: value })); + else if (key === 'focalMm' && finite(value)) lens?.setFocalLength?.(value); + else if (key === 'fStop' && finite(value)) + lens?.setDepthOfField?.({ fStop: value }); + else if (key === 'dof' && typeof value === 'boolean') + lens?.setDepthOfField?.({ enabled: value, scale: rig.settings.scale }); + else return; + publish(); + } + + // -------------------------------------------------------------- camera + function claimCamera() { + if (typeof navigation?.runImmediateNavigation !== 'function') return true; + claiming = true; + try { + return navigation.runImmediateNavigation('vcam', () => true) !== false; + } finally { + claiming = false; + } + } + + function beginDriving() { + if (!latestPose || state.driving) return; + if (!claimCamera()) { + hold('refused'); + toastPhone(t('Camera unavailable β€” exit cockpit first')); + return; + } + camera.cancelFlight?.(); + reanchor(); + state.driving = true; + state.held = false; + state.holdReason = null; + syncRenderHold(); + publish(); + } + + function stopDriving() { + state.driving = false; + lastStep = null; + syncRenderHold(); + } + + /** Start the phone's motion from the camera's current pose. */ + function reanchor() { + anchor = Cesium.Cartesian3.clone(camera.positionWC); + if (latestPose) { + rig.step({ ...latestPose, joy: [0, 0, 0, 0], dt: 0 }); + rig.recenter(camera.heading); + } + lastStep = null; + } + + function hold(reason = 'phone') { + if (state.held && state.holdReason === reason) return; + state.held = true; + state.holdReason = reason; + stopDriving(); + publish(); + } + + function release() { + if (!state.held) return; + state.held = false; + state.holdReason = null; + if (latestPose) beginDriving(); + publish(); + } + + function recenter() { + if (!state.driving) return release(); + reanchor(); + publish(); + } + + function shiftAnchor(shift) { + if (!anchor || !shift || (!shift[0] && !shift[1] && !shift[2])) return; + const frame = Cesium.Transforms.eastNorthUpToFixedFrame(anchor); + const delta = Cesium.Matrix4.multiplyByPointAsVector( + frame, + new Cesium.Cartesian3(shift[0], shift[1], shift[2]), + new Cesium.Cartesian3(), + ); + anchor = Cesium.Cartesian3.add(anchor, delta, anchor); + } + + function placeCamera() { + if (!state.driving || state.held || !latestPose || !anchor) return; + const at = now(); + // Smoothing and sticks follow wall time, so a slow frame rate does not + // slow the camera; only a long stall (a hidden tab) is cut short. + const dt = + lastStep === null ? 0 : Math.min(MAX_STEP_SEC, (at - lastStep) / 1000); + lastStep = at; + const stale = at - latestPose.at > STALE_POSE_MS; + const out = rig.step({ + ...latestPose, + joy: stale ? [0, 0, 0, 0] : latestPose.joy, + dt, + }); + shiftAnchor(out.anchorShift); + const frame = Cesium.Transforms.eastNorthUpToFixedFrame(anchor); + const destination = Cesium.Matrix4.multiplyByPoint( + frame, + Cesium.Cartesian3.fromArray(out.offset), + new Cesium.Cartesian3(), + ); + const direction = Cesium.Cartesian3.normalize( + Cesium.Matrix4.multiplyByPointAsVector( + frame, + Cesium.Cartesian3.fromArray(out.forward), + new Cesium.Cartesian3(), + ), + new Cesium.Cartesian3(), + ); + const up = Cesium.Cartesian3.normalize( + Cesium.Matrix4.multiplyByPointAsVector( + frame, + Cesium.Cartesian3.fromArray(out.up), + new Cesium.Cartesian3(), + ), + new Cesium.Cartesian3(), + ); + keepAboveGround(destination); + camera.setView({ destination, orientation: { direction, up } }); + } + + /** Stop at the terrain surface rather than passing under it. */ + function keepAboveGround(position) { + const cartographic = Cesium.Cartographic.fromCartesian(position); + if (!cartographic) return; + const ground = scene.globe?.show + ? scene.globe.getHeight(cartographic) + : undefined; + if (!finite(ground)) return; + const floor = ground + MIN_GROUND_CLEARANCE_M; + if (cartographic.height >= floor) return; + cartographic.height = floor; + Cesium.Cartographic.toCartesian(cartographic, undefined, position); + } + + /** Tap-to-focus: focus the lens on what lies under a normalized point. */ + function focusAt(x, y) { + const canvas = scene.canvas; + const point = new Cesium.Cartesian2( + Math.max(0, Math.min(1, x)) * canvas.clientWidth, + Math.max(0, Math.min(1, y)) * canvas.clientHeight, + ); + let target; + try { + if (scene.pickPositionSupported) target = scene.pickPosition(point); + } catch { + target = undefined; + } + if (!target) { + const ray = camera.getPickRay(point); + target = ray && scene.globe?.pick(ray, scene); + } + if (!target) { + toastPhone(t('Nothing to focus on there')); + return; + } + const focusM = Cesium.Cartesian3.distance(camera.positionWC, target); + lens?.setDepthOfField?.({ + enabled: true, + focusM, + scale: rig.settings.scale, + }); + publish(); + } + + // ----------------------------------------------------------- recording + function startRecording() { + if (state.recording || !session) return; + samples = []; + lastSampleAt = -Infinity; + state.recording = true; + state.recordStartedAt = now(); + syncRenderHold(); + publish(); + } + + function recordSample() { + if (!state.recording) return; + const at = now(); + if (at - lastSampleAt < RECORD_INTERVAL_MS - 1) return; + const carto = camera.positionCartographic; + if (!carto) return; + lastSampleAt = at; + const t0 = samples.length + ? state.recordStartedAt + : (state.recordStartedAt = at); + samples.push([ + round((at - t0) / 1000, 3), + round(Cesium.Math.toDegrees(carto.longitude), 7), + round(Cesium.Math.toDegrees(carto.latitude), 7), + round(carto.height, 2), + round(Cesium.Math.toDegrees(camera.heading), 2), + round(Cesium.Math.clamp(Cesium.Math.toDegrees(camera.pitch), -90, 90), 2), + round(Cesium.Math.toDegrees(camera.roll), 2), + round(lens?.getFocalLength?.() ?? 31.18, 2), + ]); + if (samples.length >= MAX_TAKE_SAMPLES) { + stopRecording(); + toastPhone(t('Take reached its length limit and was saved')); + } + } + + /** Stop recording and add the take to the selected scene. */ + function stopRecording() { + if (!state.recording) return null; + state.recording = false; + syncRenderHold(); + const take = samples; + samples = []; + let result = null; + if (take.length >= 2) { + // Playback takes the short arc between samples, so headings may wrap; + // rounding can still reach 360, which stays inside the Β±360 bound. + for (const row of take) + for (const column of [4, 6]) + row[column] = round(((row[column] % 360) + 360) % 360, 2); + const dof = lens?.getDepthOfField?.(); + result = director?.addRecordedShot?.({ + samples: take, + lens: { + focalMm: take.at(-1)[7], + ...(dof?.enabled ? { dof: { ...dof } } : {}), + }, + }); + if (result) { + state.takes += 1; + state.lastTake = { ...result, durationSec: take.at(-1)[0] }; + toastPhone(t('Take saved to the scene')); + } + } else toastPhone(t('Take too short to keep')); + publish(); + return result; + } + + // ------------------------------------------------------------- preview + function sendPreview() { + if (!state.phoneConnected || socket?.readyState !== 1 || previewBusy) + return; + const at = now(); + if (at - lastPreviewAt < 1000 / PREVIEW_FPS) return; + if (socket.bufferedAmount > PREVIEW_BACKLOG_BYTES) return; + const source = scene.canvas; + if (!source?.width || !source?.height) return; + lastPreviewAt = at; + previewBusy = true; + const width = Math.min(PREVIEW_WIDTH, source.width); + const height = Math.round((width * source.height) / source.width); + if (previewCanvas.width !== width) previewCanvas.width = width; + if (previewCanvas.height !== height) previewCanvas.height = height; + // Copy inside postRender, while the drawing buffer still holds this frame. + previewCanvas.getContext('2d').drawImage(source, 0, 0, width, height); + previewCanvas.toBlob( + async (blob) => { + try { + if (blob && socket?.readyState === 1) + socket.send(await blob.arrayBuffer()); + } finally { + previewBusy = false; + } + }, + 'image/jpeg', + PREVIEW_QUALITY, + ); + } + + function syncRenderHold() { + const wanted = + !destroyed && (state.driving || state.phoneConnected || state.recording); + if (wanted === rendering) return; + rendering = wanted; + if (wanted) holdContinuousRender(RENDER_HOLD); + else releaseContinuousRender(RENDER_HOLD); + } + + // ------------------------------------------------------------ wiring + removers.push(scene.preUpdate.addEventListener(() => placeCamera())); + removers.push( + scene.postRender.addEventListener(() => { + recordSample(); + sendPreview(); + }), + ); + const yieldToNavigation = () => { + if (!claiming && state.driving) hold('navigation'); + }; + const unsubscribeHandoff = + navigation?.subscribeCameraHandoff?.(yieldToNavigation); + if (unsubscribeHandoff) removers.push(unsubscribeHandoff); + const canvas = scene.canvas; + const yieldToMouse = () => { + if (state.driving) hold('desktop'); + }; + for (const type of ['pointerdown', 'wheel']) { + canvas?.addEventListener(type, yieldToMouse, { passive: true }); + removers.push(() => canvas?.removeEventListener(type, yieldToMouse)); + } + + return { + start, + stop, + hold: () => hold('desktop-button'), + release, + recenter, + set, + startRecording, + stopRecording, + toggleRecording: () => + state.recording ? stopRecording() : startRecording(), + setAddress(address) { + if (state.addresses.includes(address)) { + state.address = address; + publish(); + } + }, + getState: snapshot, + subscribe(listener) { + listeners.add(listener); + listener(snapshot()); + return () => listeners.delete(listener); + }, + get active() { + return !!session; + }, + destroy() { + if (destroyed) return; + const ending = session; + endLocally(null); + if (ending) + fetchImpl('/api/vcam/session', { + method: 'DELETE', + headers: { 'X-Vcam-Host-Key': ending.hostKey }, + keepalive: true, + }).catch(() => {}); + destroyed = true; + syncRenderHold(); + for (const remove of removers.splice(0)) remove(); + listeners.clear(); + }, + }; +} diff --git a/src/vcam/qr.js b/src/vcam/qr.js new file mode 100644 index 000000000..41e9eae22 --- /dev/null +++ b/src/vcam/qr.js @@ -0,0 +1,325 @@ +/** + * Minimal QR Code encoder for the phone pairing link: byte mode, error + * correction level M, versions 1–10 (up to 213 bytes). It follows ISO/IEC + * 18004 directly, so pairing needs no network service or dependency. + */ + +// Level M, indexed by version - 1. +const ECC_PER_BLOCK = [10, 16, 26, 18, 24, 16, 18, 22, 22, 26]; +const BLOCKS = [1, 1, 1, 2, 2, 4, 4, 4, 5, 5]; +const MAX_VERSION = 10; +// Format information encodes level M as 0b00. +const FORMAT_LEVEL_M = 0; + +function rawDataModules(version) { + let modules = (16 * version + 128) * version + 64; + if (version >= 2) { + const align = Math.floor(version / 7) + 2; + modules -= (25 * align - 10) * align - 55; + if (version >= 7) modules -= 36; + } + return modules; +} + +const dataCodewords = (version) => + Math.floor(rawDataModules(version) / 8) - + ECC_PER_BLOCK[version - 1] * BLOCKS[version - 1]; + +function gfMultiply(x, y) { + let z = 0; + for (let i = 7; i >= 0; i--) { + z = (z << 1) ^ ((z >>> 7) * 0x11d); + z ^= ((y >>> i) & 1) * x; + } + return z; +} + +function rsDivisor(degree) { + const result = new Array(degree).fill(0); + result[degree - 1] = 1; + let root = 1; + for (let i = 0; i < degree; i++) { + for (let j = 0; j < degree; j++) { + result[j] = gfMultiply(result[j], root); + if (j + 1 < degree) result[j] ^= result[j + 1]; + } + root = gfMultiply(root, 0x02); + } + return result; +} + +function rsRemainder(data, divisor) { + const result = new Array(divisor.length).fill(0); + for (const byte of data) { + const factor = byte ^ result.shift(); + result.push(0); + divisor.forEach((coefficient, i) => { + result[i] ^= gfMultiply(coefficient, factor); + }); + } + return result; +} + +function alignmentPositions(version, size) { + if (version === 1) return []; + const count = Math.floor(version / 7) + 2; + const step = Math.ceil((version * 4 + 4) / (count * 2 - 2)) * 2; + const result = [6]; + for (let position = size - 7; result.length < count; position -= step) + result.splice(1, 0, position); + return result; +} + +const MASKS = [ + (x, y) => (x + y) % 2 === 0, + (x, y) => y % 2 === 0, + (x) => x % 3 === 0, + (x, y) => (x + y) % 3 === 0, + (x, y) => (Math.floor(x / 3) + Math.floor(y / 2)) % 2 === 0, + (x, y) => ((x * y) % 2) + ((x * y) % 3) === 0, + (x, y) => (((x * y) % 2) + ((x * y) % 3)) % 2 === 0, + (x, y) => (((x + y) % 2) + ((x * y) % 3)) % 2 === 0, +]; + +function codewordsFor(bytes, version) { + const capacity = dataCodewords(version); + const bits = []; + const push = (value, length) => { + for (let i = length - 1; i >= 0; i--) bits.push((value >>> i) & 1); + }; + push(0b0100, 4); + push(bytes.length, version < 10 ? 8 : 16); + for (const byte of bytes) push(byte, 8); + push(0, Math.min(4, capacity * 8 - bits.length)); + push(0, (8 - (bits.length % 8)) % 8); + const data = []; + for (let i = 0; i < bits.length; i += 8) + data.push(bits.slice(i, i + 8).reduce((value, bit) => (value << 1) | bit)); + for (let pad = 0xec; data.length < capacity; pad ^= 0xec ^ 0x11) + data.push(pad); + + const blocks = BLOCKS[version - 1]; + const eccLength = ECC_PER_BLOCK[version - 1]; + const raw = Math.floor(rawDataModules(version) / 8); + const shortBlocks = blocks - (raw % blocks); + const shortLength = Math.floor(raw / blocks); + const divisor = rsDivisor(eccLength); + const filled = []; + for (let i = 0, k = 0; i < blocks; i++) { + const block = data.slice( + k, + k + shortLength - eccLength + (i < shortBlocks ? 0 : 1), + ); + k += block.length; + const ecc = rsRemainder(block, divisor); + if (i < shortBlocks) block.push(0); + filled.push(block.concat(ecc)); + } + const result = []; + for (let i = 0; i < filled[0].length; i++) + filled.forEach((block, j) => { + if (i !== shortLength - eccLength || j >= shortBlocks) + result.push(block[i]); + }); + return result; +} + +function penalty(modules) { + const size = modules.length; + let score = 0; + const lines = []; + for (let y = 0; y < size; y++) lines.push(modules[y]); + for (let x = 0; x < size; x++) lines.push(modules.map((row) => row[x])); + for (const line of lines) { + let run = 1; + for (let i = 1; i <= size; i++) { + if (i < size && line[i] === line[i - 1]) run++; + else { + if (run >= 5) score += run - 2; + run = 1; + } + } + const text = line.map(Number).join(''); + for (const pattern of ['00001011101', '10111010000']) + for ( + let at = text.indexOf(pattern); + at >= 0; + at = text.indexOf(pattern, at + 1) + ) + score += 40; + } + let dark = 0; + for (let y = 0; y < size; y++) + for (let x = 0; x < size; x++) { + if (modules[y][x]) dark++; + if ( + x < size - 1 && + y < size - 1 && + modules[y][x] === modules[y][x + 1] && + modules[y][x] === modules[y + 1][x] && + modules[y][x] === modules[y + 1][x + 1] + ) + score += 3; + } + const total = size * size; + score += (Math.ceil(Math.abs(dark * 20 - total * 10) / total) - 1) * 10; + return score; +} + +/** + * Encode text as a QR Code. + * @param {string} text + * @returns {boolean[][]} Square module matrix, `true` for dark, row-major. + */ +export function encodeQr(text) { + const bytes = [...new TextEncoder().encode(text)]; + let version = 1; + // Byte mode costs 4 + 8 (or 16) bits of header. + while ( + version <= MAX_VERSION && + 4 + (version < 10 ? 8 : 16) + bytes.length * 8 > dataCodewords(version) * 8 + ) + version++; + if (version > MAX_VERSION) + throw new RangeError('Text is too long for a QR code'); + + const size = version * 4 + 17; + const modules = Array.from({ length: size }, () => + new Array(size).fill(false), + ); + const reserved = Array.from({ length: size }, () => + new Array(size).fill(false), + ); + const set = (x, y, dark) => { + modules[y][x] = dark; + reserved[y][x] = true; + }; + + for (let i = 0; i < size; i++) { + set(6, i, i % 2 === 0); + set(i, 6, i % 2 === 0); + } + for (const [cx, cy] of [ + [3, 3], + [size - 4, 3], + [3, size - 4], + ]) + for (let dy = -4; dy <= 4; dy++) + for (let dx = -4; dx <= 4; dx++) { + const x = cx + dx; + const y = cy + dy; + if (x < 0 || y < 0 || x >= size || y >= size) continue; + const distance = Math.max(Math.abs(dx), Math.abs(dy)); + set(x, y, distance !== 2 && distance !== 4); + } + const align = alignmentPositions(version, size); + for (let i = 0; i < align.length; i++) + for (let j = 0; j < align.length; j++) { + const corner = + (i === 0 && j === 0) || + (i === 0 && j === align.length - 1) || + (i === align.length - 1 && j === 0); + if (corner) continue; + for (let dy = -2; dy <= 2; dy++) + for (let dx = -2; dx <= 2; dx++) + set( + align[i] + dx, + align[j] + dy, + Math.max(Math.abs(dx), Math.abs(dy)) !== 1, + ); + } + const drawFormat = (mask) => { + const data = (FORMAT_LEVEL_M << 3) | mask; + let rem = data; + for (let i = 0; i < 10; i++) rem = (rem << 1) ^ ((rem >>> 9) * 0x537); + const bits = ((data << 10) | rem) ^ 0x5412; + const bit = (i) => ((bits >>> i) & 1) !== 0; + for (let i = 0; i <= 5; i++) set(8, i, bit(i)); + set(8, 7, bit(6)); + set(8, 8, bit(7)); + set(7, 8, bit(8)); + for (let i = 9; i < 15; i++) set(14 - i, 8, bit(i)); + for (let i = 0; i < 8; i++) set(size - 1 - i, 8, bit(i)); + for (let i = 8; i < 15; i++) set(8, size - 15 + i, bit(i)); + set(8, size - 8, true); + }; + drawFormat(0); + if (version >= 7) { + let rem = version; + for (let i = 0; i < 12; i++) rem = (rem << 1) ^ ((rem >>> 11) * 0x1f25); + const bits = (version << 12) | rem; + for (let i = 0; i < 18; i++) { + const dark = ((bits >>> i) & 1) !== 0; + const a = size - 11 + (i % 3); + const b = Math.floor(i / 3); + set(a, b, dark); + set(b, a, dark); + } + } + + const codewords = codewordsFor(bytes, version); + let bit = 0; + for (let right = size - 1; right >= 1; right -= 2) { + if (right === 6) right = 5; + for (let vertical = 0; vertical < size; vertical++) + for (let j = 0; j < 2; j++) { + const x = right - j; + const upward = ((right + 1) & 2) === 0; + const y = upward ? size - 1 - vertical : vertical; + if (reserved[y][x] || bit >= codewords.length * 8) continue; + modules[y][x] = ((codewords[bit >>> 3] >>> (7 - (bit & 7))) & 1) !== 0; + bit++; + } + } + + let best = null; + for (let mask = 0; mask < MASKS.length; mask++) { + const candidate = modules.map((row, y) => + row.map((dark, x) => + reserved[y][x] ? dark : dark !== MASKS[mask](x, y), + ), + ); + // Format bits depend on the mask; draw them into the candidate. + const saved = modules.map((row) => [...row]); + for (let y = 0; y < size; y++) modules[y] = candidate[y]; + drawFormat(mask); + const score = penalty(modules); + if (!best || score < best.score) + best = { score, matrix: modules.map((row) => [...row]) }; + for (let y = 0; y < size; y++) modules[y] = saved[y]; + } + return best.matrix; +} + +/** + * Draw a QR matrix into a canvas with the standard four-module quiet zone. + * @param {HTMLCanvasElement} canvas + * @param {boolean[][]} matrix + * @param {number} [pixels] - Target edge length in CSS pixels. + */ +export function drawQr(canvas, matrix, pixels = 168) { + const quiet = 4; + const count = matrix.length + quiet * 2; + const scale = Math.max( + 1, + Math.floor((pixels * (window.devicePixelRatio || 1)) / count), + ); + canvas.width = canvas.height = count * scale; + canvas.style.width = + canvas.style.height = `${(count * scale) / (window.devicePixelRatio || 1)}px`; + const context = canvas.getContext('2d'); + context.fillStyle = '#fff'; + context.fillRect(0, 0, canvas.width, canvas.height); + context.fillStyle = '#000'; + matrix.forEach((row, y) => + row.forEach((dark, x) => { + if (dark) + context.fillRect( + (x + quiet) * scale, + (y + quiet) * scale, + scale, + scale, + ); + }), + ); +} diff --git a/src/vcam/qr.test.mjs b/src/vcam/qr.test.mjs new file mode 100644 index 000000000..39a2a95a5 --- /dev/null +++ b/src/vcam/qr.test.mjs @@ -0,0 +1,53 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import crypto from 'node:crypto'; +import { encodeQr } from './qr.js'; + +const hash = (matrix) => + crypto + .createHash('sha256') + .update(matrix.map((row) => row.map(Number).join('')).join('\n')) + .digest('hex'); + +test('pairing links encode to the matrices an independent decoder read back', () => { + // Both hashes were checked by decoding the matrices with jsQR, which this + // repository does not depend on; a change to the encoder must be rechecked. + assert.equal( + hash(encodeQr('https://192.168.1.23:4174/#9XUXQ4')), + '9be04e45ca9b039ed9af52ce4966541f8a60dda9a0eb8555b8664d5cdac2cb05', + ); + assert.equal( + hash(encodeQr('https://192.168.100.200:4174/#ABCDEF')), + '98e8a59c7040c74bdbe3df100b9844143e97ba74f0629d843fcac0178ccc6855', + ); +}); + +test('the symbol grows with the text and keeps its fixed patterns', () => { + for (const [text, version] of [ + ['A', 1], + ['https://10.0.0.5:4174/#ABCDEF', 3], + ['x'.repeat(150), 8], + ['z'.repeat(213), 10], + ]) { + const matrix = encodeQr(text); + assert.equal(matrix.length, version * 4 + 17, text.slice(0, 12)); + const size = matrix.length; + for (const [x, y] of [ + [0, 0], + [size - 7, 0], + [0, size - 7], + ]) { + // Finder: dark 7Γ—7 ring, light ring, dark 3Γ—3 core. + assert.equal(matrix[y][x], true); + assert.equal(matrix[y + 1][x + 1], false); + assert.equal(matrix[y + 3][x + 3], true); + } + // Timing patterns alternate between the finders. + for (let i = 8; i < size - 8; i++) { + assert.equal(matrix[6][i], i % 2 === 0); + assert.equal(matrix[i][6], i % 2 === 0); + } + assert.equal(matrix[size - 8][8], true, 'the dark module'); + } + assert.throws(() => encodeQr('q'.repeat(214)), RangeError); +}); diff --git a/src/vcam/rig.js b/src/vcam/rig.js new file mode 100644 index 000000000..cdb4cf9a4 --- /dev/null +++ b/src/vcam/rig.js @@ -0,0 +1,272 @@ +/** + * Virtual-camera rig: maps a phone's tracked pose onto a camera in the local + * east-north-up (ENU) frame of an anchor point, with no renderer dependency. + * + * The phone reports a Y-up pose (+x right, +y up, -z forward, WebXR's + * convention) whose yaw is arbitrary. Recenter aligns the phone's current + * yaw with the camera's heading and zeroes its position, so the camera never + * jumps in place or heading: it continues from where it is and follows the + * phone's motion from there, multiplied by the motion scale. A 1:1000 rig + * turns one step across the room into a kilometre over the city. + * + * Every step returns an `anchorShift`, an ENU translation the caller applies + * to its geographic anchor before reading the camera offset from it. The + * sticks, scale changes and recenters move the anchor rather than the + * phone's origin, so the camera stays continuous. + */ + +/** Motion-scale presets: walking, crane, drone, city flyover, orbit. */ +export const SCALE_PRESETS = Object.freeze([1, 10, 100, 1000, 10000]); +export const SCALE_RANGE = Object.freeze([0.01, 100000]); +/** Stick travel at full deflection, in metres per second at 1:1. */ +const STICK_SPEED_MPS = 2.5; +const STICK_YAW_RAD_PER_S = Math.PI / 3; +/** Longest smoothing time constant, at smoothing 1. */ +const MAX_SMOOTHING_SEC = 0.6; + +const scaleVector = (a, k) => [a[0] * k, a[1] * k, a[2] * k]; +const add = (a, b) => [a[0] + b[0], a[1] + b[1], a[2] + b[2]]; +const subtract = (a, b) => [a[0] - b[0], a[1] - b[1], a[2] - b[2]]; +const cross = (a, b) => [ + a[1] * b[2] - a[2] * b[1], + a[2] * b[0] - a[0] * b[2], + a[0] * b[1] - a[1] * b[0], +]; +function normalize(a) { + const length = Math.hypot(a[0], a[1], a[2]); + return length > 1e-12 ? scaleVector(a, 1 / length) : [0, 0, 0]; +} + +/** Rotate vector `v` by unit quaternion `q` ([x, y, z, w]). */ +export function rotateByQuaternion(q, v) { + const [x, y, z, w] = q; + const u = [x, y, z]; + const t = scaleVector(cross(u, v), 2); + return add(add(v, scaleVector(t, w)), cross(u, t)); +} + +function normalizeQuaternion(q) { + const length = Math.hypot(q[0], q[1], q[2], q[3]); + return length > 1e-12 ? q.map((v) => v / length) : [0, 0, 0, 1]; +} + +/** Normalized linear interpolation along the short arc. */ +export function nlerpQuaternion(a, b, k) { + const sign = + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] + a[3] * b[3] < 0 ? -1 : 1; + return normalizeQuaternion(a.map((v, i) => v + (b[i] * sign - v) * k)); +} + +/** Y-up phone vector to ENU: +x east, -z north, +y up. */ +const toEnu = (v) => [v[0], -v[2], v[1]]; + +/** Rotate an ENU vector clockwise about up by `yaw` radians (heading increases). */ +function yawEnu(v, yaw) { + const c = Math.cos(yaw); + const s = Math.sin(yaw); + return [v[0] * c + v[1] * s, -v[0] * s + v[1] * c, v[2]]; +} + +/** Heading (radians, clockwise from north) of an ENU direction, or null when vertical. */ +export function headingOfEnu(v) { + if (Math.hypot(v[0], v[1]) < 1e-6) return null; + return Math.atan2(v[0], v[1]); +} + +/** The phone's forward and up vectors in its own Y-up frame. */ +function phoneAxes(q) { + return { + forward: rotateByQuaternion(q, [0, 0, -1]), + up: rotateByQuaternion(q, [0, 1, 0]), + }; +} + +/** Yaw of a phone pose: the heading of its forward (or, looking straight down, its top edge). */ +function phoneYaw(q) { + const { forward, up } = phoneAxes(q); + return headingOfEnu(toEnu(forward)) ?? headingOfEnu(toEnu(up)) ?? 0; +} + +/** + * Create a rig. Settings may change at any time; changes that would move the + * camera instead re-zero the phone so the camera holds still. + * + * @param {object} [settings] + * @param {number} [settings.scale=1] - World metres per phone metre. + * @param {number} [settings.smoothing=0.3] - 0 (raw) to 1 (heavily damped). + * @param {boolean} [settings.heightLock=false] - Ignore the phone's vertical motion. + * @param {boolean} [settings.horizonLock=false] - Keep the horizon level (no roll). + */ +export function createRig(settings = {}) { + const state = { + scale: clampScale(settings.scale ?? 1), + smoothing: clamp01(settings.smoothing ?? 0.3), + heightLock: !!settings.heightLock, + horizonLock: !!settings.horizonLock, + yaw: 0, + origin: null, + raw: null, + smoothed: null, + centered: false, + }; + + /** Camera offset from the anchor for the smoothed pose, in ENU metres. */ + function offset() { + if (!state.smoothed?.p || !state.origin) return [0, 0, 0]; + const delta = subtract(state.smoothed.p, state.origin); + if (state.heightLock) delta[1] = 0; + return scaleVector(yawEnu(toEnu(delta), state.yaw), state.scale); + } + + function orientation() { + const q = state.smoothed?.q || [0, 0, 0, 1]; + const { forward, up } = phoneAxes(q); + const f = normalize(yawEnu(toEnu(forward), state.yaw)); + let u = normalize(yawEnu(toEnu(up), state.yaw)); + if (state.horizonLock) { + // The level up vector: world up with the forward component removed. + const level = normalize(subtract([0, 0, 1], scaleVector(f, f[2]))); + if (Math.hypot(...level) > 0.5) u = level; + } + return { forward: f, up: u }; + } + + /** + * Keep the camera where it is but restart the phone's motion from here, + * facing `heading` (radians). Returns the anchor shift this requires. + */ + function recenter(heading) { + if (!state.raw) return [0, 0, 0]; + const shift = offset(); + state.smoothed = { + p: state.raw.p && [...state.raw.p], + q: [...state.raw.q], + }; + state.origin = state.raw.p ? [...state.raw.p] : null; + state.yaw = + (Number.isFinite(heading) ? heading : 0) - phoneYaw(state.raw.q); + state.centered = true; + return shift; + } + + /** + * Advance one frame. + * @param {object} input + * @param {number[]} input.q - Phone orientation quaternion [x, y, z, w]. + * @param {number[]|null} [input.p] - Phone position in metres, or null (3DoF). + * @param {number[]} [input.joy] - Sticks [moveX, moveY, panX, craneY] in -1..1. + * @param {number} input.dt - Seconds since the previous step. + * @returns {{ anchorShift: number[], offset: number[], forward: number[], up: number[] }} + */ + function step({ q, p = null, joy = [0, 0, 0, 0], dt }) { + const pose = { + q: normalizeQuaternion(q), + p: Array.isArray(p) ? [...p] : null, + }; + let anchorShift = [0, 0, 0]; + // Switching between 6DoF and 3DoF tracking re-zeros position. + if (state.raw && !!state.raw.p !== !!pose.p) { + anchorShift = add(anchorShift, offset()); + state.origin = pose.p ? [...pose.p] : null; + if (state.smoothed) state.smoothed.p = pose.p ? [...pose.p] : null; + } + state.raw = pose; + if (!state.smoothed) + state.smoothed = { p: pose.p && [...pose.p], q: [...pose.q] }; + else { + const tau = state.smoothing * MAX_SMOOTHING_SEC; + const k = tau > 0 ? 1 - Math.exp(-Math.max(0, dt) / tau) : 1; + state.smoothed.q = nlerpQuaternion(state.smoothed.q, pose.q, k); + if (pose.p) + state.smoothed.p = state.smoothed.p + ? state.smoothed.p.map((v, i) => v + (pose.p[i] - v) * k) + : [...pose.p]; + } + if (!state.origin && pose.p) state.origin = [...pose.p]; + + const { forward, up } = orientation(); + const [moveX, moveY, panX, craneY] = joy.map( + (v) => Math.sign(v) * Math.min(1, Math.abs(v)) ** 2, + ); + const speed = STICK_SPEED_MPS * state.scale * Math.max(0, dt); + if (moveX || moveY || craneY) { + // Sticks move relative to where the camera faces, on the level plane. + const heading = headingOfEnu(forward) ?? headingOfEnu(up) ?? 0; + const ahead = [Math.sin(heading), Math.cos(heading), 0]; + const right = [Math.cos(heading), -Math.sin(heading), 0]; + anchorShift = add( + anchorShift, + add( + add( + scaleVector(ahead, moveY * speed), + scaleVector(right, moveX * speed), + ), + [0, 0, craneY * speed], + ), + ); + } + if (panX && dt > 0) { + // Pan about the camera itself: move the anchor so the camera stays put. + const before = offset(); + state.yaw += panX * STICK_YAW_RAD_PER_S * dt; + anchorShift = add(anchorShift, subtract(before, offset())); + } + const turned = panX ? orientation() : { forward, up }; + return { + anchorShift, + offset: offset(), + forward: turned.forward, + up: turned.up, + }; + } + + /** Apply settings; a scale change re-zeros the phone so the camera holds still. */ + function configure(next = {}) { + let anchorShift = [0, 0, 0]; + if (Number.isFinite(next.scale) && clampScale(next.scale) !== state.scale) { + anchorShift = offset(); + state.scale = clampScale(next.scale); + if (state.smoothed?.p) state.origin = [...state.smoothed.p]; + } + if (Number.isFinite(next.smoothing)) + state.smoothing = clamp01(next.smoothing); + if ( + typeof next.heightLock === 'boolean' && + next.heightLock !== state.heightLock + ) { + anchorShift = add(anchorShift, offset()); + state.heightLock = next.heightLock; + if (state.smoothed?.p) state.origin = [...state.smoothed.p]; + } + if (typeof next.horizonLock === 'boolean') + state.horizonLock = next.horizonLock; + return anchorShift; + } + + return { + step, + recenter, + configure, + get settings() { + return { + scale: state.scale, + smoothing: state.smoothing, + heightLock: state.heightLock, + horizonLock: state.horizonLock, + }; + }, + get centered() { + return state.centered; + }, + get tracking() { + return state.raw ? (state.raw.p ? '6dof' : '3dof') : null; + }, + }; +} + +function clamp01(value) { + return Math.max(0, Math.min(1, Number(value) || 0)); +} +function clampScale(value) { + return Math.max(SCALE_RANGE[0], Math.min(SCALE_RANGE[1], Number(value) || 1)); +} diff --git a/src/vcam/rig.test.mjs b/src/vcam/rig.test.mjs new file mode 100644 index 000000000..beb537b57 --- /dev/null +++ b/src/vcam/rig.test.mjs @@ -0,0 +1,121 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createRig, headingOfEnu, rotateByQuaternion } from './rig.js'; + +const close = (actual, expected, message, epsilon = 1e-9) => { + assert.equal(actual.length, expected.length, message); + actual.forEach((value, i) => + assert.ok(Math.abs(value - expected[i]) < epsilon, `${message}: ${actual} vs ${expected}`), + ); +}; +const axisAngle = (axis, angle) => { + const s = Math.sin(angle / 2); + return [axis[0] * s, axis[1] * s, axis[2] * s, Math.cos(angle / 2)]; +}; +const IDENTITY = [0, 0, 0, 1]; +const DEG = Math.PI / 180; + +test('recenter aligns the phone with the camera heading and keeps the camera in place', () => { + const rig = createRig({ smoothing: 0 }); + // The phone starts turned 30 degrees left of its arbitrary tracking yaw. + const turned = axisAngle([0, 1, 0], 30 * DEG); + rig.step({ q: turned, p: [0.2, 1.4, -0.3], dt: 0 }); + close(rig.recenter(90 * DEG), [0, 0, 0], 'recentering a still phone moves nothing'); + const first = rig.step({ q: turned, p: [0.2, 1.4, -0.3], dt: 0.016 }); + close(first.offset, [0, 0, 0], 'no jump on recenter'); + close(first.forward, [1, 0, 0], 'phone forward now faces the camera heading (east)'); + close(first.up, [0, 0, 1], 'level phone, level camera'); + assert.equal(rig.tracking, '6dof'); +}); + +test('phone motion maps through the aligned yaw and the motion scale', () => { + const rig = createRig({ smoothing: 0, scale: 100 }); + rig.step({ q: IDENTITY, p: [0, 0, 0], dt: 0 }); + rig.recenter(90 * DEG); + // One metre forward (phone -z) and half a metre up. + const moved = rig.step({ q: IDENTITY, p: [0, 0.5, -1], dt: 0.016 }); + close(moved.offset, [100, 0, 50], 'forward is east at 1:100'); + // A step to the phone's right is south when facing east. + close(rig.step({ q: IDENTITY, p: [1, 0, 0], dt: 0.016 }).offset, [0, -100, 0], 'right is south'); +}); + +test('scale and height-lock changes re-zero the phone so the camera holds still', () => { + const rig = createRig({ smoothing: 0, scale: 10 }); + rig.step({ q: IDENTITY, p: [0, 0, 0], dt: 0 }); + rig.recenter(0); + const before = rig.step({ q: IDENTITY, p: [0, 1, -2], dt: 0.016 }).offset; + close(before, [0, 20, 10], 'north and up at 1:10'); + const shift = rig.configure({ scale: 1000 }); + const after = rig.step({ q: IDENTITY, p: [0, 1, -2], dt: 0.016 }); + close([...shift.map((v, i) => v + after.offset[i] + after.anchorShift[i])], before, 'camera unchanged'); + const further = rig.step({ q: IDENTITY, p: [0, 1, -2.001], dt: 0.016 }); + close(further.offset, [0, 1, 0], 'one more millimetre is a metre at 1:1000', 1e-6); + rig.configure({ heightLock: true }); + close(rig.step({ q: IDENTITY, p: [0, 3, -2.001], dt: 0.016 }).offset, [0, 0, 0], 'vertical motion ignored'); + assert.deepEqual(rig.settings, { scale: 1000, smoothing: 0, heightLock: true, horizonLock: false }); +}); + +test('horizon lock removes roll but keeps pitch and heading', () => { + const rig = createRig({ smoothing: 0, horizonLock: true }); + // Pitch down 20 degrees, then roll 25 degrees. + const pitched = axisAngle([1, 0, 0], -20 * DEG); + const rolled = axisAngle([0, 0, 1], 25 * DEG); + const q = [ + pitched[3] * rolled[0] + pitched[0] * rolled[3] + pitched[1] * rolled[2] - pitched[2] * rolled[1], + pitched[3] * rolled[1] - pitched[0] * rolled[2] + pitched[1] * rolled[3] + pitched[2] * rolled[0], + pitched[3] * rolled[2] + pitched[0] * rolled[1] - pitched[1] * rolled[0] + pitched[2] * rolled[3], + pitched[3] * rolled[3] - pitched[0] * rolled[0] - pitched[1] * rolled[1] - pitched[2] * rolled[2], + ]; + rig.step({ q, p: null, dt: 0 }); + rig.recenter(0); + const { forward, up } = rig.step({ q, p: null, dt: 0.016 }); + assert.ok(Math.abs(Math.asin(forward[2]) + 20 * DEG) < 1e-9, 'pitch kept'); + assert.ok(Math.abs(headingOfEnu(forward)) < 1e-9, 'heading kept'); + assert.ok(Math.abs(up[0]) < 1e-9, 'no sideways tilt of the up vector'); + assert.equal(rig.tracking, '3dof'); + rig.configure({ horizonLock: false }); + assert.ok(Math.abs(rig.step({ q, p: null, dt: 0.016 }).up[0]) > 0.3, 'roll returns without the lock'); +}); + +test('sticks dolly along the camera heading and pan about the camera', () => { + const rig = createRig({ smoothing: 0, scale: 10 }); + rig.step({ q: IDENTITY, p: null, dt: 0 }); + rig.recenter(90 * DEG); + const dolly = rig.step({ q: IDENTITY, p: null, joy: [0, 1, 0, 0], dt: 1 }); + close(dolly.anchorShift, [25, 0, 0], 'full stick for a second at 1:10 is 25 m east'); + const half = rig.step({ q: IDENTITY, p: null, joy: [0, 0, 0, 0.5], dt: 1 }); + close(half.anchorShift, [0, 0, 6.25], 'sticks respond with a squared curve'); + + const moved = createRig({ smoothing: 0, scale: 10 }); + moved.step({ q: IDENTITY, p: [0, 0, 0], dt: 0 }); + moved.recenter(0); + const before = moved.step({ q: IDENTITY, p: [0, 0, -1], dt: 0.016 }).offset; + const panned = moved.step({ q: IDENTITY, p: [0, 0, -1], joy: [0, 0, 1, 0], dt: 0.5 }); + close(panned.anchorShift.map((v, i) => v + panned.offset[i]), before, 'the camera turns in place'); + assert.ok(Math.abs(headingOfEnu(panned.forward) - 30 * DEG) < 1e-9, 'half a second of full pan is 30 degrees'); +}); + +test('smoothing damps motion and reaches the target over time', () => { + const rig = createRig({ smoothing: 0.5, scale: 1 }); + rig.step({ q: IDENTITY, p: [0, 0, 0], dt: 0 }); + rig.recenter(0); + const first = rig.step({ q: IDENTITY, p: [0, 0, -1], dt: 1 / 60 }); + assert.ok(first.offset[1] > 0 && first.offset[1] < 0.1, 'one frame moves a little'); + let last = first; + for (let i = 0; i < 240; i++) last = rig.step({ q: IDENTITY, p: [0, 0, -1], dt: 1 / 60 }); + close(last.offset, [0, 1, 0], 'converges', 1e-4); +}); + +test('switching between 6DoF and 3DoF keeps the camera continuous', () => { + const rig = createRig({ smoothing: 0, scale: 5 }); + rig.step({ q: IDENTITY, p: [0, 0, 0], dt: 0 }); + rig.recenter(0); + const tracked = rig.step({ q: IDENTITY, p: [0, 0, -2], dt: 0.016 }); + const lost = rig.step({ q: IDENTITY, p: null, dt: 0.016 }); + close(lost.anchorShift.map((v, i) => v + lost.offset[i]), tracked.offset, 'tracking loss holds position'); +}); + +test('quaternion rotation matches the WebXR forward convention', () => { + close(rotateByQuaternion(axisAngle([0, 1, 0], 90 * DEG), [0, 0, -1]), [-1, 0, 0], 'yaw left 90'); + assert.equal(headingOfEnu([0, 0, -1]), null); +});