gods-eye-view/server/vcam/phone/phone.js

/**
 * God's Eye View virtual camera — phone side.
 *
 * Streams this phone's pose to the desktop app and shows the app's render as
 * a monitor. Tracking uses WebXR `immersive-ar` (6DoF, Android Chrome with
 * ARCore) when available, otherwise device orientation (3DoF, iPhone Safari)
 * with on-screen sticks for movement. Poses are sent in one Y-up frame for
 * both: +x right, +y up, -z forward (WebXR's convention); yaw is arbitrary
 * and the desktop aligns it on recenter.
 *
 * Transport: a WebSocket when the connection allows it, otherwise HTTPS
 * requests (iOS refuses WebSockets under a tapped-through certificate). The
 * monitor always pulls the newest frame, so a slow network drops frames
 * instead of building a backlog.
 */

import { orientationQuaternion } from './orientation.js';

const $ = (id) => document.getElementById(id);
const KEY_STORAGE = 'gevVcam.phoneKey';
const CODE_STORAGE = 'gevVcam.code';
const SCALE_PRESETS = [1, 10, 100, 1000, 10000];
const FOCAL_RANGE = [12, 400];
const FSTOP_RANGE = [1.2, 22];
const STICK_DEADZONE = 0.12;

const store = {
  get(key) {
    try {
      return sessionStorage.getItem(key);
    } catch {
      return null;
    }
  },
  set(key, value) {
    try {
      if (value === null) sessionStorage.removeItem(key);
      else sessionStorage.setItem(key, value);
    } catch {
      // Private browsing: pairing still works for this page load.
    }
  },
};

const logMap = ([low, high], v) => low * (high / low) ** v;
const logUnmap = ([low, high], value) =>
  Math.log(value / low) / Math.log(high / low);
const round = (value, places = 5) => {
  const k = 10 ** places;
  return Math.round(value * k) / k;
};
const formatScale = (scale) =>
  scale >= 1000
    ? `1:${+(scale / 1000).toFixed(scale >= 10000 ? 0 : 1)}K`
    : `1:${+scale.toFixed(scale < 10 ? 1 : 0)}`;
const formatTime = (sec) => {
  const s = Math.max(0, Math.floor(sec));
  return `${String(Math.floor(s / 60)).padStart(2, '0')}:${String(s % 60).padStart(2, '0')}`;
};

const screenAngle = () =>
  screen.orientation?.angle ?? (Number(window.orientation) || 0);

// ------------------------------------------------------------------ transport
class Link {
  constructor(key, { onMessage, onStatus, onUnpaired }) {
    this.key = key;
    this.onMessage = onMessage;
    this.onStatus = onStatus;
    this.onUnpaired = onUnpaired;
    this.ws = null;
    this.events = null;
    this.mode = 'connecting';
    this.queue = [];
    this.pendingPose = null;
    this.inFlight = false;
    this.closed = false;
    this.webSocketFailed = false;
    this.retryTimer = null;
  }

  async start() {
    if (this.closed) return;
    this.onStatus('connecting');
    if (!this.webSocketFailed && (await this.openWebSocket())) return;
    this.webSocketFailed = true;
    this.openHttp();
  }

  openWebSocket() {
    return new Promise((resolve) => {
      let settled = false;
      let ws;
      try {
        ws = new WebSocket(
          `wss://${location.host}/ws?key=${encodeURIComponent(this.key)}`,
        );
      } catch {
        resolve(false);
        return;
      }
      const done = (ok) => {
        if (settled) return;
        settled = true;
        clearTimeout(timer);
        resolve(ok);
      };
      const timer = setTimeout(() => {
        ws.close();
        done(false);
      }, 3500);
      ws.onopen = () => {
        this.ws = ws;
        this.mode = 'ws';
        this.onStatus('live');
        done(true);
      };
      ws.onerror = () => done(false);
      ws.onmessage = (event) => this.receive(event.data);
      ws.onclose = (event) => {
        done(false);
        if (this.ws !== ws) return;
        this.ws = null;
        this.mode = 'connecting';
        this.handleClose(event.code);
      };
    });
  }

  openHttp() {
    if (this.closed) return;
    this.mode = 'http';
    const events = new EventSource(
      `/api/events?key=${encodeURIComponent(this.key)}`,
    );
    this.events = events;
    events.onopen = () => this.onStatus('slow');
    events.onmessage = (event) => this.receive(event.data);
    events.onerror = async () => {
      if (events.readyState !== EventSource.CLOSED || this.closed) return;
      this.events = null;
      // A closed stream is either a lost network or an ended session.
      try {
        const probe = await fetch('/api/send', {
          method: 'POST',
          headers: {
            'Content-Type': 'application/json',
            'X-Vcam-Key': this.key,
          },
          body: '[]',
          cache: 'no-store',
        });
        if (probe.status === 403) return this.handleClose(4003);
      } catch {
        // Offline: retry below.
      }
      this.handleClose(1006);
    };
    this.flush();
  }

  handleClose(code) {
    if (this.closed) return;
    if (code === 4003 || code === 4000) {
      this.onUnpaired(code === 4000 ? 'ended' : 'unpaired');
      return;
    }
    if (code === 4001) {
      this.onStatus('replaced');
      this.close();
      return;
    }
    this.onStatus('connecting');
    clearTimeout(this.retryTimer);
    this.retryTimer = setTimeout(() => this.start(), 1000);
  }

  receive(text) {
    try {
      this.onMessage(JSON.parse(text));
    } catch {
      // Ignore malformed relay input.
    }
  }

  /** Poses are latest-wins; every other message is delivered in order. */
  send(message) {
    if (this.closed) return;
    // 1 is WebSocket.OPEN.
    if (this.ws && this.ws.readyState === 1) {
      this.ws.send(JSON.stringify(message));
      return;
    }
    if (message.type === 'pose') this.pendingPose = message;
    else this.queue.push(message);
    if (this.mode === 'http') this.flush();
  }

  async flush() {
    if (this.inFlight || this.mode !== 'http' || this.closed) return;
    const batch = this.queue.splice(0, 32);
    if (this.pendingPose) batch.push(this.pendingPose);
    this.pendingPose = null;
    if (!batch.length) return;
    this.inFlight = true;
    try {
      const response = await fetch('/api/send', {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          'X-Vcam-Key': this.key,
        },
        body: JSON.stringify(batch),
        cache: 'no-store',
      });
      if (response.status === 403) this.handleClose(4003);
    } catch {
      // The event stream's error handling owns reconnection.
    } finally {
      this.inFlight = false;
    }
    if (this.queue.length || this.pendingPose) this.flush();
  }

  close() {
    this.closed = true;
    clearTimeout(this.retryTimer);
    this.ws?.close();
    this.events?.close();
    this.ws = null;
    this.events = null;
  }
}

// --------------------------------------------------------------------- state
const app = {
  key: store.get(KEY_STORAGE),
  link: null,
  tracking: null, // { mode: '6dof'|'3dof', stop() }
  orientation: null,
  sticks: { left: [0, 0], right: [0, 0] },
  host: {
    connected: false,
    recording: false,
    elapsedSec: 0,
    scale: 1,
    focalMm: 31.18,
    fStop: 2.8,
    dof: false,
    held: false,
    smoothing: 0.3,
    heightLock: false,
    horizonLock: false,
  },
  recordingSince: 0,
  monitorRunning: false,
  wakeLock: null,
};

function show(screenId) {
  for (const id of ['pair', 'start', 'camera']) $(id).hidden = id !== screenId;
}

function setStatus(element, text, error = false) {
  element.textContent = text || '';
  element.classList.toggle('error', !!error);
}

let toastTimer = null;
function toast(text) {
  const element = $('camera-status');
  element.textContent = text;
  element.hidden = false;
  clearTimeout(toastTimer);
  toastTimer = setTimeout(() => (element.hidden = true), 2600);
}

// ------------------------------------------------------------------- pairing
async function join(code) {
  setStatus($('pair-status'), 'Pairing…');
  try {
    const response = await fetch('/api/join', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ code }),
      cache: 'no-store',
    });
    if (response.status === 403) {
      setStatus(
        $('pair-status'),
        'That code is not active. Check the code on the desktop.',
        true,
      );
      return false;
    }
    if (!response.ok) throw new Error(String(response.status));
    const { phoneKey } = await response.json();
    app.key = phoneKey;
    store.set(KEY_STORAGE, phoneKey);
    store.set(CODE_STORAGE, code);
    setStatus($('pair-status'), '');
    connect();
    showStart();
    return true;
  } catch {
    setStatus(
      $('pair-status'),
      'Could not reach the desktop. Is it on the same network?',
      true,
    );
    return false;
  }
}

function connect() {
  app.link?.close();
  app.link = new Link(app.key, {
    onMessage: handleMessage,
    onStatus: presentLink,
    onUnpaired: async (reason) => {
      app.link?.close();
      app.link = null;
      store.set(KEY_STORAGE, null);
      stopTracking();
      const code = store.get(CODE_STORAGE);
      if (reason === 'unpaired' && code && (await join(code))) return;
      show('pair');
      setStatus(
        $('pair-status'),
        reason === 'ended'
          ? 'The desktop ended this session. Enter a new code.'
          : 'Pairing expired. Enter the code shown on the desktop.',
        true,
      );
    },
  });
  app.link.start();
}

function presentLink(status) {
  const dot = $('link-dot');
  dot.classList.toggle('live', status === 'live');
  dot.classList.toggle('slow', status === 'slow');
  dot.title =
    status === 'live'
      ? 'Live connection'
      : status === 'slow'
        ? 'Fallback connection (install the certificate for lower latency)'
        : 'Connecting…';
  if (status === 'replaced') toast('Another phone took over this session');
}

async function showStart() {
  show('start');
  const xr = await xrSupported();
  $('start-xr').hidden = !xr;
  $('support').textContent = xr
    ? 'This phone supports AR tracking. Point it like a camera.'
    : 'AR tracking is not available in this browser, so the gyro camera tracks rotation and the sticks move you.';
}

async function xrSupported() {
  try {
    return !!(await navigator.xr?.isSessionSupported('immersive-ar'));
  } catch {
    return false;
  }
}

// ------------------------------------------------------------------ tracking
function sendPose(q, p) {
  const { left, right } = app.sticks;
  app.link?.send({
    type: 'pose',
    t: Math.round(performance.now()),
    mode: app.tracking?.mode,
    q: q.map((v) => round(v)),
    p: p ? p.map((v) => round(v, 4)) : null,
    j: [left[0], left[1], right[0], right[1]].map((v) => round(v, 3)),
  });
}

async function startXr() {
  setStatus($('start-status'), 'Starting AR…');
  try {
    const root = $('camera');
    const session = await navigator.xr.requestSession('immersive-ar', {
      requiredFeatures: ['local'],
      optionalFeatures: ['dom-overlay'],
      domOverlay: { root },
    });
    // Screen taps on controls must not become XR select input.
    root.addEventListener('beforexrselect', (event) => event.preventDefault());
    const canvas = document.createElement('canvas');
    const gl = canvas.getContext('webgl', { xrCompatible: true, alpha: true });
    await gl.makeXRCompatible?.();
    session.updateRenderState({ baseLayer: new XRWebGLLayer(session, gl) });
    const space = await session.requestReferenceSpace('local');
    let stopped = false;
    const onFrame = (_time, frame) => {
      if (stopped) return;
      session.requestAnimationFrame(onFrame);
      const pose = frame.getViewerPose(space);
      if (pose) {
        const { position, orientation } = pose.transform;
        sendPose(
          [orientation.x, orientation.y, orientation.z, orientation.w],
          pose.emulatedPosition ? null : [position.x, position.y, position.z],
        );
      }
      const layer = session.renderState.baseLayer;
      gl.bindFramebuffer(gl.FRAMEBUFFER, layer.framebuffer);
      gl.clearColor(0, 0, 0, 0);
      gl.clear(gl.COLOR_BUFFER_BIT);
    };
    session.requestAnimationFrame(onFrame);
    session.addEventListener('end', () => {
      stopped = true;
      if (app.tracking?.session === session) {
        app.tracking = null;
        showStart();
      }
    });
    app.tracking = {
      mode: '6dof',
      session,
      stop: () => session.end().catch(() => {}),
    };
    enterCamera();
  } catch (error) {
    setStatus(
      $('start-status'),
      `AR could not start (${error?.message || error}). Try the gyro camera.`,
      true,
    );
  }
}

async function startGyro() {
  // iOS asks for motion access only from a tap, before anything else awaits.
  try {
    if (typeof DeviceOrientationEvent?.requestPermission === 'function') {
      const answer = await DeviceOrientationEvent.requestPermission();
      if (answer !== 'granted') {
        setStatus(
          $('start-status'),
          'Motion access was denied. Allow it in Safari settings for this site, then reload.',
          true,
        );
        return;
      }
    }
  } catch {
    setStatus($('start-status'), 'Motion access is unavailable.', true);
    return;
  }
  let stopped = false;
  const onOrientation = (event) => {
    if (event.alpha === null && event.beta === null) return;
    app.orientation = orientationQuaternion(
      event.alpha,
      event.beta,
      event.gamma,
      screenAngle(),
    );
  };
  window.addEventListener('deviceorientation', onOrientation);
  const tick = () => {
    if (stopped) return;
    requestAnimationFrame(tick);
    if (app.orientation) sendPose(app.orientation, null);
  };
  requestAnimationFrame(tick);
  app.tracking = {
    mode: '3dof',
    stop: () => {
      stopped = true;
      window.removeEventListener('deviceorientation', onOrientation);
    },
  };
  document.documentElement.requestFullscreen?.().catch(() => {});
  enterCamera();
  setTimeout(() => {
    if (!app.orientation && !stopped)
      toast('No motion data yet — this browser may block the gyroscope');
  }, 1500);
}

function stopTracking() {
  app.tracking?.stop();
  app.tracking = null;
  app.monitorRunning = false;
}

async function enterCamera() {
  show('camera');
  $('mode-readout').textContent =
    app.tracking?.mode === '6dof' ? 'AR 6DOF' : 'GYRO 3DOF';
  app.link?.send({
    type: 'hello',
    mode: app.tracking?.mode,
    screen: [window.innerWidth, window.innerHeight],
  });
  try {
    app.wakeLock = await navigator.wakeLock?.request('screen');
  } catch {
    // Optional: the screen may dim during long takes.
  }
  runMonitor();
}

// ------------------------------------------------------------------- monitor
async function runMonitor() {
  if (app.monitorRunning) return;
  app.monitorRunning = true;
  const monitor = $('monitor');
  let seq = 0;
  let previousUrl = null;
  while (app.monitorRunning && app.key) {
    if (document.hidden) {
      await new Promise((resolve) => setTimeout(resolve, 500));
      continue;
    }
    try {
      const response = await fetch(
        `/api/frame?key=${encodeURIComponent(app.key)}&after=${seq}`,
        { cache: 'no-store' },
      );
      if (response.status === 403) break;
      if (response.status !== 200) continue;
      seq = Number(response.headers.get('X-Frame-Seq')) || seq + 1;
      const url = URL.createObjectURL(await response.blob());
      await new Promise((resolve) => {
        monitor.onload = resolve;
        monitor.onerror = resolve;
        monitor.src = url;
      });
      if (previousUrl) URL.revokeObjectURL(previousUrl);
      previousUrl = url;
    } catch {
      await new Promise((resolve) => setTimeout(resolve, 1000));
    }
  }
  app.monitorRunning = false;
}

// ------------------------------------------------------------------ controls
function stick(element, target) {
  const knob = element.querySelector('.knob');
  let pointer = null;
  const set = (x, y) => {
    target[0] = x;
    target[1] = y;
    const radius = element.clientWidth / 2 - 26;
    knob.style.transform = `translate(${x * radius}px, ${-y * radius}px)`;
  };
  const move = (event) => {
    const rect = element.getBoundingClientRect();
    const radius = rect.width / 2;
    let x = (event.clientX - rect.left - radius) / radius;
    let y = -(event.clientY - rect.top - radius) / radius;
    const length = Math.hypot(x, y);
    if (length > 1) {
      x /= length;
      y /= length;
    }
    // Rescale past the dead zone so small nudges stay precise.
    const live = Math.min(1, Math.hypot(x, y));
    const scale =
      live < STICK_DEADZONE
        ? 0
        : (live - STICK_DEADZONE) / (1 - STICK_DEADZONE) / live;
    set(x * scale, y * scale);
  };
  element.addEventListener('pointerdown', (event) => {
    if (pointer !== null) return;
    pointer = event.pointerId;
    element.setPointerCapture(pointer);
    move(event);
  });
  element.addEventListener('pointermove', (event) => {
    if (event.pointerId === pointer) move(event);
  });
  const release = (event) => {
    if (event.pointerId !== pointer) return;
    pointer = null;
    set(0, 0);
  };
  element.addEventListener('pointerup', release);
  element.addEventListener('pointercancel', release);
}

const sendSet = (key, value) => app.link?.send({ type: 'set', key, value });
const sendCommand = (command) => app.link?.send({ type: 'cmd', cmd: command });

function presentHost() {
  const h = app.host;
  $('record-btn').setAttribute('aria-pressed', String(h.recording));
  $('rec-readout').hidden = !h.recording;
  $('hold-btn').setAttribute('aria-pressed', String(h.held));
  $('scale-readout').textContent = formatScale(h.scale);
  $('scale-value').textContent = formatScale(h.scale);
  $('focal-readout').textContent = `${Math.round(h.focalMm)}mm`;
  $('focal-value').textContent = `${Math.round(h.focalMm)}mm`;
  $('fstop-value').textContent = `f/${h.fStop.toFixed(1)}`;
  $('smooth-value').textContent = `${Math.round(h.smoothing * 100)}%`;
  const active = (element) =>
    element === document.activeElement && element.matches(':active');
  if (!active($('scale-range')))
    $('scale-range').value = String(Math.log10(h.scale));
  if (!active($('focal-range')))
    $('focal-range').value = String(logUnmap(FOCAL_RANGE, h.focalMm));
  if (!active($('fstop-range')))
    $('fstop-range').value = String(logUnmap(FSTOP_RANGE, h.fStop));
  if (!active($('smooth-range')))
    $('smooth-range').value = String(Math.round(h.smoothing * 100));
  $('dof-toggle').checked = h.dof;
  $('height-lock').checked = h.heightLock;
  $('horizon-lock').checked = h.horizonLock;
  for (const button of $('scale-presets').querySelectorAll('button'))
    button.classList.toggle(
      'active',
      Math.abs(Number(button.dataset.scale) - h.scale) < 1e-6,
    );
  for (const button of $('focal-presets').querySelectorAll('button'))
    button.classList.toggle(
      'active',
      Math.abs(Number(button.dataset.focal) - h.focalMm) < 0.5,
    );
}

function handleMessage(message) {
  if (message.type === 'state') {
    const wasRecording = app.host.recording;
    Object.assign(app.host, message.state);
    if (app.host.recording && !wasRecording)
      app.recordingSince = performance.now() - app.host.elapsedSec * 1000;
    presentHost();
  } else if (message.type === 'peer' && message.role === 'host') {
    app.host.connected = message.connected;
    if (!message.connected) toast('Desktop disconnected — waiting…');
  } else if (message.type === 'toast' && typeof message.text === 'string') {
    toast(message.text.slice(0, 140));
  }
}

function toggleDrawer(name) {
  for (const other of ['scale', 'lens']) {
    const open = other === name && $(`${other}-drawer`).hidden;
    $(`${other}-drawer`).hidden = !open;
    $(`${other}-btn`).setAttribute('aria-expanded', String(open));
  }
}

function bindControls() {
  $('pair-form').addEventListener('submit', (event) => {
    event.preventDefault();
    const code = $('code').value.trim().toUpperCase();
    if (code) join(code);
  });
  $('start-xr').addEventListener('click', startXr);
  $('start-gyro').addEventListener('click', startGyro);
  stick($('stick-left'), app.sticks.left);
  stick($('stick-right'), app.sticks.right);
  $('record-btn').addEventListener('click', () =>
    sendCommand(app.host.recording ? 'stop' : 'record'),
  );
  $('hold-btn').addEventListener('click', () =>
    sendCommand(app.host.held ? 'release' : 'hold'),
  );
  $('recenter-btn').addEventListener('click', () => sendCommand('recenter'));
  $('scale-btn').addEventListener('click', () => toggleDrawer('scale'));
  $('lens-btn').addEventListener('click', () => toggleDrawer('lens'));
  $('scale-presets').addEventListener('click', (event) => {
    const value = Number(event.target.closest('button')?.dataset.scale);
    if (SCALE_PRESETS.includes(value)) sendSet('scale', value);
  });
  $('scale-range').addEventListener('input', (event) => {
    const scale = 10 ** Number(event.target.value);
    $('scale-value').textContent = formatScale(scale);
    sendSet('scale', round(scale, 3));
  });
  $('smooth-range').addEventListener('input', (event) =>
    sendSet('smoothing', Number(event.target.value) / 100),
  );
  $('height-lock').addEventListener('change', (event) =>
    sendSet('heightLock', event.target.checked),
  );
  $('horizon-lock').addEventListener('change', (event) =>
    sendSet('horizonLock', event.target.checked),
  );
  $('focal-range').addEventListener('input', (event) => {
    const focal = logMap(FOCAL_RANGE, Number(event.target.value));
    $('focal-value').textContent = `${Math.round(focal)}mm`;
    sendSet('focalMm', round(focal, 2));
  });
  $('focal-presets').addEventListener('click', (event) => {
    const value = Number(event.target.closest('button')?.dataset.focal);
    if (value) sendSet('focalMm', value);
  });
  $('dof-toggle').addEventListener('change', (event) =>
    sendSet('dof', event.target.checked),
  );
  $('fstop-range').addEventListener('input', (event) => {
    const fStop = logMap(FSTOP_RANGE, Number(event.target.value));
    $('fstop-value').textContent = `f/${fStop.toFixed(1)}`;
    sendSet('fStop', round(fStop, 2));
  });
  // Tap the monitor to focus: coordinates are within the shown picture.
  $('monitor').addEventListener('click', (event) => {
    const monitor = $('monitor');
    if (!monitor.naturalWidth) return;
    const rect = monitor.getBoundingClientRect();
    const fit = Math.min(
      rect.width / monitor.naturalWidth,
      rect.height / monitor.naturalHeight,
    );
    const width = monitor.naturalWidth * fit;
    const height = monitor.naturalHeight * fit;
    const x = (event.clientX - rect.left - (rect.width - width) / 2) / width;
    const y = (event.clientY - rect.top - (rect.height - height) / 2) / height;
    if (x < 0 || x > 1 || y < 0 || y > 1) return;
    app.link?.send({ type: 'focus', x: round(x, 4), y: round(y, 4) });
    const ring = $('focus-ring');
    ring.hidden = true;
    ring.style.left = `${event.clientX}px`;
    ring.style.top = `${event.clientY}px`;
    void ring.offsetWidth;
    ring.hidden = false;
  });
  setInterval(() => {
    if (app.host.recording)
      $('rec-readout').textContent =
        `● ${formatTime((performance.now() - app.recordingSince) / 1000)}`;
  }, 250);
  // The system releases a wake lock whenever the page is hidden.
  document.addEventListener('visibilitychange', async () => {
    if (document.hidden || !app.tracking) return;
    if (app.wakeLock && !app.wakeLock.released) return;
    try {
      app.wakeLock = await navigator.wakeLock?.request('screen');
    } catch {
      // Optional.
    }
  });
}

function boot() {
  bindControls();
  presentHost();
  const fromLink = decodeURIComponent(location.hash.slice(1))
    .toUpperCase()
    .replace(/[^0-9A-Z]/g, '');
  if (fromLink) {
    $('code').value = fromLink;
    store.set(CODE_STORAGE, fromLink);
    // Keep the code out of the address bar once read.
    history.replaceState(null, '', location.pathname);
  }
  if (app.key) {
    connect();
    showStart();
  } else if (fromLink) {
    join(fromLink);
  } else {
    show('pair');
  }
}

boot();