refactor: separate voice controls and configurable Realtime connections

src/radioMarkup.test.mjs

@@ -15,7 +15,7 @@ const radio = ['playback', 'interaction'].map(name =>
   readFileSync(new URL(`./layers/radio/${name}.js`, import.meta.url), 'utf8')
 ).join('\n').replace(/layerState\.|parts\.\w+\./g, '');
 const rocketLaunches = readLayerSource(new URL('./data/rocketLaunches.js', import.meta.url), 'utf8');
-const realtime = readFileSync(new URL('./voice/gevRealtime.js', import.meta.url), 'utf8');
+const realtime = readFileSync(new URL('./voice/realtimeController.js', import.meta.url), 'utf8');
 const voice = ['tools', 'instructions'].map(name => readFileSync(new URL(`../server/providers/openai/${name}.js`, import.meta.url), 'utf8')).join('\n');
 const css = readStylesheet(new URL('../style.css', import.meta.url));
 

src/standalone/application.js

@@ -13,6 +13,7 @@ export function createStandaloneApplication({
   googleApiKey,
   cesiumToken,
   geospatial = {},
+  voice = {},
   allowQaRegistration = false,
 }) {
   if (constructed)
@@ -40,6 +41,6 @@ export function createStandaloneApplication({
     createData: (context) =>
       createStandaloneData({ ...context, allowQaRegistration }),
     createTools: (context) =>
-      createStandaloneTools({ ...context, loadingScreen, placeSearch }),
+      createStandaloneTools({ ...context, loadingScreen, placeSearch, voice }),
   });
 }

src/standalone/tools.js

@@ -18,6 +18,7 @@ export function createStandaloneTools({
   data,
   loadingScreen,
   placeSearch,
+  voice = {},
   signal,
   defer,
 }) {
@@ -100,6 +101,8 @@ export function createStandaloneTools({
     if (window.__godsEyeView === debug) delete window.__godsEyeView;
   });
   const voiceCommands = initGevVoiceCommands({
+    ...voice,
+    signal,
     placeSearch,
     viewer,
     styleManager,

src/tooling/localServices.test.mjs

@@ -255,3 +255,31 @@ test('key setup writes only the supplied application root, retains request guard
   assert.equal(saved.body.includes('sk-fixture-only-not-a-real-key'), false);
   assert.equal(existsSync(path.join(untouched, '.env')), false);
 });
+
+test('Realtime service configuration selects compatible endpoint/model without forwarding request model IDs or keys', async () => {
+  const handler = install(
+    openAiRealtimeProxy({
+      realtime: {
+        endpoint: 'https://voice.example/client-secrets',
+        models: { standard: 'configured-model' },
+        resolveApiKey: () => 'server-fixture',
+        fetchImpl: async (url, options) => {
+          assert.equal(url, 'https://voice.example/client-secrets');
+          assert.equal(options.redirect, 'error');
+          assert.equal(options.headers.Authorization, 'Bearer server-fixture');
+          const payload = JSON.parse(options.body);
+          assert.equal(payload.session.model, 'configured-model');
+          assert.deepEqual(payload.session.tools, GEV_REALTIME_TOOLS);
+          return Response.json({ value: 'short-lived-fixture' });
+        },
+      },
+    }),
+  ).get('/api/realtime/token');
+  const response = await request(handler, {
+    url: '/?tier=arbitrary-model&model=other',
+  });
+  assert.equal(response.status, 200);
+  assert.equal(response.headers['x-gev-voice-model'], 'configured-model');
+  assert.equal(response.headers['cache-control'], 'no-store');
+  assert.doesNotMatch(response.body, /server-fixture|voice\.example/);
+});

src/voice/aircraftIdentityNarration.test.mjs

@@ -3,7 +3,7 @@ import { readFileSync } from 'node:fs';
 import test from 'node:test';
 
 const voiceConfig = readFileSync(new URL('../../server/providers/openai/instructions.js', import.meta.url), 'utf8');
-const realtime = readFileSync(new URL('./gevRealtime.js', import.meta.url), 'utf8');
+const realtime = readFileSync(new URL('./realtimeController.js', import.meta.url), 'utf8');
 
 test('aircraft identity narration acknowledges missing enrichment', () => {
   const start = voiceConfig.indexOf("'For \"what is this aircraft?\" answers");

src/voice/commands.js

@@ -0,0 +1,58 @@
+import {
+  GevRealtimeController,
+  shouldIgnoreVoiceButtonClick,
+} from './realtimeController.js';
+import { createVoiceControl } from './control.js';
+
+/** Bind a supplied action runner and connection controller to the voice controls. */
+export function createVoiceCommands({
+  runner,
+  dataManager,
+  annotations = null,
+  backend,
+  signal,
+  debugSink,
+  createController = (options) => new GevRealtimeController(options),
+}) {
+  if (
+    window.__gevVoiceCommands &&
+    typeof window.__gevVoiceCommands.stop === 'function'
+  ) {
+    window.__gevVoiceCommands.stop({ removeUi: true });
+  }
+  const ui = createVoiceControl({ reset: true });
+  const radioLayer = dataManager?.layers?.get('radio')?.module || null;
+  const controller = createController({
+    runner,
+    ui,
+    radioLayer,
+    dataManager,
+    backend,
+    signal,
+    debugSink,
+  });
+  // Deferred annotation outlines finish AFTER their tool result returned. Feed the
+  // final outcome (resolved / failed) into the conversation so the model can honestly
+  // confirm — or correct — what it narrated about a boundary it never saw land.
+  if (annotations && typeof annotations.onOutlineEvent === 'function') {
+    controller.annotationEventUnsubscribe = annotations.onOutlineEvent(
+      (evt) => {
+        controller.notifyMapEvent({ type: 'map_annotation_outline', ...evt });
+      },
+    );
+  }
+  controller.buttonHandler = () => {
+    if (shouldIgnoreVoiceButtonClick(controller.spaceKeyHeld)) return;
+    if (controller.isActive()) controller.stop();
+    else controller.start({ pushToTalk: false });
+  };
+  ui.button.addEventListener('click', controller.buttonHandler);
+  if (ui.tierButton) {
+    controller.tierHandler = () => controller.toggleVoiceTier();
+    ui.tierButton.addEventListener('click', controller.tierHandler);
+  }
+  controller.syncCostUi();
+  controller.bindPushToTalkShortcut();
+  window.__gevVoiceCommands = controller;
+  return controller;
+}

src/voice/control.js

@@ -0,0 +1,72 @@
+/** Build the voice control independently of its connection backend. */
+export function createVoiceControl({ reset = false } = {}) {
+  let root = document.getElementById('gev-voice-control');
+  if (root && reset) {
+    root.remove();
+    root = null;
+  }
+  if (!root) {
+    root = document.createElement('div');
+    root.id = 'gev-voice-control';
+    root.dataset.status = 'idle';
+    root.dataset.speaker = 'idle';
+    root.innerHTML = `
+      <div class="gev-voice-heading">
+        <div class="gev-voice-kicker">AI AGENT</div>
+        <div id="gev-voice-status">OFF</div>
+        <div class="gev-voice-cost">
+          <button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="Voice model tier — applies next session">STD</button>
+          <span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="Estimated session cost">~$0.00</span>
+        </div>
+      </div>
+      <button id="gev-voice-button" type="button" aria-label="Voice control — activate to toggle voice; hold Space to speak" aria-describedby="gev-voice-help">
+        <span class="gev-mic-orbit"><img src="/mic.svg" alt="" /></span>
+        <span class="gev-mic-label">ON/OFF</span>
+      </button>
+      <div class="gev-voice-visualizer" aria-hidden="true">
+        ${Array.from({ length: 15 }, (_, index) => `<span style="--bar:${index}"></span>`).join('')}
+      </div>
+      <div class="gev-voice-readout">
+        <div id="gev-voice-detail">VOICE STANDBY</div>
+      </div>
+      <div id="gev-voice-help" class="gev-voice-help-tray" role="tooltip">
+        <span class="gev-voice-help-kicker">VOICE CONTROL</span>
+        <span class="gev-voice-help-detail">Hold Space to speak · tap Space to activate focused controls</span>
+      </div>
+      <div class="gev-voice-error-tray" role="alert" aria-live="assertive">
+        <div class="gev-voice-error-header">
+          <span>VOICE SYSTEM ERROR</span>
+          <button class="gev-voice-error-dismiss" type="button">DISMISS</button>
+        </div>
+        <div id="gev-voice-error-detail"></div>
+        <div class="gev-voice-error-hint">Check microphone permission and network access, then try again.</div>
+      </div>
+    `;
+    const commandDock = document.getElementById('command-dock');
+    if (commandDock) {
+      const locationBar = document.getElementById('location-bar');
+      const controlPanel = document.getElementById('control-panel');
+      commandDock.appendChild(root);
+      if (locationBar) commandDock.insertBefore(locationBar, root);
+      if (controlPanel) commandDock.appendChild(controlPanel);
+    } else {
+      document.body.appendChild(root);
+    }
+    root
+      .querySelector('.gev-voice-error-dismiss')
+      ?.addEventListener('click', () => {
+        root.classList.add('error-dismissed');
+      });
+  }
+  return {
+    root,
+    button: root.querySelector('#gev-voice-button'),
+    buttonLabel: root.querySelector('.gev-mic-label'),
+    status: root.querySelector('#gev-voice-status'),
+    detail: root.querySelector('#gev-voice-detail'),
+    helpDetail: root.querySelector('.gev-voice-help-detail'),
+    errorDetail: root.querySelector('#gev-voice-error-detail'),
+    tierButton: root.querySelector('#gev-voice-tier'),
+    costValue: root.querySelector('#gev-voice-cost-value'),
+  };
+}

src/voice/gevActions.js

@@ -1,3 +1,5 @@
+import { readLayerLifecycleSummary } from './layerSummary.js';
+export { readLayerLifecycleSummary } from './layerSummary.js';
 import { defaultGeospatial } from '../search/defaults.js';
 import * as Cesium from 'cesium';
 import { CITY_POIS, findPoiByName, flyToGlobeView, flyToLandmark, flyToPOI, flyToPresetLocation, GLOBE_VIEW, searchAndFlyTo } from '../locations.js';
@@ -265,46 +267,7 @@ const VISIBLE_ENTITY_SHORTLIST = 64;
 const BASEMAP_CONTEXT_WAIT_MS = 1500;
 const viewTargetCache = new WeakMap();
 
-/**
- * Read one layer's authoritative visibility and lifecycle presentation state.
- * Falls back to stable enabled/disabled state for lightweight adapters that do
- * not expose the manager lifecycle API.
- * @param {object|null} dataManager Layer manager or lightweight adapter.
- * @param {string} layerId Registered layer identifier.
- * @param {object} [options] Fallback options.
- * @param {boolean} [options.fallbackEnabled=false] Observed state when no manager read is available.
- * @returns {{enabled:boolean,lifecycleState:string,lifecycleUncertain:boolean}} Lifecycle summary.
- */
-export function readLayerLifecycleSummary(dataManager, layerId, { fallbackEnabled = false } = {}) {
-  let lifecycle = null;
-  try {
-    lifecycle = dataManager?.getLayerLifecycleState?.(layerId) || null;
-  } catch {
-    lifecycle = null;
-  }
-  if (lifecycle) {
-    const enabled = Boolean(lifecycle.enabled);
-    return {
-      enabled,
-      lifecycleState: lifecycle.lifecycleState || (enabled ? 'enabled' : 'disabled'),
-      lifecycleUncertain: Boolean(lifecycle.uncertain ?? lifecycle.lifecycleUncertain),
-    };
-  }
-
-  let enabled = Boolean(fallbackEnabled);
-  try {
-    const managerEnabled = dataManager?.isEnabled?.(layerId);
-    if (typeof managerEnabled === 'boolean') enabled = managerEnabled;
-  } catch {
-    // Retain the caller's observed fallback when the lightweight adapter fails.
-  }
-  return {
-    enabled,
-    lifecycleState: enabled ? 'enabled' : 'disabled',
-    lifecycleUncertain: false,
-  };
-}
-
+/** Create application actions over the supplied scene and services. */
 export function createGevActionRunner({ viewer, styleManager, dataManager, sceneDirector = null, annotations = null, placeSearch = unavailablePlaceSearch }) {
   installViewTargetPrewarm(viewer);
   initCameraVerbs(viewer, getViewTargetCartesian);

src/voice/layerSummary.js

@@ -0,0 +1,37 @@
+/** Read layer lifecycle through the supplied manager. */
+export function readLayerLifecycleSummary(
+  dataManager,
+  layerId,
+  { fallbackEnabled = false } = {},
+) {
+  let lifecycle = null;
+  try {
+    lifecycle = dataManager?.getLayerLifecycleState?.(layerId) || null;
+  } catch {
+    lifecycle = null;
+  }
+  if (lifecycle) {
+    const enabled = Boolean(lifecycle.enabled);
+    return {
+      enabled,
+      lifecycleState:
+        lifecycle.lifecycleState || (enabled ? 'enabled' : 'disabled'),
+      lifecycleUncertain: Boolean(
+        lifecycle.uncertain ?? lifecycle.lifecycleUncertain,
+      ),
+    };
+  }
+
+  let enabled = Boolean(fallbackEnabled);
+  try {
+    const managerEnabled = dataManager?.isEnabled?.(layerId);
+    if (typeof managerEnabled === 'boolean') enabled = managerEnabled;
+  } catch {
+    // Retain the caller's observed fallback when the lightweight adapter fails.
+  }
+  return {
+    enabled,
+    lifecycleState: enabled ? 'enabled' : 'disabled',
+    lifecycleUncertain: false,
+  };
+}

src/voice/realtimeBackend.js

@@ -0,0 +1,90 @@
+import { DEFAULT_VOICE_TIER, resolveVoiceModel } from './voiceCost.js';
+
+/** Realtime-compatible token and SDP requests, independent of microphone/UI ownership. */
+export function createRealtimeBackend({
+  tokenEndpoint = '/api/realtime/token',
+  callsEndpoint = 'https://api.openai.com/v1/realtime/calls',
+  tokenTransport = (...args) => fetch(...args),
+  connectionTransport = (...args) => fetch(...args),
+  timeoutMs = 30_000,
+} = {}) {
+  const scoped = (signal) =>
+    AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)].filter(Boolean));
+  return Object.freeze({
+    protocol: 'openai-realtime',
+    async requestToken({ tier = DEFAULT_VOICE_TIER, signal } = {}) {
+      signal = scoped(signal);
+      signal.throwIfAborted();
+      const separator = tokenEndpoint.includes('?') ? '&' : '?';
+      const url = `${tokenEndpoint}${separator}tier=${encodeURIComponent(resolveVoiceModel(tier).tier)}`;
+      const response = await tokenTransport(url, {
+        signal,
+        cache: 'no-store',
+        redirect: 'error',
+      });
+      signal.throwIfAborted();
+      const data = await response.json().catch(() => null);
+      signal.throwIfAborted();
+      if (!response.ok) {
+        const reason =
+          typeof data?.error === 'string' ? data.error : data?.error?.message;
+        throw new Error(
+          reason || `Realtime token failed: HTTP ${response.status}`,
+        );
+      }
+      const token =
+        data?.value || data?.client_secret?.value || data?.client_secret;
+      if (typeof token !== 'string' || !token)
+        throw new Error(
+          'Realtime token response did not include a client secret',
+        );
+      const expiresAt = data?.expires_at ?? data?.client_secret?.expires_at;
+      if (
+        expiresAt != null &&
+        (!Number.isFinite(expiresAt) || expiresAt * 1000 <= Date.now())
+      )
+        throw new Error(
+          'Realtime client secret has expired; reconnect to request a new one',
+        );
+      return {
+        token,
+        model:
+          response.headers?.get?.('X-GEV-Voice-Model') ||
+          data?.session?.model ||
+          null,
+        tier: response.headers?.get?.('X-GEV-Voice-Tier') || null,
+        expiresAt,
+      };
+    },
+    async negotiate({ offerSdp, credential, signal }) {
+      signal = scoped(signal);
+      signal.throwIfAborted();
+      if (
+        credential.expiresAt != null &&
+        credential.expiresAt * 1000 <= Date.now()
+      )
+        throw new Error(
+          'Realtime client secret has expired; reconnect to request a new one',
+        );
+      const response = await connectionTransport(callsEndpoint, {
+        method: 'POST',
+        body: offerSdp,
+        signal,
+        redirect: 'error',
+        cache: 'no-store',
+        headers: {
+          Authorization: `Bearer ${credential.token}`,
+          'Content-Type': 'application/sdp',
+        },
+      });
+      signal.throwIfAborted();
+      if (!response.ok) {
+        await response.body?.cancel?.().catch(() => {});
+        throw new Error(`Realtime SDP failed: HTTP ${response.status}`);
+      }
+      const answer = await response.text();
+      signal.throwIfAborted();
+      return answer;
+    },
+  });
+}

src/voice/realtimeBackend.test.mjs

@@ -0,0 +1,131 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { createRealtimeBackend } from './realtimeBackend.js';
+import { GevRealtimeController } from './realtimeController.js';
+
+const tokenReply = () =>
+  Response.json({
+    value: 'ephemeral-test',
+    session: { model: 'compatible-model' },
+    expires_at: Math.floor(Date.now() / 1000) + 60,
+  });
+
+test('separate configured token and SDP transports preserve model metadata and credential scope', async () => {
+  const calls = [];
+  const backend = createRealtimeBackend({
+    tokenEndpoint: '/api/voice/token',
+    callsEndpoint: 'https://voice.example/calls',
+    tokenTransport: async (input, init) => {
+      calls.push(input);
+      assert.equal(init.redirect, 'error');
+      assert.equal(init.headers, undefined);
+      return tokenReply();
+    },
+    connectionTransport: async (input, init) => {
+      calls.push(input);
+      assert.equal(init.redirect, 'error');
+      assert.equal(init.body, 'offer');
+      assert.equal(init.headers.Authorization, 'Bearer ephemeral-test');
+      return new Response('answer');
+    },
+  });
+  const credential = await backend.requestToken({ tier: 'mini' });
+  assert.equal(credential.model, 'compatible-model');
+  assert.equal(
+    await backend.negotiate({ credential, offerSdp: 'offer' }),
+    'answer',
+  );
+  assert.deepEqual(calls, [
+    '/api/voice/token?tier=mini',
+    'https://voice.example/calls',
+  ]);
+});
+
+test('denied tokens never negotiate or retry; reconnect requests a fresh token', async () => {
+  let requests = 0;
+  const backend = createRealtimeBackend({
+    tokenTransport: async () => {
+      requests++;
+      return requests === 1
+        ? Response.json({ error: 'Unavailable' }, { status: 401 })
+        : tokenReply();
+    },
+    connectionTransport: () => assert.fail('unexpected connection'),
+  });
+  await assert.rejects(backend.requestToken(), /Unavailable/);
+  assert.equal(requests, 1);
+  assert.equal((await backend.requestToken()).token, 'ephemeral-test');
+  assert.equal(requests, 2);
+});
+
+test('expired and malformed client secrets are rejected before SDP exchange', async () => {
+  for (const data of [
+    { value: 'test', expires_at: 1 },
+    { value: {} },
+    { value: 'test', expires_at: 'later' },
+  ]) {
+    const backend = createRealtimeBackend({
+      tokenTransport: async () => Response.json(data),
+    });
+    await assert.rejects(backend.requestToken(), /expired|client secret/);
+  }
+  const backend = createRealtimeBackend({
+    connectionTransport: () => assert.fail('expired token transmitted'),
+  });
+  await assert.rejects(
+    backend.negotiate({
+      credential: { token: 'test', expiresAt: 1 },
+      offerSdp: 'offer',
+    }),
+    /expired/,
+  );
+});
+
+test('cancellation rejects late token and SDP bodies without promoting a stopped connection', async () => {
+  for (const phase of ['token', 'sdp']) {
+    const lifetime = new AbortController();
+    let release;
+    const pending = new Promise((resolve) => {
+      release = resolve;
+    });
+    const backend = createRealtimeBackend({
+      tokenTransport: async () => ({ ok: true, json: () => pending }),
+      connectionTransport: async () => ({ ok: true, text: () => pending }),
+    });
+    const result =
+      phase === 'token'
+        ? backend.requestToken({ signal: lifetime.signal })
+        : backend.negotiate({
+            credential: { token: 'test' },
+            offerSdp: 'offer',
+            signal: lifetime.signal,
+          });
+    await Promise.resolve();
+    lifetime.abort();
+    release(phase === 'token' ? { value: 'late' } : 'late answer');
+    await assert.rejects(result, { name: 'AbortError' });
+  }
+});
+
+test('controller lifetime stops pending transport and releases resources through normal teardown', () => {
+  const lifetime = new AbortController();
+  let stopped = 0;
+  const controller = new GevRealtimeController({
+    runner: async () => ({}),
+    signal: lifetime.signal,
+    debugSink: null,
+    ui: {
+      root: { dataset: {}, querySelectorAll: () => [], remove() {} },
+      status: {},
+      detail: {},
+    },
+  });
+  controller.connectionAbort = new AbortController();
+  const connection = controller.connectionAbort.signal;
+  controller.stream = { getTracks: () => [{ stop: () => stopped++ }] };
+  lifetime.abort();
+  assert.equal(connection.aborted, true);
+  assert.equal(stopped, 1);
+  assert.equal(controller.stream, null);
+  assert.equal(controller.isActive(), false);
+});