feat(voice): harden Gemini Live provider integration

server/providers/gemini.js

@@ -0,0 +1,22 @@
+import { createGeminiTokenHandler } from './gemini/realtime.js';
+
+/** Mint constrained Live API credentials; the permanent key stays on Node. */
+function geminiLiveProxy({ annotationGuidance, realtime = {} } = {}) {
+  function install(middlewares) {
+    middlewares.use(
+      '/api/gemini/token',
+      createGeminiTokenHandler({ ...realtime, annotationGuidance }),
+    );
+  }
+  return {
+    name: 'gemini-live-proxy',
+    configureServer(server) {
+      install(server.middlewares);
+    },
+    configurePreviewServer(server) {
+      install(server.middlewares);
+    },
+  };
+}
+
+export { geminiLiveProxy };

server/providers/gemini/config.js

@@ -0,0 +1,54 @@
+import { realtimeInstructions } from '../openai/instructions.js';
+import { GEV_REALTIME_TOOLS } from '../openai/tools.js';
+
+export const GEMINI_LIVE_MODEL_DEFAULT = 'gemini-3.8-live';
+export const GEMINI_TOKEN_ENDPOINT =
+  'https://generativelanguage.googleapis.com/v1beta/auth_tokens';
+export const GEMINI_LIVE_WEBSOCKET_URL =
+  'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained';
+
+export function normalizeGeminiModel(value) {
+  const model = String(value ?? GEMINI_LIVE_MODEL_DEFAULT)
+    .trim()
+    .replace(/^models\//, '');
+  return /^gemini-[a-z0-9][a-z0-9.-]{0,119}$/.test(model) ? model : null;
+}
+
+/** REST BidiGenerateContentSetup fields, excluding its model resource name. */
+export function createGeminiLiveConfig(
+  annotationGuidance,
+  inputMode = 'open-mic',
+  tools = GEV_REALTIME_TOOLS,
+  capabilities,
+) {
+  if (!['open-mic', 'push-to-talk'].includes(inputMode))
+    throw new Error('Invalid Gemini input mode');
+  const config = {
+    generationConfig: { responseModalities: ['AUDIO'] },
+    systemInstruction: {
+      parts: [{ text: realtimeInstructions(annotationGuidance, capabilities) }],
+    },
+    tools: [
+      {
+        functionDeclarations: tools.map(
+          ({ name, description, parameters }) => ({
+            name,
+            description,
+            // GEV's canonical schemas are JSON Schema, not Google's Schema.
+            parametersJsonSchema: structuredClone(parameters),
+            // App actions must finish before the model confirms their result.
+            behavior: 'BLOCKING',
+          }),
+        ),
+      },
+    ],
+    inputAudioTranscription: {},
+    outputAudioTranscription: {},
+    contextWindowCompression: { slidingWindow: {} },
+  };
+  if (inputMode === 'push-to-talk')
+    config.realtimeInputConfig = {
+      automaticActivityDetection: { disabled: true },
+    };
+  return config;
+}

server/providers/gemini/realtime.js

@@ -0,0 +1,180 @@
+import { clientKey, makeCostRateLimiter } from '../common/rate-limit.js';
+import { readResponseJsonCapped } from '../../../src/sources/httpBody.js';
+import {
+  GEMINI_TOKEN_ENDPOINT,
+  GEMINI_LIVE_WEBSOCKET_URL,
+  GEMINI_LIVE_MODEL_DEFAULT,
+  createGeminiLiveConfig,
+  normalizeGeminiModel,
+} from './config.js';
+import {
+  isGeminiRequestOriginAllowed,
+  readGeminiTokenRequest,
+} from './request.js';
+
+const UPSTREAM_RESPONSE_BYTES = 32 * 1024;
+const MINT_FAILURE = 'Gemini voice could not connect. Try again.';
+
+export function createGeminiTokenHandler({
+  annotationGuidance,
+  providerDropdown = false,
+  tools,
+  fetchImpl = (...args) => fetch(...args),
+  resolveApiKey = () => process.env.GEMINI_API_KEY,
+  resolveModel = () =>
+    process.env.GEMINI_LIVE_MODEL || GEMINI_LIVE_MODEL_DEFAULT,
+  resolveHost = () => process.env.HOST,
+  resolveAllowedHosts = () => process.env.GEV_ALLOWED_HOSTS,
+  resolveRateLimit = () => process.env.GEV_RATELIMIT_GEMINI_PER_MIN ?? '30',
+  now = Date.now,
+  timeoutMs = 15_000,
+} = {}) {
+  let limiter;
+  return async (req, res) => {
+    res.setHeader('Cache-Control', 'no-store');
+    res.setHeader('Content-Type', 'application/json; charset=utf-8');
+    res.setHeader('X-Content-Type-Options', 'nosniff');
+    const reply = (status, errorOrPayload) => {
+      if (res.destroyed || res.writableEnded) return;
+      res.statusCode = status;
+      if (status >= 400) res.setHeader('Connection', 'close');
+      res.end(
+        JSON.stringify(
+          typeof errorOrPayload === 'string'
+            ? { error: errorOrPayload }
+            : errorOrPayload,
+        ),
+      );
+    };
+    if (req.method !== 'POST') {
+      res.setHeader('Allow', 'POST');
+      return reply(405, 'Method not allowed');
+    }
+    if (
+      !isGeminiRequestOriginAllowed(req, resolveHost(), resolveAllowedHosts())
+    )
+      return reply(403, 'Gemini voice requires a same-origin app request');
+    const apiKey = String(resolveApiKey() || '').trim();
+    if (!apiKey) return reply(503, 'Add a Gemini API key in Provider Settings');
+    const model = normalizeGeminiModel(resolveModel());
+    if (!model)
+      return reply(503, 'The Gemini Live model configuration is invalid');
+    if (limiter === undefined)
+      limiter = makeCostRateLimiter(resolveRateLimit(), 30);
+    if (limiter && !limiter(clientKey(req))) {
+      res.setHeader('Retry-After', '60');
+      return reply(
+        429,
+        'Gemini voice is starting too often. Try again shortly.',
+      );
+    }
+
+    const lifetime = new AbortController();
+    let disconnected = false;
+    const disconnect = () => {
+      disconnected = true;
+      lifetime.abort(new DOMException('Request closed', 'AbortError'));
+    };
+    const responseClosed = () => {
+      if (!res.writableEnded) disconnect();
+    };
+    req.once('aborted', disconnect);
+    res.once('close', responseClosed);
+    const deadline = setTimeout(() => {
+      lifetime.abort(
+        new DOMException('Token request timed out', 'TimeoutError'),
+      );
+    }, timeoutMs);
+    deadline.unref?.();
+    try {
+      const inputMode = await readGeminiTokenRequest(req, lifetime.signal);
+      lifetime.signal.throwIfAborted();
+      const config = createGeminiLiveConfig(
+        annotationGuidance,
+        inputMode,
+        tools,
+        { providerDropdown },
+      );
+      const issuedAt = now();
+      // REST uses bidiGenerateContentSetup. liveConnectConstraints is the SDK
+      // input shape, converted before transport. With no fieldMask the complete
+      // setup is locked to this server-owned configuration.
+      const request = {
+        uses: 1,
+        expireTime: new Date(issuedAt + 30 * 60_000).toISOString(),
+        newSessionExpireTime: new Date(issuedAt + 60_000).toISOString(),
+        bidiGenerateContentSetup: { model: `models/${model}`, ...config },
+      };
+      const response = await fetchImpl(GEMINI_TOKEN_ENDPOINT, {
+        method: 'POST',
+        redirect: 'error',
+        signal: lifetime.signal,
+        headers: {
+          'x-goog-api-key': apiKey,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(request),
+      });
+      lifetime.signal.throwIfAborted();
+      if (
+        response.redirected ||
+        (response.url && response.url !== GEMINI_TOKEN_ENDPOINT)
+      )
+        throw new Error('Unexpected token response destination');
+      if (!response.ok) {
+        // Never forward upstream bodies, headers or exception messages.
+        void response.body?.cancel().catch(() => {});
+        if (response.status === 429) {
+          res.setHeader('Retry-After', '60');
+          return reply(429, 'Gemini quota is unavailable. Try again later.');
+        }
+        if (response.status === 401 || response.status === 403)
+          return reply(
+            503,
+            'Gemini access was refused. Check the API key and model access.',
+          );
+        if (response.status === 400 || response.status === 404)
+          return reply(
+            502,
+            'Gemini Live is unavailable for the configured model.',
+          );
+        return reply(502, MINT_FAILURE);
+      }
+      const data = await readResponseJsonCapped(
+        response,
+        UPSTREAM_RESPONSE_BYTES,
+        lifetime.signal,
+      );
+      lifetime.signal.throwIfAborted();
+      if (
+        !data ||
+        Array.isArray(data) ||
+        typeof data !== 'object' ||
+        typeof data.name !== 'string' ||
+        data.name.length > 8192 ||
+        !/^auth_tokens\/[A-Za-z0-9._~+/=-]+$/.test(data.name) ||
+        data.name.includes(apiKey)
+      )
+        throw new Error('Invalid ephemeral token');
+      reply(200, {
+        token: data.name,
+        model,
+        config,
+        websocketUrl: GEMINI_LIVE_WEBSOCKET_URL,
+      });
+    } catch (error) {
+      if (disconnected || req.aborted) return;
+      if (error?.code === 'BODY_TOO_LARGE')
+        return reply(413, 'Token request is too large');
+      if (error?.code === 'BAD_CONTENT_TYPE')
+        return reply(415, 'Use an empty JSON token request');
+      if (error?.code === 'BAD_BODY')
+        return reply(400, 'Use an empty JSON token request');
+      reply(lifetime.signal.aborted ? 504 : 502, MINT_FAILURE);
+    } finally {
+      clearTimeout(deadline);
+      req.off('aborted', disconnect);
+      res.off('close', responseClosed);
+    }
+  };
+}

server/providers/gemini/request.js

@@ -0,0 +1,117 @@
+import { admitSameSiteRequest } from '../../../src/localRequestGate.mjs';
+import {
+  isAllowedHost,
+  resolveAllowedHosts,
+} from '../../../build/allowedHosts.js';
+
+/** Use the app's Host and request policies, with a required browser Origin. */
+export function isGeminiRequestOriginAllowed(
+  req,
+  configuredHost = '',
+  configuredAllowedHosts,
+) {
+  const host = req.headers?.host;
+  const origin = req.headers?.origin;
+  if (
+    typeof host !== 'string' ||
+    typeof origin !== 'string' ||
+    !host ||
+    /[\s/?#@\\]/.test(host)
+  )
+    return false;
+  const protocol = req.socket?.encrypted ? 'https:' : 'http:';
+  if (
+    !admitSameSiteRequest({
+      hostHeader: host,
+      protocol,
+      origin,
+      secFetchSite: req.headers?.['sec-fetch-site'],
+      proxyHeaders: req.headers || {},
+    }).ok
+  )
+    return false;
+  const boundHost = String(configuredHost || '')
+    .trim()
+    .toLowerCase();
+  if (
+    !isAllowedHost(host, resolveAllowedHosts(configuredAllowedHosts), [
+      boundHost,
+    ])
+  )
+    return false;
+  try {
+    const authority = new URL(`${protocol}//${host}`);
+    const source = new URL(origin);
+    return (
+      source.origin === authority.origin &&
+      source.username === '' &&
+      source.password === '' &&
+      source.pathname === '/' &&
+      source.search === '' &&
+      source.hash === ''
+    );
+  } catch {
+    return false;
+  }
+}
+
+/** Admit only the existing input gesture; callers cannot supply model/setup. */
+export async function readGeminiTokenRequest(req, signal, maxBytes = 1024) {
+  const declared = Number(req.headers?.['content-length']);
+  const fail = (code) =>
+    Object.assign(new Error('Invalid token request'), { code });
+  if (Number.isFinite(declared) && declared > maxBytes)
+    throw fail('BODY_TOO_LARGE');
+  const body = await new Promise((resolve, reject) => {
+    let total = 0;
+    const chunks = [];
+    const cleanup = () => {
+      req.off('data', data);
+      req.off('end', end);
+      req.off('error', error);
+      signal.removeEventListener('abort', abort);
+    };
+    const error = (reason) => {
+      cleanup();
+      req.pause();
+      reject(reason);
+    };
+    const abort = () => error(signal.reason);
+    const data = (chunk) => {
+      total += chunk.length;
+      if (total > maxBytes) return error(fail('BODY_TOO_LARGE'));
+      chunks.push(chunk);
+    };
+    const end = () => {
+      cleanup();
+      resolve(Buffer.concat(chunks).toString('utf8'));
+    };
+    req.on('data', data);
+    req.once('end', end);
+    req.once('error', error);
+    signal.addEventListener('abort', abort, { once: true });
+    if (signal.aborted) abort();
+  });
+  if (!body.trim()) return 'open-mic';
+  if (
+    !/^application\/json(?:\s*;|$)/i.test(req.headers?.['content-type'] || '')
+  )
+    throw fail('BAD_CONTENT_TYPE');
+  let parsed;
+  try {
+    parsed = JSON.parse(body);
+  } catch {
+    throw fail('BAD_BODY');
+  }
+  if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object')
+    throw fail('BAD_BODY');
+  const keys = Object.keys(parsed);
+  if (!keys.length) return 'open-mic';
+  if (
+    keys.length === 1 &&
+    keys[0] === 'inputMode' &&
+    parsed.inputMode === 'push-to-talk'
+  )
+    return 'push-to-talk';
+  throw fail('BAD_BODY');
+}

server/providers/local.js

@@ -19,6 +19,7 @@ import { adsbLolProxy } from './aircraft/adsb-lol.js';
 import { aisLiveProxy } from './vessels/ais-live.js';
 import { trackBackfillProxies } from './aircraft/tracks.js';
 import { openAiRealtimeProxy } from './openai.js';
+import { geminiLiveProxy } from './gemini.js';
 import { googlePlacesContextProxy } from './places.js';
 import { mapillaryProxy } from './mapillary.js';
 import { keySetupEndpoint } from '../standalone/key-setup.js';
@@ -55,6 +56,7 @@ function localProviderPlugins({ realtime } = {}) {
     aisLiveProxy(),
     trackBackfillProxies(),
     openAiRealtimeProxy({ realtime }),
+    geminiLiveProxy({ realtime }),
     googlePlacesContextProxy(),
     windProxy(),
     weatherProxy(),

server/providers/openai/instructions.js

@@ -1,10 +1,15 @@
+import { voiceProviderGuidance } from '../../../src/voice/providerCapabilities.js';
+
 const DEFAULT_ANNOTATION_GUIDANCE =
   'Put related places in one annotate_map call. Set flyTo true only when the user is not already looking at the place (off-screen marks auto-frame, so when unsure leave it false). Do not say you are drawing or annotating — talk about the places while the marks appear. Marks ACCUMULATE AND PERSIST across flights and topics so the user can build up the map: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
 
 function realtimeInstructions(
   annotationGuidance = DEFAULT_ANNOTATION_GUIDANCE,
+  capabilities,
 ) {
   return [
+    'When the user explicitly requests Radio playback, the shared client handoff verifies playback before giving Radio the speaker. Enabling the Radio layer alone never requests that handoff. Successful Radio playback, requested or manual, suspends active voice. Voice resumes once when the user pauses or turns off that Radio, with no time limit; Radio-only station or volume changes keep that right. Explicit voice Stop, starting voice, provider replacement and teardown cancel it. Confirm authoritative returned playback state; a prepared result is not already playing.',
+    `For a clear standalone voice-provider request naming OpenAI or Gemini, or asking to choose a voice provider, say exactly once in your normal voice: “${voiceProviderGuidance(capabilities)}” Do not call a tool, change provider state, or claim the provider changed. Voice volume/speed and map/search providers are not voice-provider requests. Ambiguous, quoted, negated, conditional and compound text remains ordinary conversation.`,
     "You are GEV Voice Control, a concise voice controller for a Cesium geospatial app called God's Eye View.",
     'Have a natural spoken conversation with the user while the mic session is active.',
     'Do not require a wake phrase. Treat direct commands like "zoom into London" or "open datacenters" as GEV control requests.',
@@ -51,7 +56,7 @@ function realtimeInstructions(
     'HUD requests ("hud on/off", "switch to operator/minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
     'Cyber sonar requests use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch themes, reveal the HUD or enable sonar when changing a slider. If the user explicitly requests Cyber plus sonar settings, call set_hud first, wait for success, then set_cyber_sonar. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
     'Bloom/sharpen requests use set_post_processing. Scene requests ("play orbital watch", "stop the scene", "what scenes are there") use control_scene. CCTV camera requests ("next camera", "nearest camera", "select the Congress camera", "show coverage") use control_cctv — the CCTV layer must be enabled first.',
-    'Radio playback requests use control_radio. "Turn on/start the radio" means action=play; action=enable only reveals Radio markers and must be reserved for explicit "show/enable the Radio layer/markers" requests. After a prepared playback result, briefly confirm any other completed actions and say "Turning on the radio"—never claim it is already playing. The client keeps Radio muted until playback is verified, then closes voice before restoring Radio volume. Examples: "play news near Austin" → select category=news locationId=austin; "play US news" → select category=news country=US; "Radio volume 30" → volume; pause/resume/stop/next/previous use the matching action. Radio selection never moves the camera.',
+    'Radio requests use control_radio. "Turn on/start the radio" means action=enable: reveal the Radio layer/markers without starting audio or selecting a station. Showing or enabling Radio never implies playback, even when a station category or place is mentioned. Playback requires an explicit play, listen, or resume request. Use play for unqualified explicit playback, resume for explicit resume, and select for explicit playback qualified by category, station, country, coordinates, or place. After a prepared playback result, briefly confirm any other completed actions and say playback is being prepared—never claim it is already playing. The client keeps Radio muted until playback is verified, then closes voice before restoring Radio volume. Examples: "turn the radio on" → enable; "show radio stations" → enable; "play the radio" → play; "play news near Austin" → select category=news locationId=austin; "play US news" → select category=news country=US; "Radio volume 30" → volume; pause/resume/stop/next/previous use the matching action. Radio selection never moves the camera.',
     '"Track/follow <something specific>" (a callsign, ship name, satellite name) uses track_entity. "Take me to the biggest fire" uses track_entity with query "biggest fire" (the fires layer must be enabled). Bare "orbit" means camera orbit of the current landmark. "Stop following/tracking" uses stop_tracking.',
     '"Show me which planes are overhead"/"frame the ships"/"show me the satellites above" use frame_overhead with the matching target.',
     'Confirmations echo the RESULTING state, never the request: "HUD operator layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',

server/providers/openai/realtime.js

@@ -129,6 +129,7 @@ function createRealtimeOAuthLoginHandler({
 
 function createRealtimeTokenHandler({
   annotationGuidance,
+  providerDropdown = false,
   endpoint = 'https://api.openai.com/v1/realtime/client_secrets',
   fetchImpl = (...args) => fetch(...args),
   resolveApiKey = () => process.env.OPENAI_API_KEY,
@@ -298,7 +299,9 @@ function createRealtimeTokenHandler({
           },
           output: { voice },
         },
-        instructions: realtimeInstructions(annotationGuidance),
+        instructions: realtimeInstructions(annotationGuidance, {
+          providerDropdown,
+        }),
         tools,
         tool_choice: 'auto',
       },

server/providers/openai/toolDescriptions.js

@@ -314,13 +314,13 @@ export const ACTION_DESCRIPTIONS = {
   },
   control_radio: {
     description:
-      'Control Internet Radio playback without moving the map. Use select whenever the request includes a station category, name, country, coordinates, or nearby place—even when the user says play. Use play only for an unqualified "turn on/start the radio" request so the current or nearest station begins. Enable only reveals the Radio layer/markers without audio. Also supports disable, resume, pause, stop, next/previous, volume, and status.',
+      'Control Internet Radio without moving the map. "Turn on/start the radio" means enable: reveal the Radio layer/markers without starting audio or selecting a station. Playback requires an explicit play, listen, or resume request. Use select for an explicit playback request qualified by category, station name, country, coordinates, or nearby place—even when the user says play. Use play for unqualified explicit playback. Also supports disable, resume, pause, stop, next/previous, volume, and status.',
     $position: 1,
     parameters: {
       properties: {
         action: {
           description:
-            'Use select for any request qualified by category, station, country, coordinates, or place. Use play only for an unqualified turn on/start/listen request. Use enable only when the user explicitly asks to show or enable the Radio layer or its markers without requesting audio.',
+            'Use enable for turn on/start/show/enable Radio or its markers without an explicit playback request. Enable never starts audio or selects a station. Use play for an explicit unqualified play/listen request, resume for explicit resume, and select for explicit playback qualified by category, station, country, coordinates, or place.',
           $position: 2,
         },
         volumePct: {

server/standalone/vite.config.js

@@ -17,7 +17,9 @@ export default defineConfig(({ command, mode }) => {
   }
   return createBrowserViteConfig({
     plugins: [
-      ...localProviderPlugins({ realtime: { tools: standaloneVoiceTools() } }),
+      ...localProviderPlugins({
+        realtime: { tools: standaloneVoiceTools(), providerDropdown: true },
+      }),
       localMcpPlugin(),
       apiNotFoundPlugin(),
     ],