feat(voice): add typed AI chat beside voice

Chat sits next to the mic in the command dock. Replies are silent and never
request the microphone; the first message connects with the selected provider
and runs the same map actions as voice through each provider's text API
(OpenAI OPENAI_CHAT_MODEL, Gemini GEMINI_CHAT_MODEL via generateContent, which
never starts Live). Only one mode runs at a time.

- Chat panel rendered by the voice control presenter: conversation, recent and
  quick-command suggestions, Up/Down history, attachments (map-layer
  references or small UTF-8 text files, sent only with the message)
- Chat and voice panels sit above the dock or dock into the right rail's
  column; minimized bars sit centred above the dock, and height animates on
  swaps
- A centred panel steps down to its bar for the Location and Visual Presets
  trays without moving focus, rests above an open tray, and docks right while
  expanded over one; it returns to the centre once the trays close
- Minimized voice previews the last sentence from each side; the empty Voice
  Preview suggests things to say
- Settings opened from chat keep chat's frame and show the shared AI provider
  and quick commands; voice Settings show Back to voice
- Closing chat ends a typed session; the voice panel's close turns voice off
- The dock widens its voice control for the chat button and restyles its
  wings, which show icons only at 900px and below; assistant panels share the
  left panels' glass, and Cyber keeps its own surfaces
- Server text-chat handlers with body, time and per-IP limits
  (GEV_RATELIMIT_*_CHAT_PER_MIN), documented in SECURITY.md; tests and all
  eleven locales

Co-authored-by: Darren Dignam <[email protected]>

server/providers/common/textChat.js

@@ -0,0 +1,136 @@
+import { readRequestBody } from './request.js';
+import { clientKey, makeCostRateLimiter } from './rate-limit.js';
+import { waitForSignal } from '../../../src/sources/requestDeadline.js';
+import { readResponseJsonCapped } from '../../../src/sources/httpBody.js';
+import { isGeminiRequestOriginAllowed } from '../gemini/request.js';
+import { parseUsage } from './textTools.js';
+
+const MAX_BYTES = 512 * 1024;
+const FAILURE = 'Could not connect. Try again.';
+export function createTextChatHandler({
+  provider,
+  prepare,
+  parse,
+  fetchImpl = (...args) => fetch(...args),
+  resolveApiKey,
+  resolveHost = () => process.env.HOST,
+  resolveAllowedHosts = () => process.env.GEV_ALLOWED_HOSTS,
+  resolveRateLimit = () =>
+    process.env[`GEV_RATELIMIT_${provider.toUpperCase()}_CHAT_PER_MIN`] ?? '60',
+  timeoutMs = 60000,
+} = {}) {
+  let limiter;
+  return async (req, res) => {
+    if (req.aborted || res.destroyed || res.writableEnded) return;
+    res.setHeader('Cache-Control', 'no-store');
+    res.setHeader('Content-Type', 'application/json; charset=utf-8');
+    res.setHeader('X-Content-Type-Options', 'nosniff');
+    const reply = (status, value) => {
+      if (req.aborted || res.destroyed || res.writableEnded) return;
+      res.statusCode = status;
+      if (status >= 400) res.setHeader('Connection', 'close');
+      res.end(
+        JSON.stringify(typeof value === 'string' ? { error: value } : value),
+      );
+    };
+    if (req.method !== 'POST') {
+      res.setHeader('Allow', 'POST');
+      return reply(405, 'Method not allowed');
+    }
+    const lifetime = new AbortController();
+    const disconnect = () => {
+      if (!res.writableEnded) lifetime.abort();
+    };
+    req.once('aborted', disconnect);
+    res.once('close', disconnect);
+    const timer = setTimeout(() => lifetime.abort(), timeoutMs);
+    timer.unref?.();
+    let response;
+    let usage = null;
+    try {
+      if (
+        !isGeminiRequestOriginAllowed(req, resolveHost(), resolveAllowedHosts())
+      )
+        return reply(403, FAILURE);
+      const apiKey = String(resolveApiKey() || '').trim();
+      if (!apiKey) return reply(503, 'Add an API key in Provider Settings');
+      if (limiter === undefined)
+        limiter = makeCostRateLimiter(resolveRateLimit(), 60);
+      if (limiter && !limiter(clientKey(req))) {
+        res.setHeader('Retry-After', '60');
+        return reply(429, FAILURE);
+      }
+      if (
+        !/^application\/json(?:\s*;|$)/i.test(req.headers['content-type'] || '')
+      )
+        return reply(415, FAILURE);
+      if (Number(req.headers['content-length']) > MAX_BYTES)
+        return reply(413, FAILURE);
+      const raw = await readRequestBody(req, MAX_BYTES, lifetime.signal);
+      let input;
+      try {
+        input = JSON.parse(raw);
+      } catch {
+        return reply(400, FAILURE);
+      }
+      const prepared = prepare(input);
+      if (!prepared) return reply(400, FAILURE);
+      if (prepared.errorStatus) return reply(prepared.errorStatus, FAILURE);
+      const { endpoint, body, headers } = prepared;
+      lifetime.signal.throwIfAborted();
+      const acquisition = Promise.resolve(
+        fetchImpl(endpoint, {
+          method: 'POST',
+          redirect: 'error',
+          signal: lifetime.signal,
+          headers: { 'Content-Type': 'application/json', ...headers(apiKey) },
+          body: JSON.stringify(body),
+        }),
+      );
+      void acquisition.then(
+        (late) => {
+          response = late;
+          if (lifetime.signal.aborted) void late.body?.cancel().catch(() => {});
+        },
+        () => {},
+      );
+      response = await waitForSignal(acquisition, lifetime.signal);
+      lifetime.signal.throwIfAborted();
+      if (response.redirected || (response.url && response.url !== endpoint))
+        throw new Error('Unexpected destination');
+      if (!response.ok) {
+        void response.body?.cancel().catch(() => {});
+        return reply(
+          [429, 503].includes(response.status) ? response.status : 502,
+          FAILURE,
+        );
+      }
+      const data = await readResponseJsonCapped(
+        response,
+        MAX_BYTES,
+        lifetime.signal,
+      );
+      lifetime.signal.throwIfAborted();
+      usage = parseUsage(provider, data);
+      const result = parse(data);
+      if (!result || JSON.stringify(result).includes(apiKey))
+        return reply(502, { error: FAILURE, usage });
+      reply(200, { ...result, usage });
+    } catch (error) {
+      reply(
+        error?.code === 'BODY_TOO_LARGE'
+          ? 413
+          : lifetime.signal.aborted
+            ? 504
+            : 502,
+        { error: FAILURE, usage },
+      );
+    } finally {
+      if (lifetime.signal.aborted)
+        void response?.body?.cancel().catch(() => {});
+      clearTimeout(timer);
+      req.off('aborted', disconnect);
+      res.off('close', disconnect);
+    }
+  };
+}

server/providers/common/textInstructions.js

@@ -0,0 +1,31 @@
+import { realtimeDirectives } from '../openai/instructions.js';
+
+// Voice wording in the shared directives and its written form. A test checks
+// each phrase still occurs in the directives, so rewording one there fails
+// rather than leaking spoken wording into chat.
+export const TEXT_WORDING = Object.freeze([
+  ['without speaking first', 'without writing a preamble'],
+  ['before speaking', 'before confirming'],
+  ['may first say ONE sentence', 'may first write ONE sentence'],
+  ["speak the result's say line", "write the result's say line"],
+  ['on the voice card: speak a caveat', 'in the tool result: include a caveat'],
+  ['as you talk', 'as you write'],
+  ['you can read aloud', 'you can report'],
+]);
+
+// Shared-manual adaptation follows Darren Dignam's GEV COMMAND approach (#976).
+// Keep our Radio handoff and annotation outcome policies from the shared manual.
+export function textInstructions(annotationGuidance) {
+  return realtimeDirectives(annotationGuidance)
+    .map((line, index) => {
+      if (index === 0)
+        return "You are GEV Chat, a concise text assistant for a Cesium geospatial app called God's Eye View.";
+      if (index === 1)
+        return 'Have a natural written conversation. This is text-only chat; no microphone or generated audio is used.';
+      return TEXT_WORDING.reduce(
+        (text, [spoken, written]) => text.replace(spoken, written),
+        line,
+      );
+    })
+    .join('\n');
+}

server/providers/common/textTools.js

@@ -0,0 +1,54 @@
+import { validateValue } from '../../../src/tools/schema.js';
+
+export function invalidCalls(calls, tools) {
+  const schemas = new Map(tools.map((tool) => [tool.name, tool.parameters]));
+  return calls.flatMap((call, index) => {
+    const schema = schemas.get(call.name);
+    const errors = !schema
+      ? ['Unknown tool']
+      : call.parseError
+        ? ['Arguments must be valid JSON']
+        : validateValue(schema, call.args === undefined ? {} : call.args);
+    return errors.length
+      ? [{ index, error: errors.slice(0, 4).join('; ').slice(0, 1000) }]
+      : [];
+  });
+}
+
+// Missing usage stays unknown, never an invented zero-cost request.
+export function parseUsage(provider, data) {
+  const raw = provider === 'gemini' ? data?.usageMetadata : data?.usage;
+  const number = (value) =>
+    Number.isSafeInteger(value) && value >= 0 ? value : null;
+  const inputTokens = number(
+    provider === 'gemini' ? raw?.promptTokenCount : raw?.prompt_tokens,
+  );
+  const candidateTokens = number(
+    provider === 'gemini' ? raw?.candidatesTokenCount : raw?.completion_tokens,
+  );
+  const totalTokens = number(
+    provider === 'gemini' ? raw?.totalTokenCount : raw?.total_tokens,
+  );
+  if (inputTokens === null || candidateTokens === null || totalTokens === null)
+    return null;
+  const reasoningTokens =
+    number(
+      provider === 'gemini'
+        ? raw?.thoughtsTokenCount
+        : raw?.completion_tokens_details?.reasoning_tokens,
+    ) ?? 0;
+  const outputTokens =
+    candidateTokens + (provider === 'gemini' ? reasoningTokens : 0);
+  return {
+    inputTokens,
+    outputTokens,
+    totalTokens,
+    reasoningTokens,
+    cachedInputTokens:
+      number(
+        provider === 'gemini'
+          ? raw?.cachedContentTokenCount
+          : raw?.prompt_tokens_details?.cached_tokens,
+      ) ?? 0,
+  };
+}

server/providers/gemini.js

@@ -1,8 +1,13 @@
+import { createGeminiChatHandler } from './gemini/chat.js';
 import { createGeminiTokenHandler } from './gemini/realtime.js';
 
 /** Mint constrained Live API credentials; the permanent key stays on Node. */
 function geminiLiveProxy({ annotationGuidance, realtime = {} } = {}) {
   function install(middlewares) {
+    middlewares.use(
+      '/api/gemini/chat',
+      createGeminiChatHandler({ ...realtime, annotationGuidance }),
+    );
     middlewares.use(
       '/api/gemini/token',
       createGeminiTokenHandler({ ...realtime, annotationGuidance }),

server/providers/gemini/chat.js

@@ -0,0 +1,172 @@
+import { createTextChatHandler } from '../common/textChat.js';
+import { invalidCalls } from '../common/textTools.js';
+import { textInstructions } from '../common/textInstructions.js';
+import { GEV_REALTIME_TOOLS } from '../openai/tools.js';
+import { normalizeGeminiModel } from './config.js';
+
+const object = (value) =>
+  value && typeof value === 'object' && !Array.isArray(value);
+const keys = (value, allowed) =>
+  Object.keys(value).every((key) => allowed.includes(key));
+
+// Only text and function exchanges may cross this route, never media or config.
+export function validChatContents(contents, names) {
+  return (
+    Array.isArray(contents) &&
+    contents.length > 0 &&
+    contents.length <= 96 &&
+    contents.every(
+      (content) =>
+        object(content) &&
+        keys(content, ['role', 'parts']) &&
+        ['user', 'model'].includes(content.role) &&
+        Array.isArray(content.parts) &&
+        content.parts.length > 0 &&
+        content.parts.length <= 32 &&
+        content.parts.every((part) => {
+          if (
+            !object(part) ||
+            !keys(part, [
+              'text',
+              'functionCall',
+              'functionResponse',
+              'thoughtSignature',
+            ])
+          )
+            return false;
+          if (
+            part.thoughtSignature !== undefined &&
+            (typeof part.thoughtSignature !== 'string' ||
+              part.thoughtSignature.length > 32000 ||
+              content.role !== 'model')
+          )
+            return false;
+          if (Object.hasOwn(part, 'text'))
+            return (
+              keys(part, ['text', 'thoughtSignature']) &&
+              typeof part.text === 'string'
+            );
+          const call = part.functionCall ?? part.functionResponse;
+          if (
+            !object(call) ||
+            typeof call.name !== 'string' ||
+            call.name.length > 128 ||
+            (names && !names.has(call.name)) ||
+            (call.id !== undefined &&
+              (typeof call.id !== 'string' || call.id.length > 1024))
+          )
+            return false;
+          if (part.functionCall)
+            return (
+              content.role === 'model' &&
+              keys(part, ['functionCall', 'thoughtSignature']) &&
+              keys(call, ['name', 'args', 'id']) &&
+              (call.args === null || object(call.args ?? {}))
+            );
+          return (
+            content.role === 'user' &&
+            keys(part, ['functionResponse']) &&
+            keys(call, ['name', 'response', 'id']) &&
+            object(call.response)
+          );
+        }),
+    )
+  );
+}
+
+export function createGeminiChatHandler({
+  annotationGuidance,
+  tools = GEV_REALTIME_TOOLS,
+  resolveChatModel = () =>
+    process.env.GEMINI_CHAT_MODEL || 'gemini-3.5-flash-lite',
+  resolveApiKey = () => process.env.GEMINI_API_KEY,
+  ...options
+} = {}) {
+  return createTextChatHandler({
+    ...options,
+    provider: 'gemini',
+    resolveApiKey,
+    prepare(input) {
+      const model = normalizeGeminiModel(resolveChatModel());
+      if (!model || /live|audio|tts/i.test(model)) return { errorStatus: 503 };
+      if (
+        !object(input) ||
+        !keys(input, ['contents']) ||
+        !validChatContents(input.contents, null)
+      )
+        return null;
+      if (!pairedContents(input.contents)) return null;
+      return {
+        endpoint: `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`,
+        headers: (key) => ({ 'x-goog-api-key': key }),
+        body: {
+          contents: input.contents,
+          systemInstruction: {
+            parts: [{ text: textInstructions(annotationGuidance) }],
+          },
+          tools: [
+            {
+              functionDeclarations: tools.map(
+                ({ name, description, parameters }) => ({
+                  name,
+                  description,
+                  parametersJsonSchema: parameters,
+                }),
+              ),
+            },
+          ],
+          generationConfig: {
+            responseModalities: ['TEXT'],
+            maxOutputTokens: 4096,
+            thinkingConfig: { thinkingLevel: 'LOW' },
+          },
+        },
+      };
+    },
+    parse(data) {
+      const candidate = data?.candidates?.[0];
+      const content = candidate?.content;
+      if (
+        !content ||
+        !validChatContents([content], null) ||
+        content.role !== 'model'
+      )
+        return null;
+      const calls = content.parts
+        .filter((part) => part.functionCall)
+        .map((part) => part.functionCall);
+      return {
+        content,
+        finishReason: candidate.finishReason,
+        invalidCalls: invalidCalls(calls, tools),
+      };
+    },
+  });
+}
+
+// Never forward an orphan result or an unfinished batch in browser history.
+function pairedContents(contents) {
+  let pending = [];
+  for (const content of contents) {
+    const calls = content.parts
+      .filter((part) => part.functionCall)
+      .map((part) => part.functionCall);
+    const results = content.parts
+      .filter((part) => part.functionResponse)
+      .map((part) => part.functionResponse);
+    if (pending.length) {
+      if (
+        calls.length ||
+        results.length !== pending.length ||
+        results.some(
+          (result, i) =>
+            result.name !== pending[i].name || result.id !== pending[i].id,
+        )
+      )
+        return false;
+      pending = [];
+    } else if (results.length) return false;
+    if (calls.length) pending = calls;
+  }
+  return !pending.length;
+}

server/providers/openai.js

@@ -1,3 +1,4 @@
+import { createOpenAiChatHandler } from './openai/chat.js';
 import { defaultSourceRoot } from './common/source-root.js';
 import { handleHudSummary } from './openai/hud-summary.js';
 import { createDebugLogHandler } from './openai/debug-log.js';
@@ -22,6 +23,10 @@ function openAiRealtimeProxy({
 } = {}) {
   const oauthLogin = createCodexOAuthLogin(realtime);
   function install(middlewares) {
+    middlewares.use(
+      '/api/openai/chat',
+      createOpenAiChatHandler({ ...realtime, annotationGuidance }),
+    );
     // Cost-bearing and log endpoints refuse cross-site browser requests
     // (see server/providers/common/same-site.js and SECURITY.md).
     middlewares.use('/api/openai/hud-summary', sameSiteGated(handleHudSummary));

server/providers/openai/chat.js

@@ -0,0 +1,139 @@
+import { createTextChatHandler } from '../common/textChat.js';
+import { invalidCalls } from '../common/textTools.js';
+import { textInstructions } from '../common/textInstructions.js';
+import { GEV_REALTIME_TOOLS } from './tools.js';
+
+const object = (value) =>
+  value && typeof value === 'object' && !Array.isArray(value);
+const keys = (value, allowed) =>
+  Object.keys(value).every((key) => allowed.includes(key));
+const callShape = (call) =>
+  object(call) &&
+  keys(call, ['id', 'type', 'function']) &&
+  typeof call.id === 'string' &&
+  call.id.length > 0 &&
+  call.id.length <= 1024 &&
+  call.type === 'function' &&
+  object(call.function) &&
+  keys(call.function, ['name', 'arguments']) &&
+  typeof call.function.name === 'string' &&
+  call.function.name.length <= 128 &&
+  typeof call.function.arguments === 'string';
+function assistantShape(message) {
+  return (
+    object(message) &&
+    keys(message, ['role', 'content', 'tool_calls']) &&
+    message.role === 'assistant' &&
+    (message.content == null || typeof message.content === 'string') &&
+    (message.tool_calls === undefined ||
+      (Array.isArray(message.tool_calls) &&
+        message.tool_calls.length > 0 &&
+        message.tool_calls.length <= 32 &&
+        message.tool_calls.every(callShape) &&
+        new Set(message.tool_calls.map((call) => call.id)).size ===
+          message.tool_calls.length))
+  );
+}
+function validMessages(messages) {
+  if (!Array.isArray(messages) || !messages.length || messages.length > 96)
+    return false;
+  let pending = [];
+  for (const message of messages) {
+    if (!object(message)) return false;
+    if (pending.length) {
+      if (
+        !keys(message, ['role', 'content', 'tool_call_id']) ||
+        message.role !== 'tool' ||
+        typeof message.content !== 'string' ||
+        message.tool_call_id !== pending.shift()
+      )
+        return false;
+    } else if (message.role === 'user') {
+      if (
+        !keys(message, ['role', 'content']) ||
+        typeof message.content !== 'string'
+      )
+        return false;
+    } else if (assistantShape(message)) {
+      pending = (message.tool_calls || []).map((call) => call.id);
+    } else return false;
+  }
+  return !pending.length;
+}
+
+export function createOpenAiChatHandler({
+  annotationGuidance,
+  tools = GEV_REALTIME_TOOLS,
+  resolveApiKey = () => process.env.OPENAI_API_KEY,
+  resolveChatModel = () => process.env.OPENAI_CHAT_MODEL || 'gpt-4.1-mini',
+  ...options
+} = {}) {
+  return createTextChatHandler({
+    ...options,
+    provider: 'openai',
+    resolveApiKey,
+    prepare(input) {
+      const model = String(resolveChatModel()).trim();
+      if (
+        !/^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,119}$/.test(model) ||
+        /realtime|audio|tts/i.test(model)
+      )
+        return { errorStatus: 503 };
+      if (
+        !object(input) ||
+        !keys(input, ['messages']) ||
+        !validMessages(input.messages)
+      )
+        return null;
+      return {
+        endpoint: 'https://api.openai.com/v1/chat/completions',
+        headers: (key) => ({ Authorization: `Bearer ${key}` }),
+        body: {
+          model,
+          n: 1,
+          stream: false,
+          store: false,
+          max_completion_tokens: 4096,
+          messages: [
+            { role: 'system', content: textInstructions(annotationGuidance) },
+            ...input.messages,
+          ],
+          tools: tools.map(({ name, description, parameters }) => ({
+            type: 'function',
+            function: { name, description, parameters, strict: false },
+          })),
+        },
+      };
+    },
+    parse(data) {
+      const choice = data?.choices?.[0];
+      const raw = choice?.message;
+      if (!raw) return null;
+      const content = {
+        role: raw.role,
+        content: raw.content ?? null,
+        ...(raw.tool_calls?.length ? { tool_calls: raw.tool_calls } : {}),
+      };
+      if (
+        !assistantShape(content) ||
+        (!content.content && !content.tool_calls?.length)
+      )
+        return null;
+      const calls = (content.tool_calls || []).map((call) => {
+        try {
+          return {
+            name: call.function.name,
+            args: JSON.parse(call.function.arguments),
+          };
+        } catch {
+          return { name: call.function.name, parseError: true };
+        }
+      });
+      return {
+        content,
+        finishReason: choice.finish_reason,
+        invalidCalls: invalidCalls(calls, tools),
+      };
+    },
+  });
+}

server/providers/openai/instructions.js

@@ -1,9 +1,7 @@
 const DEFAULT_ANNOTATION_GUIDANCE =
   'Put related places in one annotate_map call. Set flyTo true only when the user is not already looking at the place (off-screen marks auto-frame, so when unsure leave it false). Do not say you are drawing or annotating — talk about the places while the marks appear. Marks ACCUMULATE AND PERSIST across flights and topics so the user can build up the map: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
 
-function realtimeInstructions(
-  annotationGuidance = DEFAULT_ANNOTATION_GUIDANCE,
-) {
+function realtimeDirectives(annotationGuidance = DEFAULT_ANNOTATION_GUIDANCE) {
   return [
     "You are GEV Voice Control, a concise voice controller for a Cesium geospatial app called God's Eye View.",
     'Have a natural spoken conversation with the user while the mic session is active.',
@@ -64,7 +62,11 @@ function realtimeInstructions(
     annotationGuidance,
     'annotate_map results: work any say line in naturally (places not found, a straight-line fallback with no travel time, or a full map — then ask before clearing). outlinePending means the boundary is still being traced: never claim it is drawn. A later map_annotation_outline system item reports each outcome; use it to quietly confirm or correct yourself.',
     'ROUTES vs DISTANCES: "walking/driving route from A to B" → type=route with the ordered points and the mode; the app draws the street path and reports distance and time you can read aloud. "How far is X from Y" or "X is next to Y" → type=arrow (straight-line distance), never route.',
-  ].join('\n');
+  ];
 }
 
-export { realtimeInstructions };
+function realtimeInstructions(annotationGuidance) {
+  return realtimeDirectives(annotationGuidance).join('\n');
+}
+
+export { realtimeInstructions, realtimeDirectives };