Merge branch 'main' into feat/cctv-vegvesen-kartverket-heights

server/mcp/panelKey.js

@@ -0,0 +1,117 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { randomBytes, randomUUID } from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+
+/** Where the install's panel key lives: beside the other local state, which
+ * git ignores, at the checkout root rather than the client's working
+ * directory (an MCP client may start the server anywhere). */
+export const DEFAULT_PANEL_KEY_FILE = fileURLToPath(
+  new URL('../../.gev-cache/mcp-panel-key', import.meta.url),
+);
+
+const KEY_PATTERN = /^[A-Za-z0-9_-]{43}$/;
+const newKey = () => randomBytes(32).toString('base64url');
+const notShared = (error) =>
+  `panel key not shared (${error.code || error.message}); using a key of this process`;
+
+function readKey(file) {
+  const text = fs.readFileSync(file, 'utf8').trim();
+  return KEY_PATTERN.test(text) ? text : null;
+}
+
+/** Serialize malformed-file repair; ordinary first creation still uses link(). */
+function repairKey(file, tmp, key) {
+  const lock = `${file}.repair-lock`;
+  const deadline = Date.now() + 2000;
+  const pause = new Int32Array(new SharedArrayBuffer(4));
+  let fd;
+  for (;;) {
+    try {
+      fd = fs.openSync(lock, 'wx', 0o600);
+      break;
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      // Another repair may already have published a usable key.
+      const winner = readKey(file);
+      if (winner) return winner;
+      if (Date.now() >= deadline) {
+        throw Object.assign(new Error('panel key repair is busy'), {
+          code: 'ETIMEDOUT',
+        });
+      }
+      Atomics.wait(pause, 0, 0, 10);
+    }
+  }
+  try {
+    // A repair completed between our earlier read and acquiring the lock.
+    const winner = readKey(file);
+    if (winner) return winner;
+    fs.renameSync(tmp, file);
+    return key;
+  } finally {
+    fs.closeSync(fd);
+    fs.rmSync(lock, { force: true });
+  }
+}
+
+/**
+ * The panel key every stdio server of this install shares.
+ *
+ * A client can start more than one server process for one connection: Claude
+ * Desktop was reported to start two processes; serving the panel's page from
+ * one and sending its `panel_request` calls to the other reproduces #927.
+ * Per-process keys then mismatch, and the panel refuses the request.
+ * The first process to run creates the key, without replacing one
+ * another process created at the same moment; the rest read it. When the file
+ * cannot be read or written, or repair stays busy for two seconds, the process
+ * falls back to its own key, as every process did before.
+ *
+ * As before, the key keeps `panel_request` from clients that only list it; it
+ * is not access control.
+ *
+ * @param {{ file?: string, log?: (line: string) => void }} [options]
+ * @returns {string}
+ */
+export function sharedPanelKey({
+  file = DEFAULT_PANEL_KEY_FILE,
+  log = () => {},
+} = {}) {
+  try {
+    const existing = readKey(file);
+    if (existing) return existing;
+    // Present but not a key: replace it below.
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      log(notShared(error));
+      return newKey();
+    }
+  }
+  const key = newKey();
+  let tmp = null;
+  try {
+    fs.mkdirSync(path.dirname(file), { recursive: true });
+    tmp = path.join(
+      path.dirname(file),
+      `.${path.basename(file)}.${randomUUID().slice(0, 8)}.tmp`,
+    );
+    fs.writeFileSync(tmp, `${key}\n`, { flag: 'wx', mode: 0o600 });
+    try {
+      // link() publishes the whole file at once and fails if another process
+      // got there first, so a reader never sees a half-written key and two
+      // processes starting together cannot end up with different keys.
+      fs.linkSync(tmp, file);
+      return key;
+    } catch (error) {
+      if (error.code !== 'EEXIST') throw error;
+      const winner = readKey(file);
+      if (winner) return winner;
+      return repairKey(file, tmp, key);
+    }
+  } catch (error) {
+    log(notShared(error));
+    return key;
+  } finally {
+    if (tmp) fs.rmSync(tmp, { force: true });
+  }
+}

server/mcp/server.js

@@ -27,14 +27,16 @@ const INSTRUCTIONS =
 export function createLocalMcpServer({
   apiBase = DEFAULT_API_BASE,
   fetchImpl,
+  panelKey = randomBytes(32).toString('base64url'),
 } = {}) {
   const { version } = JSON.parse(
     readFileSync(new URL('../../package.json', import.meta.url), 'utf8'),
   );
-  // Each server makes a new key for its panel. The panel's page carries it
-  // and panel_request requires it; any client may read the page, so it keeps
-  // the tool from clients that only list it, and is not access control.
-  const panelKey = randomBytes(32).toString('base64url');
+  // Unless given one, each server makes a new key for its panel. The panel's
+  // page carries it and panel_request requires it; any client may read the
+  // page, so it keeps the tool from clients that only list it, and is not
+  // access control. The stdio entry passes the install's shared key
+  // (panelKey.js), so separate processes of one client agree.
   return createMcpServer({
     catalog: catalogForSurface(
       composeCatalog({

server/mcp/stdio.js

@@ -11,6 +11,7 @@
 import { createInterface } from 'node:readline';
 import { fileURLToPath } from 'node:url';
 import { parseErrorResponse } from '../../src/tools/mcp/index.js';
+import { sharedPanelKey } from './panelKey.js';
 import { createLocalMcpServer } from './server.js';
 import { DEFAULT_API_BASE } from './services.js';
 
@@ -97,7 +98,10 @@ if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
     console.error(
       `God's Eye View MCP server reading ${options.apiBase || DEFAULT_API_BASE}`,
     );
-    await serveStdio(createLocalMcpServer(options), {
+    // A client may start several of these for one connection; they share
+    // the install's panel key so the panel works whichever one it reaches.
+    const panelKey = sharedPanelKey({ log: (line) => console.error(line) });
+    await serveStdio(createLocalMcpServer({ ...options, panelKey }), {
       input: process.stdin,
       output: process.stdout,
       log: (line) => console.error(`<- ${line}`),

server/providers/openai.js

@@ -1,20 +1,26 @@
 import { defaultSourceRoot } from './common/source-root.js';
 import { handleHudSummary } from './openai/hud-summary.js';
 import { createDebugLogHandler } from './openai/debug-log.js';
-import { createRealtimeTokenHandler } from './openai/realtime.js';
+import {
+  createRealtimeOAuthLoginHandler,
+  createRealtimeOAuthStatusHandler,
+  createRealtimeTokenHandler,
+} from './openai/realtime.js';
 import { sameSiteGated } from './common/same-site.js';
+import { createCodexOAuthLogin } from './openai/codex-auth.js';
 
 /**
  * Vite plugin: OpenAI Realtime ephemeral client secret.
  *
- * Keeps OPENAI_API_KEY server-side while the browser connects to the
- * Realtime API over WebRTC with a short-lived secret.
+ * Keeps the selected cloud credential server-side while the browser connects
+ * to the Realtime API over WebRTC with a short-lived secret.
  */
 function openAiRealtimeProxy({
   sourceRoot = defaultSourceRoot,
   annotationGuidance,
   realtime = {},
 } = {}) {
+  const oauthLogin = createCodexOAuthLogin(realtime);
   function install(middlewares) {
     // Cost-bearing and log endpoints refuse cross-site browser requests
     // (see server/providers/common/same-site.js and SECURITY.md).
@@ -31,16 +37,29 @@ function openAiRealtimeProxy({
         createRealtimeTokenHandler({ ...realtime, annotationGuidance }),
       ),
     );
+
+    middlewares.use(
+      '/api/realtime/oauth-status',
+      sameSiteGated(createRealtimeOAuthStatusHandler({ oauthLogin })),
+    );
+
+    middlewares.use(
+      '/api/realtime/oauth-login',
+      sameSiteGated(createRealtimeOAuthLoginHandler({ oauthLogin })),
+    );
   }
 
   return {
     name: 'openai-realtime-proxy',
     configureServer(server) {
       install(server.middlewares);
+      server.httpServer?.once('close', oauthLogin.dispose);
     },
     configurePreviewServer(server) {
       install(server.middlewares);
+      server.httpServer?.once('close', oauthLogin.dispose);
     },
+    closeBundle: oauthLogin.dispose,
   };
 }
 

server/providers/openai/codex-auth.js

@@ -0,0 +1,360 @@
+import { spawn } from 'node:child_process';
+import {
+  accessSync,
+  constants as fsConstants,
+  readFileSync,
+  statSync,
+} from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+export function codexAuthPath({
+  environment = process.env,
+  home = os.homedir(),
+} = {}) {
+  const explicit = String(environment.CODEX_AUTH_JSON || '').trim();
+  if (explicit) return path.resolve(explicit);
+  const codexHome = String(environment.CODEX_HOME || '').trim();
+  return path.join(
+    codexHome ? path.resolve(codexHome) : path.join(home, '.codex'),
+    'auth.json',
+  );
+}
+
+/** An optional operator lock that refuses the metered API-key voice lane. */
+export function preferCodexOAuth(environment = process.env) {
+  if (!Object.hasOwn(environment, 'GEV_PREFER_CODEX_OAUTH')) return false;
+  const value = String(environment.GEV_PREFER_CODEX_OAUTH || '')
+    .trim()
+    .toLowerCase();
+  if (['1', 'true', 'yes', 'on'].includes(value)) return true;
+  if (['0', 'false', 'no', 'off'].includes(value)) return false;
+  throw new Error(
+    'GEV_PREFER_CODEX_OAUTH must be true or false. Refusing to choose a voice auth mode.',
+  );
+}
+
+function codexOAuthError(message, code = 'CODEX_OAUTH_UNAVAILABLE') {
+  const error = new Error(message);
+  error.code = code;
+  return error;
+}
+
+export function readCodexOAuthAccessToken({
+  authPath = codexAuthPath(),
+  readFile = readFileSync,
+  statFile = statSync,
+  nowMs = Date.now(),
+} = {}) {
+  const unavailable =
+    'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.';
+  let parsed;
+  try {
+    parsed = JSON.parse(readFile(authPath, 'utf8'));
+  } catch {
+    throw codexOAuthError(unavailable);
+  }
+  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+    throw codexOAuthError(unavailable);
+  }
+  const mode = String(parsed.auth_mode || '')
+    .trim()
+    .toLowerCase();
+  const chatGptMode = mode
+    ? mode === 'chatgpt' || mode === 'chatgptauthtokens'
+    : typeof parsed.OPENAI_API_KEY !== 'string';
+  const token = parsed.tokens?.access_token;
+  if (!chatGptMode || typeof token !== 'string' || !token.trim()) {
+    throw codexOAuthError(unavailable);
+  }
+
+  // This only checks token shape and expiry. OpenAI verifies the signature when
+  // minting the short-lived Realtime credential; we never modify auth.json.
+  const parts = token.split('.');
+  if (
+    parts.length !== 3 ||
+    parts.some((part) => !/^[A-Za-z0-9_-]+$/.test(part))
+  ) {
+    throw codexOAuthError(unavailable);
+  }
+  let payload;
+  try {
+    payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
+  } catch {
+    throw codexOAuthError(unavailable);
+  }
+  if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
+    throw codexOAuthError(unavailable);
+  }
+  let expiresAtMs;
+  if (Object.hasOwn(payload, 'exp')) {
+    if (typeof payload.exp !== 'number' || !Number.isFinite(payload.exp)) {
+      throw codexOAuthError(unavailable);
+    }
+    expiresAtMs = payload.exp * 1000;
+  } else {
+    try {
+      expiresAtMs = statFile(authPath).mtimeMs + 60 * 60 * 1000;
+    } catch {
+      throw codexOAuthError(unavailable);
+    }
+  }
+  if (!Number.isFinite(expiresAtMs) || expiresAtMs <= nowMs + 60_000) {
+    throw codexOAuthError(
+      'ChatGPT sign-in has expired. Re-authenticate in Provider Settings or run codex login and try again.',
+      'CODEX_OAUTH_REAUTH_REQUIRED',
+    );
+  }
+  return token;
+}
+
+export function resolveCodexExecutable({
+  environment = process.env,
+  home = os.homedir(),
+  access = accessSync,
+} = {}) {
+  const explicit = String(environment.CODEX_BIN || '').trim();
+  if (explicit) return explicit;
+
+  for (const candidate of [
+    path.join(home, '.local', 'bin', 'codex'),
+    path.join(
+      home,
+      '.codex',
+      'packages',
+      'standalone',
+      'current',
+      'bin',
+      'codex',
+    ),
+  ]) {
+    try {
+      access(candidate, fsConstants.X_OK);
+      return candidate;
+    } catch {
+      // Fall through to the next known location, then PATH.
+    }
+  }
+  return 'codex';
+}
+
+// What `codex login` needs to find its home, open a browser and reach OpenAI.
+// The server's own environment also holds every provider secret, so the
+// child receives only these names.
+const CODEX_LOGIN_ENV =
+  /^(?:PATH|PATHEXT|HOME|USER|USERNAME|LOGNAME|SHELL|LANG|LANGUAGE|LC_[A-Z_]+|TERM|TZ|TMPDIR|TEMP|TMP|XDG_[A-Z_]+|DISPLAY|WAYLAND_DISPLAY|DBUS_SESSION_BUS_ADDRESS|BROWSER|WSL_[A-Z_]+|WSLENV|(?:HTTPS?|ALL|NO)_PROXY|SSL_CERT_(?:FILE|DIR)|NODE_EXTRA_CA_CERTS|SYSTEMROOT|WINDIR|COMSPEC|USERPROFILE|APPDATA|LOCALAPPDATA|PROGRAMDATA|HOMEDRIVE|HOMEPATH|CODEX_[A-Z_]+)$/i;
+
+/** The environment `codex login` runs with: the allowlist, then `CODEX_HOME`. */
+export function codexLoginEnvironment(environment, codexHome) {
+  const env = {};
+  for (const [name, value] of Object.entries(environment || {}))
+    if (CODEX_LOGIN_ENV.test(name) && value !== undefined) env[name] = value;
+  env.CODEX_HOME = codexHome;
+  return env;
+}
+
+export function startCodexChatGptLogin({
+  spawnImpl = spawn,
+  environment = process.env,
+  home = os.homedir(),
+  executable = resolveCodexExecutable({ environment, home }),
+} = {}) {
+  // Codex owns its credential storage. An explicit read path must agree with
+  // the CLI's write path; never silently sign in to another credential store.
+  const configuredHome = String(environment.CODEX_HOME || '').trim();
+  const loginHome = configuredHome
+    ? path.resolve(configuredHome)
+    : path.join(home, '.codex');
+  if (
+    codexAuthPath({ environment, home }) !== path.join(loginHome, 'auth.json')
+  ) {
+    return Promise.reject(
+      codexOAuthError(
+        'CODEX_AUTH_JSON does not match the Codex login storage path. Set CODEX_HOME to the matching directory or sign in to Codex separately.',
+        'CODEX_OAUTH_STORAGE_MISMATCH',
+      ),
+    );
+  }
+
+  return new Promise((resolve, reject) => {
+    let child;
+    let spawned = false;
+    let finish;
+    const completion = new Promise((done) => {
+      finish = done;
+    });
+    const fail = () => {
+      reject(
+        codexOAuthError(
+          'Could not start ChatGPT sign-in. Check that Codex is installed, or set CODEX_BIN to its executable.',
+          'CODEX_OAUTH_LOGIN_FAILED',
+        ),
+      );
+      finish({ exitCode: null });
+    };
+    try {
+      child = spawnImpl(executable, ['login'], {
+        cwd: home,
+        detached: true,
+        stdio: 'ignore',
+        env: codexLoginEnvironment(environment, loginHome),
+      });
+    } catch {
+      fail();
+      return;
+    }
+
+    child.once('error', () => {
+      if (!spawned) fail();
+      else finish({ exitCode: null });
+    });
+    child.once('exit', (exitCode) => finish({ exitCode }));
+    child.once('spawn', () => {
+      spawned = true;
+      child.unref?.();
+      resolve({
+        completion,
+        cancel() {
+          try {
+            child.kill?.('SIGTERM');
+          } catch {
+            /* Already stopped. */
+          }
+        },
+      });
+    });
+  });
+}
+
+/** One local login owner shared by the launch and status endpoints. */
+export function createCodexOAuthLogin({
+  resolveOAuthAccessToken = () => readCodexOAuthAccessToken(),
+  startOAuthLogin = () => startCodexChatGptLogin(),
+  loginTimeoutMs = 120_000,
+  setTimer = setTimeout,
+  clearTimer = clearTimeout,
+} = {}) {
+  let active = null;
+  let failure = null;
+  let disposed = false;
+
+  function credentials() {
+    try {
+      if (resolveOAuthAccessToken()) return { available: true };
+    } catch (error) {
+      const code = error?.code || 'CODEX_OAUTH_UNAVAILABLE';
+      return {
+        available: false,
+        error: error?.message || 'ChatGPT sign-in is unavailable',
+        code,
+        reauthRequired: code === 'CODEX_OAUTH_REAUTH_REQUIRED',
+      };
+    }
+    return {
+      available: false,
+      error: 'ChatGPT sign-in is unavailable',
+      code: 'CODEX_OAUTH_UNAVAILABLE',
+      reauthRequired: false,
+    };
+  }
+
+  function status() {
+    const current = credentials();
+    if (current.available) return current;
+    if (failure) return { available: false, ...failure, loginFailed: true };
+    return active ? { ...current, pending: true } : current;
+  }
+
+  function finish(attempt, error) {
+    if (active !== attempt) return;
+    clearTimer(attempt.timer);
+    active = null;
+    failure = error;
+  }
+
+  async function start() {
+    if (disposed)
+      throw codexOAuthError(
+        'ChatGPT sign-in is unavailable because the server has stopped.',
+        'CODEX_OAUTH_LOGIN_FAILED',
+      );
+    if (credentials().available)
+      return { available: true, started: false, pending: false };
+    if (active) return { available: false, started: false, pending: true };
+
+    // Reserve the attempt before awaiting spawn so concurrent POSTs share it.
+    const attempt = { process: null, timer: null };
+    active = attempt;
+    failure = null;
+    attempt.timer = setTimer(() => {
+      finish(attempt, {
+        code: 'CODEX_OAUTH_LOGIN_TIMEOUT',
+        error:
+          'ChatGPT sign-in timed out. Click USE CHATGPT OAUTH to try again.',
+      });
+      attempt.process?.cancel();
+    }, loginTimeoutMs);
+    attempt.timer?.unref?.();
+
+    try {
+      const login = await startOAuthLogin();
+      if (!login?.completion || typeof login.cancel !== 'function')
+        throw new Error('Invalid login process');
+      attempt.process = login;
+      if (active !== attempt) {
+        login.cancel();
+        return { ...status(), started: false, pending: false };
+      }
+      Promise.resolve(login.completion).then(
+        (result) => {
+          if (active !== attempt) return;
+          if (result?.exitCode !== 0) {
+            finish(attempt, {
+              code: 'CODEX_OAUTH_LOGIN_FAILED',
+              error:
+                'ChatGPT sign-in did not complete. Try again in Provider Settings, or run codex login separately.',
+            });
+          } else if (!credentials().available) {
+            finish(attempt, {
+              code: 'CODEX_OAUTH_STORAGE_UNAVAILABLE',
+              error:
+                'Codex sign-in finished, but its configured auth.json has no usable ChatGPT token. Check CODEX_HOME and Codex credential storage. GEV can read file storage only.',
+            });
+          } else finish(attempt, null);
+        },
+        () =>
+          finish(attempt, {
+            code: 'CODEX_OAUTH_LOGIN_FAILED',
+            error:
+              'ChatGPT sign-in did not complete. Try again in Provider Settings.',
+          }),
+      );
+      return { available: false, started: true, pending: true };
+    } catch (error) {
+      const code =
+        error?.code === 'CODEX_OAUTH_STORAGE_MISMATCH'
+          ? error.code
+          : 'CODEX_OAUTH_LOGIN_FAILED';
+      finish(attempt, {
+        code,
+        error:
+          code === 'CODEX_OAUTH_STORAGE_MISMATCH'
+            ? error.message
+            : 'Could not start ChatGPT sign-in. Check that Codex is installed, or set CODEX_BIN to its executable.',
+      });
+      return { ...status(), started: false, pending: false };
+    }
+  }
+
+  return {
+    status,
+    start,
+    dispose() {
+      disposed = true;
+      if (!active) return;
+      const attempt = active;
+      finish(attempt, null);
+      attempt.process?.cancel();
+    },
+  };
+}

server/providers/openai/constants.js

@@ -16,6 +16,9 @@ const OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT = 0.5;
 
 const OPENAI_HUD_SUMMARY_MODEL_DEFAULT = 'gpt-5-nano';
 
+/** Input transcription for voice captions; OPENAI_REALTIME_TRANSCRIBE_MODEL=off disables it. */
+const OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT = 'gpt-4o-mini-transcribe';
+
 export {
   OPENAI_HUD_SUMMARY_MODEL_DEFAULT,
   OPENAI_REALTIME_MODEL_MINI_DEFAULT,
@@ -24,4 +27,5 @@ export {
   OPENAI_REALTIME_REASONING_DEFAULT,
   OPENAI_REALTIME_CONTEXT_TOKENS_DEFAULT,
   OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT,
+  OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT,
 };

server/providers/openai/debug-log.js

@@ -23,7 +23,59 @@ const REALTIME_DEBUG_LOG_MAX_FILE_BYTES = 32 * 1024 * 1024;
  */
 const REALTIME_DEBUG_LOG_MAX_PER_MIN = 120;
 
-function createDebugLogHandler({ sourceRoot = defaultSourceRoot } = {}) {
+/**
+ * Fields that carry what people said or heard: transcripts, typed and spoken
+ * text, streamed deltas, tool arguments/results and the span's quoted
+ * replies. They are dropped from the persisted log unless content logging is explicitly enabled
+ * (GEV_VOICE_LOG_CONTENT=1), so timing and tool diagnostics never retain
+ * speech by default.
+ */
+const VOICE_TEXT_FIELDS = new Set([
+  'transcript',
+  'text',
+  'delta',
+  'answer_text',
+  'preamble_text',
+]);
+const VOICE_CONTENT_MAX_DEPTH = 12;
+const VOICE_CONTENT_MAX_DEPTH_MARKER = '[omitted: max depth]';
+
+/** Replace spoken or typed content and complete tool argument/result bodies. */
+function omitVoiceContent(value, depth = 0) {
+  if (depth > VOICE_CONTENT_MAX_DEPTH) return VOICE_CONTENT_MAX_DEPTH_MARKER;
+  if (value === null || typeof value !== 'object') return value;
+  if (Array.isArray(value))
+    return value.map((item) => omitVoiceContent(item, depth + 1));
+  const output = {};
+  for (const [key, item] of Object.entries(value)) {
+    // Tool bodies have open-ended schemas: omit the whole body rather than
+    // chasing query, label, say, display, or future tool-specific fields.
+    // Protocol response.output arrays still carry useful event metadata; only
+    // string output is a serialized function result and must be opaque.
+    if (
+      key === 'arguments' ||
+      key === 'result' ||
+      (key === 'output' && typeof item === 'string')
+    ) {
+      output[key] =
+        typeof item === 'string'
+          ? `[omitted ${item.length} chars]`
+          : '[omitted]';
+      continue;
+    }
+    if (VOICE_TEXT_FIELDS.has(key) && typeof item === 'string') {
+      output[key] = `[omitted ${item.length} chars]`;
+      continue;
+    }
+    output[key] = omitVoiceContent(item, depth + 1);
+  }
+  return output;
+}
+
+function createDebugLogHandler({
+  sourceRoot = defaultSourceRoot,
+  includeContent = process.env.GEV_VOICE_LOG_CONTENT === '1',
+} = {}) {
   const logDir = path.join(sourceRoot, '.gev-logs');
   const logFile = path.join(logDir, 'realtime-conversations.jsonl');
   const allow = makeRateLimiter({
@@ -79,7 +131,8 @@ function createDebugLogHandler({ sourceRoot = defaultSourceRoot } = {}) {
 
     try {
       const body = await readRequestBody(req, REALTIME_DEBUG_LOG_MAX_BYTES);
-      const record = JSON.parse(body || '{}');
+      const parsed = JSON.parse(body || '{}');
+      const record = includeContent ? parsed : omitVoiceContent(parsed);
       // The server's own timestamp comes last, so a record cannot supply one.
       await append(
         `${JSON.stringify({
@@ -104,4 +157,4 @@ function createDebugLogHandler({ sourceRoot = defaultSourceRoot } = {}) {
   };
 }
 
-export { createDebugLogHandler };
+export { createDebugLogHandler, omitVoiceContent };

server/providers/openai/instructions.js

@@ -1,5 +1,5 @@
 const DEFAULT_ANNOTATION_GUIDANCE =
-  'Use a single annotate_map call with several annotations when you are describing multiple related places at once. Set flyTo true only when the user is not already looking at the place; if every mark in a call lands off-screen the app auto-frames them, so when unsure leave flyTo false. Do NOT say out loud that you are drawing, highlighting, or annotating — just speak naturally about the places while the marks appear. ANNOTATIONS ACCUMULATE AND PERSIST — keep adding marks as you explore; you can fly around, change topic, and jump between far-apart places and the marks STAY, so the user can build up the map and show people things. Do NOT clear on your own initiative: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
+  'Put related places in one annotate_map call. Set flyTo true only when the user is not already looking at the place (off-screen marks auto-frame, so when unsure leave it false). Do not say you are drawing or annotating — talk about the places while the marks appear. Marks ACCUMULATE AND PERSIST across flights and topics so the user can build up the map: never pass clearPrevious, and call clear_annotations ONLY when the user EXPLICITLY asks to clear or reset the map.';
 
 function realtimeInstructions(
   annotationGuidance = DEFAULT_ANNOTATION_GUIDANCE,
@@ -13,27 +13,24 @@ function realtimeInstructions(
     'For requests to open, show, reveal, or focus a menu/panel, call set_panel_open or show_data_layers_menu. "Open Context" means only set_panel_open{panelId:"global-context-panel",open:true}; it does not activate a Context sub-mode. "Open Contacts" means set_context_mode{mode:"contacts"}; that action expands the parent Context panel before activating Contacts.',
     'For requests like "show me the datacenter layers", open the data layers menu and focus the matching layer row; do not enable the layer unless the user asks to turn it on.',
     'For questions like "what am I looking at?", "what is in view?", "what is this?", "that selected thing", nearby datacenter, dam, cable, ship, or current view contents, call get_entity_context first, then answer from the returned scene/entity context.',
-    'For "what is this aircraft?" answers, read the callsign, operator, registration, type, and route only from get_entity_context selected.properties. Treat route, routeOrigin, and routeDestination as the only authoritative route fields. Every aircraft identity answer MUST explicitly cover operator, type, and route. When a route is present, repeat its endpoint codes exactly; do not expand airport codes into city names. For a missing field say exactly "Operator details are unavailable", "Aircraft type is unavailable", or "Route details are unavailable" as applicable. Never silently omit missing enrichment or infer it from the callsign.',
-    'While a camera motion or route flight is active, a bare "stop" means move_camera{motion:stop} — NOT control_scene and NOT stop_tracking (those need explicit words like "stop the scene" / "stop tracking"). If move_camera stop returns stopped:false and an entity is being tracked, call stop_tracking next — the user means "stop whatever is moving". Flying somewhere while tracking automatically stops the tracking (the result says so): mention it briefly.',
+    'For "what is this aircraft?", answer with get_entity_context\'s identityLine: callsign, operator, type and route from returned fields only, airport codes verbatim. Other questions about the selection answer from its properties. Never infer them from the callsign.',
+    'While a camera motion or route flight is active, a bare "stop" means move_camera{motion:stop} — NOT control_scene and NOT stop_tracking (those need explicit words like "stop the scene" / "stop tracking"). If move_camera stop returns stopped:false and an entity is being tracked, call stop_tracking next — the user means "stop whatever is moving". Flying somewhere while tracking stops the tracking.',
     'For camera-motion requests — "orbit around this", "pan left", "tilt up", "stop moving" — call move_camera. For "fly the route" over a drawn route, call fly_route. Confirm with the RESULTING state ("Orbiting slowly", "Flying the route").',
     'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.',
-    'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data: report the returned feed source briefly, never call it an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
-    'For ANALYTICAL questions about layer data — how many / which / fastest / highest / biggest / nearest flights, ships, fires, earthquakes, satellites, datacenters, or dams ("how many flights over Texas", "biggest fire near LA", "which ships are headed to Oakland", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Narrate the count plus two or three notable examples by name, and reflect the result\'s coverage note honestly: the answer covers data loaded by enabled layers, not the whole world. If the needed layer is disabled, say so and offer to enable it. For follow-ups about the same set ("which of THOSE is closest?"), call analyst_query with followUp=true and only the new filter/sort.',
+    'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.',
+    'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.',
     'WORLD QUERIES — the query tools (get_weather, get_wind, search_places, places_nearby, plan_route, get_cyclones, get_fire_perimeters, get_bike_share, get_transit_vehicles, get_traffic_flow, situation_brief, military_awareness and the other get_* and find_* tools you have) look up data anywhere in the world without changing the map and without any layer being on. Use them when the user asks about a place they are not looking at or data whose layer is off ("what is the weather in Tokyo", "how do I drive from Austin to Dallas", "any wildfires in California"). For aircraft, ships, earthquakes, active fires, datacenters, dams and satellites overhead, use analyst_query over the layers instead, turning the layer on first when the user wants it. Use analyst_query and get_entity_context for what the app has loaded or what is on screen. Answer from the result summary and data, mention its source when it has one, and never claim the map changed. If the user also wants to SEE it, use the matching app action (fly_to_location, set_layer_visibility) as well.',
-    'FEED STATE CONTRACT — analyst_query and get_current_view_state carry feedProvenance from the same layerFeedState the Data Layers chips use (nominal/loading/degraded/stale/fallback/unavailable, or off). NEVER narrate a count as live or current when feedProvenance.overall is not nominal. Say the feedState in the same breath as the count ("12 flights in view, STALE"). If unavailable, do not invent a count; say the feed is unavailable. Fallback is valid data — name the fallback source, never call it an enable failure. Confirm only what the tool result returned.',
-    'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, "near / nearby / how many aircraft" means the Contacts window: answer from contactsWindow in the tool result — those are the exact numbers on the user\'s panel. set_context_mode, analyst_query, and get_current_view_state carry it after Contacts settles. For "Open Contacts and tell me how many aircraft are within 250 km", call set_context_mode{mode:"contacts"} first and answer from contactsWindow.aircraft; do not answer from a pre-Contacts analyst query. analyst_query\'s own count measures currently-loaded records and is usually lower; never give it as the window count. CENTER PRECEDENCE for a nearby/how-many ask, in order: an explicit place in the question ("over Texas", "near Austin") always wins and ignores Contacts state; else the CONTACTS SUBJECT when Contacts is active and has one — a selected datacenter, dam, fire, or cable does NOT silently become the center; else an entity the user explicitly names ("around this datacenter"); else the current view, said aloud ("nothing is selected, so this is the current view"). With Contacts active but NO subject yet, use the view and say so; never read an empty panel. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "about 30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two different numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge ("a few", "less than a dozen"): if a tool returns 46, say 46. (5) When it matters, add once: counts cover loaded data, and the flights layer loads where you look.',
+    'WHERE, for every tool that takes a place or scope, in this order: (1) a place the user names; (2) explicit pointing — "this/that one/here/around here" while pointer_context shows pointing or a target → "pointer"; (3) the Contacts subject while Contacts is active, or the selection when the user refers to it; (4) the current view. A selected datacenter, dam, fire or cable never silently becomes the centre.',
+    'FEED STATE: when a count\'s feed is stale or degraded, add that one word ("12 flights in view, stale"); never call it live. If unavailable, give no number: say the feed is unavailable. Fallback feeds are valid data; name them only if asked.',
+    'COUNTING CONTRACT — what "near" means. (1) While Contacts is ACTIVE, a bare "near / nearby / how many aircraft" means the settled Contacts subject window: its count is contactsWindow.aircraft. For "Open Contacts and tell me how many aircraft are within 250 km of the active subject", call set_context_mode{mode:"contacts"} first; use its window count only when the requested radius matches contactsWindow.radiusKm. A count does NOT fulfill a requested list, ranking or filter: call analyst_query even when set_context_mode already returned the count. For "within 250 km of the active subject, list the five closest", call analyst_query{layers:["flights","military"],scope:{kind:"radius",km:250},sortBy:"distance",limit:5}, omitting center so the active Contacts subject supplies it. Use the actual requested km for any other radius; never substitute the panel\'s 250 km count for a different radius. Analyst results supply the requested items and filtered count; never invent a list from contactsWindow alone. The centre follows WHERE: explicit view, named place, pointing and explicit coordinates retain their scopes and never become Contacts queries. Never invent center coordinates such as 0,0. With Contacts active but no subject yet, use the view and name that scope. (2) With Contacts OFF, "nearby" means in view; "near <place>" means a radius around that place. (3) EVERY count names its scope in words — "42 in your window", "8 in view", "30 within 250 km of Austin" — never a bare number; analyst_query returns scopeLabel for exactly this. Two numbers with named scopes are not a contradiction; say both if asked. (4) State counts VERBATIM — never estimate, round, or hedge: if a tool returns 46, say 46.',
     'While Cockpit is active, navigate with control_cockpit (next/previous, optionally targetLayer or aircraftClass). track_entity and fly_to_location are REFUSED by design while Cockpit owns the camera — that refusal is correct, not an error to retry. To go somewhere else, exit Cockpit first. control_cockpit enter establishes Contacts itself, so do not call set_context_mode before or after it.',
     'When the target layer is unknown, OMIT layerId in track_entity so it searches all enabled layers. Passing the wrong layerId ("flights" for a military contact) returns "Nothing matched" even though the contact is loaded.',
-    'If get_entity_context has no selected object or overlay entities, use its basemap context: Google Photorealistic 3D Tiles/Cesium source, center target coordinates, reverse-geocoded place, camera altitude, active style, and enabled layers. Do not say there is nothing unless the basemap target is also unavailable.',
-    'If basemap context includes knownLandmarks, prefer the nearest known landmark by name for "what am I looking at" answers. For example, if knownLandmarks includes Eiffel Tower, say Eiffel Tower.',
-    'At local zoom, use basemap nearbyPlaces, place.labels, viewportPlaces.visibleLabels, and viewportPlaces.streetLabels to identify the building, premises, roads, and named places visible around the screen target.',
-    'If basemap context includes viewportPlaces, prefer dominantCountry, dominantRegion, and dominantLocality over raw coordinates.',
-    'When basemap context includes viewportSamples or an inferred country, trust that over a single reverse-geocoded address. If most samples indicate Iran, say Iran, not the United States.',
-    'When a viewport screenshot is attached after get_entity_context, read clearly legible street, building, and place labels from it and combine them with structured label context. Respect scene viewScale: at global/continental/regional scale, avoid naming a precise street/city from one center pixel.',
+    'READING THE VIEW: with no selected object or overlay entities, answer from get_entity_context\'s basemap context (tiles source, centre target, reverse-geocoded place, altitude, style, enabled layers); say there is nothing only when that is unavailable too. Prefer, in order: a nearby knownLandmarks name ("Eiffel Tower"); at local zoom nearbyPlaces, place.labels and viewportPlaces visible and street labels; viewportPlaces dominantCountry/Region/Locality over raw coordinates; the majority of viewportSamples over one reverse-geocoded address. Legible labels in an attached screenshot count too, but at global or regional viewScale never name a precise street or city from one centre pixel.',
     'Do not mention disabled layers or stale selections.',
-    'When a request requires a tool call, do not speak in the same response as the tool call. Call the tool first.',
-    'When a single user request contains MULTIPLE changes (e.g. "switch to operator layout, use balanced detection at density 50, and switch to Bing aerial"), call ALL the corresponding tools — multiple tool calls in sequence — before speaking. Never confirm a partial subset. If a later tool fails, say which parts succeeded and which failed.',
-    'After receiving tool output, speak exactly one short confirmation. Do not repeat the confirmation.',
+    'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
+    'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
+    'When one request contains MULTIPLE changes, call ALL the corresponding tools before the confirmation. Never confirm a partial subset; if one fails, say which.',
+    'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
     'For "show/open/turn on" layer requests, enable the matching layer. For "hide/close/turn off", disable it.',
     // INSTRUCTION-ONLY mapping for the two globe-scale named views.
     //
@@ -50,27 +47,23 @@ function realtimeInstructions(
     'NAMED VIEWS are shorthand for tool calls you already have — there is no "mode" tool for them. Treat ONLY these as the shorthand: "infrastructure mode" / "the infrastructure view" / "show me global infrastructure" means three set_layer_visibility calls (local-datacenters, local-dams, telegeography-submarine-cables) plus zoom_to_globe; "environmental mode" / "earth watch" / "active events", said as the name of a view, means set_layer_visibility for local-firms and earthquakes plus zoom_to_globe. Anything vaguer is NOT this shorthand — an open-ended question about the world or the news is an ordinary question: answer it, or use analyst_query over the layers already on. Never switch a whole view on to answer a question nobody asked to see. When you do run one, make every call before speaking, then give one confirmation naming the resulting state; if the fires layer comes back unavailable because no FIRMS key is configured, say so plainly — the earthquakes still loaded. "Live contacts" and "space missions" are NOT this pattern: they stay set_context_mode{mode:"contacts"} and set_context_mode{mode:"space-missions"}.',
     'For visual filter requests, call set_visual_style with one of the allowed style IDs.',
     'SHIPS vs STYLES: "vessels" is easily heard as "visuals". "Show me vessels", "turn on vessels", "the vessels", "visuals" or "vestibules" said WITHOUT a style name means the ships layer: set_layer_visibility{layerId:"ais-live-vessels"}, exactly as "ships" or "boats" does. Call set_visual_style only when the user names a style (retro/CRT, surveillance/night vision, thermal, anime, noir, snow, normal) or says filter, effect or look. Never turn on a different data layer (such as fires) for a ships request.',
-    'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any mention of "satellite" or "satellites" ALWAYS means the satellites DATA LAYER via set_layer_visibility, never a basemap. "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
+    'Disambiguation table — basemap vs layer vs style: basemap switching requires an explicit stack name — "Bing aerial" means set_map_stack bing-aerial, "aerial with labels" means bing-labels, "OSM"/"road map" means osm, "Esri"/"Esri imagery" means esri-imagery, "Google 3D"/"photorealistic" means photoreal. Any other mention of "satellite" or "satellites" means the satellites DATA LAYER via set_layer_visibility, never a basemap — except "recent/latest satellite imagery" (the recent-imagery layer) and "weather satellite"/clouds (weather-satellite). "surveillance"/"night vision"/"thermal" are visual STYLES via set_visual_style.',
     'HUD requests ("hud on/off", "switch to operator/minimal/tactical/cyber layout") use set_hud. Detection requests ("detection on", "dense mode", "balanced mode", "sparse mode", "set density to 25", "use weighted allocation") use set_detection. Density snaps to 0/25/50/75/100 and derives Sparse/Balanced/Dense; panoptic is a legacy alias for Dense.',
     'Cyber sonar requests use set_cyber_sonar: enabled, rings, rangePct, intensityPct (Power), opacityPct, sectorDeg. For settings questions read get_current_view_state.controls.hud.sonar. Sonar is not detection. Do not silently switch themes, reveal the HUD or enable sonar when changing a slider. If the user explicitly requests Cyber plus sonar settings, call set_hud first, wait for success, then set_cyber_sonar. Confirm returned values; mapSweepActive and contactSweepActive distinguish saved settings from active map effects, including in Cockpit or reduced motion. Do not claim visible contacts solely because enabled is true.',
     'Bloom/sharpen requests use set_post_processing. Scene requests ("play orbital watch", "stop the scene", "what scenes are there") use control_scene. CCTV camera requests ("next camera", "nearest camera", "select the Congress camera", "show coverage") use control_cctv — the CCTV layer must be enabled first.',
     'Radio playback requests use control_radio. "Turn on/start the radio" means action=play; action=enable only reveals Radio markers and must be reserved for explicit "show/enable the Radio layer/markers" requests. After a prepared playback result, briefly confirm any other completed actions and say "Turning on the radio"—never claim it is already playing. The client keeps Radio muted until playback is verified, then closes voice before restoring Radio volume. Examples: "play news near Austin" → select category=news locationId=austin; "play US news" → select category=news country=US; "Radio volume 30" → volume; pause/resume/stop/next/previous use the matching action. Radio selection never moves the camera.',
     '"Track/follow <something specific>" (a callsign, ship name, satellite name) uses track_entity. "Take me to the biggest fire" uses track_entity with query "biggest fire" (the fires layer must be enabled). Bare "orbit" means camera orbit of the current landmark. "Stop following/tracking" uses stop_tracking.',
     '"Show me which planes are overhead"/"frame the ships"/"show me the satellites above" use frame_overhead with the matching target.',
-    "After frame_overhead, speak ONLY from the tool result's count field — e.g. 'Framed fourteen aircraft, labels on'; never reassess or second-guess the count aloud.",
     'Confirmations echo the RESULTING state, never the request: "HUD operator layout", "Density twenty-five percent", "Bing aerial imagery", "Tracking UAL428", "Framed fourteen aircraft". On ok=false, state the failure plainly: "Nothing matched UAL999", "No ships within 120 kilometers". Never claim an action without ok=true in the tool result.',
     'For destination requests such as "take me to Italy", "go to NYC", or "show me the Eiffel Tower", call fly_to_location. Prefer known city IDs when available; otherwise pass the plain place query.',
     'Navigation-only requests ("take me to X", "go to X", "fly to X") are NOT descriptions: call fly_to_location alone and do NOT also call annotate_map, unless the user explicitly asks to mark the place or you go on to explain specific places there. Never drop a point pin on a region-scale natural feature (a mountain range, desert, sea, or forest) — a single point in the middle of the Rockies is meaningless. If the user explicitly asks to mark such a region, prefer type=area.',
     'For country and city destinations, omit rangeM so GEV frames the whole country or city in view. For landmarks and buildings, omit rangeM so GEV chooses a close landmark view.',
     'Only supply rangeM when the user asks for a particular numeric height, distance, closer view, or wider view.',
     'For relative requests such as "zoom out a little", "pull back", "zoom in more", or "get closer", always call adjust_camera_zoom. But "globe view", "whole earth", "the whole planet", or "zoom all the way out" is an ABSOLUTE framing: call zoom_to_globe once instead — repeated adjust_camera_zoom calls can never reach the globe. Never claim the camera moved without the tool returning ok=true.',
-    'Keep spoken confirmations short, e.g. "Opening datacenters" or "Flying to London".',
-    'WHITEBOARD THE WORLD: whenever you describe or explain a specific place, building, campus, district, boundary, or a spatial relationship between places, call annotate_map to mark it visually as you talk — like sketching on the map. To call out a specific building, campus, compound, park, or district, use type=area (it traces and encloses the real footprint — a building gets a glowing volume, a district gets a draped outline). Use type=highlight only for a transient pulse on a precise spot that has no meaningful footprint, and type=pin to drop a labeled marker. Examples: "what is the Palace of Fine Arts?" → an AREA on it; "the old military base next to it" → an AREA on the Presidio; "ILM is right here" → a pin; "it sits next to the Marina" → an arrow from one to the other. Prefer place NAMES so the app resolves real positions and outlines; never invent coordinates or pixel locations.',
-    'On every annotation, also set entityKind to what the thing IS when you know it: building (one structure), compound (campus/grounds/mall/park), district (neighborhood/area of a city), street (a named road), or point_feature (a monument, statue, memorial, plaque, fountain, or other small point landmark). entityKind is a FACT about the target, independent of the mark type you chose — monuments and statues are point_feature even when you use type=area; the app then anchors them as precise points instead of guessing at a footprint.',
+    'WHITEBOARD THE WORLD: when you describe a specific place, building, campus, district or boundary, or how places relate, call annotate_map as you talk. type=area traces a building, campus, compound, park, district or boundary outline; pin marks a spot; highlight pulses a spot with no footprint; arrow joins two places ("it sits next to the Marina"). Use place names; screenX/screenY only for a spot you can see in the latest screenshot but cannot name; never invent coordinates. Set entityKind whenever you know what the target is.',
     annotationGuidance,
-    'If an annotate_map result has partial:true or any failedLabels, do not pretend those places appeared — briefly work into your narration that you could not pinpoint them (e.g. "I couldn\'t place X"). If a route comes back as a direct line (no street route was found), describe it as a straight-line distance, not a walking/driving time. If an annotate_map result has capped:true, the map is full — ASK the user whether to clear before drawing more; do not clear unprompted. outlinePending:true is NOT a failure, but it is also NOT an outline: the anchor mark is placed and the boundary is still being traced in the background. Narrate it in progress — e.g. "tracing the boundary now" — and NEVER state the outline is already drawn or visible; it may yet come back as just a point. A later system item of type map_annotation_outline reports the final outcome per mark (status resolved or failed, with its label): use it to quietly confirm, or to correct yourself if you implied a boundary that stayed a point — an honest miss beats a misleading guess.',
-    'PREFER NAMES. Only when you cannot name or geocode a place but you can clearly SEE the exact spot in the most recent viewport screenshot, fall back to screenX/screenY (normalized 0..1 from that image) to point at it; the app converts the pixel to a real world point. Never use screenX/screenY for something you could name.',
-    'PATHS vs DISTANCES: for "walking/driving route from A to B" (or through several stops), use type=route with the ordered points and the matching mode (walking/driving/cycling) — the app draws the real street-following path on the map and reports distance and travel time, which you can read aloud. For "how far is X from Y", "is it nearby", or "X is next to Y", use type=arrow between the two — it draws a floating connector and shows the straight-line distance. Do NOT use route for a simple distance/proximity question.',
+    'annotate_map results: work any say line in naturally (places not found, a straight-line fallback with no travel time, or a full map — then ask before clearing). outlinePending means the boundary is still being traced: never claim it is drawn. A later map_annotation_outline system item reports each outcome; use it to quietly confirm or correct yourself.',
+    'ROUTES vs DISTANCES: "walking/driving route from A to B" → type=route with the ordered points and the mode; the app draws the street path and reports distance and time you can read aloud. "How far is X from Y" or "X is next to Y" → type=arrow (straight-line distance), never route.',
   ].join('\n');
 }
 

server/providers/openai/realtime.js

@@ -10,15 +10,130 @@ import {
   OPENAI_REALTIME_REASONING_DEFAULT,
   OPENAI_REALTIME_CONTEXT_TOKENS_DEFAULT,
   OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT,
+  OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT,
 } from './constants.js';
 import { realtimeInstructions } from './instructions.js';
 import { GEV_REALTIME_TOOLS } from './tools.js';
+import {
+  preferCodexOAuth,
+  readCodexOAuthAccessToken,
+  createCodexOAuthLogin,
+} from './codex-auth.js';
+
+const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']);
+
+function isLoopbackRequest(req) {
+  return LOOPBACK_ADDRESSES.has(String(req.socket?.remoteAddress || ''));
+}
+
+function isSameOriginRequest(req) {
+  const origin = String(req.headers?.origin || '').trim();
+  if (!origin) return true;
+  const host = String(req.headers?.host || '').trim();
+  if (!host) return false;
+  const protocol = req.socket?.encrypted ? 'https:' : 'http:';
+  try {
+    return new URL(origin).origin === new URL(`${protocol}//${host}`).origin;
+  } catch {
+    return false;
+  }
+}
+
+function requestQuery(req) {
+  try {
+    return new URL(req.url || '', 'http://localhost').searchParams;
+  } catch {
+    return new URLSearchParams();
+  }
+}
+
+function resolveRealtimeAuthMode(query) {
+  const requested = String(query.get('auth') || 'api-key').toLowerCase();
+  if (requested === 'api-key' || requested === 'oauth') return requested;
+  return null;
+}
+
+function createRealtimeOAuthStatusHandler({
+  oauthLogin = createCodexOAuthLogin(),
+} = {}) {
+  return (req, res) => {
+    res.setHeader('Cache-Control', 'no-store');
+    res.setHeader('Content-Type', 'application/json');
+    if (req.method !== 'GET') {
+      res.statusCode = 405;
+      res.end(JSON.stringify({ error: 'Method not allowed' }));
+      return;
+    }
+    if (!isLoopbackRequest(req)) {
+      res.statusCode = 403;
+      res.end(
+        JSON.stringify({
+          available: false,
+          error: 'ChatGPT OAuth is available only from this machine',
+        }),
+      );
+      return;
+    }
+    res.statusCode = 200;
+    res.end(JSON.stringify(oauthLogin.status()));
+  };
+}
+
+function createRealtimeOAuthLoginHandler({
+  oauthLogin = createCodexOAuthLogin(),
+} = {}) {
+  return async (req, res) => {
+    res.setHeader('Cache-Control', 'no-store');
+    res.setHeader('Content-Type', 'application/json');
+    if (req.method !== 'POST') {
+      res.statusCode = 405;
+      res.end(JSON.stringify({ error: 'Method not allowed' }));
+      return;
+    }
+    if (!isLoopbackRequest(req)) {
+      res.statusCode = 403;
+      res.end(
+        JSON.stringify({
+          available: false,
+          error: 'ChatGPT OAuth sign-in is available only from this machine',
+        }),
+      );
+      return;
+    }
+    if (!isSameOriginRequest(req)) {
+      res.statusCode = 403;
+      res.end(
+        JSON.stringify({
+          available: false,
+          error: 'Cross-origin ChatGPT OAuth sign-in is not allowed',
+        }),
+      );
+      return;
+    }
+
+    try {
+      const result = await oauthLogin.start();
+      res.statusCode = result.available ? 200 : result.loginFailed ? 503 : 202;
+      res.end(JSON.stringify(result));
+    } catch {
+      res.statusCode = 503;
+      res.end(
+        JSON.stringify({
+          available: false,
+          error: 'Could not start ChatGPT sign-in',
+        }),
+      );
+    }
+  };
+}
 
 function createRealtimeTokenHandler({
   annotationGuidance,
   endpoint = 'https://api.openai.com/v1/realtime/client_secrets',
   fetchImpl = (...args) => fetch(...args),
   resolveApiKey = () => process.env.OPENAI_API_KEY,
+  resolveOAuthAccessToken = () => readCodexOAuthAccessToken(),
+  environment = process.env,
   models = {},
   tools = GEV_REALTIME_TOOLS,
 } = {}) {
@@ -31,15 +146,84 @@ function createRealtimeTokenHandler({
       return;
     }
 
+    const query = requestQuery(req);
+    const authMode = resolveRealtimeAuthMode(query);
+    if (!authMode) {
+      res.statusCode = 400;
+      res.setHeader('Content-Type', 'application/json');
+      res.end(JSON.stringify({ error: 'Unsupported cloud voice auth mode' }));
+      return;
+    }
+
+    let oauthPinned;
+    try {
+      oauthPinned = preferCodexOAuth(environment);
+    } catch (error) {
+      res.statusCode = 503;
+      res.setHeader('Content-Type', 'application/json');
+      res.end(JSON.stringify({ error: error.message }));
+      return;
+    }
+    if (oauthPinned && authMode !== 'oauth') {
+      res.statusCode = 503;
+      res.setHeader('Content-Type', 'application/json');
+      res.end(
+        JSON.stringify({
+          error:
+            'ChatGPT OAuth is required by GEV_PREFER_CODEX_OAUTH. Select USE CHATGPT OAUTH in Provider Settings.',
+          code: 'CODEX_OAUTH_REQUIRED',
+        }),
+      );
+      return;
+    }
+
     // Per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). On by default; 0 disables.
     if (!enforceRateLimit(openAiRateLimiter(), req, res)) return;
 
-    const apiKey = resolveApiKey();
-    if (!apiKey) {
-      res.statusCode = 503;
-      res.setHeader('Content-Type', 'application/json');
-      res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' }));
-      return;
+    let bearer;
+    if (authMode === 'oauth') {
+      if (!isLoopbackRequest(req)) {
+        res.statusCode = 403;
+        res.setHeader('Content-Type', 'application/json');
+        res.end(
+          JSON.stringify({
+            error: 'ChatGPT OAuth voice is available only from this machine',
+          }),
+        );
+        return;
+      }
+      try {
+        bearer = resolveOAuthAccessToken();
+      } catch (error) {
+        res.statusCode = 503;
+        res.setHeader('Content-Type', 'application/json');
+        res.end(
+          JSON.stringify({
+            error: error?.message || 'ChatGPT sign-in is unavailable',
+            code: error?.code || 'CODEX_OAUTH_UNAVAILABLE',
+          }),
+        );
+        return;
+      }
+      if (!bearer) {
+        res.statusCode = 503;
+        res.setHeader('Content-Type', 'application/json');
+        res.end(
+          JSON.stringify({
+            error: 'ChatGPT sign-in is unavailable',
+            code: 'CODEX_OAUTH_UNAVAILABLE',
+          }),
+        );
+        return;
+      }
+    } else {
+      bearer = resolveApiKey();
+      if (!bearer) {
+        res.statusCode = 503;
+        res.setHeader('Content-Type', 'application/json');
+        res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' }));
+        return;
+      }
     }
 
     // Voice model tier, requested by the client as ?tier=standard|mini.
@@ -48,15 +232,7 @@ function createRealtimeTokenHandler({
     // bad querystring degrades to a normal session rather than a dead mic.
     // The env overrides stay authoritative per tier (see .env.example) —
     // a wrong upstream model id is then a config fix, not a code change.
-    const requestedTier = (() => {
-      try {
-        return new URL(req.url || '', 'http://localhost').searchParams.get(
-          'tier',
-        );
-      } catch {
-        return null;
-      }
-    })();
+    const requestedTier = query.get('tier');
     const tier = resolveVoiceModel(requestedTier).tier;
     const model =
       tier === 'mini'
@@ -89,6 +265,14 @@ function createRealtimeTokenHandler({
           OPENAI_REALTIME_CONTEXT_RETENTION_DEFAULT,
       ),
     );
+    // User captions for the voice card. Billed separately from the session.
+    const transcribeModel =
+      process.env.OPENAI_REALTIME_TRANSCRIBE_MODEL ||
+      OPENAI_REALTIME_TRANSCRIBE_MODEL_DEFAULT;
+    const transcription =
+      transcribeModel.toLowerCase() === 'off'
+        ? {}
+        : { transcription: { model: transcribeModel } };
     const sessionConfig = {
       session: {
         type: 'realtime',
@@ -104,6 +288,7 @@ function createRealtimeTokenHandler({
         audio: {
           input: {
             noise_reduction: { type: 'near_field' },
+            ...transcription,
             turn_detection: {
               type: 'semantic_vad',
               eagerness: 'low',
@@ -125,7 +310,7 @@ function createRealtimeTokenHandler({
         redirect: 'error',
         signal: AbortSignal.timeout(30_000),
         headers: {
-          Authorization: `Bearer ${apiKey}`,
+          Authorization: `Bearer ${bearer}`,
           'Content-Type': 'application/json',
           'OpenAI-Safety-Identifier': 'gev-local-dev',
         },
@@ -139,6 +324,15 @@ function createRealtimeTokenHandler({
       // case where a bogus ?tier= was silently downgraded to standard.
       res.setHeader('X-GEV-Voice-Tier', tier);
       res.setHeader('X-GEV-Voice-Model', model);
+      res.setHeader(
+        'X-GEV-Voice-Auth',
+        authMode === 'oauth' ? 'codex-oauth' : 'env',
+      );
+      // Captions are billed separately; the client meters them with this id.
+      res.setHeader(
+        'X-GEV-Voice-Transcribe-Model',
+        transcription.transcription ? transcribeModel : 'off',
+      );
       if (requestedTier && !isKnownVoiceTier(requestedTier)) {
         res.setHeader('X-GEV-Voice-Tier-Fallback', '1');
       }
@@ -168,4 +362,8 @@ function createRealtimeTokenHandler({
   };
 }
 
-export { createRealtimeTokenHandler };
+export {
+  createRealtimeOAuthLoginHandler,
+  createRealtimeOAuthStatusHandler,
+  createRealtimeTokenHandler,
+};

server/providers/openai/toolDescriptions.js

@@ -1,3 +1,8 @@
+import {
+  voiceLayerAliasHint,
+  voiceQueryFieldHint,
+} from '../../../src/voice/layerManifest.js';
+
 export const ACTION_DESCRIPTIONS = {
   fly_to_location: {
     description:
@@ -12,7 +17,11 @@ export const ACTION_DESCRIPTIONS = {
         },
         query: {
           description:
-            'Plain place search query, e.g. "London", "Eiffel Tower", or "Dubai Marina".',
+            'Plain place search query, e.g. "London", "Eiffel Tower", or "Dubai Marina". "pointer" = the spot under the cursor.',
+          $position: 1,
+        },
+        referent: {
+          description: 'Numbered item n from the last result list.',
           $position: 1,
         },
         viewMode: {
@@ -76,13 +85,13 @@ export const ACTION_DESCRIPTIONS = {
     $position: 1,
   },
   set_layer_visibility: {
-    description: "Enable or disable one registered God's Eye View data layer.",
+    description:
+      "Enable or disable one God's Eye View data layer. Every shipped layer is in the enum; never tell the user a listed layer does not exist.",
     $position: 1,
     parameters: {
       properties: {
         layerId: {
-          description:
-            'Common-name mapping for the non-obvious ids: space mission(s) → rocket-launches; fires/wildfires/active fires → local-firms (NASA FIRMS); ships/vessels/boats → ais-live-vessels; undersea/submarine cables → telegeography-submarine-cables; datacenters → local-datacenters; dams → local-dams; bikes/bike share → bikeshare; street traffic/congestion → traffic; traffic cameras → cctv; internet radio/stations → radio; ALPR/license plate readers/Flock cameras → alpr-cameras; local ADS-B/my receiver/my antenna (aircraft heard by a local RTL-SDR receiver) → local-adsb.',
+          description: `Spoken names: ${voiceLayerAliasHint()}.`,
           $position: 1,
         },
       },
@@ -154,9 +163,13 @@ export const ACTION_DESCRIPTIONS = {
       properties: {
         scope: {
           description:
-            'Use auto by default. selected returns the clicked/selected entity; in_view returns visible entities near the screen center.',
+            'Use auto by default. selected returns the clicked/selected entity; in_view returns visible entities near the screen center; pointer returns what the cursor is on.',
           $position: 2,
         },
+        referent: {
+          description: 'Numbered item n from the last result list.',
+          $position: 1,
+        },
         layerId: {
           description: 'Optional layer filter for visible entity context.',
           $position: 2,
@@ -349,12 +362,17 @@ export const ACTION_DESCRIPTIONS = {
       properties: {
         query: {
           description:
-            'Callsign, ship name, satellite name, ICAO hex, MMSI, or NORAD id.',
+            'Callsign, ship name, satellite name, ICAO hex, MMSI, or NORAD id. "pointer" = the contact under the cursor.',
+          $position: 1,
+        },
+        referent: {
+          description:
+            'Numbered item n from the last result list (-1 = last); query may then be its label.',
           $position: 1,
         },
         layerId: {
           description:
-            'Optional layer hint: flights | military | ais-live-vessels | satellites.',
+            'Optional: flights, military, ais-live-vessels or satellites. Other layers are not trackable; for fires use query "biggest fire".',
           $position: 1,
         },
       },
@@ -381,7 +399,7 @@ export const ACTION_DESCRIPTIONS = {
   },
   annotate_map: {
     description:
-      'Draw annotations on the 3D map to visually point out what you are talking about — like sketching on a whiteboard over the world. Use this whenever you mention a specific place, building, campus, boundary, district, or a relationship between two places, so the user can SEE what you mean. Give place NAMES (preferred) or explicit lat/lng; the app resolves them to real-world positions and real building/area outlines — never guess pixel positions. Call this as you begin describing something, and you may mark several places in one call.',
+      'Mark places on the 3D map as you talk about them (see WHITEBOARD): pins, highlights, footprints and boundaries, arrows, routes and labels. Give place names (or explicit lat/lng); the app resolves real positions and outlines. Several places may go in one call.',
     $position: 1,
     parameters: {
       properties: {
@@ -398,7 +416,7 @@ export const ACTION_DESCRIPTIONS = {
               },
               target: {
                 description:
-                  'Place name to resolve, e.g. "Palace of Fine Arts, San Francisco", "the Pentagon", "Presidio of San Francisco". Preferred over coordinates. For a specific monument/statue/feature that sits within a larger landmark, use its OWN name + city ("Tejano Monument, Austin", "Texas African American History Memorial, Austin") — do NOT phrase it as "X at the Texas State Capitol", which makes the geocoder collapse several of them onto the same centroid so they stack on one spot.',
+                  'Place name to resolve ("Palace of Fine Arts, San Francisco"), or "pointer" for the spot under the cursor. A monument inside a larger landmark gets its own name + city ("Tejano Monument, Austin"), never "X at the Texas State Capitol" (that stacks them on one centroid).',
                 $position: 2,
               },
               points: {
@@ -450,27 +468,24 @@ export const ACTION_DESCRIPTIONS = {
               },
               entityKind: {
                 description:
-                  'What KIND of thing the target IS — a fact, not a style choice: building = one structure; compound = campus/grounds/mall/park; district = neighborhood or area of a city; street = a named road/corridor; point_feature = monument/statue/memorial/plaque/fountain or other small point landmark. Set it whenever you know it — it routes the resolver to the right footprint source (point_feature anchors monuments as precise points instead of adopting a nearby building outline).',
+                  'What the target IS (a fact, not a style): building = one structure; compound = campus/grounds/mall/park; district = neighborhood; street = a named road; point_feature = monument/statue/memorial/fountain, anchored as a precise point. Set it whenever you know it.',
                 $position: 2,
               },
               screenX: {
                 description:
-                  'Fallback only: when you cannot name/geocode the place but can SEE it in the latest viewport screenshot, the normalized horizontal position (0=left, 1=right) of the spot. The app converts it back to a real world point under that pixel.',
+                  'Fallback only, for a spot visible in the latest screenshot that you cannot name: normalized x (0=left, 1=right).',
                 $position: 3,
               },
               screenY: {
-                description:
-                  'Fallback only: normalized vertical position (0=top, 1=bottom) of the spot in the latest viewport screenshot.',
+                description: 'Fallback only: normalized y (0=top, 1=bottom).',
                 $position: 3,
               },
               toScreenX: {
-                description:
-                  'For type=arrow: normalized x of the arrow destination from the screenshot (pixel fallback).',
+                description: 'For type=arrow: destination x (pixel fallback).',
                 $position: 3,
               },
               toScreenY: {
-                description:
-                  'For type=arrow: normalized y of the arrow destination from the screenshot (pixel fallback).',
+                description: 'For type=arrow: destination y (pixel fallback).',
                 $position: 3,
               },
             },
@@ -528,18 +543,18 @@ export const ACTION_DESCRIPTIONS = {
   },
   analyst_query: {
     description:
-      'Answer questions ABOUT the data currently loaded on the map — counts, lists, superlatives, and attribute filters over live layers (flights, military, ships, fires, earthquakes, satellites, datacenters, dams). Examples: "how many flights over Texas", "biggest fire near LA", "which ships are headed to Oakland", "anything above 40,000 feet", "fastest thing in view". For satellites and infrastructure, counts and ranks cover only bounded examined loaded records; omitted records can change nearest/count. Queries ONLY client-side data from ENABLED layers — if the needed layer is off, say so and offer to enable it. For a follow-up about the previous answer\'s set ("which of those is closest?"), set followUp=true and send only the new filters/sort.',
+      'Counts, lists, superlatives and attribute filters over records loaded by ENABLED layers — "how many flights over Texas", "biggest fire near LA", "ships headed to Oakland", "next rocket launch". A requested list or ranking requires this tool even after set_context_mode returned a Contacts count; that count alone has no requested items. complete:false means the count is a floor ("at least") and rankings cover the records examined; partial names unanswered layers. Items carry lat/lon for a follow-up fly_to_location. ok:false codes: LAYER_OFF (offer to enable), NOT_READY (loading), FOLLOW_UP_MISMATCH (ask again without followUp), UNKNOWN_FIELD/UNKNOWN_SCOPE/BAD_VALUE (retry with the allowed values).',
     $position: 1,
     parameters: {
       properties: {
         layers: {
           description:
-            'Layers to query. fires/wildfires → local-firms; ships/vessels → ais-live-vessels.',
+            'Layers to query. fires → local-firms; ships → ais-live-vessels; buses/trains → transit; hurricanes → weather-cyclones; bases → military-installations; my receiver → local-adsb. Units differ per layer, so rank speed within one layer.',
           $position: 2,
         },
         scope: {
           description:
-            'Spatial scope. Default: view (near the camera). Use kind=region for "over Texas"-style asks; kind=anywhere for global questions.',
+            'Always set it (see WHERE). Explicit "in view" uses view; "nearby" uses view when Contacts is off. Around the active Contacts subject uses radius with the requested km and center omitted, never view: the five closest within 250 km use scope:{kind:"radius",km:250}, sortBy:"distance", limit:5. For another requested radius use its actual km, not 250. pointer = "around here/this" while pointing (radius at the cursor); region = "over Texas"; radius with center {lat, lon} = "near <place>" or explicit coordinates; anywhere = no place ("biggest anywhere", "today"). Preserve explicit view, named place, pointer and explicit center even while Contacts is active; never invent coordinates. There is no drawn-area scope yet: say so rather than answering for the view.',
           $position: 2,
           properties: {
             name: {
@@ -548,23 +563,29 @@ export const ACTION_DESCRIPTIONS = {
               $position: 1,
             },
             km: {
-              description: 'For kind=radius.',
+              description:
+                'For kind=radius or pointer: the actual requested kilometers. An active-subject 250 km request uses 250; another requested radius keeps that number.',
+              $position: 1,
+            },
+            center: {
+              description:
+                'Omit for a radius around the active Contacts subject. Supply only an explicitly requested coordinate or a resolved named-place coordinate; never invent 0,0 or replace an explicit center with the subject.',
               $position: 1,
             },
           },
         },
         filters: {
-          description:
-            'Attribute predicates, ANDed. ALTITUDE IS METERS (40,000 ft = 12192). Fields: altitudeM, speedMps, military, onGround, aircraftClass, callsign, operator, routeOrigin, routeDestination, originCountry (flights); speedKts, shipType, destination (ships); frp, confidence (fires); magnitude, depthKm, place (earthquakes).',
+          description: `ANDed; values keep their type (numbers, true/false). Altitude is meters (40,000 ft = 12192). Time fields take ISO; ageHours = hours ago. Every layer has id, lat, lon. Fields — ${voiceQueryFieldHint()}.`,
           $position: 1,
         },
         sortBy: {
-          description: 'Field to rank by, or "distance" for nearest-first.',
+          description:
+            'Field to rank by, or "distance" (nearest first unless sortDir=desc).',
           $position: 1,
         },
         followUp: {
           description:
-            'true = re-query the PREVIOUS result set instead of fresh data.',
+            'true = re-filter the last answer (its layers) instead of fresh data.',
           $position: 1,
         },
       },
@@ -572,7 +593,7 @@ export const ACTION_DESCRIPTIONS = {
   },
   next_iss_pass: {
     description:
-      "When the user asks when the ISS / the space station will next fly over: returns the next geometric ISS pass with estimated visibility for the current camera location (or an explicit lat/lon) — rise time (ISO + minutes from now), rise compass direction, peak elevation, and duration. Requires the satellites layer to have loaded its catalog at least once this session; if it hasn't, tell the user to enable the satellites layer and try again.",
+      'Next ISS pass over the camera location (or lat/lon): rise time, direction, peak elevation, duration and estimated visibility. Needs the satellites layer to have loaded once; if not, ask the user to enable it.',
     $position: 1,
     parameters: {
       properties: {