From 881e24485cf92da68af79ef91c3372ad7ba598d4 Mon Sep 17 00:00:00 2001 From: devv-shayan Date: Wed, 16 Sep 2026 17:45:27 +0500 Subject: [PATCH 01/37] feat(voice): add ChatGPT OAuth auth option --- .env.example | 5 +- README.md | 8 +- docs/CURRENT-STATE.md | 6 +- scripts/dev-fresh.sh | 2 +- scripts/package-boundaries.json | 3 + scripts/setup-doctor.mjs | 4 +- server/providers/openai.js | 20 ++- server/providers/openai/codex-auth.js | 100 +++++++++++ server/providers/openai/realtime.js | 232 ++++++++++++++++++++++++-- src/codexOauthRealtime.test.mjs | 93 +++++++++++ src/keySetup.js | 144 ++++++++++++++++ src/keySetup.test.mjs | 23 +++ src/keySetupCore.mjs | 2 +- src/setupDoctor.test.mjs | 7 + src/tooling/localServices.test.mjs | 144 +++++++++++++++- src/ui/styles/provider-settings.css | 37 ++++ src/voice/cloudVoiceAuth.js | 41 +++++ src/voice/cloudVoiceAuth.test.mjs | 29 ++++ src/voice/gevRealtime.test.mjs | 14 ++ src/voice/realtimeBackend.js | 12 +- src/voice/realtimeBackend.test.mjs | 16 ++ src/voice/realtimeConnection.js | 2 + src/voice/realtimeCost.js | 8 + 23 files changed, 918 insertions(+), 34 deletions(-) create mode 100644 server/providers/openai/codex-auth.js create mode 100644 src/codexOauthRealtime.test.mjs create mode 100644 src/voice/cloudVoiceAuth.js create mode 100644 src/voice/cloudVoiceAuth.test.mjs diff --git a/.env.example b/.env.example index e16e85a27..4c7edb372 100644 --- a/.env.example +++ b/.env.example @@ -52,7 +52,10 @@ GOOGLE_MAPS_SERVER_API_KEY= # CLIENT-EXPOSED β€” use a public assets:read token with URL restrictions. CESIUM_ION_TOKEN= -# Optional: OpenAI Realtime voice control. Do not prefix with VITE_. +# Optional: OpenAI Realtime voice control. API key is the default cloud auth. +# Provider Settings can instead opt voice into the signed-in local +# ChatGPT/Codex OAuth session; the browser still receives only a short-lived +# Realtime client secret. Do not prefix with VITE_. OPENAI_API_KEY= OPENAI_REALTIME_MODEL=gpt-realtime-2 # Cheaper voice tier, used when the GEV MIC panel's MINI toggle is on. diff --git a/README.md b/README.md index 88690d52f..9676ab76f 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ _Why cockpit mode exists: you're riding a real aircraft over real terrain β€” an ## πŸŽ™οΈ Talk to It -> Voice needs an **OpenAI key**. Without one the entire app still runs β€” the mic button just reports voice is unavailable. The same key drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. +> Voice can use an **OpenAI API key** or, on the same machine, an opt-in signed-in **ChatGPT/Codex OAuth** session. The API-key path remains the default. Selecting ChatGPT OAuth in Provider Settings reuses an existing local sign-in or starts the Codex browser sign-in flow and selects OAuth when it completes. The OpenAI key still drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. Click **GEV MIC**, grant the microphone, and just talk. This is more than a voice-controlled remote: @@ -239,7 +239,7 @@ Click **GEV MIC**, grant the microphone, and just talk. This is more than a voic - **🎯 Entity Q&A.** Click any plane, ship, or datacenter and ask _"what's this?"_ It answers using the object's live telemetry. - **πŸ‘οΈ Visual grounding.** At street level, it reads a viewport screenshot to identify legible signage and building names, and is instructed never to hallucinate labels. - **🎬 Cinematic framing.** _"Show me the planes overhead"_ pulls the camera back, angles it, and frames the live traffic like a director. -- **πŸ”’ Honest and secure.** The agent only confirms actions that succeeded. Your `OPENAI_API_KEY` never touches the browser; the client only gets a short-lived session token. +- **πŸ”’ Honest and secure.** The agent only confirms actions that succeeded. Your `OPENAI_API_KEY` never touches the browser; the client only gets a short-lived session token. If you choose **USE CHATGPT OAUTH** in Provider Settings, the local server uses that sign-in only to mint the same kind of short-lived Realtime credential, and refuses the OAuth path for non-loopback clients. Twenty-nine tools, four jobs β€” the commands below come straight from the product's voice test suite and tool playbook: @@ -412,7 +412,7 @@ Six keys. Four have a free tier, and the two πŸ”΄ ones are metered: | --- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 🟑 | **Cesium ion** | πŸ—ΊοΈ Google Photorealistic 3D, world terrain, and additional ion-hosted imagery stacks. The free Community plan is for eligible individual, personal/non-commercial use and has quotas | [cesium.com/ion](https://cesium.com/ion) β€” use a public `assets:read` token and check current [pricing/eligibility](https://cesium.com/platform/cesium-ion/pricing/) | | πŸ”΄ | **Google Maps** | Direct Google Photorealistic 3D + Google place search ([Map Tiles API](https://developers.google.com/maps/documentation/tile)) | [Google Cloud Console](https://console.cloud.google.com/) β€” URL-restrict it | -| πŸ”΄ | **OpenAI** | πŸŽ™οΈ The voice experience + AI HUD summary. The mini model works; the standard model is noticeably smarter. Want Gemini or another provider behind the mic? PRs welcome | [platform.openai.com](https://platform.openai.com) β€” metered, see costs below | +| πŸ”΄ | **OpenAI** | πŸŽ™οΈ Voice + AI HUD summary through an API key. Provider Settings also offers **USE CHATGPT OAUTH** for cloud voice without removing the API-key option | [platform.openai.com](https://platform.openai.com) β€” API-key usage is metered, see costs below | | 🟑 | **AISStream** | 🚒 Live global ships | [aisstream.io](https://aisstream.io) β€” free signup | | 🟑 | **NASA FIRMS** | πŸ”₯ Live active fires | [firms.modaps.eosdis.nasa.gov](https://firms.modaps.eosdis.nasa.gov/api/map_key/) β€” free | | 🟑 | **TomTom** | 🚦 Live flow speeds and congestion colors for the simulated traffic layer | [developer.tomtom.com](https://developer.tomtom.com) β€” free tier available | @@ -490,7 +490,7 @@ Honest numbers, roughly, as of mid-2026 β€” always check the provider pricing pa | **🟒 Most layers** | **$0, no signup.** OpenSky anon, USGS, CelesTrak, adsb.lol, city CCTV, Radio Browser, GBFS, Launch Library 2, transit feeds, OSRM routing, NOAA and ECMWF weather, OpenStreetMap ALPR mapping, bundled datasets. | | **🟑 The free-key tier** | **$0 with a signup.** AISStream, FIRMS, TomTom, OpenSky, plus Cesium ion for eligible personal/non-commercial use. Provider quotas and eligibility still apply. | | **πŸ—ΊοΈ Google 3D tiles** | **Free through an eligible Cesium ion Community account within its quota; metered through a direct Google key.** Use the direct route for Google place search or commercial deployment, verify current provider terms, and set budget alerts where billing is enabled. | -| **πŸ”΄ OpenAI voice** | **The one that costs real money β€” so the app meters it for you.** Realtime audio runs a few cents per active minute; an evening of heavy use is single-digit dollars. A live session-spend readout sits next to the mic, with an STD/MINI model toggle, a $2 warning, and a **$5 hard cap that ends the session**. The voice context window is kept deliberately short too. | +| **πŸ”΄ OpenAI voice** | API-key sessions keep the live spend readout, STD/MINI model toggle, $2 warning, and **$5 hard cap**. ChatGPT OAuth is an explicit choice in Provider Settings. Its usage is tracked without an API-dollar estimate or spend cap. The voice context window stays deliberately short. | Google's direct 3D route is surprisingly generous: the first 1,000 Photorealistic 3D Tiles sessions each month are currently free, and one root request supports diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fdb42fc0c..410e648aa 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3607,7 +3607,7 @@ silently demoting every later lookup for the session. `GEV MIC` button (bottom UI) starts an OpenAI Realtime session over WebRTC: -- **Token flow**: browser fetches a short-lived client secret from `/api/realtime/token`; the Vite middleware holds `OPENAI_API_KEY` and posts the full session config (instructions, tool schemas, VAD, truncation) to `api.openai.com/v1/realtime/client_secrets`. SDP exchange goes directly to `api.openai.com/v1/realtime/calls` with the ephemeral token. +- **Token flow**: browser fetches a short-lived client secret from `/api/realtime/token`; the Vite middleware uses `OPENAI_API_KEY` by default, or the signed-in local ChatGPT/Codex OAuth access token when OAuth was explicitly selected in Provider Settings, and posts the full session config (instructions, tool schemas, VAD, truncation) to `api.openai.com/v1/realtime/client_secrets`. Both long-lived credentials stay server-side; OAuth minting is loopback-only. SDP exchange goes directly to `api.openai.com/v1/realtime/calls` with the ephemeral token. - **Session defaults** (env-tunable): model `gpt-realtime-2` (or `gpt-realtime-2.1-mini` when the MINI tier is selected β€” see the model-tier entry below), voice `marin`, reasoning effort `low`, semantic VAD with low eagerness, no response interruption, context window truncated to ~3,000 post-instruction tokens with 0.5 retention ratio β€” the conversational window stays short because map state is fetched live per turn. - **Twenty-nine tools** (argument schemas in `src/voice/actionSchemas.js`, served with their descriptions by `server/providers/openai/tools.js`, executed client-side in `src/voice/gevActions.js`): `fly_to_location`, `select_nearest_aircraft`, `adjust_camera_zoom`, `zoom_to_globe`, `set_layer_visibility`, `show_data_layers_menu`, `set_panel_open`, `set_visual_style`, `get_entity_context`, `get_current_view_state`, `set_hud`, `set_detection`, `set_map_stack`, `set_post_processing`, `control_scene`, `control_cctv`, `set_context_mode`, `control_cockpit`, `control_radio`, `track_entity`, `stop_tracking`, `frame_overhead`, `annotate_map`, `clear_annotations`, `move_camera`, `fly_route`, `analyst_query`, `next_iss_pass`, and `next_satellite_pass`. @@ -3653,7 +3653,7 @@ silently demoting every later lookup for the session. 2. **Contacts OFF** β†’ "nearby" means **in view**; "near \" means a radius around that place. A radius query with Contacts active and no explicit centre is centred on the **active contact**, not the camera. 3. **Every count names its scope in words** β€” "42 in your window", "8 in view", "about 30 within 250 km of Austin" β€” never a bare number. `analyst_query` returns `scopeLabel` so this is mechanical. Two different numbers with named scopes are not a contradiction. 4. **The loaded-data caveat is stated once when relevant**: counts cover loaded data, and the flights layer loads where you look (appended to `coverage.note` for radius/view scopes over viewport-loaded layers). -- **Degradation**: without `OPENAI_API_KEY`, `/api/realtime/token` returns 503 and the mic button surfaces the error; the rest of the app is unaffected. +- **Degradation**: API-key mode without `OPENAI_API_KEY` returns 503 as before. Selecting OAuth without a usable local ChatGPT/Codex sign-in starts the local `codex login` browser flow, polls for the resulting local token, and selects OAuth when sign-in completes. If the Codex login flow cannot start or does not complete, the rest of the app is unaffected. ### AI HUD Summary (June 2026) @@ -3835,7 +3835,7 @@ are omitted rather than framing the wrong part of the globe. - GBFS proxy refuses upstream redirects (`redirect: 'manual'`; any 3xx becomes a 502 and the redirect target is logged server-side only) and enforces its 5 MB response cap while the body streams, cancelling the upstream read past the cap; CCTV health map is bounded. - Proxy error payloads are sanitized (no internal error details returned to clients). - That holds for the OpenAI and CCTV media paths too: `/api/openai/hud-summary` never relays OpenAI's own `error.message`, `/api/realtime/token` passes successful ephemeral-token responses through but answers with a fixed error when minting fails or upstream rejects the request, and a failed CCTV media fetch stores a fixed camera health `message` β€” `GET /api/cctv/health` serializes that field and the CCTV panel renders it as a status label, so it is a client surface as much as the response body is. -- `OPENAI_API_KEY` is server-side only; the browser receives ephemeral Realtime client secrets from `/api/realtime/token`. +- `OPENAI_API_KEY` remains server-side only and remains the default voice credential. The optional OAuth mode reads the local ChatGPT/Codex access token server-side on loopback only; the browser receives only ephemeral Realtime client secrets from `/api/realtime/token`. - `AISSTREAM_API_KEY` is server-side only; the browser reads the same-origin `/api/ais-live` cache. - `/api/google/nearby-places` keeps the Google key out of Places requests issued for voice scene context. - `/api/google/text-search` keeps the Google key server-side for view-biased Places recovery used by annotation resolution. diff --git a/scripts/dev-fresh.sh b/scripts/dev-fresh.sh index d3e175050..8c134c745 100755 --- a/scripts/dev-fresh.sh +++ b/scripts/dev-fresh.sh @@ -347,7 +347,7 @@ case "${OPENSKY_AUTH_MODE}" in echo "OpenSky auth: disabled (anonymous mode)" ;; esac -[[ -n "${OPENAI_API_KEY}" ]] && echo "OpenAI key (voice + HUD summary): configured" || echo "OpenAI key (voice + HUD summary): not set β€” GEV MIC disabled" +[[ -n "${OPENAI_API_KEY}" ]] && echo "OpenAI key (voice + HUD summary): configured" || echo "OpenAI key: not set β€” GEV MIC can use ChatGPT OAuth from Provider Settings" [[ -n "${AISSTREAM_API_KEY}" ]] && echo "AISStream key (live vessels): configured" || echo "AISStream key (live vessels): not set β€” ships layer empty" if [[ -n "${GOOGLE_MAPS_API_KEY}" ]]; then echo "Startup map: Google Photorealistic 3D Tiles (direct)" diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 189192f10..12d03ef35 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -262,6 +262,7 @@ "server/providers/common/request.js", "server/providers/common/source-root.js", "server/providers/openai.js", + "server/providers/openai/codex-auth.js", "server/providers/openai/constants.js", "server/providers/openai/debug-log.js", "server/providers/openai/hud-summary.js", @@ -1220,6 +1221,7 @@ "modules": [ "src/voice/realtimeController.js", "src/voice/realtimeBackend.js", + "src/voice/cloudVoiceAuth.js", "src/voice/control.js", "src/voice/layerSummary.js", "src/voice/voiceCost.js", @@ -1795,6 +1797,7 @@ "src/voice/gevActions.js", "src/voice/gevRealtime.js", "src/voice/layerSummary.js", + "src/voice/cloudVoiceAuth.js", "src/voice/realtimeBackend.js", "src/voice/realtimeController.js", "src/voice/voiceCost.js", diff --git a/scripts/setup-doctor.mjs b/scripts/setup-doctor.mjs index 954369030..813548dbc 100644 --- a/scripts/setup-doctor.mjs +++ b/scripts/setup-doctor.mjs @@ -175,7 +175,9 @@ export function buildCapabilitySummary( ? 'Google Photorealistic 3D Tiles through Cesium ion; Bing and world-terrain stacks available' : 'Esri World Imagery (keyless satellite basemap) with keyless terrain', flights, - voice: configured('OPENAI_API_KEY') ? 'available' : 'off until an OpenAI key is added', + voice: configured('OPENAI_API_KEY') + ? 'available' + : 'ChatGPT/Codex OAuth can be selected at runtime', vessels: configured('AISSTREAM_API_KEY') ? 'live AISStream feed' : 'off until an AISStream key is added', fires: configured('FIRMS_MAP_KEY') ? 'live NASA FIRMS feed' : 'off until a FIRMS key is added', traffic: configured('TOMTOM_API_KEY') ? 'live TomTom flow' : 'built-in traffic simulation', diff --git a/server/providers/openai.js b/server/providers/openai.js index 5dd03db5c..ac1fcd1a7 100644 --- a/server/providers/openai.js +++ b/server/providers/openai.js @@ -1,13 +1,17 @@ import { defaultSourceRoot } from './common/source-root.js'; import { handleHudSummary } from './openai/hud-summary.js'; import { createDebugLogHandler } from './openai/debug-log.js'; -import { createRealtimeTokenHandler } from './openai/realtime.js'; +import { + createRealtimeOAuthLoginHandler, + createRealtimeOAuthStatusHandler, + createRealtimeTokenHandler, +} from './openai/realtime.js'; /** * Vite plugin: OpenAI Realtime ephemeral client secret. * - * Keeps OPENAI_API_KEY server-side while the browser connects to the - * Realtime API over WebRTC with a short-lived secret. + * Keeps the selected cloud credential server-side while the browser connects + * to the Realtime API over WebRTC with a short-lived secret. */ function openAiRealtimeProxy({ sourceRoot = defaultSourceRoot, @@ -26,6 +30,16 @@ function openAiRealtimeProxy({ '/api/realtime/token', createRealtimeTokenHandler({ ...realtime, annotationGuidance }), ); + + middlewares.use( + '/api/realtime/oauth-status', + createRealtimeOAuthStatusHandler(realtime), + ); + + middlewares.use( + '/api/realtime/oauth-login', + createRealtimeOAuthLoginHandler(realtime), + ); } return { diff --git a/server/providers/openai/codex-auth.js b/server/providers/openai/codex-auth.js new file mode 100644 index 000000000..6e882accc --- /dev/null +++ b/server/providers/openai/codex-auth.js @@ -0,0 +1,100 @@ +import { spawn } from 'node:child_process'; +import { accessSync, constants as fsConstants, readFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export function codexAuthPath({ + environment = process.env, + home = os.homedir(), +} = {}) { + const explicit = String(environment.CODEX_AUTH_JSON || '').trim(); + if (explicit) return path.resolve(explicit); + const codexHome = String(environment.CODEX_HOME || '').trim(); + return path.join( + codexHome ? path.resolve(codexHome) : path.join(home, '.codex'), + 'auth.json', + ); +} + +export function readCodexOAuthAccessToken({ + authPath = codexAuthPath(), + readFile = readFileSync, +} = {}) { + let parsed; + try { + parsed = JSON.parse(readFile(authPath, 'utf8')); + } catch { + throw new Error( + 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.', + ); + } + const token = String(parsed?.tokens?.access_token || '').trim(); + if (!token) { + throw new Error( + 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.', + ); + } + return token; +} + +export function resolveCodexExecutable({ + environment = process.env, + home = os.homedir(), + access = accessSync, +} = {}) { + const explicit = String(environment.CODEX_BIN || '').trim(); + if (explicit) return explicit; + + for (const candidate of [ + path.join(home, '.local', 'bin', 'codex'), + path.join( + home, + '.codex', + 'packages', + 'standalone', + 'current', + 'bin', + 'codex', + ), + ]) { + try { + access(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Fall through to the next known location, then PATH. + } + } + return 'codex'; +} + +export function startCodexChatGptLogin({ + spawnImpl = spawn, + environment = process.env, + home = os.homedir(), + executable = resolveCodexExecutable({ environment, home }), +} = {}) { + return new Promise((resolve, reject) => { + let child; + try { + child = spawnImpl(executable, ['login'], { + cwd: home, + detached: true, + stdio: 'ignore', + env: environment, + }); + } catch (error) { + reject(error); + return; + } + + child.once('error', reject); + child.once('spawn', () => { + child.unref?.(); + resolve({ executable }); + }); + }).catch((error) => { + throw new Error( + `Could not start ChatGPT sign-in with Codex: ${error?.message || error}`, + ); + }); +} diff --git a/server/providers/openai/realtime.js b/server/providers/openai/realtime.js index b4dfb4b5a..98fea5318 100644 --- a/server/providers/openai/realtime.js +++ b/server/providers/openai/realtime.js @@ -13,12 +13,167 @@ import { } from './constants.js'; import { realtimeInstructions } from './instructions.js'; import { GEV_REALTIME_TOOLS } from './tools.js'; +import { + readCodexOAuthAccessToken, + startCodexChatGptLogin, +} from './codex-auth.js'; + +const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']); + +function isLoopbackRequest(req) { + return LOOPBACK_ADDRESSES.has(String(req.socket?.remoteAddress || '')); +} + +function isSameOriginRequest(req) { + const origin = String(req.headers?.origin || '').trim(); + if (!origin) return true; + const host = String(req.headers?.host || '').trim(); + if (!host) return false; + const protocol = req.socket?.encrypted ? 'https:' : 'http:'; + try { + return new URL(origin).origin === new URL(`${protocol}//${host}`).origin; + } catch { + return false; + } +} + +function requestQuery(req) { + try { + return new URL(req.url || '', 'http://localhost').searchParams; + } catch { + return new URLSearchParams(); + } +} + +function resolveRealtimeAuthMode(query) { + const requested = String(query.get('auth') || 'api-key').toLowerCase(); + if (requested === 'api-key' || requested === 'oauth') return requested; + return null; +} + +function createRealtimeOAuthStatusHandler({ + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), +} = {}) { + return (req, res) => { + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Content-Type', 'application/json'); + if (req.method !== 'GET') { + res.statusCode = 405; + res.end(JSON.stringify({ error: 'Method not allowed' })); + return; + } + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'ChatGPT OAuth is available only from this machine', + }), + ); + return; + } + try { + const token = resolveOAuthAccessToken(); + if (!token) throw new Error('ChatGPT sign-in is unavailable'); + res.statusCode = 200; + res.end(JSON.stringify({ available: true })); + } catch (error) { + res.statusCode = 200; + res.end( + JSON.stringify({ + available: false, + error: error?.message || 'ChatGPT sign-in is unavailable', + }), + ); + } + }; +} + +function createRealtimeOAuthLoginHandler({ + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), + startOAuthLogin = () => startCodexChatGptLogin(), + now = () => Date.now(), + restartCooldownMs = 30_000, +} = {}) { + let lastStartedAt = 0; + return async (req, res) => { + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Content-Type', 'application/json'); + if (req.method !== 'POST') { + res.statusCode = 405; + res.end(JSON.stringify({ error: 'Method not allowed' })); + return; + } + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'ChatGPT OAuth sign-in is available only from this machine', + }), + ); + return; + } + if (!isSameOriginRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'Cross-origin ChatGPT OAuth sign-in is not allowed', + }), + ); + return; + } + + try { + if (resolveOAuthAccessToken()) { + res.statusCode = 200; + res.end( + JSON.stringify({ available: true, started: false, pending: false }), + ); + return; + } + } catch { + // No usable local token yet; launch the supported Codex sign-in flow. + } + + const currentTime = now(); + if ( + lastStartedAt > 0 && + currentTime - lastStartedAt < Math.max(0, restartCooldownMs) + ) { + res.statusCode = 202; + res.end( + JSON.stringify({ available: false, started: false, pending: true }), + ); + return; + } + + try { + await startOAuthLogin(); + lastStartedAt = currentTime; + res.statusCode = 202; + res.end( + JSON.stringify({ available: false, started: true, pending: true }), + ); + } catch (error) { + res.statusCode = 503; + res.end( + JSON.stringify({ + available: false, + error: error?.message || 'Could not start ChatGPT sign-in', + }), + ); + } + }; +} function createRealtimeTokenHandler({ annotationGuidance, endpoint = 'https://api.openai.com/v1/realtime/client_secrets', fetchImpl = (...args) => fetch(...args), resolveApiKey = () => process.env.OPENAI_API_KEY, + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), models = {}, } = {}) { return async (req, res) => { @@ -30,15 +185,62 @@ function createRealtimeTokenHandler({ return; } + const query = requestQuery(req); + const authMode = resolveRealtimeAuthMode(query); + if (!authMode) { + res.statusCode = 400; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'Unsupported cloud voice auth mode' })); + return; + } + // Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). No-op when unset. if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return; - const apiKey = resolveApiKey(); - if (!apiKey) { - res.statusCode = 503; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' })); - return; + let bearer; + if (authMode === 'oauth') { + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: 'ChatGPT OAuth voice is available only from this machine', + }), + ); + return; + } + try { + bearer = resolveOAuthAccessToken(); + } catch (error) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: error?.message || 'ChatGPT sign-in is unavailable', + code: 'CODEX_OAUTH_UNAVAILABLE', + }), + ); + return; + } + if (!bearer) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: 'ChatGPT sign-in is unavailable', + code: 'CODEX_OAUTH_UNAVAILABLE', + }), + ); + return; + } + } else { + bearer = resolveApiKey(); + if (!bearer) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' })); + return; + } } // Voice model tier, requested by the client as ?tier=standard|mini. @@ -47,15 +249,7 @@ function createRealtimeTokenHandler({ // bad querystring degrades to a normal session rather than a dead mic. // The env overrides stay authoritative per tier (see .env.example) β€” // a wrong upstream model id is then a config fix, not a code change. - const requestedTier = (() => { - try { - return new URL(req.url || '', 'http://localhost').searchParams.get( - 'tier', - ); - } catch { - return null; - } - })(); + const requestedTier = query.get('tier'); const tier = resolveVoiceModel(requestedTier).tier; const model = tier === 'mini' @@ -124,7 +318,7 @@ function createRealtimeTokenHandler({ redirect: 'error', signal: AbortSignal.timeout(30_000), headers: { - Authorization: `Bearer ${apiKey}`, + Authorization: `Bearer ${bearer}`, 'Content-Type': 'application/json', 'OpenAI-Safety-Identifier': 'gev-local-dev', }, @@ -167,4 +361,8 @@ function createRealtimeTokenHandler({ }; } -export { createRealtimeTokenHandler }; +export { + createRealtimeOAuthLoginHandler, + createRealtimeOAuthStatusHandler, + createRealtimeTokenHandler, +}; diff --git a/src/codexOauthRealtime.test.mjs b/src/codexOauthRealtime.test.mjs new file mode 100644 index 000000000..4afadb0e6 --- /dev/null +++ b/src/codexOauthRealtime.test.mjs @@ -0,0 +1,93 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import test from 'node:test'; +import path from 'node:path'; +import { + codexAuthPath, + readCodexOAuthAccessToken, + resolveCodexExecutable, + startCodexChatGptLogin, +} from '../server/providers/openai/codex-auth.js'; + +test('Codex auth path follows explicit auth file, CODEX_HOME, then home directory', () => { + assert.equal( + codexAuthPath({ + environment: { CODEX_AUTH_JSON: '/tmp/custom-auth.json' }, + home: '/home/fixture', + }), + path.resolve('/tmp/custom-auth.json'), + ); + assert.equal( + codexAuthPath({ + environment: { CODEX_HOME: '/tmp/codex-home' }, + home: '/home/fixture', + }), + path.join(path.resolve('/tmp/codex-home'), 'auth.json'), + ); + assert.equal( + codexAuthPath({ environment: {}, home: '/home/fixture' }), + path.join('/home/fixture', '.codex', 'auth.json'), + ); +}); + +test('Codex OAuth token reader returns only the access token and fails closed', () => { + assert.equal( + readCodexOAuthAccessToken({ + authPath: '/unused', + readFile: () => JSON.stringify({ tokens: { access_token: ' oauth-fixture ' } }), + }), + 'oauth-fixture', + ); + for (const readFile of [ + () => '{bad json', + () => JSON.stringify({ tokens: {} }), + () => { + throw new Error('missing'); + }, + ]) { + assert.throws( + () => readCodexOAuthAccessToken({ authPath: '/unused', readFile }), + /ChatGPT sign-in is unavailable/, + ); + } +}); + +test('Codex login resolves an installed executable before PATH and starts browser auth detached', async () => { + const checked = []; + assert.equal( + resolveCodexExecutable({ + environment: {}, + home: '/home/fixture', + access(candidate) { + checked.push(candidate); + if (candidate.endsWith('/.local/bin/codex')) return; + throw new Error('missing'); + }, + }), + path.join('/home/fixture', '.local', 'bin', 'codex'), + ); + assert.equal(checked.length, 1); + + const child = new EventEmitter(); + let unref = 0; + child.unref = () => { + unref += 1; + }; + const calls = []; + const pending = startCodexChatGptLogin({ + executable: '/fixture/codex', + home: '/home/fixture', + environment: { PATH: '/fixture' }, + spawnImpl(command, args, options) { + calls.push({ command, args, options }); + queueMicrotask(() => child.emit('spawn')); + return child; + }, + }); + assert.deepEqual(await pending, { executable: '/fixture/codex' }); + assert.equal(unref, 1); + assert.equal(calls[0].command, '/fixture/codex'); + assert.deepEqual(calls[0].args, ['login']); + assert.equal(calls[0].options.detached, true); + assert.equal(calls[0].options.stdio, 'ignore'); +}); diff --git a/src/keySetup.js b/src/keySetup.js index ee1d3b2dd..e137ff4d8 100644 --- a/src/keySetup.js +++ b/src/keySetup.js @@ -1,4 +1,8 @@ import { createSurfaceKeyboard } from './ui/surfaceKeyboard.js'; +import { + readStoredCloudVoiceAuthMode, + writeStoredCloudVoiceAuthMode, +} from './voice/cloudVoiceAuth.js'; /** * The POWER UP surface β€” paste a key, get a power. @@ -38,6 +42,46 @@ export function collectKeyUpdates(fields) { return updates; } +function sleepWithSignal(ms, signal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason || new DOMException('Aborted', 'AbortError')); + return; + } + const onAbort = () => { + clearTimeout(timer); + reject(signal.reason || new DOMException('Aborted', 'AbortError')); + }; + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + +export async function waitForChatGptOAuth({ + fetchImpl, + signal, + timeoutMs = 120_000, + pollMs = 1_000, + now = () => Date.now(), + sleep = sleepWithSignal, +} = {}) { + const deadline = now() + Math.max(0, timeoutMs); + while (!signal?.aborted && now() <= deadline) { + const response = await fetchImpl('/api/realtime/oauth-status', { + cache: 'no-store', + signal, + }); + const payload = await response.json().catch(() => ({})); + if (response.ok && payload.available) return true; + if (now() >= deadline) break; + await sleep(Math.max(0, pollMs), signal); + } + return false; +} + /** * After the FIRST Google key lands, the restart's reload should boot the * photoreal default β€” not faithfully restore the auto-selected keyless OSM @@ -144,6 +188,26 @@ function buildRow(documentRef, key) { } row.append(fields); } + if (key.id === 'openai') { + const auth = documentRef.createElement('div'); + auth.className = 'key-setup-cloud-auth'; + const mode = readStoredCloudVoiceAuthMode(); + const state = documentRef.createElement('span'); + state.className = 'key-setup-cloud-auth-state'; + state.textContent = + mode === 'oauth' ? 'VOICE AUTH Β· CHATGPT OAUTH' : 'VOICE AUTH Β· API KEY'; + const toggle = documentRef.createElement('button'); + toggle.type = 'button'; + toggle.className = 'key-setup-cloud-auth-toggle'; + toggle.dataset.cloudVoiceAuthToggle = 'true'; + toggle.textContent = mode === 'oauth' ? 'USE API KEY' : 'USE CHATGPT OAUTH'; + toggle.title = + mode === 'oauth' + ? 'Use OPENAI_API_KEY for the next cloud voice session' + : 'Use the signed-in local ChatGPT/Codex OAuth session for the next cloud voice session'; + auth.append(state, toggle); + row.append(auth); + } return row; } @@ -363,6 +427,86 @@ export async function initKeySetup({ applyButton?.addEventListener('click', onApply); // Remove buttons are rendered per row; delegate so re-renders stay wired. rowsHost?.addEventListener('click', (event) => { + const authButton = event.target?.closest?.( + '[data-cloud-voice-auth-toggle]', + ); + if (authButton && !disposed) { + void (async () => { + const current = readStoredCloudVoiceAuthMode(); + if (current === 'oauth') { + writeStoredCloudVoiceAuthMode('api-key'); + render(status); + say('Cloud voice will use OPENAI_API_KEY on the next session.'); + return; + } + authButton.disabled = true; + say('Checking local ChatGPT OAuth sign-in…'); + try { + const response = await doFetch('/api/realtime/oauth-status', { + cache: 'no-store', + signal: lifetime.signal, + }); + const payload = await response.json().catch(() => ({})); + if (response.ok && payload.available) { + writeStoredCloudVoiceAuthMode('oauth'); + render(status); + say( + 'ChatGPT OAuth selected for cloud voice. Your API key stays saved and available.', + ); + return; + } + + say('Opening ChatGPT sign-in in your browser…'); + const loginResponse = await doFetch('/api/realtime/oauth-login', { + method: 'POST', + cache: 'no-store', + signal: lifetime.signal, + }); + const loginPayload = await loginResponse.json().catch(() => ({})); + if (!loginResponse.ok && loginResponse.status !== 202) { + say( + loginPayload.error || + payload.error || + 'Could not start ChatGPT sign-in on this machine.', + ); + return; + } + if (loginPayload.available) { + writeStoredCloudVoiceAuthMode('oauth'); + render(status); + say( + 'ChatGPT OAuth selected for cloud voice. Your API key stays saved and available.', + ); + return; + } + + say( + 'Finish ChatGPT sign-in in the browser. Waiting for it to complete…', + ); + const available = await waitForChatGptOAuth({ + fetchImpl: doFetch, + signal: lifetime.signal, + }); + if (!available) { + say( + 'ChatGPT sign-in is still pending. Finish it in the browser, then click USE CHATGPT OAUTH again.', + ); + return; + } + writeStoredCloudVoiceAuthMode('oauth'); + render(status); + say( + 'ChatGPT sign-in complete. OAuth will be used for the next cloud voice session.', + ); + } catch (error) { + if (lifetime.signal.aborted) return; + say(`OAuth check failed: ${error?.message || error}`); + } finally { + authButton.disabled = false; + } + })(); + return; + } const button = event.target?.closest?.('[data-key-setup-remove]'); if (disposed || !button || busy) return; let envVars = []; diff --git a/src/keySetup.test.mjs b/src/keySetup.test.mjs index c0d9520c0..a03f2d556 100644 --- a/src/keySetup.test.mjs +++ b/src/keySetup.test.mjs @@ -5,6 +5,7 @@ import { keySetupChipLabel, stripKeylessBasemapFromHash, bindKeySetupPlacement, + waitForChatGptOAuth, } from './keySetup.js'; test('setup placement moves the same button only in Cyber and disconnects on teardown', () => { @@ -61,6 +62,28 @@ test('collectKeyUpdates keeps only non-empty trimmed values', () => { assert.deepEqual(collectKeyUpdates(null), {}); }); +test('OAuth login polling stops as soon as local ChatGPT auth becomes available', async () => { + let checks = 0; + let clock = 0; + const available = await waitForChatGptOAuth({ + fetchImpl: async () => { + checks += 1; + return { + ok: true, + json: async () => ({ available: checks >= 3 }), + }; + }, + timeoutMs: 10_000, + pollMs: 100, + now: () => clock, + sleep: async (ms) => { + clock += ms; + }, + }); + assert.equal(available, true); + assert.equal(checks, 3); +}); + test('the first Google key strips ONLY the keyless OSM basemap from the share hash', () => { const stripped = stripKeylessBasemapFromHash('lat=30.2&lon=-97.7&map=osm&style=normal'); assert.ok(stripped !== null); diff --git a/src/keySetupCore.mjs b/src/keySetupCore.mjs index e8bfef92c..a23c9825c 100644 --- a/src/keySetupCore.mjs +++ b/src/keySetupCore.mjs @@ -52,7 +52,7 @@ export const KEY_SETUP_KEYS = Object.freeze([ Object.freeze({ id: 'openai', title: 'OPENAI', - unlocks: 'Voice control β€” talk to the planet', + unlocks: 'Voice control β€” API key or ChatGPT OAuth', getUrl: 'https://platform.openai.com/api-keys', envVars: Object.freeze(['OPENAI_API_KEY']), tier: 'metered', diff --git a/src/setupDoctor.test.mjs b/src/setupDoctor.test.mjs index c66d9e686..aa825f126 100644 --- a/src/setupDoctor.test.mjs +++ b/src/setupDoctor.test.mjs @@ -224,6 +224,13 @@ test('doctor describes the credential ladder without exposing values', () => { assert.match(capabilities.map, /Google Photorealistic 3D Tiles through Cesium ion/); assert.match(capabilities.map, /Bing and world-terrain stacks/); assert.equal(capabilities.voice, 'available'); + assert.equal( + buildCapabilitySummary({ + ...credentials, + OPENAI_API_KEY: { configured: false }, + }).voice, + 'ChatGPT/Codex OAuth can be selected at runtime', + ); assert.match(capabilities.missions, /token allowance/); assert.equal(capabilities.flights, 'OpenSky keyless anonymous access (rate-limited)'); diff --git a/src/tooling/localServices.test.mjs b/src/tooling/localServices.test.mjs index 26eebfa08..7014659b7 100644 --- a/src/tooling/localServices.test.mjs +++ b/src/tooling/localServices.test.mjs @@ -37,6 +37,7 @@ function request( url = '/', body = '', origin = 'http://localhost:4173', + remoteAddress = '127.0.0.1', } = {}, ) { return new Promise((resolve, reject) => { @@ -49,7 +50,7 @@ function request( origin, 'content-type': 'application/json', }, - socket: { remoteAddress: '127.0.0.1' }, + socket: { remoteAddress }, }); const headers = {}; const res = { @@ -195,6 +196,147 @@ test('Realtime handler preserves tools and default instructions, isolates suppli assert.equal(sent[0].session.instructions, sent[2].session.instructions); }); +test('Realtime handler keeps API key as default and uses OAuth only when requested', async (t) => { + env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined); + let oauthReads = 0; + const authorizations = []; + const routes = install( + openAiRealtimeProxy({ + realtime: { + resolveApiKey: () => 'fixture-api-key', + resolveOAuthAccessToken: () => { + oauthReads += 1; + return 'fixture-oauth-token'; + }, + fetchImpl: async (_url, options) => { + authorizations.push(options.headers.Authorization); + return Response.json({ value: 'fixture-ephemeral' }); + }, + }, + }), + ); + const token = routes.get('/api/realtime/token'); + + assert.equal((await request(token)).status, 200); + assert.equal(oauthReads, 0, 'default cloud voice does not read OAuth'); + assert.equal(authorizations.at(-1), 'Bearer fixture-api-key'); + + assert.equal((await request(token, { url: '/?auth=oauth' })).status, 200); + assert.equal(oauthReads, 1); + assert.equal(authorizations.at(-1), 'Bearer fixture-oauth-token'); +}); + +test('OAuth status and token minting never expose local OAuth to remote clients', async () => { + let oauthReads = 0; + const routes = install( + openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => { + oauthReads += 1; + return 'must-not-be-read-remotely'; + }, + fetchImpl: () => + assert.fail('remote OAuth request must not reach OpenAI'), + }, + }), + ); + const remote = { remoteAddress: '192.168.1.20' }; + assert.equal( + (await request(routes.get('/api/realtime/oauth-status'), remote)).status, + 403, + ); + assert.equal( + ( + await request(routes.get('/api/realtime/token'), { + ...remote, + url: '/?auth=oauth', + }) + ).status, + 403, + ); + assert.equal(oauthReads, 0); +}); + +test('OAuth status reports local sign-in availability without returning the token', async () => { + const routes = install( + openAiRealtimeProxy({ + realtime: { resolveOAuthAccessToken: () => 'fixture-oauth-secret' }, + }), + ); + const response = await request(routes.get('/api/realtime/oauth-status')); + assert.equal(response.status, 200); + assert.deepEqual(response.json(), { available: true }); + assert.equal(response.body.includes('fixture-oauth-secret'), false); +}); + +test('OAuth login endpoint launches local Codex sign-in once and never exposes credentials', async () => { + let starts = 0; + let clock = 1_000; + const routes = install( + openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => { + throw new Error('not signed in'); + }, + startOAuthLogin: async () => { + starts += 1; + }, + now: () => clock, + }, + }), + ); + const handler = routes.get('/api/realtime/oauth-login'); + + const started = await request(handler, { method: 'POST' }); + assert.equal(started.status, 202); + assert.deepEqual(started.json(), { + available: false, + started: true, + pending: true, + }); + assert.equal(starts, 1); + + clock += 1_000; + const duplicate = await request(handler, { method: 'POST' }); + assert.equal(duplicate.status, 202); + assert.equal(duplicate.json().pending, true); + assert.equal(starts, 1, 'rapid retries must not spawn another login'); + + const remote = await request(handler, { + method: 'POST', + remoteAddress: '192.168.1.20', + }); + assert.equal(remote.status, 403); + assert.equal(starts, 1); + + const crossOrigin = await request(handler, { + method: 'POST', + origin: 'https://example.com', + }); + assert.equal(crossOrigin.status, 403); + assert.equal(starts, 1); +}); + +test('OAuth login endpoint selects an existing local sign-in without launching Codex', async () => { + const handler = install( + openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => 'fixture-existing-secret', + startOAuthLogin: () => + assert.fail('existing auth must not launch login'), + }, + }), + ).get('/api/realtime/oauth-login'); + const response = await request(handler, { method: 'POST' }); + assert.equal(response.status, 200); + assert.deepEqual(response.json(), { + available: true, + started: false, + pending: false, + }); + assert.equal(response.body.includes('fixture-existing-secret'), false); +}); + test('debug logging resolves each supplied application directory independently', async (t) => { const first = root(t), second = root(t); diff --git a/src/ui/styles/provider-settings.css b/src/ui/styles/provider-settings.css index b68593990..043e046d6 100644 --- a/src/ui/styles/provider-settings.css +++ b/src/ui/styles/provider-settings.css @@ -300,6 +300,43 @@ color: rgba(148, 178, 188, 0.42); } +.key-setup-cloud-auth { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.key-setup-cloud-auth-state { + color: rgba(190, 214, 222, 0.7); + font-family: var(--font-mono); + font-size: 0.56rem; + letter-spacing: 0.05rem; +} + +.key-setup-cloud-auth-toggle { + margin-left: auto; + padding: 0.28rem 0.55rem; + color: rgba(54, 220, 255, 0.9); + font-family: var(--font-mono); + font-size: 0.55rem; + letter-spacing: 0.06rem; + background: rgba(54, 220, 255, 0.08); + border: 1px solid rgba(54, 220, 255, 0.28); + border-radius: 0.32rem; + cursor: pointer; +} + +.key-setup-cloud-auth-toggle:hover, +.key-setup-cloud-auth-toggle:focus-visible { + border-color: rgba(54, 220, 255, 0.65); + outline: none; +} + +.key-setup-cloud-auth-toggle:disabled { + cursor: wait; + opacity: 0.5; +} + .key-setup-footer { margin-top: 0.85rem; } diff --git a/src/voice/cloudVoiceAuth.js b/src/voice/cloudVoiceAuth.js new file mode 100644 index 000000000..77feae8aa --- /dev/null +++ b/src/voice/cloudVoiceAuth.js @@ -0,0 +1,41 @@ +export const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']); +export const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key'; +export const CLOUD_VOICE_AUTH_STORAGE_KEY = 'godsEyeView.voice.cloudAuth'; + +function storageRef(storage) { + if (storage) return storage; + try { + return typeof localStorage === 'undefined' ? null : localStorage; + } catch { + return null; + } +} + +export function normalizeCloudVoiceAuthMode(mode) { + const normalized = String(mode || '') + .trim() + .toLowerCase(); + return CLOUD_VOICE_AUTH_MODES.includes(normalized) + ? normalized + : DEFAULT_CLOUD_VOICE_AUTH_MODE; +} + +export function readStoredCloudVoiceAuthMode(storage) { + try { + return normalizeCloudVoiceAuthMode( + storageRef(storage)?.getItem(CLOUD_VOICE_AUTH_STORAGE_KEY), + ); + } catch { + return DEFAULT_CLOUD_VOICE_AUTH_MODE; + } +} + +export function writeStoredCloudVoiceAuthMode(mode, storage) { + const resolved = normalizeCloudVoiceAuthMode(mode); + try { + storageRef(storage)?.setItem(CLOUD_VOICE_AUTH_STORAGE_KEY, resolved); + } catch { + /* best effort */ + } + return resolved; +} diff --git a/src/voice/cloudVoiceAuth.test.mjs b/src/voice/cloudVoiceAuth.test.mjs new file mode 100644 index 000000000..c7a4c2e4d --- /dev/null +++ b/src/voice/cloudVoiceAuth.test.mjs @@ -0,0 +1,29 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + CLOUD_VOICE_AUTH_STORAGE_KEY, + readStoredCloudVoiceAuthMode, + writeStoredCloudVoiceAuthMode, +} from './cloudVoiceAuth.js'; + +function memoryStorage(seed = {}) { + const values = new Map(Object.entries(seed)); + return { + getItem: (key) => values.get(key) ?? null, + setItem: (key, value) => values.set(key, value), + }; +} + +test('cloud voice auth defaults to API key and persists OAuth independently', () => { + const storage = memoryStorage(); + assert.equal(readStoredCloudVoiceAuthMode(storage), 'api-key'); + assert.equal(writeStoredCloudVoiceAuthMode('oauth', storage), 'oauth'); + assert.equal(readStoredCloudVoiceAuthMode(storage), 'oauth'); + assert.equal(writeStoredCloudVoiceAuthMode('unknown', storage), 'api-key'); + assert.equal( + readStoredCloudVoiceAuthMode( + memoryStorage({ [CLOUD_VOICE_AUTH_STORAGE_KEY]: 'oauth' }), + ), + 'oauth', + ); +}); diff --git a/src/voice/gevRealtime.test.mjs b/src/voice/gevRealtime.test.mjs index d7056a0e3..66e1b1367 100644 --- a/src/voice/gevRealtime.test.mjs +++ b/src/voice/gevRealtime.test.mjs @@ -3194,6 +3194,20 @@ test('F1: the toggle still records the next-session preference while live', () = assert.match(ui.tierButton.title, /this session stays on/i); }); +test('OAuth cloud sessions hide API spend and do not trip the API spend guard', () => { + const { controller, ui } = costControllerHarness(); + controller.status = 'listening'; + controller._cost.cloudVoiceAuth = 'oauth'; + controller._cost.sessionCloudVoiceAuth = 'oauth'; + controller.syncCostUi(); + const before = controller.costTracker.state(); + assert.equal(ui.costValue.hidden, true); + assert.equal(controller.recordUsage(usdUsage(100)), null); + const after = controller.costTracker.state(); + assert.equal(after.totalUsd, before.totalUsd); + assert.equal(controller.costCapStopped, false); +}); + test('F1: when idle, toggling does re-price the preview meter', () => { const { controller } = costControllerHarness(); controller.status = 'idle'; diff --git a/src/voice/realtimeBackend.js b/src/voice/realtimeBackend.js index 2a8af2d0d..b429e17c5 100644 --- a/src/voice/realtimeBackend.js +++ b/src/voice/realtimeBackend.js @@ -15,11 +15,19 @@ export function createRealtimeBackend({ ); return Object.freeze({ protocol: 'openai-realtime', - async requestToken({ tier = DEFAULT_VOICE_TIER, signal } = {}) { + async requestToken({ + tier = DEFAULT_VOICE_TIER, + authMode = 'api-key', + signal, + } = {}) { signal = scoped(signal); signal.throwIfAborted(); const separator = tokenEndpoint.includes('?') ? '&' : '?'; - const url = `${tokenEndpoint}${separator}tier=${encodeURIComponent(resolveVoiceModel(tier).tier)}`; + const query = new URLSearchParams({ + tier: resolveVoiceModel(tier).tier, + }); + if (authMode === 'oauth') query.set('auth', 'oauth'); + const url = `${tokenEndpoint}${separator}${query}`; const response = await tokenTransport(url, { signal, cache: 'no-store', diff --git a/src/voice/realtimeBackend.test.mjs b/src/voice/realtimeBackend.test.mjs index 1fc21da9d..d179f5b6b 100644 --- a/src/voice/realtimeBackend.test.mjs +++ b/src/voice/realtimeBackend.test.mjs @@ -41,6 +41,22 @@ test('separate configured token and SDP transports preserve model metadata and c ]); }); +test('OAuth is an additive token query while API key stays the default', async () => { + const calls = []; + const backend = createRealtimeBackend({ + tokenTransport: async (input) => { + calls.push(input); + return tokenReply(); + }, + }); + await backend.requestToken(); + await backend.requestToken({ authMode: 'oauth' }); + assert.deepEqual(calls, [ + '/api/realtime/token?tier=standard', + '/api/realtime/token?tier=standard&auth=oauth', + ]); +}); + test('denied tokens never negotiate or retry; reconnect requests a fresh token', async () => { let requests = 0; const backend = createRealtimeBackend({ diff --git a/src/voice/realtimeConnection.js b/src/voice/realtimeConnection.js index b20afb67d..b0409eb92 100644 --- a/src/voice/realtimeConnection.js +++ b/src/voice/realtimeConnection.js @@ -82,6 +82,7 @@ export class RealtimeConnection { this.debugLog('session.starting', { epoch, tier: this.cost.voiceTier, + authMode: this.cost.sessionCloudVoiceAuth, connection: this.connectionDiagnostics(), }); let localStream = null; @@ -89,6 +90,7 @@ export class RealtimeConnection { try { const minted = await this.backend.requestToken({ tier: this.cost.voiceTier, + authMode: this.cost.sessionCloudVoiceAuth, signal, }); const token = minted.token; diff --git a/src/voice/realtimeCost.js b/src/voice/realtimeCost.js index 9c1169490..83d19785e 100644 --- a/src/voice/realtimeCost.js +++ b/src/voice/realtimeCost.js @@ -4,6 +4,7 @@ import { writeStoredVoiceTier, writeStoredVoiceLimits, } from './realtimePreferences.js'; +import { readStoredCloudVoiceAuthMode } from './cloudVoiceAuth.js'; import { createVoiceCostTracker, resolveVoiceModel, @@ -16,6 +17,8 @@ export class RealtimeCost { Object.assign(this, { readUi, readStatus }, operations); this.voiceTier = readStoredVoiceTier(); this.voiceLimits = readStoredVoiceLimits(); + this.cloudVoiceAuth = readStoredCloudVoiceAuthMode(); + this.sessionCloudVoiceAuth = null; this.costTracker = createVoiceCostTracker({ tier: this.voiceTier, limits: this.voiceLimits, @@ -56,6 +59,8 @@ export class RealtimeCost { }; applies next session`; } if (this.ui?.costValue) { + const liveAuth = this.sessionCloudVoiceAuth || this.cloudVoiceAuth; + this.ui.costValue.hidden = liveAuth === 'oauth'; this.ui.costValue.textContent = state.display; this.ui.costValue.dataset.level = state.level; this.ui.costValue.title = @@ -136,6 +141,7 @@ export class RealtimeCost { */ recordUsage(usage) { if (!usage) return null; + if (this.sessionCloudVoiceAuth === 'oauth') return null; const state = this.costTracker.record(usage); this.syncCostUi(); if (state.warnCrossed) { @@ -190,6 +196,8 @@ export class RealtimeCost { prepareSession() { this.voiceTier = readStoredVoiceTier(); this.voiceLimits = readStoredVoiceLimits(); + this.cloudVoiceAuth = readStoredCloudVoiceAuthMode(); + this.sessionCloudVoiceAuth = this.cloudVoiceAuth; this.costCapStopped = false; // Provisional meter (tier-priced) so the readout shows $0.00 while // connecting. It is REPLACED below with one bound to the model the server From a398a259341537dc45497acbad2f1f17ce82eac4 Mon Sep 17 00:00:00 2001 From: devv-shayan Date: Sun, 27 Sep 2026 21:09:26 +0500 Subject: [PATCH 02/37] fix(voice): validate Codex OAuth and report unknown usage --- .env.example | 3 + server/providers/openai/codex-auth.js | 85 +++++++++++++++++++++++--- server/providers/openai/realtime.js | 28 +++++++++ src/codexOauthRealtime.test.mjs | 86 +++++++++++++++++++++++++-- src/tooling/localServices.test.mjs | 33 +++++++++- src/voice/gevRealtime.test.mjs | 17 +++++- src/voice/realtimeController.js | 5 +- src/voice/realtimeCost.js | 27 ++++++++- 8 files changed, 265 insertions(+), 19 deletions(-) diff --git a/.env.example b/.env.example index 4c7edb372..552962ba1 100644 --- a/.env.example +++ b/.env.example @@ -57,6 +57,9 @@ CESIUM_ION_TOKEN= # ChatGPT/Codex OAuth session; the browser still receives only a short-lived # Realtime client secret. Do not prefix with VITE_. OPENAI_API_KEY= +# Optional: forbid API-key voice sessions. Select ChatGPT OAuth in Provider +# Settings; missing or expired Codex sign-in then fails closed. +# GEV_PREFER_CODEX_OAUTH=true OPENAI_REALTIME_MODEL=gpt-realtime-2 # Cheaper voice tier, used when the GEV MIC panel's MINI toggle is on. # Model ids and prices are external facts that drift β€” if OpenAI moves this id, diff --git a/server/providers/openai/codex-auth.js b/server/providers/openai/codex-auth.js index 6e882accc..68c20e929 100644 --- a/server/providers/openai/codex-auth.js +++ b/server/providers/openai/codex-auth.js @@ -1,5 +1,10 @@ import { spawn } from 'node:child_process'; -import { accessSync, constants as fsConstants, readFileSync } from 'node:fs'; +import { + accessSync, + constants as fsConstants, + readFileSync, + statSync, +} from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -16,22 +21,88 @@ export function codexAuthPath({ ); } +/** An optional operator lock that refuses the metered API-key voice lane. */ +export function preferCodexOAuth(environment = process.env) { + if (!Object.hasOwn(environment, 'GEV_PREFER_CODEX_OAUTH')) return false; + const value = String(environment.GEV_PREFER_CODEX_OAUTH || '') + .trim() + .toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(value)) return true; + if (['0', 'false', 'no', 'off'].includes(value)) return false; + throw new Error( + 'GEV_PREFER_CODEX_OAUTH must be true or false. Refusing to choose a voice auth mode.', + ); +} + export function readCodexOAuthAccessToken({ authPath = codexAuthPath(), readFile = readFileSync, + statFile = statSync, + nowMs = Date.now(), } = {}) { + const unavailable = + 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.'; let parsed; try { parsed = JSON.parse(readFile(authPath, 'utf8')); } catch { - throw new Error( - 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.', - ); + throw new Error(unavailable); } - const token = String(parsed?.tokens?.access_token || '').trim(); - if (!token) { + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(unavailable); + } + const mode = String(parsed.auth_mode || '') + .trim() + .toLowerCase(); + const chatGptMode = mode + ? mode === 'chatgpt' || mode === 'chatgptauthtokens' + : typeof parsed.OPENAI_API_KEY !== 'string'; + const token = parsed.tokens?.access_token; + const refresh = parsed.tokens?.refresh_token; + if ( + !chatGptMode || + typeof token !== 'string' || + !token.trim() || + typeof refresh !== 'string' || + !refresh.trim() + ) { + throw new Error(unavailable); + } + + // This only checks token shape and expiry. OpenAI verifies the signature when + // minting the short-lived Realtime credential; we never modify auth.json. + const parts = token.split('.'); + if ( + parts.length !== 3 || + parts.some((part) => !/^[A-Za-z0-9_-]+$/.test(part)) + ) { + throw new Error(unavailable); + } + let payload; + try { + payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')); + } catch { + throw new Error(unavailable); + } + if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { + throw new Error(unavailable); + } + let expiresAtMs; + if (Object.hasOwn(payload, 'exp')) { + if (typeof payload.exp !== 'number' || !Number.isFinite(payload.exp)) { + throw new Error(unavailable); + } + expiresAtMs = payload.exp * 1000; + } else { + try { + expiresAtMs = statFile(authPath).mtimeMs + 60 * 60 * 1000; + } catch { + throw new Error(unavailable); + } + } + if (!Number.isFinite(expiresAtMs) || expiresAtMs <= nowMs + 60_000) { throw new Error( - 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.', + 'ChatGPT sign-in has expired. Run codex login and try again.', ); } return token; diff --git a/server/providers/openai/realtime.js b/server/providers/openai/realtime.js index 98fea5318..1869ebd24 100644 --- a/server/providers/openai/realtime.js +++ b/server/providers/openai/realtime.js @@ -14,6 +14,7 @@ import { import { realtimeInstructions } from './instructions.js'; import { GEV_REALTIME_TOOLS } from './tools.js'; import { + preferCodexOAuth, readCodexOAuthAccessToken, startCodexChatGptLogin, } from './codex-auth.js'; @@ -174,6 +175,7 @@ function createRealtimeTokenHandler({ fetchImpl = (...args) => fetch(...args), resolveApiKey = () => process.env.OPENAI_API_KEY, resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), + environment = process.env, models = {}, } = {}) { return async (req, res) => { @@ -194,6 +196,28 @@ function createRealtimeTokenHandler({ return; } + let oauthPinned; + try { + oauthPinned = preferCodexOAuth(environment); + } catch (error) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: error.message })); + return; + } + if (oauthPinned && authMode !== 'oauth') { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: + 'ChatGPT OAuth is required by GEV_PREFER_CODEX_OAUTH. Select USE CHATGPT OAUTH in Provider Settings.', + code: 'CODEX_OAUTH_REQUIRED', + }), + ); + return; + } + // Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). No-op when unset. if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return; @@ -332,6 +356,10 @@ function createRealtimeTokenHandler({ // case where a bogus ?tier= was silently downgraded to standard. res.setHeader('X-GEV-Voice-Tier', tier); res.setHeader('X-GEV-Voice-Model', model); + res.setHeader( + 'X-GEV-Voice-Auth', + authMode === 'oauth' ? 'codex-oauth' : 'env', + ); if (requestedTier && !isKnownVoiceTier(requestedTier)) { res.setHeader('X-GEV-Voice-Tier-Fallback', '1'); } diff --git a/src/codexOauthRealtime.test.mjs b/src/codexOauthRealtime.test.mjs index 4afadb0e6..ffd2b169f 100644 --- a/src/codexOauthRealtime.test.mjs +++ b/src/codexOauthRealtime.test.mjs @@ -4,11 +4,32 @@ import test from 'node:test'; import path from 'node:path'; import { codexAuthPath, + preferCodexOAuth, readCodexOAuthAccessToken, resolveCodexExecutable, startCodexChatGptLogin, } from '../server/providers/openai/codex-auth.js'; +const nowMs = Date.UTC(2026, 8, 27); +const jwt = (payload) => + `${Buffer.from('{}').toString('base64url')}.${Buffer.from(JSON.stringify(payload)).toString('base64url')}.signature`; +const auth = (token, extras = {}) => + JSON.stringify({ + auth_mode: 'chatgpt', + tokens: { access_token: token, refresh_token: 'refresh-fixture' }, + ...extras, + }); + +test('optional Codex OAuth pin fails closed on ambiguous values', () => { + assert.equal(preferCodexOAuth({}), false); + assert.equal(preferCodexOAuth({ GEV_PREFER_CODEX_OAUTH: 'true' }), true); + assert.equal(preferCodexOAuth({ GEV_PREFER_CODEX_OAUTH: 'off' }), false); + assert.throws( + () => preferCodexOAuth({ GEV_PREFER_CODEX_OAUTH: 'maybe' }), + /must be true or false/, + ); +}); + test('Codex auth path follows explicit auth file, CODEX_HOME, then home directory', () => { assert.equal( codexAuthPath({ @@ -30,28 +51,85 @@ test('Codex auth path follows explicit auth file, CODEX_HOME, then home director ); }); -test('Codex OAuth token reader returns only the access token and fails closed', () => { +test('Codex OAuth reader accepts a current ChatGPT token and rejects unusable credentials', () => { + const current = jwt({ exp: nowMs / 1000 + 3600 }); assert.equal( readCodexOAuthAccessToken({ authPath: '/unused', - readFile: () => JSON.stringify({ tokens: { access_token: ' oauth-fixture ' } }), + nowMs, + readFile: () => auth(current), }), - 'oauth-fixture', + current, ); for (const readFile of [ () => '{bad json', () => JSON.stringify({ tokens: {} }), + () => auth(current, { auth_mode: 'apikey' }), + () => + JSON.stringify({ + OPENAI_API_KEY: 'sk-fixture', + tokens: { access_token: current, refresh_token: 'refresh-fixture' }, + }), + () => + JSON.stringify({ + auth_mode: 'chatgpt', + tokens: { access_token: current }, + }), + () => auth('opaque-token'), + () => auth('header.not-json.signature'), + () => auth(jwt({ exp: 'tomorrow' })), () => { throw new Error('missing'); }, ]) { assert.throws( - () => readCodexOAuthAccessToken({ authPath: '/unused', readFile }), + () => readCodexOAuthAccessToken({ authPath: '/unused', readFile, nowMs }), /ChatGPT sign-in is unavailable/, ); } }); +test('Codex OAuth reader enforces expiry margin and bounds tokens without exp by file age', () => { + for (const exp of [nowMs / 1000 - 1, nowMs / 1000 + 59]) { + assert.throws( + () => + readCodexOAuthAccessToken({ + authPath: '/unused', + nowMs, + readFile: () => auth(jwt({ exp })), + }), + /sign-in has expired/, + ); + } + const noExp = jwt({ sub: 'fixture' }); + const base = { authPath: '/unused', nowMs, readFile: () => auth(noExp) }; + assert.equal( + readCodexOAuthAccessToken({ + ...base, + statFile: () => ({ mtimeMs: nowMs }), + }), + noExp, + ); + assert.throws( + () => + readCodexOAuthAccessToken({ + ...base, + statFile: () => ({ mtimeMs: nowMs - 3600_000 }), + }), + /sign-in has expired/, + ); + assert.throws( + () => + readCodexOAuthAccessToken({ + ...base, + statFile: () => { + throw new Error('missing'); + }, + }), + /sign-in is unavailable/, + ); +}); + test('Codex login resolves an installed executable before PATH and starts browser auth detached', async () => { const checked = []; assert.equal( diff --git a/src/tooling/localServices.test.mjs b/src/tooling/localServices.test.mjs index 7014659b7..408dbbbc3 100644 --- a/src/tooling/localServices.test.mjs +++ b/src/tooling/localServices.test.mjs @@ -217,15 +217,44 @@ test('Realtime handler keeps API key as default and uses OAuth only when request ); const token = routes.get('/api/realtime/token'); - assert.equal((await request(token)).status, 200); + const apiResponse = await request(token); + assert.equal(apiResponse.status, 200); + assert.equal(apiResponse.headers['x-gev-voice-auth'], 'env'); assert.equal(oauthReads, 0, 'default cloud voice does not read OAuth'); assert.equal(authorizations.at(-1), 'Bearer fixture-api-key'); - assert.equal((await request(token, { url: '/?auth=oauth' })).status, 200); + const oauthResponse = await request(token, { url: '/?auth=oauth' }); + assert.equal(oauthResponse.status, 200); + assert.equal(oauthResponse.headers['x-gev-voice-auth'], 'codex-oauth'); assert.equal(oauthReads, 1); assert.equal(authorizations.at(-1), 'Bearer fixture-oauth-token'); }); +test('a Codex OAuth pin refuses API-key minting without silently switching modes', async (t) => { + env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined); + const authorizations = []; + const routes = install( + openAiRealtimeProxy({ + realtime: { + environment: { GEV_PREFER_CODEX_OAUTH: 'true' }, + resolveApiKey: () => assert.fail('pinned OAuth cannot read an API key'), + resolveOAuthAccessToken: () => 'fixture-oauth-token', + fetchImpl: async (_url, options) => { + authorizations.push(options.headers.Authorization); + return Response.json({ value: 'fixture-ephemeral' }); + }, + }, + }), + ); + const token = routes.get('/api/realtime/token'); + const refused = await request(token); + assert.equal(refused.status, 503); + assert.equal(refused.json().code, 'CODEX_OAUTH_REQUIRED'); + assert.equal(authorizations.length, 0); + assert.equal((await request(token, { url: '/?auth=oauth' })).status, 200); + assert.deepEqual(authorizations, ['Bearer fixture-oauth-token']); +}); + test('OAuth status and token minting never expose local OAuth to remote clients', async () => { let oauthReads = 0; const routes = install( diff --git a/src/voice/gevRealtime.test.mjs b/src/voice/gevRealtime.test.mjs index 66e1b1367..f16e02a91 100644 --- a/src/voice/gevRealtime.test.mjs +++ b/src/voice/gevRealtime.test.mjs @@ -3194,18 +3194,29 @@ test('F1: the toggle still records the next-session preference while live', () = assert.match(ui.tierButton.title, /this session stays on/i); }); -test('OAuth cloud sessions hide API spend and do not trip the API spend guard', () => { +test('OAuth cloud sessions report unknown cost and usage without tripping the API spend guard', () => { const { controller, ui } = costControllerHarness(); controller.status = 'listening'; controller._cost.cloudVoiceAuth = 'oauth'; controller._cost.sessionCloudVoiceAuth = 'oauth'; controller.syncCostUi(); const before = controller.costTracker.state(); - assert.equal(ui.costValue.hidden, true); - assert.equal(controller.recordUsage(usdUsage(100)), null); + assert.equal(ui.costValue.hidden, false); + assert.equal(ui.costValue.textContent, 'COST UNKNOWN'); + assert.deepEqual(controller.recordUsage(usdUsage(100)), { + responses: 1, + input: 0, + output: 1562500, + costKnown: false, + }); const after = controller.costTracker.state(); assert.equal(after.totalUsd, before.totalUsd); assert.equal(controller.costCapStopped, false); + assert.match(ui.costValue.title, /1 response\(s\).*1562500 output tokens/); + controller.dc = {}; + controller.responseActive = true; + controller.stop(); + assert.match(ui.costValue.title, /Usage is incomplete/); }); test('F1: when idle, toggling does re-price the preview meter', () => { diff --git a/src/voice/realtimeController.js b/src/voice/realtimeController.js index d0c9f1bae..999297c27 100644 --- a/src/voice/realtimeController.js +++ b/src/voice/realtimeController.js @@ -239,7 +239,10 @@ export class GevRealtimeController extends RealtimeFacade { status: this.status, connection: this.connectionDiagnostics(), }); - if (this.dc && this.responseActive) this.costTracker.markIncomplete(); + if (this.dc && this.responseActive) { + this.costTracker.markIncomplete(); + this.syncCostUi(); + } this._connection.closeTransport(); this.stopVoiceVisualizer(); this._connection.releaseMedia(); diff --git a/src/voice/realtimeCost.js b/src/voice/realtimeCost.js index 83d19785e..0de89065c 100644 --- a/src/voice/realtimeCost.js +++ b/src/voice/realtimeCost.js @@ -19,6 +19,7 @@ export class RealtimeCost { this.voiceLimits = readStoredVoiceLimits(); this.cloudVoiceAuth = readStoredCloudVoiceAuthMode(); this.sessionCloudVoiceAuth = null; + this.oauthUsage = { responses: 0, input: 0, output: 0 }; this.costTracker = createVoiceCostTracker({ tier: this.voiceTier, limits: this.voiceLimits, @@ -60,7 +61,20 @@ export class RealtimeCost { } if (this.ui?.costValue) { const liveAuth = this.sessionCloudVoiceAuth || this.cloudVoiceAuth; - this.ui.costValue.hidden = liveAuth === 'oauth'; + if (liveAuth === 'oauth') { + const { responses, input, output } = this.oauthUsage; + this.ui.costValue.hidden = false; + this.ui.costValue.textContent = 'COST UNKNOWN'; + this.ui.costValue.dataset.level = 'unknown'; + this.ui.costValue.title = + `ChatGPT OAuth session: ${responses} response(s), ${input} input and ${output} output tokens reported. ` + + 'USD cost and the API spend cap are unavailable for this auth mode.' + + (state.incomplete + ? ' Usage is incomplete because a response was still in flight when the session ended.' + : ''); + return; + } + this.ui.costValue.hidden = false; this.ui.costValue.textContent = state.display; this.ui.costValue.dataset.level = state.level; this.ui.costValue.title = @@ -141,7 +155,15 @@ export class RealtimeCost { */ recordUsage(usage) { if (!usage) return null; - if (this.sessionCloudVoiceAuth === 'oauth') return null; + if (this.sessionCloudVoiceAuth === 'oauth') { + const count = (value) => + Number.isFinite(value) && value > 0 ? value : 0; + this.oauthUsage.responses += 1; + this.oauthUsage.input += count(usage.input_tokens); + this.oauthUsage.output += count(usage.output_tokens); + this.syncCostUi(); + return { ...this.oauthUsage, costKnown: false }; + } const state = this.costTracker.record(usage); this.syncCostUi(); if (state.warnCrossed) { @@ -198,6 +220,7 @@ export class RealtimeCost { this.voiceLimits = readStoredVoiceLimits(); this.cloudVoiceAuth = readStoredCloudVoiceAuthMode(); this.sessionCloudVoiceAuth = this.cloudVoiceAuth; + this.oauthUsage = { responses: 0, input: 0, output: 0 }; this.costCapStopped = false; // Provisional meter (tier-priced) so the readout shows $0.00 while // connecting. It is REPLACED below with one bound to the model the server From a19897ad1521b86189dd7bb8f515249ed51ebc63 Mon Sep 17 00:00:00 2001 From: Bilawal Sidhu Date: Mon, 28 Sep 2026 21:11:47 -0500 Subject: [PATCH 03/37] =?UTF-8?q?feat(voice):=20voice=20core=20=E2=80=94?= =?UTF-8?q?=20honest=20layer=20reach,=20spoken=20summary=20with=20on-scree?= =?UTF-8?q?n=20card,=20narration,=20point-and-ask?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Voice layer manifest (src/voice/layerManifest.js) generates the layer enums, aliases and analyst field hints; a test fails when a catalog layer has no entry. - analyst_query validates fields, operators, units, values and scopes, counts up to 250,000 loaded records per layer, reports complete/partial/unanswered, tells LAYER_OFF / NOT_READY / FEED_UNAVAILABLE apart from zero, refuses mismatched follow-ups and never lets a superseded query replace follow-up memory. Seven more layers expose analyst records. - Tool results carry a code-built say line and a display card; the voice card in the mic control shows captions, plan steps and the result (src/voice/resultDisplay.js is the one adapter). Slow tools narrate their current step out of band; holding Space while the assistant talks cuts it off. Turn spans go to the debug log; caption transcription is metered; the log omits what was said unless GEV_VOICE_LOG_CONTENT=1. - Point and ask: the Space-down pointer snapshot resolves "this", "that one" and "around here" through 'pointer' targets, and "the second one" through referent:n from the last numbered list. - Prompt states each policy once (instructions + tools 48.5k -> 46.7k chars). - Voice benchmark harness (scripts/qa-voice-bench.mjs, scripts/voice-bench/) shares its phrase table and grading with scripts/qa-voice-routing.mjs. --- .env.example | 7 + CHANGELOG.md | 48 + docs/CURRENT-STATE.md | 54 +- docs/VOICE-OWNERSHIP.md | 8 +- scripts/package-boundaries.json | 46 +- scripts/qa-voice-bench.mjs | 411 ++++++++ scripts/qa-voice-routing.mjs | 712 ++++++++++--- scripts/voice-bench/adapters/geminiLive.mjs | 179 ++++ scripts/voice-bench/adapters/localChat.mjs | 263 +++++ .../voice-bench/adapters/openaiRealtime.mjs | 167 +++ .../voice-bench/baseline-tools-4b56d0e9.json | 1 + scripts/voice-bench/grade.mjs | 268 +++++ scripts/voice-bench/keys.mjs | 35 + scripts/voice-bench/phrases.mjs | 600 +++++++++++ scripts/voice-bench/pricing.mjs | 58 ++ scripts/voice-bench/productionConfig.mjs | 36 + scripts/voice-bench/report.mjs | 272 +++++ scripts/voice-bench/stub.mjs | 21 + scripts/voice-bench/toolFormats.mjs | 119 +++ scripts/voice-turn-spans.mjs | 111 ++ server/providers/openai/constants.js | 4 + server/providers/openai/debug-log.js | 40 +- server/providers/openai/instructions.js | 41 +- server/providers/openai/realtime.js | 15 + server/providers/openai/toolDescriptions.js | 67 +- src/data/analystEngine.js | 971 +++++++++++++++--- src/data/analystEngine.test.mjs | 260 ++++- src/data/firmsHeatmap.test.mjs | 1 + src/data/pickRegistry.js | 17 + src/data/pickedContext.js | 128 +++ src/data/pickedContext.test.mjs | 75 ++ src/firstRunExperience.test.mjs | 10 +- src/layers/alpr/index.js | 31 + src/layers/bikeshare/controls.js | 33 + src/layers/cyclones/index.js | 29 + src/layers/firms/model.js | 4 +- src/layers/installations/controls.js | 29 + src/layers/launches/controls.js | 34 + src/layers/localAdsb/index.js | 34 + src/layers/transit/queries.js | 32 + src/radioMarkup.test.mjs | 29 +- src/tooling/localServices.test.mjs | 59 ++ src/tooling/voiceBench.test.mjs | 285 +++++ src/ui/styles/voice-card.css | 378 +++++++ src/voice/actionSchemas.js | 87 +- src/voice/actionSchemas.test.mjs | 60 +- src/voice/aircraftIdentityNarration.test.mjs | 66 +- src/voice/analystAccessors.test.mjs | 225 ++++ src/voice/control.js | 23 + src/voice/deixis.js | 363 +++++++ src/voice/deixis.test.mjs | 149 +++ src/voice/earcon.js | 66 ++ src/voice/gevActions.js | 726 +++++++++---- src/voice/gevActions.test.mjs | 225 +++- src/voice/gevRealtime.js | 35 +- src/voice/gevRealtime.test.mjs | 44 + src/voice/integrationContract.test.mjs | 301 ++++++ src/voice/layerManifest.js | 518 ++++++++++ src/voice/layerManifest.test.mjs | 112 ++ src/voice/narration.js | 270 +++++ src/voice/narration.test.mjs | 163 +++ src/voice/pointAndAsk.test.mjs | 352 +++++++ src/voice/pointAndAskReview.test.mjs | 261 +++++ src/voice/pointerContext.js | 366 +++++++ src/voice/pointerContext.test.mjs | 184 ++++ src/voice/pointerCrop.test.mjs | 232 +++++ src/voice/pointerPick.js | 61 ++ src/voice/pointerReticle.js | 135 +++ src/voice/pointerReticle.test.mjs | 103 ++ src/voice/realtimeBackend.js | 2 + src/voice/realtimeConnection.js | 1 + src/voice/realtimeController.js | 142 +++ src/voice/realtimeCost.js | 25 +- src/voice/realtimeFacade.js | 4 + src/voice/realtimeInput.js | 13 + src/voice/realtimeNarration.test.mjs | 258 +++++ src/voice/realtimeProtocol.js | 96 +- src/voice/realtimeRadio.js | 13 + src/voice/realtimeTurns.js | 376 ++++++- src/voice/realtimeViewport.js | 210 +++- src/voice/referents.js | 98 ++ src/voice/resultDisplay.js | 112 ++ src/voice/session.js | 24 +- src/voice/session.test.mjs | 27 + src/voice/sessionCommands.js | 50 + src/voice/speech.js | 545 ++++++++++ src/voice/speech.test.mjs | 437 ++++++++ src/voice/speechLint.js | 54 + src/voice/turnMetrics.js | 314 ++++++ src/voice/turnMetrics.test.mjs | 218 ++++ src/voice/voiceCard.js | 203 ++++ src/voice/voiceCard.test.mjs | 233 +++++ src/voice/voiceCardPresentation.js | 287 ++++++ src/voice/voiceCost.js | 72 +- src/voice/voiceCost.test.mjs | 26 + src/voice/voiceWorkflowHarness.mjs | 108 ++ style.css | 1 + 97 files changed, 14343 insertions(+), 755 deletions(-) create mode 100644 scripts/qa-voice-bench.mjs create mode 100644 scripts/voice-bench/adapters/geminiLive.mjs create mode 100644 scripts/voice-bench/adapters/localChat.mjs create mode 100644 scripts/voice-bench/adapters/openaiRealtime.mjs create mode 100644 scripts/voice-bench/baseline-tools-4b56d0e9.json create mode 100644 scripts/voice-bench/grade.mjs create mode 100644 scripts/voice-bench/keys.mjs create mode 100644 scripts/voice-bench/phrases.mjs create mode 100644 scripts/voice-bench/pricing.mjs create mode 100644 scripts/voice-bench/productionConfig.mjs create mode 100644 scripts/voice-bench/report.mjs create mode 100644 scripts/voice-bench/stub.mjs create mode 100644 scripts/voice-bench/toolFormats.mjs create mode 100644 scripts/voice-turn-spans.mjs create mode 100644 src/data/pickedContext.js create mode 100644 src/data/pickedContext.test.mjs create mode 100644 src/tooling/voiceBench.test.mjs create mode 100644 src/ui/styles/voice-card.css create mode 100644 src/voice/analystAccessors.test.mjs create mode 100644 src/voice/deixis.js create mode 100644 src/voice/deixis.test.mjs create mode 100644 src/voice/earcon.js create mode 100644 src/voice/integrationContract.test.mjs create mode 100644 src/voice/layerManifest.js create mode 100644 src/voice/layerManifest.test.mjs create mode 100644 src/voice/narration.js create mode 100644 src/voice/narration.test.mjs create mode 100644 src/voice/pointAndAsk.test.mjs create mode 100644 src/voice/pointAndAskReview.test.mjs create mode 100644 src/voice/pointerContext.js create mode 100644 src/voice/pointerContext.test.mjs create mode 100644 src/voice/pointerCrop.test.mjs create mode 100644 src/voice/pointerPick.js create mode 100644 src/voice/pointerReticle.js create mode 100644 src/voice/pointerReticle.test.mjs create mode 100644 src/voice/realtimeNarration.test.mjs create mode 100644 src/voice/referents.js create mode 100644 src/voice/resultDisplay.js create mode 100644 src/voice/speech.js create mode 100644 src/voice/speech.test.mjs create mode 100644 src/voice/speechLint.js create mode 100644 src/voice/turnMetrics.js create mode 100644 src/voice/turnMetrics.test.mjs create mode 100644 src/voice/voiceCard.js create mode 100644 src/voice/voiceCard.test.mjs create mode 100644 src/voice/voiceCardPresentation.js create mode 100644 src/voice/voiceWorkflowHarness.mjs diff --git a/.env.example b/.env.example index 67a1e74a7..a28f375d4 100644 --- a/.env.example +++ b/.env.example @@ -64,6 +64,13 @@ OPENAI_REALTIME_REASONING_EFFORT=low # Keep only a short conversational window. Current map state is fetched live. OPENAI_REALTIME_CONTEXT_TOKENS=3000 OPENAI_REALTIME_CONTEXT_RETENTION=0.5 +# Voice-card captions of what you said. Billed separately by OpenAI and +# counted in the session cost meter and cap; off disables them. +OPENAI_REALTIME_TRANSCRIBE_MODEL=gpt-4o-mini-transcribe +# The voice debug log (.gev-logs/realtime-conversations.jsonl, rotated at +# 32 MB, never expired) omits transcripts and typed or spoken text by default. +# Set to 1 to keep them while debugging; delete the log when you are done. +GEV_VOICE_LOG_CONTENT= OPENAI_HUD_SUMMARY_MODEL=gpt-5-nano # Optional: opt-in per-IP rate limit for the OpenAI cost endpoints # (/api/realtime/token + /api/openai/hud-summary), in requests per minute. diff --git a/CHANGELOG.md b/CHANGELOG.md index 58103d04d..08c7c47d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +- The voice card shows analyst counts ("At least 250,000 aircraft …") with + their scope, feed window and caveats, and marks partial answers. +- Voice answers are shorter and fill slow work instead of going silent. Slow + or multi-step requests may open with one short spoken plan; instant ones + just happen. Tool results carry a code-built `say` line and a `display` + card: sources and notes stay on screen, a caveat is spoken only when it + changes the answer, and "at least" and partial answers are always said. A card above the mic shows captions, the plan's steps and the + result. When a tool that reports steps (layers, place lookups, outlines, + nearest-aircraft) is still running after 1.5 s its current step is spoken + once; others say "still working" once after 8 s. A soft + tick plays in push-to-talk. Holding Space while the assistant talks cuts it + off. Each turn writes latency and speech figures to the debug log + (`node scripts/voice-turn-spans.mjs`). Captions of what you say are + counted in the voice cost meter; the debug log leaves out what was said + unless `GEV_VOICE_LOG_CONTENT=1`. +- Point and ask: hold Space with the cursor on an aircraft, ship or spot on + the map and say "what's that", "track that one" or "how many flights + around here". A reticle marks the point and the voice card shows what + "this" meant. In open mic it works when you have just moved the cursor + there. "The second one" picks from the last numbered list. Asking about + bare ground up close gives the assistant a small crop of the map around the + point. + - Public Overpass instances are no longer used by default. Street Traffic roads come from TomTom flow tiles, OpenFreeMap vector tiles, or both, chosen on the layer row (TomTom / OSM / Hybrid) or with `?trafficRoads=`. With a @@ -149,6 +172,7 @@ geocode and admin-boundary fallback answers `region-timeout` after 3 s instead of holding the reply. + - Transit and Directions rows repaint as soon as their data lands again: `refreshLayerStats()` now lives on the layer lifecycle, not only on the compatibility facade. `scripts/qa-radio.mjs` uses it instead of a private @@ -182,6 +206,30 @@ (`src/ui/imagerySplit.js`, `src/maps/imageryComparison.js`), and `MapSourceController.subscribe()` reports every settled map switch. +## Unreleased β€” voice reaches every layer; honest analyst answers + +- Voice can switch on every shipped data layer (the two scene-driven + Bhote Koshi layers stay scene-only), including transit, Recent + Imagery, wind, the weather layers, cyclones and mapped installations, and + understands their spoken names ("buses", "hurricanes", "military bases"). + One voice manifest (`src/voice/layerManifest.js`) generates the tool enums + and aliases, and a unit test fails when a new layer has no entry. +- `analyst_query` counts up to 250,000 loaded records per layer instead of the + first 2,000, and says when a layer went past that (`complete: false`). It + also answers over bikeshare stations, transit vehicles, rocket launches, + cyclones, ALPR cameras, mapped installations, local ADS-B aircraft and fire perimeters; fire + records carry their sensor (VIIRS or MODIS). +- Unknown fields, operators, units, value types, scope kinds and invalid + centres are refused with the allowed values instead of answering zero or widening to anywhere. A + layer that is off, still loading or unavailable is reported as such, not + as a count of zero; a partly answerable query is marked `partial`. + Distance sorts honour `sortDir`, a written unit such as `altitudeM(ft)` + converts, and large rankings no longer sort the whole set. +- Results carry `lat`/`lon` for each item and a `display` object with the + scope, the feed window (earthquakes and fires cover the last 24 h) and any + caveat for the screen. The model is asked to answer in one short phrase. +- The weather and Recent Imagery panels can be opened by voice. + ## Unreleased β€” local receiver feeds - The Local ADS-B layer also reads local 1090 MHz and 978 MHz UAT decoder diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 22ce1eb3b..92442b023 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -222,7 +222,7 @@ Consult the linked official advisory for safety decisions. Voice and HUD snapshots reuse the existing feedState classifier. Analyst follow-ups retain their original data provenance; current-view results append provenance without replacing legacy fields. HUD context and deterministic telemetry include non-nominal feed state. -Satellite and local infrastructure layers expose on-demand analyst records through their current factory owners. Analyst counts and ranks explicitly cover only bounded examined loaded records (default 2,000 per new layer, core satellite rows before dense extras); omitted records can change nearest/count and satellite distance is ground distance. Existing tools and result fields remain available. +Analyst records come on demand from each layer's `getAnalystRecords`: flights, military, local ADS-B, vessels, satellites, launches, earthquakes, FIRMS (VIIRS and MODIS, with a `sensor` field), fire perimeters, cyclones, transit, bikeshare, ALPR cameras, mapped installations, datacenters and dams. `analyst_query` counts the whole loaded set (up to 250,000 records per layer; past that `coverage.records` reports `returned`, `total` and `truncated: true` and the result carries `complete: false`). Ranking keeps a top-k and scans in 20,000-row slices that yield to the page and stop when superseded. It refuses unknown fields, operators, units that do not convert (`altitudeM(ft)` converts), wrong-typed values, invalid centres, unknown scope kinds and sort fields with `code` and `allowed`, refuses a follow-up with no previous answer (`NO_RESULT_CONTEXT`) or naming other layers than that answer (`FOLLOW_UP_MISMATCH`), and tells `LAYER_OFF`, `NOT_READY` and `FEED_UNAVAILABLE` apart from a real zero; when only some layers can answer the result is `partial` with `unanswered` ids. The Contacts-window path applies the same filters, ranking and item shape to the panel cohort. A superseded query never replaces follow-up memory. Items carry `lat`/`lon`; the feed window (earthquakes and FIRMS: last 24 h), the precise scope and any caveat sit in `display`, which is on-screen detail rather than something the model recites. "In view" is still a radius around the view centre, not the visible footprint. Satellite distance is ground distance. Keyless terrain tiles retry when Re:Earth throttles them. The browser fetches `terrain.reearth.land/cesium-mesh/ellipsoid/{z}/{x}/{y}.terrain` directly; no @@ -297,6 +297,45 @@ replacement session; audio meters release failed initialization and reject revok frames. Delayed action results and post-capture continuations cannot resume a stopped conversation or send output into a replacement. See [voice ownership](VOICE-OWNERSHIP.md). +Voice speech: `src/voice/speech.js` adds `{ say, display, referents?, schedule? }` +to layer toggles, fly_to_location, frame_overhead, select_nearest_aircraft, +get_entity_context and annotate_map (analyst_query plugs in through the +runner's `speechBuilders`). `get_current_view_state` lists enabled layers only. +The voice card nested in the mic control (`voiceCard.js`) shows captions, plan +steps and the result. `resultDisplay.js` is the one adapter from a result to +the card: builder results show as built, and analyst results show a title +with their scope, window and caveats. Caveats are on the card; the model speaks +one only when it changes the answer, and always says lower bounds and partial +answers. `narration.js` speaks one +code-authored line for the current step of a tool still running at 1.5 s +(out of band, never after the result). Tools that report steps: layer +toggles, place lookups, select_nearest_aircraft and annotate_map (finding, +outlining). Other tools get one "still working" line at 8 s. The +earcon plays in push-to-talk only. A claimed Space hold while the assistant +speaks is a barge-in (WebRTC cancel + audio clear), refused while Radio holds +the speaker (`mayVoiceClaimSpeaker()`). Progress lines stay outside the reply +lifecycle and Radio handoff. `turn.span` debug records carry generation and +playback latency, silence, preamble and word-count figures. Caption +transcription (`OPENAI_REALTIME_TRANSCRIBE_MODEL`, default on) is metered into +the session cost and cap. The debug log omits transcripts and text unless +`GEV_VOICE_LOG_CONTENT=1`. + +Point and ask: `pointerContext.js` reads what the cursor is on. A Space hold +is the pointing gesture: its keydown snapshot (released with the key) is sent +as an inert `pointer_context` item with the target, with reticle and chip. +Open-mic speech and typed turns count only a pointer that moved on the map in +the last 5 s and send only the kind (`pointing:"aircraft"`/`"ground"`); the chip appears when a tool +resolves it. Losing the pointer sends one `target:"none"` item. `deixis.js` resolves `'pointer'` in +track_entity/fly_to_location queries, get_entity_context scope, analyst_query +`scope.kind` and annotate_map targets, plus `referent:n` from the latest +numbered result (`referents.js`, cleared at session end). A new result set (an analyst count) replaces +the numbered list even when it is empty. A reticle marks the +point during the turn and the voice card shows "THIS"/"HERE"; both hide in +Clean-UI and recording. A ground-only pointer lookup at local/city scale sends a ringed 512 px crop in +the one retained image slot, dropped if the camera moved or a newer capture or +turn started; screenX/screenY against the crop are mapped back to the canvas. Harness: `qa-voice-routing.mjs --layer deixis`, +`--layer deixis-live`, `--layer behavior --deixis-only`. + The application shell composes focused state owners for navigation, destination lookup/orbit, Cockpit, visual settings, panel layout, aircraft display and layer bindings. Keyboard/display subscriptions have a separate lifetime; existing @@ -363,9 +402,11 @@ Reference feed construction is exported through `sources/reference`; the cable s Source factories have dedicated `layers//source` exports. ALPR and earthquake record normalization and CCTV source policy no longer pull rendering into source consumers. Catalog construction and voice feed reads use their focused owners; compatibility entries remain available. Settings filesystem hardening lives under `server/standalone/`. Import-direction checks complement export ownership checks; source behavior, settings policy and rendering are unchanged. -Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/result, interruption and completion events; stopping or replacing a session cancels pending actions. Alternate adapters can use the same controls and action runner. +Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. Alternate adapters can use the same controls and action runner. -Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 29 tools, retaining the existing 28 names and argument contracts apart from the three additive analyst-layer enum values using separate description-only metadata. Description customization cannot replace argument types, enum values or required fields. +Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 30 tools using separate description-only metadata. Description customization cannot replace argument types, enum values or required fields. + +`src/voice/layerManifest.js` declares each catalog layer's voice capabilities: spoken aliases, `toggle`, in-view `context`, `track`, and analyst `query` fields with type and unit, or `noQuery` with the reason. The `set_layer_visibility`, `show_data_layers_menu`, `get_entity_context` and `analyst_query` layer enums, the runner's alias table and the analyst field hints are generated from it. `src/voice/layerManifest.test.mjs` fails when a catalog layer is neither listed nor named in `VOICE_OFF_LAYERS` (today only the two scene-driven Bhote Koshi layers). Global Context is listed with `toggle: false`; `set_context_mode` reaches it. `set_panel_open` also opens `weather-panel` and `recent-imagery-panel`. Portable source exports provide Radio Browser station normalization, CCTV feed types and regional records independently of HTTP middleware. Radio normalization accepts an explicit URL policy; the standalone directory retains its existing HTTPS rules. Tile coordinate validation accepts explicit zoom bounds with unchanged traffic defaults. Cache, request and rendering owners remain unchanged. @@ -408,7 +449,10 @@ clears caches and removes map-stack listeners. Standalone setup supplies the existing service clients. HUD summaries, regional brief, cockpit weather, location framing and annotation boundaries receive their service instances explicitly. Compatibility entrypoints retain defaults for direct callers; normal startup does -not change global service slots. Analyst follow-up memory belongs to its voice runner. +not change global service slots. Analyst follow-up memory belongs to its voice +runner and ends with the voice session: stopping voice invalidates writes still in +flight and forgets the last answer, the pointer and the numbered list; removing +voice disposes the runner. Application startup constructs fresh layer instances from explicit source objects. Standalone composition selects the existing providers. Controls, launch orbit @@ -3849,7 +3893,7 @@ and unreachable upstream (502/504) separately from road geometry. - **Reliability**: tool-call dedupe (2.5s window across call/item/args keys); response-create queueing that respects active responses and defers follow-ups when the user starts speaking; per-tool follow-up instructions so the agent confirms only what actually happened (zoom confirms only on `ok=true`). - **Aircraft identity honesty:** β€œWhat is this aircraft?” reads callsign, operator, registration, type, and route only from the selected contact context. Missing operator, route, or type enrichment is named explicitly rather than silently omitted or inferred from the callsign. - **Diagnostics**: every client/server event is posted to `/api/realtime/debug-log` and appended to `.gev-logs/realtime-conversations.jsonl` (gitignored) with secret/image redaction; last 30 errors persist in `localStorage` (`gev-realtime-errors`); `window.__gevVoiceCommands.getDiagnostics()` in the console. -- **Counting semantics ("near")** β€” a CONTRACT; new count-bearing tool work inherits it. Three honest numbers exist for one question: the Contacts cohort (250 km around the subject, what the panel shows), `analyst_query`'s count of _currently-loaded_ records for the requested scope, and the layer-wide loaded total in `coverage.layersQueried[].records`. They diverge legitimately β€” the flights layer loads by viewport, so after a camera dive the loaded set can hold a fraction of the cohort (field case: panel 42, analyst 8). The contract: +- **Counting semantics ("near")** β€” a CONTRACT; new count-bearing tool work inherits it. Three honest numbers exist for one question: the Contacts cohort (250 km around the subject, what the panel shows), `analyst_query`'s count of _currently-loaded_ records for the requested scope, and the layer-wide loaded total in `coverage.layersQueried[].total`. They diverge legitimately β€” the flights layer loads by viewport, so after a camera dive the loaded set can hold a fraction of the cohort (field case: panel 42, analyst 8). The contract: 1. **Contacts ACTIVE** β†’ "near / nearby / how many aircraft" means the **Contacts window** β€” the panel's numbers, spoken verbatim. Mechanism: `contactsWindow` (`{centeredOn, radiusKm, flights, military, vessels}`), carried by both `analyst_query` and `get_current_view_state`, derived by `contactsWindowFromSnapshot()` from the same snapshot the panel renders so the two cannot drift. A cohort whose feed cannot answer reports `'unknown'`, never a confident zero. 2. **Contacts OFF** β†’ "nearby" means **in view**; "near \" means a radius around that place. A radius query with Contacts active and no explicit centre is centred on the **active contact**, not the camera. 3. **Every count names its scope in words** β€” "42 in your window", "8 in view", "about 30 within 250 km of Austin" β€” never a bare number. `analyst_query` returns `scopeLabel` so this is mechanical. Two different numbers with named scopes are not a contradiction. diff --git a/docs/VOICE-OWNERSHIP.md b/docs/VOICE-OWNERSHIP.md index d5553df00..4dac83a51 100644 --- a/docs/VOICE-OWNERSHIP.md +++ b/docs/VOICE-OWNERSHIP.md @@ -14,6 +14,12 @@ The session adapter, action runner and backend interfaces remain unchanged. | `realtimeCost.js` | Next-session preferences and the current session's model-bound meter | | `realtimeViewport.js` | One retained image, bounded deletion identities and capture generation | | `realtimeDiagnostics.js` | Bounded error history and sanitized optional diagnostics | +| `turnMetrics.js` | Per-turn latency and speech spans written to the debug log | +| `narration.js` | Progress lines and the earcon while tools run; spoken through the turns owner | +| `pointerContext.js` | Pointer position, dwell pick and the current turn's pointer snapshot (session-scoped) | +| `referents.js` | Numbered items from the latest result for `referent:n` (session-scoped) | +| `resultDisplay.js` | What the voice card shows for a result and which items it numbers (read by the card and `referents.js`) | +| `gevActions.js` runner | Analyst follow-up memory (session: `resetConversation()` on stop, `dispose()` on removal) | Pure input policy, preferences and protocol-response policy have separate modules. Owners receive named readers, operations or focused collaborators. Cross-owner @@ -29,7 +35,7 @@ Action and post-capture continuations retain their conversation identity before publishing output, queueing a response or changing status. Behavioral invariants remain in force: a 500 ms Space hold claims push-to-talk; -short control taps and text entry remain native; a click-started session stays +short control taps and text entry remain native; a claimed hold while the assistant speaks interrupts it (never during a Radio handoff); a click-started session stays open-mic. Every accepted or refused tool call receives its terminal output once. New typed input supersedes old intent. Radio waits for the spoken confirmation and verified muted playback. A live cost meter stays bound to its negotiated model, diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 276b9e0db..5e27c4db7 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -282,6 +282,7 @@ "src/data/installationFeedback.js", "src/voice/voiceCost.js", "src/voice/actionSchemas.js", + "src/voice/layerManifest.js", "server/providers/openai/toolDescriptions.js", "src/sources/rateLimit.js" ], @@ -649,6 +650,7 @@ "src/ui/styles/command-dock-trays.css", "src/ui/styles/command-dock-sliding.css", "src/ui/styles/voice-cost.css", + "src/ui/styles/voice-card.css", "src/ui/styles/first-run.css", "src/ui/styles/provider-settings.css", "src/ui/styles/weather.css", @@ -1285,7 +1287,19 @@ "src/voice/realtimeFacade.js", "src/voice/realtimeRadio.js", "src/voice/realtimeTurns.js", - "src/voice/realtimeConnection.js" + "src/voice/realtimeConnection.js", + "src/voice/turnMetrics.js", + "src/voice/speechLint.js", + "src/voice/voiceCard.js", + "src/voice/voiceCardPresentation.js", + "src/voice/resultDisplay.js", + "src/voice/referents.js", + "src/voice/layerManifest.js", + "src/voice/speech.js", + "src/voice/narration.js", + "src/voice/earcon.js", + "src/voice/pointerContext.js", + "src/voice/pointerReticle.js" ], "external": [] }, @@ -1385,6 +1399,7 @@ "src/data/aisLiveVessels.js", "src/data/alprCameras.js", "src/data/analystEngine.js", + "src/voice/layerManifest.js", "src/data/bikeshare.js", "src/data/cctv.js", "src/data/cctvCards.js", @@ -1851,6 +1866,14 @@ "src/voice/commands.js", "src/voice/control.js", "src/voice/gevActions.js", + "src/voice/speech.js", + "src/voice/earcon.js", + "src/voice/narration.js", + "src/voice/turnMetrics.js", + "src/voice/speechLint.js", + "src/voice/voiceCard.js", + "src/voice/voiceCardPresentation.js", + "src/voice/resultDisplay.js", "src/voice/gevRealtime.js", "src/voice/layerSummary.js", "src/voice/realtimeBackend.js", @@ -2014,6 +2037,12 @@ "src/maps/imageryHost.js", "src/ui/recentImagery.js", "src/ui/imageryBoxTool.js", + "src/voice/pointerContext.js", + "src/voice/pointerReticle.js", + "src/voice/deixis.js", + "src/voice/referents.js", + "src/voice/pointerPick.js", + "src/data/pickedContext.js", "src/layers/alpr/tileRecords.js", "src/sources/capability.js", "src/sources/openFreeMap.js", @@ -2736,7 +2765,11 @@ }, "voice-action-schemas": { "exports": ["./voice/action-schemas"], - "modules": ["src/voice/actionSchemas.js", "src/hudLayoutPolicy.js"], + "modules": [ + "src/voice/actionSchemas.js", + "src/voice/layerManifest.js", + "src/hudLayoutPolicy.js" + ], "external": [] }, "voice-session": { @@ -2749,7 +2782,14 @@ "modules": [ "src/voice/sessionCommands.js", "src/voice/session.js", - "src/voice/control.js" + "src/voice/control.js", + "src/voice/voiceCard.js", + "src/voice/voiceCardPresentation.js", + "src/voice/resultDisplay.js", + "src/voice/referents.js", + "src/voice/layerManifest.js", + "src/voice/speech.js", + "src/voice/pointerReticle.js" ], "external": [] }, diff --git a/scripts/qa-voice-bench.mjs b/scripts/qa-voice-bench.mjs new file mode 100644 index 000000000..f2f0f973b --- /dev/null +++ b/scripts/qa-voice-bench.mjs @@ -0,0 +1,411 @@ +#!/usr/bin/env node +/** + * qa-voice-bench.mjs β€” provider-neutral TEXT-mode voice-routing benchmark. + * + * Every provider gets the same prompt: production's realtimeInstructions() + * and GEV_REALTIME_TOOLS (the Realtime adapter uses the exact session body the + * token handler mints). Two suites, reported separately: + * core the qa-voice-routing phrase table, graded with its semantics + * (plus benchmark-only typed-argument checks) + * coverage the newer-layers coverage probe, graded correct / + * honest-refusal / hallucinated-capability / wrong-tool + * dialogue two-turn dialogues in one context; turn 2 depends on turn 1 + * capability area handles / imagery / OSM / point-and-ask phrases (and two + * dialogues) for the tools added after 4b56d0e9 + * Coverage rows are graded twice: v1 (original rubric, 4b56d0e9 tool schema) + * and v2 (rubric for the current tool surface). + * + * Methodology: a fresh context per phrase (new Realtime/Live session, or a + * stateless chat request); stub tool results; continuations until the model + * stops calling tools (max 4). No app page is involved. + * + * Usage: + * node scripts/qa-voice-bench.mjs --provider openai-realtime --model gpt-realtime-2 --reps 2 + * node scripts/qa-voice-bench.mjs --provider gemini-live --model gemini-3.8-live + * node scripts/qa-voice-bench.mjs --provider ollama --model gemma4:e4b + * node scripts/qa-voice-bench.mjs --provider openai-chat --base http://localhost:1234/v1 --model + * node scripts/qa-voice-bench.mjs --summarize --out qa-shots/voice-bench/ + * Common flags: --out (reuse one dir to collect several models), + * --suite core,coverage --only --budget + * --variant