From 881e24485cf92da68af79ef91c3372ad7ba598d4 Mon Sep 17 00:00:00 2001 From: devv-shayan Date: Wed, 16 Sep 2026 17:45:27 +0500 Subject: [PATCH] feat(voice): add ChatGPT OAuth auth option --- .env.example | 5 +- README.md | 8 +- docs/CURRENT-STATE.md | 6 +- scripts/dev-fresh.sh | 2 +- scripts/package-boundaries.json | 3 + scripts/setup-doctor.mjs | 4 +- server/providers/openai.js | 20 ++- server/providers/openai/codex-auth.js | 100 +++++++++++ server/providers/openai/realtime.js | 232 ++++++++++++++++++++++++-- src/codexOauthRealtime.test.mjs | 93 +++++++++++ src/keySetup.js | 144 ++++++++++++++++ src/keySetup.test.mjs | 23 +++ src/keySetupCore.mjs | 2 +- src/setupDoctor.test.mjs | 7 + src/tooling/localServices.test.mjs | 144 +++++++++++++++- src/ui/styles/provider-settings.css | 37 ++++ src/voice/cloudVoiceAuth.js | 41 +++++ src/voice/cloudVoiceAuth.test.mjs | 29 ++++ src/voice/gevRealtime.test.mjs | 14 ++ src/voice/realtimeBackend.js | 12 +- src/voice/realtimeBackend.test.mjs | 16 ++ src/voice/realtimeConnection.js | 2 + src/voice/realtimeCost.js | 8 + 23 files changed, 918 insertions(+), 34 deletions(-) create mode 100644 server/providers/openai/codex-auth.js create mode 100644 src/codexOauthRealtime.test.mjs create mode 100644 src/voice/cloudVoiceAuth.js create mode 100644 src/voice/cloudVoiceAuth.test.mjs diff --git a/.env.example b/.env.example index e16e85a27..4c7edb372 100644 --- a/.env.example +++ b/.env.example @@ -52,7 +52,10 @@ GOOGLE_MAPS_SERVER_API_KEY= # CLIENT-EXPOSED β€” use a public assets:read token with URL restrictions. CESIUM_ION_TOKEN= -# Optional: OpenAI Realtime voice control. Do not prefix with VITE_. +# Optional: OpenAI Realtime voice control. API key is the default cloud auth. +# Provider Settings can instead opt voice into the signed-in local +# ChatGPT/Codex OAuth session; the browser still receives only a short-lived +# Realtime client secret. Do not prefix with VITE_. OPENAI_API_KEY= OPENAI_REALTIME_MODEL=gpt-realtime-2 # Cheaper voice tier, used when the GEV MIC panel's MINI toggle is on. diff --git a/README.md b/README.md index 88690d52f..9676ab76f 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,7 @@ _Why cockpit mode exists: you're riding a real aircraft over real terrain β€” an ## πŸŽ™οΈ Talk to It -> Voice needs an **OpenAI key**. Without one the entire app still runs β€” the mic button just reports voice is unavailable. The same key drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. +> Voice can use an **OpenAI API key** or, on the same machine, an opt-in signed-in **ChatGPT/Codex OAuth** session. The API-key path remains the default. Selecting ChatGPT OAuth in Provider Settings reuses an existing local sign-in or starts the Codex browser sign-in flow and selects OAuth when it completes. The OpenAI key still drives the **AI HUD summary**: a terse, five-word intelligence-style readout of the current view that regenerates as you move. Click **GEV MIC**, grant the microphone, and just talk. This is more than a voice-controlled remote: @@ -239,7 +239,7 @@ Click **GEV MIC**, grant the microphone, and just talk. This is more than a voic - **🎯 Entity Q&A.** Click any plane, ship, or datacenter and ask _"what's this?"_ It answers using the object's live telemetry. - **πŸ‘οΈ Visual grounding.** At street level, it reads a viewport screenshot to identify legible signage and building names, and is instructed never to hallucinate labels. - **🎬 Cinematic framing.** _"Show me the planes overhead"_ pulls the camera back, angles it, and frames the live traffic like a director. -- **πŸ”’ Honest and secure.** The agent only confirms actions that succeeded. Your `OPENAI_API_KEY` never touches the browser; the client only gets a short-lived session token. +- **πŸ”’ Honest and secure.** The agent only confirms actions that succeeded. Your `OPENAI_API_KEY` never touches the browser; the client only gets a short-lived session token. If you choose **USE CHATGPT OAUTH** in Provider Settings, the local server uses that sign-in only to mint the same kind of short-lived Realtime credential, and refuses the OAuth path for non-loopback clients. Twenty-nine tools, four jobs β€” the commands below come straight from the product's voice test suite and tool playbook: @@ -412,7 +412,7 @@ Six keys. Four have a free tier, and the two πŸ”΄ ones are metered: | --- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | 🟑 | **Cesium ion** | πŸ—ΊοΈ Google Photorealistic 3D, world terrain, and additional ion-hosted imagery stacks. The free Community plan is for eligible individual, personal/non-commercial use and has quotas | [cesium.com/ion](https://cesium.com/ion) β€” use a public `assets:read` token and check current [pricing/eligibility](https://cesium.com/platform/cesium-ion/pricing/) | | πŸ”΄ | **Google Maps** | Direct Google Photorealistic 3D + Google place search ([Map Tiles API](https://developers.google.com/maps/documentation/tile)) | [Google Cloud Console](https://console.cloud.google.com/) β€” URL-restrict it | -| πŸ”΄ | **OpenAI** | πŸŽ™οΈ The voice experience + AI HUD summary. The mini model works; the standard model is noticeably smarter. Want Gemini or another provider behind the mic? PRs welcome | [platform.openai.com](https://platform.openai.com) β€” metered, see costs below | +| πŸ”΄ | **OpenAI** | πŸŽ™οΈ Voice + AI HUD summary through an API key. Provider Settings also offers **USE CHATGPT OAUTH** for cloud voice without removing the API-key option | [platform.openai.com](https://platform.openai.com) β€” API-key usage is metered, see costs below | | 🟑 | **AISStream** | 🚒 Live global ships | [aisstream.io](https://aisstream.io) β€” free signup | | 🟑 | **NASA FIRMS** | πŸ”₯ Live active fires | [firms.modaps.eosdis.nasa.gov](https://firms.modaps.eosdis.nasa.gov/api/map_key/) β€” free | | 🟑 | **TomTom** | 🚦 Live flow speeds and congestion colors for the simulated traffic layer | [developer.tomtom.com](https://developer.tomtom.com) β€” free tier available | @@ -490,7 +490,7 @@ Honest numbers, roughly, as of mid-2026 β€” always check the provider pricing pa | **🟒 Most layers** | **$0, no signup.** OpenSky anon, USGS, CelesTrak, adsb.lol, city CCTV, Radio Browser, GBFS, Launch Library 2, transit feeds, OSRM routing, NOAA and ECMWF weather, OpenStreetMap ALPR mapping, bundled datasets. | | **🟑 The free-key tier** | **$0 with a signup.** AISStream, FIRMS, TomTom, OpenSky, plus Cesium ion for eligible personal/non-commercial use. Provider quotas and eligibility still apply. | | **πŸ—ΊοΈ Google 3D tiles** | **Free through an eligible Cesium ion Community account within its quota; metered through a direct Google key.** Use the direct route for Google place search or commercial deployment, verify current provider terms, and set budget alerts where billing is enabled. | -| **πŸ”΄ OpenAI voice** | **The one that costs real money β€” so the app meters it for you.** Realtime audio runs a few cents per active minute; an evening of heavy use is single-digit dollars. A live session-spend readout sits next to the mic, with an STD/MINI model toggle, a $2 warning, and a **$5 hard cap that ends the session**. The voice context window is kept deliberately short too. | +| **πŸ”΄ OpenAI voice** | API-key sessions keep the live spend readout, STD/MINI model toggle, $2 warning, and **$5 hard cap**. ChatGPT OAuth is an explicit choice in Provider Settings. Its usage is tracked without an API-dollar estimate or spend cap. The voice context window stays deliberately short. | Google's direct 3D route is surprisingly generous: the first 1,000 Photorealistic 3D Tiles sessions each month are currently free, and one root request supports diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index fdb42fc0c..410e648aa 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -3607,7 +3607,7 @@ silently demoting every later lookup for the session. `GEV MIC` button (bottom UI) starts an OpenAI Realtime session over WebRTC: -- **Token flow**: browser fetches a short-lived client secret from `/api/realtime/token`; the Vite middleware holds `OPENAI_API_KEY` and posts the full session config (instructions, tool schemas, VAD, truncation) to `api.openai.com/v1/realtime/client_secrets`. SDP exchange goes directly to `api.openai.com/v1/realtime/calls` with the ephemeral token. +- **Token flow**: browser fetches a short-lived client secret from `/api/realtime/token`; the Vite middleware uses `OPENAI_API_KEY` by default, or the signed-in local ChatGPT/Codex OAuth access token when OAuth was explicitly selected in Provider Settings, and posts the full session config (instructions, tool schemas, VAD, truncation) to `api.openai.com/v1/realtime/client_secrets`. Both long-lived credentials stay server-side; OAuth minting is loopback-only. SDP exchange goes directly to `api.openai.com/v1/realtime/calls` with the ephemeral token. - **Session defaults** (env-tunable): model `gpt-realtime-2` (or `gpt-realtime-2.1-mini` when the MINI tier is selected β€” see the model-tier entry below), voice `marin`, reasoning effort `low`, semantic VAD with low eagerness, no response interruption, context window truncated to ~3,000 post-instruction tokens with 0.5 retention ratio β€” the conversational window stays short because map state is fetched live per turn. - **Twenty-nine tools** (argument schemas in `src/voice/actionSchemas.js`, served with their descriptions by `server/providers/openai/tools.js`, executed client-side in `src/voice/gevActions.js`): `fly_to_location`, `select_nearest_aircraft`, `adjust_camera_zoom`, `zoom_to_globe`, `set_layer_visibility`, `show_data_layers_menu`, `set_panel_open`, `set_visual_style`, `get_entity_context`, `get_current_view_state`, `set_hud`, `set_detection`, `set_map_stack`, `set_post_processing`, `control_scene`, `control_cctv`, `set_context_mode`, `control_cockpit`, `control_radio`, `track_entity`, `stop_tracking`, `frame_overhead`, `annotate_map`, `clear_annotations`, `move_camera`, `fly_route`, `analyst_query`, `next_iss_pass`, and `next_satellite_pass`. @@ -3653,7 +3653,7 @@ silently demoting every later lookup for the session. 2. **Contacts OFF** β†’ "nearby" means **in view**; "near \" means a radius around that place. A radius query with Contacts active and no explicit centre is centred on the **active contact**, not the camera. 3. **Every count names its scope in words** β€” "42 in your window", "8 in view", "about 30 within 250 km of Austin" β€” never a bare number. `analyst_query` returns `scopeLabel` so this is mechanical. Two different numbers with named scopes are not a contradiction. 4. **The loaded-data caveat is stated once when relevant**: counts cover loaded data, and the flights layer loads where you look (appended to `coverage.note` for radius/view scopes over viewport-loaded layers). -- **Degradation**: without `OPENAI_API_KEY`, `/api/realtime/token` returns 503 and the mic button surfaces the error; the rest of the app is unaffected. +- **Degradation**: API-key mode without `OPENAI_API_KEY` returns 503 as before. Selecting OAuth without a usable local ChatGPT/Codex sign-in starts the local `codex login` browser flow, polls for the resulting local token, and selects OAuth when sign-in completes. If the Codex login flow cannot start or does not complete, the rest of the app is unaffected. ### AI HUD Summary (June 2026) @@ -3835,7 +3835,7 @@ are omitted rather than framing the wrong part of the globe. - GBFS proxy refuses upstream redirects (`redirect: 'manual'`; any 3xx becomes a 502 and the redirect target is logged server-side only) and enforces its 5 MB response cap while the body streams, cancelling the upstream read past the cap; CCTV health map is bounded. - Proxy error payloads are sanitized (no internal error details returned to clients). - That holds for the OpenAI and CCTV media paths too: `/api/openai/hud-summary` never relays OpenAI's own `error.message`, `/api/realtime/token` passes successful ephemeral-token responses through but answers with a fixed error when minting fails or upstream rejects the request, and a failed CCTV media fetch stores a fixed camera health `message` β€” `GET /api/cctv/health` serializes that field and the CCTV panel renders it as a status label, so it is a client surface as much as the response body is. -- `OPENAI_API_KEY` is server-side only; the browser receives ephemeral Realtime client secrets from `/api/realtime/token`. +- `OPENAI_API_KEY` remains server-side only and remains the default voice credential. The optional OAuth mode reads the local ChatGPT/Codex access token server-side on loopback only; the browser receives only ephemeral Realtime client secrets from `/api/realtime/token`. - `AISSTREAM_API_KEY` is server-side only; the browser reads the same-origin `/api/ais-live` cache. - `/api/google/nearby-places` keeps the Google key out of Places requests issued for voice scene context. - `/api/google/text-search` keeps the Google key server-side for view-biased Places recovery used by annotation resolution. diff --git a/scripts/dev-fresh.sh b/scripts/dev-fresh.sh index d3e175050..8c134c745 100755 --- a/scripts/dev-fresh.sh +++ b/scripts/dev-fresh.sh @@ -347,7 +347,7 @@ case "${OPENSKY_AUTH_MODE}" in echo "OpenSky auth: disabled (anonymous mode)" ;; esac -[[ -n "${OPENAI_API_KEY}" ]] && echo "OpenAI key (voice + HUD summary): configured" || echo "OpenAI key (voice + HUD summary): not set β€” GEV MIC disabled" +[[ -n "${OPENAI_API_KEY}" ]] && echo "OpenAI key (voice + HUD summary): configured" || echo "OpenAI key: not set β€” GEV MIC can use ChatGPT OAuth from Provider Settings" [[ -n "${AISSTREAM_API_KEY}" ]] && echo "AISStream key (live vessels): configured" || echo "AISStream key (live vessels): not set β€” ships layer empty" if [[ -n "${GOOGLE_MAPS_API_KEY}" ]]; then echo "Startup map: Google Photorealistic 3D Tiles (direct)" diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 189192f10..12d03ef35 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -262,6 +262,7 @@ "server/providers/common/request.js", "server/providers/common/source-root.js", "server/providers/openai.js", + "server/providers/openai/codex-auth.js", "server/providers/openai/constants.js", "server/providers/openai/debug-log.js", "server/providers/openai/hud-summary.js", @@ -1220,6 +1221,7 @@ "modules": [ "src/voice/realtimeController.js", "src/voice/realtimeBackend.js", + "src/voice/cloudVoiceAuth.js", "src/voice/control.js", "src/voice/layerSummary.js", "src/voice/voiceCost.js", @@ -1795,6 +1797,7 @@ "src/voice/gevActions.js", "src/voice/gevRealtime.js", "src/voice/layerSummary.js", + "src/voice/cloudVoiceAuth.js", "src/voice/realtimeBackend.js", "src/voice/realtimeController.js", "src/voice/voiceCost.js", diff --git a/scripts/setup-doctor.mjs b/scripts/setup-doctor.mjs index 954369030..813548dbc 100644 --- a/scripts/setup-doctor.mjs +++ b/scripts/setup-doctor.mjs @@ -175,7 +175,9 @@ export function buildCapabilitySummary( ? 'Google Photorealistic 3D Tiles through Cesium ion; Bing and world-terrain stacks available' : 'Esri World Imagery (keyless satellite basemap) with keyless terrain', flights, - voice: configured('OPENAI_API_KEY') ? 'available' : 'off until an OpenAI key is added', + voice: configured('OPENAI_API_KEY') + ? 'available' + : 'ChatGPT/Codex OAuth can be selected at runtime', vessels: configured('AISSTREAM_API_KEY') ? 'live AISStream feed' : 'off until an AISStream key is added', fires: configured('FIRMS_MAP_KEY') ? 'live NASA FIRMS feed' : 'off until a FIRMS key is added', traffic: configured('TOMTOM_API_KEY') ? 'live TomTom flow' : 'built-in traffic simulation', diff --git a/server/providers/openai.js b/server/providers/openai.js index 5dd03db5c..ac1fcd1a7 100644 --- a/server/providers/openai.js +++ b/server/providers/openai.js @@ -1,13 +1,17 @@ import { defaultSourceRoot } from './common/source-root.js'; import { handleHudSummary } from './openai/hud-summary.js'; import { createDebugLogHandler } from './openai/debug-log.js'; -import { createRealtimeTokenHandler } from './openai/realtime.js'; +import { + createRealtimeOAuthLoginHandler, + createRealtimeOAuthStatusHandler, + createRealtimeTokenHandler, +} from './openai/realtime.js'; /** * Vite plugin: OpenAI Realtime ephemeral client secret. * - * Keeps OPENAI_API_KEY server-side while the browser connects to the - * Realtime API over WebRTC with a short-lived secret. + * Keeps the selected cloud credential server-side while the browser connects + * to the Realtime API over WebRTC with a short-lived secret. */ function openAiRealtimeProxy({ sourceRoot = defaultSourceRoot, @@ -26,6 +30,16 @@ function openAiRealtimeProxy({ '/api/realtime/token', createRealtimeTokenHandler({ ...realtime, annotationGuidance }), ); + + middlewares.use( + '/api/realtime/oauth-status', + createRealtimeOAuthStatusHandler(realtime), + ); + + middlewares.use( + '/api/realtime/oauth-login', + createRealtimeOAuthLoginHandler(realtime), + ); } return { diff --git a/server/providers/openai/codex-auth.js b/server/providers/openai/codex-auth.js new file mode 100644 index 000000000..6e882accc --- /dev/null +++ b/server/providers/openai/codex-auth.js @@ -0,0 +1,100 @@ +import { spawn } from 'node:child_process'; +import { accessSync, constants as fsConstants, readFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +export function codexAuthPath({ + environment = process.env, + home = os.homedir(), +} = {}) { + const explicit = String(environment.CODEX_AUTH_JSON || '').trim(); + if (explicit) return path.resolve(explicit); + const codexHome = String(environment.CODEX_HOME || '').trim(); + return path.join( + codexHome ? path.resolve(codexHome) : path.join(home, '.codex'), + 'auth.json', + ); +} + +export function readCodexOAuthAccessToken({ + authPath = codexAuthPath(), + readFile = readFileSync, +} = {}) { + let parsed; + try { + parsed = JSON.parse(readFile(authPath, 'utf8')); + } catch { + throw new Error( + 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.', + ); + } + const token = String(parsed?.tokens?.access_token || '').trim(); + if (!token) { + throw new Error( + 'ChatGPT sign-in is unavailable. Sign in to Codex or the ChatGPT desktop app and try again.', + ); + } + return token; +} + +export function resolveCodexExecutable({ + environment = process.env, + home = os.homedir(), + access = accessSync, +} = {}) { + const explicit = String(environment.CODEX_BIN || '').trim(); + if (explicit) return explicit; + + for (const candidate of [ + path.join(home, '.local', 'bin', 'codex'), + path.join( + home, + '.codex', + 'packages', + 'standalone', + 'current', + 'bin', + 'codex', + ), + ]) { + try { + access(candidate, fsConstants.X_OK); + return candidate; + } catch { + // Fall through to the next known location, then PATH. + } + } + return 'codex'; +} + +export function startCodexChatGptLogin({ + spawnImpl = spawn, + environment = process.env, + home = os.homedir(), + executable = resolveCodexExecutable({ environment, home }), +} = {}) { + return new Promise((resolve, reject) => { + let child; + try { + child = spawnImpl(executable, ['login'], { + cwd: home, + detached: true, + stdio: 'ignore', + env: environment, + }); + } catch (error) { + reject(error); + return; + } + + child.once('error', reject); + child.once('spawn', () => { + child.unref?.(); + resolve({ executable }); + }); + }).catch((error) => { + throw new Error( + `Could not start ChatGPT sign-in with Codex: ${error?.message || error}`, + ); + }); +} diff --git a/server/providers/openai/realtime.js b/server/providers/openai/realtime.js index b4dfb4b5a..98fea5318 100644 --- a/server/providers/openai/realtime.js +++ b/server/providers/openai/realtime.js @@ -13,12 +13,167 @@ import { } from './constants.js'; import { realtimeInstructions } from './instructions.js'; import { GEV_REALTIME_TOOLS } from './tools.js'; +import { + readCodexOAuthAccessToken, + startCodexChatGptLogin, +} from './codex-auth.js'; + +const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1', '::ffff:127.0.0.1']); + +function isLoopbackRequest(req) { + return LOOPBACK_ADDRESSES.has(String(req.socket?.remoteAddress || '')); +} + +function isSameOriginRequest(req) { + const origin = String(req.headers?.origin || '').trim(); + if (!origin) return true; + const host = String(req.headers?.host || '').trim(); + if (!host) return false; + const protocol = req.socket?.encrypted ? 'https:' : 'http:'; + try { + return new URL(origin).origin === new URL(`${protocol}//${host}`).origin; + } catch { + return false; + } +} + +function requestQuery(req) { + try { + return new URL(req.url || '', 'http://localhost').searchParams; + } catch { + return new URLSearchParams(); + } +} + +function resolveRealtimeAuthMode(query) { + const requested = String(query.get('auth') || 'api-key').toLowerCase(); + if (requested === 'api-key' || requested === 'oauth') return requested; + return null; +} + +function createRealtimeOAuthStatusHandler({ + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), +} = {}) { + return (req, res) => { + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Content-Type', 'application/json'); + if (req.method !== 'GET') { + res.statusCode = 405; + res.end(JSON.stringify({ error: 'Method not allowed' })); + return; + } + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'ChatGPT OAuth is available only from this machine', + }), + ); + return; + } + try { + const token = resolveOAuthAccessToken(); + if (!token) throw new Error('ChatGPT sign-in is unavailable'); + res.statusCode = 200; + res.end(JSON.stringify({ available: true })); + } catch (error) { + res.statusCode = 200; + res.end( + JSON.stringify({ + available: false, + error: error?.message || 'ChatGPT sign-in is unavailable', + }), + ); + } + }; +} + +function createRealtimeOAuthLoginHandler({ + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), + startOAuthLogin = () => startCodexChatGptLogin(), + now = () => Date.now(), + restartCooldownMs = 30_000, +} = {}) { + let lastStartedAt = 0; + return async (req, res) => { + res.setHeader('Cache-Control', 'no-store'); + res.setHeader('Content-Type', 'application/json'); + if (req.method !== 'POST') { + res.statusCode = 405; + res.end(JSON.stringify({ error: 'Method not allowed' })); + return; + } + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'ChatGPT OAuth sign-in is available only from this machine', + }), + ); + return; + } + if (!isSameOriginRequest(req)) { + res.statusCode = 403; + res.end( + JSON.stringify({ + available: false, + error: 'Cross-origin ChatGPT OAuth sign-in is not allowed', + }), + ); + return; + } + + try { + if (resolveOAuthAccessToken()) { + res.statusCode = 200; + res.end( + JSON.stringify({ available: true, started: false, pending: false }), + ); + return; + } + } catch { + // No usable local token yet; launch the supported Codex sign-in flow. + } + + const currentTime = now(); + if ( + lastStartedAt > 0 && + currentTime - lastStartedAt < Math.max(0, restartCooldownMs) + ) { + res.statusCode = 202; + res.end( + JSON.stringify({ available: false, started: false, pending: true }), + ); + return; + } + + try { + await startOAuthLogin(); + lastStartedAt = currentTime; + res.statusCode = 202; + res.end( + JSON.stringify({ available: false, started: true, pending: true }), + ); + } catch (error) { + res.statusCode = 503; + res.end( + JSON.stringify({ + available: false, + error: error?.message || 'Could not start ChatGPT sign-in', + }), + ); + } + }; +} function createRealtimeTokenHandler({ annotationGuidance, endpoint = 'https://api.openai.com/v1/realtime/client_secrets', fetchImpl = (...args) => fetch(...args), resolveApiKey = () => process.env.OPENAI_API_KEY, + resolveOAuthAccessToken = () => readCodexOAuthAccessToken(), models = {}, } = {}) { return async (req, res) => { @@ -30,15 +185,62 @@ function createRealtimeTokenHandler({ return; } + const query = requestQuery(req); + const authMode = resolveRealtimeAuthMode(query); + if (!authMode) { + res.statusCode = 400; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'Unsupported cloud voice auth mode' })); + return; + } + // Opt-in per-IP throttle (GEV_RATELIMIT_OPENAI_PER_MIN). No-op when unset. if (!enforceOptInRateLimit(openAiRateLimiter(), req, res)) return; - const apiKey = resolveApiKey(); - if (!apiKey) { - res.statusCode = 503; - res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' })); - return; + let bearer; + if (authMode === 'oauth') { + if (!isLoopbackRequest(req)) { + res.statusCode = 403; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: 'ChatGPT OAuth voice is available only from this machine', + }), + ); + return; + } + try { + bearer = resolveOAuthAccessToken(); + } catch (error) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: error?.message || 'ChatGPT sign-in is unavailable', + code: 'CODEX_OAUTH_UNAVAILABLE', + }), + ); + return; + } + if (!bearer) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end( + JSON.stringify({ + error: 'ChatGPT sign-in is unavailable', + code: 'CODEX_OAUTH_UNAVAILABLE', + }), + ); + return; + } + } else { + bearer = resolveApiKey(); + if (!bearer) { + res.statusCode = 503; + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' })); + return; + } } // Voice model tier, requested by the client as ?tier=standard|mini. @@ -47,15 +249,7 @@ function createRealtimeTokenHandler({ // bad querystring degrades to a normal session rather than a dead mic. // The env overrides stay authoritative per tier (see .env.example) β€” // a wrong upstream model id is then a config fix, not a code change. - const requestedTier = (() => { - try { - return new URL(req.url || '', 'http://localhost').searchParams.get( - 'tier', - ); - } catch { - return null; - } - })(); + const requestedTier = query.get('tier'); const tier = resolveVoiceModel(requestedTier).tier; const model = tier === 'mini' @@ -124,7 +318,7 @@ function createRealtimeTokenHandler({ redirect: 'error', signal: AbortSignal.timeout(30_000), headers: { - Authorization: `Bearer ${apiKey}`, + Authorization: `Bearer ${bearer}`, 'Content-Type': 'application/json', 'OpenAI-Safety-Identifier': 'gev-local-dev', }, @@ -167,4 +361,8 @@ function createRealtimeTokenHandler({ }; } -export { createRealtimeTokenHandler }; +export { + createRealtimeOAuthLoginHandler, + createRealtimeOAuthStatusHandler, + createRealtimeTokenHandler, +}; diff --git a/src/codexOauthRealtime.test.mjs b/src/codexOauthRealtime.test.mjs new file mode 100644 index 000000000..4afadb0e6 --- /dev/null +++ b/src/codexOauthRealtime.test.mjs @@ -0,0 +1,93 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import test from 'node:test'; +import path from 'node:path'; +import { + codexAuthPath, + readCodexOAuthAccessToken, + resolveCodexExecutable, + startCodexChatGptLogin, +} from '../server/providers/openai/codex-auth.js'; + +test('Codex auth path follows explicit auth file, CODEX_HOME, then home directory', () => { + assert.equal( + codexAuthPath({ + environment: { CODEX_AUTH_JSON: '/tmp/custom-auth.json' }, + home: '/home/fixture', + }), + path.resolve('/tmp/custom-auth.json'), + ); + assert.equal( + codexAuthPath({ + environment: { CODEX_HOME: '/tmp/codex-home' }, + home: '/home/fixture', + }), + path.join(path.resolve('/tmp/codex-home'), 'auth.json'), + ); + assert.equal( + codexAuthPath({ environment: {}, home: '/home/fixture' }), + path.join('/home/fixture', '.codex', 'auth.json'), + ); +}); + +test('Codex OAuth token reader returns only the access token and fails closed', () => { + assert.equal( + readCodexOAuthAccessToken({ + authPath: '/unused', + readFile: () => JSON.stringify({ tokens: { access_token: ' oauth-fixture ' } }), + }), + 'oauth-fixture', + ); + for (const readFile of [ + () => '{bad json', + () => JSON.stringify({ tokens: {} }), + () => { + throw new Error('missing'); + }, + ]) { + assert.throws( + () => readCodexOAuthAccessToken({ authPath: '/unused', readFile }), + /ChatGPT sign-in is unavailable/, + ); + } +}); + +test('Codex login resolves an installed executable before PATH and starts browser auth detached', async () => { + const checked = []; + assert.equal( + resolveCodexExecutable({ + environment: {}, + home: '/home/fixture', + access(candidate) { + checked.push(candidate); + if (candidate.endsWith('/.local/bin/codex')) return; + throw new Error('missing'); + }, + }), + path.join('/home/fixture', '.local', 'bin', 'codex'), + ); + assert.equal(checked.length, 1); + + const child = new EventEmitter(); + let unref = 0; + child.unref = () => { + unref += 1; + }; + const calls = []; + const pending = startCodexChatGptLogin({ + executable: '/fixture/codex', + home: '/home/fixture', + environment: { PATH: '/fixture' }, + spawnImpl(command, args, options) { + calls.push({ command, args, options }); + queueMicrotask(() => child.emit('spawn')); + return child; + }, + }); + assert.deepEqual(await pending, { executable: '/fixture/codex' }); + assert.equal(unref, 1); + assert.equal(calls[0].command, '/fixture/codex'); + assert.deepEqual(calls[0].args, ['login']); + assert.equal(calls[0].options.detached, true); + assert.equal(calls[0].options.stdio, 'ignore'); +}); diff --git a/src/keySetup.js b/src/keySetup.js index ee1d3b2dd..e137ff4d8 100644 --- a/src/keySetup.js +++ b/src/keySetup.js @@ -1,4 +1,8 @@ import { createSurfaceKeyboard } from './ui/surfaceKeyboard.js'; +import { + readStoredCloudVoiceAuthMode, + writeStoredCloudVoiceAuthMode, +} from './voice/cloudVoiceAuth.js'; /** * The POWER UP surface β€” paste a key, get a power. @@ -38,6 +42,46 @@ export function collectKeyUpdates(fields) { return updates; } +function sleepWithSignal(ms, signal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason || new DOMException('Aborted', 'AbortError')); + return; + } + const onAbort = () => { + clearTimeout(timer); + reject(signal.reason || new DOMException('Aborted', 'AbortError')); + }; + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, ms); + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + +export async function waitForChatGptOAuth({ + fetchImpl, + signal, + timeoutMs = 120_000, + pollMs = 1_000, + now = () => Date.now(), + sleep = sleepWithSignal, +} = {}) { + const deadline = now() + Math.max(0, timeoutMs); + while (!signal?.aborted && now() <= deadline) { + const response = await fetchImpl('/api/realtime/oauth-status', { + cache: 'no-store', + signal, + }); + const payload = await response.json().catch(() => ({})); + if (response.ok && payload.available) return true; + if (now() >= deadline) break; + await sleep(Math.max(0, pollMs), signal); + } + return false; +} + /** * After the FIRST Google key lands, the restart's reload should boot the * photoreal default β€” not faithfully restore the auto-selected keyless OSM @@ -144,6 +188,26 @@ function buildRow(documentRef, key) { } row.append(fields); } + if (key.id === 'openai') { + const auth = documentRef.createElement('div'); + auth.className = 'key-setup-cloud-auth'; + const mode = readStoredCloudVoiceAuthMode(); + const state = documentRef.createElement('span'); + state.className = 'key-setup-cloud-auth-state'; + state.textContent = + mode === 'oauth' ? 'VOICE AUTH Β· CHATGPT OAUTH' : 'VOICE AUTH Β· API KEY'; + const toggle = documentRef.createElement('button'); + toggle.type = 'button'; + toggle.className = 'key-setup-cloud-auth-toggle'; + toggle.dataset.cloudVoiceAuthToggle = 'true'; + toggle.textContent = mode === 'oauth' ? 'USE API KEY' : 'USE CHATGPT OAUTH'; + toggle.title = + mode === 'oauth' + ? 'Use OPENAI_API_KEY for the next cloud voice session' + : 'Use the signed-in local ChatGPT/Codex OAuth session for the next cloud voice session'; + auth.append(state, toggle); + row.append(auth); + } return row; } @@ -363,6 +427,86 @@ export async function initKeySetup({ applyButton?.addEventListener('click', onApply); // Remove buttons are rendered per row; delegate so re-renders stay wired. rowsHost?.addEventListener('click', (event) => { + const authButton = event.target?.closest?.( + '[data-cloud-voice-auth-toggle]', + ); + if (authButton && !disposed) { + void (async () => { + const current = readStoredCloudVoiceAuthMode(); + if (current === 'oauth') { + writeStoredCloudVoiceAuthMode('api-key'); + render(status); + say('Cloud voice will use OPENAI_API_KEY on the next session.'); + return; + } + authButton.disabled = true; + say('Checking local ChatGPT OAuth sign-in…'); + try { + const response = await doFetch('/api/realtime/oauth-status', { + cache: 'no-store', + signal: lifetime.signal, + }); + const payload = await response.json().catch(() => ({})); + if (response.ok && payload.available) { + writeStoredCloudVoiceAuthMode('oauth'); + render(status); + say( + 'ChatGPT OAuth selected for cloud voice. Your API key stays saved and available.', + ); + return; + } + + say('Opening ChatGPT sign-in in your browser…'); + const loginResponse = await doFetch('/api/realtime/oauth-login', { + method: 'POST', + cache: 'no-store', + signal: lifetime.signal, + }); + const loginPayload = await loginResponse.json().catch(() => ({})); + if (!loginResponse.ok && loginResponse.status !== 202) { + say( + loginPayload.error || + payload.error || + 'Could not start ChatGPT sign-in on this machine.', + ); + return; + } + if (loginPayload.available) { + writeStoredCloudVoiceAuthMode('oauth'); + render(status); + say( + 'ChatGPT OAuth selected for cloud voice. Your API key stays saved and available.', + ); + return; + } + + say( + 'Finish ChatGPT sign-in in the browser. Waiting for it to complete…', + ); + const available = await waitForChatGptOAuth({ + fetchImpl: doFetch, + signal: lifetime.signal, + }); + if (!available) { + say( + 'ChatGPT sign-in is still pending. Finish it in the browser, then click USE CHATGPT OAUTH again.', + ); + return; + } + writeStoredCloudVoiceAuthMode('oauth'); + render(status); + say( + 'ChatGPT sign-in complete. OAuth will be used for the next cloud voice session.', + ); + } catch (error) { + if (lifetime.signal.aborted) return; + say(`OAuth check failed: ${error?.message || error}`); + } finally { + authButton.disabled = false; + } + })(); + return; + } const button = event.target?.closest?.('[data-key-setup-remove]'); if (disposed || !button || busy) return; let envVars = []; diff --git a/src/keySetup.test.mjs b/src/keySetup.test.mjs index c0d9520c0..a03f2d556 100644 --- a/src/keySetup.test.mjs +++ b/src/keySetup.test.mjs @@ -5,6 +5,7 @@ import { keySetupChipLabel, stripKeylessBasemapFromHash, bindKeySetupPlacement, + waitForChatGptOAuth, } from './keySetup.js'; test('setup placement moves the same button only in Cyber and disconnects on teardown', () => { @@ -61,6 +62,28 @@ test('collectKeyUpdates keeps only non-empty trimmed values', () => { assert.deepEqual(collectKeyUpdates(null), {}); }); +test('OAuth login polling stops as soon as local ChatGPT auth becomes available', async () => { + let checks = 0; + let clock = 0; + const available = await waitForChatGptOAuth({ + fetchImpl: async () => { + checks += 1; + return { + ok: true, + json: async () => ({ available: checks >= 3 }), + }; + }, + timeoutMs: 10_000, + pollMs: 100, + now: () => clock, + sleep: async (ms) => { + clock += ms; + }, + }); + assert.equal(available, true); + assert.equal(checks, 3); +}); + test('the first Google key strips ONLY the keyless OSM basemap from the share hash', () => { const stripped = stripKeylessBasemapFromHash('lat=30.2&lon=-97.7&map=osm&style=normal'); assert.ok(stripped !== null); diff --git a/src/keySetupCore.mjs b/src/keySetupCore.mjs index e8bfef92c..a23c9825c 100644 --- a/src/keySetupCore.mjs +++ b/src/keySetupCore.mjs @@ -52,7 +52,7 @@ export const KEY_SETUP_KEYS = Object.freeze([ Object.freeze({ id: 'openai', title: 'OPENAI', - unlocks: 'Voice control β€” talk to the planet', + unlocks: 'Voice control β€” API key or ChatGPT OAuth', getUrl: 'https://platform.openai.com/api-keys', envVars: Object.freeze(['OPENAI_API_KEY']), tier: 'metered', diff --git a/src/setupDoctor.test.mjs b/src/setupDoctor.test.mjs index c66d9e686..aa825f126 100644 --- a/src/setupDoctor.test.mjs +++ b/src/setupDoctor.test.mjs @@ -224,6 +224,13 @@ test('doctor describes the credential ladder without exposing values', () => { assert.match(capabilities.map, /Google Photorealistic 3D Tiles through Cesium ion/); assert.match(capabilities.map, /Bing and world-terrain stacks/); assert.equal(capabilities.voice, 'available'); + assert.equal( + buildCapabilitySummary({ + ...credentials, + OPENAI_API_KEY: { configured: false }, + }).voice, + 'ChatGPT/Codex OAuth can be selected at runtime', + ); assert.match(capabilities.missions, /token allowance/); assert.equal(capabilities.flights, 'OpenSky keyless anonymous access (rate-limited)'); diff --git a/src/tooling/localServices.test.mjs b/src/tooling/localServices.test.mjs index 26eebfa08..7014659b7 100644 --- a/src/tooling/localServices.test.mjs +++ b/src/tooling/localServices.test.mjs @@ -37,6 +37,7 @@ function request( url = '/', body = '', origin = 'http://localhost:4173', + remoteAddress = '127.0.0.1', } = {}, ) { return new Promise((resolve, reject) => { @@ -49,7 +50,7 @@ function request( origin, 'content-type': 'application/json', }, - socket: { remoteAddress: '127.0.0.1' }, + socket: { remoteAddress }, }); const headers = {}; const res = { @@ -195,6 +196,147 @@ test('Realtime handler preserves tools and default instructions, isolates suppli assert.equal(sent[0].session.instructions, sent[2].session.instructions); }); +test('Realtime handler keeps API key as default and uses OAuth only when requested', async (t) => { + env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined); + let oauthReads = 0; + const authorizations = []; + const routes = install( + openAiRealtimeProxy({ + realtime: { + resolveApiKey: () => 'fixture-api-key', + resolveOAuthAccessToken: () => { + oauthReads += 1; + return 'fixture-oauth-token'; + }, + fetchImpl: async (_url, options) => { + authorizations.push(options.headers.Authorization); + return Response.json({ value: 'fixture-ephemeral' }); + }, + }, + }), + ); + const token = routes.get('/api/realtime/token'); + + assert.equal((await request(token)).status, 200); + assert.equal(oauthReads, 0, 'default cloud voice does not read OAuth'); + assert.equal(authorizations.at(-1), 'Bearer fixture-api-key'); + + assert.equal((await request(token, { url: '/?auth=oauth' })).status, 200); + assert.equal(oauthReads, 1); + assert.equal(authorizations.at(-1), 'Bearer fixture-oauth-token'); +}); + +test('OAuth status and token minting never expose local OAuth to remote clients', async () => { + let oauthReads = 0; + const routes = install( + openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => { + oauthReads += 1; + return 'must-not-be-read-remotely'; + }, + fetchImpl: () => + assert.fail('remote OAuth request must not reach OpenAI'), + }, + }), + ); + const remote = { remoteAddress: '192.168.1.20' }; + assert.equal( + (await request(routes.get('/api/realtime/oauth-status'), remote)).status, + 403, + ); + assert.equal( + ( + await request(routes.get('/api/realtime/token'), { + ...remote, + url: '/?auth=oauth', + }) + ).status, + 403, + ); + assert.equal(oauthReads, 0); +}); + +test('OAuth status reports local sign-in availability without returning the token', async () => { + const routes = install( + openAiRealtimeProxy({ + realtime: { resolveOAuthAccessToken: () => 'fixture-oauth-secret' }, + }), + ); + const response = await request(routes.get('/api/realtime/oauth-status')); + assert.equal(response.status, 200); + assert.deepEqual(response.json(), { available: true }); + assert.equal(response.body.includes('fixture-oauth-secret'), false); +}); + +test('OAuth login endpoint launches local Codex sign-in once and never exposes credentials', async () => { + let starts = 0; + let clock = 1_000; + const routes = install( + openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => { + throw new Error('not signed in'); + }, + startOAuthLogin: async () => { + starts += 1; + }, + now: () => clock, + }, + }), + ); + const handler = routes.get('/api/realtime/oauth-login'); + + const started = await request(handler, { method: 'POST' }); + assert.equal(started.status, 202); + assert.deepEqual(started.json(), { + available: false, + started: true, + pending: true, + }); + assert.equal(starts, 1); + + clock += 1_000; + const duplicate = await request(handler, { method: 'POST' }); + assert.equal(duplicate.status, 202); + assert.equal(duplicate.json().pending, true); + assert.equal(starts, 1, 'rapid retries must not spawn another login'); + + const remote = await request(handler, { + method: 'POST', + remoteAddress: '192.168.1.20', + }); + assert.equal(remote.status, 403); + assert.equal(starts, 1); + + const crossOrigin = await request(handler, { + method: 'POST', + origin: 'https://example.com', + }); + assert.equal(crossOrigin.status, 403); + assert.equal(starts, 1); +}); + +test('OAuth login endpoint selects an existing local sign-in without launching Codex', async () => { + const handler = install( + openAiRealtimeProxy({ + realtime: { + resolveOAuthAccessToken: () => 'fixture-existing-secret', + startOAuthLogin: () => + assert.fail('existing auth must not launch login'), + }, + }), + ).get('/api/realtime/oauth-login'); + const response = await request(handler, { method: 'POST' }); + assert.equal(response.status, 200); + assert.deepEqual(response.json(), { + available: true, + started: false, + pending: false, + }); + assert.equal(response.body.includes('fixture-existing-secret'), false); +}); + test('debug logging resolves each supplied application directory independently', async (t) => { const first = root(t), second = root(t); diff --git a/src/ui/styles/provider-settings.css b/src/ui/styles/provider-settings.css index b68593990..043e046d6 100644 --- a/src/ui/styles/provider-settings.css +++ b/src/ui/styles/provider-settings.css @@ -300,6 +300,43 @@ color: rgba(148, 178, 188, 0.42); } +.key-setup-cloud-auth { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.key-setup-cloud-auth-state { + color: rgba(190, 214, 222, 0.7); + font-family: var(--font-mono); + font-size: 0.56rem; + letter-spacing: 0.05rem; +} + +.key-setup-cloud-auth-toggle { + margin-left: auto; + padding: 0.28rem 0.55rem; + color: rgba(54, 220, 255, 0.9); + font-family: var(--font-mono); + font-size: 0.55rem; + letter-spacing: 0.06rem; + background: rgba(54, 220, 255, 0.08); + border: 1px solid rgba(54, 220, 255, 0.28); + border-radius: 0.32rem; + cursor: pointer; +} + +.key-setup-cloud-auth-toggle:hover, +.key-setup-cloud-auth-toggle:focus-visible { + border-color: rgba(54, 220, 255, 0.65); + outline: none; +} + +.key-setup-cloud-auth-toggle:disabled { + cursor: wait; + opacity: 0.5; +} + .key-setup-footer { margin-top: 0.85rem; } diff --git a/src/voice/cloudVoiceAuth.js b/src/voice/cloudVoiceAuth.js new file mode 100644 index 000000000..77feae8aa --- /dev/null +++ b/src/voice/cloudVoiceAuth.js @@ -0,0 +1,41 @@ +export const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']); +export const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key'; +export const CLOUD_VOICE_AUTH_STORAGE_KEY = 'godsEyeView.voice.cloudAuth'; + +function storageRef(storage) { + if (storage) return storage; + try { + return typeof localStorage === 'undefined' ? null : localStorage; + } catch { + return null; + } +} + +export function normalizeCloudVoiceAuthMode(mode) { + const normalized = String(mode || '') + .trim() + .toLowerCase(); + return CLOUD_VOICE_AUTH_MODES.includes(normalized) + ? normalized + : DEFAULT_CLOUD_VOICE_AUTH_MODE; +} + +export function readStoredCloudVoiceAuthMode(storage) { + try { + return normalizeCloudVoiceAuthMode( + storageRef(storage)?.getItem(CLOUD_VOICE_AUTH_STORAGE_KEY), + ); + } catch { + return DEFAULT_CLOUD_VOICE_AUTH_MODE; + } +} + +export function writeStoredCloudVoiceAuthMode(mode, storage) { + const resolved = normalizeCloudVoiceAuthMode(mode); + try { + storageRef(storage)?.setItem(CLOUD_VOICE_AUTH_STORAGE_KEY, resolved); + } catch { + /* best effort */ + } + return resolved; +} diff --git a/src/voice/cloudVoiceAuth.test.mjs b/src/voice/cloudVoiceAuth.test.mjs new file mode 100644 index 000000000..c7a4c2e4d --- /dev/null +++ b/src/voice/cloudVoiceAuth.test.mjs @@ -0,0 +1,29 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + CLOUD_VOICE_AUTH_STORAGE_KEY, + readStoredCloudVoiceAuthMode, + writeStoredCloudVoiceAuthMode, +} from './cloudVoiceAuth.js'; + +function memoryStorage(seed = {}) { + const values = new Map(Object.entries(seed)); + return { + getItem: (key) => values.get(key) ?? null, + setItem: (key, value) => values.set(key, value), + }; +} + +test('cloud voice auth defaults to API key and persists OAuth independently', () => { + const storage = memoryStorage(); + assert.equal(readStoredCloudVoiceAuthMode(storage), 'api-key'); + assert.equal(writeStoredCloudVoiceAuthMode('oauth', storage), 'oauth'); + assert.equal(readStoredCloudVoiceAuthMode(storage), 'oauth'); + assert.equal(writeStoredCloudVoiceAuthMode('unknown', storage), 'api-key'); + assert.equal( + readStoredCloudVoiceAuthMode( + memoryStorage({ [CLOUD_VOICE_AUTH_STORAGE_KEY]: 'oauth' }), + ), + 'oauth', + ); +}); diff --git a/src/voice/gevRealtime.test.mjs b/src/voice/gevRealtime.test.mjs index d7056a0e3..66e1b1367 100644 --- a/src/voice/gevRealtime.test.mjs +++ b/src/voice/gevRealtime.test.mjs @@ -3194,6 +3194,20 @@ test('F1: the toggle still records the next-session preference while live', () = assert.match(ui.tierButton.title, /this session stays on/i); }); +test('OAuth cloud sessions hide API spend and do not trip the API spend guard', () => { + const { controller, ui } = costControllerHarness(); + controller.status = 'listening'; + controller._cost.cloudVoiceAuth = 'oauth'; + controller._cost.sessionCloudVoiceAuth = 'oauth'; + controller.syncCostUi(); + const before = controller.costTracker.state(); + assert.equal(ui.costValue.hidden, true); + assert.equal(controller.recordUsage(usdUsage(100)), null); + const after = controller.costTracker.state(); + assert.equal(after.totalUsd, before.totalUsd); + assert.equal(controller.costCapStopped, false); +}); + test('F1: when idle, toggling does re-price the preview meter', () => { const { controller } = costControllerHarness(); controller.status = 'idle'; diff --git a/src/voice/realtimeBackend.js b/src/voice/realtimeBackend.js index 2a8af2d0d..b429e17c5 100644 --- a/src/voice/realtimeBackend.js +++ b/src/voice/realtimeBackend.js @@ -15,11 +15,19 @@ export function createRealtimeBackend({ ); return Object.freeze({ protocol: 'openai-realtime', - async requestToken({ tier = DEFAULT_VOICE_TIER, signal } = {}) { + async requestToken({ + tier = DEFAULT_VOICE_TIER, + authMode = 'api-key', + signal, + } = {}) { signal = scoped(signal); signal.throwIfAborted(); const separator = tokenEndpoint.includes('?') ? '&' : '?'; - const url = `${tokenEndpoint}${separator}tier=${encodeURIComponent(resolveVoiceModel(tier).tier)}`; + const query = new URLSearchParams({ + tier: resolveVoiceModel(tier).tier, + }); + if (authMode === 'oauth') query.set('auth', 'oauth'); + const url = `${tokenEndpoint}${separator}${query}`; const response = await tokenTransport(url, { signal, cache: 'no-store', diff --git a/src/voice/realtimeBackend.test.mjs b/src/voice/realtimeBackend.test.mjs index 1fc21da9d..d179f5b6b 100644 --- a/src/voice/realtimeBackend.test.mjs +++ b/src/voice/realtimeBackend.test.mjs @@ -41,6 +41,22 @@ test('separate configured token and SDP transports preserve model metadata and c ]); }); +test('OAuth is an additive token query while API key stays the default', async () => { + const calls = []; + const backend = createRealtimeBackend({ + tokenTransport: async (input) => { + calls.push(input); + return tokenReply(); + }, + }); + await backend.requestToken(); + await backend.requestToken({ authMode: 'oauth' }); + assert.deepEqual(calls, [ + '/api/realtime/token?tier=standard', + '/api/realtime/token?tier=standard&auth=oauth', + ]); +}); + test('denied tokens never negotiate or retry; reconnect requests a fresh token', async () => { let requests = 0; const backend = createRealtimeBackend({ diff --git a/src/voice/realtimeConnection.js b/src/voice/realtimeConnection.js index b20afb67d..b0409eb92 100644 --- a/src/voice/realtimeConnection.js +++ b/src/voice/realtimeConnection.js @@ -82,6 +82,7 @@ export class RealtimeConnection { this.debugLog('session.starting', { epoch, tier: this.cost.voiceTier, + authMode: this.cost.sessionCloudVoiceAuth, connection: this.connectionDiagnostics(), }); let localStream = null; @@ -89,6 +90,7 @@ export class RealtimeConnection { try { const minted = await this.backend.requestToken({ tier: this.cost.voiceTier, + authMode: this.cost.sessionCloudVoiceAuth, signal, }); const token = minted.token; diff --git a/src/voice/realtimeCost.js b/src/voice/realtimeCost.js index 9c1169490..83d19785e 100644 --- a/src/voice/realtimeCost.js +++ b/src/voice/realtimeCost.js @@ -4,6 +4,7 @@ import { writeStoredVoiceTier, writeStoredVoiceLimits, } from './realtimePreferences.js'; +import { readStoredCloudVoiceAuthMode } from './cloudVoiceAuth.js'; import { createVoiceCostTracker, resolveVoiceModel, @@ -16,6 +17,8 @@ export class RealtimeCost { Object.assign(this, { readUi, readStatus }, operations); this.voiceTier = readStoredVoiceTier(); this.voiceLimits = readStoredVoiceLimits(); + this.cloudVoiceAuth = readStoredCloudVoiceAuthMode(); + this.sessionCloudVoiceAuth = null; this.costTracker = createVoiceCostTracker({ tier: this.voiceTier, limits: this.voiceLimits, @@ -56,6 +59,8 @@ export class RealtimeCost { }; applies next session`; } if (this.ui?.costValue) { + const liveAuth = this.sessionCloudVoiceAuth || this.cloudVoiceAuth; + this.ui.costValue.hidden = liveAuth === 'oauth'; this.ui.costValue.textContent = state.display; this.ui.costValue.dataset.level = state.level; this.ui.costValue.title = @@ -136,6 +141,7 @@ export class RealtimeCost { */ recordUsage(usage) { if (!usage) return null; + if (this.sessionCloudVoiceAuth === 'oauth') return null; const state = this.costTracker.record(usage); this.syncCostUi(); if (state.warnCrossed) { @@ -190,6 +196,8 @@ export class RealtimeCost { prepareSession() { this.voiceTier = readStoredVoiceTier(); this.voiceLimits = readStoredVoiceLimits(); + this.cloudVoiceAuth = readStoredCloudVoiceAuthMode(); + this.sessionCloudVoiceAuth = this.cloudVoiceAuth; this.costCapStopped = false; // Provisional meter (tier-priced) so the readout shows $0.00 while // connecting. It is REPLACED below with one bound to the model the server