From 09d02b6e7af9d40f67d3720569b25d43e9b5a8f4 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 6 Nov 2026 00:52:49 +0000 Subject: [PATCH 1/4] Keyless voice: say where to add the OpenAI key instead of blaming the microphone --- server/providers/openai/realtime.js | 7 ++++++- src/ui/views/panels/VoiceControl.vue | 2 +- src/voice/control.js | 2 +- src/voice/realtimeBackend.js | 5 +++-- src/voice/realtimeBackend.test.mjs | 14 ++++++++++++++ src/voice/realtimeController.js | 5 ++++- src/voice/realtimeDiagnostics.js | 11 +++++++++++ 7 files changed, 40 insertions(+), 6 deletions(-) diff --git a/server/providers/openai/realtime.js b/server/providers/openai/realtime.js index 4412eb331..f88171e4a 100644 --- a/server/providers/openai/realtime.js +++ b/server/providers/openai/realtime.js @@ -238,7 +238,12 @@ function createRealtimeTokenHandler({ if (!bearer) { res.statusCode = 503; res.setHeader('Content-Type', 'application/json'); - res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' })); + res.end( + JSON.stringify({ + error: 'OPENAI_API_KEY is not set', + code: 'API_KEY_MISSING', + }), + ); return; } } diff --git a/src/ui/views/panels/VoiceControl.vue b/src/ui/views/panels/VoiceControl.vue index 8c1144e52..172077425 100644 --- a/src/ui/views/panels/VoiceControl.vue +++ b/src/ui/views/panels/VoiceControl.vue @@ -166,7 +166,7 @@ const frameStyle = (panel) => ({
{{ view.error.text }}
-
{{ t('Check microphone permission and network access, then try again.') }}
+
{{ t('Check microphone permission and network access, then try again.') }}
diff --git a/src/voice/control.js b/src/voice/control.js index 61da65bfc..58cf08f2f 100644 --- a/src/voice/control.js +++ b/src/voice/control.js @@ -107,7 +107,7 @@ function initialState(panel) { t('Hold Space to speak · tap Space to activate focused controls'), }, // A typed chat error reads as chat's, without the microphone hint. - error: { text: '', typed: false }, + error: { text: '', typed: false, hint: true }, // Connection and turn state beside the selected mode's indicator. phase: { text: '' }, voicePanel: { diff --git a/src/voice/realtimeBackend.js b/src/voice/realtimeBackend.js index ca38432ee..a9c10f2af 100644 --- a/src/voice/realtimeBackend.js +++ b/src/voice/realtimeBackend.js @@ -39,8 +39,9 @@ export function createRealtimeBackend({ if (!response.ok) { const reason = typeof data?.error === 'string' ? data.error : data?.error?.message; - throw new Error( - reason || `Realtime token failed: HTTP ${response.status}`, + throw Object.assign( + new Error(reason || `Realtime token failed: HTTP ${response.status}`), + typeof data?.code === 'string' ? { code: data.code } : {}, ); } const token = diff --git a/src/voice/realtimeBackend.test.mjs b/src/voice/realtimeBackend.test.mjs index ac96e2374..b60bf8406 100644 --- a/src/voice/realtimeBackend.test.mjs +++ b/src/voice/realtimeBackend.test.mjs @@ -143,3 +143,17 @@ test('controller lifetime stops pending transport and releases resources through assert.equal(controller.stream, null); assert.equal(controller.isActive(), false); }); + +test('a missing server key keeps its code, so the UI can point to Provider Settings', async () => { + const backend = createRealtimeBackend({ + tokenTransport: async () => + Response.json( + { error: 'OPENAI_API_KEY is not set', code: 'API_KEY_MISSING' }, + { status: 503 }, + ), + }); + await assert.rejects(backend.requestToken(), { + message: 'OPENAI_API_KEY is not set', + code: 'API_KEY_MISSING', + }); +}); diff --git a/src/voice/realtimeController.js b/src/voice/realtimeController.js index e5f2efd60..a7dd07021 100644 --- a/src/voice/realtimeController.js +++ b/src/voice/realtimeController.js @@ -18,7 +18,7 @@ import { } from './pointerContext.js'; import { shouldPauseRadioForVoice } from './realtimeProtocol.js'; -import { postDebugLog } from './realtimeDiagnostics.js'; +import { missingVoiceKeyText, postDebugLog } from './realtimeDiagnostics.js'; export { readStoredVoiceTier, @@ -437,6 +437,9 @@ export class GevRealtimeController extends RealtimeFacade { status === 'error' ? resolvedDetail() || t('Voice session could not be started.') : '', + // "Check microphone permission" is wrong advice for a missing key. + hint: () => + !(status === 'error' && resolvedDetail() === missingVoiceKeyText()), }); if (status === 'idle' || status === 'connecting' || status === 'error') { this.setVoiceSpeaker('idle'); diff --git a/src/voice/realtimeDiagnostics.js b/src/voice/realtimeDiagnostics.js index 55e078280..dcdacf08e 100644 --- a/src/voice/realtimeDiagnostics.js +++ b/src/voice/realtimeDiagnostics.js @@ -1,3 +1,11 @@ +import { t } from '../i18n/index.js'; + +/** The voice error when the server has no OpenAI key: where to add it. */ +export const missingVoiceKeyText = () => + t('Needs {envVars} — add it in Provider Settings', { + envVars: 'OPENAI_API_KEY', + }); + const ERROR_LOG_LIMIT = 30; const ERROR_STORAGE_KEY = 'gev-realtime-errors'; @@ -83,6 +91,7 @@ function createErrorRecord(source, error, extra = {}) { String(error?.message || '').trim() || 'No browser error message supplied', errorDetail: rtcError?.errorDetail || null, + code: typeof rtcError?.code === 'string' ? rtcError.code : null, sctpCauseCode: rtcError?.sctpCauseCode ?? null, receivedAlert: rtcError?.receivedAlert ?? null, sentAlert: rtcError?.sentAlert ?? null, @@ -91,6 +100,8 @@ function createErrorRecord(source, error, extra = {}) { } function formatErrorForDisplay(record) { + // A missing key is a setup step, not a connection fault: say where to add it. + if (record.code === 'API_KEY_MISSING') return missingVoiceKeyText(); const primary = [record.source, record.message].filter(Boolean).join(': '); const state = [ record.errorDetail && `detail=${record.errorDetail}`, From c03a54de40ceff918b9eb04f84f0dc68bd371612 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 6 Nov 2026 11:55:31 +0000 Subject: [PATCH 2/4] Tests: pin the API_KEY_MISSING code on the keyless voice token response --- src/hudSummaryResponse.test.mjs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hudSummaryResponse.test.mjs b/src/hudSummaryResponse.test.mjs index 097d7133d..6effd2eee 100644 --- a/src/hudSummaryResponse.test.mjs +++ b/src/hudSummaryResponse.test.mjs @@ -150,7 +150,10 @@ test('the installed keyless HUD route stays successful after the voice quota is const firstToken = await invokeRoute(token); const secondToken = await invokeRoute(token); assert.equal(firstToken.statusCode, 503); - assert.deepEqual(firstToken.body, { error: 'OPENAI_API_KEY is not set' }); + assert.deepEqual(firstToken.body, { + error: 'OPENAI_API_KEY is not set', + code: 'API_KEY_MISSING', + }); assert.equal(secondToken.statusCode, 429); for (let attempt = 0; attempt < 2; attempt += 1) { From 0687e0e3dae9fd265c11e4802da465b96fa138c3 Mon Sep 17 00:00:00 2001 From: Said Turksever Date: Fri, 6 Nov 2026 11:59:28 +0000 Subject: [PATCH 3/4] Changelog: 1599 fragment --- changelog.d/1599-keyless-voice-key-hint.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/1599-keyless-voice-key-hint.md diff --git a/changelog.d/1599-keyless-voice-key-hint.md b/changelog.d/1599-keyless-voice-key-hint.md new file mode 100644 index 000000000..dfa2c5bb1 --- /dev/null +++ b/changelog.d/1599-keyless-voice-key-hint.md @@ -0,0 +1,4 @@ +- Without `OPENAI_API_KEY`, the mic now says the key is missing and to add it in + Provider Settings, instead of showing the raw server error with a hint to + check microphone permissions. `/api/realtime/token` returns + `code: "API_KEY_MISSING"` with its 503 (asturksever, #1599). From f86b8954f664a052bcbd36c275cce86369418223 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Tue, 10 Nov 2026 16:01:50 -0800 Subject: [PATCH 4/4] fix(voice): a missing-key error keeps its wording and hides the microphone hint after a language change (#1599) The voice error was matched against the translated key message to decide whether to show the microphone hint, so switching language brought the hint back and left the message in the old language. The error now carries a missing-key flag into the status, which words the message at render time. A controller test covers the language switch, and CURRENT-STATE.md documents the keyless voice response. --- docs/CURRENT-STATE.md | 2 +- src/voice/gevRealtime.test.mjs | 30 ++++++++++++++++++++++++++++++ src/voice/realtimeController.js | 16 ++++++++++++---- src/voice/realtimeDiagnostics.js | 6 +++++- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index b04d4cf2e..d9721b2a3 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -6269,7 +6269,7 @@ and unreachable upstream (502/504) separately from road geometry. `GEV MIC` button (bottom UI) starts an OpenAI Realtime session over WebRTC: -- **Token flow**: browser fetches a short-lived client secret from `/api/realtime/token`; the Vite middleware uses `OPENAI_API_KEY` by default, or the signed-in local ChatGPT/Codex OAuth access token when OAuth was explicitly selected in Provider Settings, and posts the full session config (instructions, tool schemas, VAD, truncation) to `api.openai.com/v1/realtime/client_secrets`. Both long-lived credentials stay server-side; OAuth minting is loopback-only. SDP exchange goes directly to `api.openai.com/v1/realtime/calls` with the ephemeral token. +- **Token flow**: browser fetches a short-lived client secret from `/api/realtime/token`; the Vite middleware uses `OPENAI_API_KEY` by default, or the signed-in local ChatGPT/Codex OAuth access token when OAuth was explicitly selected in Provider Settings, and posts the full session config (instructions, tool schemas, VAD, truncation) to `api.openai.com/v1/realtime/client_secrets`. Both long-lived credentials stay server-side; OAuth minting is loopback-only. SDP exchange goes directly to `api.openai.com/v1/realtime/calls` with the ephemeral token. With no OpenAI key the token route answers `503` with code `API_KEY_MISSING`, and the voice control shows `Needs OPENAI_API_KEY — add it in Provider Settings` in the interface language, reworded on a language change, without the microphone-permission hint. - **Session defaults** (env-tunable): model `gpt-realtime-2` (or `gpt-realtime-2.1-mini` when the MINI tier is selected — see the model-tier entry below), voice `marin`, reasoning effort `low`, semantic VAD with low eagerness, no response interruption, context window truncated to ~3,000 post-instruction tokens with 0.5 retention ratio — the conversational window stays short because map state is fetched live per turn. - **Thirty tools** (argument schemas in `src/voice/actionSchemas.js`, served with their descriptions by `server/providers/openai/tools.js`, executed client-side in `src/voice/gevActions.js`): `fly_to_location`, `select_nearest_aircraft`, `adjust_camera_zoom`, `zoom_to_globe`, `set_layer_visibility`, `show_data_layers_menu`, `set_panel_open`, `set_visual_style`, `get_entity_context`, `get_current_view_state`, `set_hud`, `set_cyber_sonar`, `set_detection`, `set_map_stack`, `set_post_processing`, `control_scene`, `control_cctv`, `set_context_mode`, `control_cockpit`, `control_radio`, `track_entity`, `stop_tracking`, `frame_overhead`, `annotate_map`, `clear_annotations`, `move_camera`, `fly_route`, `analyst_query`, `next_iss_pass`, and `next_satellite_pass`. diff --git a/src/voice/gevRealtime.test.mjs b/src/voice/gevRealtime.test.mjs index 859932e08..426a15f5c 100644 --- a/src/voice/gevRealtime.test.mjs +++ b/src/voice/gevRealtime.test.mjs @@ -35,6 +35,7 @@ import { } from './gevRealtime.js'; import { createVoiceCostTracker } from './voiceCost.js'; import { createTestVoiceUi } from '../testSupport/voiceUi.mjs'; +import { setLocale } from '../i18n/index.js'; test('push-to-talk recognizes Space by code or key', () => { assert.equal(PUSH_TO_TALK_HOLD_DELAY_MS, 500); @@ -3734,3 +3735,32 @@ test('the controller forwards viewport escalation diagnostics to its debug log', }], ); }); + +test('a missing server key says where to add it, without the microphone hint, in every language', () => { + const ui = createTestVoiceUi(); + const controller = new GevRealtimeController({ + ui, + runner: async () => ({ ok: true }), + radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} }, + }); + try { + controller.reportError( + 'Realtime connection', + Object.assign(new Error('OPENAI_API_KEY is not set'), { code: 'API_KEY_MISSING' }), + ); + assert.equal(ui.state.error.text, 'Needs OPENAI_API_KEY — add it in Provider Settings'); + assert.equal(ui.state.error.hint, false); + + setLocale('es'); + assert.notEqual(ui.state.error.text, 'Needs OPENAI_API_KEY — add it in Provider Settings'); + assert.match(ui.state.error.text, /OPENAI_API_KEY/); + assert.equal(ui.state.error.hint, false); + setLocale('en'); + + controller.reportError('Realtime connection', new Error('ICE failed')); + assert.match(ui.state.error.text, /ICE failed/); + assert.equal(ui.state.error.hint, true); + } finally { + setLocale('en'); + } +}); diff --git a/src/voice/realtimeController.js b/src/voice/realtimeController.js index a7dd07021..56f501f95 100644 --- a/src/voice/realtimeController.js +++ b/src/voice/realtimeController.js @@ -407,7 +407,14 @@ export class GevRealtimeController extends RealtimeFacade { } } - setStatus(status, detail) { + /** + * @param {string} status + * @param {string} [detail] + * @param {{missingKey?: boolean}} [options] True when the error is a + * missing server key: it is worded in the current language and shown + * without the microphone hint. + */ + setStatus(status, detail, { missingKey = false } = {}) { this.status = status; this.emitSessionEvent({ type: 'state', state: status, detail }); this.ui.update('root', { @@ -435,11 +442,12 @@ export class GevRealtimeController extends RealtimeFacade { this.ui.update('error', { text: () => status === 'error' - ? resolvedDetail() || t('Voice session could not be started.') + ? (missingKey && missingVoiceKeyText()) || + resolvedDetail() || + t('Voice session could not be started.') : '', // "Check microphone permission" is wrong advice for a missing key. - hint: () => - !(status === 'error' && resolvedDetail() === missingVoiceKeyText()), + hint: () => !(status === 'error' && missingKey), }); if (status === 'idle' || status === 'connecting' || status === 'error') { this.setVoiceSpeaker('idle'); diff --git a/src/voice/realtimeDiagnostics.js b/src/voice/realtimeDiagnostics.js index dcdacf08e..6efa71dfc 100644 --- a/src/voice/realtimeDiagnostics.js +++ b/src/voice/realtimeDiagnostics.js @@ -183,7 +183,11 @@ export class RealtimeDiagnostics { storeErrors(this.errors); console.error('[GEV Realtime]', record); this.debugLog('error', record); - this.setStatus('error', formatErrorForDisplay(record)); + this.setStatus( + 'error', + formatErrorForDisplay(record), + record.code === 'API_KEY_MISSING' ? { missingKey: true } : undefined, + ); return record; }