Merge pull request #1599 from bilawalsidhu/fix/keyless-voice-key-hint

fix(voice): keyless mic says where to add the OpenAI key

changelog.d/1599-keyless-voice-key-hint.md

@@ -0,0 +1,4 @@
+- Without `OPENAI_API_KEY`, the mic now says the key is missing and to add it in
+  Provider Settings, instead of showing the raw server error with a hint to
+  check microphone permissions. `/api/realtime/token` returns
+  `code: "API_KEY_MISSING"` with its 503 (asturksever, #1599).

docs/CURRENT-STATE.md

@@ -6269,7 +6269,7 @@ and unreachable upstream (502/504) separately from road geometry.
 
 `GEV MIC` button (bottom UI) starts an OpenAI Realtime session over WebRTC:
 
-- **Token flow**: browser fetches a short-lived client secret from `/api/realtime/token`; the Vite middleware uses `OPENAI_API_KEY` by default, or the signed-in local ChatGPT/Codex OAuth access token when OAuth was explicitly selected in Provider Settings, and posts the full session config (instructions, tool schemas, VAD, truncation) to `api.openai.com/v1/realtime/client_secrets`. Both long-lived credentials stay server-side; OAuth minting is loopback-only. SDP exchange goes directly to `api.openai.com/v1/realtime/calls` with the ephemeral token.
+- **Token flow**: browser fetches a short-lived client secret from `/api/realtime/token`; the Vite middleware uses `OPENAI_API_KEY` by default, or the signed-in local ChatGPT/Codex OAuth access token when OAuth was explicitly selected in Provider Settings, and posts the full session config (instructions, tool schemas, VAD, truncation) to `api.openai.com/v1/realtime/client_secrets`. Both long-lived credentials stay server-side; OAuth minting is loopback-only. SDP exchange goes directly to `api.openai.com/v1/realtime/calls` with the ephemeral token. With no OpenAI key the token route answers `503` with code `API_KEY_MISSING`, and the voice control shows `Needs OPENAI_API_KEY — add it in Provider Settings` in the interface language, reworded on a language change, without the microphone-permission hint.
 - **Session defaults** (env-tunable): model `gpt-realtime-2` (or `gpt-realtime-2.1-mini` when the MINI tier is selected — see the model-tier entry below), voice `marin`, reasoning effort `low`, semantic VAD with low eagerness, no response interruption, context window truncated to ~3,000 post-instruction tokens with 0.5 retention ratio — the conversational window stays short because map state is fetched live per turn.
 - **Thirty tools** (argument schemas in `src/voice/actionSchemas.js`, served with their descriptions by `server/providers/openai/tools.js`, executed client-side in `src/voice/gevActions.js`): `fly_to_location`, `select_nearest_aircraft`, `adjust_camera_zoom`, `zoom_to_globe`, `set_layer_visibility`, `show_data_layers_menu`, `set_panel_open`, `set_visual_style`, `get_entity_context`, `get_current_view_state`, `set_hud`, `set_cyber_sonar`, `set_detection`, `set_map_stack`, `set_post_processing`, `control_scene`, `control_cctv`, `set_context_mode`, `control_cockpit`, `control_radio`, `track_entity`, `stop_tracking`, `frame_overhead`, `annotate_map`, `clear_annotations`, `move_camera`, `fly_route`, `analyst_query`, `next_iss_pass`, and `next_satellite_pass`.
 

server/providers/openai/realtime.js

@@ -238,7 +238,12 @@ function createRealtimeTokenHandler({
       if (!bearer) {
         res.statusCode = 503;
         res.setHeader('Content-Type', 'application/json');
-        res.end(JSON.stringify({ error: 'OPENAI_API_KEY is not set' }));
+        res.end(
+          JSON.stringify({
+            error: 'OPENAI_API_KEY is not set',
+            code: 'API_KEY_MISSING',
+          }),
+        );
         return;
       }
     }

src/hudSummaryResponse.test.mjs

@@ -150,7 +150,10 @@ test('the installed keyless HUD route stays successful after the voice quota is
     const firstToken = await invokeRoute(token);
     const secondToken = await invokeRoute(token);
     assert.equal(firstToken.statusCode, 503);
-    assert.deepEqual(firstToken.body, { error: 'OPENAI_API_KEY is not set' });
+    assert.deepEqual(firstToken.body, {
+      error: 'OPENAI_API_KEY is not set',
+      code: 'API_KEY_MISSING',
+    });
     assert.equal(secondToken.statusCode, 429);
 
     for (let attempt = 0; attempt < 2; attempt += 1) {

src/ui/views/panels/VoiceControl.vue

@@ -166,7 +166,7 @@ const frameStyle = (panel) => ({
           <button class="gev-voice-error-dismiss" type="button" @click="controls?.dismissError()">{{ t('DISMISS') }}</button>
         </div>
         <div id="gev-voice-error-detail">{{ view.error.text }}</div>
-        <div class="gev-voice-error-hint" :hidden="view.error.typed">{{ t('Check microphone permission and network access, then try again.') }}</div>
+        <div class="gev-voice-error-hint" :hidden="view.error.typed || view.error.hint === false">{{ t('Check microphone permission and network access, then try again.') }}</div>
       </div>
   </div>
 </template>

src/voice/control.js

@@ -107,7 +107,7 @@ function initialState(panel) {
         t('Hold Space to speak · tap Space to activate focused controls'),
     },
     // A typed chat error reads as chat's, without the microphone hint.
-    error: { text: '', typed: false },
+    error: { text: '', typed: false, hint: true },
     // Connection and turn state beside the selected mode's indicator.
     phase: { text: '' },
     voicePanel: {

src/voice/gevRealtime.test.mjs

@@ -35,6 +35,7 @@ import {
 } from './gevRealtime.js';
 import { createVoiceCostTracker } from './voiceCost.js';
 import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
+import { setLocale } from '../i18n/index.js';
 
 test('push-to-talk recognizes Space by code or key', () => {
   assert.equal(PUSH_TO_TALK_HOLD_DELAY_MS, 500);
@@ -3734,3 +3735,32 @@ test('the controller forwards viewport escalation diagnostics to its debug log',
     }],
   );
 });
+
+test('a missing server key says where to add it, without the microphone hint, in every language', () => {
+  const ui = createTestVoiceUi();
+  const controller = new GevRealtimeController({
+    ui,
+    runner: async () => ({ ok: true }),
+    radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} },
+  });
+  try {
+    controller.reportError(
+      'Realtime connection',
+      Object.assign(new Error('OPENAI_API_KEY is not set'), { code: 'API_KEY_MISSING' }),
+    );
+    assert.equal(ui.state.error.text, 'Needs OPENAI_API_KEY — add it in Provider Settings');
+    assert.equal(ui.state.error.hint, false);
+
+    setLocale('es');
+    assert.notEqual(ui.state.error.text, 'Needs OPENAI_API_KEY — add it in Provider Settings');
+    assert.match(ui.state.error.text, /OPENAI_API_KEY/);
+    assert.equal(ui.state.error.hint, false);
+    setLocale('en');
+
+    controller.reportError('Realtime connection', new Error('ICE failed'));
+    assert.match(ui.state.error.text, /ICE failed/);
+    assert.equal(ui.state.error.hint, true);
+  } finally {
+    setLocale('en');
+  }
+});

src/voice/realtimeBackend.js

@@ -39,8 +39,9 @@ export function createRealtimeBackend({
       if (!response.ok) {
         const reason =
           typeof data?.error === 'string' ? data.error : data?.error?.message;
-        throw new Error(
-          reason || `Realtime token failed: HTTP ${response.status}`,
+        throw Object.assign(
+          new Error(reason || `Realtime token failed: HTTP ${response.status}`),
+          typeof data?.code === 'string' ? { code: data.code } : {},
         );
       }
       const token =

src/voice/realtimeBackend.test.mjs

@@ -143,3 +143,17 @@ test('controller lifetime stops pending transport and releases resources through
   assert.equal(controller.stream, null);
   assert.equal(controller.isActive(), false);
 });
+
+test('a missing server key keeps its code, so the UI can point to Provider Settings', async () => {
+  const backend = createRealtimeBackend({
+    tokenTransport: async () =>
+      Response.json(
+        { error: 'OPENAI_API_KEY is not set', code: 'API_KEY_MISSING' },
+        { status: 503 },
+      ),
+  });
+  await assert.rejects(backend.requestToken(), {
+    message: 'OPENAI_API_KEY is not set',
+    code: 'API_KEY_MISSING',
+  });
+});

src/voice/realtimeController.js

@@ -18,7 +18,7 @@ import {
 } from './pointerContext.js';
 
 import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
-import { postDebugLog } from './realtimeDiagnostics.js';
+import { missingVoiceKeyText, postDebugLog } from './realtimeDiagnostics.js';
 
 export {
   readStoredVoiceTier,
@@ -407,7 +407,14 @@ export class GevRealtimeController extends RealtimeFacade {
     }
   }
 
-  setStatus(status, detail) {
+  /**
+   * @param {string} status
+   * @param {string} [detail]
+   * @param {{missingKey?: boolean}} [options] True when the error is a
+   *   missing server key: it is worded in the current language and shown
+   *   without the microphone hint.
+   */
+  setStatus(status, detail, { missingKey = false } = {}) {
     this.status = status;
     this.emitSessionEvent({ type: 'state', state: status, detail });
     this.ui.update('root', {
@@ -435,8 +442,12 @@ export class GevRealtimeController extends RealtimeFacade {
     this.ui.update('error', {
       text: () =>
         status === 'error'
-          ? resolvedDetail() || t('Voice session could not be started.')
+          ? (missingKey && missingVoiceKeyText()) ||
+            resolvedDetail() ||
+            t('Voice session could not be started.')
           : '',
+      // "Check microphone permission" is wrong advice for a missing key.
+      hint: () => !(status === 'error' && missingKey),
     });
     if (status === 'idle' || status === 'connecting' || status === 'error') {
       this.setVoiceSpeaker('idle');

src/voice/realtimeDiagnostics.js

@@ -1,3 +1,11 @@
+import { t } from '../i18n/index.js';
+
+/** The voice error when the server has no OpenAI key: where to add it. */
+export const missingVoiceKeyText = () =>
+  t('Needs {envVars} — add it in Provider Settings', {
+    envVars: 'OPENAI_API_KEY',
+  });
+
 const ERROR_LOG_LIMIT = 30;
 
 const ERROR_STORAGE_KEY = 'gev-realtime-errors';
@@ -83,6 +91,7 @@ function createErrorRecord(source, error, extra = {}) {
       String(error?.message || '').trim() ||
       'No browser error message supplied',
     errorDetail: rtcError?.errorDetail || null,
+    code: typeof rtcError?.code === 'string' ? rtcError.code : null,
     sctpCauseCode: rtcError?.sctpCauseCode ?? null,
     receivedAlert: rtcError?.receivedAlert ?? null,
     sentAlert: rtcError?.sentAlert ?? null,
@@ -91,6 +100,8 @@ function createErrorRecord(source, error, extra = {}) {
 }
 
 function formatErrorForDisplay(record) {
+  // A missing key is a setup step, not a connection fault: say where to add it.
+  if (record.code === 'API_KEY_MISSING') return missingVoiceKeyText();
   const primary = [record.source, record.message].filter(Boolean).join(': ');
   const state = [
     record.errorDetail && `detail=${record.errorDetail}`,
@@ -172,7 +183,11 @@ export class RealtimeDiagnostics {
     storeErrors(this.errors);
     console.error('[GEV Realtime]', record);
     this.debugLog('error', record);
-    this.setStatus('error', formatErrorForDisplay(record));
+    this.setStatus(
+      'error',
+      formatErrorForDisplay(record),
+      record.code === 'API_KEY_MISSING' ? { missingKey: true } : undefined,
+    );
     return record;
   }