Merge pull request #1661 from bilawalsidhu/sameh/gemini-error-locale

fix(voice): Gemini voice messages follow a language change

changelog.d/1661-gemini-voice-message-locale.md

@@ -0,0 +1,4 @@
+- Gemini voice messages follow a language change. A Gemini Live status or
+  error, such as "Gemini quota is unavailable", is shown in the new interface
+  language after you switch it, instead of staying in the old one, as OpenAI
+  voice messages already do (Sameh Khamis, @samehkhamis, #1661).

docs/CURRENT-STATE.md

@@ -1135,7 +1135,7 @@ Source factories have dedicated `layers/<family>/source` exports. ALPR and earth
 
 Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. An active session's normal Stop publishes one idle transition, including adapters that emit their own terminal state; a restart triggered by that transition keeps ownership of its new startup state. Alternate adapters can use the same controls and action runner.
 
-The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter.
+The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Voice status and error messages keep the message as given on the session's `state` event (`sourceDetail`, English catalog text when it is the app's own, with `sourceParams` for its placeholders), beside the worded `detail`, so the voice control words them again after a language change. Typed-chat errors and the chat panel are worded once. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter.
 
 Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 33 tools using separate description-only metadata; a session is offered 32 of them unless `OVERPASS_UPSTREAMS` is set, which adds `osm_query` and its one prompt sentence (`server/providers/openai/tools.js`). Description customization cannot replace argument types, enum values or required fields.
 

src/voice/geminiSession.js

@@ -71,7 +71,9 @@ export function geminiSocketUrl({ token, websocketUrl }) {
   return url.href;
 }
 
-// Known local-server failures are translated; unexpected diagnostics stay intact.
+// The local server's known failures, marked for translation. Each is kept as
+// its English catalog text, which the status words when it is shown, so a
+// language change words it again; unexpected diagnostics stay intact.
 function tokenErrorMessage(error) {
   switch (error) {
     case msg('Gemini voice could not connect. Try again.'):
@@ -85,7 +87,6 @@ function tokenErrorMessage(error) {
     case msg('Token request is too large'):
     case msg('Use an empty JSON token request'):
     case msg('Method not allowed'):
-      return t(error);
     default:
       return error;
   }
@@ -360,19 +361,31 @@ export class GeminiSessionController {
   }
 
   /**
-   * @param {{missingKey?: boolean}} [options] True when the server has no
-   *   Gemini key: the control then drops its microphone hint.
+   * @param {{missingKey?: boolean, params?: object}} [options] `missingKey`
+   *   is true when the server has no Gemini key: the control then drops its
+   *   microphone hint. `params` fill the placeholders of `detail`.
    */
-  setStatus(status, detail, recovery, { missingKey = false } = {}) {
+  setStatus(
+    status,
+    detail,
+    recovery,
+    { missingKey = false, params = null } = {},
+  ) {
     this.status = status;
     if (status === 'listening' && this.input.pushToTalkMode)
       detail = this.input.pushToTalkKeyHeld
         ? msg('Release Space to send')
         : msg('Hold Space to talk');
+    // `detail` is worded in the current language; `sourceDetail` keeps the
+    // message as given (English catalog text, with `sourceParams` for its
+    // placeholders, when it is ours), so a view can word it again after a
+    // language change.
     const event = {
       type: 'state',
       state: status,
-      detail: translateKnown(detail),
+      detail: params ? t(detail, params) : translateKnown(detail),
+      ...(typeof detail === 'string' && { sourceDetail: detail }),
+      ...(params && { sourceParams: params }),
     };
     if (recovery) event.recovery = translateKnown(recovery);
     if (missingKey) event.missingKey = true;
@@ -598,6 +611,15 @@ export class GeminiSessionController {
                 }),
           ),
           typeof data?.code === 'string' ? { code: data.code } : {},
+          // The status words this again in the current language when shown.
+          typeof data?.error === 'string'
+            ? {}
+            : {
+                wording: {
+                  text: msg('Gemini token request failed (HTTP {status})'),
+                  params: { status: response.status },
+                },
+              },
         );
       if (
         typeof data?.model !== 'string' ||
@@ -971,11 +993,14 @@ export class GeminiSessionController {
           )
         : error?.name === 'TimeoutError'
           ? msg('Gemini connection timed out. Try again.')
-          : error?.message || msg('Gemini voice could not start');
-    this.lastError = detail;
+          : error?.wording?.text ||
+            error?.message ||
+            msg('Gemini voice could not start');
+    this.lastError = error?.wording ? error.message : detail;
     this.stop({ preserveStatus: true });
     this.setStatus('error', detail, recovery, {
       missingKey: error?.code === 'API_KEY_MISSING',
+      params: error?.wording?.params,
     });
   }
 

src/voice/geminiSession.test.mjs

@@ -4,6 +4,7 @@ import { geminiSocketUrl } from './geminiSession.js';
 import { createVoiceSession } from './session.js';
 import { createPointerTracker } from './pointerContext.js';
 import { withToolCatalog } from './gevRealtime.js';
+import { setLocale, t } from '../i18n/index.js';
 import {
   harness,
   tick,
@@ -3662,3 +3663,49 @@ test('a missing server key reaches the error state flagged, other token failures
     assert.equal(h.sockets.length, 0);
   }
 });
+
+test('an error keeps its English catalog text beside the worded detail', async (t2) => {
+  t2.after(() => setLocale('en'));
+  setLocale('es');
+  const h = harness({
+    fetchImpl: async () =>
+      Response.json(
+        { error: 'Gemini quota is unavailable. Try again later.' },
+        { status: 503 },
+      ),
+  });
+  await h.c.start();
+  const error = h.events
+    .filter((event) => event.type === 'state' && event.state === 'error')
+    .at(-1);
+  assert.equal(
+    error.sourceDetail,
+    'Gemini quota is unavailable. Try again later.',
+  );
+  assert.equal(
+    error.detail,
+    t('Gemini quota is unavailable. Try again later.'),
+  );
+  assert.notEqual(error.detail, error.sourceDetail);
+});
+
+test('a token failure without a message keeps its status for the next language', async (t2) => {
+  t2.after(() => setLocale('en'));
+  setLocale('es');
+  const h = harness({
+    fetchImpl: async () => Response.json({}, { status: 502 }),
+  });
+  await h.c.start();
+  const error = h.events
+    .filter((event) => event.type === 'state' && event.state === 'error')
+    .at(-1);
+  assert.equal(
+    error.sourceDetail,
+    'Gemini token request failed (HTTP {status})',
+  );
+  assert.deepEqual(error.sourceParams, { status: 502 });
+  assert.equal(
+    error.detail,
+    t('Gemini token request failed (HTTP {status})', { status: 502 }),
+  );
+});

src/voice/gevRealtime.test.mjs

@@ -35,7 +35,7 @@ import {
 } from './gevRealtime.js';
 import { createVoiceCostTracker } from './voiceCost.js';
 import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
-import { setLocale } from '../i18n/index.js';
+import { setLocale, t } from '../i18n/index.js';
 
 test('push-to-talk recognizes Space by code or key', () => {
   assert.equal(PUSH_TO_TALK_HOLD_DELAY_MS, 500);
@@ -3764,3 +3764,21 @@ test('a missing server key says where to add it, without the microphone hint, in
     setLocale('en');
   }
 });
+
+test('a status from a shared helper is worded again after a language change', () => {
+  const ui = createTestVoiceUi();
+  const controller = new GevRealtimeController({
+    ui,
+    runner: async () => ({ ok: true }),
+    radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} },
+  });
+  try {
+    controller.setStatus('listening', 'Radio did not start');
+    assert.equal(ui.state.detail.text, 'Radio did not start');
+    setLocale('es');
+    assert.equal(ui.state.detail.text, t('Radio did not start'));
+    assert.notEqual(ui.state.detail.text, 'Radio did not start');
+  } finally {
+    setLocale('en');
+  }
+});

src/voice/providerCommands.test.mjs

@@ -11,6 +11,7 @@ import { createGevActionRunner } from './gevActions.js';
 import { createPointerTracker } from './pointerContext.js';
 import { createReferentRegistry } from './referents.js';
 import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
+import { setLocale, t } from '../i18n/index.js';
 
 const PROVIDER_KEY = 'gev.voice.provider';
 const settle = () => new Promise((resolve) => setImmediate(resolve));
@@ -897,3 +898,57 @@ test('a missing-key error hides the microphone hint; other errors keep it', asyn
   assert.equal(ui.state.error.text, 'Socket closed');
   assert.equal(ui.state.error.hint, true);
 });
+
+test('a provider message is worded again after a language change', async (t2) => {
+  t2.after(() => setLocale('en'));
+  const f = compositionFixture(t2, {});
+  f.select('gemini');
+  await settle();
+  const instance = f.instances.at(-1);
+  const ui = f.views.at(-1);
+  const quota = 'Gemini quota is unavailable. Try again later.';
+  instance.hooks.emit({
+    type: 'state',
+    state: 'error',
+    detail: quota,
+    sourceDetail: quota,
+  });
+  assert.equal(ui.state.error.text, quota);
+  setLocale('es');
+  assert.equal(ui.state.error.text, t(quota));
+  assert.notEqual(ui.state.error.text, quota);
+  setLocale('en');
+  instance.hooks.emit({
+    type: 'state',
+    state: 'listening',
+    detail: 'Ask or command',
+    sourceDetail: 'Ask or command',
+  });
+  setLocale('es');
+  assert.equal(ui.state.detail.text, t('Ask or command'));
+  assert.notEqual(ui.state.detail.text, 'Ask or command');
+});
+
+test('a provider message with placeholders is worded again with them', async (t2) => {
+  t2.after(() => setLocale('en'));
+  const f = compositionFixture(t2, {});
+  f.select('gemini');
+  await settle();
+  const ui = f.views.at(-1);
+  const text = 'Gemini token request failed (HTTP {status})';
+  f.instances.at(-1).hooks.emit({
+    type: 'state',
+    state: 'error',
+    detail: t(text, { status: 502 }),
+    sourceDetail: text,
+    sourceParams: { status: 502 },
+  });
+  assert.equal(ui.state.error.text, 'Gemini token request failed (HTTP 502)');
+  setLocale('es');
+  assert.equal(ui.state.error.text, t(text, { status: 502 }));
+  assert.match(ui.state.error.text, /502/);
+  assert.notEqual(
+    ui.state.error.text,
+    'Gemini token request failed (HTTP 502)',
+  );
+});

src/voice/realtimeController.js

@@ -54,7 +54,7 @@ export {
 
 import { createRealtimeBackend } from './realtimeBackend.js';
 
-import { t } from '../i18n/index.js';
+import { t, translateKnown } from '../i18n/index.js';
 import { voiceStatusLabel } from './control.js';
 
 // Machine states (dataset.status, CSS); voiceStatusLabel() translates labels.
@@ -416,10 +416,13 @@ export class GevRealtimeController extends RealtimeFacade {
    */
   setStatus(status, detail, { missingKey = false } = {}) {
     this.status = status;
+    // A detail may be English catalog text (from the helpers this controller
+    // shares with Gemini): it is worded in the current language when shown.
     this.emitSessionEvent({
       type: 'state',
       state: status,
-      detail,
+      detail: translateKnown(detail),
+      ...(typeof detail === 'string' && { sourceDetail: detail }),
       ...(missingKey && { missingKey: true }),
     });
     this.ui.update('root', {
@@ -437,7 +440,7 @@ export class GevRealtimeController extends RealtimeFacade {
         ? held
           ? t('Release Space to send')
           : t('Hold Space to talk')
-        : detail;
+        : translateKnown(detail);
     const primaryDetail = () =>
       status === 'error'
         ? t('VOICE UNAVAILABLE')

src/voice/realtimeInput.js

@@ -1,4 +1,4 @@
-import { t } from '../i18n/index.js';
+import { msg, t } from '../i18n/index.js';
 import {
   PUSH_TO_TALK_HOLD_DELAY_MS,
   MICROPHONE_VISUALIZER_GATE,
@@ -124,7 +124,7 @@ export class RealtimeInput {
           this.ui.update('root', { pushToTalk: 'held' });
           this.setMicrophoneEnabled(true);
           if (this.status === 'listening')
-            this.setStatus('listening', t('Release Space to send'));
+            this.setStatus('listening', msg('Release Space to send'));
         } else {
           this.start({ pushToTalk: true });
           // start() performs a controlled stop() before connecting. Restore the
@@ -201,7 +201,7 @@ export class RealtimeInput {
     if (!this.pushToTalkMode) return;
     this.setMicrophoneEnabled(false);
     if (this.status === 'listening')
-      this.setStatus('listening', t('Hold Space to talk'));
+      this.setStatus('listening', msg('Hold Space to talk'));
     else this.updateVoiceButtonLabel();
   }
 

src/voice/realtimeRadio.js

@@ -1,4 +1,4 @@
-import { t } from '../i18n/index.js';
+import { msg } from '../i18n/index.js';
 import {
   silenceRadioForVoice,
   startPreparedRadioAfterPlaybackReady,
@@ -261,7 +261,7 @@ export class RealtimeRadio {
     if (radioHandoff.result?.ok || radioHandoff.cancelled || !stillCurrent)
       return;
     if (this.dc?.readyState === 'open' && !this.userTurnPending) {
-      this.setStatus('listening', t('Radio did not start'));
+      this.setStatus('listening', msg('Radio did not start'));
       this.queueResponseCreate(
         'Say exactly one short correction: “The Radio station could not start. Voice is still on.”',
       );

src/voice/sessionCommands.js

@@ -7,7 +7,7 @@ import { createVoiceSession, isStoppable } from './session.js';
 import { VoiceCardControls } from './voiceCard.js';
 import { PointerReticle } from './pointerReticle.js';
 import { VOICE_PROVIDERS } from './providerCapabilities.js';
-import { t } from '../i18n/index.js';
+import { t, translateKnown } from '../i18n/index.js';
 
 function createDefaultReticle(pointer) {
   const element = pointer?.element;
@@ -114,11 +114,20 @@ export function createVoiceCommands({
     // States are machine values; only the label is translated.
     ui.update('status', { text: () => voiceStatusLabel(event.state) });
     // The portable session reports 'Voice off' in English: show the idle
-    // label instead. Other details are provider messages, shown as they come.
+    // label instead. Other details are provider messages; one that keeps its
+    // English catalog text is worded again in the current language.
     const idle = event.state === 'idle';
-    const detail = event.detail === 'Voice off' ? '' : event.detail;
+    const source = event.sourceDetail ?? event.detail;
+    const detail = () =>
+      source === 'Voice off'
+        ? ''
+        : event.sourceDetail
+          ? event.sourceParams
+            ? t(event.sourceDetail, event.sourceParams)
+            : translateKnown(event.sourceDetail)
+          : event.detail;
     ui.update('detail', {
-      text: () => detail || (idle ? t('Voice off') : t('Voice active')),
+      text: () => detail() || (idle ? t('Voice off') : t('Voice active')),
     });
     // A typed chat session is not the microphone.
     ui.update('button', {
@@ -127,7 +136,7 @@ export function createVoiceCommands({
     ui.update('error', {
       text: () =>
         event.state === 'error'
-          ? event.detail || t('Voice could not be started.')
+          ? detail() || t('Voice could not be started.')
           : '',
       // "Check microphone permission" is wrong advice for a missing key.
       hint: () => !(event.state === 'error' && event.missingKey === true),