From 874f83ba48fa0ee667f8063f010a4072e0c38d23 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Tue, 10 Nov 2026 18:11:07 -0800 Subject: [PATCH] fix(voice): messages worded before reaching the status also follow a language change (#1661) The radio and push-to-talk helpers shared by both voice providers now pass English catalog text, and the OpenAI controller words its detail when shown, like the Gemini session. A token failure without a server message carries its status as sourceParams, so it is worded again with the number. The docs say sourceDetail is the message as given, catalog text when it is the app's own. --- docs/CURRENT-STATE.md | 2 +- src/voice/geminiSession.js | 36 ++++++++++++++++++++++------- src/voice/geminiSession.test.mjs | 21 +++++++++++++++++ src/voice/gevRealtime.test.mjs | 20 +++++++++++++++- src/voice/providerCommands.test.mjs | 24 +++++++++++++++++++ src/voice/realtimeController.js | 9 +++++--- src/voice/realtimeInput.js | 6 ++--- src/voice/realtimeRadio.js | 4 ++-- src/voice/sessionCommands.js | 4 +++- 9 files changed, 107 insertions(+), 19 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index adbf20824..9563a3aff 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1135,7 +1135,7 @@ Source factories have dedicated `layers//source` exports. ALPR and earth Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. An active session's normal Stop publishes one idle transition, including adapters that emit their own terminal state; a restart triggered by that transition keeps ownership of its new startup state. Alternate adapters can use the same controls and action runner. -The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Gemini's status and error messages keep their English catalog text on the session's `state` event (`sourceDetail`, beside the worded `detail`), so the voice control words them again after a language change. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. +The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Voice status and error messages keep the message as given on the session's `state` event (`sourceDetail`, English catalog text when it is the app's own, with `sourceParams` for its placeholders), beside the worded `detail`, so the voice control words them again after a language change. Typed-chat errors and the chat panel are worded once. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 33 tools using separate description-only metadata; a session is offered 32 of them unless `OVERPASS_UPSTREAMS` is set, which adds `osm_query` and its one prompt sentence (`server/providers/openai/tools.js`). Description customization cannot replace argument types, enum values or required fields. diff --git a/src/voice/geminiSession.js b/src/voice/geminiSession.js index 8900c1552..572db471a 100644 --- a/src/voice/geminiSession.js +++ b/src/voice/geminiSession.js @@ -361,23 +361,31 @@ export class GeminiSessionController { } /** - * @param {{missingKey?: boolean}} [options] True when the server has no - * Gemini key: the control then drops its microphone hint. + * @param {{missingKey?: boolean, params?: object}} [options] `missingKey` + * is true when the server has no Gemini key: the control then drops its + * microphone hint. `params` fill the placeholders of `detail`. */ - setStatus(status, detail, recovery, { missingKey = false } = {}) { + setStatus( + status, + detail, + recovery, + { missingKey = false, params = null } = {}, + ) { this.status = status; if (status === 'listening' && this.input.pushToTalkMode) detail = this.input.pushToTalkKeyHeld ? msg('Release Space to send') : msg('Hold Space to talk'); // `detail` is worded in the current language; `sourceDetail` keeps the - // English catalog text, so a view can word it again after a language - // change. + // message as given (English catalog text, with `sourceParams` for its + // placeholders, when it is ours), so a view can word it again after a + // language change. const event = { type: 'state', state: status, - detail: translateKnown(detail), + detail: params ? t(detail, params) : translateKnown(detail), ...(typeof detail === 'string' && { sourceDetail: detail }), + ...(params && { sourceParams: params }), }; if (recovery) event.recovery = translateKnown(recovery); if (missingKey) event.missingKey = true; @@ -603,6 +611,15 @@ export class GeminiSessionController { }), ), typeof data?.code === 'string' ? { code: data.code } : {}, + // The status words this again in the current language when shown. + typeof data?.error === 'string' + ? {} + : { + wording: { + text: msg('Gemini token request failed (HTTP {status})'), + params: { status: response.status }, + }, + }, ); if ( typeof data?.model !== 'string' || @@ -976,11 +993,14 @@ export class GeminiSessionController { ) : error?.name === 'TimeoutError' ? msg('Gemini connection timed out. Try again.') - : error?.message || msg('Gemini voice could not start'); - this.lastError = detail; + : error?.wording?.text || + error?.message || + msg('Gemini voice could not start'); + this.lastError = error?.wording ? error.message : detail; this.stop({ preserveStatus: true }); this.setStatus('error', detail, recovery, { missingKey: error?.code === 'API_KEY_MISSING', + params: error?.wording?.params, }); } diff --git a/src/voice/geminiSession.test.mjs b/src/voice/geminiSession.test.mjs index 6e921d827..e441e6915 100644 --- a/src/voice/geminiSession.test.mjs +++ b/src/voice/geminiSession.test.mjs @@ -3688,3 +3688,24 @@ test('an error keeps its English catalog text beside the worded detail', async ( ); assert.notEqual(error.detail, error.sourceDetail); }); + +test('a token failure without a message keeps its status for the next language', async (t2) => { + t2.after(() => setLocale('en')); + setLocale('es'); + const h = harness({ + fetchImpl: async () => Response.json({}, { status: 502 }), + }); + await h.c.start(); + const error = h.events + .filter((event) => event.type === 'state' && event.state === 'error') + .at(-1); + assert.equal( + error.sourceDetail, + 'Gemini token request failed (HTTP {status})', + ); + assert.deepEqual(error.sourceParams, { status: 502 }); + assert.equal( + error.detail, + t('Gemini token request failed (HTTP {status})', { status: 502 }), + ); +}); diff --git a/src/voice/gevRealtime.test.mjs b/src/voice/gevRealtime.test.mjs index 426a15f5c..752dbf9c8 100644 --- a/src/voice/gevRealtime.test.mjs +++ b/src/voice/gevRealtime.test.mjs @@ -35,7 +35,7 @@ import { } from './gevRealtime.js'; import { createVoiceCostTracker } from './voiceCost.js'; import { createTestVoiceUi } from '../testSupport/voiceUi.mjs'; -import { setLocale } from '../i18n/index.js'; +import { setLocale, t } from '../i18n/index.js'; test('push-to-talk recognizes Space by code or key', () => { assert.equal(PUSH_TO_TALK_HOLD_DELAY_MS, 500); @@ -3764,3 +3764,21 @@ test('a missing server key says where to add it, without the microphone hint, in setLocale('en'); } }); + +test('a status from a shared helper is worded again after a language change', () => { + const ui = createTestVoiceUi(); + const controller = new GevRealtimeController({ + ui, + runner: async () => ({ ok: true }), + radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} }, + }); + try { + controller.setStatus('listening', 'Radio did not start'); + assert.equal(ui.state.detail.text, 'Radio did not start'); + setLocale('es'); + assert.equal(ui.state.detail.text, t('Radio did not start')); + assert.notEqual(ui.state.detail.text, 'Radio did not start'); + } finally { + setLocale('en'); + } +}); diff --git a/src/voice/providerCommands.test.mjs b/src/voice/providerCommands.test.mjs index 96c19ecf4..dc358834d 100644 --- a/src/voice/providerCommands.test.mjs +++ b/src/voice/providerCommands.test.mjs @@ -928,3 +928,27 @@ test('a provider message is worded again after a language change', async (t2) => assert.equal(ui.state.detail.text, t('Ask or command')); assert.notEqual(ui.state.detail.text, 'Ask or command'); }); + +test('a provider message with placeholders is worded again with them', async (t2) => { + t2.after(() => setLocale('en')); + const f = compositionFixture(t2, {}); + f.select('gemini'); + await settle(); + const ui = f.views.at(-1); + const text = 'Gemini token request failed (HTTP {status})'; + f.instances.at(-1).hooks.emit({ + type: 'state', + state: 'error', + detail: t(text, { status: 502 }), + sourceDetail: text, + sourceParams: { status: 502 }, + }); + assert.equal(ui.state.error.text, 'Gemini token request failed (HTTP 502)'); + setLocale('es'); + assert.equal(ui.state.error.text, t(text, { status: 502 })); + assert.match(ui.state.error.text, /502/); + assert.notEqual( + ui.state.error.text, + 'Gemini token request failed (HTTP 502)', + ); +}); diff --git a/src/voice/realtimeController.js b/src/voice/realtimeController.js index 58b8aec77..b85439759 100644 --- a/src/voice/realtimeController.js +++ b/src/voice/realtimeController.js @@ -54,7 +54,7 @@ export { import { createRealtimeBackend } from './realtimeBackend.js'; -import { t } from '../i18n/index.js'; +import { t, translateKnown } from '../i18n/index.js'; import { voiceStatusLabel } from './control.js'; // Machine states (dataset.status, CSS); voiceStatusLabel() translates labels. @@ -416,10 +416,13 @@ export class GevRealtimeController extends RealtimeFacade { */ setStatus(status, detail, { missingKey = false } = {}) { this.status = status; + // A detail may be English catalog text (from the helpers this controller + // shares with Gemini): it is worded in the current language when shown. this.emitSessionEvent({ type: 'state', state: status, - detail, + detail: translateKnown(detail), + ...(typeof detail === 'string' && { sourceDetail: detail }), ...(missingKey && { missingKey: true }), }); this.ui.update('root', { @@ -437,7 +440,7 @@ export class GevRealtimeController extends RealtimeFacade { ? held ? t('Release Space to send') : t('Hold Space to talk') - : detail; + : translateKnown(detail); const primaryDetail = () => status === 'error' ? t('VOICE UNAVAILABLE') diff --git a/src/voice/realtimeInput.js b/src/voice/realtimeInput.js index 9c9fd85e7..0a43fa6ec 100644 --- a/src/voice/realtimeInput.js +++ b/src/voice/realtimeInput.js @@ -1,4 +1,4 @@ -import { t } from '../i18n/index.js'; +import { msg, t } from '../i18n/index.js'; import { PUSH_TO_TALK_HOLD_DELAY_MS, MICROPHONE_VISUALIZER_GATE, @@ -124,7 +124,7 @@ export class RealtimeInput { this.ui.update('root', { pushToTalk: 'held' }); this.setMicrophoneEnabled(true); if (this.status === 'listening') - this.setStatus('listening', t('Release Space to send')); + this.setStatus('listening', msg('Release Space to send')); } else { this.start({ pushToTalk: true }); // start() performs a controlled stop() before connecting. Restore the @@ -201,7 +201,7 @@ export class RealtimeInput { if (!this.pushToTalkMode) return; this.setMicrophoneEnabled(false); if (this.status === 'listening') - this.setStatus('listening', t('Hold Space to talk')); + this.setStatus('listening', msg('Hold Space to talk')); else this.updateVoiceButtonLabel(); } diff --git a/src/voice/realtimeRadio.js b/src/voice/realtimeRadio.js index 065c55c3b..90abce9e8 100644 --- a/src/voice/realtimeRadio.js +++ b/src/voice/realtimeRadio.js @@ -1,4 +1,4 @@ -import { t } from '../i18n/index.js'; +import { msg } from '../i18n/index.js'; import { silenceRadioForVoice, startPreparedRadioAfterPlaybackReady, @@ -261,7 +261,7 @@ export class RealtimeRadio { if (radioHandoff.result?.ok || radioHandoff.cancelled || !stillCurrent) return; if (this.dc?.readyState === 'open' && !this.userTurnPending) { - this.setStatus('listening', t('Radio did not start')); + this.setStatus('listening', msg('Radio did not start')); this.queueResponseCreate( 'Say exactly one short correction: “The Radio station could not start. Voice is still on.”', ); diff --git a/src/voice/sessionCommands.js b/src/voice/sessionCommands.js index c3ef1a917..2add30636 100644 --- a/src/voice/sessionCommands.js +++ b/src/voice/sessionCommands.js @@ -122,7 +122,9 @@ export function createVoiceCommands({ source === 'Voice off' ? '' : event.sourceDetail - ? translateKnown(event.sourceDetail) + ? event.sourceParams + ? t(event.sourceDetail, event.sourceParams) + : translateKnown(event.sourceDetail) : event.detail; ui.update('detail', { text: () => detail() || (idle ? t('Voice off') : t('Voice active')),