diff --git a/changelog.d/1661-gemini-voice-message-locale.md b/changelog.d/1661-gemini-voice-message-locale.md new file mode 100644 index 000000000..c53270375 --- /dev/null +++ b/changelog.d/1661-gemini-voice-message-locale.md @@ -0,0 +1,4 @@ +- Gemini voice messages follow a language change. A Gemini Live status or + error, such as "Gemini quota is unavailable", is shown in the new interface + language after you switch it, instead of staying in the old one, as OpenAI + voice messages already do (Sameh Khamis, @samehkhamis, #1661). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9a2f938e6..9563a3aff 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1135,7 +1135,7 @@ Source factories have dedicated `layers//source` exports. ALPR and earth Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. An active session's normal Stop publishes one idle transition, including adapters that emit their own terminal state; a restart triggered by that transition keeps ownership of its new startup state. Alternate adapters can use the same controls and action runner. -The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. +The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Voice status and error messages keep the message as given on the session's `state` event (`sourceDetail`, English catalog text when it is the app's own, with `sourceParams` for its placeholders), beside the worded `detail`, so the voice control words them again after a language change. Typed-chat errors and the chat panel are worded once. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 33 tools using separate description-only metadata; a session is offered 32 of them unless `OVERPASS_UPSTREAMS` is set, which adds `osm_query` and its one prompt sentence (`server/providers/openai/tools.js`). Description customization cannot replace argument types, enum values or required fields. diff --git a/src/voice/geminiSession.js b/src/voice/geminiSession.js index 0ee9fb313..572db471a 100644 --- a/src/voice/geminiSession.js +++ b/src/voice/geminiSession.js @@ -71,7 +71,9 @@ export function geminiSocketUrl({ token, websocketUrl }) { return url.href; } -// Known local-server failures are translated; unexpected diagnostics stay intact. +// The local server's known failures, marked for translation. Each is kept as +// its English catalog text, which the status words when it is shown, so a +// language change words it again; unexpected diagnostics stay intact. function tokenErrorMessage(error) { switch (error) { case msg('Gemini voice could not connect. Try again.'): @@ -85,7 +87,6 @@ function tokenErrorMessage(error) { case msg('Token request is too large'): case msg('Use an empty JSON token request'): case msg('Method not allowed'): - return t(error); default: return error; } @@ -360,19 +361,31 @@ export class GeminiSessionController { } /** - * @param {{missingKey?: boolean}} [options] True when the server has no - * Gemini key: the control then drops its microphone hint. + * @param {{missingKey?: boolean, params?: object}} [options] `missingKey` + * is true when the server has no Gemini key: the control then drops its + * microphone hint. `params` fill the placeholders of `detail`. */ - setStatus(status, detail, recovery, { missingKey = false } = {}) { + setStatus( + status, + detail, + recovery, + { missingKey = false, params = null } = {}, + ) { this.status = status; if (status === 'listening' && this.input.pushToTalkMode) detail = this.input.pushToTalkKeyHeld ? msg('Release Space to send') : msg('Hold Space to talk'); + // `detail` is worded in the current language; `sourceDetail` keeps the + // message as given (English catalog text, with `sourceParams` for its + // placeholders, when it is ours), so a view can word it again after a + // language change. const event = { type: 'state', state: status, - detail: translateKnown(detail), + detail: params ? t(detail, params) : translateKnown(detail), + ...(typeof detail === 'string' && { sourceDetail: detail }), + ...(params && { sourceParams: params }), }; if (recovery) event.recovery = translateKnown(recovery); if (missingKey) event.missingKey = true; @@ -598,6 +611,15 @@ export class GeminiSessionController { }), ), typeof data?.code === 'string' ? { code: data.code } : {}, + // The status words this again in the current language when shown. + typeof data?.error === 'string' + ? {} + : { + wording: { + text: msg('Gemini token request failed (HTTP {status})'), + params: { status: response.status }, + }, + }, ); if ( typeof data?.model !== 'string' || @@ -971,11 +993,14 @@ export class GeminiSessionController { ) : error?.name === 'TimeoutError' ? msg('Gemini connection timed out. Try again.') - : error?.message || msg('Gemini voice could not start'); - this.lastError = detail; + : error?.wording?.text || + error?.message || + msg('Gemini voice could not start'); + this.lastError = error?.wording ? error.message : detail; this.stop({ preserveStatus: true }); this.setStatus('error', detail, recovery, { missingKey: error?.code === 'API_KEY_MISSING', + params: error?.wording?.params, }); } diff --git a/src/voice/geminiSession.test.mjs b/src/voice/geminiSession.test.mjs index 3c91d3a0a..e441e6915 100644 --- a/src/voice/geminiSession.test.mjs +++ b/src/voice/geminiSession.test.mjs @@ -4,6 +4,7 @@ import { geminiSocketUrl } from './geminiSession.js'; import { createVoiceSession } from './session.js'; import { createPointerTracker } from './pointerContext.js'; import { withToolCatalog } from './gevRealtime.js'; +import { setLocale, t } from '../i18n/index.js'; import { harness, tick, @@ -3662,3 +3663,49 @@ test('a missing server key reaches the error state flagged, other token failures assert.equal(h.sockets.length, 0); } }); + +test('an error keeps its English catalog text beside the worded detail', async (t2) => { + t2.after(() => setLocale('en')); + setLocale('es'); + const h = harness({ + fetchImpl: async () => + Response.json( + { error: 'Gemini quota is unavailable. Try again later.' }, + { status: 503 }, + ), + }); + await h.c.start(); + const error = h.events + .filter((event) => event.type === 'state' && event.state === 'error') + .at(-1); + assert.equal( + error.sourceDetail, + 'Gemini quota is unavailable. Try again later.', + ); + assert.equal( + error.detail, + t('Gemini quota is unavailable. Try again later.'), + ); + assert.notEqual(error.detail, error.sourceDetail); +}); + +test('a token failure without a message keeps its status for the next language', async (t2) => { + t2.after(() => setLocale('en')); + setLocale('es'); + const h = harness({ + fetchImpl: async () => Response.json({}, { status: 502 }), + }); + await h.c.start(); + const error = h.events + .filter((event) => event.type === 'state' && event.state === 'error') + .at(-1); + assert.equal( + error.sourceDetail, + 'Gemini token request failed (HTTP {status})', + ); + assert.deepEqual(error.sourceParams, { status: 502 }); + assert.equal( + error.detail, + t('Gemini token request failed (HTTP {status})', { status: 502 }), + ); +}); diff --git a/src/voice/gevRealtime.test.mjs b/src/voice/gevRealtime.test.mjs index 426a15f5c..752dbf9c8 100644 --- a/src/voice/gevRealtime.test.mjs +++ b/src/voice/gevRealtime.test.mjs @@ -35,7 +35,7 @@ import { } from './gevRealtime.js'; import { createVoiceCostTracker } from './voiceCost.js'; import { createTestVoiceUi } from '../testSupport/voiceUi.mjs'; -import { setLocale } from '../i18n/index.js'; +import { setLocale, t } from '../i18n/index.js'; test('push-to-talk recognizes Space by code or key', () => { assert.equal(PUSH_TO_TALK_HOLD_DELAY_MS, 500); @@ -3764,3 +3764,21 @@ test('a missing server key says where to add it, without the microphone hint, in setLocale('en'); } }); + +test('a status from a shared helper is worded again after a language change', () => { + const ui = createTestVoiceUi(); + const controller = new GevRealtimeController({ + ui, + runner: async () => ({ ok: true }), + radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} }, + }); + try { + controller.setStatus('listening', 'Radio did not start'); + assert.equal(ui.state.detail.text, 'Radio did not start'); + setLocale('es'); + assert.equal(ui.state.detail.text, t('Radio did not start')); + assert.notEqual(ui.state.detail.text, 'Radio did not start'); + } finally { + setLocale('en'); + } +}); diff --git a/src/voice/providerCommands.test.mjs b/src/voice/providerCommands.test.mjs index dcc38d6c5..dc358834d 100644 --- a/src/voice/providerCommands.test.mjs +++ b/src/voice/providerCommands.test.mjs @@ -11,6 +11,7 @@ import { createGevActionRunner } from './gevActions.js'; import { createPointerTracker } from './pointerContext.js'; import { createReferentRegistry } from './referents.js'; import { createTestVoiceUi } from '../testSupport/voiceUi.mjs'; +import { setLocale, t } from '../i18n/index.js'; const PROVIDER_KEY = 'gev.voice.provider'; const settle = () => new Promise((resolve) => setImmediate(resolve)); @@ -897,3 +898,57 @@ test('a missing-key error hides the microphone hint; other errors keep it', asyn assert.equal(ui.state.error.text, 'Socket closed'); assert.equal(ui.state.error.hint, true); }); + +test('a provider message is worded again after a language change', async (t2) => { + t2.after(() => setLocale('en')); + const f = compositionFixture(t2, {}); + f.select('gemini'); + await settle(); + const instance = f.instances.at(-1); + const ui = f.views.at(-1); + const quota = 'Gemini quota is unavailable. Try again later.'; + instance.hooks.emit({ + type: 'state', + state: 'error', + detail: quota, + sourceDetail: quota, + }); + assert.equal(ui.state.error.text, quota); + setLocale('es'); + assert.equal(ui.state.error.text, t(quota)); + assert.notEqual(ui.state.error.text, quota); + setLocale('en'); + instance.hooks.emit({ + type: 'state', + state: 'listening', + detail: 'Ask or command', + sourceDetail: 'Ask or command', + }); + setLocale('es'); + assert.equal(ui.state.detail.text, t('Ask or command')); + assert.notEqual(ui.state.detail.text, 'Ask or command'); +}); + +test('a provider message with placeholders is worded again with them', async (t2) => { + t2.after(() => setLocale('en')); + const f = compositionFixture(t2, {}); + f.select('gemini'); + await settle(); + const ui = f.views.at(-1); + const text = 'Gemini token request failed (HTTP {status})'; + f.instances.at(-1).hooks.emit({ + type: 'state', + state: 'error', + detail: t(text, { status: 502 }), + sourceDetail: text, + sourceParams: { status: 502 }, + }); + assert.equal(ui.state.error.text, 'Gemini token request failed (HTTP 502)'); + setLocale('es'); + assert.equal(ui.state.error.text, t(text, { status: 502 })); + assert.match(ui.state.error.text, /502/); + assert.notEqual( + ui.state.error.text, + 'Gemini token request failed (HTTP 502)', + ); +}); diff --git a/src/voice/realtimeController.js b/src/voice/realtimeController.js index 58b8aec77..b85439759 100644 --- a/src/voice/realtimeController.js +++ b/src/voice/realtimeController.js @@ -54,7 +54,7 @@ export { import { createRealtimeBackend } from './realtimeBackend.js'; -import { t } from '../i18n/index.js'; +import { t, translateKnown } from '../i18n/index.js'; import { voiceStatusLabel } from './control.js'; // Machine states (dataset.status, CSS); voiceStatusLabel() translates labels. @@ -416,10 +416,13 @@ export class GevRealtimeController extends RealtimeFacade { */ setStatus(status, detail, { missingKey = false } = {}) { this.status = status; + // A detail may be English catalog text (from the helpers this controller + // shares with Gemini): it is worded in the current language when shown. this.emitSessionEvent({ type: 'state', state: status, - detail, + detail: translateKnown(detail), + ...(typeof detail === 'string' && { sourceDetail: detail }), ...(missingKey && { missingKey: true }), }); this.ui.update('root', { @@ -437,7 +440,7 @@ export class GevRealtimeController extends RealtimeFacade { ? held ? t('Release Space to send') : t('Hold Space to talk') - : detail; + : translateKnown(detail); const primaryDetail = () => status === 'error' ? t('VOICE UNAVAILABLE') diff --git a/src/voice/realtimeInput.js b/src/voice/realtimeInput.js index 9c9fd85e7..0a43fa6ec 100644 --- a/src/voice/realtimeInput.js +++ b/src/voice/realtimeInput.js @@ -1,4 +1,4 @@ -import { t } from '../i18n/index.js'; +import { msg, t } from '../i18n/index.js'; import { PUSH_TO_TALK_HOLD_DELAY_MS, MICROPHONE_VISUALIZER_GATE, @@ -124,7 +124,7 @@ export class RealtimeInput { this.ui.update('root', { pushToTalk: 'held' }); this.setMicrophoneEnabled(true); if (this.status === 'listening') - this.setStatus('listening', t('Release Space to send')); + this.setStatus('listening', msg('Release Space to send')); } else { this.start({ pushToTalk: true }); // start() performs a controlled stop() before connecting. Restore the @@ -201,7 +201,7 @@ export class RealtimeInput { if (!this.pushToTalkMode) return; this.setMicrophoneEnabled(false); if (this.status === 'listening') - this.setStatus('listening', t('Hold Space to talk')); + this.setStatus('listening', msg('Hold Space to talk')); else this.updateVoiceButtonLabel(); } diff --git a/src/voice/realtimeRadio.js b/src/voice/realtimeRadio.js index 065c55c3b..90abce9e8 100644 --- a/src/voice/realtimeRadio.js +++ b/src/voice/realtimeRadio.js @@ -1,4 +1,4 @@ -import { t } from '../i18n/index.js'; +import { msg } from '../i18n/index.js'; import { silenceRadioForVoice, startPreparedRadioAfterPlaybackReady, @@ -261,7 +261,7 @@ export class RealtimeRadio { if (radioHandoff.result?.ok || radioHandoff.cancelled || !stillCurrent) return; if (this.dc?.readyState === 'open' && !this.userTurnPending) { - this.setStatus('listening', t('Radio did not start')); + this.setStatus('listening', msg('Radio did not start')); this.queueResponseCreate( 'Say exactly one short correction: “The Radio station could not start. Voice is still on.”', ); diff --git a/src/voice/sessionCommands.js b/src/voice/sessionCommands.js index ad5ce59bd..2add30636 100644 --- a/src/voice/sessionCommands.js +++ b/src/voice/sessionCommands.js @@ -7,7 +7,7 @@ import { createVoiceSession, isStoppable } from './session.js'; import { VoiceCardControls } from './voiceCard.js'; import { PointerReticle } from './pointerReticle.js'; import { VOICE_PROVIDERS } from './providerCapabilities.js'; -import { t } from '../i18n/index.js'; +import { t, translateKnown } from '../i18n/index.js'; function createDefaultReticle(pointer) { const element = pointer?.element; @@ -114,11 +114,20 @@ export function createVoiceCommands({ // States are machine values; only the label is translated. ui.update('status', { text: () => voiceStatusLabel(event.state) }); // The portable session reports 'Voice off' in English: show the idle - // label instead. Other details are provider messages, shown as they come. + // label instead. Other details are provider messages; one that keeps its + // English catalog text is worded again in the current language. const idle = event.state === 'idle'; - const detail = event.detail === 'Voice off' ? '' : event.detail; + const source = event.sourceDetail ?? event.detail; + const detail = () => + source === 'Voice off' + ? '' + : event.sourceDetail + ? event.sourceParams + ? t(event.sourceDetail, event.sourceParams) + : translateKnown(event.sourceDetail) + : event.detail; ui.update('detail', { - text: () => detail || (idle ? t('Voice off') : t('Voice active')), + text: () => detail() || (idle ? t('Voice off') : t('Voice active')), }); // A typed chat session is not the microphone. ui.update('button', { @@ -127,7 +136,7 @@ export function createVoiceCommands({ ui.update('error', { text: () => event.state === 'error' - ? event.detail || t('Voice could not be started.') + ? detail() || t('Voice could not be started.') : '', // "Check microphone permission" is wrong advice for a missing key. hint: () => !(event.state === 'error' && event.missingKey === true),