diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 9a2f938e6..adbf20824 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1135,7 +1135,7 @@ Source factories have dedicated `layers//source` exports. ALPR and earth Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. An active session's normal Stop publishes one idle transition, including adapters that emit their own terminal state; a restart triggered by that transition keeps ownership of its new startup state. Alternate adapters can use the same controls and action runner. -The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. +The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Gemini's status and error messages keep their English catalog text on the session's `state` event (`sourceDetail`, beside the worded `detail`), so the voice control words them again after a language change. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 33 tools using separate description-only metadata; a session is offered 32 of them unless `OVERPASS_UPSTREAMS` is set, which adds `osm_query` and its one prompt sentence (`server/providers/openai/tools.js`). Description customization cannot replace argument types, enum values or required fields. diff --git a/src/voice/geminiSession.js b/src/voice/geminiSession.js index 0ee9fb313..8900c1552 100644 --- a/src/voice/geminiSession.js +++ b/src/voice/geminiSession.js @@ -71,7 +71,9 @@ export function geminiSocketUrl({ token, websocketUrl }) { return url.href; } -// Known local-server failures are translated; unexpected diagnostics stay intact. +// The local server's known failures, marked for translation. Each is kept as +// its English catalog text, which the status words when it is shown, so a +// language change words it again; unexpected diagnostics stay intact. function tokenErrorMessage(error) { switch (error) { case msg('Gemini voice could not connect. Try again.'): @@ -85,7 +87,6 @@ function tokenErrorMessage(error) { case msg('Token request is too large'): case msg('Use an empty JSON token request'): case msg('Method not allowed'): - return t(error); default: return error; } @@ -369,10 +370,14 @@ export class GeminiSessionController { detail = this.input.pushToTalkKeyHeld ? msg('Release Space to send') : msg('Hold Space to talk'); + // `detail` is worded in the current language; `sourceDetail` keeps the + // English catalog text, so a view can word it again after a language + // change. const event = { type: 'state', state: status, detail: translateKnown(detail), + ...(typeof detail === 'string' && { sourceDetail: detail }), }; if (recovery) event.recovery = translateKnown(recovery); if (missingKey) event.missingKey = true; diff --git a/src/voice/geminiSession.test.mjs b/src/voice/geminiSession.test.mjs index 3c91d3a0a..6e921d827 100644 --- a/src/voice/geminiSession.test.mjs +++ b/src/voice/geminiSession.test.mjs @@ -4,6 +4,7 @@ import { geminiSocketUrl } from './geminiSession.js'; import { createVoiceSession } from './session.js'; import { createPointerTracker } from './pointerContext.js'; import { withToolCatalog } from './gevRealtime.js'; +import { setLocale, t } from '../i18n/index.js'; import { harness, tick, @@ -3662,3 +3663,28 @@ test('a missing server key reaches the error state flagged, other token failures assert.equal(h.sockets.length, 0); } }); + +test('an error keeps its English catalog text beside the worded detail', async (t2) => { + t2.after(() => setLocale('en')); + setLocale('es'); + const h = harness({ + fetchImpl: async () => + Response.json( + { error: 'Gemini quota is unavailable. Try again later.' }, + { status: 503 }, + ), + }); + await h.c.start(); + const error = h.events + .filter((event) => event.type === 'state' && event.state === 'error') + .at(-1); + assert.equal( + error.sourceDetail, + 'Gemini quota is unavailable. Try again later.', + ); + assert.equal( + error.detail, + t('Gemini quota is unavailable. Try again later.'), + ); + assert.notEqual(error.detail, error.sourceDetail); +}); diff --git a/src/voice/providerCommands.test.mjs b/src/voice/providerCommands.test.mjs index dcc38d6c5..96c19ecf4 100644 --- a/src/voice/providerCommands.test.mjs +++ b/src/voice/providerCommands.test.mjs @@ -11,6 +11,7 @@ import { createGevActionRunner } from './gevActions.js'; import { createPointerTracker } from './pointerContext.js'; import { createReferentRegistry } from './referents.js'; import { createTestVoiceUi } from '../testSupport/voiceUi.mjs'; +import { setLocale, t } from '../i18n/index.js'; const PROVIDER_KEY = 'gev.voice.provider'; const settle = () => new Promise((resolve) => setImmediate(resolve)); @@ -897,3 +898,33 @@ test('a missing-key error hides the microphone hint; other errors keep it', asyn assert.equal(ui.state.error.text, 'Socket closed'); assert.equal(ui.state.error.hint, true); }); + +test('a provider message is worded again after a language change', async (t2) => { + t2.after(() => setLocale('en')); + const f = compositionFixture(t2, {}); + f.select('gemini'); + await settle(); + const instance = f.instances.at(-1); + const ui = f.views.at(-1); + const quota = 'Gemini quota is unavailable. Try again later.'; + instance.hooks.emit({ + type: 'state', + state: 'error', + detail: quota, + sourceDetail: quota, + }); + assert.equal(ui.state.error.text, quota); + setLocale('es'); + assert.equal(ui.state.error.text, t(quota)); + assert.notEqual(ui.state.error.text, quota); + setLocale('en'); + instance.hooks.emit({ + type: 'state', + state: 'listening', + detail: 'Ask or command', + sourceDetail: 'Ask or command', + }); + setLocale('es'); + assert.equal(ui.state.detail.text, t('Ask or command')); + assert.notEqual(ui.state.detail.text, 'Ask or command'); +}); diff --git a/src/voice/sessionCommands.js b/src/voice/sessionCommands.js index ad5ce59bd..c3ef1a917 100644 --- a/src/voice/sessionCommands.js +++ b/src/voice/sessionCommands.js @@ -7,7 +7,7 @@ import { createVoiceSession, isStoppable } from './session.js'; import { VoiceCardControls } from './voiceCard.js'; import { PointerReticle } from './pointerReticle.js'; import { VOICE_PROVIDERS } from './providerCapabilities.js'; -import { t } from '../i18n/index.js'; +import { t, translateKnown } from '../i18n/index.js'; function createDefaultReticle(pointer) { const element = pointer?.element; @@ -114,11 +114,18 @@ export function createVoiceCommands({ // States are machine values; only the label is translated. ui.update('status', { text: () => voiceStatusLabel(event.state) }); // The portable session reports 'Voice off' in English: show the idle - // label instead. Other details are provider messages, shown as they come. + // label instead. Other details are provider messages; one that keeps its + // English catalog text is worded again in the current language. const idle = event.state === 'idle'; - const detail = event.detail === 'Voice off' ? '' : event.detail; + const source = event.sourceDetail ?? event.detail; + const detail = () => + source === 'Voice off' + ? '' + : event.sourceDetail + ? translateKnown(event.sourceDetail) + : event.detail; ui.update('detail', { - text: () => detail || (idle ? t('Voice off') : t('Voice active')), + text: () => detail() || (idle ? t('Voice off') : t('Voice active')), }); // A typed chat session is not the microphone. ui.update('button', { @@ -127,7 +134,7 @@ export function createVoiceCommands({ ui.update('error', { text: () => event.state === 'error' - ? event.detail || t('Voice could not be started.') + ? detail() || t('Voice could not be started.') : '', // "Check microphone permission" is wrong advice for a missing key. hint: () => !(event.state === 'error' && event.missingKey === true),