From b89f81b7290fe9a118d0c7ed29dfbaa633b0788b Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Tue, 10 Nov 2026 17:06:50 -0800 Subject: [PATCH] fix(voice): keyless Gemini voice drops the microphone hint The Gemini token route now answers a missing key with code API_KEY_MISSING, the session carries it to the error state as missingKey, and the voice control hides "Check microphone permission" for that error, as it already does for OpenAI. The OpenAI controller sends the same flag, so the shared control and the controller agree whichever updates last. --- docs/CURRENT-STATE.md | 2 +- server/providers/gemini/realtime.js | 5 ++++- src/geminiServer.test.mjs | 7 ++++++- src/voice/geminiSession.js | 26 ++++++++++++++++++-------- src/voice/geminiSession.test.mjs | 18 ++++++++++++++++++ src/voice/providerCommands.test.mjs | 19 +++++++++++++++++++ src/voice/realtimeController.js | 7 ++++++- src/voice/sessionCommands.js | 2 ++ 8 files changed, 74 insertions(+), 12 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index d9721b2a3..9a2f938e6 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -1135,7 +1135,7 @@ Source factories have dedicated `layers//source` exports. ALPR and earth Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. An active session's normal Stop publishes one idle transition, including adapters that emit their own terminal state; a restart triggered by that transition keeps ownership of its new startup state. Alternate adapters can use the same controls and action runner. -The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. +The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter. Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 33 tools using separate description-only metadata; a session is offered 32 of them unless `OVERPASS_UPSTREAMS` is set, which adds `osm_query` and its one prompt sentence (`server/providers/openai/tools.js`). Description customization cannot replace argument types, enum values or required fields. diff --git a/server/providers/gemini/realtime.js b/server/providers/gemini/realtime.js index fa8821afa..ec2c87cd3 100644 --- a/server/providers/gemini/realtime.js +++ b/server/providers/gemini/realtime.js @@ -63,7 +63,10 @@ export function createGeminiTokenHandler({ return reply(403, 'Gemini voice requires a same-origin app request'); apiKey = String(resolveApiKey() || '').trim(); if (!apiKey) - return reply(503, 'Add a Gemini API key in Provider Settings'); + return reply(503, { + error: 'Add a Gemini API key in Provider Settings', + code: 'API_KEY_MISSING', + }); model = normalizeGeminiModel(resolveModel()); if (!model) return reply(503, 'The Gemini Live model configuration is invalid'); diff --git a/src/geminiServer.test.mjs b/src/geminiServer.test.mjs index e6d8d2de9..de8211f98 100644 --- a/src/geminiServer.test.mjs +++ b/src/geminiServer.test.mjs @@ -251,7 +251,12 @@ test('mint locks canonical action schemas and server instructions with no perman test('missing key and malformed model fail without an upstream request', async (t) => { const missing = await fixture(t, { resolveApiKey: () => '' }); - assert.equal((await missing.send()).status, 503); + const missingResponse = await missing.send(); + assert.equal(missingResponse.status, 503); + assert.deepEqual(await missingResponse.json(), { + error: 'Add a Gemini API key in Provider Settings', + code: 'API_KEY_MISSING', + }); assert.equal(missing.calls.length, 0); for (const model of [ '', diff --git a/src/voice/geminiSession.js b/src/voice/geminiSession.js index 8996774b3..0ee9fb313 100644 --- a/src/voice/geminiSession.js +++ b/src/voice/geminiSession.js @@ -359,7 +359,11 @@ export class GeminiSessionController { ); } - setStatus(status, detail, recovery) { + /** + * @param {{missingKey?: boolean}} [options] True when the server has no + * Gemini key: the control then drops its microphone hint. + */ + setStatus(status, detail, recovery, { missingKey = false } = {}) { this.status = status; if (status === 'listening' && this.input.pushToTalkMode) detail = this.input.pushToTalkKeyHeld @@ -371,6 +375,7 @@ export class GeminiSessionController { detail: translateKnown(detail), }; if (recovery) event.recovery = translateKnown(recovery); + if (missingKey) event.missingKey = true; this.emit(event); this.input.updateVoiceButtonLabel(); if (['idle', 'connecting', 'error'].includes(status)) @@ -584,12 +589,15 @@ export class GeminiSessionController { const data = await readResponseJsonCapped(response, 256 * 1024, signal); signal.throwIfAborted(); if (!response.ok) - throw new Error( - typeof data?.error === 'string' - ? tokenErrorMessage(data.error) - : t('Gemini token request failed (HTTP {status})', { - status: response.status, - }), + throw Object.assign( + new Error( + typeof data?.error === 'string' + ? tokenErrorMessage(data.error) + : t('Gemini token request failed (HTTP {status})', { + status: response.status, + }), + ), + typeof data?.code === 'string' ? { code: data.code } : {}, ); if ( typeof data?.model !== 'string' || @@ -966,7 +974,9 @@ export class GeminiSessionController { : error?.message || msg('Gemini voice could not start'); this.lastError = detail; this.stop({ preserveStatus: true }); - this.setStatus('error', detail, recovery); + this.setStatus('error', detail, recovery, { + missingKey: error?.code === 'API_KEY_MISSING', + }); } abortTools() { diff --git a/src/voice/geminiSession.test.mjs b/src/voice/geminiSession.test.mjs index 82f403ef7..a97b5caac 100644 --- a/src/voice/geminiSession.test.mjs +++ b/src/voice/geminiSession.test.mjs @@ -3638,3 +3638,21 @@ test('cancelling a queued globe reset also skips its dependent rotation', async h.c.stop(); } }); + +test('a missing server key reaches the error state flagged, other token failures do not', async () => { + for (const [body, missingKey] of [ + [{ error: 'Add a Gemini API key in Provider Settings', code: 'API_KEY_MISSING' }, true], + [{ error: 'Gemini quota is unavailable. Try again later.' }, undefined], + ]) { + const h = harness({ + fetchImpl: async () => Response.json(body, { status: 503 }), + }); + await h.c.start(); + const error = h.events.filter( + (event) => event.type === 'state' && event.state === 'error', + ).at(-1); + assert.equal(error?.detail, body.error); + assert.equal(error?.missingKey, missingKey); + assert.equal(h.sockets.length, 0); + } +}); diff --git a/src/voice/providerCommands.test.mjs b/src/voice/providerCommands.test.mjs index a9acd2804..b2fed2721 100644 --- a/src/voice/providerCommands.test.mjs +++ b/src/voice/providerCommands.test.mjs @@ -871,3 +871,22 @@ test('a provider switch hands chat Settings to the new mount', async (t) => { assert.deepEqual(f.mounts[1].options.presentation, shown); assert.equal(f.facade.provider, next); }); + +test('a missing-key error hides the microphone hint; other errors keep it', async (t) => { + const f = compositionFixture(t, {}); + f.select('gemini'); + await settle(); + const instance = f.instances.at(-1); + const ui = f.views.at(-1); + instance.hooks.emit({ + type: 'state', + state: 'error', + detail: 'Add a Gemini API key in Provider Settings', + missingKey: true, + }); + assert.equal(ui.state.error.text, 'Add a Gemini API key in Provider Settings'); + assert.equal(ui.state.error.hint, false); + instance.hooks.emit({ type: 'state', state: 'error', detail: 'Socket closed' }); + assert.equal(ui.state.error.text, 'Socket closed'); + assert.equal(ui.state.error.hint, true); +}); diff --git a/src/voice/realtimeController.js b/src/voice/realtimeController.js index 56f501f95..58b8aec77 100644 --- a/src/voice/realtimeController.js +++ b/src/voice/realtimeController.js @@ -416,7 +416,12 @@ export class GevRealtimeController extends RealtimeFacade { */ setStatus(status, detail, { missingKey = false } = {}) { this.status = status; - this.emitSessionEvent({ type: 'state', state: status, detail }); + this.emitSessionEvent({ + type: 'state', + state: status, + detail, + ...(missingKey && { missingKey: true }), + }); this.ui.update('root', { status, ...(status === 'error' && { errorDismissed: false }), diff --git a/src/voice/sessionCommands.js b/src/voice/sessionCommands.js index 242a2126b..ad5ce59bd 100644 --- a/src/voice/sessionCommands.js +++ b/src/voice/sessionCommands.js @@ -129,6 +129,8 @@ export function createVoiceCommands({ event.state === 'error' ? event.detail || t('Voice could not be started.') : '', + // "Check microphone permission" is wrong advice for a missing key. + hint: () => !(event.state === 'error' && event.missingKey === true), }); }); // Captions, plan and result card nested in the control; it only reads