Merge pull request #1660 from bilawalsidhu/sameh/gemini-keyless-mic-hint

fix(voice): keyless Gemini voice drops the microphone hint

changelog.d/1660-keyless-gemini-mic-hint.md

@@ -0,0 +1,4 @@
+- Keyless Gemini voice no longer blames the microphone. Without
+  `GEMINI_API_KEY`, the voice control shows only "Add a Gemini API key in
+  Provider Settings", without the microphone-permission hint, as OpenAI voice
+  already does (Sameh Khamis, @samehkhamis, #1660).

docs/CURRENT-STATE.md

@@ -1135,7 +1135,7 @@ Source factories have dedicated `layers/<family>/source` exports. ALPR and earth
 
 Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. An active session's normal Stop publishes one idle transition, including adapters that emit their own terminal state; a restart triggered by that transition keeps ownership of its new startup state. Alternate adapters can use the same controls and action runner.
 
-The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter.
+The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter.
 
 Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 33 tools using separate description-only metadata; a session is offered 32 of them unless `OVERPASS_UPSTREAMS` is set, which adds `osm_query` and its one prompt sentence (`server/providers/openai/tools.js`). Description customization cannot replace argument types, enum values or required fields.
 

server/providers/gemini/realtime.js

@@ -63,7 +63,10 @@ export function createGeminiTokenHandler({
         return reply(403, 'Gemini voice requires a same-origin app request');
       apiKey = String(resolveApiKey() || '').trim();
       if (!apiKey)
-        return reply(503, 'Add a Gemini API key in Provider Settings');
+        return reply(503, {
+          error: 'Add a Gemini API key in Provider Settings',
+          code: 'API_KEY_MISSING',
+        });
       model = normalizeGeminiModel(resolveModel());
       if (!model)
         return reply(503, 'The Gemini Live model configuration is invalid');

src/geminiServer.test.mjs

@@ -251,7 +251,12 @@ test('mint locks canonical action schemas and server instructions with no perman
 
 test('missing key and malformed model fail without an upstream request', async (t) => {
   const missing = await fixture(t, { resolveApiKey: () => '' });
-  assert.equal((await missing.send()).status, 503);
+  const missingResponse = await missing.send();
+  assert.equal(missingResponse.status, 503);
+  assert.deepEqual(await missingResponse.json(), {
+    error: 'Add a Gemini API key in Provider Settings',
+    code: 'API_KEY_MISSING',
+  });
   assert.equal(missing.calls.length, 0);
   for (const model of [
     '',

src/voice/geminiSession.js

@@ -359,7 +359,11 @@ export class GeminiSessionController {
     );
   }
 
-  setStatus(status, detail, recovery) {
+  /**
+   * @param {{missingKey?: boolean}} [options] True when the server has no
+   *   Gemini key: the control then drops its microphone hint.
+   */
+  setStatus(status, detail, recovery, { missingKey = false } = {}) {
     this.status = status;
     if (status === 'listening' && this.input.pushToTalkMode)
       detail = this.input.pushToTalkKeyHeld
@@ -371,6 +375,7 @@ export class GeminiSessionController {
       detail: translateKnown(detail),
     };
     if (recovery) event.recovery = translateKnown(recovery);
+    if (missingKey) event.missingKey = true;
     this.emit(event);
     this.input.updateVoiceButtonLabel();
     if (['idle', 'connecting', 'error'].includes(status))
@@ -584,12 +589,15 @@ export class GeminiSessionController {
       const data = await readResponseJsonCapped(response, 256 * 1024, signal);
       signal.throwIfAborted();
       if (!response.ok)
-        throw new Error(
-          typeof data?.error === 'string'
-            ? tokenErrorMessage(data.error)
-            : t('Gemini token request failed (HTTP {status})', {
-                status: response.status,
-              }),
+        throw Object.assign(
+          new Error(
+            typeof data?.error === 'string'
+              ? tokenErrorMessage(data.error)
+              : t('Gemini token request failed (HTTP {status})', {
+                  status: response.status,
+                }),
+          ),
+          typeof data?.code === 'string' ? { code: data.code } : {},
         );
       if (
         typeof data?.model !== 'string' ||
@@ -966,7 +974,9 @@ export class GeminiSessionController {
           : error?.message || msg('Gemini voice could not start');
     this.lastError = detail;
     this.stop({ preserveStatus: true });
-    this.setStatus('error', detail, recovery);
+    this.setStatus('error', detail, recovery, {
+      missingKey: error?.code === 'API_KEY_MISSING',
+    });
   }
 
   abortTools() {

src/voice/geminiSession.test.mjs

@@ -3638,3 +3638,27 @@ test('cancelling a queued globe reset also skips its dependent rotation', async
     h.c.stop();
   }
 });
+
+test('a missing server key reaches the error state flagged, other token failures do not', async () => {
+  for (const [body, missingKey] of [
+    [
+      {
+        error: 'Add a Gemini API key in Provider Settings',
+        code: 'API_KEY_MISSING',
+      },
+      true,
+    ],
+    [{ error: 'Gemini quota is unavailable. Try again later.' }, undefined],
+  ]) {
+    const h = harness({
+      fetchImpl: async () => Response.json(body, { status: 503 }),
+    });
+    await h.c.start();
+    const error = h.events
+      .filter((event) => event.type === 'state' && event.state === 'error')
+      .at(-1);
+    assert.equal(error?.detail, body.error);
+    assert.equal(error?.missingKey, missingKey);
+    assert.equal(h.sockets.length, 0);
+  }
+});

src/voice/providerCommands.test.mjs

@@ -871,3 +871,29 @@ test('a provider switch hands chat Settings to the new mount', async (t) => {
   assert.deepEqual(f.mounts[1].options.presentation, shown);
   assert.equal(f.facade.provider, next);
 });
+
+test('a missing-key error hides the microphone hint; other errors keep it', async (t) => {
+  const f = compositionFixture(t, {});
+  f.select('gemini');
+  await settle();
+  const instance = f.instances.at(-1);
+  const ui = f.views.at(-1);
+  instance.hooks.emit({
+    type: 'state',
+    state: 'error',
+    detail: 'Add a Gemini API key in Provider Settings',
+    missingKey: true,
+  });
+  assert.equal(
+    ui.state.error.text,
+    'Add a Gemini API key in Provider Settings',
+  );
+  assert.equal(ui.state.error.hint, false);
+  instance.hooks.emit({
+    type: 'state',
+    state: 'error',
+    detail: 'Socket closed',
+  });
+  assert.equal(ui.state.error.text, 'Socket closed');
+  assert.equal(ui.state.error.hint, true);
+});

src/voice/realtimeController.js

@@ -416,7 +416,12 @@ export class GevRealtimeController extends RealtimeFacade {
    */
   setStatus(status, detail, { missingKey = false } = {}) {
     this.status = status;
-    this.emitSessionEvent({ type: 'state', state: status, detail });
+    this.emitSessionEvent({
+      type: 'state',
+      state: status,
+      detail,
+      ...(missingKey && { missingKey: true }),
+    });
     this.ui.update('root', {
       status,
       ...(status === 'error' && { errorDismissed: false }),

src/voice/sessionCommands.js

@@ -129,6 +129,8 @@ export function createVoiceCommands({
         event.state === 'error'
           ? event.detail || t('Voice could not be started.')
           : '',
+      // "Check microphone permission" is wrong advice for a missing key.
+      hint: () => !(event.state === 'error' && event.missingKey === true),
     });
   });
   // Captions, plan and result card nested in the control; it only reads