fix(voice): keyless Gemini voice drops the microphone hint

The Gemini token route now answers a missing key with code API_KEY_MISSING,
the session carries it to the error state as missingKey, and the voice
control hides "Check microphone permission" for that error, as it already
does for OpenAI. The OpenAI controller sends the same flag, so the shared
control and the controller agree whichever updates last.
This commit is contained in:
Sameh Khamis committed 2026-11-10 17:06:50 -08:00
1 parent 9560a6dcd1
commit b89f81b729
8 files changed
+74 -12

No files matched your search

+1 -1
View File
@@ -1135,7 +1135,7 @@ Source factories have dedicated `layers/<family>/source` exports. ALPR and earth
Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/progress/result, interruption, completion and turn-metrics events; stopping or replacing a session cancels pending actions. An active session's normal Stop publishes one idle transition, including adapters that emit their own terminal state; a restart triggered by that transition keeps ownership of its new startup state. Alternate adapters can use the same controls and action runner.
The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter.
The standalone app offers OpenAI and Gemini Live through the **AI Provider** setting, shared by voice and typed chat. Gemini keeps its permanent API key on the local server and mints a short-lived, single-use credential with server-owned instructions and the standalone action catalog. With no Gemini key the token route answers `503` with code `API_KEY_MISSING`, and the voice control says `Add a Gemini API key in Provider Settings` without the microphone-permission hint. Its native WebSocket adapter supports open microphone and push-to-talk input, streaming audio, captions and the existing action runner. In push-to-talk, a tool call that arrives with no input, reply or tool result outstanding is answered with a failure and never runs, and neither does any other call in the reply it starts; open-microphone turns start on Gemini's server-side activity detection, which reports no boundary to check. Switching providers ends the old conversation and releases its microphone and connection; only the input mode carries over. Gemini uses bounded shutdown draining and explicit manual restart, without automatic reconnection or a promise of uninterrupted session duration. Hosts that do not opt into the provider facade retain their existing default or custom adapter.
Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds 33 tools using separate description-only metadata; a session is offered 32 of them unless `OVERPASS_UPSTREAMS` is set, which adds `osm_query` and its one prompt sentence (`server/providers/openai/tools.js`). Description customization cannot replace argument types, enum values or required fields.
+4 -1
View File
@@ -63,7 +63,10 @@ export function createGeminiTokenHandler({
return reply(403, 'Gemini voice requires a same-origin app request');
apiKey = String(resolveApiKey() || '').trim();
if (!apiKey)
return reply(503, 'Add a Gemini API key in Provider Settings');
return reply(503, {
error: 'Add a Gemini API key in Provider Settings',
code: 'API_KEY_MISSING',
});
model = normalizeGeminiModel(resolveModel());
if (!model)
return reply(503, 'The Gemini Live model configuration is invalid');
+6 -1
View File
@@ -251,7 +251,12 @@ test('mint locks canonical action schemas and server instructions with no perman
test('missing key and malformed model fail without an upstream request', async (t) => {
const missing = await fixture(t, { resolveApiKey: () => '' });
assert.equal((await missing.send()).status, 503);
const missingResponse = await missing.send();
assert.equal(missingResponse.status, 503);
assert.deepEqual(await missingResponse.json(), {
error: 'Add a Gemini API key in Provider Settings',
code: 'API_KEY_MISSING',
});
assert.equal(missing.calls.length, 0);
for (const model of [
'',
+18 -8
View File
@@ -359,7 +359,11 @@ export class GeminiSessionController {
);
}
setStatus(status, detail, recovery) {
/**
* @param {{missingKey?: boolean}} [options] True when the server has no
* Gemini key: the control then drops its microphone hint.
*/
setStatus(status, detail, recovery, { missingKey = false } = {}) {
this.status = status;
if (status === 'listening' && this.input.pushToTalkMode)
detail = this.input.pushToTalkKeyHeld
@@ -371,6 +375,7 @@ export class GeminiSessionController {
detail: translateKnown(detail),
};
if (recovery) event.recovery = translateKnown(recovery);
if (missingKey) event.missingKey = true;
this.emit(event);
this.input.updateVoiceButtonLabel();
if (['idle', 'connecting', 'error'].includes(status))
@@ -584,12 +589,15 @@ export class GeminiSessionController {
const data = await readResponseJsonCapped(response, 256 * 1024, signal);
signal.throwIfAborted();
if (!response.ok)
throw new Error(
typeof data?.error === 'string'
? tokenErrorMessage(data.error)
: t('Gemini token request failed (HTTP {status})', {
status: response.status,
}),
throw Object.assign(
new Error(
typeof data?.error === 'string'
? tokenErrorMessage(data.error)
: t('Gemini token request failed (HTTP {status})', {
status: response.status,
}),
),
typeof data?.code === 'string' ? { code: data.code } : {},
);
if (
typeof data?.model !== 'string' ||
@@ -966,7 +974,9 @@ export class GeminiSessionController {
: error?.message || msg('Gemini voice could not start');
this.lastError = detail;
this.stop({ preserveStatus: true });
this.setStatus('error', detail, recovery);
this.setStatus('error', detail, recovery, {
missingKey: error?.code === 'API_KEY_MISSING',
});
}
abortTools() {
+18
View File
@@ -3638,3 +3638,21 @@ test('cancelling a queued globe reset also skips its dependent rotation', async
h.c.stop();
}
});
test('a missing server key reaches the error state flagged, other token failures do not', async () => {
for (const [body, missingKey] of [
[{ error: 'Add a Gemini API key in Provider Settings', code: 'API_KEY_MISSING' }, true],
[{ error: 'Gemini quota is unavailable. Try again later.' }, undefined],
]) {
const h = harness({
fetchImpl: async () => Response.json(body, { status: 503 }),
});
await h.c.start();
const error = h.events.filter(
(event) => event.type === 'state' && event.state === 'error',
).at(-1);
assert.equal(error?.detail, body.error);
assert.equal(error?.missingKey, missingKey);
assert.equal(h.sockets.length, 0);
}
});
+19
View File
@@ -871,3 +871,22 @@ test('a provider switch hands chat Settings to the new mount', async (t) => {
assert.deepEqual(f.mounts[1].options.presentation, shown);
assert.equal(f.facade.provider, next);
});
test('a missing-key error hides the microphone hint; other errors keep it', async (t) => {
const f = compositionFixture(t, {});
f.select('gemini');
await settle();
const instance = f.instances.at(-1);
const ui = f.views.at(-1);
instance.hooks.emit({
type: 'state',
state: 'error',
detail: 'Add a Gemini API key in Provider Settings',
missingKey: true,
});
assert.equal(ui.state.error.text, 'Add a Gemini API key in Provider Settings');
assert.equal(ui.state.error.hint, false);
instance.hooks.emit({ type: 'state', state: 'error', detail: 'Socket closed' });
assert.equal(ui.state.error.text, 'Socket closed');
assert.equal(ui.state.error.hint, true);
});
+6 -1
View File
@@ -416,7 +416,12 @@ export class GevRealtimeController extends RealtimeFacade {
*/
setStatus(status, detail, { missingKey = false } = {}) {
this.status = status;
this.emitSessionEvent({ type: 'state', state: status, detail });
this.emitSessionEvent({
type: 'state',
state: status,
detail,
...(missingKey && { missingKey: true }),
});
this.ui.update('root', {
status,
...(status === 'error' && { errorDismissed: false }),
+2
View File
@@ -129,6 +129,8 @@ export function createVoiceCommands({
event.state === 'error'
? event.detail || t('Voice could not be started.')
: '',
// "Check microphone permission" is wrong advice for a missing key.
hint: () => !(event.state === 'error' && event.missingKey === true),
});
});
// Captions, plan and result card nested in the control; it only reads