feat(voice): add typed AI chat beside voice

Chat sits next to the mic in the command dock. Replies are silent and never
request the microphone; the first message connects with the selected provider
and runs the same map actions as voice through each provider's text API
(OpenAI OPENAI_CHAT_MODEL, Gemini GEMINI_CHAT_MODEL via generateContent, which
never starts Live). Only one mode runs at a time.

- Chat panel rendered by the voice control presenter: conversation, recent and
  quick-command suggestions, Up/Down history, attachments (map-layer
  references or small UTF-8 text files, sent only with the message)
- Chat and voice panels sit above the dock or dock into the right rail's
  column; minimized bars sit centred above the dock, and height animates on
  swaps
- A centred panel steps down to its bar for the Location and Visual Presets
  trays without moving focus, rests above an open tray, and docks right while
  expanded over one; it returns to the centre once the trays close
- Minimized voice previews the last sentence from each side; the empty Voice
  Preview suggests things to say
- Settings opened from chat keep chat's frame and show the shared AI provider
  and quick commands; voice Settings show Back to voice
- Closing chat ends a typed session; the voice panel's close turns voice off
- The dock widens its voice control for the chat button and restyles its
  wings, which show icons only at 900px and below; assistant panels share the
  left panels' glass, and Cyber keeps its own surfaces
- Server text-chat handlers with body, time and per-IP limits
  (GEV_RATELIMIT_*_CHAT_PER_MIN), documented in SECURITY.md; tests and all
  eleven locales

Co-authored-by: Darren Dignam <[email protected]>
This commit is contained in:
manjunath22466andDarren Dignam committed 2026-11-02 20:56:39 +05:30
1 parent 720260af58
commit af461febae
57 files changed
+6500 -63

No files matched your search

+492
View File
@@ -0,0 +1,492 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createGeminiTextChat } from './geminiTextChat.js';
import { boundedToolResult } from './textProtocol.js';
import { createTextChat } from './textChat.js';
import { createVoiceSession } from './session.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
const tick = () => new Promise((resolve) => setImmediate(resolve));
const final = () =>
Response.json({
content: { role: 'model', parts: [{ text: 'Done' }] },
finishReason: 'STOP',
});
test('text chat executes fresh calls and preserves signatures and function results without media', async () => {
const requests = [],
calls = [],
events = [];
let count = 0;
const session = createVoiceSession({
runner: async (name, args) => {
calls.push({ name, args });
return { ok: true, say: 'Moved', records: ['界'.repeat(15000)] };
},
createAdapter: (hooks) =>
createGeminiTextChat({
...hooks,
fetchImpl: async (url, options) => {
requests.push({ url, body: JSON.parse(options.body) });
return ++count === 1
? Response.json({
content: {
role: 'model',
parts: [
{
thoughtSignature: 'opaque',
functionCall: {
name: 'fly_to',
args: { lat: 1, lon: 2 },
id: 'call',
},
},
],
},
usage: { inputTokens: 10, outputTokens: 2, totalTokens: 12 },
})
: Response.json({
content: { role: 'model', parts: [{ text: 'Done' }] },
finishReason: 'STOP',
usage: { inputTokens: 20, outputTokens: 3, totalTokens: 23 },
});
},
}),
});
session.subscribe((event) => events.push(event));
await session.start({ textOnly: true });
assert.equal(session.sendText('Go there'), true);
for (let i = 0; i < 5; i++) await tick();
assert.equal(requests.length, 2);
assert.equal(requests[0].url, '/api/gemini/chat');
assert.equal(calls.length, 1);
assert.equal(
requests[1].body.contents[1].parts[0].thoughtSignature,
'opaque',
);
assert.equal(
requests[1].body.contents[2].parts[0].functionResponse.id,
'call',
);
assert.ok(
events.some(
(event) => event.type === 'completion' && event.status === 'completed',
),
);
const result =
requests[1].body.contents[2].parts[0].functionResponse.response;
assert.equal(result.ok, true);
assert.equal(result.say, 'Moved');
assert.equal(result.truncated, true);
assert.ok(new TextEncoder().encode(JSON.stringify(result)).length <= 12000);
const usage = events.find((event) => event.type === 'completion').usage;
assert.equal(usage.requests, 2);
assert.equal(usage.totalTokens, 35);
assert.equal(usage.unknownRequests, 0);
session.destroy();
});
test('stop cancels a pending tool and prevents stale follow-up requests or completion', async () => {
let resolve,
actionSignal,
requests = 0;
const events = [];
const session = createVoiceSession({
runner: (_name, _args, options) => {
actionSignal = options.signal;
return new Promise((done) => {
resolve = done;
});
},
createAdapter: (hooks) =>
createGeminiTextChat({
...hooks,
fetchImpl: async () => {
requests++;
return Response.json({
content: {
role: 'model',
parts: [{ functionCall: { name: 'fly_to', args: {} } }],
},
});
},
}),
});
session.subscribe((event) => events.push(event));
await session.start();
session.sendText('Go');
for (let i = 0; i < 5 && !resolve; i++) await tick();
session.stop();
assert.equal(actionSignal.aborted, true);
resolve({ ok: true });
await tick();
assert.equal(requests, 1);
assert.ok(!events.some((event) => event.type === 'completion'));
session.destroy();
});
test('runaway tool responses stop at the bounded turn limit', async () => {
let requests = 0;
const events = [];
const chat = createGeminiTextChat({
emit: (event) => events.push(event),
runAction: async () => ({ ok: true }),
fetchImpl: async () => {
requests++;
return Response.json({
content: {
role: 'model',
parts: [{ functionCall: { name: 'fly_to', args: {} } }],
},
});
},
});
chat.start();
chat.sendText('Go');
for (let i = 0; i < 20; i++) await tick();
assert.equal(requests, 8);
assert.ok(events.some((event) => event.state === 'error'));
chat.stop();
});
test('composite chat never starts Live and the next keyboard start resets its mode', async () => {
const { createGeminiSession } = await import('./geminiSession.js');
const events = [];
const adapter = createGeminiSession({
emit: (event) => events.push(event),
runAction: async () => ({ ok: true }),
ui: createTestVoiceUi(),
fetchImpl: () => {
throw Error('No startup request');
},
getUserMedia: () => {
throw Error('No microphone');
},
createAudio: () => {
throw Error('No audio');
},
});
await adapter.start({ textOnly: true });
assert.equal(adapter.getStartOptions().textOnly, true);
assert.equal(events.at(-1).state, 'listening');
adapter.stop();
adapter.controller.start = async () => {};
adapter.controller.sendTextCommand = () => 'live';
await adapter.controller.input.start({ pushToTalk: true });
assert.equal(adapter.getStartOptions().textOnly, undefined);
assert.equal(adapter.sendText('Hello'), 'live');
adapter.stop({ removeUi: true });
});
test('failed Radio startup cancels its stream before releasing the mute', async () => {
const actions = [];
let count = 0;
const chat = createGeminiTextChat({
emit: () => {},
runAction: async () => ({ ok: true, radioPlaybackRequested: true }),
radioLayer: {
setVoiceDucked: (v) => actions.push(v),
playForVoice: async () => {
throw Error('blocked');
},
stopPlayback: () => actions.push('stop'),
},
fetchImpl: async () =>
++count === 1
? Response.json({
content: {
role: 'model',
parts: [
{
functionCall: {
name: 'control_radio',
args: { action: 'play' },
},
},
],
},
})
: final(),
});
chat.start();
chat.sendText('Play radio');
for (let i = 0; i < 5; i++) await tick();
assert.deepEqual(actions, [true, 'stop', false]);
chat.stop();
});
test('retired chat handoff cannot unmute the next voice session', async () => {
const actions = [];
let resolve;
const chat = createGeminiTextChat({
emit: () => {},
runAction: async () => ({ ok: true, radioPlaybackRequested: true }),
radioLayer: {
setVoiceDucked: (v) => actions.push(v),
playForVoice: () =>
new Promise((done) => {
resolve = done;
}),
stopPlayback: () => actions.push('stop'),
},
fetchImpl: async () =>
Response.json({
content: {
role: 'model',
parts: [
{
functionCall: { name: 'control_radio', args: { action: 'play' } },
},
],
},
}),
});
chat.start();
chat.sendText('Play');
for (let i = 0; i < 5 && !resolve; i++) await tick();
chat.stop();
actions.push('new-voice-muted');
resolve(true);
await tick();
assert.deepEqual(actions, [true, 'stop', false, 'new-voice-muted']);
});
for (const intent of ['pause', 'off'])
test(`user Radio ${intent} cancels delayed text preparation`, async () => {
let controls,
visibility,
resolve,
toolSignal,
plays = 0;
const chat = createGeminiTextChat({
emit: () => {},
runAction: (_name, _args, options) => {
toolSignal = options.signal;
return new Promise((done) => {
resolve = done;
});
},
radioLayer: {
subscribePlaybackControls: (cb) => {
controls = cb;
return () => {};
},
playForVoice: () => {
plays++;
return true;
},
},
dataManager: {
subscribeVisibilityRequests: (cb) => {
visibility = cb;
return () => {};
},
},
fetchImpl: async () =>
Response.json({
content: {
role: 'model',
parts: [
{
functionCall: {
name: 'control_radio',
args: { action: 'play' },
},
},
],
},
}),
});
chat.start();
chat.sendText('Play');
for (let i = 0; i < 5 && !resolve; i++) await tick();
if (intent === 'pause') controls({ origin: 'user', action: 'pause' });
else visibility({ origin: 'user', layerId: 'radio', enabled: false });
assert.equal(toolSignal.aborted, true);
resolve({ ok: true, radioPlaybackRequested: true });
await tick();
assert.equal(plays, 0);
});
test('a tool deadline reports one error, aborts work and admits a successor', async () => {
const events = [];
let actionSignal,
requests = 0;
const chat = createGeminiTextChat({
emit: (e) => events.push(e),
timeoutMs: 30,
runAction: (_n, _a, o) => {
actionSignal = o.signal;
return new Promise(() => {});
},
fetchImpl: async () =>
++requests === 1
? Response.json({
content: {
role: 'model',
parts: [{ functionCall: { name: 'fly_to', args: {} } }],
},
usage: { inputTokens: 5, outputTokens: 1, totalTokens: 6 },
})
: final(),
});
chat.start();
chat.sendText('Go');
await new Promise((resolve) => setTimeout(resolve, 70));
assert.equal(actionSignal.aborted, true);
assert.equal(events.filter((e) => e.state === 'error').length, 1);
assert.equal(requests, 1);
assert.equal(events.find((e) => e.state === 'error').usage.totalTokens, 6);
assert.ok(!events.some((e) => e.type === 'completion'));
chat.start();
assert.equal(chat.sendText('Hello'), true);
for (let i = 0; i < 5; i++) await tick();
assert.ok(events.some((e) => e.type === 'completion'));
chat.stop();
});
test('usage survives a later failed request and explicitly marks its unknown tokens', async () => {
const events = [];
let count = 0;
const chat = createGeminiTextChat({
emit: (e) => events.push(e),
runAction: async () => ({ ok: true }),
fetchImpl: async () =>
++count === 1
? Response.json({
content: {
role: 'model',
parts: [{ functionCall: { name: 'fly_to', args: {} } }],
},
usage: { inputTokens: 5, outputTokens: 1, totalTokens: 6 },
})
: Response.json({ error: 'unavailable' }, { status: 503 }),
});
chat.start();
chat.sendText('Go');
for (let i = 0; i < 6; i++) await tick();
const usage = events.find((e) => e.state === 'error').usage;
assert.equal(usage.totalTokens, 6);
assert.equal(usage.requests, 2);
assert.equal(usage.unknownRequests, 1);
chat.stop();
});
test('invalid mixed batches preserve every pair and a corrected reset can execute', async () => {
const requests = [],
executed = [];
let count = 0;
const call = (name, id) => ({ functionCall: { name, id, args: {} } });
const chat = createGeminiTextChat({
emit: () => {},
runAction: async (name) => {
executed.push(name);
return { ok: true };
},
fetchImpl: async (_url, options) => {
requests.push(JSON.parse(options.body));
count++;
return count === 1
? Response.json({
content: {
role: 'model',
parts: [
{ text: 'Planning' },
call('zoom_to_globe', 'bad'),
call('fly_to', 'sibling'),
],
},
invalidCalls: [{ index: 0, error: 'Invalid arguments' }],
})
: count === 2
? Response.json({
content: {
role: 'model',
parts: [call('zoom_to_globe', 'corrected')],
},
})
: final();
},
});
chat.start();
chat.sendText('Reset');
for (let i = 0; i < 8; i++) await tick();
assert.deepEqual(executed, ['zoom_to_globe']);
const parts = requests[1].contents[2].parts;
assert.deepEqual(
parts.map((p) => p.functionResponse.id),
['bad', 'sibling'],
);
assert.ok(parts.every((p) => p.functionResponse.response.ok === false));
assert.equal(
requests[2].contents[4].parts[0].functionResponse.response.ok,
true,
);
chat.stop();
});
test('result reduction stays bounded JSON when essential details or serialization fail', () => {
const result = boundedToolResult({ ok: false, error: '界'.repeat(20000) });
assert.ok(new TextEncoder().encode(JSON.stringify(result)).length <= 12000);
assert.equal(result.ok, false);
const cycle = { ok: true };
cycle.self = cycle;
assert.equal(boundedToolResult(cycle).ok, false);
});
test('OpenAI native tool pairs round-trip malformed arguments and incomplete usage', async () => {
const requests = [],
events = [],
actions = [];
let count = 0;
const chat = createTextChat({
provider: 'openai',
emit: (e) => events.push(e),
runAction: async (name) => {
actions.push(name);
return { ok: true };
},
fetchImpl: async (url, options) => {
assert.equal(url, '/api/openai/chat');
requests.push(JSON.parse(options.body));
count++;
return Response.json(
count === 1
? {
content: {
role: 'assistant',
content: null,
tool_calls: [
{
id: 'a',
type: 'function',
function: { name: 'fly_to', arguments: '{' },
},
{
id: 'b',
type: 'function',
function: { name: 'fly_to', arguments: '{}' },
},
],
},
invalidCalls: [{ index: 0, error: 'JSON invalid' }],
usage: { inputTokens: 2, outputTokens: 1, totalTokens: 3 },
}
: {
content: { role: 'assistant', content: 'Partial' },
finishReason: 'length',
usage: { inputTokens: 3, outputTokens: 2, totalTokens: 5 },
},
);
},
});
chat.start();
chat.sendText('Go');
for (let i = 0; i < 6; i++) await tick();
assert.deepEqual(actions, []);
assert.deepEqual(
requests[1].messages.slice(2).map((m) => m.tool_call_id),
['a', 'b'],
);
assert.equal(events.find((e) => e.state === 'error').usage.totalTokens, 8);
chat.stop();
});