feat(voice): add typed AI chat beside voice

Chat sits next to the mic in the command dock. Replies are silent and never
request the microphone; the first message connects with the selected provider
and runs the same map actions as voice through each provider's text API
(OpenAI OPENAI_CHAT_MODEL, Gemini GEMINI_CHAT_MODEL via generateContent, which
never starts Live). Only one mode runs at a time.

- Chat panel rendered by the voice control presenter: conversation, recent and
  quick-command suggestions, Up/Down history, attachments (map-layer
  references or small UTF-8 text files, sent only with the message)
- Chat and voice panels sit above the dock or dock into the right rail's
  column; minimized bars sit centred above the dock, and height animates on
  swaps
- A centred panel steps down to its bar for the Location and Visual Presets
  trays without moving focus, rests above an open tray, and docks right while
  expanded over one; it returns to the centre once the trays close
- Minimized voice previews the last sentence from each side; the empty Voice
  Preview suggests things to say
- Settings opened from chat keep chat's frame and show the shared AI provider
  and quick commands; voice Settings show Back to voice
- Closing chat ends a typed session; the voice panel's close turns voice off
- The dock widens its voice control for the chat button and restyles its
  wings, which show icons only at 900px and below; assistant panels share the
  left panels' glass, and Cyber keeps its own surfaces
- Server text-chat handlers with body, time and per-IP limits
  (GEV_RATELIMIT_*_CHAT_PER_MIN), documented in SECURITY.md; tests and all
  eleven locales

Co-authored-by: Darren Dignam <[email protected]>
This commit is contained in:
manjunath22466andDarren Dignam committed 2026-11-02 20:56:39 +05:30
1 parent 720260af58
commit af461febae
57 files changed
+6500 -63

No files matched your search

+276
View File
@@ -0,0 +1,276 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import http from 'node:http';
import { once } from 'node:events';
import { createGeminiChatHandler } from '../server/providers/gemini/chat.js';
import { createOpenAiChatHandler } from '../server/providers/openai/chat.js';
import {
realtimeDirectives,
realtimeInstructions,
} from '../server/providers/openai/instructions.js';
import {
TEXT_WORDING,
textInstructions,
} from '../server/providers/common/textInstructions.js';
const tools = [
{
name: 'fly_to',
description: 'Fly',
parameters: {
type: 'object',
properties: { lat: { type: 'number', minimum: -90, maximum: 90 } },
required: ['lat'],
additionalProperties: false,
},
},
];
const providers = {
gemini: {
create: createGeminiChatHandler,
input: { contents: [{ role: 'user', parts: [{ text: 'Hello' }] }] },
reply: (bad = false) => ({
candidates: [
{
content: {
role: 'model',
parts: [
{
thoughtSignature: 'opaque',
functionCall: {
name: 'fly_to',
id: 'c',
args: { lat: bad ? 100 : 1 },
},
},
],
},
finishReason: 'STOP',
},
],
usageMetadata: {
promptTokenCount: 7,
candidatesTokenCount: 2,
totalTokenCount: 10,
thoughtsTokenCount: 1,
},
}),
},
openai: {
create: createOpenAiChatHandler,
input: { messages: [{ role: 'user', content: 'Hello' }] },
reply: (bad = false) => ({
choices: [
{
message: {
role: 'assistant',
content: null,
tool_calls: [
{
type: 'function',
id: 'c',
function: {
name: 'fly_to',
arguments: bad ? '{' : '{"lat":1}',
},
},
],
},
finish_reason: 'tool_calls',
},
],
usage: { prompt_tokens: 7, completion_tokens: 3, total_tokens: 10 },
}),
},
};
async function fixture(t, provider, fetchImpl) {
const calls = [];
const handler = provider.create({
tools,
resolveApiKey: () => 'fixture-only',
resolveRateLimit: () => '0',
resolveHost: () => '',
resolveAllowedHosts: () => '',
fetchImpl:
fetchImpl ||
(async (url, options) => {
calls.push({ url, body: JSON.parse(options.body) });
return Response.json(provider.reply(true));
}),
});
const server = http.createServer((req, res) => void handler(req, res));
server.listen(0, '127.0.0.1');
await once(server, 'listening');
t.after(async () => {
server.closeAllConnections();
await new Promise((resolve) => server.close(resolve));
});
const origin = `http://127.0.0.1:${server.address().port}`;
return {
calls,
origin,
send: (input) =>
fetch(origin, {
method: 'POST',
headers: { Origin: origin, 'Content-Type': 'application/json' },
body: JSON.stringify(input ?? provider.input),
}),
};
}
for (const [name, provider] of Object.entries(providers)) {
test(`${name} text route validates calls, retains usage and only accepts paired history`, async (t) => {
const f = await fixture(t, provider);
const response = await f.send();
assert.equal(response.status, 200);
const body = await response.json();
assert.equal(body.invalidCalls[0].index, 0);
assert.equal(body.usage.totalTokens, 10);
assert.equal(body.usage.outputTokens, 3);
const nullCall = provider.reply();
if (name === 'gemini')
nullCall.candidates[0].content.parts[0].functionCall.args = null;
else nullCall.choices[0].message.tool_calls[0].function.arguments = 'null';
const nullFixture = await fixture(t, provider, async () =>
Response.json(nullCall),
);
const nullBody = await (await nullFixture.send()).json();
assert.equal(nullBody.invalidCalls.length, 1);
assert.match(nullBody.invalidCalls[0].error, /must be an object/);
const input =
name === 'gemini'
? {
contents: [
...provider.input.contents,
body.content,
{
role: 'user',
parts: [
{
functionResponse: {
name: 'fly_to',
id: 'c',
response: { ok: false, error: 'Invalid' },
},
},
],
},
],
}
: {
messages: [
...provider.input.messages,
body.content,
{ role: 'tool', tool_call_id: 'c', content: '{"ok":false}' },
],
};
assert.equal((await f.send(input)).status, 200);
const orphan =
name === 'gemini'
? { contents: [...input.contents.slice(0, -1)] }
: { messages: input.messages.slice(0, -1) };
assert.equal((await f.send(orphan)).status, 400);
assert.equal(
(await f.send({ ...provider.input, model: 'caller-model' })).status,
400,
);
if (name === 'openai') {
assert.equal(
f.calls[0].url,
'https://api.openai.com/v1/chat/completions',
);
assert.equal(f.calls[0].body.stream, false);
assert.equal(f.calls[0].body.n, 1);
assert.equal(f.calls[0].body.model, 'gpt-4.1-mini');
assert.equal(f.calls[0].body.messages[0].role, 'system');
assert.equal(
(await f.send({ messages: [{ role: 'system', content: 'override' }] }))
.status,
400,
);
}
});
test(`${name} client disconnect aborts upstream and cancels a late body`, async (t) => {
let finish,
signal,
cancelled = 0,
ready;
const acquired = new Promise((resolve) => (ready = resolve));
const f = await fixture(t, provider, (_url, options) => {
signal = options.signal;
ready();
return new Promise((resolve) => (finish = resolve));
});
const request = http.request(f.origin, {
method: 'POST',
headers: { Origin: f.origin, 'Content-Type': 'application/json' },
});
request.on('error', () => {});
request.end(JSON.stringify(provider.input));
await acquired;
const aborted = once(signal, 'abort');
request.destroy();
await aborted;
assert.equal(signal.aborted, true);
finish({
body: {
cancel: async () => {
cancelled++;
},
},
});
await new Promise((resolve) => setImmediate(resolve));
assert.ok(cancelled > 0);
});
test(`${name} invalid upstream content still returns reported usage`, async (t) => {
const f = await fixture(t, provider, async () =>
Response.json({
...provider.reply(),
...(name === 'gemini' ? { candidates: [] } : { choices: [] }),
}),
);
const response = await f.send();
assert.equal(response.status, 502);
assert.equal((await response.json()).usage.totalTokens, 10);
});
}
test('text adapts speech wording while retaining shared Radio and annotation policies', () => {
const directives = realtimeDirectives();
assert.equal(realtimeInstructions(), directives.join('\n'));
const text = textInstructions();
assert.match(text, /text-only chat/);
assert.ok(!text.includes('mic session is active'));
for (const line of directives.filter(
(line) =>
(line.includes('radio') && !line.startsWith('PREAMBLES:')) ||
line.startsWith('annotate_map results:'),
))
assert.ok(text.includes(line));
});
test('every spoken phrase chat rewrites still occurs in the voice directives', () => {
const voice = realtimeDirectives().join('\n');
const text = textInstructions();
for (const [spoken, written] of TEXT_WORDING) {
assert.ok(voice.includes(spoken), `voice directives lost "${spoken}"`);
assert.ok(!text.includes(spoken), `chat still says "${spoken}"`);
assert.ok(text.includes(written), `chat lacks "${written}"`);
}
});
test('OpenAI empty tool arrays are removed before history is returned', async (t) => {
const f = await fixture(t, providers.openai, async () =>
Response.json({
choices: [
{
message: { role: 'assistant', content: 'Hello', tool_calls: [] },
finish_reason: 'stop',
},
],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
}),
);
const body = await (await f.send()).json();
assert.equal(body.content.tool_calls, undefined);
});