feat(voice): add typed AI chat beside voice

Chat sits next to the mic in the command dock. Replies are silent and never
request the microphone; the first message connects with the selected provider
and runs the same map actions as voice through each provider's text API
(OpenAI OPENAI_CHAT_MODEL, Gemini GEMINI_CHAT_MODEL via generateContent, which
never starts Live). Only one mode runs at a time.

- Chat panel rendered by the voice control presenter: conversation, recent and
  quick-command suggestions, Up/Down history, attachments (map-layer
  references or small UTF-8 text files, sent only with the message)
- Chat and voice panels sit above the dock or dock into the right rail's
  column; minimized bars sit centred above the dock, and height animates on
  swaps
- A centred panel steps down to its bar for the Location and Visual Presets
  trays without moving focus, rests above an open tray, and docks right while
  expanded over one; it returns to the centre once the trays close
- Minimized voice previews the last sentence from each side; the empty Voice
  Preview suggests things to say
- Settings opened from chat keep chat's frame and show the shared AI provider
  and quick commands; voice Settings show Back to voice
- Closing chat ends a typed session; the voice panel's close turns voice off
- The dock widens its voice control for the chat button and restyles its
  wings, which show icons only at 900px and below; assistant panels share the
  left panels' glass, and Cyber keeps its own surfaces
- Server text-chat handlers with body, time and per-IP limits
  (GEV_RATELIMIT_*_CHAT_PER_MIN), documented in SECURITY.md; tests and all
  eleven locales

Co-authored-by: Darren Dignam <[email protected]>
This commit is contained in:
manjunath22466andDarren Dignam committed 2026-11-02 20:56:39 +05:30
1 parent 720260af58
commit af461febae
57 files changed
+6500 -63

No files matched your search

+153
View File
@@ -0,0 +1,153 @@
/** Only the wire format differs; the UI and action lifetime use one text loop. */
export const textProtocols = {
gemini: {
endpoint: '/api/gemini/chat',
user: (text) => ({ role: 'user', parts: [{ text }] }),
isUser: (item) =>
item.role === 'user' &&
item.parts.some((part) => typeof part.text === 'string'),
request: (contents) => ({ contents }),
decode(data) {
const content = data?.content;
if (
content?.role !== 'model' ||
!Array.isArray(content.parts) ||
!content.parts.length ||
content.parts.length > 32
)
throw Error('Invalid response');
return {
content,
text: content.parts
.filter((part) => typeof part.text === 'string' && !part.thought)
.map((part) => part.text)
.join(''),
calls: content.parts
.filter((part) => part.functionCall)
.map((part) => part.functionCall),
finished: data.finishReason === 'STOP',
};
},
results: (pairs) => [
{
role: 'user',
parts: pairs.map(({ call, result }) => ({
functionResponse: {
name: call.name,
...(call.id ? { id: call.id } : {}),
response: result,
},
})),
},
],
},
openai: {
endpoint: '/api/openai/chat',
user: (content) => ({ role: 'user', content }),
isUser: (item) => item.role === 'user',
request: (messages) => ({ messages }),
decode(data) {
const content = data?.content;
if (
content?.role !== 'assistant' ||
(content.content != null && typeof content.content !== 'string') ||
(content.tool_calls !== undefined && !Array.isArray(content.tool_calls))
)
throw Error('Invalid response');
return {
content,
text: content.content || '',
calls: (content.tool_calls || []).map((call) => {
let args;
try {
args = JSON.parse(call.function.arguments);
} catch {
/* Server supplies the paired validation error. */
}
return { id: call.id, name: call.function.name, args };
}),
finished: data.finishReason === 'stop',
};
},
results: (pairs) =>
pairs.map(({ call, result }) => ({
role: 'tool',
tool_call_id: call.id,
content: JSON.stringify(result),
})),
},
};
export const emptyTextUsage = () => ({
requests: 0,
reportedRequests: 0,
unknownRequests: 0,
inputTokens: 0,
outputTokens: 0,
totalTokens: 0,
reasoningTokens: 0,
cachedInputTokens: 0,
});
export function addTextUsage(total, usage) {
total.requests++;
if (
!usage ||
!['inputTokens', 'outputTokens', 'totalTokens'].every(
(key) => Number.isSafeInteger(usage[key]) && usage[key] >= 0,
)
) {
total.unknownRequests++;
return;
}
total.reportedRequests++;
for (const key of [
'inputTokens',
'outputTokens',
'totalTokens',
'reasoningTokens',
'cachedInputTokens',
]) {
if (Number.isSafeInteger(usage[key]) && usage[key] >= 0)
total[key] += usage[key];
}
}
const bytes = (value) => new TextEncoder().encode(JSON.stringify(value)).length;
// Adapted from Darren Dignam's GEV COMMAND (#976): reduce values, never cut JSON.
// Byte limits also cover multibyte strings; success/failure and the say line survive.
export function boundedToolResult(result, maxBytes = 12000) {
let value;
try {
value = JSON.parse(JSON.stringify(result ?? { ok: false }));
} catch {
return { ok: false, error: 'Tool result could not be serialized' };
}
if (!value || typeof value !== 'object' || Array.isArray(value))
value = { ok: false, value };
if (bytes(value) <= maxBytes) return value;
const essential = new Set([
'ok',
'action',
'error',
'say',
'partial',
'scopeLabel',
'count',
'reason',
'responseGuidance',
]);
value.truncated = true;
for (const key of Object.keys(value)
.filter((key) => !essential.has(key) && key !== 'truncated')
.sort((a, b) => bytes(value[b]) - bytes(value[a]))) {
value[key] = '[omitted oversized field]';
if (bytes(value) <= maxBytes) return value;
}
// Preserve the action outcome while explicitly omitting oversized details.
return {
ok: value.ok === true,
truncated: true,
partial: true,
error: 'Detailed tool result exceeded the response limit',
};
}