feat(voice): add typed AI chat beside voice

Chat sits next to the mic in the command dock. Replies are silent and never
request the microphone; the first message connects with the selected provider
and runs the same map actions as voice through each provider's text API
(OpenAI OPENAI_CHAT_MODEL, Gemini GEMINI_CHAT_MODEL via generateContent, which
never starts Live). Only one mode runs at a time.

- Chat panel rendered by the voice control presenter: conversation, recent and
  quick-command suggestions, Up/Down history, attachments (map-layer
  references or small UTF-8 text files, sent only with the message)
- Chat and voice panels sit above the dock or dock into the right rail's
  column; minimized bars sit centred above the dock, and height animates on
  swaps
- A centred panel steps down to its bar for the Location and Visual Presets
  trays without moving focus, rests above an open tray, and docks right while
  expanded over one; it returns to the centre once the trays close
- Minimized voice previews the last sentence from each side; the empty Voice
  Preview suggests things to say
- Settings opened from chat keep chat's frame and show the shared AI provider
  and quick commands; voice Settings show Back to voice
- Closing chat ends a typed session; the voice panel's close turns voice off
- The dock widens its voice control for the chat button and restyles its
  wings, which show icons only at 900px and below; assistant panels share the
  left panels' glass, and Cyber keeps its own surfaces
- Server text-chat handlers with body, time and per-IP limits
  (GEV_RATELIMIT_*_CHAT_PER_MIN), documented in SECURITY.md; tests and all
  eleven locales

Co-authored-by: Darren Dignam <[email protected]>
This commit is contained in:
manjunath22466andDarren Dignam committed 2026-11-02 20:56:39 +05:30
1 parent 720260af58
commit af461febae
57 files changed
+6500 -63

No files matched your search

+78 -3
View File
@@ -67,6 +67,12 @@ function initialState(panel) {
microphone: null,
rowGrace: false,
errorDismissed: false,
// Which assistant the dock shows as selected: idle, voice or chat.
assistantMode: 'idle',
chatOpen: false,
chatBusy: false,
// Set once either side has spoken this voice session.
voiceExchange: false,
},
// The provider field and its select.
provider: {
@@ -100,7 +106,10 @@ function initialState(panel) {
text: () =>
t('Hold Space to speak · tap Space to activate focused controls'),
},
error: { text: '' },
// A typed chat error reads as chat's, without the microphone hint.
error: { text: '', typed: false },
// Connection and turn state beside the selected mode's indicator.
phase: { text: '' },
voicePanel: {
hidden: true,
view: null,
@@ -108,7 +117,41 @@ function initialState(panel) {
settingsHidden: true,
settingsExpanded: false,
selected: 'preview',
minimized: false,
// 'chat' while Settings were opened from chat.
context: null,
// Chat's frame, so Settings opened from chat keep its size.
frameWidth: null,
frameHeight: null,
},
// The typed chat panel; its messages publish apart (setChatMessages).
chat: {
open: false,
minimized: false,
busy: false,
status: () => t('Silent chat · microphone off'),
empty: true,
quickChecked: true,
// The latest reply, shown above the minimized bar.
miniReply: '',
suggestions: [],
attachments: [],
attachMenuOpen: false,
layers: [],
locked: false,
sendDisabled: true,
// The input's text; bumped revision means the owner wrote it.
draft: '',
draftRevision: 0,
rows: 2,
backVisible: false,
},
// The minimized voice bar's last sentence from each side.
mini: { user: '', assistant: '' },
// Above the dock, or docked into the right rail's lane.
// `yielded` names the panel minimized to make room for a dock tray;
// `trayDocked` marks a dock taken only while a tray was open.
layout: { docked: false, yielded: null, trayDocked: false },
previewEmpty: {
hidden: false,
text: () => t('Voice replies appear here.'),
@@ -140,7 +183,16 @@ const ELEMENT_IDS = Object.freeze({
previewTab: 'gev-voice-preview-tab',
settingsTab: 'gev-voice-settings-tab',
settingsButton: 'gev-voice-settings-button',
settingsClose: 'gev-voice-settings-close',
card: 'gev-voice-card',
chatButton: 'gev-chat-button',
chatPanel: 'gev-chat-panel',
chatInput: 'gev-chat-input',
chatLayer: 'gev-chat-layer',
chatFile: 'gev-chat-file',
chatBack: 'gev-chat-back',
voiceBack: 'gev-voice-back',
chatMiniReply: 'gev-chat-mini-reply',
});
let controlCount = 0;
@@ -195,6 +247,7 @@ export function createVoiceControl({
let defaultCard = true;
// Each bar's `--audio-level` and `--audio-opacity`, or null at rest.
let bars = null;
let chatMessages = [];
let removed = false;
const owns = () => !removed && store.controls === ui;
const publish = () => {
@@ -241,8 +294,14 @@ export function createVoiceControl({
* Listen for an interaction the view reports: `button`, `tier`,
* `provider-change`, `settings-button`, `settings-close`, `tab`,
* `tab-keydown`, `panel-keydown`, `pointerenter`, `pointerleave`,
* `preview-change`, `timeout-select`, `timeout-input`, `card-dismiss`;
* and `locale` after a locale change, for an owner's own words.
* `preview-change`, `timeout-select`, `timeout-input`, `card-dismiss`,
* `voice-back`, `voice-minimize`, `layout-dock`; the chat's
* `chat-button`, `chat-close`, `chat-minimize`, `chat-restore`,
* `chat-back`, `chat-input`, `chat-keydown`, `chat-submit`, `chat-stop`,
* `chat-prompt`, `chat-suggestion`, `chat-quick-change`,
* `chat-attach-toggle`, `chat-layer`, `chat-file-button`, `chat-files`,
* `chat-attachment-remove`; and `locale` after a locale change, for an
* owner's own words.
* @returns {() => void} Removes the listener.
*/
on(name, listener) {
@@ -260,6 +319,20 @@ export function createVoiceControl({
get bars() {
return bars;
},
/** The chat conversation, as last published. */
get chatMessages() {
return chatMessages;
},
/**
* Show the chat conversation. Published apart from the rest of the
* control: a typed reply reveals frame by frame.
* @param {Array<{id: number, role: string, text: string}>} next
*/
setChatMessages(next) {
if (removed) return;
chatMessages = next;
if (owns()) store.publishChat(chatMessages);
},
/**
* Show the waveform's bars (null rests it). Published apart from the
* rest of the control, and only when they change.
@@ -301,6 +374,7 @@ export function createVoiceControl({
store.connect(null);
store.publish(null);
store.publishBars(null);
store.publishChat([]);
}
removed = true;
offLocale();
@@ -320,6 +394,7 @@ export function createVoiceControl({
});
store.connect(ui);
store.publishBars(null);
store.publishChat(chatMessages);
publish();
return ui;
}