fix(voice): address review of typed AI chat

Typed chat sessions no longer end on the voice inactivity timeout, and a
failed or stopped reply keeps the conversation: Stop cancels only the
reply in progress, and map actions that already ran stay in the model's
context. Each upstream request and map action gets its own deadline, old
tool results shrink instead of the history overflowing, and step limits,
a missing or rejected key, the OAuth lock and throttling each get their
own message.

- Server: OpenAI chat is off under GEV_PREFER_CODEX_OAUTH; chat throttles
  default to the provider's throttle; replies carry API_KEY_MISSING,
  API_KEY_REJECTED, CODEX_OAUTH_REQUIRED and RATE_LIMITED codes; upstream
  Retry-After passes through, and a rejected key (Gemini's invalid-key 400
  included) cools down for a minute
- Pinokio forwards the chat throttle settings
- UI: the reply reveal publishes only the message list; the border orbit
  runs only while busy; layers attach with a button or Enter; the log is
  aria-busy while a reply types out; auto-scroll respects a reader who
  scrolled up; Escape closes the attach menu; the chat gear says Settings;
  a note says chat bills the OpenAI key when voice uses ChatGPT sign-in
- Layout does less work while panel height animates; unused blurs removed
- Gemini voice start ends chat; the voice preview clears on every new
  session
- Tests for the route gates, client limits, attachments and the fixes
This commit is contained in:
manjunath22466 committed 2026-11-03 22:01:08 +05:30
1 parent 11cd0142ac
commit da8b0dd8cd
49 files changed
+2121 -160

No files matched your search

+15 -3
View File
@@ -66,6 +66,14 @@ export function bindChatAttachments({ ui, dataManager, onChange, setStatus }) {
setStatus(error.message);
}
}),
// Escape closes the menu and returns to its Attach button.
ui.on('chat-attach-keydown', (event) => {
if (event.key !== 'Escape' || !ui.state.chat.attachMenuOpen) return;
event.preventDefault();
event.stopPropagation();
close();
ui.element('chatPanel')?.querySelector?.('#gev-chat-attach')?.focus?.();
}),
ui.on('chat-file-button', () => {
if (!locked) ui.element('chatFile')?.click();
}),
@@ -82,10 +90,14 @@ export function bindChatAttachments({ ui, dataManager, onChange, setStatus }) {
);
if (file.size > 10000)
throw new Error(t('Each file must be 10 KB or smaller.'));
const content = new TextDecoder('utf-8', { fatal: true }).decode(
await file.arrayBuffer(),
);
const bytes = await file.arrayBuffer();
if (disposed) return;
let content = '';
try {
content = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
} catch {
// Not UTF-8: the decoder's own message is not the person's words.
}
if (!content.trim() || /[\u0000-\u0008\u000e-\u001f]/.test(content))
throw new Error(t('Choose a non-empty UTF-8 text file.'));
add(`file:${file.name}`, { kind: 'file', name: file.name, content });
+139
View File
@@ -0,0 +1,139 @@
// src/voice/chatAttachments.test.mjs — the chat's draft attachments: what
// they accept and refuse, and how their menu closes.
import '../testSupport/registerDom.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
import { bindChatAttachments } from './chatAttachments.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import { mountVoiceView } from '../testSupport/voiceView.mjs';
const $ = (selector) => document.querySelector(selector);
function layers(count) {
return new Map(
Array.from({ length: count }, (_, index) => [
`layer-${index + 1}`,
{ module: { name: `Layer ${index + 1}` } },
]),
);
}
function attachments(t, { ui = createTestVoiceUi({ panel: true }) } = {}) {
const statuses = [];
const bound = bindChatAttachments({
ui,
dataManager: { layers: layers(6), isEnabled: () => true },
onChange() {},
setStatus: (text) => statuses.push(text),
});
t.after(() => bound.destroy());
const names = () => ui.state.chat.attachments.map((item) => item.name);
// The file input as the view hands it over; the reading is asynchronous.
const choose = async (...files) => {
const input = { files, value: 'C:\\fakepath\\file' };
ui.emit('chat-files', input);
for (let i = 0; i < 5; i++) await new Promise((r) => setTimeout(r, 0));
return input;
};
return { ui, bound, statuses, names, choose };
}
const file = (name, content) => new File([content], name);
test('a fifth attachment is refused: the limit is four', (t) => {
const { ui, statuses, names } = attachments(t);
for (let i = 1; i <= 4; i++) ui.emit('chat-layer', `layer-${i}`);
assert.deepEqual(names(), ['Layer 1', 'Layer 2', 'Layer 3', 'Layer 4']);
assert.deepEqual(statuses, []);
ui.emit('chat-layer', 'layer-5');
assert.equal(names().length, 4);
assert.deepEqual(statuses, ['You can attach up to four items.']);
// Choosing an attached layer again replaces it and is not a fifth.
ui.emit('chat-layer', 'layer-2');
assert.equal(names().length, 4);
assert.equal(statuses.length, 1);
});
test('a 10,001-byte file is refused: each file may be up to 10 KB', async (t) => {
const { statuses, names, choose } = attachments(t);
const input = await choose(file('big.txt', 'a'.repeat(10001)));
assert.deepEqual(names(), []);
assert.deepEqual(statuses, ['Each file must be 10 KB or smaller.']);
// The input clears so the same file can be chosen again.
assert.equal(input.value, '');
await choose(file('notes.txt', 'a'.repeat(9000)));
assert.deepEqual(names(), ['notes.txt']);
assert.equal(statuses.length, 1);
});
test('only text file types are accepted', async (t) => {
const { statuses, names, choose } = attachments(t);
await choose(file('setup.exe', 'MZ'));
assert.deepEqual(names(), []);
assert.deepEqual(statuses, [
'Choose a TXT, Markdown, CSV, JSON or GeoJSON file.',
]);
await choose(file('places.GeoJSON', '{"type":"FeatureCollection"}'));
assert.deepEqual(names(), ['places.GeoJSON']);
});
test('a file that is not UTF-8 is refused in the person’s words', async (t) => {
const { statuses, names, choose } = attachments(t);
await choose(file('latin1.csv', new Uint8Array([0x63, 0x61, 0x66, 0xe9])));
assert.deepEqual(names(), []);
assert.deepEqual(statuses, ['Choose a non-empty UTF-8 text file.']);
});
test('control characters and empty files are refused; tabs and line breaks are text', async (t) => {
const { statuses, names, choose } = attachments(t);
await choose(file('binary.txt', 'name\u0001value'));
await choose(file('blank.md', ' \n\t '));
assert.deepEqual(names(), []);
assert.deepEqual(statuses, [
'Choose a non-empty UTF-8 text file.',
'Choose a non-empty UTF-8 text file.',
]);
await choose(file('table.csv', 'a\tb\r\nc\td\n'));
assert.deepEqual(names(), ['table.csv']);
assert.equal(statuses.length, 2);
});
test('attachments over 10 KB together are refused', async (t) => {
const { statuses, names, choose } = attachments(t);
await choose(file('one.txt', 'a'.repeat(6000)));
await choose(file('two.txt', 'b'.repeat(6000)));
assert.deepEqual(names(), ['one.txt']);
assert.deepEqual(statuses, [
'Attachments are too large together. Use smaller files.',
]);
});
test('Escape closes the attach menu and returns to the Attach button', async (t) => {
const { pinia, settle } = mountVoiceView(t);
const ui = createTestVoiceUi({ panel: true, pinia });
attachments(t, { ui });
ui.update('chat', { open: true });
await settle();
$('#gev-chat-attach').click();
await settle();
assert.equal($('#gev-chat-attach-menu').hidden, false);
assert.equal(document.activeElement?.id, 'gev-chat-layer');
const escape = new KeyboardEvent('keydown', {
key: 'Escape',
bubbles: true,
cancelable: true,
});
$('#gev-chat-layer').dispatchEvent(escape);
await settle();
assert.equal(escape.defaultPrevented, true);
assert.equal($('#gev-chat-attach-menu').hidden, true);
assert.equal(document.activeElement?.id, 'gev-chat-attach');
// With the menu closed, Escape is left to whatever else handles it.
const again = new KeyboardEvent('keydown', {
key: 'Escape',
bubbles: true,
cancelable: true,
});
$('#gev-chat-attach').dispatchEvent(again);
assert.equal(again.defaultPrevented, false);
});
+64 -15
View File
@@ -1,5 +1,6 @@
import { bindChatAttachments } from './chatAttachments.js';
import { bindChatPreferences } from './chatPreferences.js';
import { readStoredCloudVoiceAuthMode } from './cloudVoiceAuth.js';
import { t } from '../i18n/index.js';
/**
@@ -21,6 +22,8 @@ export function bindChatControl({ ui, session, onOpen, dataManager }) {
let latestReply;
let typingFrame = 0;
let lastTypingTime = 0;
// Set by Stop until the cancelled turn reports its end or the next send.
let stopping = false;
const isTyped = () => Boolean(session.adapter.getStartOptions?.().textOnly);
const chat = () => ui.state.chat;
// Chat Settings are still chat: the dock keeps its chat state.
@@ -81,11 +84,19 @@ export function bindChatControl({ ui, session, onOpen, dataManager }) {
}
// Layout moves (yielding to a dock tray, say) pass `focus: false`, so they
// never take focus from what the user is typing in.
// ChatGPT sign-in covers OpenAI voice only; typed chat still uses the key.
const authNote = () =>
ui.state.root.provider === 'openai' &&
readStoredCloudVoiceAuthMode() === 'oauth'
? t(
'Typed chat uses your OpenAI API key. ChatGPT sign-in covers voice only.',
)
: '';
function open({ focus = true } = {}) {
leaveChatSettings();
if (!isTyped() && session.isActive()) session.stop();
onOpen?.();
ui.update('chat', { open: true, minimized: false, rows: 2 });
ui.update('chat', { open: true, minimized: false, rows: 2, authNote });
sync();
if (focus) void ui.focus(['chatInput']);
}
@@ -110,6 +121,20 @@ export function bindChatControl({ ui, session, onOpen, dataManager }) {
return entry;
}
const preview = (text) => ui.update('chat', { miniReply: text });
// The reveal writes only the message list each frame. The minimized bar's
// reply and the log's busy state change once, as the reveal starts and
// ends, so the frames do not republish the whole control.
function startReveal() {
if (typingFrame) return;
ui.update('chat', { revealing: true });
typingFrame = requestAnimationFrame(animateReply);
}
function endReveal() {
ui.update('chat', {
revealing: false,
...(latestReply && { miniReply: latestReply.entry.text }),
});
}
function animateReply(time) {
if (disposed) return;
const elapsed = Math.min(100, time - (lastTypingTime || time - 16));
@@ -122,26 +147,35 @@ export function bindChatControl({ ui, session, onOpen, dataManager }) {
Math.ceil(remaining / 40),
);
reply.entry.text = reply.text.slice(0, reply.entry.text.length + step);
if (reply === latestReply) preview(reply.entry.text);
if (reply.entry.text === reply.text) typing.delete(reply);
}
publishMessages();
typingFrame = typing.size ? requestAnimationFrame(animateReply) : 0;
if (!typingFrame) lastTypingTime = 0;
if (!typingFrame) {
lastTypingTime = 0;
endReveal();
}
}
function finishTyping() {
const revealing = Boolean(typingFrame);
if (typingFrame) cancelAnimationFrame(typingFrame);
typingFrame = 0;
lastTypingTime = 0;
for (const reply of typing) reply.entry.text = reply.text;
if (typing.size) publishMessages();
typing.clear();
if (revealing) endReveal();
}
const unsubscribe = session.subscribe((event) => {
if (disposed) return;
const typed = isTyped();
if (event.type === 'state') sync();
if (event.type === 'transcript' && event.role === 'assistant' && typed) {
if (
event.type === 'transcript' &&
event.role === 'assistant' &&
typed &&
!stopping
) {
const id = event.itemId || event.responseId || 'current';
let reply = replies.get(id);
if (!reply) {
@@ -152,12 +186,12 @@ export function bindChatControl({ ui, session, onOpen, dataManager }) {
reply.text = event.final ? event.text : reply.text + event.text;
if (globalThis.matchMedia?.('(prefers-reduced-motion: reduce)').matches) {
reply.entry.text = reply.text;
preview(reply.text);
if (event.final) preview(reply.text);
typing.delete(reply);
publishMessages();
} else {
typing.add(reply);
if (!typingFrame) typingFrame = requestAnimationFrame(animateReply);
startReveal();
}
}
if (event.type === 'action-call' && busy) {
@@ -167,14 +201,19 @@ export function bindChatControl({ ui, session, onOpen, dataManager }) {
if (typed && event.type === 'completion' && !event.intermediate) {
busy = false;
const completed = event.status === 'completed';
setStatus(() =>
completed
? t('Silent chat · microphone off')
: t('Reply interrupted. You can try again.'),
);
if (!completed) {
finishTyping();
preview(t('Reply interrupted. You can try again.'));
// A turn the person stopped keeps its "Reply stopped." status.
if (stopping) stopping = false;
else {
setStatus(() =>
completed
? t('Silent chat · microphone off')
: t('Reply interrupted. You can try again.'),
);
if (!completed) {
finishTyping();
preview(t('Reply interrupted. You can try again.'));
} else if (latestReply && !typing.has(latestReply))
preview(latestReply.entry.text);
}
sync();
}
@@ -242,8 +281,10 @@ export function bindChatControl({ ui, session, onOpen, dataManager }) {
const attempt = ++revision;
busy = true;
connecting = true;
stopping = false;
finishTyping();
replies.clear();
latestReply = undefined;
sync();
setStatus(() => t('Connecting…'));
preview(t('Connecting…'));
@@ -312,13 +353,21 @@ export function bindChatControl({ ui, session, onOpen, dataManager }) {
}
}),
ui.on('chat-submit', () => void submit()),
// Stop ends only the reply in flight: the session and conversation stay.
// An adapter without turn cancellation ends the session instead.
ui.on('chat-stop', () => {
if (!busy) return;
finishTyping();
session.stop();
const cancelTurn = session.adapter.cancelTurn?.bind(session.adapter);
if (cancelTurn && !connecting) {
stopping = true;
busy = false;
cancelTurn();
} else session.stop();
if (disposed) return;
setStatus(() => t('Reply stopped.'));
preview(t('Reply stopped.'));
sync();
}),
ui.on('chat-prompt', (text) => {
setDraft(text);
+331
View File
@@ -0,0 +1,331 @@
// src/voice/chatControlView.test.mjs — the typed chat's owner and its view:
// the reply reveal, Stop, the attach menu's layer choice, the conversation's
// scrolling and what screen readers are told.
import '../testSupport/registerDom.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
import { bindChatControl } from './chatControl.js';
import { CLOUD_VOICE_AUTH_STORAGE_KEY } from './cloudVoiceAuth.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import { mountVoiceView } from '../testSupport/voiceView.mjs';
const $ = (selector) => document.querySelector(selector);
const flush = async () => {
for (let i = 0; i < 5; i++) await new Promise((r) => setTimeout(r, 0));
};
// A typed session that connects at once and sends what it is given.
function typedSession({ cancelTurn = false } = {}) {
const listeners = new Set();
const session = {
state: 'idle',
sent: [],
stops: 0,
cancels: 0,
adapter: {
getStartOptions: () => ({ textOnly: true }),
...(cancelTurn && {
cancelTurn() {
session.cancels++;
},
}),
},
subscribe(fn) {
listeners.add(fn);
return () => listeners.delete(fn);
},
emit(event) {
for (const fn of [...listeners]) fn(event);
},
setState(state) {
session.state = state;
session.emit({ type: 'state', state });
},
isActive: () => !['idle', 'error'].includes(session.state),
start() {
session.setState('listening');
},
stop() {
session.stops++;
session.setState('idle');
},
sendText(text) {
session.sent.push(text);
return true;
},
};
return session;
}
// Animation frames run only when the test steps them.
function manualFrames(t) {
const original = {
request: globalThis.requestAnimationFrame,
cancel: globalThis.cancelAnimationFrame,
};
const queue = new Map();
let next = 0;
let time = 0;
globalThis.requestAnimationFrame = (fn) => {
queue.set(++next, fn);
return next;
};
globalThis.cancelAnimationFrame = (id) => queue.delete(id);
t.after(() => {
globalThis.requestAnimationFrame = original.request;
globalThis.cancelAnimationFrame = original.cancel;
});
return {
get pending() {
return queue.size;
},
step() {
time += 16;
const due = [...queue.values()];
queue.clear();
for (const fn of due) fn(time);
},
};
}
function chat(t, { pinia, session = typedSession(), dataManager } = {}) {
const ui = createTestVoiceUi({ panel: true, pinia });
const control = bindChatControl({ ui, session, dataManager });
t.after(() => control.destroy());
const send = async (text) => {
ui.emit('chat-input', text);
ui.emit('chat-submit');
await flush();
};
const reply = (text, extra = {}) =>
session.emit({
type: 'transcript',
role: 'assistant',
itemId: 'reply-1',
text,
...extra,
});
const texts = () => ui.chatMessages.map((m) => `${m.role}:${m.text}`);
return { ui, session, control, send, reply, texts };
}
test('a revealed reply writes only the message list each frame', async (t) => {
const frames = manualFrames(t);
const { ui, send, reply, texts } = chat(t);
ui.emit('chat-button');
await send('Fly to Tokyo');
assert.equal(ui.state.chat.miniReply, 'Thinking…');
const text = 'Flying to Tokyo now. '.repeat(8).trim();
reply(text, { final: true });
assert.equal(ui.state.chat.revealing, true);
const control = ui.state;
const seen = new Set();
for (let guard = 0; frames.pending; guard++) {
if (guard > 200) assert.fail('the reveal did not finish');
frames.step();
if (!frames.pending) break;
seen.add(texts().at(-1));
// Mid-reveal the control itself is not republished.
assert.equal(ui.state, control);
}
assert.ok(seen.size > 1, 'the reply revealed over several frames');
assert.equal(texts().at(-1), `assistant:${text}`);
// The minimized bar's reply and the log's busy state change once, at the end.
assert.equal(ui.state.chat.revealing, false);
assert.equal(ui.state.chat.miniReply, text);
});
test('Stop cancels only the reply in flight and keeps the conversation', async (t) => {
manualFrames(t);
const session = typedSession({ cancelTurn: true });
const { ui, send, reply, texts } = chat(t, { session });
ui.emit('chat-button');
await send('Fly to Tokyo');
reply('Flying');
assert.equal(ui.state.chat.busy, true);
ui.emit('chat-stop');
assert.equal(session.cancels, 1);
assert.equal(session.stops, 0);
assert.equal(session.isActive(), true);
assert.equal(ui.state.chat.busy, false);
assert.equal(ui.state.chat.status, 'Reply stopped.');
assert.equal(ui.state.chat.miniReply, 'Reply stopped.');
assert.deepEqual(texts(), ['user:Fly to Tokyo', 'assistant:Flying']);
// The cancelled turn's own end and late text keep the stop as it was.
reply(' to Tokyo.');
session.emit({ type: 'completion', status: 'cancelled' });
assert.equal(ui.state.chat.status, 'Reply stopped.');
assert.deepEqual(texts(), ['user:Fly to Tokyo', 'assistant:Flying']);
// The same session takes the next message.
await send('Show live flights');
assert.equal(session.sent.length, 2);
assert.equal(ui.state.chat.busy, true);
});
test('Stop ends the session when its adapter cannot cancel a turn', async (t) => {
manualFrames(t);
const { ui, session, send } = chat(t);
ui.emit('chat-button');
await send('Fly to Tokyo');
ui.emit('chat-stop');
assert.equal(session.stops, 1);
assert.equal(ui.state.chat.busy, false);
assert.equal(ui.state.chat.status, 'Reply stopped.');
});
test('with ChatGPT sign-in chosen for OpenAI voice, chat says it uses the API key', (t) => {
t.after(() => localStorage.removeItem(CLOUD_VOICE_AUTH_STORAGE_KEY));
const { ui } = chat(t);
const note =
'Typed chat uses your OpenAI API key. ChatGPT sign-in covers voice only.';
ui.update('root', { provider: 'openai' });
ui.emit('chat-button');
assert.equal(ui.state.chat.authNote, '');
ui.emit('chat-close');
localStorage.setItem(CLOUD_VOICE_AUTH_STORAGE_KEY, 'oauth');
ui.emit('chat-button');
assert.equal(ui.state.chat.authNote, note);
ui.emit('chat-close');
ui.update('root', { provider: 'gemini' });
ui.emit('chat-button');
assert.equal(ui.state.chat.authNote, '');
});
test('the conversation is busy for screen readers while a reply types out', async (t) => {
const frames = manualFrames(t);
const { pinia, settle } = mountVoiceView(t);
const { ui, session, send, reply } = chat(t, { pinia });
ui.emit('chat-button');
await settle();
const log = $('#gev-chat-messages');
const phase = $('#gev-assistant-phase');
assert.equal(log.getAttribute('aria-busy'), null);
// In chat, the chat's own status line is the one status region.
assert.equal(phase.getAttribute('aria-hidden'), 'true');
assert.equal(phase.getAttribute('aria-live'), 'off');
await send('Fly to Tokyo');
await settle();
assert.equal(log.getAttribute('aria-busy'), 'true');
reply('Flying to Tokyo now.', { final: true });
session.emit({ type: 'completion', status: 'completed' });
frames.step();
await settle();
// Done thinking, still typing out.
assert.equal(ui.state.chat.busy, false);
assert.equal(log.getAttribute('aria-busy'), 'true');
while (frames.pending) frames.step();
await settle();
assert.equal(log.getAttribute('aria-busy'), null);
ui.emit('chat-close');
await settle();
assert.equal(phase.getAttribute('aria-hidden'), null);
assert.equal(phase.getAttribute('aria-live'), 'polite');
});
test('a layer attaches with the Attach button or Enter, not as the choice moves', async (t) => {
const { pinia, settle } = mountVoiceView(t);
const dataManager = {
layers: new Map([
['flights', { module: { name: 'Live flights' } }],
['quakes', { module: { name: 'Earthquakes' } }],
]),
isEnabled: () => true,
};
const { ui } = chat(t, { pinia, dataManager });
ui.emit('chat-button');
await settle();
$('#gev-chat-attach').click();
await settle();
const select = $('#gev-chat-layer');
const attach = $('#gev-chat-layer-attach');
assert.equal(attach.disabled, true);
// Arrow keys on a closed select change it once per press.
for (const value of ['flights', 'quakes']) {
select.value = value;
select.dispatchEvent(new Event('change', { bubbles: true }));
await settle();
}
assert.deepEqual(ui.state.chat.attachments, []);
assert.equal(attach.disabled, false);
attach.click();
await settle();
assert.deepEqual(
ui.state.chat.attachments.map((item) => item.name),
['Earthquakes'],
);
$('#gev-chat-attach').click();
await settle();
select.value = 'flights';
select.dispatchEvent(new Event('change', { bubbles: true }));
const enter = new KeyboardEvent('keydown', {
key: 'Enter',
bubbles: true,
cancelable: true,
});
select.dispatchEvent(enter);
await settle();
assert.equal(enter.defaultPrevented, true);
assert.deepEqual(
ui.state.chat.attachments.map((item) => item.name),
['Earthquakes', 'Live flights'],
);
});
test('the conversation follows new messages only while read at the bottom', async (t) => {
const { pinia, settle } = mountVoiceView(t);
const { ui } = chat(t, { pinia });
ui.emit('chat-button');
await settle();
const list = $('#gev-chat-messages');
let top = 0;
Object.defineProperties(list, {
scrollHeight: { configurable: true, get: () => 1000 },
clientHeight: { configurable: true, get: () => 200 },
scrollTop: {
configurable: true,
get: () => top,
set: (value) => {
top = Math.min(value, 800);
},
},
});
const show = async (...messages) => {
ui.setChatMessages(
messages.map(([role, text], index) => ({ id: index + 1, role, text })),
);
await settle();
};
await show(['user', 'Fly to Tokyo'], ['assistant', 'Fly']);
assert.equal(top, 800);
// The reader scrolls up; the reply keeps typing out below them.
top = 300;
list.dispatchEvent(new Event('scroll'));
await show(['user', 'Fly to Tokyo'], ['assistant', 'Flying to Tokyo.']);
assert.equal(top, 300);
// Sending brings them back to the newest message.
await show(
['user', 'Fly to Tokyo'],
['assistant', 'Flying to Tokyo.'],
['user', 'Show live flights'],
);
assert.equal(top, 800);
list.dispatchEvent(new Event('scroll'));
await show(
['user', 'Fly to Tokyo'],
['assistant', 'Flying to Tokyo.'],
['user', 'Show live flights'],
['assistant', 'Showing'],
);
assert.equal(top, 800);
});
test('the chat gear is named for the settings it opens', async (t) => {
const { pinia, settle } = mountVoiceView(t);
const { ui } = chat(t, { pinia });
ui.emit('chat-button');
await settle();
assert.equal(
$('#gev-voice-settings-button').getAttribute('aria-label'),
'Settings',
);
});
+4
View File
@@ -134,6 +134,10 @@ function initialState(panel) {
quickChecked: true,
// The latest reply, shown above the minimized bar.
miniReply: '',
// A reply is typing out; the conversation is busy for screen readers.
revealing: false,
// Says chat bills the OpenAI key while voice uses ChatGPT sign-in.
authNote: '',
suggestions: [],
attachments: [],
attachMenuOpen: false,
+3
View File
@@ -2545,6 +2545,7 @@ export function createGeminiSession(options) {
});
let textOnly = false;
const startVoice = (settings) => {
chat.stop({ preserveStatus: true });
textOnly = false;
return controller.start(settings);
};
@@ -2571,6 +2572,8 @@ export function createGeminiSession(options) {
chat.stop({ ...settings, preserveStatus: true });
controller.stop(settings);
},
// Chat Stop: abort the reply in progress, keep the conversation.
cancelTurn: () => textOnly && chat.cancelTurn(),
sendText: (text) =>
textOnly ? chat.sendText(text) : controller.sendTextCommand(text),
sendMapEvent: (event) =>
+57 -2
View File
@@ -177,6 +177,58 @@ test('composite chat never starts Live and the next keyboard start resets its mo
adapter.stop({ removeUi: true });
});
test('Gemini composite: chat Stop keeps the conversation and voice start ends chat', async () => {
const { createGeminiSession } = await import('./geminiSession.js');
const requests = [];
const signals = [];
const adapter = createGeminiSession({
emit: () => {},
runAction: (_name, _args, options) => {
signals.push(options.signal);
return new Promise(() => {});
},
ui: createTestVoiceUi(),
fetchImpl: async (_url, options) => {
requests.push(JSON.parse(options.body));
return requests.length === 1
? final()
: Response.json({
content: {
role: 'model',
parts: [{ functionCall: { name: 'fly_to', args: {} } }],
},
});
},
getUserMedia: () => {
throw Error('No microphone');
},
createAudio: () => {
throw Error('No audio');
},
});
await adapter.start({ textOnly: true });
adapter.sendText('First');
for (let i = 0; i < 5; i++) await tick();
adapter.sendText('Second');
for (let i = 0; i < 5 && !signals[0]; i++) await tick();
assert.equal(adapter.cancelTurn(), true);
assert.equal(signals[0].aborted, true);
assert.equal(adapter.sendText('Third'), true);
for (let i = 0; i < 5 && !signals[1]; i++) await tick();
assert.deepEqual(
requests[2].contents
.filter((item) => item.parts[0].text)
.map((item) => item.parts[0].text),
['First', 'Done', 'Third'],
);
// Starting voice retires the typed turn in progress.
adapter.controller.start = async () => {};
await adapter.controller.input.start({ pushToTalk: true });
assert.equal(signals[1].aborted, true);
assert.equal(adapter.cancelTurn(), false);
adapter.stop({ removeUi: true });
});
test('failed Radio startup cancels its stream before releasing the mute', async () => {
const actions = [];
let count = 0;
@@ -307,7 +359,8 @@ for (const intent of ['pause', 'off'])
assert.equal(plays, 0);
});
test('a tool deadline reports one error, aborts work and admits a successor', async () => {
test('a tool deadline reports one error, aborts work and admits a successor', async (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
const events = [];
let actionSignal,
requests = 0;
@@ -331,7 +384,9 @@ test('a tool deadline reports one error, aborts work and admits a successor', as
});
chat.start();
chat.sendText('Go');
await new Promise((resolve) => setTimeout(resolve, 70));
for (let i = 0; i < 5 && !actionSignal; i++) await tick();
assert.equal(actionSignal.aborted, false);
t.mock.timers.tick(30);
assert.equal(actionSignal.aborted, true);
assert.equal(events.filter((e) => e.state === 'error').length, 1);
assert.equal(requests, 1);
+3 -1
View File
@@ -289,7 +289,9 @@ export function createVoicePanelHost({
signal: session.signal,
...clock,
onExpire: () => {
if (!owns()) return;
// Typed chat has no microphone or audio to time out, and stopping it
// would drop the conversation the chat log still shows.
if (!owns() || session.adapter?.getStartOptions?.().textOnly) return;
const ticket = revision;
session.stop();
closeAfterStop(ticket);
+30
View File
@@ -472,6 +472,25 @@ test('native connecting phases retain output observation; idle starts before fir
assert.equal(adapter.stops, 1);
});
test('the voice inactivity timeout never ends a typed chat session', async (t) => {
const f = mounted(t, { native: true });
const adapter = f.instances[0];
adapter.getStartOptions = () => ({ textOnly: true });
await f.commands.session.start({ textOnly: true });
adapter.controller.handleMessage({});
f.clock.advance(600000);
assert.equal(adapter.stops, 0);
assert.equal(f.commands.session.isActive(), true);
// A quiet voice session started afterwards still times out.
f.commands.session.stop();
const stops = adapter.stops;
adapter.getStartOptions = () => ({});
await f.commands.session.start();
adapter.controller.handleMessage({});
f.clock.advance(600000);
assert.equal(adapter.stops, stops + 1);
});
async function directActions(
t,
{ storage = null, delayExit = false, exposePanel = true } = {},
@@ -1134,6 +1153,17 @@ test('closing chat Settings ends a typed session', async (t) => {
assert.equal($('#gev-chat-panel').hidden, true);
});
test('opening chat stops a live voice session', async (t) => {
const f = mounted(t, { view: true });
await f.commands.session.start();
await f.settle();
assert.equal(f.commands.session.isActive(), true);
$('#gev-chat-button').click();
await f.settle();
assert.equal(f.commands.session.isActive(), false);
assert.equal($('#gev-chat-panel').hidden, false);
});
test('the mic button closes chat and minimize collapses voice to a bar', async (t) => {
const f = mounted(t, { view: true });
await f.settle();
+54 -11
View File
@@ -145,6 +145,8 @@ function layOut({ ui, name, minimize, restore }) {
let lastHeight = 0;
let animateNext = false;
let heightAnimation = null;
// What the last placement depended on, short of measuring.
let placedInputs = '';
const animateHeight = (from, to) => {
if (view.matchMedia?.('(prefers-reduced-motion: reduce)').matches) return;
heightAnimation?.cancel();
@@ -155,12 +157,35 @@ function layOut({ ui, name, minimize, restore }) {
],
{ duration: 240, easing: 'cubic-bezier(0.2, 0.8, 0.2, 1)' },
);
// Frames skipped while it runs are placed once it ends.
if (heightAnimation) heightAnimation.onfinish = schedule;
};
const place = () => {
const minimized = Boolean(panel.dataset.minimized);
// Minimized bars always sit centred above the dock; only an expanded
// docked panel takes the right lane.
const inLane = isDocked() && !minimized && !isMobile();
const mobile = isMobile();
const inLane = isDocked() && !minimized && !mobile;
// The height animation resizes the panel every frame. While it runs and
// nothing it was placed for has changed, leave the placement as it is;
// it is placed again when the animation finishes.
const inputs = [
minimized,
isDocked(),
mobile,
panel.hidden,
root.dataset?.chatOpen,
root.dataset?.assistantMode,
view.innerWidth,
view.innerHeight,
].join('|');
if (
heightAnimation?.playState === 'running' &&
!animateNext &&
inputs === placedInputs
)
return;
placedInputs = inputs;
panel.dataset.placement = inLane ? 'docked' : 'compact';
syncRail();
if (lifetime.signal.aborted) return;
@@ -174,7 +199,7 @@ function layOut({ ui, name, minimize, restore }) {
const expanding = !appearing && wasMinimized && !minimized;
seen = true;
wasMinimized = minimized;
if (!isDocked() && !isMobile()) {
if (!isDocked() && !mobile) {
// A panel opened or switched to while the trays hold the centre
// starts as its bar.
if (appearing && !minimized && (ui.state.layout.yielded || trayOpen())) {
@@ -198,6 +223,7 @@ function layOut({ ui, name, minimize, restore }) {
top: box.top + (root.clientTop || 0),
};
const anchorBottom = anchor.top + (root.clientHeight || box.height);
let measured = null;
const w = view.innerWidth,
h = view.innerHeight;
if (inLane) {
@@ -227,7 +253,7 @@ function layOut({ ui, name, minimize, restore }) {
.getElementById?.('command-dock')
?.getBoundingClientRect();
panel.style.removeProperty('height');
if (isMobile() && !minimized) panel.style.width = `${w - 16}px`;
if (mobile && !minimized) panel.style.width = `${w - 16}px`;
else panel.style.removeProperty('width');
const rect = panel.getBoundingClientRect();
const centre = dockRect?.width
@@ -253,13 +279,16 @@ function layOut({ ui, name, minimize, restore }) {
panel.style.left = `${x - anchor.left}px`;
panel.style.top = 'auto';
panel.style.bottom = `${anchorBottom - (y + rect.height)}px`;
measured = rect.height;
}
panel.style.transform = 'none';
// Grow from the panel shown before (chat and voice swap), or from this
// panel's last height when its view changes. A running animation is
// mid-flight, so its frames are not recorded as the panel's height.
if (heightAnimation?.playState === 'running' && !animateNext) return;
const height = panel.getBoundingClientRect().height;
// A centred panel was measured above; placing it moves it but keeps its
// size. The lane's height is set here, so it is measured again.
const height = measured ?? panel.getBoundingClientRect().height;
const previous = handoffs.get(root);
const from = minimized
? 0
@@ -314,7 +343,23 @@ function layOut({ ui, name, minimize, restore }) {
.map((item) => item.id),
);
let railOpen = expandedRailPanels();
const railChanges = new MutationObserver(() => {
// Rail panels open and close through their own class, so only the rail's
// direct children are watched, each as it joins the rail.
const watchedRailPanels = new WeakSet();
const watchRailPanels = () => {
for (const item of rail?.children ?? []) {
if (watchedRailPanels.has(item)) continue;
watchedRailPanels.add(item);
railChanges.observe(item, {
attributes: true,
attributeFilter: ['class'],
});
}
};
const railChanges = new MutationObserver((records) => {
if (records.some((record) => record.type === 'childList'))
watchRailPanels();
if (!records.some((record) => record.type === 'attributes')) return;
const next = expandedRailPanels();
const opened = [...next].some((id) => !railOpen.has(id));
railOpen = next;
@@ -326,12 +371,10 @@ function layOut({ ui, name, minimize, restore }) {
)
minimize();
});
if (rail)
railChanges.observe(rail, {
attributes: true,
attributeFilter: ['class'],
subtree: true,
});
if (rail) {
railChanges.observe(rail, { childList: true });
watchRailPanels();
}
// Reaching for a wing's button steps a centred panel down to its bar,
// which then rests above the tray.
const reachWing = ({ target }) => {
+2
View File
@@ -47,6 +47,8 @@ export function createRealtimeSession({
chat.stop({ ...settings, preserveStatus: true });
controller.stop(settings);
},
// Chat Stop: abort the reply in progress, keep the conversation.
cancelTurn: () => textOnly && chat.cancelTurn(),
sendText: (text) =>
textOnly ? chat.sendText(text) : controller.sendTextCommand(text),
sendMapEvent: (event) =>
+50
View File
@@ -650,3 +650,53 @@ test('OpenAI composite starts HTTP chat without Realtime and restores voice keyb
assert.equal(adapter.sendText('Hello'), 'voice');
adapter.stop();
});
test('OpenAI composite chat Stop cancels only the reply in progress', async () => {
const requests = [];
let pendingSignal;
const controller = {
_input: {},
_radio: {},
isActive: () => true,
start: async () => {},
stop: () => {},
sendTextCommand: () => 'voice',
notifyMapEvent: () => 'voice-event',
};
const adapter = createRealtimeSession({
emit: () => {},
runAction: async () => ({ ok: true }),
createController: () => controller,
fetchImpl: (_url, options) => {
requests.push(JSON.parse(options.body));
if (requests.length !== 2)
return Response.json({
content: { role: 'assistant', content: 'Hello' },
finishReason: 'stop',
});
pendingSignal = options.signal;
return new Promise(() => {});
},
});
const settle = async () => {
for (let i = 0; i < 5; i++)
await new Promise((resolve) => setImmediate(resolve));
};
await adapter.start({ textOnly: true });
adapter.sendText('First');
await settle();
adapter.sendText('Second');
await settle();
assert.equal(adapter.cancelTurn(), true);
assert.equal(pendingSignal.aborted, true);
assert.equal(adapter.sendText('Third'), true);
await settle();
assert.deepEqual(
requests[2].messages.map((m) => m.content),
['First', 'Hello', 'Third'],
);
adapter.stop();
await controller._input.start({ pushToTalk: true });
assert.equal(adapter.cancelTurn(), false);
adapter.stop();
});
+149 -36
View File
@@ -12,6 +12,17 @@ import {
responseInstructionForToolResult,
} from './realtimeProtocol.js';
const encoder = new TextEncoder();
const byteSize = (value) =>
encoder.encode(typeof value === 'string' ? value : JSON.stringify(value))
.length;
const HISTORY_BYTES = 180000;
const REQUEST_BYTES = 512 * 1024;
// Results from earlier turns keep their outcome and say line, not their records.
const COMPACT_RESULT_BYTES = 1500;
const PARTIAL_NOTE =
'This reply stopped before it finished. The map actions above already ran.';
/** Text-only HTTP turns: no Live token, socket, microphone or audio context. */
export function createTextChat({
provider,
@@ -28,6 +39,8 @@ export function createTextChat({
const sessionUsage = emptyTextUsage();
let active = false;
let turn = null;
let turnUsage = null;
let partial = null;
let sequence = 0;
let history = [];
let mapEvents = [];
@@ -43,6 +56,37 @@ export function createTextChat({
radioLayer?.setVoiceDucked?.(false);
};
const state = (value) => emit({ type: 'state', state: value });
// Every call keeps its paired result; only the result body shrinks.
const compact = (items) =>
items.map((item) => protocol.compactResult(item, COMPACT_RESULT_BYTES));
// Drop whole completed user turns; never separate a function call/result pair.
const trim = (items, maxBytes) => {
while (items.length && (items.length > 64 || byteSize(items) > maxBytes)) {
const next = items.findIndex(
(item, index) => index > 0 && protocol.isUser(item),
);
items = next < 0 ? [] : items.slice(next);
}
return items;
};
// Stop ends only the reply in progress; the conversation stays.
const cancelTurn = () => {
const owner = turn;
if (!active || !owner || signal?.aborted) return false;
if (partial?.owner === owner) partial.keep();
turn = null;
owner.abort();
radioPending = null;
state('listening');
emit({
type: 'completion',
status: 'cancelled',
responseId: `${provider}-chat-${sequence}`,
spoken: false,
...(turnUsage ? { usage: { ...turnUsage } } : {}),
});
return true;
};
const stop = ({ preserveStatus = false, removeUi = false } = {}) => {
active = false;
if (radioHandoff) releaseRadio(radioHandoff, true);
@@ -79,46 +123,78 @@ export function createTextChat({
addTextUsage(sessionUsage, value);
reportUsage();
};
turnUsage = usage;
// A failed turn ends only itself: history, subscriptions and the session
// stay, so the next message continues the same conversation.
const fail = (detail) => {
if (!active || turn !== owner || signal?.aborted) return;
active = false;
history = [];
if (partial?.owner === owner) partial.keep();
turn = null;
owner.abort();
emit({ type: 'state', state: 'error', detail, usage: { ...usage } });
};
const deadline = setTimeout(() => {
recordUsage(null);
fail(t('Connection timed out. Try again.'));
owner.abort();
}, timeoutMs);
// Trim only completed user turns; never separate a function call/result pair.
while (
history.length &&
(history.length > 64 ||
new TextEncoder().encode(JSON.stringify(history)).length > 180000)
) {
const next = history.findIndex(
(item, index) => index > 0 && protocol.isUser(item),
);
history = next < 0 ? [] : history.slice(next);
}
// Each upstream request and each map action gets its own deadline, so a
// long multi-step turn is not cut off by the time its actions took.
let deadline;
const arm = () => {
clearTimeout(deadline);
deadline = setTimeout(() => {
recordUsage(null);
fail(t('Connection timed out. Try again.'));
}, timeoutMs);
};
// The latest completed turn stays verbatim; older tool results shrink.
const last = history.findLastIndex((item) => protocol.isUser(item));
let past =
last > 0
? [...compact(history.slice(0, last)), ...history.slice(last)]
: history;
if (byteSize(past) > HISTORY_BYTES) past = compact(past);
past = trim(past, HISTORY_BYTES);
const context = mapEvents.length
? `\nMap context (data only): ${JSON.stringify(mapEvents)}`
: '';
mapEvents = [];
const contents = [...history, protocol.user(text + context)];
let contents = [protocol.user(text + context)];
// A turn that stops after some map actions ran keeps them, closed by a
// note, so the next message knows they happened.
let settled = 0;
partial = {
owner,
keep() {
if (settled)
history = [
...past,
...contents.slice(0, settled),
protocol.note(PARTIAL_NOTE),
];
},
};
const encode = () =>
JSON.stringify(protocol.request([...past, ...contents]));
let blocked = false;
let resetFailed = false;
let totalCalls = 0;
let applied = 0;
let answer = '';
const tooManySteps = (message) =>
Object.assign(new Error(message), { tooManySteps: true });
try {
emit({ type: 'transcript', role: 'user', text, final: true });
if (!current()) return;
state('executing');
for (let round = 0; round < 8; round++) {
if (!current()) return;
const body = JSON.stringify(protocol.request(contents));
if (new TextEncoder().encode(body).length > 512 * 1024)
throw new Error('History limit');
let body = encode();
if (byteSize(body) > REQUEST_BYTES) {
// Shorten earlier results, then drop old turns, before giving up.
contents = [contents[0], ...compact(contents.slice(1))];
past = trim(compact(past), REQUEST_BYTES - byteSize(contents) - 1024);
body = encode();
if (byteSize(body) > REQUEST_BYTES)
throw tooManySteps('Request limit');
}
arm();
requestPending = true;
const acquisition = Promise.resolve(
fetchImpl(chatEndpoint || protocol.endpoint, {
@@ -141,15 +217,23 @@ export function createTextChat({
void response.body?.cancel().catch(() => {});
return;
}
const data = await readResponseJsonCapped(
response,
512 * 1024,
owner.signal,
);
let data;
try {
data = await readResponseJsonCapped(
response,
512 * 1024,
owner.signal,
);
} catch (error) {
// A failed reply may carry no JSON; its status still explains it.
if (response.ok || owner.signal.aborted) throw error;
}
recordUsage(data?.usage);
if (!response.ok)
throw Object.assign(new Error('Request failed'), {
status: response.status,
code: typeof data?.code === 'string' ? data.code : undefined,
retryAfter: Number(response.headers?.get?.('retry-after')) || 0,
});
if (!current()) return;
const {
@@ -174,7 +258,7 @@ export function createTextChat({
throw Object.assign(new Error('Incomplete response'), {
incomplete: true,
});
history = contents;
history = [...past, ...contents];
turn = null;
state('listening');
if (
@@ -193,7 +277,7 @@ export function createTextChat({
return;
}
totalCalls += calls.length;
if (totalCalls > 16) throw new Error('Tool limit');
if (totalCalls > 16) throw tooManySteps('Tool limit');
const pairs = [];
const invalid = new Map(
(data.invalidCalls || []).map((item) => [item.index, item.error]),
@@ -226,6 +310,7 @@ export function createTextChat({
: 'globe-view-required',
};
} else {
arm();
try {
if (
call.name === 'control_radio' ||
@@ -284,6 +369,7 @@ export function createTextChat({
}
}
if (!current()) return;
if (result?.ok === true) applied++;
if (!invalid.size && isGlobeViewRefusal(result)) blocked = true;
if (
!invalid.size &&
@@ -300,21 +386,47 @@ export function createTextChat({
});
}
contents.push(...protocol.results(pairs));
settled = contents.length;
}
throw new Error('Round limit');
throw tooManySteps('Round limit');
} catch (error) {
recordUsage(null);
fail(
error?.status === 504 || owner.signal.aborted
? t('Connection timed out. Try again.')
: error?.incomplete
? t('Reply interrupted. You can try again.')
: t('Could not connect. Try again.'),
error?.code === 'API_KEY_MISSING'
? t('Needs {envVars} — add it in Provider Settings', {
envVars: protocol.keyEnv,
})
: error?.code === 'CODEX_OAUTH_REQUIRED'
? t(
'Typed chat is unavailable while GEV_PREFER_CODEX_OAUTH requires ChatGPT OAuth.',
)
: error?.code === 'API_KEY_REJECTED'
? t(
'The provider rejected the API key. Check it in Provider Settings.',
)
: error?.code === 'RATE_LIMITED' || error?.status === 429
? t('Too many chat requests. Try again in {seconds} s.', {
seconds: Math.min(error.retryAfter || 60, 3600),
})
: error?.status === 504 || owner.signal.aborted
? t('Connection timed out. Try again.')
: error?.incomplete
? t('Reply interrupted. You can try again.')
: error?.tooManySteps
? applied
? t(
'That request needed too many steps. Changes already made to the map stay in place.',
)
: t(
'That request needed too many steps. Try a simpler request.',
)
: t('Could not connect. Try again.'),
);
} finally {
recordUsage(null);
clearTimeout(deadline);
if (turn === owner) turn = null;
if (turnUsage === usage) turnUsage = null;
}
}
return {
@@ -322,7 +434,7 @@ export function createTextChat({
if (signal?.aborted) return;
active = true;
const cancelRadioTurn = () => {
if (radioPending && radioPending === turn) stop();
if (radioPending && radioPending === turn) cancelTurn();
};
unsubscribeControls?.();
unsubscribeVisibility?.();
@@ -352,6 +464,7 @@ export function createTextChat({
state('listening');
},
stop,
cancelTurn,
sendText(text) {
if (
!active ||
+392
View File
@@ -0,0 +1,392 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createTextChat } from './textChat.js';
import { t } from '../i18n/index.js';
const tick = () => new Promise((resolve) => setImmediate(resolve));
const settle = async (count = 8) => {
for (let i = 0; i < count; i++) await tick();
};
const bytes = (value) =>
new TextEncoder().encode(
typeof value === 'string' ? value : JSON.stringify(value),
).length;
// The wire shapes differ; the turn loop under test is shared.
const wire = {
openai: {
final: (text) => ({
content: { role: 'assistant', content: text },
finishReason: 'stop',
}),
calls: (count, prefix = 'call') => ({
content: {
role: 'assistant',
content: null,
tool_calls: Array.from({ length: count }, (_, index) => ({
id: `${prefix}-${index}`,
type: 'function',
function: { name: 'fly_to', arguments: '{}' },
})),
},
}),
items: (body) => body.messages,
userTexts: (items) =>
items.filter((item) => item.role === 'user').map((item) => item.content),
keyEnv: 'OPENAI_API_KEY',
},
gemini: {
final: (text) => ({
content: { role: 'model', parts: [{ text }] },
finishReason: 'STOP',
}),
calls: (count, prefix = 'call') => ({
content: {
role: 'model',
parts: Array.from({ length: count }, (_, index) => ({
functionCall: { name: 'fly_to', args: {}, id: `${prefix}-${index}` },
})),
},
}),
items: (body) => body.contents,
userTexts: (items) =>
items
.filter((item) => item.role === 'user' && item.parts[0].text)
.map((item) => item.parts[0].text),
keyEnv: 'GEMINI_API_KEY',
},
};
function harness(provider, replies, options = {}) {
const events = [];
const requests = [];
const actions = [];
const chat = createTextChat({
provider,
emit: (event) => events.push(event),
runAction: async (name, args, context) => {
actions.push(name);
return options.runAction
? options.runAction(name, args, context)
: { ok: true };
},
fetchImpl: async (_url, init) => {
const body = JSON.parse(init.body);
requests.push(body);
const reply = replies(requests.length, body, init);
return reply instanceof Response ? reply : Response.json(reply);
},
...options.chat,
});
const errors = () => events.filter((event) => event.state === 'error');
const completions = (status = 'completed') =>
events.filter(
(event) => event.type === 'completion' && event.status === status,
);
return { chat, events, requests, actions, errors, completions };
}
for (const provider of ['openai', 'gemini']) {
const w = wire[provider];
test(`${provider}: a failed turn keeps the conversation for the next message`, async () => {
const h = harness(provider, (n) =>
n === 2
? Response.json({ error: 'unavailable' }, { status: 503 })
: w.final(`Reply ${n}`),
);
h.chat.start();
h.chat.sendText('First');
await settle();
h.chat.sendText('Second');
await settle();
assert.equal(h.errors().length, 1);
assert.equal(h.errors()[0].detail, t('Could not connect. Try again.'));
// The session stays usable: no restart is needed before the next send.
assert.equal(h.chat.sendText('Third'), true);
await settle();
assert.deepEqual(w.userTexts(w.items(h.requests[2])), ['First', 'Third']);
assert.equal(h.completions().length, 2);
h.chat.stop();
});
test(`${provider}: a turn that fails after map actions keeps them for the next message`, async () => {
const h = harness(provider, (n) =>
n === 1
? w.calls(1)
: n === 2
? Response.json({ error: 'unavailable' }, { status: 503 })
: w.final('Done'),
);
h.chat.start();
h.chat.sendText('Fly there');
await settle();
assert.equal(h.actions.length, 1);
assert.equal(h.errors().length, 1);
h.chat.sendText('Continue');
await settle();
const items = w.items(h.requests[2]);
assert.deepEqual(w.userTexts(items), ['Fly there', 'Continue']);
// Call, result, then a note that closes the stopped reply.
const note = items.at(-2);
const noteText = provider === 'openai' ? note.content : note.parts[0].text;
assert.equal(note.role, provider === 'openai' ? 'assistant' : 'model');
assert.match(noteText, /map actions above already ran/);
assert.equal(items.length, 5);
h.chat.stop();
});
test(`${provider}: cancelTurn aborts only the reply in progress`, async () => {
let toolSignal;
const h = harness(
provider,
(n) => (n === 2 ? w.calls(1) : w.final(`Reply ${n}`)),
{
runAction: (_name, _args, context) => {
toolSignal = context.signal;
return new Promise(() => {});
},
},
);
h.chat.start();
h.chat.sendText('First');
await settle();
h.chat.sendText('Second');
await settle();
assert.ok(toolSignal);
assert.equal(h.chat.cancelTurn(), true);
assert.equal(toolSignal.aborted, true);
assert.equal(h.chat.cancelTurn(), false);
assert.equal(h.completions('cancelled').length, 1);
assert.equal(h.events.at(-2).state, 'listening');
await settle();
assert.equal(h.errors().length, 0);
assert.equal(h.chat.sendText('Third'), true);
await settle();
assert.equal(h.requests.length, 3);
assert.deepEqual(w.userTexts(w.items(h.requests[2])), ['First', 'Third']);
// Ending the session still starts the next one fresh.
h.chat.stop();
h.chat.start();
h.chat.sendText('Fresh');
await settle();
assert.deepEqual(w.userTexts(w.items(h.requests.at(-1))), ['Fresh']);
h.chat.stop();
});
for (const [code, expected] of [
[
'API_KEY_MISSING',
() =>
t('Needs {envVars} — add it in Provider Settings', {
envVars: w.keyEnv,
}),
],
[
'CODEX_OAUTH_REQUIRED',
() =>
t(
'Typed chat is unavailable while GEV_PREFER_CODEX_OAUTH requires ChatGPT OAuth.',
),
],
])
test(`${provider}: a 503 ${code} reply names the fix`, async () => {
const h = harness(provider, () =>
Response.json({ error: 'Provider refused', code }, { status: 503 }),
);
h.chat.start();
h.chat.sendText('Hello');
await settle();
assert.equal(h.errors().length, 1);
assert.equal(h.errors()[0].detail, expected());
h.chat.stop();
});
for (const [status, body, headers, expected] of [
[
502,
{ error: 'Refused', code: 'API_KEY_REJECTED' },
{},
() =>
t('The provider rejected the API key. Check it in Provider Settings.'),
],
[
429,
{ error: 'Refused', code: 'RATE_LIMITED' },
{ 'Retry-After': '42' },
() =>
t('Too many chat requests. Try again in {seconds} s.', { seconds: 42 }),
],
[
429,
{ error: 'Refused' },
{},
() =>
t('Too many chat requests. Try again in {seconds} s.', { seconds: 60 }),
],
])
test(`${provider}: a ${status} ${body.code ?? 'throttle'} reply says what to do`, async () => {
const h = harness(provider, () =>
Response.json(body, { status, headers }),
);
h.chat.start();
h.chat.sendText('Hello');
await settle();
assert.equal(h.errors()[0].detail, expected());
h.chat.stop();
});
test(`${provider}: a gateway timeout without a JSON body still reads as a timeout`, async () => {
const h = harness(
provider,
() => new Response('<html>Gateway Timeout</html>', { status: 504 }),
);
h.chat.start();
h.chat.sendText('Hello');
await settle();
assert.equal(h.errors()[0].detail, t('Connection timed out. Try again.'));
h.chat.stop();
});
test(`${provider}: a round with 17 calls hits the call cap before any action`, async () => {
const h = harness(provider, () => w.calls(17));
h.chat.start();
h.chat.sendText('Do everything');
await settle();
assert.equal(h.requests.length, 1);
assert.deepEqual(h.actions, []);
assert.equal(
h.errors()[0].detail,
t('That request needed too many steps. Try a simpler request.'),
);
h.chat.stop();
});
test(`${provider}: the round limit says applied map changes stay`, async () => {
const h = harness(provider, (n) => w.calls(1, `round-${n}`));
h.chat.start();
h.chat.sendText('Keep going');
await settle(30);
assert.equal(h.requests.length, 8);
assert.equal(h.actions.length, 8);
assert.equal(
h.errors()[0].detail,
t(
'That request needed too many steps. Changes already made to the map stay in place.',
),
);
h.chat.stop();
});
test(`${provider}: the deadline covers each request, not the whole turn`, async (context) => {
context.mock.timers.enable({ apis: ['setTimeout'] });
let pending = 0;
const h = harness(
provider,
(n) => (n < 3 ? w.calls(1, `round-${n}`) : w.final('Done')),
{
chat: { timeoutMs: 50 },
runAction: () =>
new Promise((resolve) => {
pending++;
setTimeout(() => resolve({ ok: true }), 40);
}),
},
);
h.chat.start();
h.chat.sendText('Two slow steps');
for (const expected of [1, 2]) {
await settle();
assert.equal(pending, expected);
context.mock.timers.tick(40);
}
await settle();
assert.equal(h.errors().length, 0);
assert.equal(h.completions().length, 1);
h.chat.stop();
});
}
test('long history shortens old tool results and keeps every call paired', async () => {
const w = wire.openai;
let turn = 0;
const h = harness(
'openai',
(n, body) => {
const last = body.messages.at(-1);
return last.role === 'user'
? w.calls(1, `turn-${++turn}`)
: w.final('Ok');
},
{
runAction: async () => ({
ok: true,
say: 'Found',
records: ['x'.repeat(11000)],
}),
},
);
h.chat.start();
for (let i = 1; i <= 20; i++) {
assert.equal(h.chat.sendText(`Turn ${i}`), true);
await settle();
}
assert.equal(h.completions().length, 20);
const messages = h.requests.at(-1).messages;
assert.equal(messages[0].role, 'user');
assert.ok(messages.length <= 64 + 3);
const calls = new Set();
for (const message of messages) {
for (const call of message.tool_calls || []) calls.add(call.id);
if (message.role === 'tool') {
assert.ok(calls.delete(message.tool_call_id), message.tool_call_id);
const result = JSON.parse(message.content);
assert.equal(result.ok, true);
assert.equal(result.say, 'Found');
}
}
assert.deepEqual([...calls], []);
const results = messages.filter((message) => message.role === 'tool');
// Earlier turns are compact; the latest turns keep their full result.
assert.ok(results.slice(0, -2).every((item) => bytes(item.content) <= 1500));
assert.ok(results.slice(-2).every((item) => bytes(item.content) > 10000));
});
test('a request over the size cap drops old turns instead of failing', async () => {
const w = wire.gemini;
const big = (char, size) => char.repeat(size);
let turn = 0;
const h = harness('gemini', (n, body) => {
const last = body.contents.at(-1);
const text = last.parts[0].text;
if (text && turn < 4) {
turn++;
return w.final(big('a', 40000));
}
if (text)
return {
content: {
role: 'model',
parts: [
{ text: big('b', 380000) },
{ functionCall: { name: 'fly_to', args: {}, id: 'large' } },
],
},
};
return w.final('Done');
});
h.chat.start();
for (let i = 1; i <= 5; i++) {
h.chat.sendText(`Turn ${i}`);
await settle();
}
assert.equal(h.errors().length, 0);
assert.equal(h.completions().length, 5);
const request = h.requests.at(-1);
assert.ok(bytes(request) <= 512 * 1024);
const texts = w.userTexts(request.contents);
assert.equal(texts.at(-1), 'Turn 5');
assert.ok(!texts.includes('Turn 1'));
assert.equal(request.contents[0].role, 'user');
assert.ok(request.contents[0].parts[0].text);
});
+37
View File
@@ -2,7 +2,9 @@
export const textProtocols = {
gemini: {
endpoint: '/api/gemini/chat',
keyEnv: 'GEMINI_API_KEY',
user: (text) => ({ role: 'user', parts: [{ text }] }),
note: (text) => ({ role: 'model', parts: [{ text }] }),
isUser: (item) =>
item.role === 'user' &&
item.parts.some((part) => typeof part.text === 'string'),
@@ -40,10 +42,32 @@ export const textProtocols = {
})),
},
],
compactResult: (item, maxBytes) =>
item.role === 'user' && item.parts.some((part) => part.functionResponse)
? {
...item,
parts: item.parts.map((part) =>
part.functionResponse
? {
...part,
functionResponse: {
...part.functionResponse,
response: boundedToolResult(
part.functionResponse.response,
maxBytes,
),
},
}
: part,
),
}
: item,
},
openai: {
endpoint: '/api/openai/chat',
keyEnv: 'OPENAI_API_KEY',
user: (content) => ({ role: 'user', content }),
note: (content) => ({ role: 'assistant', content }),
isUser: (item) => item.role === 'user',
request: (messages) => ({ messages }),
decode(data) {
@@ -75,6 +99,19 @@ export const textProtocols = {
tool_call_id: call.id,
content: JSON.stringify(result),
})),
compactResult(item, maxBytes) {
if (item.role !== 'tool') return item;
let result;
try {
result = JSON.parse(item.content);
} catch {
result = { ok: false, error: 'Tool result could not be read' };
}
return {
...item,
content: JSON.stringify(boundedToolResult(result, maxBytes)),
};
},
},
};
+3 -1
View File
@@ -52,8 +52,10 @@ export function bindVoiceMiniPreview({ ui, session }) {
};
clear();
return session.subscribe((event) => {
if (session.adapter?.getStartOptions?.().textOnly) return;
// Any new session starts empty, even one whose start is still reported
// as typed chat while voice takes over.
if (event.type === 'state' && event.state === 'connecting') clear();
if (session.adapter?.getStartOptions?.().textOnly) return;
if (event.type !== 'transcript' || !event.text) return;
if (event.role === 'user') show('user', event.text);
if (event.role !== 'assistant') return;
+15 -2
View File
@@ -23,8 +23,9 @@ test('lastSentence keeps only the final sentence across scripts', () => {
function fixture({ textOnly = false } = {}) {
const ui = createTestVoiceUi({ panel: true });
const listeners = new Set();
const mode = { textOnly };
const session = {
adapter: { getStartOptions: () => ({ textOnly }) },
adapter: { getStartOptions: () => ({ textOnly: mode.textOnly }) },
subscribe(fn) {
listeners.add(fn);
return () => listeners.delete(fn);
@@ -32,7 +33,7 @@ function fixture({ textOnly = false } = {}) {
};
const emit = (event) => listeners.forEach((fn) => fn(event));
const unbind = bindVoiceMiniPreview({ ui, session });
return { ui, emit, unbind, listeners };
return { ui, emit, unbind, listeners, mode };
}
test('voice preview shows the last user and assistant sentence', () => {
@@ -90,3 +91,15 @@ test('typed chat turns do not reach the voice preview', () => {
assert.deepEqual(f.ui.state.mini, { user: '', assistant: '' });
assert.equal(f.ui.state.root.voiceExchange, false);
});
test('a new session clears the preview even while the adapter still reports chat', () => {
const f = fixture();
f.emit({ type: 'transcript', role: 'user', text: 'Show live flights.' });
assert.equal(f.ui.state.mini.user, 'Show live flights.');
// Voice -> chat -> voice: the next start's connecting can arrive before
// the adapter has left typed mode.
f.mode.textOnly = true;
f.emit({ type: 'state', state: 'connecting' });
assert.deepEqual(f.ui.state.mini, { user: '', assistant: '' });
assert.equal(f.ui.state.root.voiceExchange, false);
});