refactor: separate voice sessions from protocol adapters

This commit is contained in:
Sameh Khamis committed 2026-09-14 20:27:12 -07:00
1 parent 584c9608d5
commit 4507f3a1d0
13 files changed
+735 -62

No files matched your search

+5
View File
@@ -77,3 +77,8 @@
#command-dock.tray-open .gev-voice-cost {
display: none;
}
.gev-voice-tier-btn[hidden],
.gev-voice-cost-value[hidden] {
display: none;
}
+7 -55
View File
@@ -1,58 +1,10 @@
import {
GevRealtimeController,
shouldIgnoreVoiceButtonClick,
} from './realtimeController.js';
import { createVoiceControl } from './control.js';
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
import { createRealtimeSession } from './realtimeSession.js';
/** Bind a supplied action runner and connection controller to the voice controls. */
export function createVoiceCommands({
runner,
dataManager,
annotations = null,
backend,
signal,
debugSink,
createController = (options) => new GevRealtimeController(options),
}) {
if (
window.__gevVoiceCommands &&
typeof window.__gevVoiceCommands.stop === 'function'
) {
window.__gevVoiceCommands.stop({ removeUi: true });
}
const ui = createVoiceControl({ reset: true });
const radioLayer = dataManager?.layers?.get('radio')?.module || null;
const controller = createController({
runner,
ui,
radioLayer,
dataManager,
backend,
signal,
debugSink,
/** Default composition; callers may supply another session adapter factory. */
export function createVoiceCommands(options) {
return bindVoiceCommands({
createSession: createRealtimeSession,
...options,
});
// Deferred annotation outlines finish AFTER their tool result returned. Feed the
// final outcome (resolved / failed) into the conversation so the model can honestly
// confirm — or correct — what it narrated about a boundary it never saw land.
if (annotations && typeof annotations.onOutlineEvent === 'function') {
controller.annotationEventUnsubscribe = annotations.onOutlineEvent(
(evt) => {
controller.notifyMapEvent({ type: 'map_annotation_outline', ...evt });
},
);
}
controller.buttonHandler = () => {
if (shouldIgnoreVoiceButtonClick(controller.spaceKeyHeld)) return;
if (controller.isActive()) controller.stop();
else controller.start({ pushToTalk: false });
};
ui.button.addEventListener('click', controller.buttonHandler);
if (ui.tierButton) {
controller.tierHandler = () => controller.toggleVoiceTier();
ui.tierButton.addEventListener('click', controller.tierHandler);
}
controller.syncCostUi();
controller.bindPushToTalkShortcut();
window.__gevVoiceCommands = controller;
return controller;
}
+13 -2
View File
@@ -1,3 +1,4 @@
import { realtimeSessionEvent } from './realtimeEvents.js';
import { readLayerLifecycleSummary } from './layerSummary.js';
import { createRealtimeBackend } from './realtimeBackend.js';
import {
@@ -195,7 +196,9 @@ const SUPERSEDED_RESPONSE_MEMORY = 8;
export class GevRealtimeController {
constructor({ runner, ui, radioLayer = null, dataManager = null,
backend = createRealtimeBackend(), signal, debugSink = postDebugLog }) {
backend = createRealtimeBackend(), signal, debugSink = postDebugLog, actionExecutor, onSessionEvent }) {
this.actionExecutor = actionExecutor;
this.onSessionEvent = onSessionEvent;
this.backend = backend;
this.lifetimeSignal = signal;
this.connectionAbort = null;
@@ -926,6 +929,7 @@ export class GevRealtimeController {
this.setStatus('idle', 'Voice off');
}
this.setRadioVoiceDucking(false);
if (removeUi) this.emitSessionEvent({ type: 'disposed' });
}
/**
@@ -1039,6 +1043,8 @@ export class GevRealtimeController {
} catch {
return;
}
const sessionEvent = realtimeSessionEvent(payload);
if (sessionEvent) this.emitSessionEvent(sessionEvent);
this.debugLog('server.event', {
type: payload.type,
eventId: payload.event_id || null,
@@ -1245,7 +1251,7 @@ export class GevRealtimeController {
authorityDomain: radioAuthorityDomain,
});
}
result = await this.runner(call.name, parsedArguments, {
result = await (this.actionExecutor || this.runner)(call.name, parsedArguments, {
signal: toolController.signal,
isCurrent: () => (
this.activeToolAbortControllers.has(toolController)
@@ -1467,8 +1473,13 @@ export class GevRealtimeController {
return sent;
}
emitSessionEvent(event) {
try { this.onSessionEvent?.(event); } catch { /* Observers cannot interrupt voice. */ }
}
setStatus(status, detail) {
this.status = status;
this.emitSessionEvent({ type: 'state', state: status, detail });
this.ui.root.dataset.status = status;
if (status === 'error') this.ui.root.classList.remove('error-dismissed');
this.updateVoiceButtonLabel();
+45
View File
@@ -0,0 +1,45 @@
/** Only this adapter knows the Realtime transcript and response event names. */
export function realtimeSessionEvent(payload) {
const type = payload?.type;
if (type === 'input_audio_buffer.speech_started')
return { type: 'interruption', reason: 'user-speech' };
if (type === 'conversation.item.input_audio_transcription.completed')
return {
type: 'transcript',
role: 'user',
text: payload.transcript || '',
final: true,
itemId: payload.item_id || null,
};
if (
[
'response.output_audio_transcript.delta',
'response.output_audio_transcript.done',
'response.output_text.delta',
'response.output_text.done',
'response.audio_transcript.delta',
'response.audio_transcript.done',
'response.text.delta',
'response.text.done',
].includes(type)
) {
const final = type.endsWith('.done');
return {
type: 'transcript',
role: 'assistant',
text: final
? (payload.transcript ?? payload.text ?? '')
: payload.delta || '',
final,
itemId: payload.item_id || null,
responseId: payload.response_id || null,
};
}
if (type === 'response.done')
return {
type: 'completion',
responseId: payload.response?.id || null,
status: payload.response?.status || 'unknown',
};
return null;
}
+35
View File
@@ -0,0 +1,35 @@
import { GevRealtimeController } from './realtimeController.js';
/** Adapt the existing WebRTC implementation to the common voice session. */
export function createRealtimeSession({
emit,
runAction,
createController = (options) => new GevRealtimeController(options),
...options
}) {
const controller = createController({
...options,
actionExecutor: runAction,
onSessionEvent: emit,
});
return {
controller,
capabilities: { costControls: true, pushToTalk: true },
start: (settings) => controller.start(settings),
stop: (settings) => controller.stop(settings),
sendText: (text) => controller.sendTextCommand(text),
sendMapEvent: (event) => controller.notifyMapEvent(event),
ignoreButtonClick: () => Boolean(controller.spaceKeyHeld),
bindControls() {
if (controller.ui.tierButton) {
controller.tierHandler = () => controller.toggleVoiceTier();
controller.ui.tierButton.addEventListener(
'click',
controller.tierHandler,
);
}
controller.syncCostUi();
controller.bindPushToTalkShortcut();
},
};
}
+157
View File
@@ -0,0 +1,157 @@
/**
* Protocol-independent voice lifetime and action dispatch.
* An adapter receives { emit, runAction, signal } and supplies start, stop,
* sendText and sendMapEvent. It owns its microphone/connection and wire messages.
* Events: state, transcript, action-call, action-result, interruption, completion.
*/
export function createVoiceSession({ createAdapter, runner, signal }) {
if (typeof createAdapter !== 'function' || typeof runner !== 'function')
throw new TypeError('Voice requires an adapter factory and action runner');
const lifetime = new AbortController();
const listeners = new Set();
const actions = new Set();
let adapter;
let state = 'idle';
let disposed = false;
let generation = 0;
let startAttempt = 0;
function cancelActions() {
generation++;
for (const action of actions) action.abort();
actions.clear();
}
function emit(event) {
if (disposed || !event || typeof event.type !== 'string') return;
if (event.type === 'disposed') {
dispose(false);
return;
}
if (event.type === 'state') {
state = event.state;
if (['idle', 'error', 'connecting'].includes(state)) cancelActions();
}
if (event.type === 'interruption') cancelActions();
for (const listener of [...listeners]) {
try {
listener(event);
} catch {
/* Observers cannot break a session. */
}
}
}
async function runAction(name, args, options = {}) {
if (
disposed ||
lifetime.signal.aborted ||
['idle', 'error'].includes(state)
)
throw abortError();
const action = new AbortController();
const epoch = generation;
actions.add(action);
const actionSignal = AbortSignal.any([
lifetime.signal,
action.signal,
...(options.signal ? [options.signal] : []),
]);
const isCurrent = () =>
!actionSignal.aborted &&
epoch === generation &&
(!options.isCurrent || options.isCurrent());
try {
if (!isCurrent()) throw abortError();
emit({ type: 'action-call', name, arguments: args });
if (!isCurrent()) throw abortError();
const result = await runner(name, args, {
...options,
signal: actionSignal,
isCurrent,
});
if (!isCurrent()) throw abortError();
emit({ type: 'action-result', name, result });
return result;
} finally {
actions.delete(action);
}
}
function dispose(stopAdapter = true) {
if (disposed) return;
disposed = true;
startAttempt++;
state = 'idle';
cancelActions();
lifetime.abort();
signal?.removeEventListener('abort', abort);
listeners.clear();
if (stopAdapter) adapter?.stop({ removeUi: true });
}
const abort = () => dispose();
try {
adapter = createAdapter({ emit, runAction, signal: lifetime.signal });
for (const method of ['start', 'stop', 'sendText', 'sendMapEvent']) {
if (typeof adapter?.[method] !== 'function')
throw new TypeError('Voice adapter must implement ' + method);
}
} catch (error) {
dispose();
throw error;
}
signal?.addEventListener('abort', abort, { once: true });
if (signal?.aborted) dispose();
return {
adapter,
signal: lifetime.signal,
get state() {
return state;
},
get disposed() {
return disposed;
},
isActive: () => !disposed && !['idle', 'error'].includes(state),
subscribe(listener) {
if (disposed) return () => {};
listeners.add(listener);
return () => listeners.delete(listener);
},
async start(options) {
if (disposed || !['idle', 'error'].includes(state)) return;
emit({ type: 'state', state: 'connecting' });
const attempt = ++startAttempt;
try {
await adapter.start(options);
} catch (error) {
if (!disposed && attempt === startAttempt) {
try {
adapter.stop({ preserveStatus: true });
} finally {
emit({ type: 'state', state: 'error', detail: error.message });
}
}
}
},
stop(options = {}) {
if (options.removeUi) return dispose();
if (disposed) return;
startAttempt++;
cancelActions();
try {
adapter.stop(options);
} finally {
if (!options.preserveStatus)
emit({ type: 'state', state: 'idle', detail: 'Voice off' });
}
},
sendText: (text) => !disposed && adapter.sendText(text),
sendMapEvent: (event) => !disposed && adapter.sendMapEvent(event),
destroy: () => dispose(),
};
}
function abortError() {
return new DOMException('Voice action cancelled', 'AbortError');
}
+348
View File
@@ -0,0 +1,348 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceSession } from './session.js';
import { realtimeSessionEvent } from './realtimeEvents.js';
import { createVoiceCommands } from './sessionCommands.js';
function deferred() {
let resolve;
const promise = new Promise((done) => {
resolve = done;
});
return { promise, resolve };
}
function fixture(
runner = async (name, args) => ({ ok: true, name, ...args }),
signal,
) {
let hooks;
const sent = [];
let stops = 0;
const session = createVoiceSession({
runner,
signal,
createAdapter(options) {
hooks = options;
return {
start() {
hooks.emit({ type: 'state', state: 'listening' });
},
stop() {
stops++;
hooks.emit({ type: 'state', state: 'idle' });
},
sendText(text) {
sent.push(['text', text]);
return true;
},
sendMapEvent(event) {
sent.push(['map', event]);
return true;
},
};
},
});
return {
session,
hooks,
sent,
get stops() {
return stops;
},
};
}
test('alternate protocol shares session events and action execution without Realtime', async () => {
const f = fixture();
const events = [];
f.session.subscribe(() => {
throw new Error('observer');
});
const unsubscribe = f.session.subscribe((event) => events.push(event));
await f.session.start();
assert.equal(f.session.isActive(), true);
f.hooks.emit({
type: 'transcript',
role: 'user',
text: 'London',
final: true,
});
assert.deepEqual(
await f.hooks.runAction('fly_to_location', { query: 'London' }),
{ ok: true, name: 'fly_to_location', query: 'London' },
);
f.hooks.emit({ type: 'completion', status: 'completed' });
f.session.sendText('Next');
f.session.sendMapEvent({
type: 'map_annotation_outline',
status: 'resolved',
});
assert.deepEqual(
events.map((event) => event.type),
[
'state',
'state',
'transcript',
'action-call',
'action-result',
'completion',
],
);
assert.equal(f.sent.length, 2);
unsubscribe();
f.session.stop();
assert.equal(f.session.state, 'idle');
await f.session.start();
assert.equal(f.session.state, 'listening');
f.session.destroy();
assert.equal(f.session.disposed, true);
assert.equal(f.session.sendText('stale'), false);
});
test('stop, interruption and lifetime abort cancel actions and suppress stale results', async () => {
for (const cancel of ['stop', 'interruption', 'lifetime']) {
const result = deferred();
const lifetime = new AbortController();
let observed;
const f = fixture(async (_name, _args, options) => {
observed = options;
return result.promise;
}, lifetime.signal);
const events = [];
f.session.subscribe((event) => events.push(event));
await f.session.start();
const pending = f.hooks.runAction('fly_to_location', {});
if (cancel === 'stop') f.session.stop();
if (cancel === 'interruption')
f.hooks.emit({ type: 'interruption', reason: 'user-speech' });
if (cancel === 'lifetime') lifetime.abort();
assert.equal(observed.signal.aborted, true);
assert.equal(observed.isCurrent(), false);
result.resolve({ ok: true });
await assert.rejects(pending, { name: 'AbortError' });
assert.equal(
events.some((event) => event.type === 'action-result'),
false,
);
f.session.destroy();
}
});
test('caller cancellation and sibling actions preserve independent ownership', async () => {
const first = deferred();
const f = fixture(async (name) =>
name === 'first' ? first.promise : { ok: true },
);
await f.session.start();
const signal = new AbortController();
const pending = f.hooks.runAction('first', {}, { signal: signal.signal });
signal.abort();
assert.deepEqual(await f.hooks.runAction('second', {}), { ok: true });
first.resolve({ ok: true });
await assert.rejects(pending, { name: 'AbortError' });
f.session.destroy();
});
test('stale connect failure cannot stop a replacement session', async () => {
let reject;
let calls = 0;
let hooks;
const session = createVoiceSession({
runner: async () => {},
createAdapter(options) {
hooks = options;
return {
start() {
if (++calls === 1)
return new Promise((_resolve, fail) => {
reject = fail;
});
hooks.emit({ type: 'state', state: 'listening' });
},
stop() {},
sendText() {},
sendMapEvent() {},
};
},
});
const old = session.start();
session.stop();
await session.start();
reject(new Error('old connection failed'));
await old;
assert.equal(session.state, 'listening');
session.destroy();
});
test('wire events project only normalized transcript, interruption and completion fields', () => {
assert.deepEqual(
realtimeSessionEvent({
type: 'response.output_audio_transcript.delta',
delta: 'Hello',
item_id: 'i',
response_id: 'r',
extra: 'ignored',
}),
{
type: 'transcript',
role: 'assistant',
text: 'Hello',
final: false,
itemId: 'i',
responseId: 'r',
},
);
assert.deepEqual(
realtimeSessionEvent({
type: 'conversation.item.input_audio_transcription.completed',
transcript: 'Go',
}),
{
type: 'transcript',
role: 'user',
text: 'Go',
final: true,
itemId: null,
},
);
assert.equal(
realtimeSessionEvent({ type: 'response.output_text.done', text: 'Done' })
.final,
true,
);
assert.equal(
realtimeSessionEvent({ type: 'input_audio_buffer.speech_started' }).type,
'interruption',
);
assert.equal(
realtimeSessionEvent({
type: 'response.done',
response: { id: 'r', status: 'cancelled' },
}).status,
'cancelled',
);
assert.equal(
realtimeSessionEvent({
type: 'session.created',
session: { secret: 'excluded' },
}),
null,
);
});
test('common button and annotation bindings work with an alternate adapter and clean up', async () => {
const previous = globalThis.window;
globalThis.window = {};
try {
const button = new EventTarget();
button.setAttribute = () => {};
let removed = 0;
let subscribed = null;
let calls = 0;
const events = [];
const ui = {
button,
root: {
dataset: {},
remove() {
removed++;
},
},
status: {},
detail: {},
tierButton: {},
costValue: {},
helpDetail: {},
};
const lifetime = new AbortController();
const controls = createVoiceCommands({
runner: async () => {
calls++;
return { ok: true };
},
signal: lifetime.signal,
createControl: () => ui,
annotations: {
onOutlineEvent(listener) {
subscribed = listener;
return () => {
subscribed = null;
};
},
},
createSession({ emit, runAction }) {
return {
async start() {
emit({ type: 'state', state: 'listening', detail: 'Ready' });
await runAction('fly_to_location', { query: 'London' });
},
stop() {
emit({ type: 'state', state: 'idle' });
},
sendText() {},
sendMapEvent(event) {
events.push(event);
},
};
},
});
button.dispatchEvent(new Event('click'));
await new Promise((done) => setTimeout(done, 0));
assert.equal(calls, 1);
assert.equal(ui.tierButton.hidden, true);
assert.equal(ui.costValue.hidden, true);
assert.equal(ui.helpDetail.textContent, 'Activate to toggle voice');
assert.equal(ui.status.textContent, 'LISTENING');
assert.equal(ui.detail.textContent, 'Ready');
subscribed({ status: 'resolved' });
assert.deepEqual(events, [
{ type: 'map_annotation_outline', status: 'resolved' },
]);
button.dispatchEvent(new Event('click'));
assert.equal(controls.state, 'idle');
lifetime.abort();
assert.equal(subscribed, null);
assert.equal(removed, 1);
button.dispatchEvent(new Event('click'));
assert.equal(calls, 1);
} finally {
globalThis.window = previous;
}
});
test('late actions cannot run after stop and failed startup closes the adapter', async () => {
let calls = 0;
const f = fixture(async () => {
calls++;
});
await f.session.start();
f.session.stop();
await assert.rejects(f.hooks.runAction('fly_to_location', {}), {
name: 'AbortError',
});
assert.equal(calls, 0);
f.session.destroy();
let stopped = 0;
const session = createVoiceSession({
runner: async () => {},
createAdapter({ emit }) {
return {
start() {
emit({ type: 'state', state: 'connecting' });
throw new Error('Connection refused');
},
stop() {
stopped++;
emit({ type: 'state', state: 'idle' });
},
sendText() {},
sendMapEvent() {},
};
},
});
await session.start();
assert.equal(session.state, 'error');
assert.equal(stopped, 1);
session.destroy();
});
+86
View File
@@ -0,0 +1,86 @@
import { createVoiceControl } from './control.js';
import { createVoiceSession } from './session.js';
/** Bind common controls to a supplied voice-session adapter. */
export function createVoiceCommands({
runner,
dataManager,
annotations = null,
createSession,
createController,
backend,
signal,
debugSink,
createControl = createVoiceControl,
}) {
window.__gevVoiceCommands?.stop?.({ removeUi: true });
const ui = createControl({ reset: true });
const session = createVoiceSession({
runner,
signal,
createAdapter: (hooks) =>
createSession({
...hooks,
runner,
ui,
dataManager,
backend,
debugSink,
createController,
radioLayer: dataManager?.layers?.get('radio')?.module || null,
}),
});
const adapter = session.adapter;
const capabilities = adapter.capabilities || {};
if (ui.tierButton) ui.tierButton.hidden = !capabilities.costControls;
if (ui.costValue) ui.costValue.hidden = !capabilities.costControls;
if (!capabilities.pushToTalk) {
ui.button.setAttribute('aria-label', 'Toggle voice control');
if (ui.helpDetail) ui.helpDetail.textContent = 'Activate to toggle voice';
}
// Retain the existing controller's inspection surface for browser tools.
const controls = adapter.controller || session;
controls.session = session;
const updateStatus = session.subscribe((event) => {
if (event.type !== 'state') return;
ui.root.dataset.status = event.state;
ui.status.textContent =
event.state === 'idle' ? 'OFF' : event.state.toUpperCase();
ui.detail.textContent =
event.detail || (event.state === 'idle' ? 'Voice off' : 'Voice active');
ui.button.setAttribute('aria-pressed', String(session.isActive()));
if (ui.errorDetail)
ui.errorDetail.textContent =
event.state === 'error'
? event.detail || 'Voice could not be started.'
: '';
if (event.state === 'error') ui.root.classList?.remove('error-dismissed');
});
const annotationUnsubscribe = annotations?.onOutlineEvent?.((event) => {
session.sendMapEvent({ type: 'map_annotation_outline', ...event });
});
const buttonHandler = () => {
if (adapter.ignoreButtonClick?.()) return;
if (session.isActive()) session.stop();
else void session.start({ pushToTalk: false });
};
ui.button.addEventListener('click', buttonHandler);
session.signal.addEventListener(
'abort',
() => {
ui.button.removeEventListener('click', buttonHandler);
annotationUnsubscribe?.();
updateStatus();
ui.root.remove();
},
{ once: true },
);
if (session.disposed) {
ui.button.removeEventListener('click', buttonHandler);
annotationUnsubscribe?.();
updateStatus();
ui.root.remove();
} else adapter.bindControls?.();
window.__gevVoiceCommands = controls;
return controls;
}