mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-06 11:24:00 +00:00
Merge pull request #578 from bilawalsidhu/refactor/voice-session-interface
refactor: separate voice sessions from protocol adapters
This commit is contained in:
13 files changed
+735
-62
No files matched your search
@@ -1,5 +1,6 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
- Separate voice session lifetime and common controls from the default Realtime protocol adapter.
|
||||||
- Separate canonical voice action arguments from descriptive wording, preserving the existing Realtime tool inventory.
|
- Separate canonical voice action arguments from descriptive wording, preserving the existing Realtime tool inventory.
|
||||||
|
|
||||||
- Expose portable radio, camera-type and regional source helpers; keep HTTP transport separate from record normalization.
|
- Expose portable radio, camera-type and regional source helpers; keep HTTP transport separate from record normalization.
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# God's Eye View Current State
|
# God's Eye View Current State
|
||||||
|
|
||||||
|
Voice controls bind to a protocol-independent session factory. The default WebRTC adapter preserves the existing Realtime connection, push-to-talk, cost controls and radio handoff. Session subscriptions expose state, transcript, action call/result, interruption and completion events; stopping or replacing a session cancels pending actions. Alternate adapters can use the same controls and action runner.
|
||||||
|
|
||||||
Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds the same 28 tools using separate description-only metadata. Description customization cannot replace argument types, enum values or required fields.
|
Voice action argument schemas have one portable owner under `src/voice/actionSchemas.js`. The Realtime provider builds the same 28 tools using separate description-only metadata. Description customization cannot replace argument types, enum values or required fields.
|
||||||
|
|
||||||
Portable source exports provide Radio Browser station normalization, CCTV feed types and regional records independently of HTTP middleware. Radio normalization accepts an explicit URL policy; the standalone directory retains its existing HTTPS rules. Tile coordinate validation accepts explicit zoom bounds with unchanged traffic defaults. Cache, request and rendering owners remain unchanged.
|
Portable source exports provide Radio Browser station normalization, CCTV feed types and regional records independently of HTTP middleware. Radio normalization accepts an explicit URL policy; the standalone directory retains its existing HTTPS rules. Tile coordinate validation accepts explicit zoom bounds with unchanged traffic defaults. Cache, request and rendering owners remain unchanged.
|
||||||
|
|||||||
+4
-1
@@ -211,6 +211,9 @@
|
|||||||
"./sources/radio-browser": "./src/sources/radioBrowser.js",
|
"./sources/radio-browser": "./src/sources/radioBrowser.js",
|
||||||
"./sources/cctv-types": "./src/sources/cctvTypes.js",
|
"./sources/cctv-types": "./src/sources/cctvTypes.js",
|
||||||
"./sources/regional": "./src/data/regionalModel.js",
|
"./sources/regional": "./src/data/regionalModel.js",
|
||||||
"./voice/action-schemas": "./src/voice/actionSchemas.js"
|
"./voice/action-schemas": "./src/voice/actionSchemas.js",
|
||||||
|
"./voice/session": "./src/voice/session.js",
|
||||||
|
"./voice/session-controls": "./src/voice/sessionCommands.js",
|
||||||
|
"./voice/realtime-session": "./src/voice/realtimeSession.js"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -654,5 +654,10 @@
|
|||||||
"src/sources/protocols.test.mjs",
|
"src/sources/protocols.test.mjs",
|
||||||
"src/voice/actionSchemas.js",
|
"src/voice/actionSchemas.js",
|
||||||
"src/voice/actionSchemas.test.mjs",
|
"src/voice/actionSchemas.test.mjs",
|
||||||
"server/providers/openai/toolDescriptions.js"
|
"server/providers/openai/toolDescriptions.js",
|
||||||
|
"src/voice/session.js",
|
||||||
|
"src/voice/sessionCommands.js",
|
||||||
|
"src/voice/realtimeSession.js",
|
||||||
|
"src/voice/realtimeEvents.js",
|
||||||
|
"src/voice/session.test.mjs"
|
||||||
]
|
]
|
||||||
@@ -1058,7 +1058,8 @@
|
|||||||
"./voice/control",
|
"./voice/control",
|
||||||
"./voice/realtime-backend",
|
"./voice/realtime-backend",
|
||||||
"./voice/commands",
|
"./voice/commands",
|
||||||
"./voice/cost"
|
"./voice/cost",
|
||||||
|
"./voice/realtime-session"
|
||||||
],
|
],
|
||||||
"modules": [
|
"modules": [
|
||||||
"src/voice/realtimeController.js",
|
"src/voice/realtimeController.js",
|
||||||
@@ -1066,7 +1067,11 @@
|
|||||||
"src/voice/control.js",
|
"src/voice/control.js",
|
||||||
"src/voice/layerSummary.js",
|
"src/voice/layerSummary.js",
|
||||||
"src/voice/voiceCost.js",
|
"src/voice/voiceCost.js",
|
||||||
"src/voice/commands.js"
|
"src/voice/commands.js",
|
||||||
|
"src/voice/session.js",
|
||||||
|
"src/voice/sessionCommands.js",
|
||||||
|
"src/voice/realtimeSession.js",
|
||||||
|
"src/voice/realtimeEvents.js"
|
||||||
],
|
],
|
||||||
"external": []
|
"external": []
|
||||||
},
|
},
|
||||||
@@ -1599,7 +1604,11 @@
|
|||||||
"src/app/layers/directions.js",
|
"src/app/layers/directions.js",
|
||||||
"src/search/coordinateGeocoder.js",
|
"src/search/coordinateGeocoder.js",
|
||||||
"src/search/coordinateParser.js",
|
"src/search/coordinateParser.js",
|
||||||
"src/search/presetGeocoder.js"
|
"src/search/presetGeocoder.js",
|
||||||
|
"src/voice/session.js",
|
||||||
|
"src/voice/sessionCommands.js",
|
||||||
|
"src/voice/realtimeSession.js",
|
||||||
|
"src/voice/realtimeEvents.js"
|
||||||
],
|
],
|
||||||
"external": [
|
"external": [
|
||||||
"@mapbox/vector-tile",
|
"@mapbox/vector-tile",
|
||||||
@@ -2143,5 +2152,19 @@
|
|||||||
"exports": ["./voice/action-schemas"],
|
"exports": ["./voice/action-schemas"],
|
||||||
"modules": ["src/voice/actionSchemas.js"],
|
"modules": ["src/voice/actionSchemas.js"],
|
||||||
"external": []
|
"external": []
|
||||||
|
},
|
||||||
|
"voice-session": {
|
||||||
|
"exports": ["./voice/session"],
|
||||||
|
"modules": ["src/voice/session.js"],
|
||||||
|
"external": []
|
||||||
|
},
|
||||||
|
"voice-session-controls": {
|
||||||
|
"exports": ["./voice/session-controls"],
|
||||||
|
"modules": [
|
||||||
|
"src/voice/sessionCommands.js",
|
||||||
|
"src/voice/session.js",
|
||||||
|
"src/voice/control.js"
|
||||||
|
],
|
||||||
|
"external": []
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -77,3 +77,8 @@
|
|||||||
#command-dock.tray-open .gev-voice-cost {
|
#command-dock.tray-open .gev-voice-cost {
|
||||||
display: none;
|
display: none;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.gev-voice-tier-btn[hidden],
|
||||||
|
.gev-voice-cost-value[hidden] {
|
||||||
|
display: none;
|
||||||
|
}
|
||||||
+7
-55
@@ -1,58 +1,10 @@
|
|||||||
import {
|
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
|
||||||
GevRealtimeController,
|
import { createRealtimeSession } from './realtimeSession.js';
|
||||||
shouldIgnoreVoiceButtonClick,
|
|
||||||
} from './realtimeController.js';
|
|
||||||
import { createVoiceControl } from './control.js';
|
|
||||||
|
|
||||||
/** Bind a supplied action runner and connection controller to the voice controls. */
|
/** Default composition; callers may supply another session adapter factory. */
|
||||||
export function createVoiceCommands({
|
export function createVoiceCommands(options) {
|
||||||
runner,
|
return bindVoiceCommands({
|
||||||
dataManager,
|
createSession: createRealtimeSession,
|
||||||
annotations = null,
|
...options,
|
||||||
backend,
|
|
||||||
signal,
|
|
||||||
debugSink,
|
|
||||||
createController = (options) => new GevRealtimeController(options),
|
|
||||||
}) {
|
|
||||||
if (
|
|
||||||
window.__gevVoiceCommands &&
|
|
||||||
typeof window.__gevVoiceCommands.stop === 'function'
|
|
||||||
) {
|
|
||||||
window.__gevVoiceCommands.stop({ removeUi: true });
|
|
||||||
}
|
|
||||||
const ui = createVoiceControl({ reset: true });
|
|
||||||
const radioLayer = dataManager?.layers?.get('radio')?.module || null;
|
|
||||||
const controller = createController({
|
|
||||||
runner,
|
|
||||||
ui,
|
|
||||||
radioLayer,
|
|
||||||
dataManager,
|
|
||||||
backend,
|
|
||||||
signal,
|
|
||||||
debugSink,
|
|
||||||
});
|
});
|
||||||
// Deferred annotation outlines finish AFTER their tool result returned. Feed the
|
|
||||||
// final outcome (resolved / failed) into the conversation so the model can honestly
|
|
||||||
// confirm — or correct — what it narrated about a boundary it never saw land.
|
|
||||||
if (annotations && typeof annotations.onOutlineEvent === 'function') {
|
|
||||||
controller.annotationEventUnsubscribe = annotations.onOutlineEvent(
|
|
||||||
(evt) => {
|
|
||||||
controller.notifyMapEvent({ type: 'map_annotation_outline', ...evt });
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
controller.buttonHandler = () => {
|
|
||||||
if (shouldIgnoreVoiceButtonClick(controller.spaceKeyHeld)) return;
|
|
||||||
if (controller.isActive()) controller.stop();
|
|
||||||
else controller.start({ pushToTalk: false });
|
|
||||||
};
|
|
||||||
ui.button.addEventListener('click', controller.buttonHandler);
|
|
||||||
if (ui.tierButton) {
|
|
||||||
controller.tierHandler = () => controller.toggleVoiceTier();
|
|
||||||
ui.tierButton.addEventListener('click', controller.tierHandler);
|
|
||||||
}
|
|
||||||
controller.syncCostUi();
|
|
||||||
controller.bindPushToTalkShortcut();
|
|
||||||
window.__gevVoiceCommands = controller;
|
|
||||||
return controller;
|
|
||||||
}
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { realtimeSessionEvent } from './realtimeEvents.js';
|
||||||
import { readLayerLifecycleSummary } from './layerSummary.js';
|
import { readLayerLifecycleSummary } from './layerSummary.js';
|
||||||
import { createRealtimeBackend } from './realtimeBackend.js';
|
import { createRealtimeBackend } from './realtimeBackend.js';
|
||||||
import {
|
import {
|
||||||
@@ -195,7 +196,9 @@ const SUPERSEDED_RESPONSE_MEMORY = 8;
|
|||||||
|
|
||||||
export class GevRealtimeController {
|
export class GevRealtimeController {
|
||||||
constructor({ runner, ui, radioLayer = null, dataManager = null,
|
constructor({ runner, ui, radioLayer = null, dataManager = null,
|
||||||
backend = createRealtimeBackend(), signal, debugSink = postDebugLog }) {
|
backend = createRealtimeBackend(), signal, debugSink = postDebugLog, actionExecutor, onSessionEvent }) {
|
||||||
|
this.actionExecutor = actionExecutor;
|
||||||
|
this.onSessionEvent = onSessionEvent;
|
||||||
this.backend = backend;
|
this.backend = backend;
|
||||||
this.lifetimeSignal = signal;
|
this.lifetimeSignal = signal;
|
||||||
this.connectionAbort = null;
|
this.connectionAbort = null;
|
||||||
@@ -926,6 +929,7 @@ export class GevRealtimeController {
|
|||||||
this.setStatus('idle', 'Voice off');
|
this.setStatus('idle', 'Voice off');
|
||||||
}
|
}
|
||||||
this.setRadioVoiceDucking(false);
|
this.setRadioVoiceDucking(false);
|
||||||
|
if (removeUi) this.emitSessionEvent({ type: 'disposed' });
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1039,6 +1043,8 @@ export class GevRealtimeController {
|
|||||||
} catch {
|
} catch {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const sessionEvent = realtimeSessionEvent(payload);
|
||||||
|
if (sessionEvent) this.emitSessionEvent(sessionEvent);
|
||||||
this.debugLog('server.event', {
|
this.debugLog('server.event', {
|
||||||
type: payload.type,
|
type: payload.type,
|
||||||
eventId: payload.event_id || null,
|
eventId: payload.event_id || null,
|
||||||
@@ -1245,7 +1251,7 @@ export class GevRealtimeController {
|
|||||||
authorityDomain: radioAuthorityDomain,
|
authorityDomain: radioAuthorityDomain,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
result = await this.runner(call.name, parsedArguments, {
|
result = await (this.actionExecutor || this.runner)(call.name, parsedArguments, {
|
||||||
signal: toolController.signal,
|
signal: toolController.signal,
|
||||||
isCurrent: () => (
|
isCurrent: () => (
|
||||||
this.activeToolAbortControllers.has(toolController)
|
this.activeToolAbortControllers.has(toolController)
|
||||||
@@ -1467,8 +1473,13 @@ export class GevRealtimeController {
|
|||||||
return sent;
|
return sent;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
emitSessionEvent(event) {
|
||||||
|
try { this.onSessionEvent?.(event); } catch { /* Observers cannot interrupt voice. */ }
|
||||||
|
}
|
||||||
|
|
||||||
setStatus(status, detail) {
|
setStatus(status, detail) {
|
||||||
this.status = status;
|
this.status = status;
|
||||||
|
this.emitSessionEvent({ type: 'state', state: status, detail });
|
||||||
this.ui.root.dataset.status = status;
|
this.ui.root.dataset.status = status;
|
||||||
if (status === 'error') this.ui.root.classList.remove('error-dismissed');
|
if (status === 'error') this.ui.root.classList.remove('error-dismissed');
|
||||||
this.updateVoiceButtonLabel();
|
this.updateVoiceButtonLabel();
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
/** Only this adapter knows the Realtime transcript and response event names. */
|
||||||
|
export function realtimeSessionEvent(payload) {
|
||||||
|
const type = payload?.type;
|
||||||
|
if (type === 'input_audio_buffer.speech_started')
|
||||||
|
return { type: 'interruption', reason: 'user-speech' };
|
||||||
|
if (type === 'conversation.item.input_audio_transcription.completed')
|
||||||
|
return {
|
||||||
|
type: 'transcript',
|
||||||
|
role: 'user',
|
||||||
|
text: payload.transcript || '',
|
||||||
|
final: true,
|
||||||
|
itemId: payload.item_id || null,
|
||||||
|
};
|
||||||
|
if (
|
||||||
|
[
|
||||||
|
'response.output_audio_transcript.delta',
|
||||||
|
'response.output_audio_transcript.done',
|
||||||
|
'response.output_text.delta',
|
||||||
|
'response.output_text.done',
|
||||||
|
'response.audio_transcript.delta',
|
||||||
|
'response.audio_transcript.done',
|
||||||
|
'response.text.delta',
|
||||||
|
'response.text.done',
|
||||||
|
].includes(type)
|
||||||
|
) {
|
||||||
|
const final = type.endsWith('.done');
|
||||||
|
return {
|
||||||
|
type: 'transcript',
|
||||||
|
role: 'assistant',
|
||||||
|
text: final
|
||||||
|
? (payload.transcript ?? payload.text ?? '')
|
||||||
|
: payload.delta || '',
|
||||||
|
final,
|
||||||
|
itemId: payload.item_id || null,
|
||||||
|
responseId: payload.response_id || null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (type === 'response.done')
|
||||||
|
return {
|
||||||
|
type: 'completion',
|
||||||
|
responseId: payload.response?.id || null,
|
||||||
|
status: payload.response?.status || 'unknown',
|
||||||
|
};
|
||||||
|
return null;
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
import { GevRealtimeController } from './realtimeController.js';
|
||||||
|
|
||||||
|
/** Adapt the existing WebRTC implementation to the common voice session. */
|
||||||
|
export function createRealtimeSession({
|
||||||
|
emit,
|
||||||
|
runAction,
|
||||||
|
createController = (options) => new GevRealtimeController(options),
|
||||||
|
...options
|
||||||
|
}) {
|
||||||
|
const controller = createController({
|
||||||
|
...options,
|
||||||
|
actionExecutor: runAction,
|
||||||
|
onSessionEvent: emit,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
controller,
|
||||||
|
capabilities: { costControls: true, pushToTalk: true },
|
||||||
|
start: (settings) => controller.start(settings),
|
||||||
|
stop: (settings) => controller.stop(settings),
|
||||||
|
sendText: (text) => controller.sendTextCommand(text),
|
||||||
|
sendMapEvent: (event) => controller.notifyMapEvent(event),
|
||||||
|
ignoreButtonClick: () => Boolean(controller.spaceKeyHeld),
|
||||||
|
bindControls() {
|
||||||
|
if (controller.ui.tierButton) {
|
||||||
|
controller.tierHandler = () => controller.toggleVoiceTier();
|
||||||
|
controller.ui.tierButton.addEventListener(
|
||||||
|
'click',
|
||||||
|
controller.tierHandler,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
controller.syncCostUi();
|
||||||
|
controller.bindPushToTalkShortcut();
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
/**
|
||||||
|
* Protocol-independent voice lifetime and action dispatch.
|
||||||
|
* An adapter receives { emit, runAction, signal } and supplies start, stop,
|
||||||
|
* sendText and sendMapEvent. It owns its microphone/connection and wire messages.
|
||||||
|
* Events: state, transcript, action-call, action-result, interruption, completion.
|
||||||
|
*/
|
||||||
|
export function createVoiceSession({ createAdapter, runner, signal }) {
|
||||||
|
if (typeof createAdapter !== 'function' || typeof runner !== 'function')
|
||||||
|
throw new TypeError('Voice requires an adapter factory and action runner');
|
||||||
|
const lifetime = new AbortController();
|
||||||
|
const listeners = new Set();
|
||||||
|
const actions = new Set();
|
||||||
|
let adapter;
|
||||||
|
let state = 'idle';
|
||||||
|
let disposed = false;
|
||||||
|
let generation = 0;
|
||||||
|
let startAttempt = 0;
|
||||||
|
|
||||||
|
function cancelActions() {
|
||||||
|
generation++;
|
||||||
|
for (const action of actions) action.abort();
|
||||||
|
actions.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
function emit(event) {
|
||||||
|
if (disposed || !event || typeof event.type !== 'string') return;
|
||||||
|
if (event.type === 'disposed') {
|
||||||
|
dispose(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (event.type === 'state') {
|
||||||
|
state = event.state;
|
||||||
|
if (['idle', 'error', 'connecting'].includes(state)) cancelActions();
|
||||||
|
}
|
||||||
|
if (event.type === 'interruption') cancelActions();
|
||||||
|
for (const listener of [...listeners]) {
|
||||||
|
try {
|
||||||
|
listener(event);
|
||||||
|
} catch {
|
||||||
|
/* Observers cannot break a session. */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runAction(name, args, options = {}) {
|
||||||
|
if (
|
||||||
|
disposed ||
|
||||||
|
lifetime.signal.aborted ||
|
||||||
|
['idle', 'error'].includes(state)
|
||||||
|
)
|
||||||
|
throw abortError();
|
||||||
|
const action = new AbortController();
|
||||||
|
const epoch = generation;
|
||||||
|
actions.add(action);
|
||||||
|
const actionSignal = AbortSignal.any([
|
||||||
|
lifetime.signal,
|
||||||
|
action.signal,
|
||||||
|
...(options.signal ? [options.signal] : []),
|
||||||
|
]);
|
||||||
|
const isCurrent = () =>
|
||||||
|
!actionSignal.aborted &&
|
||||||
|
epoch === generation &&
|
||||||
|
(!options.isCurrent || options.isCurrent());
|
||||||
|
try {
|
||||||
|
if (!isCurrent()) throw abortError();
|
||||||
|
emit({ type: 'action-call', name, arguments: args });
|
||||||
|
if (!isCurrent()) throw abortError();
|
||||||
|
const result = await runner(name, args, {
|
||||||
|
...options,
|
||||||
|
signal: actionSignal,
|
||||||
|
isCurrent,
|
||||||
|
});
|
||||||
|
if (!isCurrent()) throw abortError();
|
||||||
|
emit({ type: 'action-result', name, result });
|
||||||
|
return result;
|
||||||
|
} finally {
|
||||||
|
actions.delete(action);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function dispose(stopAdapter = true) {
|
||||||
|
if (disposed) return;
|
||||||
|
disposed = true;
|
||||||
|
startAttempt++;
|
||||||
|
state = 'idle';
|
||||||
|
cancelActions();
|
||||||
|
lifetime.abort();
|
||||||
|
signal?.removeEventListener('abort', abort);
|
||||||
|
listeners.clear();
|
||||||
|
if (stopAdapter) adapter?.stop({ removeUi: true });
|
||||||
|
}
|
||||||
|
const abort = () => dispose();
|
||||||
|
try {
|
||||||
|
adapter = createAdapter({ emit, runAction, signal: lifetime.signal });
|
||||||
|
for (const method of ['start', 'stop', 'sendText', 'sendMapEvent']) {
|
||||||
|
if (typeof adapter?.[method] !== 'function')
|
||||||
|
throw new TypeError('Voice adapter must implement ' + method);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
dispose();
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
signal?.addEventListener('abort', abort, { once: true });
|
||||||
|
if (signal?.aborted) dispose();
|
||||||
|
|
||||||
|
return {
|
||||||
|
adapter,
|
||||||
|
signal: lifetime.signal,
|
||||||
|
get state() {
|
||||||
|
return state;
|
||||||
|
},
|
||||||
|
get disposed() {
|
||||||
|
return disposed;
|
||||||
|
},
|
||||||
|
isActive: () => !disposed && !['idle', 'error'].includes(state),
|
||||||
|
subscribe(listener) {
|
||||||
|
if (disposed) return () => {};
|
||||||
|
listeners.add(listener);
|
||||||
|
return () => listeners.delete(listener);
|
||||||
|
},
|
||||||
|
async start(options) {
|
||||||
|
if (disposed || !['idle', 'error'].includes(state)) return;
|
||||||
|
emit({ type: 'state', state: 'connecting' });
|
||||||
|
const attempt = ++startAttempt;
|
||||||
|
try {
|
||||||
|
await adapter.start(options);
|
||||||
|
} catch (error) {
|
||||||
|
if (!disposed && attempt === startAttempt) {
|
||||||
|
try {
|
||||||
|
adapter.stop({ preserveStatus: true });
|
||||||
|
} finally {
|
||||||
|
emit({ type: 'state', state: 'error', detail: error.message });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
stop(options = {}) {
|
||||||
|
if (options.removeUi) return dispose();
|
||||||
|
if (disposed) return;
|
||||||
|
startAttempt++;
|
||||||
|
cancelActions();
|
||||||
|
try {
|
||||||
|
adapter.stop(options);
|
||||||
|
} finally {
|
||||||
|
if (!options.preserveStatus)
|
||||||
|
emit({ type: 'state', state: 'idle', detail: 'Voice off' });
|
||||||
|
}
|
||||||
|
},
|
||||||
|
sendText: (text) => !disposed && adapter.sendText(text),
|
||||||
|
sendMapEvent: (event) => !disposed && adapter.sendMapEvent(event),
|
||||||
|
destroy: () => dispose(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortError() {
|
||||||
|
return new DOMException('Voice action cancelled', 'AbortError');
|
||||||
|
}
|
||||||
@@ -0,0 +1,348 @@
|
|||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { createVoiceSession } from './session.js';
|
||||||
|
import { realtimeSessionEvent } from './realtimeEvents.js';
|
||||||
|
import { createVoiceCommands } from './sessionCommands.js';
|
||||||
|
|
||||||
|
function deferred() {
|
||||||
|
let resolve;
|
||||||
|
const promise = new Promise((done) => {
|
||||||
|
resolve = done;
|
||||||
|
});
|
||||||
|
return { promise, resolve };
|
||||||
|
}
|
||||||
|
|
||||||
|
function fixture(
|
||||||
|
runner = async (name, args) => ({ ok: true, name, ...args }),
|
||||||
|
signal,
|
||||||
|
) {
|
||||||
|
let hooks;
|
||||||
|
const sent = [];
|
||||||
|
let stops = 0;
|
||||||
|
const session = createVoiceSession({
|
||||||
|
runner,
|
||||||
|
signal,
|
||||||
|
createAdapter(options) {
|
||||||
|
hooks = options;
|
||||||
|
return {
|
||||||
|
start() {
|
||||||
|
hooks.emit({ type: 'state', state: 'listening' });
|
||||||
|
},
|
||||||
|
stop() {
|
||||||
|
stops++;
|
||||||
|
hooks.emit({ type: 'state', state: 'idle' });
|
||||||
|
},
|
||||||
|
sendText(text) {
|
||||||
|
sent.push(['text', text]);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
sendMapEvent(event) {
|
||||||
|
sent.push(['map', event]);
|
||||||
|
return true;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
session,
|
||||||
|
hooks,
|
||||||
|
sent,
|
||||||
|
get stops() {
|
||||||
|
return stops;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('alternate protocol shares session events and action execution without Realtime', async () => {
|
||||||
|
const f = fixture();
|
||||||
|
const events = [];
|
||||||
|
f.session.subscribe(() => {
|
||||||
|
throw new Error('observer');
|
||||||
|
});
|
||||||
|
const unsubscribe = f.session.subscribe((event) => events.push(event));
|
||||||
|
await f.session.start();
|
||||||
|
assert.equal(f.session.isActive(), true);
|
||||||
|
f.hooks.emit({
|
||||||
|
type: 'transcript',
|
||||||
|
role: 'user',
|
||||||
|
text: 'London',
|
||||||
|
final: true,
|
||||||
|
});
|
||||||
|
assert.deepEqual(
|
||||||
|
await f.hooks.runAction('fly_to_location', { query: 'London' }),
|
||||||
|
{ ok: true, name: 'fly_to_location', query: 'London' },
|
||||||
|
);
|
||||||
|
f.hooks.emit({ type: 'completion', status: 'completed' });
|
||||||
|
f.session.sendText('Next');
|
||||||
|
f.session.sendMapEvent({
|
||||||
|
type: 'map_annotation_outline',
|
||||||
|
status: 'resolved',
|
||||||
|
});
|
||||||
|
assert.deepEqual(
|
||||||
|
events.map((event) => event.type),
|
||||||
|
[
|
||||||
|
'state',
|
||||||
|
'state',
|
||||||
|
'transcript',
|
||||||
|
'action-call',
|
||||||
|
'action-result',
|
||||||
|
'completion',
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert.equal(f.sent.length, 2);
|
||||||
|
unsubscribe();
|
||||||
|
f.session.stop();
|
||||||
|
assert.equal(f.session.state, 'idle');
|
||||||
|
await f.session.start();
|
||||||
|
assert.equal(f.session.state, 'listening');
|
||||||
|
f.session.destroy();
|
||||||
|
assert.equal(f.session.disposed, true);
|
||||||
|
assert.equal(f.session.sendText('stale'), false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('stop, interruption and lifetime abort cancel actions and suppress stale results', async () => {
|
||||||
|
for (const cancel of ['stop', 'interruption', 'lifetime']) {
|
||||||
|
const result = deferred();
|
||||||
|
const lifetime = new AbortController();
|
||||||
|
let observed;
|
||||||
|
const f = fixture(async (_name, _args, options) => {
|
||||||
|
observed = options;
|
||||||
|
return result.promise;
|
||||||
|
}, lifetime.signal);
|
||||||
|
const events = [];
|
||||||
|
f.session.subscribe((event) => events.push(event));
|
||||||
|
await f.session.start();
|
||||||
|
const pending = f.hooks.runAction('fly_to_location', {});
|
||||||
|
if (cancel === 'stop') f.session.stop();
|
||||||
|
if (cancel === 'interruption')
|
||||||
|
f.hooks.emit({ type: 'interruption', reason: 'user-speech' });
|
||||||
|
if (cancel === 'lifetime') lifetime.abort();
|
||||||
|
assert.equal(observed.signal.aborted, true);
|
||||||
|
assert.equal(observed.isCurrent(), false);
|
||||||
|
result.resolve({ ok: true });
|
||||||
|
await assert.rejects(pending, { name: 'AbortError' });
|
||||||
|
assert.equal(
|
||||||
|
events.some((event) => event.type === 'action-result'),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
f.session.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('caller cancellation and sibling actions preserve independent ownership', async () => {
|
||||||
|
const first = deferred();
|
||||||
|
const f = fixture(async (name) =>
|
||||||
|
name === 'first' ? first.promise : { ok: true },
|
||||||
|
);
|
||||||
|
await f.session.start();
|
||||||
|
const signal = new AbortController();
|
||||||
|
const pending = f.hooks.runAction('first', {}, { signal: signal.signal });
|
||||||
|
signal.abort();
|
||||||
|
assert.deepEqual(await f.hooks.runAction('second', {}), { ok: true });
|
||||||
|
first.resolve({ ok: true });
|
||||||
|
await assert.rejects(pending, { name: 'AbortError' });
|
||||||
|
f.session.destroy();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('stale connect failure cannot stop a replacement session', async () => {
|
||||||
|
let reject;
|
||||||
|
let calls = 0;
|
||||||
|
let hooks;
|
||||||
|
const session = createVoiceSession({
|
||||||
|
runner: async () => {},
|
||||||
|
createAdapter(options) {
|
||||||
|
hooks = options;
|
||||||
|
return {
|
||||||
|
start() {
|
||||||
|
if (++calls === 1)
|
||||||
|
return new Promise((_resolve, fail) => {
|
||||||
|
reject = fail;
|
||||||
|
});
|
||||||
|
hooks.emit({ type: 'state', state: 'listening' });
|
||||||
|
},
|
||||||
|
stop() {},
|
||||||
|
sendText() {},
|
||||||
|
sendMapEvent() {},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const old = session.start();
|
||||||
|
session.stop();
|
||||||
|
await session.start();
|
||||||
|
reject(new Error('old connection failed'));
|
||||||
|
await old;
|
||||||
|
assert.equal(session.state, 'listening');
|
||||||
|
session.destroy();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('wire events project only normalized transcript, interruption and completion fields', () => {
|
||||||
|
assert.deepEqual(
|
||||||
|
realtimeSessionEvent({
|
||||||
|
type: 'response.output_audio_transcript.delta',
|
||||||
|
delta: 'Hello',
|
||||||
|
item_id: 'i',
|
||||||
|
response_id: 'r',
|
||||||
|
extra: 'ignored',
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
type: 'transcript',
|
||||||
|
role: 'assistant',
|
||||||
|
text: 'Hello',
|
||||||
|
final: false,
|
||||||
|
itemId: 'i',
|
||||||
|
responseId: 'r',
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.deepEqual(
|
||||||
|
realtimeSessionEvent({
|
||||||
|
type: 'conversation.item.input_audio_transcription.completed',
|
||||||
|
transcript: 'Go',
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
type: 'transcript',
|
||||||
|
role: 'user',
|
||||||
|
text: 'Go',
|
||||||
|
final: true,
|
||||||
|
itemId: null,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
realtimeSessionEvent({ type: 'response.output_text.done', text: 'Done' })
|
||||||
|
.final,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
realtimeSessionEvent({ type: 'input_audio_buffer.speech_started' }).type,
|
||||||
|
'interruption',
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
realtimeSessionEvent({
|
||||||
|
type: 'response.done',
|
||||||
|
response: { id: 'r', status: 'cancelled' },
|
||||||
|
}).status,
|
||||||
|
'cancelled',
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
realtimeSessionEvent({
|
||||||
|
type: 'session.created',
|
||||||
|
session: { secret: 'excluded' },
|
||||||
|
}),
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('common button and annotation bindings work with an alternate adapter and clean up', async () => {
|
||||||
|
const previous = globalThis.window;
|
||||||
|
globalThis.window = {};
|
||||||
|
try {
|
||||||
|
const button = new EventTarget();
|
||||||
|
button.setAttribute = () => {};
|
||||||
|
let removed = 0;
|
||||||
|
let subscribed = null;
|
||||||
|
let calls = 0;
|
||||||
|
const events = [];
|
||||||
|
const ui = {
|
||||||
|
button,
|
||||||
|
root: {
|
||||||
|
dataset: {},
|
||||||
|
remove() {
|
||||||
|
removed++;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
status: {},
|
||||||
|
detail: {},
|
||||||
|
tierButton: {},
|
||||||
|
costValue: {},
|
||||||
|
helpDetail: {},
|
||||||
|
};
|
||||||
|
const lifetime = new AbortController();
|
||||||
|
const controls = createVoiceCommands({
|
||||||
|
runner: async () => {
|
||||||
|
calls++;
|
||||||
|
return { ok: true };
|
||||||
|
},
|
||||||
|
signal: lifetime.signal,
|
||||||
|
createControl: () => ui,
|
||||||
|
annotations: {
|
||||||
|
onOutlineEvent(listener) {
|
||||||
|
subscribed = listener;
|
||||||
|
return () => {
|
||||||
|
subscribed = null;
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
createSession({ emit, runAction }) {
|
||||||
|
return {
|
||||||
|
async start() {
|
||||||
|
emit({ type: 'state', state: 'listening', detail: 'Ready' });
|
||||||
|
await runAction('fly_to_location', { query: 'London' });
|
||||||
|
},
|
||||||
|
stop() {
|
||||||
|
emit({ type: 'state', state: 'idle' });
|
||||||
|
},
|
||||||
|
sendText() {},
|
||||||
|
sendMapEvent(event) {
|
||||||
|
events.push(event);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
button.dispatchEvent(new Event('click'));
|
||||||
|
await new Promise((done) => setTimeout(done, 0));
|
||||||
|
assert.equal(calls, 1);
|
||||||
|
assert.equal(ui.tierButton.hidden, true);
|
||||||
|
assert.equal(ui.costValue.hidden, true);
|
||||||
|
assert.equal(ui.helpDetail.textContent, 'Activate to toggle voice');
|
||||||
|
assert.equal(ui.status.textContent, 'LISTENING');
|
||||||
|
assert.equal(ui.detail.textContent, 'Ready');
|
||||||
|
subscribed({ status: 'resolved' });
|
||||||
|
assert.deepEqual(events, [
|
||||||
|
{ type: 'map_annotation_outline', status: 'resolved' },
|
||||||
|
]);
|
||||||
|
button.dispatchEvent(new Event('click'));
|
||||||
|
assert.equal(controls.state, 'idle');
|
||||||
|
lifetime.abort();
|
||||||
|
assert.equal(subscribed, null);
|
||||||
|
assert.equal(removed, 1);
|
||||||
|
button.dispatchEvent(new Event('click'));
|
||||||
|
assert.equal(calls, 1);
|
||||||
|
} finally {
|
||||||
|
globalThis.window = previous;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('late actions cannot run after stop and failed startup closes the adapter', async () => {
|
||||||
|
let calls = 0;
|
||||||
|
const f = fixture(async () => {
|
||||||
|
calls++;
|
||||||
|
});
|
||||||
|
await f.session.start();
|
||||||
|
f.session.stop();
|
||||||
|
await assert.rejects(f.hooks.runAction('fly_to_location', {}), {
|
||||||
|
name: 'AbortError',
|
||||||
|
});
|
||||||
|
assert.equal(calls, 0);
|
||||||
|
f.session.destroy();
|
||||||
|
let stopped = 0;
|
||||||
|
const session = createVoiceSession({
|
||||||
|
runner: async () => {},
|
||||||
|
createAdapter({ emit }) {
|
||||||
|
return {
|
||||||
|
start() {
|
||||||
|
emit({ type: 'state', state: 'connecting' });
|
||||||
|
throw new Error('Connection refused');
|
||||||
|
},
|
||||||
|
stop() {
|
||||||
|
stopped++;
|
||||||
|
emit({ type: 'state', state: 'idle' });
|
||||||
|
},
|
||||||
|
sendText() {},
|
||||||
|
sendMapEvent() {},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
});
|
||||||
|
await session.start();
|
||||||
|
assert.equal(session.state, 'error');
|
||||||
|
assert.equal(stopped, 1);
|
||||||
|
session.destroy();
|
||||||
|
});
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
import { createVoiceControl } from './control.js';
|
||||||
|
import { createVoiceSession } from './session.js';
|
||||||
|
|
||||||
|
/** Bind common controls to a supplied voice-session adapter. */
|
||||||
|
export function createVoiceCommands({
|
||||||
|
runner,
|
||||||
|
dataManager,
|
||||||
|
annotations = null,
|
||||||
|
createSession,
|
||||||
|
createController,
|
||||||
|
backend,
|
||||||
|
signal,
|
||||||
|
debugSink,
|
||||||
|
createControl = createVoiceControl,
|
||||||
|
}) {
|
||||||
|
window.__gevVoiceCommands?.stop?.({ removeUi: true });
|
||||||
|
const ui = createControl({ reset: true });
|
||||||
|
const session = createVoiceSession({
|
||||||
|
runner,
|
||||||
|
signal,
|
||||||
|
createAdapter: (hooks) =>
|
||||||
|
createSession({
|
||||||
|
...hooks,
|
||||||
|
runner,
|
||||||
|
ui,
|
||||||
|
dataManager,
|
||||||
|
backend,
|
||||||
|
debugSink,
|
||||||
|
createController,
|
||||||
|
radioLayer: dataManager?.layers?.get('radio')?.module || null,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const adapter = session.adapter;
|
||||||
|
const capabilities = adapter.capabilities || {};
|
||||||
|
if (ui.tierButton) ui.tierButton.hidden = !capabilities.costControls;
|
||||||
|
if (ui.costValue) ui.costValue.hidden = !capabilities.costControls;
|
||||||
|
if (!capabilities.pushToTalk) {
|
||||||
|
ui.button.setAttribute('aria-label', 'Toggle voice control');
|
||||||
|
if (ui.helpDetail) ui.helpDetail.textContent = 'Activate to toggle voice';
|
||||||
|
}
|
||||||
|
// Retain the existing controller's inspection surface for browser tools.
|
||||||
|
const controls = adapter.controller || session;
|
||||||
|
controls.session = session;
|
||||||
|
const updateStatus = session.subscribe((event) => {
|
||||||
|
if (event.type !== 'state') return;
|
||||||
|
ui.root.dataset.status = event.state;
|
||||||
|
ui.status.textContent =
|
||||||
|
event.state === 'idle' ? 'OFF' : event.state.toUpperCase();
|
||||||
|
ui.detail.textContent =
|
||||||
|
event.detail || (event.state === 'idle' ? 'Voice off' : 'Voice active');
|
||||||
|
ui.button.setAttribute('aria-pressed', String(session.isActive()));
|
||||||
|
if (ui.errorDetail)
|
||||||
|
ui.errorDetail.textContent =
|
||||||
|
event.state === 'error'
|
||||||
|
? event.detail || 'Voice could not be started.'
|
||||||
|
: '';
|
||||||
|
if (event.state === 'error') ui.root.classList?.remove('error-dismissed');
|
||||||
|
});
|
||||||
|
const annotationUnsubscribe = annotations?.onOutlineEvent?.((event) => {
|
||||||
|
session.sendMapEvent({ type: 'map_annotation_outline', ...event });
|
||||||
|
});
|
||||||
|
const buttonHandler = () => {
|
||||||
|
if (adapter.ignoreButtonClick?.()) return;
|
||||||
|
if (session.isActive()) session.stop();
|
||||||
|
else void session.start({ pushToTalk: false });
|
||||||
|
};
|
||||||
|
ui.button.addEventListener('click', buttonHandler);
|
||||||
|
session.signal.addEventListener(
|
||||||
|
'abort',
|
||||||
|
() => {
|
||||||
|
ui.button.removeEventListener('click', buttonHandler);
|
||||||
|
annotationUnsubscribe?.();
|
||||||
|
updateStatus();
|
||||||
|
ui.root.remove();
|
||||||
|
},
|
||||||
|
{ once: true },
|
||||||
|
);
|
||||||
|
if (session.disposed) {
|
||||||
|
ui.button.removeEventListener('click', buttonHandler);
|
||||||
|
annotationUnsubscribe?.();
|
||||||
|
updateStatus();
|
||||||
|
ui.root.remove();
|
||||||
|
} else adapter.bindControls?.();
|
||||||
|
window.__gevVoiceCommands = controls;
|
||||||
|
return controls;
|
||||||
|
}
|
||||||
Reference in new issue
Block a user