mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 16:22:20 +00:00
createVoiceControl now returns a presenter instead of building the control with innerHTML: the session, cost, input, card and panel owners write its state by part and listen for interactions, and VoiceControl.vue renders it in the command dock, between the location bar and the visual presets as before. Focus moves wait for the render; the waveform's bars publish apart from the control and only when they change; the minutes field keeps a draft until its owner writes it. Owner tests use a real presenter on their own store, and view tests render the control.
255 lines
12 KiB
JavaScript
255 lines
12 KiB
JavaScript
import assert from 'node:assert/strict';
|
|
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
|
|
import test from 'node:test';
|
|
import { GevRealtimeController } from './realtimeController.js';
|
|
import { PROGRESS_MAX_OUTPUT_TOKENS } from './realtimeTurns.js';
|
|
|
|
function controllerFixture({ runner = async () => ({ ok: true }) } = {}) {
|
|
const sent = [];
|
|
const statuses = [];
|
|
const usage = [];
|
|
const controller = new GevRealtimeController({
|
|
ui: createTestVoiceUi(),
|
|
runner,
|
|
debugSink: null,
|
|
});
|
|
const setStatus = controller.setStatus.bind(controller);
|
|
controller.setStatus = (status, detail) => {
|
|
statuses.push(status);
|
|
return setStatus(status, detail);
|
|
};
|
|
const recordUsage = controller.recordUsage.bind(controller);
|
|
controller.recordUsage = (value) => {
|
|
if (value) usage.push(value);
|
|
return recordUsage(value);
|
|
};
|
|
controller.dc = {
|
|
readyState: 'open',
|
|
send: (body) => sent.push(JSON.parse(body)),
|
|
close() {},
|
|
};
|
|
const event = (payload) =>
|
|
controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
|
|
// Server errors carry their own event_id; the client's id is in error.event_id.
|
|
let serverSeq = 0;
|
|
const rejection = (clientEventId, code = 'invalid_request_error') =>
|
|
event({
|
|
type: 'error',
|
|
event_id: `event_server_${++serverSeq}`,
|
|
error: { type: 'invalid_request_error', code, message: 'rejected', event_id: clientEventId },
|
|
});
|
|
const progressLine = async (id = 'resp_p', line = 'Heading to Austin.') => {
|
|
assert.equal(controller._turns.speakProgressLine(line), true);
|
|
const request = sent.at(-1);
|
|
await event({ type: 'response.created', response: { id, metadata: { gev: 'progress' } } });
|
|
return request;
|
|
};
|
|
return { controller, sent, statuses, usage, event, rejection, progressLine };
|
|
}
|
|
|
|
const types = (sent) => sent.map((message) => message.type);
|
|
|
|
test('Space barge-in cancels the reply and clears audio; WebRTC truncates, so no manual truncate', async () => {
|
|
const { controller, sent, statuses, event, rejection } = controllerFixture();
|
|
await event({ type: 'response.created', response: { id: 'resp_1' } });
|
|
await event({ type: 'output_audio_buffer.started', response_id: 'resp_1' });
|
|
controller.pendingResponseInstructions = 'queued follow-up';
|
|
|
|
assert.equal(controller._turns.bargeIn(), true);
|
|
assert.deepEqual(types(sent), ['response.cancel', 'output_audio_buffer.clear']);
|
|
assert.equal(sent[0].response_id, 'resp_1');
|
|
assert.equal(controller.pendingResponseInstructions, null, 'no queued follow-up talks over the user');
|
|
assert.equal(controller.isSupersededResponse('resp_1'), true, 'late calls from the cut reply are refused');
|
|
|
|
// The target may already have finished. The rejection names the client id
|
|
// in error.event_id while the envelope has a different server id.
|
|
await rejection(sent[0].event_id, 'response_cancel_not_active');
|
|
await rejection(sent[1].event_id);
|
|
assert.equal(statuses.includes('error'), false);
|
|
});
|
|
|
|
test('an unrelated server error with a colliding top-level id is still fatal', async () => {
|
|
const { controller, sent, statuses, event } = controllerFixture();
|
|
await event({ type: 'response.created', response: { id: 'resp_1' } });
|
|
controller._turns.bargeIn();
|
|
await event({ type: 'error', event_id: sent[0].event_id, error: { code: 'server_error', message: 'boom' } });
|
|
assert.equal(statuses.includes('error'), true, 'only error.event_id correlates a client event');
|
|
});
|
|
|
|
test('a rejected progress line releases the waiting reply without an error state', async () => {
|
|
const { controller, sent, statuses, rejection } = controllerFixture();
|
|
controller._turns.speakProgressLine('Heading to Austin.');
|
|
controller.queueResponseCreate('Speak the say line.');
|
|
assert.equal(sent.length, 1, 'the follow-up waits behind the line');
|
|
await rejection(sent[0].event_id);
|
|
assert.equal(statuses.includes('error'), false);
|
|
assert.equal(sent.at(-1).type, 'response.create');
|
|
assert.equal(sent.at(-1).response.instructions, 'Speak the say line.');
|
|
});
|
|
|
|
test('progress requests are out of band, bounded, and carry the line only as data', async () => {
|
|
const { controller, sent } = controllerFixture();
|
|
const line = 'Finding Texas State Capitol on the map.';
|
|
assert.equal(controller._turns.speakProgressLine(line), true);
|
|
const { response } = sent[0];
|
|
assert.equal(response.conversation, 'none');
|
|
assert.equal(response.tool_choice, 'none');
|
|
assert.equal(response.max_output_tokens, PROGRESS_MAX_OUTPUT_TOKENS);
|
|
assert.ok(PROGRESS_MAX_OUTPUT_TOKENS <= 100);
|
|
assert.deepEqual(response.metadata, { gev: 'progress' });
|
|
assert.equal(response.input[0].content[0].text, line);
|
|
assert.equal(response.instructions.includes('Capitol'), false, 'labels never enter instructions');
|
|
assert.equal(controller._turns.speakProgressLine('Another line.'), false, 'one line at a time');
|
|
});
|
|
|
|
test('a cancelled progress response never clears a pending Radio handoff', async () => {
|
|
const { controller, sent, event, progressLine, usage } = controllerFixture();
|
|
await progressLine('resp_p');
|
|
// Radio prepared its playback while the line was generating.
|
|
controller.pendingRadioPlaybackResult = { ok: true, radioPlaybackRequested: true };
|
|
controller.queueResponseCreate('Say "Turning on the radio."');
|
|
controller._turns.retractProgressLine();
|
|
await event({
|
|
type: 'response.done',
|
|
response: { id: 'resp_p', status: 'cancelled', metadata: { gev: 'progress' }, usage: { input_tokens: 5, output_tokens: 3 } },
|
|
});
|
|
assert.ok(controller.pendingRadioPlaybackResult, 'Radio stays pending');
|
|
assert.equal(sent.at(-1).type, 'response.create');
|
|
assert.match(sent.at(-1).response.instructions, /Turning on the radio/, 'the confirmation is not stranded');
|
|
assert.equal(usage.length, 1, 'progress usage is still billed');
|
|
});
|
|
|
|
test('an old progress completion leaves the active in-band response alone', async () => {
|
|
const { controller, event, progressLine } = controllerFixture();
|
|
await progressLine('resp_p');
|
|
// Server VAD starts a real reply while the line is still generating.
|
|
await event({ type: 'response.created', response: { id: 'resp_real' } });
|
|
await event({ type: 'response.output_audio_transcript.done', response_id: 'resp_p', transcript: 'Heading to Austin.' });
|
|
await event({ type: 'response.done', response: { id: 'resp_p', status: 'completed', metadata: { gev: 'progress' } } });
|
|
assert.equal(controller.activeResponseId, 'resp_real');
|
|
assert.equal(controller.responseActive, true);
|
|
});
|
|
|
|
test('user speech stops an audible progress line: cancel and clear its audio', async () => {
|
|
const { sent, event, progressLine } = controllerFixture();
|
|
await progressLine('resp_p');
|
|
await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' });
|
|
sent.length = 0;
|
|
await event({ type: 'input_audio_buffer.speech_started' });
|
|
assert.deepEqual(types(sent), ['response.cancel', 'output_audio_buffer.clear']);
|
|
assert.equal(sent[0].response_id, 'resp_p');
|
|
});
|
|
|
|
test('a typed command stops an audible progress line too', async () => {
|
|
const { controller, sent, event, progressLine } = controllerFixture();
|
|
await progressLine('resp_p');
|
|
await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' });
|
|
await event({ type: 'response.done', response: { id: 'resp_p', status: 'completed', metadata: { gev: 'progress' } } });
|
|
sent.length = 0;
|
|
controller.sendTextCommand('go to Paris');
|
|
assert.equal(types(sent)[0], 'output_audio_buffer.clear', 'playback still owned after generation ended');
|
|
});
|
|
|
|
test('a finished tool lets a heard line finish, and playback ownership outlives response.done', async () => {
|
|
const { controller, sent, event, progressLine } = controllerFixture();
|
|
await progressLine('resp_p');
|
|
await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' });
|
|
controller._turns.retractProgressLine();
|
|
assert.equal(types(sent).includes('response.cancel'), false, 'audible lines are not cut');
|
|
controller.queueResponseCreate('Speak the say line.');
|
|
assert.equal(types(sent).filter((t) => t === 'response.create').length, 1, 'follow-up waits for generation');
|
|
await event({ type: 'response.done', response: { id: 'resp_p', status: 'completed', metadata: { gev: 'progress' } } });
|
|
assert.equal(types(sent).filter((t) => t === 'response.create').length, 2, 'follow-up after the line');
|
|
assert.equal(controller._turns.canNarrate(), false, 'still playing: no new line');
|
|
await event({ type: 'output_audio_buffer.stopped', response_id: 'resp_p' });
|
|
assert.equal(controller._turns.progressResponse, null);
|
|
});
|
|
|
|
test('an unheard line is cancelled when its tool finishes', async () => {
|
|
const { controller, sent, progressLine } = controllerFixture();
|
|
await progressLine('resp_p');
|
|
controller._turns.retractProgressLine();
|
|
assert.equal(sent.at(-1).type, 'response.cancel');
|
|
assert.equal(sent.at(-1).response_id, 'resp_p');
|
|
});
|
|
|
|
test('Radio handoff phases block barge-in and narration: prepared, reserved and starting playback', async () => {
|
|
const { controller, sent, event } = controllerFixture();
|
|
await event({ type: 'response.created', response: { id: 'resp_r' } });
|
|
await event({ type: 'response.done', response: { id: 'resp_r', status: 'completed' } });
|
|
await event({ type: 'output_audio_buffer.started', response_id: 'resp_r' });
|
|
const radio = controller._radio;
|
|
const phases = [
|
|
() => { radio.pendingRadioPlaybackResult = { ok: true }; },
|
|
() => { radio.radioVisibilityOffPending = true; },
|
|
() => { radio.radioHandoffInFlight = true; },
|
|
];
|
|
for (const enter of phases) {
|
|
radio.pendingRadioPlaybackResult = null;
|
|
radio.radioVisibilityOffPending = false;
|
|
radio.radioHandoffInFlight = false;
|
|
enter();
|
|
assert.equal(radio.mayVoiceClaimSpeaker(), false);
|
|
assert.equal(controller._turns.bargeIn(), false);
|
|
assert.equal(controller._turns.canNarrate(), false);
|
|
}
|
|
assert.deepEqual(sent, []);
|
|
});
|
|
|
|
test('caption transcription is metered with the served transcription model', async () => {
|
|
const { controller, event } = controllerFixture();
|
|
controller._cost.bindTranscriptionModel('gpt-4o-mini-transcribe');
|
|
const before = controller.costTracker.state().totalUsd;
|
|
await event({
|
|
type: 'conversation.item.input_audio_transcription.completed',
|
|
item_id: 'item_u',
|
|
transcript: 'take me to Tokyo',
|
|
usage: { type: 'tokens', input_tokens: 1000, input_token_details: { audio_tokens: 1000 }, output_tokens: 10 },
|
|
});
|
|
const after = controller.costTracker.state().totalUsd;
|
|
assert.ok(Math.abs(after - before - (1000 * 3 + 10 * 5) / 1_000_000) < 1e-12);
|
|
});
|
|
|
|
test('tools report start, progress and finish to narration; Radio stays silent', async () => {
|
|
const calls = [];
|
|
const { controller, event } = controllerFixture({
|
|
runner: async (name, args, options) => {
|
|
options.progress?.({ step: 'resolve', label: 'Zilker Park' });
|
|
return { ok: true, action: name };
|
|
},
|
|
});
|
|
controller._turns.narration = {
|
|
toolStarted: (id, info) => calls.push(['start', id, info.name, info.narrate, info.label]),
|
|
progress: (id, update) => calls.push(['progress', id, update.step]),
|
|
toolFinished: (id) => calls.push(['finish', id]),
|
|
preamble: () => calls.push(['preamble']),
|
|
userTurnEnded() {},
|
|
cancel() {},
|
|
audible() {},
|
|
quiet() {},
|
|
};
|
|
await event({ type: 'response.output_audio_transcript.delta', response_id: 'resp_t', delta: 'Finding ' });
|
|
await event({
|
|
type: 'response.function_call_arguments.done',
|
|
response_id: 'resp_t',
|
|
call_id: 'call_1',
|
|
name: 'annotate_map',
|
|
arguments: JSON.stringify({ annotations: [{ target: 'Zilker Park' }] }),
|
|
});
|
|
assert.deepEqual(calls, [
|
|
['preamble'],
|
|
['start', 'call_1', 'annotate_map', true, 'Zilker Park'],
|
|
['progress', 'call_1', 'resolve'],
|
|
['finish', 'call_1'],
|
|
]);
|
|
calls.length = 0;
|
|
await event({
|
|
type: 'response.function_call_arguments.done',
|
|
response_id: 'resp_r',
|
|
call_id: 'call_2',
|
|
name: 'control_radio',
|
|
arguments: JSON.stringify({ action: 'volume', volumePct: 20 }),
|
|
});
|
|
assert.equal(calls.find((call) => call[0] === 'start')?.[3], false);
|
|
});
|