import assert from 'node:assert/strict'; import test from 'node:test'; import { GevRealtimeController } from './realtimeController.js'; import { PROGRESS_MAX_OUTPUT_TOKENS } from './realtimeTurns.js'; function controllerFixture({ runner = async () => ({ ok: true }) } = {}) { const sent = []; const statuses = []; const usage = []; const controller = new GevRealtimeController({ ui: { root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] }, status: { textContent: '' }, detail: { textContent: '', title: '' }, errorDetail: { textContent: '' }, }, runner, debugSink: null, }); const setStatus = controller.setStatus.bind(controller); controller.setStatus = (status, detail) => { statuses.push(status); return setStatus(status, detail); }; const recordUsage = controller.recordUsage.bind(controller); controller.recordUsage = (value) => { if (value) usage.push(value); return recordUsage(value); }; controller.dc = { readyState: 'open', send: (body) => sent.push(JSON.parse(body)), close() {}, }; const event = (payload) => controller.handleRealtimeEvent({ data: JSON.stringify(payload) }); // Server errors carry their own event_id; the client's id is in error.event_id. let serverSeq = 0; const rejection = (clientEventId, code = 'invalid_request_error') => event({ type: 'error', event_id: `event_server_${++serverSeq}`, error: { type: 'invalid_request_error', code, message: 'rejected', event_id: clientEventId }, }); const progressLine = async (id = 'resp_p', line = 'Heading to Austin.') => { assert.equal(controller._turns.speakProgressLine(line), true); const request = sent.at(-1); await event({ type: 'response.created', response: { id, metadata: { gev: 'progress' } } }); return request; }; return { controller, sent, statuses, usage, event, rejection, progressLine }; } const types = (sent) => sent.map((message) => message.type); test('Space barge-in cancels the reply and clears audio; WebRTC truncates, so no manual truncate', async () => { const { controller, sent, statuses, event, rejection } = controllerFixture(); await event({ type: 'response.created', response: { id: 'resp_1' } }); await event({ type: 'output_audio_buffer.started', response_id: 'resp_1' }); controller.pendingResponseInstructions = 'queued follow-up'; assert.equal(controller._turns.bargeIn(), true); assert.deepEqual(types(sent), ['response.cancel', 'output_audio_buffer.clear']); assert.equal(sent[0].response_id, 'resp_1'); assert.equal(controller.pendingResponseInstructions, null, 'no queued follow-up talks over the user'); assert.equal(controller.isSupersededResponse('resp_1'), true, 'late calls from the cut reply are refused'); // The target may already have finished. The rejection names the client id // in error.event_id while the envelope has a different server id. await rejection(sent[0].event_id, 'response_cancel_not_active'); await rejection(sent[1].event_id); assert.equal(statuses.includes('error'), false); }); test('an unrelated server error with a colliding top-level id is still fatal', async () => { const { controller, sent, statuses, event } = controllerFixture(); await event({ type: 'response.created', response: { id: 'resp_1' } }); controller._turns.bargeIn(); await event({ type: 'error', event_id: sent[0].event_id, error: { code: 'server_error', message: 'boom' } }); assert.equal(statuses.includes('error'), true, 'only error.event_id correlates a client event'); }); test('a rejected progress line releases the waiting reply without an error state', async () => { const { controller, sent, statuses, rejection } = controllerFixture(); controller._turns.speakProgressLine('Heading to Austin.'); controller.queueResponseCreate('Speak the say line.'); assert.equal(sent.length, 1, 'the follow-up waits behind the line'); await rejection(sent[0].event_id); assert.equal(statuses.includes('error'), false); assert.equal(sent.at(-1).type, 'response.create'); assert.equal(sent.at(-1).response.instructions, 'Speak the say line.'); }); test('progress requests are out of band, bounded, and carry the line only as data', async () => { const { controller, sent } = controllerFixture(); const line = 'Finding Texas State Capitol on the map.'; assert.equal(controller._turns.speakProgressLine(line), true); const { response } = sent[0]; assert.equal(response.conversation, 'none'); assert.equal(response.tool_choice, 'none'); assert.equal(response.max_output_tokens, PROGRESS_MAX_OUTPUT_TOKENS); assert.ok(PROGRESS_MAX_OUTPUT_TOKENS <= 100); assert.deepEqual(response.metadata, { gev: 'progress' }); assert.equal(response.input[0].content[0].text, line); assert.equal(response.instructions.includes('Capitol'), false, 'labels never enter instructions'); assert.equal(controller._turns.speakProgressLine('Another line.'), false, 'one line at a time'); }); test('a cancelled progress response never clears a pending Radio handoff', async () => { const { controller, sent, event, progressLine, usage } = controllerFixture(); await progressLine('resp_p'); // Radio prepared its playback while the line was generating. controller.pendingRadioPlaybackResult = { ok: true, radioPlaybackRequested: true }; controller.queueResponseCreate('Say "Turning on the radio."'); controller._turns.retractProgressLine(); await event({ type: 'response.done', response: { id: 'resp_p', status: 'cancelled', metadata: { gev: 'progress' }, usage: { input_tokens: 5, output_tokens: 3 } }, }); assert.ok(controller.pendingRadioPlaybackResult, 'Radio stays pending'); assert.equal(sent.at(-1).type, 'response.create'); assert.match(sent.at(-1).response.instructions, /Turning on the radio/, 'the confirmation is not stranded'); assert.equal(usage.length, 1, 'progress usage is still billed'); }); test('an old progress completion leaves the active in-band response alone', async () => { const { controller, event, progressLine } = controllerFixture(); await progressLine('resp_p'); // Server VAD starts a real reply while the line is still generating. await event({ type: 'response.created', response: { id: 'resp_real' } }); await event({ type: 'response.output_audio_transcript.done', response_id: 'resp_p', transcript: 'Heading to Austin.' }); await event({ type: 'response.done', response: { id: 'resp_p', status: 'completed', metadata: { gev: 'progress' } } }); assert.equal(controller.activeResponseId, 'resp_real'); assert.equal(controller.responseActive, true); }); test('user speech stops an audible progress line: cancel and clear its audio', async () => { const { sent, event, progressLine } = controllerFixture(); await progressLine('resp_p'); await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' }); sent.length = 0; await event({ type: 'input_audio_buffer.speech_started' }); assert.deepEqual(types(sent), ['response.cancel', 'output_audio_buffer.clear']); assert.equal(sent[0].response_id, 'resp_p'); }); test('a typed command stops an audible progress line too', async () => { const { controller, sent, event, progressLine } = controllerFixture(); await progressLine('resp_p'); await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' }); await event({ type: 'response.done', response: { id: 'resp_p', status: 'completed', metadata: { gev: 'progress' } } }); sent.length = 0; controller.sendTextCommand('go to Paris'); assert.equal(types(sent)[0], 'output_audio_buffer.clear', 'playback still owned after generation ended'); }); test('a finished tool lets a heard line finish, and playback ownership outlives response.done', async () => { const { controller, sent, event, progressLine } = controllerFixture(); await progressLine('resp_p'); await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' }); controller._turns.retractProgressLine(); assert.equal(types(sent).includes('response.cancel'), false, 'audible lines are not cut'); controller.queueResponseCreate('Speak the say line.'); assert.equal(types(sent).filter((t) => t === 'response.create').length, 1, 'follow-up waits for generation'); await event({ type: 'response.done', response: { id: 'resp_p', status: 'completed', metadata: { gev: 'progress' } } }); assert.equal(types(sent).filter((t) => t === 'response.create').length, 2, 'follow-up after the line'); assert.equal(controller._turns.canNarrate(), false, 'still playing: no new line'); await event({ type: 'output_audio_buffer.stopped', response_id: 'resp_p' }); assert.equal(controller._turns.progressResponse, null); }); test('an unheard line is cancelled when its tool finishes', async () => { const { controller, sent, progressLine } = controllerFixture(); await progressLine('resp_p'); controller._turns.retractProgressLine(); assert.equal(sent.at(-1).type, 'response.cancel'); assert.equal(sent.at(-1).response_id, 'resp_p'); }); test('Radio handoff phases block barge-in and narration: prepared, reserved and starting playback', async () => { const { controller, sent, event } = controllerFixture(); await event({ type: 'response.created', response: { id: 'resp_r' } }); await event({ type: 'response.done', response: { id: 'resp_r', status: 'completed' } }); await event({ type: 'output_audio_buffer.started', response_id: 'resp_r' }); const radio = controller._radio; const phases = [ () => { radio.pendingRadioPlaybackResult = { ok: true }; }, () => { radio.radioVisibilityOffPending = true; }, () => { radio.radioHandoffInFlight = true; }, ]; for (const enter of phases) { radio.pendingRadioPlaybackResult = null; radio.radioVisibilityOffPending = false; radio.radioHandoffInFlight = false; enter(); assert.equal(radio.mayVoiceClaimSpeaker(), false); assert.equal(controller._turns.bargeIn(), false); assert.equal(controller._turns.canNarrate(), false); } assert.deepEqual(sent, []); }); test('caption transcription is metered with the served transcription model', async () => { const { controller, event } = controllerFixture(); controller._cost.bindTranscriptionModel('gpt-4o-mini-transcribe'); const before = controller.costTracker.state().totalUsd; await event({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'item_u', transcript: 'take me to Tokyo', usage: { type: 'tokens', input_tokens: 1000, input_token_details: { audio_tokens: 1000 }, output_tokens: 10 }, }); const after = controller.costTracker.state().totalUsd; assert.ok(Math.abs(after - before - (1000 * 3 + 10 * 5) / 1_000_000) < 1e-12); }); test('tools report start, progress and finish to narration; Radio stays silent', async () => { const calls = []; const { controller, event } = controllerFixture({ runner: async (name, args, options) => { options.progress?.({ step: 'resolve', label: 'Zilker Park' }); return { ok: true, action: name }; }, }); controller._turns.narration = { toolStarted: (id, info) => calls.push(['start', id, info.name, info.narrate, info.label]), progress: (id, update) => calls.push(['progress', id, update.step]), toolFinished: (id) => calls.push(['finish', id]), preamble: () => calls.push(['preamble']), userTurnEnded() {}, cancel() {}, audible() {}, quiet() {}, }; await event({ type: 'response.output_audio_transcript.delta', response_id: 'resp_t', delta: 'Finding ' }); await event({ type: 'response.function_call_arguments.done', response_id: 'resp_t', call_id: 'call_1', name: 'annotate_map', arguments: JSON.stringify({ annotations: [{ target: 'Zilker Park' }] }), }); assert.deepEqual(calls, [ ['preamble'], ['start', 'call_1', 'annotate_map', true, 'Zilker Park'], ['progress', 'call_1', 'resolve'], ['finish', 'call_1'], ]); calls.length = 0; await event({ type: 'response.function_call_arguments.done', response_id: 'resp_r', call_id: 'call_2', name: 'control_radio', arguments: JSON.stringify({ action: 'volume', volumePct: 20 }), }); assert.equal(calls.find((call) => call[0] === 'start')?.[3], false); });