import test from 'node:test'; import assert from 'node:assert/strict'; import { geminiSocketUrl } from './geminiSession.js'; import { createVoiceSession } from './session.js'; import { createPointerTracker } from './pointerContext.js'; import { withToolCatalog } from './gevRealtime.js'; import { harness, tick, deferred, } from '../testSupport/geminiNativeHarness.mjs'; const endpoint = 'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained'; const credential = { token: 'auth_tokens/test', model: 'gemini-3.8-live', websocketUrl: endpoint, config: { generationConfig: { responseModalities: ['AUDIO'] } }, }; const manualCredential = { ...credential, config: { ...credential.config, realtimeInputConfig: { automaticActivityDetection: { disabled: true } }, }, }; test('ephemeral credentials are sent only to the fixed constrained Google endpoint', () => { const url = new URL(geminiSocketUrl(credential)); assert.equal(url.searchParams.get('access_token'), credential.token); for (const websocketUrl of [ endpoint + '?key=foo', endpoint.replace('googleapis.com', 'example.com'), endpoint.replace('wss:', 'ws:'), endpoint.replace('Constrained', ''), endpoint.replace('wss://', 'wss://user@'), ]) assert.throws( () => geminiSocketUrl({ ...credential, websocketUrl }), /endpoint/, ); assert.throws( () => geminiSocketUrl({ ...credential, token: 'AIza-not-an-ephemeral-token' }), /ephemeral/, ); }); test('PTT locks manual activity and serializes cold and repeated held turns', async () => { const h = harness(); h.c.input.pushToTalkKeyHeld = true; h.c.input.spaceKeyHeld = true; await h.c.start({ pushToTalk: true }); assert.equal(h.requests[0].body, '{"inputMode":"push-to-talk"}'); const ws = h.sockets[0]; assert.deepEqual(ws.sent[0].setup.realtimeInputConfig, { automaticActivityDetection: { disabled: true }, }); const audio = h.captures[0]; const pcm = { data: 'AAABAA==', mimeType: 'audio/pcm;rate=16000' }; audio.callbacks.onAudioStart(audio.metadata); // A duplicate notification cannot reopen. audio.callbacks.onAudio(pcm, audio.metadata); h.c.input.setMicrophoneEnabled(false); audio.callbacks.onAudioEnd(audio.metadata); audio.callbacks.onAudio(pcm, audio.metadata); // Unbracketed/stale PCM is not forwarded. assert.equal(h.tracks[0].enabled, false); h.c.input.setMicrophoneEnabled(true); audio.callbacks.onAudio(pcm, audio.metadata); h.c.input.setMicrophoneEnabled(false); assert.deepEqual(ws.sent.slice(1), [ { realtimeInput: { activityStart: {} } }, { realtimeInput: { audio: pcm } }, { realtimeInput: { activityEnd: {} } }, { realtimeInput: { activityStart: {} } }, { realtimeInput: { audio: pcm } }, { realtimeInput: { activityEnd: {} } }, ]); assert.equal(h.c.status, 'listening'); assert.equal(h.c.input.pushToTalkMode, true); assert.equal(h.tracks[0].enabled, false); h.c.stop(); }); test('ordinary microphone keeps its default setup and automatic stream-end message', async () => { const h = harness(); await h.c.start(); assert.equal(h.requests[0].body, '{}'); assert.deepEqual(h.sockets[0].sent, [ { setup: { ...credential.config, model: 'models/gemini-3.8-live' } }, ]); h.c.input.setMicrophoneEnabled(false); assert.deepEqual(h.sockets[0].sent.at(-1), { realtimeInput: { audioStreamEnd: true }, }); h.c.stop(); }); for (const phase of ['token', 'setup', 'microphone']) { test(`PTT release during ${phase} startup stays muted until a later hold`, async () => { const pending = deferred(); const h = harness({ ...(phase === 'token' ? { fetchImpl: () => pending.promise } : {}), ...(phase === 'setup' ? { autoSetup: false } : {}), ...(phase === 'microphone' ? { getUserMedia: () => pending.promise } : {}), }); h.c.input.pushToTalkKeyHeld = true; h.c.input.spaceKeyHeld = true; const start = h.c.start({ pushToTalk: true }); await tick(); assert.equal(h.media.length, 0); assert.equal( h.sockets[0]?.sent.filter((value) => value.realtimeInput).length || 0, 0, ); h.c.input.pushToTalkKeyHeld = false; h.c.input.spaceKeyHeld = false; h.c.input.setMicrophoneEnabled(false); if (phase === 'token') pending.resolve(Response.json(manualCredential)); else if (phase === 'setup') h.sockets[0].message({ setupComplete: {} }); else pending.resolve(h.makeStream()); await start; assert.equal(h.c.status, 'listening'); assert.equal(h.tracks[0].enabled, false); assert.equal(h.media[0].enabled, false); assert.equal( h.sockets[0].sent.length, 1, 'no phantom activity for a released gesture', ); h.c.input.pushToTalkKeyHeld = true; h.c.input.setMicrophoneEnabled(true); assert.deepEqual(h.sockets[0].sent.at(-1), { realtimeInput: { activityStart: {} }, }); h.c.input.setMicrophoneEnabled(false); assert.deepEqual(h.sockets[0].sent.at(-1), { realtimeInput: { activityEnd: {} }, }); h.c.stop(); }); } test('a token config that disagrees with the gesture fails before socket creation and releases consented microphone', async () => { for (const [pushToTalk, supplied] of [ [true, credential], [false, manualCredential], ]) { const h = harness({ fetchImpl: async () => Response.json(supplied), }); await h.c.start({ pushToTalk }); assert.equal(h.c.status, 'error'); assert.match(h.c.lastError, /input mode/); assert.equal(h.sockets.length, 0); assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(h.captures[0].closed, true); } }); test('old PTT callbacks cannot send to a stopped or replacement session', async () => { const h = harness(); h.c.input.pushToTalkKeyHeld = true; await h.c.start({ pushToTalk: true }); const old = h.captures[0].callbacks; const socket = h.sockets[0]; h.c.stop(); const oldCount = socket.sent.length; await h.c.start(); const replacementCount = h.sockets[1].sent.length; old.onAudioStart(h.captures[0].metadata); old.onAudio( { data: 'AAAA', mimeType: 'audio/pcm;rate=16000' }, h.captures[0].metadata, ); old.onAudioEnd(h.captures[0].metadata); assert.equal(socket.sent.length, oldCount); assert.equal(h.sockets[1].sent.length, replacementCount); assert.equal(h.c.status, 'listening'); assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(h.tracks[1].enabled, true); h.c.stop({ removeUi: true }); const latest = h.captures[1].callbacks; latest.onAudioStart(h.captures[1].metadata); latest.onAudioEnd(h.captures[1].metadata); assert.equal(h.sockets[1].sent.length, replacementCount); }); test('session waits for setup, captures PCM, plays all parts, emits transcripts and releases every resource', async () => { const h = harness(); await h.c.start(); assert.equal(h.c.status, 'listening'); assert.equal(h.media.length, 1); const ws = h.sockets[0]; assert.equal(ws.sent[0].setup.model, 'models/gemini-3.8-live'); h.captures[0].callbacks.onAudio( { data: 'AAAA', mimeType: 'audio/pcm;rate=16000', }, h.captures[0].metadata, ); assert.equal(ws.sent.at(-1).realtimeInput.audio.data, 'AAAA'); ws.message({ serverContent: { inputTranscription: { text: 'Hello' }, outputTranscription: { text: 'Hi' }, modelTurn: { parts: [ { inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } }, { inlineData: { data: 'AQAB', mimeType: 'audio/pcm;rate=24000' } }, ], }, turnComplete: true, }, }); assert.equal(h.captures[0].played.length, 2); assert.equal( h.events.some((event) => event.type === 'completion'), false, 'server turn completion waits for queued audio to drain', ); assert.ok( h.events.some( (event) => event.type === 'transcript' && event.role === 'assistant' && event.final && event.text === 'Hi', ), ); assert.equal(h.c.getDiagnostics().completedTurns, 1); h.captures[0].drain(); assert.equal( h.events.some( (event) => event.type === 'completion' && event.status === 'completed', ), true, ); h.c.stop(); assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(ws.closed, true); assert.equal(h.captures[0].closed, true); assert.equal(h.state.ducked, false); assert.equal(h.c.status, 'idle'); }); test('goAway deadline tears down pending work and an explicit restart creates a fresh conversation', async () => { const pending = deferred(); let issued = 0; let pendingOptions; const h = harness({ fetchImpl: async () => Response.json({ ...credential, token: `auth_tokens/session-${++issued}`, }), runAction: async (name, _args, options) => { if (name === 'control_radio') return { ok: true, action: 'control_radio', radioAction: 'play', radioPlaybackRequested: true, }; pendingOptions = options; return pending.promise; }, }); await h.c.start(); establishToolInput(h); const oldSocket = h.sockets[0]; const oldAudio = h.captures[0]; const oldCallbacks = oldAudio.callbacks; oldSocket.message({ toolCall: { functionCalls: [ { id: 'radio', name: 'control_radio', args: { action: 'play' } }, { id: 'pending', name: 'get_current_view_state', args: {} }, ], }, }); await tick(); await tick(); assert.ok(h.c.radio.pendingRadioPlaybackResult); assert.equal(h.c.pendingTools, 1); oldSocket.message({ serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, }, }); assert.equal(oldAudio.played.length, 1); let sentBeforeExpiry = oldSocket.sent.length; oldSocket.message({ goAway: { timeLeft: '5s' } }); sentBeforeExpiry = oldSocket.sent.length; // Microphone stream-end is a reliable lifecycle marker. assert.equal(h.c.status, 'executing'); assert.equal(pendingOptions.signal.aborted, false); assert.equal(h.c.sendTextCommand('Do not admit a new turn'), false); h.advance(5000); assert.equal(h.c.status, 'error'); assert.equal( h.c.lastError, 'Gemini Live connection is expiring. Start voice again for a new conversation.', ); assert.equal( h.events.at(-1).recovery, 'Start voice again for a fresh token and a new conversation.', ); assert.equal(oldSocket.closed, true); assert.equal(oldAudio.closed, true); assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(h.c.pendingTools, 0); assert.equal(h.c.radio.pendingRadioPlaybackResult, null); assert.equal(pendingOptions.signal.aborted, true); oldCallbacks.onAudioStart(h.captures[0].metadata); oldCallbacks.onAudio( { data: 'AQAB', mimeType: 'audio/pcm;rate=16000' }, h.captures[0].metadata, ); oldCallbacks.onAudioEnd(h.captures[0].metadata); oldCallbacks.onDrain(); oldCallbacks.onError(new Error('late retired audio')); oldSocket.message({ serverContent: { turnComplete: true } }); oldSocket.error(); oldSocket.remoteClose(); pending.resolve({ ok: true }); await tick(); await tick(); assert.equal(oldSocket.sent.length, sentBeforeExpiry); assert.equal(oldAudio.played.length, 1); assert.equal( oldSocket.sent.filter((entry) => entry.toolResponse).length, 1, 'only the completed Radio result was sent before expiry', ); assert.equal(h.state.playing, 0); await h.c.start(); assert.equal(issued, 2); assert.equal(h.sockets.length, 2); assert.equal( new URL(h.sockets[1].url).searchParams.get('access_token'), 'auth_tokens/session-2', ); assert.equal(h.c.status, 'listening'); assert.equal(h.c.completedTurns, 0); assert.equal(h.captures[1].played.length, 0); assert.equal(h.tracks[1].readyState, 'live'); h.c.stop(); }); test('pre-setup rejection or close releases startup and asks for a fresh session', async () => { for (const terminal of ['rejected', 'closed']) { const h = harness({ autoSetup: false }); const start = h.c.start(); await tick(); if (terminal === 'rejected') h.sockets[0].message({ error: { message: 'credential rejected' } }); else h.sockets[0].remoteClose(); await start; assert.equal(h.c.status, 'error', terminal); assert.equal(h.tracks[0].readyState, 'ended', terminal); assert.equal(h.captures[0].closed, true, terminal); assert.equal(h.sockets[0].closed, true, terminal); assert.match(h.c.lastError, /Start voice again/, terminal); assert.match(h.events.at(-1).recovery, /fresh session/, terminal); assert.equal(h.sockets.length, 1, 'no automatic retry'); } }); test('network close after setup releases media and starts no replacement', async () => { const h = harness(); await h.c.start(); h.sockets[0].remoteClose(); assert.equal(h.c.status, 'error'); assert.equal( h.c.lastError, 'Gemini Live connection ended. Start voice again for a new conversation.', ); assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(h.captures[0].closed, true); assert.equal(h.sockets.length, 1); assert.doesNotMatch(h.events.at(-1).recovery, /microphone|API key/i); }); test('OFF during unresolved setup blocks late setup, error and close callbacks', async () => { const h = harness({ autoSetup: false }); const start = h.c.start(); await tick(); const socket = h.sockets[0]; h.c.stop(); const eventCount = h.events.length; socket.message({ setupComplete: {} }); socket.message({ error: { message: 'late rejection' } }); socket.error(); socket.remoteClose(); await start; assert.equal(h.c.status, 'idle'); assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(h.captures[0].closed, true); assert.equal(h.events.length, eventCount); }); test('a disposed pending microphone is stopped and cannot update the replacement UI', async () => { const microphone = deferred(); const h = harness({ getUserMedia: () => microphone.promise }); const start = h.c.start(); await tick(); h.c.stop({ removeUi: true }); const eventCount = h.events.length; microphone.resolve(h.makeStream()); await start; assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(h.events.length, eventCount); assert.equal(h.media.length, 0); }); test('late token rejection after stop cannot replace a successor state', async () => { const response = deferred(); const h = harness({ fetchImpl: () => response.promise }); const start = h.c.start(); h.c.stop({ removeUi: true }); const count = h.events.length; response.resolve(Response.json({ error: 'late rejection' }, { status: 503 })); await start; assert.equal(h.events.length, count); assert.equal(h.sockets.length, 0); }); test('tool cancellation covers queued calls and interrupted results never return to the socket', async () => { const first = deferred(); const names = []; const h = harness({ runAction: async (name) => { names.push(name); return first.promise; }, }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; ws.message({ toolCall: { functionCalls: [ { id: 'a', name: 'one', args: {} }, { id: 'b', name: 'two', args: {} }, ], }, }); await tick(); ws.message({ toolCallCancellation: { ids: ['b'] } }); first.resolve({ ok: true }); await h.c.toolQueue; assert.deepEqual(names, ['one']); assert.equal(h.c.pendingTools, 0); assert.equal(ws.sent.filter((entry) => entry.toolResponse).length, 1); const second = deferred(); h.c.runAction = () => second.promise; ws.message({ toolCall: { functionCalls: [{ id: 'c', name: 'three', args: {} }] }, }); await tick(); ws.message({ serverContent: { interrupted: true } }); second.resolve({ ok: true }); await tick(); assert.equal(ws.sent.filter((entry) => entry.toolResponse).length, 1); assert.equal(h.captures[0].cleared, true); h.c.stop(); }); test('wire voiceActivity.type start aborts pending work and discards old playback', async () => { const pending = deferred(); let signal; const h = harness({ runAction: (_name, _args, options) => { signal = options.signal; return pending.promise; }, }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; ws.message({ serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } }, }); ws.message({ toolCall: { functionCalls: [ { id: 'old', name: 'fly_to_location', args: { query: 'Austin' } }, ], }, }); await tick(); const turn = h.c.turn; assert.equal(signal.aborted, false); assert.equal(h.captures[0].pending, true); ws.message({ voiceActivity: { type: 'ACTIVITY_START', audioOffset: '5.240s' }, }); assert.equal(signal.aborted, true); assert.ok(h.c.turn > turn); assert.equal(h.c.pendingTools, 0); assert.equal(h.captures[0].cleared, true); assert.equal(h.c.status, 'listening'); pending.resolve({ ok: true }); await tick(); assert.equal( ws.sent.some((entry) => entry.toolResponse), false, ); h.c.stop(); }); test('unknown or malformed voice activity cannot revoke the current turn', async () => { const h = harness(); await h.c.start(); const turn = h.c.turn; for (const voiceActivity of [ null, {}, { type: 'TYPE_UNSPECIFIED' }, { type: 'unknown' }, { type: 1 }, ]) { h.sockets[0].message({ voiceActivity }); assert.equal(h.c.turn, turn); assert.equal(h.c.status, 'listening'); assert.equal(h.c.narration.state().pendingTimers, 0); } h.c.stop(); }); for (const pushToTalk of [false, true]) { test(`wire voiceActivity.type end respects ${pushToTalk ? 'manual' : 'automatic'} turn ownership`, async () => { const h = harness(); await h.c.start({ pushToTalk }); if (!pushToTalk) h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_START' } }); h.c.narration.cancel(); h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_END', audioOffset: '9.080s' }, }); assert.equal(h.c.narration.state().pendingTimers, pushToTalk ? 0 : 1); assert.equal(h.c.status, 'listening'); h.c.stop(); assert.equal(h.timers.size, 0); }); } test('radio handoff waits for a post-tool reply and actual playback drain', async () => { const h = harness({ runAction: async () => ({ ok: true, action: 'control_radio', radioAction: 'play', radioPlaybackRequested: true, }), }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; ws.message({ toolCall: { functionCalls: [ { id: 'radio', name: 'control_radio', args: { action: 'play' } }, ], }, }); await h.c.toolQueue; ws.message({ serverContent: { turnComplete: true } }); assert.equal( h.state.playing, 0, 'the tool turn alone must not release Radio', ); ws.message({ serverContent: { modelTurn: { parts: [ { inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } }, ], }, turnComplete: true, }, }); assert.equal( h.state.playing, 0, 'queued assistant audio still owns the speaker', ); h.captures[0].drain(); await tick(); assert.equal(h.state.playing, 1); assert.equal(h.state.ducked, false); assert.equal(h.c.status, 'idle'); }); test('explicit Radio stop cancels a handoff while confirmation is playing', async () => { const h = harness({ runAction: async () => ({ ok: true, action: 'control_radio', radioPlaybackRequested: true, }), }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; ws.message({ toolCall: { functionCalls: [ { id: 'r', name: 'control_radio', args: { action: 'play' } }, ], }, }); await h.c.toolQueue; ws.message({ serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, turnComplete: true, }, }); h.radioControl({ action: 'stop', origin: 'user' }); h.captures[0].drain(); await tick(); assert.equal(h.state.playing, 0); assert.equal(h.c.status, 'listening'); h.c.stop(); }); test('eligible entity context sends a bounded fresh image before its function result', async () => { const h = harness({ runAction: async () => ({ ok: true, action: 'get_entity_context', scene: { basemap: { viewScale: 'local' } }, }), captureViewport: async () => 'data:image/jpeg;base64,aW1hZ2U=', }); await h.c.start(); establishToolInput(h); h.sockets[0].message({ toolCall: { functionCalls: [{ id: 'view', name: 'get_entity_context', args: {} }], }, }); await h.c.toolQueue; const sent = h.sockets[0].sent; const imageIndex = sent.findIndex((entry) => entry.realtimeInput?.video); assert.ok(imageIndex > 0); assert.ok(imageIndex < sent.findIndex((entry) => entry.toolResponse)); h.c.stop(); }); test('socket pressure drops PCM without failing the active session and delivery recovers', async () => { const h = harness(); await h.c.start(); const ws = h.sockets[0], audio = h.captures[0], before = ws.sent.length; ws.bufferedAmount = 300000; audio.callbacks.onAudio( { data: 'AA==', mimeType: 'audio/pcm;rate=16000' }, audio.metadata, ); assert.equal(ws.sent.length, before); assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1); assert.equal(h.c.status, 'listening'); assert.equal(h.tracks[0].readyState, 'live'); ws.bufferedAmount = 0; audio.callbacks.onAudio( { data: 'AQ==', mimeType: 'audio/pcm;rate=16000' }, audio.metadata, ); assert.equal(ws.sent.at(-1).realtimeInput.audio.data, 'AQ=='); h.c.stop(); }); test('passive map context never interrupts spoken output and is delivered at the next explicit turn', async () => { const h = harness(); await h.c.start(); const ws = h.sockets[0]; ws.message({ serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } }, }); const before = ws.sent.length; const event = { type: 'map_annotation_outline', label: 'Austin', status: 'resolved', }; assert.equal(h.c.notifyMapEvent(event), true); event.label = 'changed after enqueue'; assert.equal( ws.sent.length, before, 'passive context sends no clientContent while speech is active', ); assert.equal(h.captures[0].pending, true); h.c.sendTextCommand('What did you outline?'); const parts = ws.sent.at(-1).clientContent.turns[0].parts; assert.equal(JSON.parse(parts[0].text).events[0].label, 'Austin'); assert.equal(parts[1].text, 'What did you outline?'); assert.equal(h.c.pendingMapEvents.length, 0); for (let i = 0; i < 30; i++) h.c.notifyMapEvent({ index: i }); assert.equal(h.c.pendingMapEvents.length, 16); assert.equal(h.c.notifyMapEvent({ text: 'x'.repeat(16000) }), false); h.c.stop(); assert.equal(h.c.pendingMapEvents.length, 0); }); test('passive context joins the next tool response and survives a failed explicit send', async () => { const h = harness(); await h.c.start(); h.c.notifyMapEvent({ type: 'map_annotation_outline', label: 'London' }); h.sockets[0].readyState = 0; assert.equal(h.c.sendTextCommand('Continue'), false); assert.equal(h.c.pendingMapEvents.length, 1); h.sockets[0].readyState = 1; h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_START' } }); h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_END' } }); h.sockets[0].message({ serverContent: { inputTranscription: { text: 'Continue' } }, }); h.sockets[0].message({ toolCall: { functionCalls: [{ id: 'context', name: 'get_entity_context', args: {} }], }, }); await h.c.toolQueue; assert.equal( h.sockets[0].sent.at(-1).toolResponse.functionResponses[0].response .mapEvents[0].label, 'London', ); assert.equal(h.c.pendingMapEvents.length, 0); h.c.stop(); }); test('a delayed permission prompt does not mint a token or consume the network deadline', async () => { const microphone = deferred(); const h = harness({ startupTimeoutMs: 40, getUserMedia: () => microphone.promise, }); const start = h.c.start(); await new Promise((resolve) => setTimeout(resolve, 80)); assert.equal(h.c.status, 'connecting'); assert.equal(h.requests.length, 0); assert.equal(h.sockets.length, 0); microphone.resolve(h.makeStream()); await start; assert.equal(h.c.status, 'listening'); assert.equal(h.requests.length, 1); h.c.stop(); }); test('Stop during a permission prompt releases a late grant without touching the successor', async () => { const microphone = deferred(); const h = harness({ getUserMedia: () => microphone.promise }); const start = h.c.start(); await tick(); h.c.stop(); await start; const lateStream = h.makeStream(); h.c.getUserMedia = async () => h.makeStream(); await h.c.start(); microphone.resolve(lateStream); await tick(); assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(h.tracks[1].readyState, 'live'); assert.equal(h.c.status, 'listening'); h.c.stop(); }); test('stalled audio initialization also obeys the startup deadline', async () => { const initialization = deferred(); const h = harness({ startupTimeoutMs: 40 }); const createAudio = h.c.createAudio; h.c.createAudio = (callbacks) => { const audio = createAudio(callbacks); audio.initialize = () => initialization.promise; return audio; }; await Promise.all([ h.c.start(), new Promise((resolve) => setTimeout(resolve, 80)), ]); assert.equal(h.c.status, 'error'); assert.equal(h.captures[0].closed, true); assert.equal(h.sockets.length, 0); initialization.resolve(); await tick(); assert.equal(h.c.status, 'error'); }); for (const activity of [ 'activity-start', 'push-to-talk', 'interrupted', 'user-text', ]) { test(`new ${activity} revokes a pending Radio preflight after the old spoken turn ended`, async () => { const ready = deferred(); const h = harness({ runAction: async () => ({ ok: true, action: 'control_radio', radioPlaybackRequested: true, }), }); h.c.radio.radioLayer.playForVoice = () => ready.promise; await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; ws.message({ toolCall: { functionCalls: [ { id: 'radio', name: 'control_radio', args: { action: 'play' } }, ], }, }); await h.c.toolQueue; ws.message({ serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, turnComplete: true, }, }); h.captures[0].drain(); assert.equal(h.c.radio.radioHandoffInFlight, true); if (activity === 'activity-start') ws.message({ voiceActivity: { type: 'ACTIVITY_START', audioOffset: '12s', }, }); else if (activity === 'interrupted') ws.message({ serverContent: { interrupted: true } }); else if (activity === 'user-text') h.c.sendTextCommand('Actually stop'); else h.c.input.setMicrophoneEnabled(true); ready.resolve(true); await tick(); assert.equal(h.c.status, 'listening'); assert.equal(h.c.stream.getTracks()[0].readyState, 'live'); assert.equal(h.state.ducked, true); assert.ok(h.state.stopped > 0); h.c.stop(); }); } test('uncorrelated late transcription preserves its own successful Radio action and preflight', async () => { const tool = deferred(); const ready = deferred(); const h = harness({ runAction: () => tool.promise }); h.c.radio.radioLayer.playForVoice = () => ready.promise; await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; ws.message({ toolCall: { functionCalls: [ { id: 'radio-late-transcript', name: 'control_radio', args: { action: 'play' }, }, ], }, }); await tick(); const handoffEpoch = h.c.radio.radioHandoffEpoch; ws.message({ serverContent: { inputTranscription: { text: 'Play the radio' }, interimInputTranscription: { text: 'Play the radio' }, }, }); assert.equal(h.c.radio.radioHandoffEpoch, handoffEpoch); tool.resolve({ ok: true, action: 'control_radio', radioPlaybackRequested: true, }); await h.c.toolQueue; const response = ws.sent.find((message) => message.toolResponse).toolResponse .functionResponses[0].response; assert.equal(response.ok, true); assert.equal(response.radioPlaybackRequested, true); ws.message({ serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, turnComplete: true, }, }); h.captures[0].drain(); assert.equal(h.c.radio.radioHandoffInFlight, true); ws.message({ serverContent: { inputTranscription: { text: ' please' } } }); assert.equal(h.c.radio.radioHandoffInFlight, true); ready.resolve(true); await tick(); assert.equal(h.c.status, 'idle'); assert.equal(h.state.ducked, false); }); function radioPlaybackResult() { return { ok: true, action: 'control_radio', radioAction: 'play', radioPlaybackRequested: true, }; } function establishToolInput(h, text = 'Run the requested action') { assert.equal(h.c.sendTextCommand(text), true); } function requestTool( ws, id, name = 'control_radio', args = { action: 'play' }, ) { ws.message({ toolCall: { functionCalls: [{ id, name, args }] } }); } function drainRadioReply(h) { h.sockets[0].message({ serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] }, turnComplete: true, }, }); h.captures[0].drain(); } test('cancelling one queued Radio call preserves active and queued same-feature siblings', async () => { const first = deferred(); const executions = []; const h = harness({ runAction: (name, args, options) => { executions.push({ args, options }); return args.station === 'first' ? first.promise : radioPlaybackResult(); }, }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; for (const station of ['first', 'cancelled', 'last']) requestTool(ws, station, 'control_radio', { action: 'select', station }); await tick(); const epoch = h.c.radio.radioHandoffEpoch; ws.message({ toolCallCancellation: { ids: ['cancelled'] } }); assert.equal(h.c.radio.radioHandoffEpoch, epoch); assert.equal(executions[0].options.isCurrent(), true); first.resolve(radioPlaybackResult()); await h.c.toolQueue; assert.deepEqual( executions.map((entry) => entry.args.station), ['first', 'last'], ); assert.ok(executions.every((entry) => !entry.options.signal.aborted)); assert.equal(h.c.pendingTools, 0); const newer = h.c.radio.pendingRadioPlaybackResult; ws.message({ toolCallCancellation: { ids: ['first', 'unknown'] } }); assert.equal(h.c.radio.pendingRadioPlaybackResult, newer); assert.equal(h.c.radio.radioHandoffEpoch, epoch); h.c.stop(); }); test('matching active Radio cancellation aborts its work and permits its sibling to finish', async () => { const first = deferred(); const executions = []; const h = harness({ runAction: (name, args, options) => { executions.push(options); return args.action === 'play' ? first.promise : radioPlaybackResult(); }, }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; requestTool(ws, 'old'); requestTool(ws, 'new', 'control_radio', { action: 'select' }); await tick(); const epoch = h.c.radio.radioHandoffEpoch; ws.message({ toolCallCancellation: { ids: ['old'] } }); assert.equal(executions[0].signal.aborted, true); assert.equal(executions[0].isCurrent(), false); await h.c.toolQueue; const pending = h.c.radio.pendingRadioPlaybackResult; assert.ok(pending); assert.equal(executions[1].isCurrent(), true); assert.equal(h.c.radio.radioHandoffEpoch, epoch); first.resolve(radioPlaybackResult()); await tick(); assert.equal(h.c.radio.pendingRadioPlaybackResult, pending); assert.deepEqual( ws.sent .flatMap((m) => m.toolResponse?.functionResponses || []) .map((r) => r.id), ['new'], ); h.c.stop(); }); test('matching prepared cancellation preserves a stronger sibling reservation and signal', async () => { const stronger = deferred(); let strongerOptions; const h = harness({ runAction: (name, args, options) => { if (args.action === 'stop') { strongerOptions = options; return stronger.promise; } return radioPlaybackResult(); }, }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; requestTool(ws, 'play'); await h.c.toolQueue; requestTool(ws, 'stop', 'control_radio', { action: 'stop' }); await tick(); const reservations = [...h.c.radio.radioToolHandoffReservations.keys()]; assert.equal(reservations.length, 1); const epoch = h.c.radio.radioHandoffEpoch; ws.message({ toolCallCancellation: { ids: ['play', 'unknown'] } }); assert.equal(h.c.radio.pendingRadioPlaybackResult, null); assert.equal(strongerOptions.signal.aborted, false); assert.deepEqual( [...h.c.radio.radioToolHandoffReservations.keys()], reservations, ); assert.equal(h.c.radio.radioHandoffEpoch, epoch); stronger.resolve({ ok: false, error: 'Refused fixture stop' }); await h.c.toolQueue; assert.equal(h.c.radio.radioToolHandoffReservations.size, 0); drainRadioReply(h); await tick(); assert.equal(h.state.playing, 0); assert.equal(h.c.status, 'listening'); h.c.stop(); }); test('cancelled stronger call releases only its reservation and retains prepared playback', async () => { const stronger = deferred(); const h = harness({ runAction: (name, args) => args.action === 'stop' ? stronger.promise : radioPlaybackResult(), }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; requestTool(ws, 'play'); await h.c.toolQueue; const prepared = h.c.radio.pendingRadioPlaybackResult; requestTool(ws, 'stop', 'control_radio', { action: 'stop' }); await tick(); const separateReservation = h.c.radio.reserveRadioToolHandoff(); ws.message({ toolCallCancellation: { ids: ['stop'] } }); await h.c.toolQueue; assert.equal(h.c.radio.pendingRadioPlaybackResult, prepared); assert.deepEqual( [...h.c.radio.radioToolHandoffReservations.keys()], [separateReservation], ); stronger.resolve({ ok: true, action: 'control_radio' }); await tick(); assert.equal(h.c.radio.pendingRadioPlaybackResult, prepared); h.c.radio.settleRadioToolHandoffReservation(separateReservation); drainRadioReply(h); await tick(); assert.equal(h.state.playing, 1); assert.equal(h.c.status, 'idle'); }); test('matching preflight cancellation and late completion cannot revoke a newer reused-result handoff', async () => { const sharedResult = radioPlaybackResult(); const starts = []; const stops = []; let playingAttempt = null; const h = harness({ runAction: () => sharedResult }); h.c.radio.radioLayer.playForVoice = ({ attemptId }) => { const ready = deferred(); starts.push({ attemptId, ready }); playingAttempt = attemptId; return ready.promise; }; h.c.radio.radioLayer.stopPlayback = ({ attemptId }) => { stops.push(attemptId); if (playingAttempt === attemptId) playingAttempt = null; }; await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; requestTool(ws, 'old'); await h.c.toolQueue; drainRadioReply(h); const oldResult = h.c.radio.radioHandoffInFlightResult; establishToolInput(h, 'Play the radio again'); requestTool(ws, 'new'); await h.c.toolQueue; const newerResult = h.c.radio.pendingRadioPlaybackResult; assert.notEqual(oldResult, newerResult); assert.notEqual(newerResult, sharedResult); const epoch = h.c.radio.radioHandoffEpoch; ws.message({ toolCallCancellation: { ids: ['old'] } }); assert.equal(h.c.radio.radioHandoffEpoch, epoch); assert.equal(h.c.radio.pendingRadioPlaybackResult, newerResult); assert.equal(h.c.radio.radioHandoffInFlight, false); assert.deepEqual(stops, [starts[0].attemptId]); drainRadioReply(h); assert.equal(starts.length, 2); const newerAttempt = starts[1].attemptId; starts[0].ready.resolve(true); await tick(); assert.equal(h.c.status, 'listening'); assert.equal(h.c.radio.radioHandoffAttemptId, newerAttempt); assert.equal(playingAttempt, newerAttempt); assert.ok(stops.every((id) => id === starts[0].attemptId)); ws.message({ toolCallCancellation: { ids: ['old', 'unknown'] } }); assert.equal(h.c.radio.radioHandoffAttemptId, newerAttempt); starts[1].ready.resolve(true); await tick(); assert.equal(h.c.status, 'idle'); assert.equal(playingAttempt, newerAttempt); }); test('cancelled preflight rejection cannot emit a stale Radio failure correction', async () => { let rejectReady; const ready = new Promise((_, reject) => { rejectReady = reject; }); const corrections = []; const h = harness({ runAction: () => radioPlaybackResult() }); h.c.radio.radioLayer.playForVoice = () => ready; h.c.radio.queueResponseCreate = (text) => corrections.push(text); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; requestTool(ws, 'old'); await h.c.toolQueue; drainRadioReply(h); ws.message({ toolCallCancellation: { ids: ['old'] } }); rejectReady(new Error('late obsolete stream rejection')); await tick(); assert.deepEqual(corrections, []); assert.equal(h.c.status, 'listening'); h.c.stop(); }); test('completed query and unknown cancellation IDs are inert and cannot poison future calls', async () => { const executions = []; const h = harness({ runAction: (name, args) => { executions.push(args.action); return args.action === 'status' ? { ok: true, action: name } : radioPlaybackResult(); }, }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; requestTool(ws, 'query', 'control_radio', { action: 'status' }); await h.c.toolQueue; requestTool(ws, 'owner'); await h.c.toolQueue; const prepared = h.c.radio.pendingRadioPlaybackResult; const epoch = h.c.radio.radioHandoffEpoch; ws.message({ toolCallCancellation: { ids: ['query', 'future'] } }); assert.equal(h.c.radio.pendingRadioPlaybackResult, prepared); assert.equal(h.c.radio.radioHandoffEpoch, epoch); requestTool(ws, 'future', 'control_radio', { action: 'select' }); await h.c.toolQueue; assert.deepEqual(executions, ['status', 'play', 'select']); h.c.stop(); }); for (const replacement of ['user-activity', 'stop']) { test(`${replacement} still revokes all prepared and preflight Radio ownership`, async () => { const ready = deferred(); const h = harness({ runAction: () => radioPlaybackResult() }); h.c.radio.radioLayer.playForVoice = () => ready.promise; await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; requestTool(ws, 'old'); await h.c.toolQueue; drainRadioReply(h); h.c.radio.setPendingPlayback(radioPlaybackResult()); assert.ok(h.c.radio.pendingRadioPlaybackResult); assert.equal(h.c.radio.radioHandoffInFlight, true); if (replacement === 'user-activity') h.c.noteUserActivity(); else h.c.stop(); assert.equal(h.c.radio.pendingRadioPlaybackResult, null); assert.equal(h.c.radio.radioHandoffInFlight, false); assert.equal(h.c.radio.radioToolHandoffReservations.size, 0); assert.ok(h.state.stopped > 0); ready.resolve(true); await tick(); assert.equal(h.c.status, replacement === 'stop' ? 'idle' : 'listening'); if (replacement === 'user-activity') { assert.equal(h.c.stream.getTracks()[0].readyState, 'live'); h.c.stop(); } }); } test('successful stronger Radio stop still invalidates the prepared group', async () => { const h = harness({ runAction: (name, args) => args.action === 'stop' ? { ok: true, action: name, radioAction: 'stop' } : radioPlaybackResult(), }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0]; requestTool(ws, 'play'); await h.c.toolQueue; const epoch = h.c.radio.radioHandoffEpoch; requestTool(ws, 'stop', 'control_radio', { action: 'stop' }); await h.c.toolQueue; assert.equal(h.c.radio.pendingRadioPlaybackResult, null); assert.ok(h.c.radio.radioHandoffEpoch > epoch); drainRadioReply(h); await tick(); assert.equal(h.state.playing, 0); assert.equal(h.c.status, 'listening'); h.c.stop(); }); test('slow Gemini tools reuse shared progress timing without injecting a conversation turn', async () => { const pending = deferred(); let options; const h = harness({ runAction: (_name, _args, next) => { options = next; next.progress({ step: 'search', label: 'Austin' }); return pending.promise; }, }); await h.c.start({ pushToTalk: true }); h.c.sendTextCommand('Fly to Austin'); const ws = h.sockets[0]; requestTool(ws, 'flight', 'fly_to_location', { query: 'Austin' }); await tick(); assert.equal( options.callId, 'flight', 'shared card and narration use one call identity', ); assert.equal(h.c.narration.state().tools, 1); h.advance(699); assert.equal(h.earcon.playing, false); h.advance(1); assert.equal(h.earcon.playing, true); h.advance(7500); assert.equal( ws.sent.length, 2, 'progress never sends clientContent or an early tool response', ); assert.equal(h.c.narration.state().stepLines, 0); assert.equal(h.c.narration.state().stillSpoken, false); pending.resolve({ ok: true, action: 'fly_to_location' }); await h.c.toolQueue; assert.equal(ws.sent.at(-1).toolResponse.functionResponses[0].id, 'flight'); assert.equal(h.c.narration.state().tools, 0); h.advance(3000); assert.equal(h.earcon.playing, false, 'reply grace bounds the final earcon'); assert.equal(h.timers.size, 0); h.c.stop(); }); test('fast Gemini tools retract pending progress and Radio tools do not schedule lines', async () => { for (const name of ['fly_to_location', 'control_radio']) { const h = harness(); await h.c.start({ pushToTalk: true }); h.c.sendTextCommand('Run command'); requestTool( h.sockets[0], 'fast', name, name === 'control_radio' ? { action: 'status' } : {}, ); await h.c.toolQueue; h.advance(20000); assert.equal(h.earcon.starts, 0); assert.equal(h.timers.size, 0); assert.equal( h.sockets[0].sent.filter((message) => message.clientContent).length, 1, ); h.c.stop(); } }); test('Gemini model preamble and playback take priority over the working earcon', async () => { const pending = deferred(); const h = harness({ runAction: () => pending.promise }); await h.c.start({ pushToTalk: true }); h.c.sendTextCommand('Fly to Austin'); const ws = h.sockets[0]; ws.message({ serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } }, }); requestTool(ws, 'slow', 'fly_to_location', { query: 'Austin' }); await tick(); assert.equal(h.c.narration.state().preambleSpoken, true); h.advance(1500); assert.equal(h.earcon.starts, 0, 'queued model audio owns the speaker'); h.captures[0].drain(); h.advance(700); assert.equal(h.earcon.playing, true); ws.message({ serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } }, }); assert.equal( h.earcon.playing, false, 'new model audio immediately stops the tone', ); h.c.stop(); pending.resolve({ ok: true }); await tick(); }); test('PTT release starts shared progress timing and a new hold cancels it', async () => { const pending = deferred(); let options; const h = harness({ runAction: (_name, _args, next) => { options = next; return pending.promise; }, }); h.c.input.pushToTalkKeyHeld = true; await h.c.start({ pushToTalk: true }); assert.equal(h.timers.size, 0); h.c.input.pushToTalkKeyHeld = false; h.c.input.setMicrophoneEnabled(false); h.sockets[0].message({ serverContent: { inputTranscription: { text: 'fly to Austin' } }, }); requestTool(h.sockets[0], 'held', 'fly_to_location', { query: 'Austin' }); await tick(); h.advance(700); assert.equal(h.earcon.playing, true); h.c.input.pushToTalkKeyHeld = true; assert.equal(h.c.input.mayClaimSpeaker(), true); assert.equal(h.c.input.bargeIn(), true); h.c.input.setMicrophoneEnabled(true); assert.equal(h.earcon.playing, false); assert.ok(h.c.retiredBoundaryWait, 'interruption retains its recovery bound'); assert.deepEqual([...h.timers.keys()], [h.c.retiredBoundaryWait.timer]); assert.equal(options.signal.aborted, true); assert.equal(h.captures[0].cleared, true); assert.equal(h.c.narration.state().preambleSpoken, false); pending.resolve({ ok: true }); await tick(); assert.equal( h.sockets[0].sent.some((message) => message.toolResponse), false, ); h.c.stop(); }); test('open microphone and Radio speaker ownership suppress Gemini earcons', async () => { for (const mode of ['open-mic', 'held-ptt', 'radio']) { const pending = deferred(); const h = harness({ runAction: () => pending.promise }); await h.c.start({ pushToTalk: mode !== 'open-mic' }); h.c.sendTextCommand('Fly to Austin'); requestTool(h.sockets[0], 'slow', 'fly_to_location', { query: 'Austin' }); await tick(); if (mode === 'held-ptt') h.c.input.pushToTalkKeyHeld = true; if (mode === 'radio') { h.c.radio.setPendingPlayback(radioPlaybackResult()); assert.equal(h.c.input.mayClaimSpeaker(), false); assert.equal( h.c.input.bargeIn(), false, 'a hold cannot steal a Radio handoff', ); assert.equal(h.c.pendingTools, 1); } h.advance(20000); assert.equal(h.earcon.starts, 0, mode); h.c.stop(); pending.resolve({ ok: true }); await tick(); } }); test('Gemini progress is bounded and all terminal paths release narration ownership', async () => { for (const end of ['stop', 'goAway', 'destroy', 'interruption']) { const pending = deferred(); const h = harness({ runAction: () => pending.promise }); await h.c.start({ pushToTalk: true }); h.c.sendTextCommand('Fly to Austin'); const ws = h.sockets[0]; requestTool(ws, 'slow', 'fly_to_location', { query: 'Austin' }); await tick(); h.advance(700); assert.equal(h.earcon.playing, true); h.advance(15000); assert.equal( h.earcon.playing, false, 'working tone never runs beyond the shared limit', ); const disposals = h.earcon.disposals; if (end === 'goAway') ws.message({ goAway: { timeLeft: '0s' } }); else if (end === 'interruption') ws.message({ serverContent: { interrupted: true } }); else h.c.stop({ removeUi: end === 'destroy' }); assert.equal(h.c.narration.state().tools, 0, end); const retirementTimer = h.c.retiredBoundaryWait?.timer; assert.deepEqual( [...h.timers.keys()], end === 'interruption' ? [retirementTimer] : [], 'only the interrupted reply recovery timer may remain', ); if (end === 'interruption') assert.ok(h.c.retiredBoundaryWait); assert.equal(h.earcon.playing, false, end); if (end !== 'interruption') assert.equal(h.earcon.disposals, disposals + 1, end); pending.resolve({ ok: true }); await tick(); h.advance(20000); assert.equal( ws.sent.some((message) => message.toolResponse), false, end, ); assert.equal(h.timers.size, 0, end); h.c.stop(); } }); test('late tool and audio callbacks cannot change a successor Gemini narrator', async () => { const first = deferred(); const second = deferred(); const options = []; const h = harness({ runAction: (_name, _args, next) => { options.push(next); return options.length === 1 ? first.promise : second.promise; }, }); await h.c.start({ pushToTalk: true }); h.c.sendTextCommand('Fly to Austin'); requestTool(h.sockets[0], 'reused', 'fly_to_location', { query: 'Austin' }); await tick(); const oldAudio = h.captures[0].callbacks; h.c.stop(); await h.c.start({ pushToTalk: true }); h.c.sendTextCommand('Fly to Tokyo'); requestTool(h.sockets[1], 'reused', 'fly_to_location', { query: 'Tokyo' }); await tick(); const before = h.c.narration.state(); options[0].progress({ step: 'fly', label: 'Austin' }); oldAudio.onAudioEnd(h.captures[0].metadata); oldAudio.onDrain(); first.resolve({ ok: true }); await tick(); assert.deepEqual(h.c.narration.state(), before); assert.equal(h.c.pendingTools, 1); h.advance(700); assert.equal(h.earcon.playing, true); second.resolve({ ok: true }); await h.c.toolQueue; assert.equal(h.c.narration.state().tools, 0); h.c.stop(); assert.equal(h.timers.size, 0); }); test('Gemini tool progress reaches the shared card event contract with its call id', async () => { const pending = deferred(); const events = []; let options; let h; const session = createVoiceSession({ runner: (_name, _args, next) => { options = next; next.progress({ step: 'search', label: 'Austin' }); return pending.promise; }, createAdapter: (shared) => { h = harness(shared); return { start: (settings) => h.c.start(settings), stop: (settings) => h.c.stop(settings), sendText: (text) => h.c.sendTextCommand(text), sendMapEvent: (event) => h.c.notifyMapEvent(event), }; }, }); session.subscribe((event) => events.push(event)); await session.start({ pushToTalk: true }); establishToolInput(h); requestTool(h.sockets[0], 'shared-call', 'fly_to_location', { query: 'Austin', }); await tick(); assert.equal( events.find((event) => event.type === 'action-call').callId, 'shared-call', ); assert.deepEqual( events.find((event) => event.type === 'progress'), { type: 'progress', name: 'fly_to_location', callId: 'shared-call', step: 'search', label: 'Austin', }, ); pending.resolve({ ok: true, action: 'fly_to_location' }); await h.c.toolQueue; assert.equal( events.find((event) => event.type === 'action-result').callId, 'shared-call', ); session.stop(); const progressCount = events.filter( (event) => event.type === 'progress', ).length; options.progress({ step: 'fly', label: 'Stale Austin' }); assert.equal( events.filter((event) => event.type === 'progress').length, progressCount, ); assert.equal(h.timers.size, 0); session.destroy(); }); function pointerHarness() { let clock = 1000; const pointer = createPointerTracker({ element: { clientWidth: 1000, clientHeight: 500, addEventListener() {}, }, now: () => clock, setTimer: () => 1, clearTimer() {}, pick: (x) => ({ entity: { layerId: 'flights', id: String(x), label: `Aircraft ${x}`, kind: 'aircraft', lat: 30, lon: -97, }, }), }); return { pointer, advance: (ms) => { clock += ms; }, }; } const replyAudio = (data) => ({ modelTurn: { parts: [{ inlineData: { data, mimeType: 'audio/pcm;rate=24000' } }], }, }); test('B48 pointer: a cold held PTT turn freezes keydown and sends quiet context before activity', async () => { const { pointer } = pointerHarness(); const h = harness({ pointer }); pointer.moveTo(300, 200); h.c.input.holdPointer(); pointer.moveTo(700, 200); h.c.input.pushToTalkKeyHeld = true; h.c.input.spaceKeyHeld = true; await h.c.start({ pushToTalk: true }); assert.equal(pointer.activeSnapshot().entity.id, '300'); assert.deepEqual( h.events.filter((event) => event.type === 'pointer').at(-1).pointer, { kind: 'this', label: 'Aircraft 300', screenPx: { x: 300, y: 200 }, }, ); const start = h.sockets[0].sent.find( (value) => value.realtimeInput?.activityStart, ); const context = h.sockets[0].sent.find((value) => value.clientContent); assert.equal( JSON.parse(context.clientContent.turns[0].parts[0].text).target.label, 'Aircraft 300', ); assert.equal(context.clientContent.turns[0].role, 'user'); assert.equal(context.clientContent.turnComplete, false); assert.deepEqual(start, { realtimeInput: { activityStart: {} } }); assert.equal( h.sockets[0].sent.indexOf(context) < h.sockets[0].sent.indexOf(start), true, ); assert.equal( h.c.replyPending, false, 'context and an empty hold request no reply', ); assert.equal( h.sockets[0].sent.some( (value) => value.clientContent?.turnComplete === true, ), false, ); h.c.input.releasePointerHold(); h.c.input.setMicrophoneEnabled(false); // A delayed server onset must not replace the released gesture's referent. h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_START' } }); assert.equal(pointer.activeSnapshot().entity.id, '300'); h.sockets[0].message({ serverContent: { inputTranscription: { text: 'what is this' } }, }); h.sockets[0].message({ toolCall: { functionCalls: [ { id: 'pointed', name: 'get_entity_context', args: { scope: 'pointer' }, }, ], }, }); await h.c.toolQueue; assert.equal(h.calls[0].args.scope, 'pointer'); assert.equal(pointer.activeSnapshot().entity.id, '300'); h.c.stop(); assert.equal(pointer.activeSnapshot(), null); pointer.destroy(); }); test('B48 pointer: text uses recent availability, clears stale context and sends one turn', async () => { const { pointer, advance } = pointerHarness(); const h = harness({ pointer }); await h.c.start(); pointer.moveTo(400, 200); h.c.sendTextCommand('what is this?'); assert.equal(pointer.activeSnapshot().entity.id, '400'); let parts = h.sockets[0].sent.at(-1).clientContent.turns[0].parts; assert.deepEqual(JSON.parse(parts[0].text), { type: 'pointer_context', at: 'text', pointing: 'aircraft', }); assert.equal(parts.at(-1).text, 'what is this?'); assert.equal( h.sockets[0].sent.filter((value) => value.clientContent).length, 1, ); assert.equal( h.events.filter((event) => event.type === 'pointer').at(-1).pointer, null, ); h.c.runAction = async () => { assert.equal(pointer.activeSnapshot().entity.id, '400'); assert.equal(h.c.announcePointer(), true); assert.equal(h.c.announcePointer(), false); return { ok: true }; }; h.sockets[0].message({ toolCall: { functionCalls: [ { id: 'implicit-pointer', name: 'get_entity_context', args: { scope: 'pointer' }, }, ], }, }); await h.c.toolQueue; assert.equal( h.events.filter((event) => event.type === 'pointer').at(-1).pointer.label, 'Aircraft 400', ); h.sockets[0].message({ serverContent: { turnComplete: true } }); advance(5001); h.c.sendTextCommand('what is here?'); assert.equal(pointer.activeSnapshot(), null); parts = h.sockets[0].sent.at(-1).clientContent.turns[0].parts; assert.deepEqual(JSON.parse(parts[0].text), { type: 'pointer_context', at: 'text', target: 'none', }); assert.equal(parts.at(-1).text, 'what is here?'); h.c.stop(); pointer.destroy(); }); test('B48 pointer: automatic speech snapshots once per onset and late input transcript keeps it', async () => { const { pointer } = pointerHarness(); const h = harness({ pointer }); await h.c.start(); pointer.moveTo(450, 200); const ws = h.sockets[0]; ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); const turn = h.c.turn; assert.equal(pointer.activeSnapshot().entity.id, '450'); assert.deepEqual(JSON.parse(ws.sent.at(-1).realtimeInput.text), { type: 'pointer_context', at: 'speech_start', pointing: 'aircraft', }); pointer.moveTo(750, 200); ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); ws.message({ serverContent: { inputTranscription: { text: 'what is this?' } }, }); assert.equal( h.c.turn, turn, 'duplicate onset does not revoke the current turn', ); assert.equal(pointer.activeSnapshot().entity.id, '450'); assert.equal( ws.sent.some((value) => value.clientContent), false, ); ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); ws.message({ serverContent: { turnComplete: true } }); ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); assert.equal(pointer.activeSnapshot().entity.id, '750'); h.c.stop(); pointer.destroy(); }); for (const route of ['push-to-talk', 'user-text']) { for (const idle of ['first', 'completed']) { test(`B48 fence: ${route} permits the ${idle} idle reply without an interruption marker`, async () => { const h = harness(); await h.c.start({ pushToTalk: route === 'push-to-talk' }); const ws = h.sockets[0]; const audio = h.captures[0]; if (idle === 'completed') { ws.message({ serverContent: { ...replyAudio('previous'), turnComplete: true }, }); audio.drain(); } if (route === 'user-text') h.c.sendTextCommand('first live request'); else { h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); h.c.input.setMicrophoneEnabled(false); ws.message({ serverContent: { inputTranscription: { text: 'what is in view' } }, }); } ws.message({ toolCall: { functionCalls: [ { id: 'valid-first-call', name: 'get_entity_context', args: {} }, ], }, }); await h.c.toolQueue; ws.message({ serverContent: { ...replyAudio('legitimate'), turnComplete: true }, }); assert.equal(h.calls.length, 1); assert.equal(audio.played.at(-1).data, 'legitimate'); h.c.stop(); }); } } test('B48 fence: an empty PTT hold creates no retired boundary for the next command', async () => { const h = harness(); await h.c.start({ pushToTalk: true }); h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); h.c.input.setMicrophoneEnabled(false); h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); h.captures[0].callbacks.onAudio( { data: 'AAAA', mimeType: 'audio/pcm;rate=16000', }, h.captures[0].metadata, ); h.c.input.setMicrophoneEnabled(false); h.sockets[0].message({ serverContent: { ...replyAudio('first-real'), turnComplete: true }, }); assert.equal(h.captures[0].played.at(-1).data, 'first-real'); h.c.stop(); }); test('B48 fence: a coalesced interrupted boundary keeps successor speech and its pointer', async () => { const { pointer } = pointerHarness(); const h = harness({ pointer }); await h.c.start(); const ws = h.sockets[0]; ws.message({ serverContent: replyAudio('old') }); pointer.moveTo(600, 200); ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); const turn = h.c.turn; ws.message({ voiceActivity: { type: 'ACTIVITY_START' }, serverContent: { ...replyAudio('boundary-old'), interrupted: true, turnComplete: true, inputTranscription: { text: 'new pointed question' }, outputTranscription: { text: 'old answer' }, }, }); assert.equal(h.c.turn, turn); assert.equal(pointer.activeSnapshot().entity.id, '600'); assert.equal(h.captures[0].played.length, 1); assert.equal( h.events.some((event) => event.type === 'completion'), false, ); assert.equal( h.events.some( (event) => event.role === 'user' && event.text === 'new pointed question', ), true, ); ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); ws.message({ serverContent: { ...replyAudio('successor'), turnComplete: true }, }); assert.equal(h.captures[0].played.at(-1).data, 'successor'); h.c.stop(); pointer.destroy(); }); for (const route of ['push-to-talk', 'user-text']) { test(`B48 queue: rapid ${route} keeps only the latest pointer/input and serializes reply boundaries`, async () => { const { pointer } = pointerHarness(); const h = harness({ pointer }); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; const audio = h.captures[0]; ws.message({ serverContent: replyAudio('old') }); const submit = (x) => { pointer.moveTo(x, 200); if (route === 'user-text') return h.c.sendTextCommand(`request ${x}`); h.c.input.holdPointer(); h.c.input.bargeIn(); h.c.input.beginPointerTurn('keydown'); h.c.input.setMicrophoneEnabled(true); audio.callbacks.onAudio( { data: String(x), mimeType: 'audio/pcm;rate=16000', }, audio.metadata, ); h.c.input.releasePointerHold(); h.c.input.setMicrophoneEnabled(false); }; submit(300); // This start/command must reach the provider to cause interruption. const sentFirst = ws.sent.length; submit(500); submit(700); // Newer intent replaces the still-unsent second turn. assert.equal( ws.sent.length, sentFirst, 'later input waits without opening another server generation', ); assert.equal(pointer.activeSnapshot().entity.id, '700'); const latestTurn = h.c.turn; ws.message({ serverContent: { interrupted: true, turnComplete: true } }); const flushed = ws.sent.slice(sentFirst); if (route === 'user-text') { assert.equal(flushed.length, 1); assert.equal( flushed[0].clientContent.turns[0].parts.at(-1).text, 'request 700', ); } else { assert.deepEqual( flushed.map((value) => value.clientContent ? ['context'] : Object.keys(value.realtimeInput), ), [['context'], ['activityStart'], ['audio'], ['activityEnd']], ); assert.equal( JSON.parse(flushed[0].clientContent.turns[0].parts[0].text).target .label, 'Aircraft 700', ); assert.equal(flushed[0].clientContent.turnComplete, false); assert.equal(flushed[2].realtimeInput.audio.data, '700'); } ws.message({ serverContent: { ...replyAudio('first-successor-retired'), outputTranscription: { text: 'stale second answer' }, }, }); ws.message({ toolCall: { functionCalls: [ { id: 'queued-old-call', name: 'fly_to_location', args: { query: 'stale' }, }, ], }, }); assert.equal(audio.played.length, 1); ws.message({ serverContent: { interrupted: true, turnComplete: true } }); assert.equal(h.c.turn, latestTurn); ws.message({ serverContent: { inputTranscription: { text: 'what is this' } }, }); ws.message({ toolCall: { functionCalls: [ { id: 'latest-call', name: 'get_entity_context', args: { scope: 'pointer' }, }, ], }, }); if (route === 'push-to-talk') { await tick(); h.advance(5_000); } await h.c.toolQueue; assert.equal(h.calls.length, 1); assert.equal(pointer.activeSnapshot().entity.id, '700'); ws.message({ serverContent: { ...replyAudio('latest'), turnComplete: true }, }); assert.equal(audio.played.at(-1).data, 'latest'); h.c.stop(); pointer.destroy(); }); } test('B48 queue: wait timeout fails closed and Stop releases queued input without replay', async () => { for (const exit of ['timeout', 'stop']) { const h = harness({ inputBoundaryTimeoutMs: 10 }); await h.c.start(); const ws = h.sockets[0]; ws.message({ serverContent: replyAudio('old') }); h.c.sendTextCommand('sent successor'); h.c.sendTextCommand('queued latest'); const sent = ws.sent.length; if (exit === 'stop') h.c.stop(); else h.advance(25); assert.equal(ws.closed, true); assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(h.c.status, exit === 'stop' ? 'idle' : 'error'); assert.equal(h.c.deferredInput, null); ws.message({ serverContent: { interrupted: true, turnComplete: true } }); assert.equal( ws.sent.length, sent, 'released queue cannot replay on a late boundary', ); h.c.stop(); } }); test('B48 queue: 256 KiB input bound fails closed without flushing partial queued speech', async () => { const h = harness(); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; ws.message({ serverContent: replyAudio('old') }); h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); h.captures[0].callbacks.onAudio( { data: 'first', mimeType: 'audio/pcm;rate=16000', }, h.captures[0].metadata, ); h.c.input.setMicrophoneEnabled(false); h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); const sent = ws.sent.length; h.captures[0].callbacks.onAudio( { data: 'A'.repeat(256 * 1024), mimeType: 'audio/pcm;rate=16000', }, h.captures[0].metadata, ); assert.equal(h.c.status, 'error'); assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(ws.closed, true); assert.equal(h.c.deferredInput, null); assert.equal(ws.sent.length, sent); h.c.stop(); }); test('B48 fence: duplicate automatic activity end cannot retire an idle next command', async () => { const h = harness(); await h.c.start(); const ws = h.sockets[0]; ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); ws.message({ serverContent: { ...replyAudio('finished'), turnComplete: true }, }); h.captures[0].drain(); ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); h.c.sendTextCommand('new idle turn'); ws.message({ serverContent: { ...replyAudio('new idle answer'), turnComplete: true }, }); assert.equal(h.captures[0].played.at(-1).data, 'new idle answer'); h.c.stop(); }); test('B48 queue: a boundary before deferred PTT worklet start still fences the submitted successor', async () => { const h = harness(); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; ws.message({ serverContent: replyAudio('old') }); h.c.sendTextCommand('first successor'); h.c.input.bargeIn(); // Audio-start callback is deliberately delayed. ws.message({ serverContent: { interrupted: true, turnComplete: true } }); ws.message({ serverContent: replyAudio('first-successor-stale') }); assert.equal(h.captures[0].played.length, 1); h.c.input.setMicrophoneEnabled(true); h.captures[0].callbacks.onAudio( { data: 'latest', mimeType: 'audio/pcm;rate=16000', }, h.captures[0].metadata, ); h.c.input.setMicrophoneEnabled(false); ws.message({ serverContent: { interrupted: true, turnComplete: true } }); ws.message({ serverContent: { ...replyAudio('latest'), turnComplete: true }, }); assert.equal(h.captures[0].played.at(-1).data, 'latest'); h.c.stop(); }); for (const latest of ['push-to-talk', 'user-text']) { test(`B48 queue: ${latest} can replace the other unsent input mode without stale pointer context`, async () => { const { pointer, advance } = pointerHarness(); const h = harness({ pointer }); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; const audio = h.captures[0]; ws.message({ serverContent: replyAudio('old') }); pointer.moveTo(300, 200); h.c.sendTextCommand('first successor'); const speak = (x) => { pointer.moveTo(x, 200); h.c.input.holdPointer(); h.c.input.bargeIn(); h.c.input.beginPointerTurn('keydown'); h.c.input.setMicrophoneEnabled(true); audio.callbacks.onAudio( { data: String(x), mimeType: 'audio/pcm;rate=16000', }, audio.metadata, ); h.c.input.releasePointerHold(); h.c.input.setMicrophoneEnabled(false); }; if (latest === 'push-to-talk') { pointer.moveTo(500, 200); h.c.sendTextCommand('discard queued text'); speak(700); } else { speak(500); advance(5001); h.c.sendTextCommand('latest text without pointer'); } const sent = ws.sent.length; ws.message({ serverContent: { turnComplete: true } }); // Natural completion races the claim. const frames = ws.sent.slice(sent); if (latest === 'push-to-talk') { assert.equal( frames.some((message) => message.clientContent?.turnComplete === true), false, ); assert.equal(frames[0].clientContent.turnComplete, false); assert.deepEqual(frames[1], { realtimeInput: { activityStart: {} } }); assert.equal( JSON.parse(frames[0].clientContent.turns[0].parts[0].text).target.label, 'Aircraft 700', ); } else { assert.equal(frames.length, 1); assert.deepEqual( JSON.parse(frames[0].clientContent.turns[0].parts[0].text), { type: 'pointer_context', at: 'text', target: 'none' }, ); assert.equal(pointer.activeSnapshot(), null); } ws.message({ serverContent: { interrupted: true, turnComplete: true } }); ws.message({ serverContent: { ...replyAudio('latest'), turnComplete: true }, }); assert.equal(audio.played.at(-1).data, 'latest'); h.c.stop(); pointer.destroy(); }); } test('R51: typed override closes forwarded PTT before its queued text and suppresses the old flush', async () => { const h = harness(); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; const audio = h.captures[0]; ws.message({ serverContent: replyAudio('retired A') }); h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); audio.callbacks.onAudio( { data: 'spoken B', mimeType: 'audio/pcm;rate=16000', }, audio.metadata, ); const before = ws.sent.length; h.c.sendTextCommand('latest typed C'); assert.equal( ws.sent.at(-1).realtimeInput?.activityEnd !== undefined, true, 'the forwarded speech must end when typed intent supersedes it', ); assert.equal(ws.sent.length, before + 1); audio.callbacks.onAudio( { data: 'retired buffered B', mimeType: 'audio/pcm;rate=16000', }, audio.metadata, ); h.c.input.setMicrophoneEnabled(false); assert.equal( ws.sent.length, before + 1, 'late old PCM/end cannot extend or duplicate closed activity', ); ws.message({ serverContent: { interrupted: true, turnComplete: true } }); const flushed = ws.sent.slice(before); assert.equal(flushed.length, 2); assert.equal(flushed[0].realtimeInput?.activityEnd !== undefined, true); assert.equal( flushed[1].clientContent.turns[0].parts.at(-1).text, 'latest typed C', ); h.c.stop(); }); test('R51: a delayed old worklet start cannot reclaim a typed turn; a fresh hold still works', async () => { const h = harness(); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; const audio = h.captures[0]; h.c.sendTextCommand('current text'); const before = ws.sent.length; // This is the prior gesture's callback, delayed until after typed intent. audio.callbacks.onAudioStart(audio.metadata); audio.callbacks.onAudio( { data: 'old gesture', mimeType: 'audio/pcm;rate=16000', }, audio.metadata, ); audio.callbacks.onAudioEnd(audio.metadata); assert.equal(ws.sent.length, before); ws.message({ serverContent: { ...replyAudio('text completed'), turnComplete: true }, }); audio.drain(); h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); audio.callbacks.onAudio( { data: 'fresh hold', mimeType: 'audio/pcm;rate=16000', }, audio.metadata, ); h.c.input.setMicrophoneEnabled(false); const frames = ws.sent.slice(before); assert.equal(frames[0].realtimeInput.activityStart !== undefined, true); assert.equal(frames[1].realtimeInput.audio.data, 'fresh hold'); assert.equal(frames[2].realtimeInput.activityEnd !== undefined, true); h.c.stop(); }); test('R51: replaced unsent PTT contributes no unmatched end to the forwarded stream', async () => { const h = harness(); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; const audio = h.captures[0]; ws.message({ serverContent: replyAudio('retired A') }); h.c.sendTextCommand('submitted B'); h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); audio.callbacks.onAudio( { data: 'unsent C', mimeType: 'audio/pcm;rate=16000', }, audio.metadata, ); h.c.sendTextCommand('newest typed D'); const before = ws.sent.length; h.c.input.setMicrophoneEnabled(false); assert.equal(ws.sent.length, before); ws.message({ serverContent: { interrupted: true, turnComplete: true } }); const flushed = ws.sent.slice(before); assert.equal(flushed.length, 1); assert.equal( flushed[0].clientContent.turns[0].parts.at(-1).text, 'newest typed D', ); h.c.stop(); }); test('R51-2: FIFO old capture messages stay retired after typed replacement and a fresh claim', async () => { const { pointer } = pointerHarness(); const h = harness({ pointer }); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; const audio = h.captures[0]; // Delay only the worklet message port; gate/gesture callbacks still run now. const port = []; const callbacks = audio.callbacks; const enable = audio.setMicrophoneEnabled.bind(audio); audio.setMicrophoneEnabled = (value, metadata) => { if (value && !audio.enabled) port.push(() => callbacks.onAudioStart(metadata)); if (!value && audio.enabled) { const ending = audio.metadata; port.push(() => callbacks.onAudioEnd(ending)); } audio.enabled = value; if (value) audio.metadata = metadata; }; ws.message({ serverContent: replyAudio('outstanding A') }); const claim = (x) => { pointer.moveTo(x, 200); h.c.input.holdPointer(); h.c.input.bargeIn(); h.c.input.beginPointerTurn('keydown'); h.c.input.setMicrophoneEnabled(true); return audio.metadata; }; const b = claim(300); port.push(() => callbacks.onAudio( { data: 'retired B', mimeType: 'audio/pcm;rate=16000' }, b, ), ); h.c.sendTextCommand('typed C'); h.c.input.releasePointerHold(); h.c.input.setMicrophoneEnabled(false); const d = claim(700); port.push(() => callbacks.onAudio( { data: 'current D', mimeType: 'audio/pcm;rate=16000' }, d, ), ); h.c.input.releasePointerHold(); h.c.input.setMicrophoneEnabled(false); const before = ws.sent.length; // Allowed port FIFO: start B / PCM B / end B / start D / PCM D / end D. for (const deliver of port) deliver(); assert.equal(ws.sent.length, before); const queued = h.c.deferredInput.messages; assert.deepEqual( queued.map((message) => message.clientContent ? 'context' : message.realtimeInput.audio?.data || (message.realtimeInput.activityStart ? 'start' : 'end'), ), ['context', 'start', 'current D', 'end'], ); assert.equal( JSON.parse(queued[0].clientContent.turns[0].parts[0].text).target.label, 'Aircraft 700', ); ws.message({ serverContent: { interrupted: true, turnComplete: true } }); assert.deepEqual(ws.sent.slice(before), queued); assert.equal(JSON.stringify(ws.sent).includes('captureId'), false); audio.setMicrophoneEnabled = enable; h.c.stop(); }); test('R51-2: a normal release keeps its delayed final PCM and end, without relabelling duplicate gates or tool aborts', async () => { const h = harness(); h.c.input.pushToTalkKeyHeld = true; await h.c.start({ pushToTalk: true }); const audio = h.captures[0]; const metadata = audio.metadata; const ws = h.sockets[0]; h.c.input.setMicrophoneEnabled(true); assert.equal(audio.metadata.captureId, metadata.captureId); h.c.abortTools(); // Radio cleanup is not a new physical capture. const end = audio.callbacks.onAudioEnd; let released; audio.callbacks.onAudioEnd = (identity) => { released = identity; }; h.c.input.setMicrophoneEnabled(false); assert.equal(released.captureId, metadata.captureId); const before = ws.sent.length; audio.callbacks.onAudio( { data: 'released tail', mimeType: 'audio/pcm;rate=16000' }, metadata, ); end(released); assert.deepEqual(ws.sent.slice(before), [ { realtimeInput: { audio: { data: 'released tail', mimeType: 'audio/pcm;rate=16000' }, }, }, { realtimeInput: { activityEnd: {} } }, ]); audio.callbacks.onAudioEnd = end; h.c.stop(); }); test('R51-2: a forwarded old capture closes once before fresh FIFO start and cannot close the new activity', async () => { const h = harness(); await h.c.start({ pushToTalk: true }); const audio = h.captures[0]; const ws = h.sockets[0]; h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); const b = audio.metadata; audio.callbacks.onAudio( { data: 'forwarded B', mimeType: 'audio/pcm;rate=16000' }, b, ); const start = audio.callbacks.onAudioStart; const end = audio.callbacks.onAudioEnd; const port = []; audio.callbacks.onAudioStart = (identity) => port.push(() => start(identity)); audio.callbacks.onAudioEnd = (identity) => port.push(() => end(identity)); h.c.input.setMicrophoneEnabled(false); h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); const d = audio.metadata; const before = ws.sent.length; for (const deliver of port) deliver(); // end B, start D in port FIFO. assert.deepEqual(ws.sent.slice(before), [ { realtimeInput: { activityEnd: {} } }, { realtimeInput: { activityStart: {} } }, ]); assert.equal(h.c.forwardedCaptureId, d.captureId); end(b); // A duplicate old end cannot close D. assert.equal(ws.sent.length, before + 2); assert.equal(h.c.forwardedActivityOpen, true); audio.callbacks.onAudio( { data: 'current D', mimeType: 'audio/pcm;rate=16000' }, d, ); audio.callbacks.onAudioStart = start; audio.callbacks.onAudioEnd = end; h.c.input.setMicrophoneEnabled(false); assert.equal(ws.sent.at(-1).realtimeInput.activityEnd !== undefined, true); assert.equal(h.c.forwardedActivityOpen, false); h.c.stop(); }); test('R51-2: absent metadata fails closed and Stop/error restart never reuse capture identity', async () => { const h = harness(); let previous = 0; let old; let oldMetadata; for (const terminate of ['stop', 'error', 'stop']) { h.c.input.pushToTalkKeyHeld = true; await h.c.start({ pushToTalk: true }); const audio = h.captures.at(-1); const ws = h.sockets.at(-1); assert.ok(audio.metadata.captureId > previous); previous = audio.metadata.captureId; const before = ws.sent.length; for (const captureId of [undefined, null, 0, -1, 1.5, '1', Infinity]) { const metadata = captureId === undefined ? undefined : { captureId }; audio.callbacks.onAudioStart(metadata); audio.callbacks.onAudio( { data: 'must reject', mimeType: 'audio/pcm;rate=16000' }, metadata, ); audio.callbacks.onAudioEnd(metadata); } if (old) { old.onAudioStart(oldMetadata); old.onAudio( { data: 'old epoch', mimeType: 'audio/pcm;rate=16000' }, oldMetadata, ); old.onAudioEnd(oldMetadata); } assert.equal(ws.sent.length, before); assert.equal(h.c.forwardedActivityOpen, true); old = audio.callbacks; oldMetadata = audio.metadata; if (terminate === 'error') h.c.fail(new Error('capture failed')); else h.c.stop(); assert.equal(h.c.captureOwner, null); assert.equal(h.c.captureEnabled, false); assert.equal(h.c.forwardedCaptureId, null); } }); for (const queued of ['empty', 'text']) { test(`R52: ${queued} flush keeps the original retirement wait bound`, async () => { const h = harness({ inputBoundaryTimeoutMs: 40 }); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; ws.message({ serverContent: replyAudio('old') }); h.c.sendTextCommand('first submitted successor'); if (queued === 'empty') h.c.input.bargeIn(); else h.c.sendTextCommand('newest queued text'); // A physical empty hold no longer submits/retires a successor. The first // submitted text still owns the reply when the old native boundary arrives. if (queued === 'empty') { assert.equal(h.c.deferredInput, null); ws.message({ serverContent: { interrupted: true, turnComplete: true } }); assert.equal(h.c.retiredReplyPending, false); assert.equal(h.c.replyPending, true); h.advance(60); assert.equal(h.c.status, 'listening'); assert.equal(ws.closed, undefined); h.c.stop(); return; } const deadline = h.c.deferredInput.deadline; ws.message({ serverContent: { interrupted: true, turnComplete: true } }); assert.equal(h.c.deferredInput, null); assert.equal(h.c.retiredReplyPending, true); h.advance(deadline + 20); assert.equal( h.c.status, 'error', 'missing genuine boundary cannot leave voice waiting indefinitely', ); assert.equal(ws.closed, true); assert.equal(h.tracks[0].readyState, 'ended'); h.c.stop(); }); } test('R52: replacement cannot extend a flushed retirement deadline and Stop cancels it', async () => { for (const exit of ['timeout', 'stop', 'boundary']) { const h = harness({ inputBoundaryTimeoutMs: 60 }); await h.c.start(); const ws = h.sockets[0]; ws.message({ serverContent: replyAudio('old') }); h.c.sendTextCommand('first successor'); h.c.sendTextCommand('queued successor'); const deadline = h.c.deferredInput.deadline; ws.message({ serverContent: { interrupted: true, turnComplete: true } }); h.advance(15); h.c.sendTextCommand('replacement while successor retired'); assert.equal(h.c.deferredInput.deadline, deadline); if (exit === 'stop') h.c.stop(); if (exit === 'boundary') { ws.message({ serverContent: { interrupted: true, turnComplete: true } }); ws.message({ serverContent: { interrupted: true, turnComplete: true } }); ws.message({ serverContent: { ...replyAudio('latest'), turnComplete: true }, }); h.captures[0].drain(); } h.advance(deadline + 20); assert.equal( h.c.status, exit === 'timeout' ? 'error' : exit === 'stop' ? 'idle' : 'listening', ); assert.equal(Boolean(ws.closed), exit !== 'boundary'); h.c.stop(); } }); test('PTT pointer context is quiet, ordered once, and only PCM creates a pending reply', async () => { const { pointer } = pointerHarness(); const h = harness({ pointer }); pointer.moveTo(300, 200); h.c.input.holdPointer(); h.c.input.pushToTalkKeyHeld = true; h.c.input.spaceKeyHeld = true; await h.c.start({ pushToTalk: true }); const ws = h.sockets[0], audio = h.captures[0]; const [context, start] = ws.sent.slice(1); assert.deepEqual(Object.keys(context), ['clientContent']); assert.equal(context.clientContent.turnComplete, false); assert.equal(context.clientContent.turns.length, 1); assert.equal(context.clientContent.turns[0].role, 'user'); assert.equal( JSON.parse(context.clientContent.turns[0].parts[0].text).target.label, 'Aircraft 300', ); assert.deepEqual(start, { realtimeInput: { activityStart: {} } }); assert.equal(h.c.replyPending, false); assert.equal(audio.played.length, 0); assert.equal(h.calls.length, 0); audio.callbacks.onAudioStart(audio.metadata); assert.equal( ws.sent.length, 3, 'duplicate start does not resend quiet context', ); audio.callbacks.onAudio( { data: 'AAABAA==', mimeType: 'audio/pcm;rate=16000' }, audio.metadata, ); assert.equal(h.c.replyPending, true); assert.equal( ws.sent.some((message) => message.realtimeInput?.text), false, ); h.c.stop(); pointer.destroy(); }); test('PTT pointer context rejected by the channel fails closed without opening activity', async () => { const { pointer } = pointerHarness(); const h = harness({ pointer }); await h.c.start({ pushToTalk: true }); pointer.moveTo(300, 200); h.c.input.holdPointer(); const ws = h.sockets[0], before = ws.sent.length; h.c.channel.readyState = 'connecting'; // The adapter emits one admitted start, not a retryable channel-open event. h.c.input.setMicrophoneEnabled(true); assert.equal(ws.sent.length, before); assert.equal(h.c.forwardedActivityOpen, false); assert.equal(h.c.pointerSentKey, null); assert.equal(h.c.captureOwner, null); assert.equal(h.c.userTurnPending, false); assert.equal(h.c.status, 'error'); assert.equal(ws.closed, true); assert.equal(h.tracks[0].readyState, 'ended'); h.c.stop(); pointer.destroy(); }); test('quiet pointer context rejects stale or absent capture metadata without touching the new pointer', async () => { const { pointer } = pointerHarness(); const h = harness({ pointer }); await h.c.start({ pushToTalk: true }); pointer.moveTo(300, 200); h.c.input.holdPointer(); h.c.input.setMicrophoneEnabled(true); const old = h.captures[0].metadata, context = h.c.captureOwner.context; h.c.input.setMicrophoneEnabled(false); pointer.moveTo(700, 200); h.c.input.holdPointer(); h.c.input.setMicrophoneEnabled(true); const before = h.sockets[0].sent.length, key = h.c.pointerSentKey; assert.equal(h.c.sendPointerContext(context, old), false); assert.equal(h.c.sendPointerContext(context), false); assert.equal(h.c.sendPointerContext(context, { captureId: 0 }), false); assert.equal(h.sockets[0].sent.length, before); assert.equal(h.c.pointerSentKey, key); assert.equal(pointer.activeSnapshot().entity.id, '700'); h.c.stop(); pointer.destroy(); }); test('replacing a deferred quiet PTT context preserves its deadline and only flushes the latest owner', async () => { const { pointer } = pointerHarness(); const h = harness({ pointer }); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0], audio = h.captures[0]; ws.message({ serverContent: replyAudio('old A') }); const claim = (x, pcm) => { pointer.moveTo(x, 200); h.c.input.holdPointer(); h.c.input.bargeIn(); h.c.input.beginPointerTurn('keydown'); h.c.input.setMicrophoneEnabled(true); if (pcm) audio.callbacks.onAudio( { data: pcm, mimeType: 'audio/pcm;rate=16000' }, audio.metadata, ); h.c.input.releasePointerHold(); h.c.input.setMicrophoneEnabled(false); }; claim(300, 'submitted B'); claim(500); const deadline = h.c.deferredInput.deadline; const sent = ws.sent.length; claim(700); assert.equal(h.c.deferredInput.deadline, deadline); assert.equal(h.c.deferredInput.kind, 'push-to-talk'); assert.equal(h.c.deferredInput.captureId, h.c.captureOwner.captureId); assert.equal(h.c.deferredInput.messages.length, 3); assert.equal( JSON.parse( h.c.deferredInput.messages[0].clientContent.turns[0].parts[0].text, ).target.label, 'Aircraft 700', ); assert.equal( JSON.stringify(h.c.deferredInput.messages).includes('Aircraft 500'), false, ); assert.equal(ws.sent.length, sent); // The genuine retired boundary may coalesce content; it must discard it // while releasing only the latest quiet context and its activity markers. ws.message({ serverContent: { ...replyAudio('retired coalesced A'), interrupted: true, turnComplete: true, }, }); const flushed = ws.sent.slice(sent); assert.equal(flushed.length, 3); assert.equal(flushed[0].clientContent.turnComplete, false); assert.deepEqual(flushed[1], { realtimeInput: { activityStart: {} } }); assert.deepEqual(flushed[2], { realtimeInput: { activityEnd: {} } }); assert.equal(audio.played.at(-1).data, 'old A'); assert.equal( h.c.replyPending, false, 'flushed context without PCM asks for no reply', ); assert.equal(h.c.retiredBoundaryWait.deadline, deadline); assert.equal(h.c.retiredReplyPending, true); assert.equal(h.c.pointerSentKey, h.c.captureOwner.context.key); h.c.stop(); pointer.destroy(); }); test('projected PCM bytes and optional viewport frames drop while reliable tool results wait', async () => { const h = harness({ captureViewport: async () => 'data:image/jpeg;base64,aW1hZ2U=', }); await h.c.start(); establishToolInput(h); const ws = h.sockets[0], audio = h.captures[0], before = ws.sent.length; ws.bufferedAmount = 262140; audio.callbacks.onAudio( { data: 'AA==', mimeType: 'audio/pcm;rate=16000' }, audio.metadata, ); ws.message({ toolCall: { functionCalls: [ { id: 'image-pressure', name: 'get_entity_context', args: {} }, ], }, }); h.c.runAction = async () => ({ ok: true, action: 'get_entity_context', scene: { basemap: { viewScale: 'local' } }, }); await h.c.toolQueue; assert.equal(h.c.status, 'listening'); assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1); assert.equal(h.c.getDiagnostics().droppedImages, 1); assert.equal(ws.sent.length, before); assert.equal(h.c.controlQueue.length, 1); ws.bufferedAmount = 0; h.advance(25); assert.equal( ws.sent.at(-1).toolResponse.functionResponses[0].id, 'image-pressure', ); assert.match( ws.sent.at(-1).toolResponse.functionResponses[0].response.responseGuidance, /briefly|request|read|view/i, ); assert.equal(h.c.controlQueue.length, 0); h.c.stop(); }); test('congested PTT markers remain ordered and forwarding flags commit only at wire delivery', async () => { const h = harness(); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0], audio = h.captures[0], before = ws.sent.length; ws.bufferedAmount = 300000; h.c.input.setMicrophoneEnabled(true); const capture = audio.metadata; audio.callbacks.onAudio( { data: 'discarded PCM', mimeType: 'audio/pcm;rate=16000' }, capture, ); h.c.input.setMicrophoneEnabled(false); assert.equal(h.c.forwardedActivityOpen, false); assert.equal(h.c.controlQueue.length, 2); assert.equal(ws.sent.length, before); assert.equal(h.c.replyPending, false); ws.bufferedAmount = 0; h.advance(25); assert.deepEqual(ws.sent.slice(before), [ { realtimeInput: { activityStart: {} } }, { realtimeInput: { activityEnd: {} } }, ]); assert.equal(h.c.forwardedActivityOpen, false); h.c.stop(); }); test('saturation then typed override and a fresh hold cannot flush retired pointer or capture markers', async () => { const { pointer } = pointerHarness(); const h = harness({ pointer }); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0], audio = h.captures[0], before = ws.sent.length; ws.bufferedAmount = 300000; pointer.moveTo(300, 200); h.c.input.holdPointer(); h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); const old = audio.metadata; h.c.input.setMicrophoneEnabled(false); h.c.sendTextCommand('unsent typed override'); assert.equal( h.c.replyPending, false, 'queued text creates no phantom retired reply', ); pointer.moveTo(700, 200); h.c.input.holdPointer(); h.c.input.bargeIn(); h.c.input.setMicrophoneEnabled(true); h.c.input.setMicrophoneEnabled(false); audio.callbacks.onAudioStart(old); audio.callbacks.onAudioEnd(old); assert.equal( h.c.pointerSentKey, null, 'queued context is not consumed before wire delivery', ); ws.bufferedAmount = 0; h.advance(25); const frames = ws.sent.slice(before); assert.equal(JSON.stringify(frames).includes('Aircraft 300'), false); assert.equal(JSON.stringify(frames).includes('unsent typed override'), false); assert.equal( JSON.parse(frames[0].clientContent.turns[0].parts[0].text).target.label, 'Aircraft 700', ); assert.deepEqual(frames.slice(1), [ { realtimeInput: { activityStart: {} } }, { realtimeInput: { activityEnd: {} } }, ]); assert.equal(h.c.forwardedActivityOpen, false); h.c.stop(); pointer.destroy(); }); for (const cancellation of ['new-turn', 'server-call']) { test(`a queued tool response is revoked by ${cancellation} before pressure clears`, async () => { const h = harness(); await h.c.start(); const ws = h.sockets[0], before = ws.sent.length; ws.bufferedAmount = 300000; ws.message({ toolCall: { functionCalls: [ { id: 'queued-old', name: 'get_current_view_state', args: {} }, ], }, }); await h.c.toolQueue; assert.equal(h.c.controlQueue.length, 1); if (cancellation === 'new-turn') h.c.sendTextCommand('new owned input'); else ws.message({ toolCallCancellation: { ids: ['queued-old'] } }); ws.bufferedAmount = 0; h.advance(25); assert.equal( ws.sent.slice(before).some((frame) => frame.toolResponse), false, ); h.c.stop(); }); } test('reliable pressure has an absolute deadline and Stop releases its poll without successor effects', async () => { for (const mode of ['deadline', 'stop']) { const h = harness(); await h.c.start(); const ws = h.sockets[0]; ws.bufferedAmount = 300000; h.c.send({ realtimeInput: { text: 'reliable authored context' } }); h.advance(4975); assert.equal(h.c.status, 'listening'); if (mode === 'stop') { h.c.stop(); await h.c.start(); } h.advance(25); assert.equal(h.c.status, mode === 'deadline' ? 'error' : 'listening'); assert.equal(h.c.controlQueue.length, 0); assert.equal(h.timers.size, 0); h.c.stop(); } }); test('reliable queue is byte and message bounded without partially sending oversized frames', async () => { for (const kind of ['bytes', 'messages']) { const h = harness(); await h.c.start(); const ws = h.sockets[0], before = ws.sent.length; ws.bufferedAmount = 300000; if (kind === 'bytes') { assert.throws( () => h.c.send({ realtimeInput: { text: 'x'.repeat(1024 * 1024) } }), /queue limit/, ); } else { for (let i = 0; i < 64; i++) assert.equal(h.c.send({ realtimeInput: { text: String(i) } }), true); assert.throws( () => h.c.send({ realtimeInput: { text: 'overflow' } }), /queue limit/, ); } assert.equal(ws.sent.length, before); h.c.stop(); assert.equal(h.timers.size, 0); } }); test('goAway permits the current tool/reply/audio drain and denies successor input', async () => { const tool = deferred(); const h = harness({ runAction: () => tool.promise }); await h.c.start(); const ws = h.sockets[0], audio = h.captures[0]; h.c.sendTextCommand('read the view'); ws.message({ toolCall: { functionCalls: [{ id: 'current', name: 'get_entity_context', args: {} }], }, }); await tick(); ws.message({ goAway: { timeLeft: '5s' } }); assert.equal(h.c.goAway.deadline, 5000); assert.equal(h.tracks[0].enabled, false); assert.equal(h.c.sendTextCommand('successor is disallowed'), false); assert.equal(h.c.bargeIn(), false); h.c.input.setMicrophoneEnabled(true); assert.equal(h.tracks[0].enabled, false); tool.resolve({ ok: true, action: 'get_entity_context' }); await h.c.toolQueue; assert.equal(ws.sent.at(-1).toolResponse.functionResponses[0].id, 'current'); ws.message({ serverContent: { ...replyAudio('accepted reply'), outputTranscription: { text: 'Current result' }, turnComplete: true, }, }); assert.equal(h.c.isActive(), true); assert.equal(audio.pending, true); audio.drain(); assert.equal(h.c.status, 'error'); assert.equal(ws.closed, true); assert.equal(h.timers.size, 0); assert.equal( h.events.findLast((event) => event.type === 'completion').audioDrained, true, ); h.c.stop(); }); test('goAway chooses the earliest bounded deadline and Stop cancels it for a fresh epoch', async () => { const h = harness(); await h.c.start(); const ws = h.sockets[0]; ws.message({ serverContent: replyAudio('playing') }); ws.message({ goAway: { timeLeft: '1.250s' } }); h.advance(250); ws.message({ goAway: { timeLeft: '500s' } }); assert.equal(h.c.goAway.deadline, 1250); ws.message({ goAway: { timeLeft: '0.5s' } }); assert.equal(h.c.goAway.deadline, 750); h.c.stop(); await h.c.start(); h.advance(1000); assert.equal(h.c.status, 'listening'); assert.equal(h.sockets.length, 2); assert.equal(h.c.goAway, null); h.c.stop(); }); for (const duration of ['bad', '-1s', 'Infinitys', '10000000s', null]) { test(`malformed goAway timeLeft ${duration} closes safely without an unbounded timer`, async () => { const h = harness(); await h.c.start(); h.sockets[0].message({ serverContent: replyAudio('playing') }); h.sockets[0].message({ goAway: { timeLeft: duration } }); assert.equal(h.c.status, 'error'); assert.equal(h.timers.size, 0); }); } test('close diagnostics preserve bounded code/category but never provider reason or credential text', async () => { const h = harness(); await h.c.start(); h.sockets[0].onclose({ code: 1008, reason: 'auth_tokens/private-secret user-private-text', }); assert.deepEqual(h.c.getDiagnostics().close, { code: 1008, category: 'policy', }); assert.doesNotMatch( JSON.stringify(h.c.getDiagnostics()), /private-secret|user-private-text/, ); await h.c.start(); h.sockets[1].onclose({ code: Infinity, reason: 'private' }); assert.deepEqual(h.c.getDiagnostics().close, { code: null, category: 'connection', }); h.c.stop(); }); test('goAway never flushes an unsent successor at the retired response boundary', async () => { const h = harness(); await h.c.start(); const ws = h.sockets[0]; ws.message({ serverContent: replyAudio('retired previous') }); h.c.sendTextCommand('already forwarded successor'); h.c.sendTextCommand('unsent newest successor'); assert.ok(h.c.deferredInput); const before = ws.sent.length; ws.message({ goAway: { timeLeft: '5s' } }); ws.message({ serverContent: { interrupted: true, turnComplete: true } }); assert.equal( ws.sent.slice(before).some((message) => message.clientContent), false, ); assert.equal(h.c.status, 'error'); assert.equal(h.c.deferredInput, null); assert.equal(h.timers.size, 0); h.c.stop(); }); test('permission rejection opens no token/socket and initialized audio is released', async () => { const h = harness({ getUserMedia: async () => { throw new DOMException('not allowed', 'NotAllowedError'); }, }); await h.c.start(); assert.equal(h.c.status, 'error'); assert.equal(h.requests.length, 0); assert.equal(h.sockets.length, 0); assert.equal(h.captures[0].closed, true); assert.match(h.c.lastError, /permission was denied/); }); test('consented microphone stays muted through the network phase and its deadline releases tracks', async () => { const pending = deferred(); const h = harness({ startupTimeoutMs: 20, fetchImpl: () => pending.promise }); const start = h.c.start(); await tick(); assert.equal(h.tracks[0].enabled, false); await Promise.all([start, new Promise((resolve) => setTimeout(resolve, 40))]); assert.equal(h.c.status, 'error'); assert.equal(h.tracks[0].readyState, 'ended'); assert.equal(h.captures[0].closed, true); pending.resolve(Response.json(credential)); await tick(); assert.equal(h.sockets.length, 0); h.c.stop(); }); test('valid huge goAway duration is clamped to a bounded timer', async () => { const h = harness(); await h.c.start(); h.sockets[0].message({ serverContent: replyAudio('playing') }); h.sockets[0].message({ goAway: { timeLeft: '999999s' } }); assert.equal(h.c.goAway.deadline, 600000); h.c.stop(); assert.equal(h.timers.size, 0); }); test('cancelling a pressure-queued response retains passive map context for the next owned text', async () => { const h = harness(); await h.c.start(); const ws = h.sockets[0]; h.c.notifyMapEvent({ type: 'map_annotation_outline', label: 'Austin' }); ws.bufferedAmount = 300000; ws.message({ toolCall: { functionCalls: [ { id: 'old-map-envelope', name: 'get_current_view_state', args: {} }, ], }, }); await h.c.toolQueue; assert.equal( h.c.pendingMapEvents.length, 1, 'unsent data remains owned by the context queue', ); ws.message({ toolCallCancellation: { ids: ['old-map-envelope'] } }); h.c.sendTextCommand('use the map context'); ws.bufferedAmount = 0; h.advance(25); const frame = ws.sent.at(-1); assert.equal( JSON.parse(frame.clientContent.turns[0].parts[0].text).events[0].label, 'Austin', ); assert.equal(h.c.pendingMapEvents.length, 0); h.c.stop(); }); test('an expired goAway deadline rejects late output before the timer callback can run', async () => { let now = 0; const h = harness({ now: () => now }); await h.c.start(); const ws = h.sockets[0]; h.c.sendTextCommand('current request'); ws.message({ serverContent: replyAudio('initial output') }); ws.message({ goAway: { timeLeft: '1s' } }); now = 1000; ws.message({ serverContent: { ...replyAudio('too late'), turnComplete: true }, }); assert.equal(h.c.status, 'error'); assert.equal(h.captures[0].played.length, 1); assert.equal( h.events.some((event) => event.type === 'completion'), false, ); assert.equal(h.timers.size, 0); }); function guardedNativeHarness(overrides = {}) { const mutations = []; const observed = []; const guarded = withToolCatalog( async (name, args, options) => { mutations.push({ name, args, options }); return { ok: true, action: name }; }, async () => null, ); const h = harness({ ...overrides, runAction: async (name, args, options) => { observed.push({ name, args, options }); return guarded(name, args, options); }, }); return { ...h, mutations, observed }; } for (const stage of ['deferred', 'flushed']) { test(`Sameh P3-1: ${stage} retirement deadline uses the injected clock and releases timers`, async () => { const h = harness({ inputBoundaryTimeoutMs: 60 }); try { await h.c.start(); const ws = h.sockets[0]; ws.message({ serverContent: replyAudio('old') }); h.c.sendTextCommand('first successor'); h.c.sendTextCommand('queued successor'); assert.equal(h.c.deferredInput.deadline, 60); h.advance(15); h.c.sendTextCommand('replacement'); assert.equal(h.c.deferredInput.deadline, 60); if (stage === 'flushed') { ws.message({ serverContent: { interrupted: true, turnComplete: true }, }); assert.equal(h.c.retiredBoundaryWait.deadline, 60); } h.advance(44); assert.notEqual(h.c.status, 'error'); h.advance(1); assert.equal(h.c.status, 'error'); assert.equal(ws.closed, true); assert.equal(h.timers.size, 0); } finally { h.c.stop(); } }); } for (const exit of ['stop', 'boundary']) { test(`Sameh P3-1: ${exit} clears an injected retirement deadline before replacement`, async () => { const h = harness({ inputBoundaryTimeoutMs: 60 }); try { await h.c.start(); const ws = h.sockets[0]; ws.message({ serverContent: replyAudio('old') }); h.c.sendTextCommand('first successor'); h.c.sendTextCommand('queued successor'); const deadline = h.c.deferredInput.deadline; h.advance(15); h.c.sendTextCommand('newest successor'); assert.equal(h.c.deferredInput.deadline, deadline); ws.message({ serverContent: { interrupted: true, turnComplete: true } }); assert.equal(h.c.retiredBoundaryWait.deadline, deadline); assert.equal( [...h.timers.values()].some((timer) => timer.at === 60), true, 'retirement wait must belong to the injected timer registry', ); if (exit === 'stop') h.c.stop(); else { ws.message({ serverContent: { interrupted: true, turnComplete: true }, }); ws.message({ serverContent: { turnComplete: true } }); } assert.equal( [...h.timers.values()].some((timer) => timer.at === 60), false, ); h.advance(100); assert.equal(h.c.status, exit === 'stop' ? 'idle' : 'listening'); assert.equal(Boolean(ws.closed), exit === 'stop'); } finally { h.c.stop(); } }); } for (const segments of [['use foo as my voice provider', 'and zoom out']]) { test(`native advisory caption text ${JSON.stringify(segments)} releases tools`, async () => { const h = guardedNativeHarness(); try { await h.c.start(); const ws = h.sockets[0]; ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); ws.message({ voiceActivity: { type: 'ACTIVITY_END' } }); for (const text of segments) ws.message({ serverContent: { inputTranscription: { text } } }); requestTool(ws, 'ordinary-port', 'zoom_to_globe', {}); await h.c.toolQueue; assert.equal(h.mutations.length, 1); } finally { h.c.stop(); } assert.equal(h.timers.size, 0); }); } const zoomCall = (id) => ({ toolCall: { functionCalls: [ { id, name: 'adjust_camera_zoom', args: { direction: 'in' } }, ], }, }); const toolResponses = (ws, from = 0) => ws.sent .slice(from) .flatMap((message) => message.toolResponse?.functionResponses || []); async function completedPttTurn() { const h = harness(); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; establishToolInput(h); ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } }); h.captures[0].drain(); await tick(); return { h, ws }; } test('push-to-talk refuses an unrequested tool call and every call in its reply', async () => { const { h, ws } = await completedPttTurn(); const sent = ws.sent.length; ws.message(zoomCall('unrequested')); ws.message(zoomCall('follow-up')); await h.c.toolQueue; assert.deepEqual(h.calls, []); assert.deepEqual( toolResponses(ws, sent).map(({ id, response }) => [id, response.ok]), [ ['unrequested', false], ['follow-up', false], ], ); ws.message({ serverContent: { turnComplete: true } }); establishToolInput(h, 'Zoom in'); // A refused ID stays spent; only the newly requested call runs. ws.message(zoomCall('unrequested')); ws.message(zoomCall('requested')); await h.c.toolQueue; assert.deepEqual( h.calls.map((call) => call.name), ['adjust_camera_zoom'], ); assert.equal(toolResponses(ws, sent).length, 3); h.c.stop(); }); test('push-to-talk refuses a call after unprompted speech that started the reply', async () => { const { h, ws } = await completedPttTurn(); const sent = ws.sent.length; ws.message({ serverContent: { outputTranscription: { text: 'Let me zoom in' } }, }); ws.message(zoomCall('after-speech')); await h.c.toolQueue; assert.deepEqual(h.calls, []); assert.deepEqual( toolResponses(ws, sent).map(({ id, response }) => [id, response.ok]), [['after-speech', false]], ); h.c.stop(); }); test('a refusal does not hold back the completion of the reply still playing', async () => { const h = harness(); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; establishToolInput(h); ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } }); ws.message(zoomCall('unrequested')); h.captures[0].drain(); await tick(); assert.equal(h.events.filter((e) => e.type === 'completion').length, 1); assert.deepEqual(h.calls, []); h.c.stop(); }); test('open microphone still runs a call Gemini makes without a local boundary', async () => { const h = harness(); await h.c.start(); h.sockets[0].message(zoomCall('server-detected')); await h.c.toolQueue; assert.deepEqual( h.calls.map((call) => call.name), ['adjust_camera_zoom'], ); h.c.stop(); }); test('a late caption after a completed push-to-talk turn cannot retire the next request', async () => { const { h, ws } = await completedPttTurn(); ws.message({ serverContent: { outputTranscription: { text: ' done.' } } }); establishToolInput(h, 'Zoom in'); ws.message(zoomCall('requested')); await h.c.toolQueue; h.advance(5_000); assert.deepEqual( h.calls.map((call) => call.name), ['adjust_camera_zoom'], ); assert.equal(h.c.lastError, null); h.c.stop(); }); test('an open-mic reply that starts with a caption is retired by newer typed input', async () => { const h = harness(); await h.c.start(); const ws = h.sockets[0]; establishToolInput(h); ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } }); h.captures[0].drain(); await tick(); ws.message({ serverContent: { outputTranscription: { text: 'Zooming' } } }); establishToolInput(h, 'Replacement request'); ws.message(zoomCall('earlier-reply')); ws.message({ serverContent: { turnComplete: true } }); await h.c.toolQueue; assert.deepEqual(h.calls, []); h.c.stop(); }); test('a refused reply boundary that arrives after new push-to-talk input cannot close it', async () => { const { h, ws } = await completedPttTurn(); const audio = h.captures[0]; ws.message(zoomCall('unrequested')); h.c.input.pushToTalkKeyHeld = true; h.c.input.setMicrophoneEnabled(true); audio.callbacks.onAudio( { data: 'AAAB', mimeType: 'audio/pcm;rate=16000' }, audio.metadata, ); h.c.input.pushToTalkKeyHeld = false; h.c.input.setMicrophoneEnabled(false); const completions = h.events.filter((e) => e.type === 'completion').length; ws.message({ serverContent: { turnComplete: true } }); assert.equal( h.events.filter((e) => e.type === 'completion').length, completions, ); ws.message(zoomCall('requested')); await h.c.toolQueue; assert.deepEqual( h.calls.map((call) => call.name), ['adjust_camera_zoom'], ); h.c.stop(); }); test('push-to-talk admits a call made while an earlier requested tool is still running', async () => { const running = deferred(); const h = harness({ runAction: async (name) => { h.calls.push({ name }); if (h.calls.length === 1) await running.promise; return { ok: true, action: name }; }, }); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; establishToolInput(h, 'Zoom in'); ws.message(zoomCall('first')); await tick(); ws.message({ serverContent: { turnComplete: true } }); ws.message(zoomCall('second')); running.resolve(); await h.c.toolQueue; assert.equal(h.calls.length, 2); assert.equal( toolResponses(ws).some(({ response }) => response.ok === false), false, ); h.c.stop(); }); test('a narration callback that submits a successor keeps the old packet tools from running', async () => { const h = harness(); await h.c.start({ pushToTalk: true }); const ws = h.sockets[0]; establishToolInput(h, 'Zoom in'); const audible = h.c.narration.audible.bind(h.c.narration); let submitted = false; h.c.narration.audible = () => { audible(); if (submitted) return; submitted = true; assert.equal(h.c.sendTextCommand('New request'), true); }; ws.message({ serverContent: replyAudio('old'), toolCall: { functionCalls: [ { id: 'old', name: 'adjust_camera_zoom', args: { direction: 'in' } }, ], }, }); await h.c.toolQueue; assert.equal(submitted, true); assert.deepEqual(h.calls, []); h.c.stop(); }); for (const reply of ['typed', 'push-to-talk', 'open-mic']) { test(`globe refusal blocks queued and later tools until fresh ${reply} input`, async () => { const names = []; const h = harness({ runAction: async (name) => { names.push(name); return name === 'move_camera' ? { ok: false, action: 'move_camera', reason: 'globe-view-required' } : { ok: true, action: name }; }, }); try { await h.c.start({ pushToTalk: reply === 'push-to-talk' }); establishToolInput(h, 'rotate the globe'); const ws = h.sockets[0]; ws.message({ toolCall: { functionCalls: [ { id: 'rotate', name: 'move_camera', args: { motion: 'rotate_globe' }, }, { id: 'queued-reset', name: 'zoom_to_globe', args: {} }, ], }, }); await h.c.toolQueue; assert.deepEqual(names, ['move_camera']); const blocked = ws.sent .flatMap((entry) => entry.toolResponse?.functionResponses || []) .find((call) => call.id === 'queued-reset'); assert.equal(blocked.response.ok, false); assert.match(blocked.response.error, /fresh user input/i); assert.equal(blocked.response.skipped, true); assert.match(blocked.response.responseGuidance, /did not run/i); assert.doesNotMatch( blocked.response.responseGuidance, /confirm.*completed/i, ); requestTool(ws, 'repeated-rotation', 'move_camera', { motion: 'rotate_globe', }); await h.c.toolQueue; assert.deepEqual(names, ['move_camera']); const repeated = toolResponses(ws).find( (call) => call.id === 'repeated-rotation', ).response; assert.equal(repeated.ok, false); assert.equal(repeated.skipped, true); assert.match(repeated.responseGuidance, /did not run/i); assert.doesNotMatch( repeated.responseGuidance, /confirm.*completed/i, ); if (reply === 'typed') { const stimuli = [ () => {}, () => ws.message({ serverContent: { interrupted: true } }), () => ws.message({ serverContent: { inputTranscription: { text: 'reset to globe view' }, }, }), () => { h.c.input.setMicrophoneEnabled(false); h.c.input.setMicrophoneEnabled(true); }, ]; for (const [i, stimulus] of stimuli.entries()) { stimulus(); requestTool(ws, `unrequested-reset-${i}`, 'zoom_to_globe', {}); await h.c.toolQueue; assert.deepEqual(names, ['move_camera']); } } // Finish the refusal reply before admitting the actual next user input. ws.message({ serverContent: { turnComplete: true } }); if (reply === 'typed') assert.equal(h.c.sendTextCommand('reset to globe view'), true); else if (reply === 'push-to-talk') { h.c.input.pushToTalkKeyHeld = true; h.c.input.spaceKeyHeld = true; h.c.input.setMicrophoneEnabled(true); h.c.input.setMicrophoneEnabled(false); } else ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); requestTool(ws, 'accepted-reset', 'zoom_to_globe', {}); await h.c.toolQueue; assert.deepEqual(names, ['move_camera', 'zoom_to_globe']); } finally { h.c.stop(); } }); } test('a cancelled globe reset skips queued rotation, preserves Stop and recovers on fresh input', async () => { const calls = []; let resets = 0; const h = harness({ runAction: async (name, args) => { calls.push(name === 'move_camera' ? args.motion : name); return name === 'zoom_to_globe' && resets++ === 0 ? { ok: false, action: name, cancelled: true } : { ok: true, action: name }; }, }); try { await h.c.start(); establishToolInput(h, 'reset to globe view and rotate'); const ws = h.sockets[0]; ws.message({ toolCall: { functionCalls: [ { id: 'reset-failed', name: 'zoom_to_globe', args: {} }, { id: 'rotate-skipped', name: 'move_camera', args: { motion: 'rotate_globe' }, }, { id: 'stop-allowed', name: 'move_camera', args: { motion: 'stop' } }, ], }, }); await h.c.toolQueue; assert.deepEqual(calls, ['zoom_to_globe', 'stop']); const skipped = toolResponses(ws).find( (call) => call.id === 'rotate-skipped', ).response; assert.equal(skipped.ok, false); assert.equal(skipped.reason, 'globe-reset-failed'); assert.equal(skipped.skipped, true); assert.equal( toolResponses(ws).find((call) => call.id === 'stop-allowed').response.ok, true, ); ws.message({ serverContent: { turnComplete: true } }); establishToolInput(h, 'reset to globe view and rotate'); ws.message({ toolCall: { functionCalls: [ { id: 'reset-success', name: 'zoom_to_globe', args: {} }, { id: 'rotate-success', name: 'move_camera', args: { motion: 'rotate_globe' }, }, ], }, }); await h.c.toolQueue; assert.deepEqual(calls, [ 'zoom_to_globe', 'stop', 'zoom_to_globe', 'rotate_globe', ]); } finally { h.c.stop(); } }); test('cancelling a queued globe reset also skips its dependent rotation', async () => { const blocker = deferred(); const calls = []; const h = harness({ runAction: async (name, args) => { calls.push(name === 'move_camera' ? args.motion : name); return args.motion === 'pan' ? blocker.promise : { ok: true, action: name }; }, }); try { await h.c.start(); establishToolInput(h, 'pan, reset to globe view and rotate'); const ws = h.sockets[0]; ws.message({ toolCall: { functionCalls: [ { id: 'blocker', name: 'move_camera', args: { motion: 'pan', direction: 'left' }, }, { id: 'reset-cancelled', name: 'zoom_to_globe', args: {} }, { id: 'rotate-after-cancel', name: 'move_camera', args: { motion: 'rotate_globe' }, }, ], }, }); await tick(); ws.message({ toolCallCancellation: { ids: ['reset-cancelled'] } }); blocker.resolve({ ok: true, action: 'move_camera' }); await h.c.toolQueue; assert.deepEqual(calls, ['pan']); const skipped = toolResponses(ws).find( (call) => call.id === 'rotate-after-cancel', ).response; assert.equal(skipped.reason, 'globe-reset-failed'); assert.equal(skipped.skipped, true); } finally { blocker.resolve({ ok: true }); h.c.stop(); } });