mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-04 10:24:02 +00:00
Add continuous globe rotation with direction and speed controls, viewport fit checks, camera ownership handoff, and voice refusal guards.
3613 lines
116 KiB
JavaScript
3613 lines
116 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { geminiSocketUrl } from './geminiSession.js';
|
|
import { createVoiceSession } from './session.js';
|
|
import { createPointerTracker } from './pointerContext.js';
|
|
import { withToolCatalog } from './gevRealtime.js';
|
|
import {
|
|
harness,
|
|
tick,
|
|
deferred,
|
|
} from '../testSupport/geminiNativeHarness.mjs';
|
|
|
|
const endpoint =
|
|
'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained';
|
|
|
|
const credential = {
|
|
token: 'auth_tokens/test',
|
|
model: 'gemini-3.8-live',
|
|
websocketUrl: endpoint,
|
|
config: { generationConfig: { responseModalities: ['AUDIO'] } },
|
|
};
|
|
|
|
const manualCredential = {
|
|
...credential,
|
|
config: {
|
|
...credential.config,
|
|
realtimeInputConfig: { automaticActivityDetection: { disabled: true } },
|
|
},
|
|
};
|
|
|
|
test('ephemeral credentials are sent only to the fixed constrained Google endpoint', () => {
|
|
const url = new URL(geminiSocketUrl(credential));
|
|
assert.equal(url.searchParams.get('access_token'), credential.token);
|
|
for (const websocketUrl of [
|
|
endpoint + '?key=foo',
|
|
endpoint.replace('googleapis.com', 'example.com'),
|
|
endpoint.replace('wss:', 'ws:'),
|
|
endpoint.replace('Constrained', ''),
|
|
endpoint.replace('wss://', 'wss://user@'),
|
|
])
|
|
assert.throws(
|
|
() => geminiSocketUrl({ ...credential, websocketUrl }),
|
|
/endpoint/,
|
|
);
|
|
assert.throws(
|
|
() =>
|
|
geminiSocketUrl({ ...credential, token: 'AIza-not-an-ephemeral-token' }),
|
|
/ephemeral/,
|
|
);
|
|
});
|
|
|
|
test('PTT locks manual activity and serializes cold and repeated held turns', async () => {
|
|
const h = harness();
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
h.c.input.spaceKeyHeld = true;
|
|
await h.c.start({ pushToTalk: true });
|
|
assert.equal(h.requests[0].body, '{"inputMode":"push-to-talk"}');
|
|
const ws = h.sockets[0];
|
|
assert.deepEqual(ws.sent[0].setup.realtimeInputConfig, {
|
|
automaticActivityDetection: { disabled: true },
|
|
});
|
|
const audio = h.captures[0];
|
|
const pcm = { data: 'AAABAA==', mimeType: 'audio/pcm;rate=16000' };
|
|
audio.callbacks.onAudioStart(audio.metadata); // A duplicate notification cannot reopen.
|
|
audio.callbacks.onAudio(pcm, audio.metadata);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
audio.callbacks.onAudioEnd(audio.metadata);
|
|
audio.callbacks.onAudio(pcm, audio.metadata); // Unbracketed/stale PCM is not forwarded.
|
|
assert.equal(h.tracks[0].enabled, false);
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
audio.callbacks.onAudio(pcm, audio.metadata);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
assert.deepEqual(ws.sent.slice(1), [
|
|
{ realtimeInput: { activityStart: {} } },
|
|
{ realtimeInput: { audio: pcm } },
|
|
{ realtimeInput: { activityEnd: {} } },
|
|
{ realtimeInput: { activityStart: {} } },
|
|
{ realtimeInput: { audio: pcm } },
|
|
{ realtimeInput: { activityEnd: {} } },
|
|
]);
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.c.input.pushToTalkMode, true);
|
|
assert.equal(h.tracks[0].enabled, false);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('ordinary microphone keeps its default setup and automatic stream-end message', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
assert.equal(h.requests[0].body, '{}');
|
|
assert.deepEqual(h.sockets[0].sent, [
|
|
{ setup: { ...credential.config, model: 'models/gemini-3.8-live' } },
|
|
]);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
assert.deepEqual(h.sockets[0].sent.at(-1), {
|
|
realtimeInput: { audioStreamEnd: true },
|
|
});
|
|
h.c.stop();
|
|
});
|
|
|
|
for (const phase of ['token', 'setup', 'microphone']) {
|
|
test(`PTT release during ${phase} startup stays muted until a later hold`, async () => {
|
|
const pending = deferred();
|
|
const h = harness({
|
|
...(phase === 'token' ? { fetchImpl: () => pending.promise } : {}),
|
|
...(phase === 'setup' ? { autoSetup: false } : {}),
|
|
...(phase === 'microphone'
|
|
? { getUserMedia: () => pending.promise }
|
|
: {}),
|
|
});
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
h.c.input.spaceKeyHeld = true;
|
|
const start = h.c.start({ pushToTalk: true });
|
|
await tick();
|
|
assert.equal(h.media.length, 0);
|
|
assert.equal(
|
|
h.sockets[0]?.sent.filter((value) => value.realtimeInput).length || 0,
|
|
0,
|
|
);
|
|
h.c.input.pushToTalkKeyHeld = false;
|
|
h.c.input.spaceKeyHeld = false;
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
if (phase === 'token') pending.resolve(Response.json(manualCredential));
|
|
else if (phase === 'setup') h.sockets[0].message({ setupComplete: {} });
|
|
else pending.resolve(h.makeStream());
|
|
await start;
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.tracks[0].enabled, false);
|
|
assert.equal(h.media[0].enabled, false);
|
|
assert.equal(
|
|
h.sockets[0].sent.length,
|
|
1,
|
|
'no phantom activity for a released gesture',
|
|
);
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
assert.deepEqual(h.sockets[0].sent.at(-1), {
|
|
realtimeInput: { activityStart: {} },
|
|
});
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
assert.deepEqual(h.sockets[0].sent.at(-1), {
|
|
realtimeInput: { activityEnd: {} },
|
|
});
|
|
h.c.stop();
|
|
});
|
|
}
|
|
|
|
test('a token config that disagrees with the gesture fails before socket creation and releases consented microphone', async () => {
|
|
for (const [pushToTalk, supplied] of [
|
|
[true, credential],
|
|
[false, manualCredential],
|
|
]) {
|
|
const h = harness({
|
|
fetchImpl: async () => Response.json(supplied),
|
|
});
|
|
await h.c.start({ pushToTalk });
|
|
assert.equal(h.c.status, 'error');
|
|
assert.match(h.c.lastError, /input mode/);
|
|
assert.equal(h.sockets.length, 0);
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(h.captures[0].closed, true);
|
|
}
|
|
});
|
|
|
|
test('old PTT callbacks cannot send to a stopped or replacement session', async () => {
|
|
const h = harness();
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
await h.c.start({ pushToTalk: true });
|
|
const old = h.captures[0].callbacks;
|
|
const socket = h.sockets[0];
|
|
h.c.stop();
|
|
const oldCount = socket.sent.length;
|
|
await h.c.start();
|
|
const replacementCount = h.sockets[1].sent.length;
|
|
old.onAudioStart(h.captures[0].metadata);
|
|
old.onAudio(
|
|
{ data: 'AAAA', mimeType: 'audio/pcm;rate=16000' },
|
|
h.captures[0].metadata,
|
|
);
|
|
old.onAudioEnd(h.captures[0].metadata);
|
|
assert.equal(socket.sent.length, oldCount);
|
|
assert.equal(h.sockets[1].sent.length, replacementCount);
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(h.tracks[1].enabled, true);
|
|
h.c.stop({ removeUi: true });
|
|
const latest = h.captures[1].callbacks;
|
|
latest.onAudioStart(h.captures[1].metadata);
|
|
latest.onAudioEnd(h.captures[1].metadata);
|
|
assert.equal(h.sockets[1].sent.length, replacementCount);
|
|
});
|
|
|
|
test('session waits for setup, captures PCM, plays all parts, emits transcripts and releases every resource', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.media.length, 1);
|
|
const ws = h.sockets[0];
|
|
assert.equal(ws.sent[0].setup.model, 'models/gemini-3.8-live');
|
|
h.captures[0].callbacks.onAudio(
|
|
{
|
|
data: 'AAAA',
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
h.captures[0].metadata,
|
|
);
|
|
assert.equal(ws.sent.at(-1).realtimeInput.audio.data, 'AAAA');
|
|
ws.message({
|
|
serverContent: {
|
|
inputTranscription: { text: 'Hello' },
|
|
outputTranscription: { text: 'Hi' },
|
|
modelTurn: {
|
|
parts: [
|
|
{ inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } },
|
|
{ inlineData: { data: 'AQAB', mimeType: 'audio/pcm;rate=24000' } },
|
|
],
|
|
},
|
|
turnComplete: true,
|
|
},
|
|
});
|
|
assert.equal(h.captures[0].played.length, 2);
|
|
assert.equal(
|
|
h.events.some((event) => event.type === 'completion'),
|
|
false,
|
|
'server turn completion waits for queued audio to drain',
|
|
);
|
|
assert.ok(
|
|
h.events.some(
|
|
(event) =>
|
|
event.type === 'transcript' &&
|
|
event.role === 'assistant' &&
|
|
event.final &&
|
|
event.text === 'Hi',
|
|
),
|
|
);
|
|
assert.equal(h.c.getDiagnostics().completedTurns, 1);
|
|
h.captures[0].drain();
|
|
assert.equal(
|
|
h.events.some(
|
|
(event) => event.type === 'completion' && event.status === 'completed',
|
|
),
|
|
true,
|
|
);
|
|
h.c.stop();
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(ws.closed, true);
|
|
assert.equal(h.captures[0].closed, true);
|
|
assert.equal(h.state.ducked, false);
|
|
assert.equal(h.c.status, 'idle');
|
|
});
|
|
|
|
test('goAway deadline tears down pending work and an explicit restart creates a fresh conversation', async () => {
|
|
const pending = deferred();
|
|
let issued = 0;
|
|
let pendingOptions;
|
|
const h = harness({
|
|
fetchImpl: async () =>
|
|
Response.json({
|
|
...credential,
|
|
token: `auth_tokens/session-${++issued}`,
|
|
}),
|
|
runAction: async (name, _args, options) => {
|
|
if (name === 'control_radio')
|
|
return {
|
|
ok: true,
|
|
action: 'control_radio',
|
|
radioAction: 'play',
|
|
radioPlaybackRequested: true,
|
|
};
|
|
pendingOptions = options;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const oldSocket = h.sockets[0];
|
|
const oldAudio = h.captures[0];
|
|
const oldCallbacks = oldAudio.callbacks;
|
|
oldSocket.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'radio', name: 'control_radio', args: { action: 'play' } },
|
|
{ id: 'pending', name: 'get_current_view_state', args: {} },
|
|
],
|
|
},
|
|
});
|
|
await tick();
|
|
await tick();
|
|
assert.ok(h.c.radio.pendingRadioPlaybackResult);
|
|
assert.equal(h.c.pendingTools, 1);
|
|
oldSocket.message({
|
|
serverContent: {
|
|
modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] },
|
|
},
|
|
});
|
|
assert.equal(oldAudio.played.length, 1);
|
|
let sentBeforeExpiry = oldSocket.sent.length;
|
|
|
|
oldSocket.message({ goAway: { timeLeft: '5s' } });
|
|
sentBeforeExpiry = oldSocket.sent.length; // Microphone stream-end is a reliable lifecycle marker.
|
|
assert.equal(h.c.status, 'executing');
|
|
assert.equal(pendingOptions.signal.aborted, false);
|
|
assert.equal(h.c.sendTextCommand('Do not admit a new turn'), false);
|
|
h.advance(5000);
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(
|
|
h.c.lastError,
|
|
'Gemini Live connection is expiring. Start voice again for a new conversation.',
|
|
);
|
|
assert.equal(
|
|
h.events.at(-1).recovery,
|
|
'Start voice again for a fresh token and a new conversation.',
|
|
);
|
|
assert.equal(oldSocket.closed, true);
|
|
assert.equal(oldAudio.closed, true);
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(h.c.pendingTools, 0);
|
|
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
|
assert.equal(pendingOptions.signal.aborted, true);
|
|
|
|
oldCallbacks.onAudioStart(h.captures[0].metadata);
|
|
oldCallbacks.onAudio(
|
|
{ data: 'AQAB', mimeType: 'audio/pcm;rate=16000' },
|
|
h.captures[0].metadata,
|
|
);
|
|
oldCallbacks.onAudioEnd(h.captures[0].metadata);
|
|
oldCallbacks.onDrain();
|
|
oldCallbacks.onError(new Error('late retired audio'));
|
|
oldSocket.message({ serverContent: { turnComplete: true } });
|
|
oldSocket.error();
|
|
oldSocket.remoteClose();
|
|
pending.resolve({ ok: true });
|
|
await tick();
|
|
await tick();
|
|
assert.equal(oldSocket.sent.length, sentBeforeExpiry);
|
|
assert.equal(oldAudio.played.length, 1);
|
|
assert.equal(
|
|
oldSocket.sent.filter((entry) => entry.toolResponse).length,
|
|
1,
|
|
'only the completed Radio result was sent before expiry',
|
|
);
|
|
assert.equal(h.state.playing, 0);
|
|
|
|
await h.c.start();
|
|
assert.equal(issued, 2);
|
|
assert.equal(h.sockets.length, 2);
|
|
assert.equal(
|
|
new URL(h.sockets[1].url).searchParams.get('access_token'),
|
|
'auth_tokens/session-2',
|
|
);
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.c.completedTurns, 0);
|
|
assert.equal(h.captures[1].played.length, 0);
|
|
assert.equal(h.tracks[1].readyState, 'live');
|
|
h.c.stop();
|
|
});
|
|
|
|
test('pre-setup rejection or close releases startup and asks for a fresh session', async () => {
|
|
for (const terminal of ['rejected', 'closed']) {
|
|
const h = harness({ autoSetup: false });
|
|
const start = h.c.start();
|
|
await tick();
|
|
if (terminal === 'rejected')
|
|
h.sockets[0].message({ error: { message: 'credential rejected' } });
|
|
else h.sockets[0].remoteClose();
|
|
await start;
|
|
assert.equal(h.c.status, 'error', terminal);
|
|
assert.equal(h.tracks[0].readyState, 'ended', terminal);
|
|
assert.equal(h.captures[0].closed, true, terminal);
|
|
assert.equal(h.sockets[0].closed, true, terminal);
|
|
assert.match(h.c.lastError, /Start voice again/, terminal);
|
|
assert.match(h.events.at(-1).recovery, /fresh session/, terminal);
|
|
assert.equal(h.sockets.length, 1, 'no automatic retry');
|
|
}
|
|
});
|
|
|
|
test('network close after setup releases media and starts no replacement', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
h.sockets[0].remoteClose();
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(
|
|
h.c.lastError,
|
|
'Gemini Live connection ended. Start voice again for a new conversation.',
|
|
);
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(h.captures[0].closed, true);
|
|
assert.equal(h.sockets.length, 1);
|
|
assert.doesNotMatch(h.events.at(-1).recovery, /microphone|API key/i);
|
|
});
|
|
|
|
test('OFF during unresolved setup blocks late setup, error and close callbacks', async () => {
|
|
const h = harness({ autoSetup: false });
|
|
const start = h.c.start();
|
|
await tick();
|
|
const socket = h.sockets[0];
|
|
h.c.stop();
|
|
const eventCount = h.events.length;
|
|
socket.message({ setupComplete: {} });
|
|
socket.message({ error: { message: 'late rejection' } });
|
|
socket.error();
|
|
socket.remoteClose();
|
|
await start;
|
|
assert.equal(h.c.status, 'idle');
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(h.captures[0].closed, true);
|
|
assert.equal(h.events.length, eventCount);
|
|
});
|
|
|
|
test('a disposed pending microphone is stopped and cannot update the replacement UI', async () => {
|
|
const microphone = deferred();
|
|
const h = harness({ getUserMedia: () => microphone.promise });
|
|
const start = h.c.start();
|
|
await tick();
|
|
h.c.stop({ removeUi: true });
|
|
const eventCount = h.events.length;
|
|
microphone.resolve(h.makeStream());
|
|
await start;
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(h.events.length, eventCount);
|
|
assert.equal(h.media.length, 0);
|
|
});
|
|
|
|
test('late token rejection after stop cannot replace a successor state', async () => {
|
|
const response = deferred();
|
|
const h = harness({ fetchImpl: () => response.promise });
|
|
const start = h.c.start();
|
|
h.c.stop({ removeUi: true });
|
|
const count = h.events.length;
|
|
response.resolve(Response.json({ error: 'late rejection' }, { status: 503 }));
|
|
await start;
|
|
assert.equal(h.events.length, count);
|
|
assert.equal(h.sockets.length, 0);
|
|
});
|
|
|
|
test('tool cancellation covers queued calls and interrupted results never return to the socket', async () => {
|
|
const first = deferred();
|
|
const names = [];
|
|
const h = harness({
|
|
runAction: async (name) => {
|
|
names.push(name);
|
|
return first.promise;
|
|
},
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'a', name: 'one', args: {} },
|
|
{ id: 'b', name: 'two', args: {} },
|
|
],
|
|
},
|
|
});
|
|
await tick();
|
|
ws.message({ toolCallCancellation: { ids: ['b'] } });
|
|
first.resolve({ ok: true });
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(names, ['one']);
|
|
assert.equal(h.c.pendingTools, 0);
|
|
assert.equal(ws.sent.filter((entry) => entry.toolResponse).length, 1);
|
|
const second = deferred();
|
|
h.c.runAction = () => second.promise;
|
|
ws.message({
|
|
toolCall: { functionCalls: [{ id: 'c', name: 'three', args: {} }] },
|
|
});
|
|
await tick();
|
|
ws.message({ serverContent: { interrupted: true } });
|
|
second.resolve({ ok: true });
|
|
await tick();
|
|
assert.equal(ws.sent.filter((entry) => entry.toolResponse).length, 1);
|
|
assert.equal(h.captures[0].cleared, true);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('wire voiceActivity.type start aborts pending work and discards old playback', async () => {
|
|
const pending = deferred();
|
|
let signal;
|
|
const h = harness({
|
|
runAction: (_name, _args, options) => {
|
|
signal = options.signal;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } },
|
|
});
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'old', name: 'fly_to_location', args: { query: 'Austin' } },
|
|
],
|
|
},
|
|
});
|
|
await tick();
|
|
const turn = h.c.turn;
|
|
assert.equal(signal.aborted, false);
|
|
assert.equal(h.captures[0].pending, true);
|
|
ws.message({
|
|
voiceActivity: { type: 'ACTIVITY_START', audioOffset: '5.240s' },
|
|
});
|
|
assert.equal(signal.aborted, true);
|
|
assert.ok(h.c.turn > turn);
|
|
assert.equal(h.c.pendingTools, 0);
|
|
assert.equal(h.captures[0].cleared, true);
|
|
assert.equal(h.c.status, 'listening');
|
|
pending.resolve({ ok: true });
|
|
await tick();
|
|
assert.equal(
|
|
ws.sent.some((entry) => entry.toolResponse),
|
|
false,
|
|
);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('unknown or malformed voice activity cannot revoke the current turn', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const turn = h.c.turn;
|
|
for (const voiceActivity of [
|
|
null,
|
|
{},
|
|
{ type: 'TYPE_UNSPECIFIED' },
|
|
{ type: 'unknown' },
|
|
{ type: 1 },
|
|
]) {
|
|
h.sockets[0].message({ voiceActivity });
|
|
assert.equal(h.c.turn, turn);
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.c.narration.state().pendingTimers, 0);
|
|
}
|
|
h.c.stop();
|
|
});
|
|
|
|
for (const pushToTalk of [false, true]) {
|
|
test(`wire voiceActivity.type end respects ${pushToTalk ? 'manual' : 'automatic'} turn ownership`, async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk });
|
|
if (!pushToTalk)
|
|
h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
|
h.c.narration.cancel();
|
|
h.sockets[0].message({
|
|
voiceActivity: { type: 'ACTIVITY_END', audioOffset: '9.080s' },
|
|
});
|
|
assert.equal(h.c.narration.state().pendingTimers, pushToTalk ? 0 : 1);
|
|
assert.equal(h.c.status, 'listening');
|
|
h.c.stop();
|
|
assert.equal(h.timers.size, 0);
|
|
});
|
|
}
|
|
|
|
test('radio handoff waits for a post-tool reply and actual playback drain', async () => {
|
|
const h = harness({
|
|
runAction: async () => ({
|
|
ok: true,
|
|
action: 'control_radio',
|
|
radioAction: 'play',
|
|
radioPlaybackRequested: true,
|
|
}),
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'radio', name: 'control_radio', args: { action: 'play' } },
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
ws.message({ serverContent: { turnComplete: true } });
|
|
assert.equal(
|
|
h.state.playing,
|
|
0,
|
|
'the tool turn alone must not release Radio',
|
|
);
|
|
ws.message({
|
|
serverContent: {
|
|
modelTurn: {
|
|
parts: [
|
|
{ inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } },
|
|
],
|
|
},
|
|
turnComplete: true,
|
|
},
|
|
});
|
|
assert.equal(
|
|
h.state.playing,
|
|
0,
|
|
'queued assistant audio still owns the speaker',
|
|
);
|
|
h.captures[0].drain();
|
|
await tick();
|
|
assert.equal(h.state.playing, 1);
|
|
assert.equal(h.state.ducked, false);
|
|
assert.equal(h.c.status, 'idle');
|
|
});
|
|
|
|
test('explicit Radio stop cancels a handoff while confirmation is playing', async () => {
|
|
const h = harness({
|
|
runAction: async () => ({
|
|
ok: true,
|
|
action: 'control_radio',
|
|
radioPlaybackRequested: true,
|
|
}),
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'r', name: 'control_radio', args: { action: 'play' } },
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
ws.message({
|
|
serverContent: {
|
|
modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] },
|
|
turnComplete: true,
|
|
},
|
|
});
|
|
h.radioControl({ action: 'stop', origin: 'user' });
|
|
h.captures[0].drain();
|
|
await tick();
|
|
assert.equal(h.state.playing, 0);
|
|
assert.equal(h.c.status, 'listening');
|
|
h.c.stop();
|
|
});
|
|
|
|
test('eligible entity context sends a bounded fresh image before its function result', async () => {
|
|
const h = harness({
|
|
runAction: async () => ({
|
|
ok: true,
|
|
action: 'get_entity_context',
|
|
scene: { basemap: { viewScale: 'local' } },
|
|
}),
|
|
captureViewport: async () => 'data:image/jpeg;base64,aW1hZ2U=',
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
h.sockets[0].message({
|
|
toolCall: {
|
|
functionCalls: [{ id: 'view', name: 'get_entity_context', args: {} }],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
const sent = h.sockets[0].sent;
|
|
const imageIndex = sent.findIndex((entry) => entry.realtimeInput?.video);
|
|
assert.ok(imageIndex > 0);
|
|
assert.ok(imageIndex < sent.findIndex((entry) => entry.toolResponse));
|
|
h.c.stop();
|
|
});
|
|
|
|
test('socket pressure drops PCM without failing the active session and delivery recovers', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const ws = h.sockets[0],
|
|
audio = h.captures[0],
|
|
before = ws.sent.length;
|
|
ws.bufferedAmount = 300000;
|
|
audio.callbacks.onAudio(
|
|
{ data: 'AA==', mimeType: 'audio/pcm;rate=16000' },
|
|
audio.metadata,
|
|
);
|
|
assert.equal(ws.sent.length, before);
|
|
assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1);
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.tracks[0].readyState, 'live');
|
|
ws.bufferedAmount = 0;
|
|
audio.callbacks.onAudio(
|
|
{ data: 'AQ==', mimeType: 'audio/pcm;rate=16000' },
|
|
audio.metadata,
|
|
);
|
|
assert.equal(ws.sent.at(-1).realtimeInput.audio.data, 'AQ==');
|
|
h.c.stop();
|
|
});
|
|
|
|
test('passive map context never interrupts spoken output and is delivered at the next explicit turn', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } },
|
|
});
|
|
const before = ws.sent.length;
|
|
const event = {
|
|
type: 'map_annotation_outline',
|
|
label: 'Austin',
|
|
status: 'resolved',
|
|
};
|
|
assert.equal(h.c.notifyMapEvent(event), true);
|
|
event.label = 'changed after enqueue';
|
|
assert.equal(
|
|
ws.sent.length,
|
|
before,
|
|
'passive context sends no clientContent while speech is active',
|
|
);
|
|
assert.equal(h.captures[0].pending, true);
|
|
h.c.sendTextCommand('What did you outline?');
|
|
const parts = ws.sent.at(-1).clientContent.turns[0].parts;
|
|
assert.equal(JSON.parse(parts[0].text).events[0].label, 'Austin');
|
|
assert.equal(parts[1].text, 'What did you outline?');
|
|
assert.equal(h.c.pendingMapEvents.length, 0);
|
|
for (let i = 0; i < 30; i++) h.c.notifyMapEvent({ index: i });
|
|
assert.equal(h.c.pendingMapEvents.length, 16);
|
|
assert.equal(h.c.notifyMapEvent({ text: 'x'.repeat(16000) }), false);
|
|
h.c.stop();
|
|
assert.equal(h.c.pendingMapEvents.length, 0);
|
|
});
|
|
|
|
test('passive context joins the next tool response and survives a failed explicit send', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
h.c.notifyMapEvent({ type: 'map_annotation_outline', label: 'London' });
|
|
h.sockets[0].readyState = 0;
|
|
assert.equal(h.c.sendTextCommand('Continue'), false);
|
|
assert.equal(h.c.pendingMapEvents.length, 1);
|
|
h.sockets[0].readyState = 1;
|
|
h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
|
h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
|
h.sockets[0].message({
|
|
serverContent: { inputTranscription: { text: 'Continue' } },
|
|
});
|
|
h.sockets[0].message({
|
|
toolCall: {
|
|
functionCalls: [{ id: 'context', name: 'get_entity_context', args: {} }],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.equal(
|
|
h.sockets[0].sent.at(-1).toolResponse.functionResponses[0].response
|
|
.mapEvents[0].label,
|
|
'London',
|
|
);
|
|
assert.equal(h.c.pendingMapEvents.length, 0);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('a delayed permission prompt does not mint a token or consume the network deadline', async () => {
|
|
const microphone = deferred();
|
|
const h = harness({
|
|
startupTimeoutMs: 40,
|
|
getUserMedia: () => microphone.promise,
|
|
});
|
|
const start = h.c.start();
|
|
await new Promise((resolve) => setTimeout(resolve, 80));
|
|
assert.equal(h.c.status, 'connecting');
|
|
assert.equal(h.requests.length, 0);
|
|
assert.equal(h.sockets.length, 0);
|
|
microphone.resolve(h.makeStream());
|
|
await start;
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.requests.length, 1);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('Stop during a permission prompt releases a late grant without touching the successor', async () => {
|
|
const microphone = deferred();
|
|
const h = harness({ getUserMedia: () => microphone.promise });
|
|
const start = h.c.start();
|
|
await tick();
|
|
h.c.stop();
|
|
await start;
|
|
const lateStream = h.makeStream();
|
|
h.c.getUserMedia = async () => h.makeStream();
|
|
await h.c.start();
|
|
microphone.resolve(lateStream);
|
|
await tick();
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(h.tracks[1].readyState, 'live');
|
|
assert.equal(h.c.status, 'listening');
|
|
h.c.stop();
|
|
});
|
|
|
|
test('stalled audio initialization also obeys the startup deadline', async () => {
|
|
const initialization = deferred();
|
|
const h = harness({ startupTimeoutMs: 40 });
|
|
const createAudio = h.c.createAudio;
|
|
h.c.createAudio = (callbacks) => {
|
|
const audio = createAudio(callbacks);
|
|
audio.initialize = () => initialization.promise;
|
|
return audio;
|
|
};
|
|
await Promise.all([
|
|
h.c.start(),
|
|
new Promise((resolve) => setTimeout(resolve, 80)),
|
|
]);
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(h.captures[0].closed, true);
|
|
assert.equal(h.sockets.length, 0);
|
|
initialization.resolve();
|
|
await tick();
|
|
assert.equal(h.c.status, 'error');
|
|
});
|
|
|
|
for (const activity of [
|
|
'activity-start',
|
|
'push-to-talk',
|
|
'interrupted',
|
|
'user-text',
|
|
]) {
|
|
test(`new ${activity} revokes a pending Radio preflight after the old spoken turn ended`, async () => {
|
|
const ready = deferred();
|
|
const h = harness({
|
|
runAction: async () => ({
|
|
ok: true,
|
|
action: 'control_radio',
|
|
radioPlaybackRequested: true,
|
|
}),
|
|
});
|
|
h.c.radio.radioLayer.playForVoice = () => ready.promise;
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'radio', name: 'control_radio', args: { action: 'play' } },
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
ws.message({
|
|
serverContent: {
|
|
modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] },
|
|
turnComplete: true,
|
|
},
|
|
});
|
|
h.captures[0].drain();
|
|
assert.equal(h.c.radio.radioHandoffInFlight, true);
|
|
if (activity === 'activity-start')
|
|
ws.message({
|
|
voiceActivity: {
|
|
type: 'ACTIVITY_START',
|
|
audioOffset: '12s',
|
|
},
|
|
});
|
|
else if (activity === 'interrupted')
|
|
ws.message({ serverContent: { interrupted: true } });
|
|
else if (activity === 'user-text') h.c.sendTextCommand('Actually stop');
|
|
else h.c.input.setMicrophoneEnabled(true);
|
|
ready.resolve(true);
|
|
await tick();
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.c.stream.getTracks()[0].readyState, 'live');
|
|
assert.equal(h.state.ducked, true);
|
|
assert.ok(h.state.stopped > 0);
|
|
h.c.stop();
|
|
});
|
|
}
|
|
|
|
test('uncorrelated late transcription preserves its own successful Radio action and preflight', async () => {
|
|
const tool = deferred();
|
|
const ready = deferred();
|
|
const h = harness({ runAction: () => tool.promise });
|
|
h.c.radio.radioLayer.playForVoice = () => ready.promise;
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{
|
|
id: 'radio-late-transcript',
|
|
name: 'control_radio',
|
|
args: { action: 'play' },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
await tick();
|
|
const handoffEpoch = h.c.radio.radioHandoffEpoch;
|
|
ws.message({
|
|
serverContent: {
|
|
inputTranscription: { text: 'Play the radio' },
|
|
interimInputTranscription: { text: 'Play the radio' },
|
|
},
|
|
});
|
|
assert.equal(h.c.radio.radioHandoffEpoch, handoffEpoch);
|
|
tool.resolve({
|
|
ok: true,
|
|
action: 'control_radio',
|
|
radioPlaybackRequested: true,
|
|
});
|
|
await h.c.toolQueue;
|
|
const response = ws.sent.find((message) => message.toolResponse).toolResponse
|
|
.functionResponses[0].response;
|
|
assert.equal(response.ok, true);
|
|
assert.equal(response.radioPlaybackRequested, true);
|
|
ws.message({
|
|
serverContent: {
|
|
modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] },
|
|
turnComplete: true,
|
|
},
|
|
});
|
|
h.captures[0].drain();
|
|
assert.equal(h.c.radio.radioHandoffInFlight, true);
|
|
ws.message({ serverContent: { inputTranscription: { text: ' please' } } });
|
|
assert.equal(h.c.radio.radioHandoffInFlight, true);
|
|
ready.resolve(true);
|
|
await tick();
|
|
assert.equal(h.c.status, 'idle');
|
|
assert.equal(h.state.ducked, false);
|
|
});
|
|
|
|
function radioPlaybackResult() {
|
|
return {
|
|
ok: true,
|
|
action: 'control_radio',
|
|
radioAction: 'play',
|
|
radioPlaybackRequested: true,
|
|
};
|
|
}
|
|
|
|
function establishToolInput(h, text = 'Run the requested action') {
|
|
assert.equal(h.c.sendTextCommand(text), true);
|
|
}
|
|
|
|
function requestTool(
|
|
ws,
|
|
id,
|
|
name = 'control_radio',
|
|
args = { action: 'play' },
|
|
) {
|
|
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
|
|
}
|
|
|
|
function drainRadioReply(h) {
|
|
h.sockets[0].message({
|
|
serverContent: {
|
|
modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] },
|
|
turnComplete: true,
|
|
},
|
|
});
|
|
h.captures[0].drain();
|
|
}
|
|
|
|
test('cancelling one queued Radio call preserves active and queued same-feature siblings', async () => {
|
|
const first = deferred();
|
|
const executions = [];
|
|
const h = harness({
|
|
runAction: (name, args, options) => {
|
|
executions.push({ args, options });
|
|
return args.station === 'first' ? first.promise : radioPlaybackResult();
|
|
},
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
for (const station of ['first', 'cancelled', 'last'])
|
|
requestTool(ws, station, 'control_radio', { action: 'select', station });
|
|
await tick();
|
|
const epoch = h.c.radio.radioHandoffEpoch;
|
|
ws.message({ toolCallCancellation: { ids: ['cancelled'] } });
|
|
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
|
|
assert.equal(executions[0].options.isCurrent(), true);
|
|
first.resolve(radioPlaybackResult());
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(
|
|
executions.map((entry) => entry.args.station),
|
|
['first', 'last'],
|
|
);
|
|
assert.ok(executions.every((entry) => !entry.options.signal.aborted));
|
|
assert.equal(h.c.pendingTools, 0);
|
|
const newer = h.c.radio.pendingRadioPlaybackResult;
|
|
ws.message({ toolCallCancellation: { ids: ['first', 'unknown'] } });
|
|
assert.equal(h.c.radio.pendingRadioPlaybackResult, newer);
|
|
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('matching active Radio cancellation aborts its work and permits its sibling to finish', async () => {
|
|
const first = deferred();
|
|
const executions = [];
|
|
const h = harness({
|
|
runAction: (name, args, options) => {
|
|
executions.push(options);
|
|
return args.action === 'play' ? first.promise : radioPlaybackResult();
|
|
},
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
requestTool(ws, 'old');
|
|
requestTool(ws, 'new', 'control_radio', { action: 'select' });
|
|
await tick();
|
|
const epoch = h.c.radio.radioHandoffEpoch;
|
|
ws.message({ toolCallCancellation: { ids: ['old'] } });
|
|
assert.equal(executions[0].signal.aborted, true);
|
|
assert.equal(executions[0].isCurrent(), false);
|
|
await h.c.toolQueue;
|
|
const pending = h.c.radio.pendingRadioPlaybackResult;
|
|
assert.ok(pending);
|
|
assert.equal(executions[1].isCurrent(), true);
|
|
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
|
|
first.resolve(radioPlaybackResult());
|
|
await tick();
|
|
assert.equal(h.c.radio.pendingRadioPlaybackResult, pending);
|
|
assert.deepEqual(
|
|
ws.sent
|
|
.flatMap((m) => m.toolResponse?.functionResponses || [])
|
|
.map((r) => r.id),
|
|
['new'],
|
|
);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('matching prepared cancellation preserves a stronger sibling reservation and signal', async () => {
|
|
const stronger = deferred();
|
|
let strongerOptions;
|
|
const h = harness({
|
|
runAction: (name, args, options) => {
|
|
if (args.action === 'stop') {
|
|
strongerOptions = options;
|
|
return stronger.promise;
|
|
}
|
|
return radioPlaybackResult();
|
|
},
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
requestTool(ws, 'play');
|
|
await h.c.toolQueue;
|
|
requestTool(ws, 'stop', 'control_radio', { action: 'stop' });
|
|
await tick();
|
|
const reservations = [...h.c.radio.radioToolHandoffReservations.keys()];
|
|
assert.equal(reservations.length, 1);
|
|
const epoch = h.c.radio.radioHandoffEpoch;
|
|
ws.message({ toolCallCancellation: { ids: ['play', 'unknown'] } });
|
|
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
|
assert.equal(strongerOptions.signal.aborted, false);
|
|
assert.deepEqual(
|
|
[...h.c.radio.radioToolHandoffReservations.keys()],
|
|
reservations,
|
|
);
|
|
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
|
|
stronger.resolve({ ok: false, error: 'Refused fixture stop' });
|
|
await h.c.toolQueue;
|
|
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
|
drainRadioReply(h);
|
|
await tick();
|
|
assert.equal(h.state.playing, 0);
|
|
assert.equal(h.c.status, 'listening');
|
|
h.c.stop();
|
|
});
|
|
|
|
test('cancelled stronger call releases only its reservation and retains prepared playback', async () => {
|
|
const stronger = deferred();
|
|
const h = harness({
|
|
runAction: (name, args) =>
|
|
args.action === 'stop' ? stronger.promise : radioPlaybackResult(),
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
requestTool(ws, 'play');
|
|
await h.c.toolQueue;
|
|
const prepared = h.c.radio.pendingRadioPlaybackResult;
|
|
requestTool(ws, 'stop', 'control_radio', { action: 'stop' });
|
|
await tick();
|
|
const separateReservation = h.c.radio.reserveRadioToolHandoff();
|
|
ws.message({ toolCallCancellation: { ids: ['stop'] } });
|
|
await h.c.toolQueue;
|
|
assert.equal(h.c.radio.pendingRadioPlaybackResult, prepared);
|
|
assert.deepEqual(
|
|
[...h.c.radio.radioToolHandoffReservations.keys()],
|
|
[separateReservation],
|
|
);
|
|
stronger.resolve({ ok: true, action: 'control_radio' });
|
|
await tick();
|
|
assert.equal(h.c.radio.pendingRadioPlaybackResult, prepared);
|
|
h.c.radio.settleRadioToolHandoffReservation(separateReservation);
|
|
drainRadioReply(h);
|
|
await tick();
|
|
assert.equal(h.state.playing, 1);
|
|
assert.equal(h.c.status, 'idle');
|
|
});
|
|
|
|
test('matching preflight cancellation and late completion cannot revoke a newer reused-result handoff', async () => {
|
|
const sharedResult = radioPlaybackResult();
|
|
const starts = [];
|
|
const stops = [];
|
|
let playingAttempt = null;
|
|
const h = harness({ runAction: () => sharedResult });
|
|
h.c.radio.radioLayer.playForVoice = ({ attemptId }) => {
|
|
const ready = deferred();
|
|
starts.push({ attemptId, ready });
|
|
playingAttempt = attemptId;
|
|
return ready.promise;
|
|
};
|
|
h.c.radio.radioLayer.stopPlayback = ({ attemptId }) => {
|
|
stops.push(attemptId);
|
|
if (playingAttempt === attemptId) playingAttempt = null;
|
|
};
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
requestTool(ws, 'old');
|
|
await h.c.toolQueue;
|
|
drainRadioReply(h);
|
|
const oldResult = h.c.radio.radioHandoffInFlightResult;
|
|
establishToolInput(h, 'Play the radio again');
|
|
requestTool(ws, 'new');
|
|
await h.c.toolQueue;
|
|
const newerResult = h.c.radio.pendingRadioPlaybackResult;
|
|
assert.notEqual(oldResult, newerResult);
|
|
assert.notEqual(newerResult, sharedResult);
|
|
const epoch = h.c.radio.radioHandoffEpoch;
|
|
ws.message({ toolCallCancellation: { ids: ['old'] } });
|
|
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
|
|
assert.equal(h.c.radio.pendingRadioPlaybackResult, newerResult);
|
|
assert.equal(h.c.radio.radioHandoffInFlight, false);
|
|
assert.deepEqual(stops, [starts[0].attemptId]);
|
|
drainRadioReply(h);
|
|
assert.equal(starts.length, 2);
|
|
const newerAttempt = starts[1].attemptId;
|
|
starts[0].ready.resolve(true);
|
|
await tick();
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.c.radio.radioHandoffAttemptId, newerAttempt);
|
|
assert.equal(playingAttempt, newerAttempt);
|
|
assert.ok(stops.every((id) => id === starts[0].attemptId));
|
|
ws.message({ toolCallCancellation: { ids: ['old', 'unknown'] } });
|
|
assert.equal(h.c.radio.radioHandoffAttemptId, newerAttempt);
|
|
starts[1].ready.resolve(true);
|
|
await tick();
|
|
assert.equal(h.c.status, 'idle');
|
|
assert.equal(playingAttempt, newerAttempt);
|
|
});
|
|
|
|
test('cancelled preflight rejection cannot emit a stale Radio failure correction', async () => {
|
|
let rejectReady;
|
|
const ready = new Promise((_, reject) => {
|
|
rejectReady = reject;
|
|
});
|
|
const corrections = [];
|
|
const h = harness({ runAction: () => radioPlaybackResult() });
|
|
h.c.radio.radioLayer.playForVoice = () => ready;
|
|
h.c.radio.queueResponseCreate = (text) => corrections.push(text);
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
requestTool(ws, 'old');
|
|
await h.c.toolQueue;
|
|
drainRadioReply(h);
|
|
ws.message({ toolCallCancellation: { ids: ['old'] } });
|
|
rejectReady(new Error('late obsolete stream rejection'));
|
|
await tick();
|
|
assert.deepEqual(corrections, []);
|
|
assert.equal(h.c.status, 'listening');
|
|
h.c.stop();
|
|
});
|
|
|
|
test('completed query and unknown cancellation IDs are inert and cannot poison future calls', async () => {
|
|
const executions = [];
|
|
const h = harness({
|
|
runAction: (name, args) => {
|
|
executions.push(args.action);
|
|
return args.action === 'status'
|
|
? { ok: true, action: name }
|
|
: radioPlaybackResult();
|
|
},
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
requestTool(ws, 'query', 'control_radio', { action: 'status' });
|
|
await h.c.toolQueue;
|
|
requestTool(ws, 'owner');
|
|
await h.c.toolQueue;
|
|
const prepared = h.c.radio.pendingRadioPlaybackResult;
|
|
const epoch = h.c.radio.radioHandoffEpoch;
|
|
ws.message({ toolCallCancellation: { ids: ['query', 'future'] } });
|
|
assert.equal(h.c.radio.pendingRadioPlaybackResult, prepared);
|
|
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
|
|
requestTool(ws, 'future', 'control_radio', { action: 'select' });
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(executions, ['status', 'play', 'select']);
|
|
h.c.stop();
|
|
});
|
|
|
|
for (const replacement of ['user-activity', 'stop']) {
|
|
test(`${replacement} still revokes all prepared and preflight Radio ownership`, async () => {
|
|
const ready = deferred();
|
|
const h = harness({ runAction: () => radioPlaybackResult() });
|
|
h.c.radio.radioLayer.playForVoice = () => ready.promise;
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
requestTool(ws, 'old');
|
|
await h.c.toolQueue;
|
|
drainRadioReply(h);
|
|
h.c.radio.setPendingPlayback(radioPlaybackResult());
|
|
assert.ok(h.c.radio.pendingRadioPlaybackResult);
|
|
assert.equal(h.c.radio.radioHandoffInFlight, true);
|
|
if (replacement === 'user-activity') h.c.noteUserActivity();
|
|
else h.c.stop();
|
|
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
|
assert.equal(h.c.radio.radioHandoffInFlight, false);
|
|
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
|
assert.ok(h.state.stopped > 0);
|
|
ready.resolve(true);
|
|
await tick();
|
|
assert.equal(h.c.status, replacement === 'stop' ? 'idle' : 'listening');
|
|
if (replacement === 'user-activity') {
|
|
assert.equal(h.c.stream.getTracks()[0].readyState, 'live');
|
|
h.c.stop();
|
|
}
|
|
});
|
|
}
|
|
|
|
test('successful stronger Radio stop still invalidates the prepared group', async () => {
|
|
const h = harness({
|
|
runAction: (name, args) =>
|
|
args.action === 'stop'
|
|
? { ok: true, action: name, radioAction: 'stop' }
|
|
: radioPlaybackResult(),
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0];
|
|
requestTool(ws, 'play');
|
|
await h.c.toolQueue;
|
|
const epoch = h.c.radio.radioHandoffEpoch;
|
|
requestTool(ws, 'stop', 'control_radio', { action: 'stop' });
|
|
await h.c.toolQueue;
|
|
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
|
assert.ok(h.c.radio.radioHandoffEpoch > epoch);
|
|
drainRadioReply(h);
|
|
await tick();
|
|
assert.equal(h.state.playing, 0);
|
|
assert.equal(h.c.status, 'listening');
|
|
h.c.stop();
|
|
});
|
|
|
|
test('slow Gemini tools reuse shared progress timing without injecting a conversation turn', async () => {
|
|
const pending = deferred();
|
|
let options;
|
|
const h = harness({
|
|
runAction: (_name, _args, next) => {
|
|
options = next;
|
|
next.progress({ step: 'search', label: 'Austin' });
|
|
return pending.promise;
|
|
},
|
|
});
|
|
await h.c.start({ pushToTalk: true });
|
|
h.c.sendTextCommand('Fly to Austin');
|
|
const ws = h.sockets[0];
|
|
requestTool(ws, 'flight', 'fly_to_location', { query: 'Austin' });
|
|
await tick();
|
|
assert.equal(
|
|
options.callId,
|
|
'flight',
|
|
'shared card and narration use one call identity',
|
|
);
|
|
assert.equal(h.c.narration.state().tools, 1);
|
|
h.advance(699);
|
|
assert.equal(h.earcon.playing, false);
|
|
h.advance(1);
|
|
assert.equal(h.earcon.playing, true);
|
|
h.advance(7500);
|
|
assert.equal(
|
|
ws.sent.length,
|
|
2,
|
|
'progress never sends clientContent or an early tool response',
|
|
);
|
|
assert.equal(h.c.narration.state().stepLines, 0);
|
|
assert.equal(h.c.narration.state().stillSpoken, false);
|
|
pending.resolve({ ok: true, action: 'fly_to_location' });
|
|
await h.c.toolQueue;
|
|
assert.equal(ws.sent.at(-1).toolResponse.functionResponses[0].id, 'flight');
|
|
assert.equal(h.c.narration.state().tools, 0);
|
|
h.advance(3000);
|
|
assert.equal(h.earcon.playing, false, 'reply grace bounds the final earcon');
|
|
assert.equal(h.timers.size, 0);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('fast Gemini tools retract pending progress and Radio tools do not schedule lines', async () => {
|
|
for (const name of ['fly_to_location', 'control_radio']) {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
h.c.sendTextCommand('Run command');
|
|
requestTool(
|
|
h.sockets[0],
|
|
'fast',
|
|
name,
|
|
name === 'control_radio' ? { action: 'status' } : {},
|
|
);
|
|
await h.c.toolQueue;
|
|
h.advance(20000);
|
|
assert.equal(h.earcon.starts, 0);
|
|
assert.equal(h.timers.size, 0);
|
|
assert.equal(
|
|
h.sockets[0].sent.filter((message) => message.clientContent).length,
|
|
1,
|
|
);
|
|
h.c.stop();
|
|
}
|
|
});
|
|
|
|
test('Gemini model preamble and playback take priority over the working earcon', async () => {
|
|
const pending = deferred();
|
|
const h = harness({ runAction: () => pending.promise });
|
|
await h.c.start({ pushToTalk: true });
|
|
h.c.sendTextCommand('Fly to Austin');
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } },
|
|
});
|
|
requestTool(ws, 'slow', 'fly_to_location', { query: 'Austin' });
|
|
await tick();
|
|
assert.equal(h.c.narration.state().preambleSpoken, true);
|
|
h.advance(1500);
|
|
assert.equal(h.earcon.starts, 0, 'queued model audio owns the speaker');
|
|
h.captures[0].drain();
|
|
h.advance(700);
|
|
assert.equal(h.earcon.playing, true);
|
|
ws.message({
|
|
serverContent: { modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] } },
|
|
});
|
|
assert.equal(
|
|
h.earcon.playing,
|
|
false,
|
|
'new model audio immediately stops the tone',
|
|
);
|
|
h.c.stop();
|
|
pending.resolve({ ok: true });
|
|
await tick();
|
|
});
|
|
|
|
test('PTT release starts shared progress timing and a new hold cancels it', async () => {
|
|
const pending = deferred();
|
|
let options;
|
|
const h = harness({
|
|
runAction: (_name, _args, next) => {
|
|
options = next;
|
|
return pending.promise;
|
|
},
|
|
});
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
await h.c.start({ pushToTalk: true });
|
|
assert.equal(h.timers.size, 0);
|
|
h.c.input.pushToTalkKeyHeld = false;
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
h.sockets[0].message({
|
|
serverContent: { inputTranscription: { text: 'fly to Austin' } },
|
|
});
|
|
requestTool(h.sockets[0], 'held', 'fly_to_location', { query: 'Austin' });
|
|
await tick();
|
|
h.advance(700);
|
|
assert.equal(h.earcon.playing, true);
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
assert.equal(h.c.input.mayClaimSpeaker(), true);
|
|
assert.equal(h.c.input.bargeIn(), true);
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
assert.equal(h.earcon.playing, false);
|
|
assert.ok(h.c.retiredBoundaryWait, 'interruption retains its recovery bound');
|
|
assert.deepEqual([...h.timers.keys()], [h.c.retiredBoundaryWait.timer]);
|
|
assert.equal(options.signal.aborted, true);
|
|
assert.equal(h.captures[0].cleared, true);
|
|
assert.equal(h.c.narration.state().preambleSpoken, false);
|
|
pending.resolve({ ok: true });
|
|
await tick();
|
|
assert.equal(
|
|
h.sockets[0].sent.some((message) => message.toolResponse),
|
|
false,
|
|
);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('open microphone and Radio speaker ownership suppress Gemini earcons', async () => {
|
|
for (const mode of ['open-mic', 'held-ptt', 'radio']) {
|
|
const pending = deferred();
|
|
const h = harness({ runAction: () => pending.promise });
|
|
await h.c.start({ pushToTalk: mode !== 'open-mic' });
|
|
h.c.sendTextCommand('Fly to Austin');
|
|
requestTool(h.sockets[0], 'slow', 'fly_to_location', { query: 'Austin' });
|
|
await tick();
|
|
if (mode === 'held-ptt') h.c.input.pushToTalkKeyHeld = true;
|
|
if (mode === 'radio') {
|
|
h.c.radio.setPendingPlayback(radioPlaybackResult());
|
|
assert.equal(h.c.input.mayClaimSpeaker(), false);
|
|
assert.equal(
|
|
h.c.input.bargeIn(),
|
|
false,
|
|
'a hold cannot steal a Radio handoff',
|
|
);
|
|
assert.equal(h.c.pendingTools, 1);
|
|
}
|
|
h.advance(20000);
|
|
assert.equal(h.earcon.starts, 0, mode);
|
|
h.c.stop();
|
|
pending.resolve({ ok: true });
|
|
await tick();
|
|
}
|
|
});
|
|
|
|
test('Gemini progress is bounded and all terminal paths release narration ownership', async () => {
|
|
for (const end of ['stop', 'goAway', 'destroy', 'interruption']) {
|
|
const pending = deferred();
|
|
const h = harness({ runAction: () => pending.promise });
|
|
await h.c.start({ pushToTalk: true });
|
|
h.c.sendTextCommand('Fly to Austin');
|
|
const ws = h.sockets[0];
|
|
requestTool(ws, 'slow', 'fly_to_location', { query: 'Austin' });
|
|
await tick();
|
|
h.advance(700);
|
|
assert.equal(h.earcon.playing, true);
|
|
h.advance(15000);
|
|
assert.equal(
|
|
h.earcon.playing,
|
|
false,
|
|
'working tone never runs beyond the shared limit',
|
|
);
|
|
const disposals = h.earcon.disposals;
|
|
if (end === 'goAway') ws.message({ goAway: { timeLeft: '0s' } });
|
|
else if (end === 'interruption')
|
|
ws.message({ serverContent: { interrupted: true } });
|
|
else h.c.stop({ removeUi: end === 'destroy' });
|
|
assert.equal(h.c.narration.state().tools, 0, end);
|
|
const retirementTimer = h.c.retiredBoundaryWait?.timer;
|
|
assert.deepEqual(
|
|
[...h.timers.keys()],
|
|
end === 'interruption' ? [retirementTimer] : [],
|
|
'only the interrupted reply recovery timer may remain',
|
|
);
|
|
if (end === 'interruption') assert.ok(h.c.retiredBoundaryWait);
|
|
assert.equal(h.earcon.playing, false, end);
|
|
if (end !== 'interruption')
|
|
assert.equal(h.earcon.disposals, disposals + 1, end);
|
|
pending.resolve({ ok: true });
|
|
await tick();
|
|
h.advance(20000);
|
|
assert.equal(
|
|
ws.sent.some((message) => message.toolResponse),
|
|
false,
|
|
end,
|
|
);
|
|
assert.equal(h.timers.size, 0, end);
|
|
h.c.stop();
|
|
}
|
|
});
|
|
|
|
test('late tool and audio callbacks cannot change a successor Gemini narrator', async () => {
|
|
const first = deferred();
|
|
const second = deferred();
|
|
const options = [];
|
|
const h = harness({
|
|
runAction: (_name, _args, next) => {
|
|
options.push(next);
|
|
return options.length === 1 ? first.promise : second.promise;
|
|
},
|
|
});
|
|
await h.c.start({ pushToTalk: true });
|
|
h.c.sendTextCommand('Fly to Austin');
|
|
requestTool(h.sockets[0], 'reused', 'fly_to_location', { query: 'Austin' });
|
|
await tick();
|
|
const oldAudio = h.captures[0].callbacks;
|
|
h.c.stop();
|
|
await h.c.start({ pushToTalk: true });
|
|
h.c.sendTextCommand('Fly to Tokyo');
|
|
requestTool(h.sockets[1], 'reused', 'fly_to_location', { query: 'Tokyo' });
|
|
await tick();
|
|
const before = h.c.narration.state();
|
|
options[0].progress({ step: 'fly', label: 'Austin' });
|
|
oldAudio.onAudioEnd(h.captures[0].metadata);
|
|
oldAudio.onDrain();
|
|
first.resolve({ ok: true });
|
|
await tick();
|
|
assert.deepEqual(h.c.narration.state(), before);
|
|
assert.equal(h.c.pendingTools, 1);
|
|
h.advance(700);
|
|
assert.equal(h.earcon.playing, true);
|
|
second.resolve({ ok: true });
|
|
await h.c.toolQueue;
|
|
assert.equal(h.c.narration.state().tools, 0);
|
|
h.c.stop();
|
|
assert.equal(h.timers.size, 0);
|
|
});
|
|
|
|
test('Gemini tool progress reaches the shared card event contract with its call id', async () => {
|
|
const pending = deferred();
|
|
const events = [];
|
|
let options;
|
|
let h;
|
|
const session = createVoiceSession({
|
|
runner: (_name, _args, next) => {
|
|
options = next;
|
|
next.progress({ step: 'search', label: 'Austin' });
|
|
return pending.promise;
|
|
},
|
|
createAdapter: (shared) => {
|
|
h = harness(shared);
|
|
return {
|
|
start: (settings) => h.c.start(settings),
|
|
stop: (settings) => h.c.stop(settings),
|
|
sendText: (text) => h.c.sendTextCommand(text),
|
|
sendMapEvent: (event) => h.c.notifyMapEvent(event),
|
|
};
|
|
},
|
|
});
|
|
session.subscribe((event) => events.push(event));
|
|
await session.start({ pushToTalk: true });
|
|
establishToolInput(h);
|
|
requestTool(h.sockets[0], 'shared-call', 'fly_to_location', {
|
|
query: 'Austin',
|
|
});
|
|
await tick();
|
|
assert.equal(
|
|
events.find((event) => event.type === 'action-call').callId,
|
|
'shared-call',
|
|
);
|
|
assert.deepEqual(
|
|
events.find((event) => event.type === 'progress'),
|
|
{
|
|
type: 'progress',
|
|
name: 'fly_to_location',
|
|
callId: 'shared-call',
|
|
step: 'search',
|
|
label: 'Austin',
|
|
},
|
|
);
|
|
pending.resolve({ ok: true, action: 'fly_to_location' });
|
|
await h.c.toolQueue;
|
|
assert.equal(
|
|
events.find((event) => event.type === 'action-result').callId,
|
|
'shared-call',
|
|
);
|
|
session.stop();
|
|
const progressCount = events.filter(
|
|
(event) => event.type === 'progress',
|
|
).length;
|
|
options.progress({ step: 'fly', label: 'Stale Austin' });
|
|
assert.equal(
|
|
events.filter((event) => event.type === 'progress').length,
|
|
progressCount,
|
|
);
|
|
assert.equal(h.timers.size, 0);
|
|
session.destroy();
|
|
});
|
|
|
|
function pointerHarness() {
|
|
let clock = 1000;
|
|
const pointer = createPointerTracker({
|
|
element: {
|
|
clientWidth: 1000,
|
|
clientHeight: 500,
|
|
addEventListener() {},
|
|
},
|
|
now: () => clock,
|
|
setTimer: () => 1,
|
|
clearTimer() {},
|
|
pick: (x) => ({
|
|
entity: {
|
|
layerId: 'flights',
|
|
id: String(x),
|
|
label: `Aircraft ${x}`,
|
|
kind: 'aircraft',
|
|
lat: 30,
|
|
lon: -97,
|
|
},
|
|
}),
|
|
});
|
|
return {
|
|
pointer,
|
|
advance: (ms) => {
|
|
clock += ms;
|
|
},
|
|
};
|
|
}
|
|
|
|
const replyAudio = (data) => ({
|
|
modelTurn: {
|
|
parts: [{ inlineData: { data, mimeType: 'audio/pcm;rate=24000' } }],
|
|
},
|
|
});
|
|
|
|
test('B48 pointer: a cold held PTT turn freezes keydown and sends quiet context before activity', async () => {
|
|
const { pointer } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
pointer.moveTo(300, 200);
|
|
h.c.input.holdPointer();
|
|
pointer.moveTo(700, 200);
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
h.c.input.spaceKeyHeld = true;
|
|
await h.c.start({ pushToTalk: true });
|
|
assert.equal(pointer.activeSnapshot().entity.id, '300');
|
|
assert.deepEqual(
|
|
h.events.filter((event) => event.type === 'pointer').at(-1).pointer,
|
|
{
|
|
kind: 'this',
|
|
label: 'Aircraft 300',
|
|
screenPx: { x: 300, y: 200 },
|
|
},
|
|
);
|
|
const start = h.sockets[0].sent.find(
|
|
(value) => value.realtimeInput?.activityStart,
|
|
);
|
|
const context = h.sockets[0].sent.find((value) => value.clientContent);
|
|
assert.equal(
|
|
JSON.parse(context.clientContent.turns[0].parts[0].text).target.label,
|
|
'Aircraft 300',
|
|
);
|
|
assert.equal(context.clientContent.turns[0].role, 'user');
|
|
assert.equal(context.clientContent.turnComplete, false);
|
|
assert.deepEqual(start, { realtimeInput: { activityStart: {} } });
|
|
assert.equal(
|
|
h.sockets[0].sent.indexOf(context) < h.sockets[0].sent.indexOf(start),
|
|
true,
|
|
);
|
|
assert.equal(
|
|
h.c.replyPending,
|
|
false,
|
|
'context and an empty hold request no reply',
|
|
);
|
|
assert.equal(
|
|
h.sockets[0].sent.some(
|
|
(value) => value.clientContent?.turnComplete === true,
|
|
),
|
|
false,
|
|
);
|
|
h.c.input.releasePointerHold();
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
// A delayed server onset must not replace the released gesture's referent.
|
|
h.sockets[0].message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
|
assert.equal(pointer.activeSnapshot().entity.id, '300');
|
|
h.sockets[0].message({
|
|
serverContent: { inputTranscription: { text: 'what is this' } },
|
|
});
|
|
h.sockets[0].message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{
|
|
id: 'pointed',
|
|
name: 'get_entity_context',
|
|
args: { scope: 'pointer' },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.equal(h.calls[0].args.scope, 'pointer');
|
|
assert.equal(pointer.activeSnapshot().entity.id, '300');
|
|
h.c.stop();
|
|
assert.equal(pointer.activeSnapshot(), null);
|
|
pointer.destroy();
|
|
});
|
|
|
|
test('B48 pointer: text uses recent availability, clears stale context and sends one turn', async () => {
|
|
const { pointer, advance } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
await h.c.start();
|
|
pointer.moveTo(400, 200);
|
|
h.c.sendTextCommand('what is this?');
|
|
assert.equal(pointer.activeSnapshot().entity.id, '400');
|
|
let parts = h.sockets[0].sent.at(-1).clientContent.turns[0].parts;
|
|
assert.deepEqual(JSON.parse(parts[0].text), {
|
|
type: 'pointer_context',
|
|
at: 'text',
|
|
pointing: 'aircraft',
|
|
});
|
|
assert.equal(parts.at(-1).text, 'what is this?');
|
|
assert.equal(
|
|
h.sockets[0].sent.filter((value) => value.clientContent).length,
|
|
1,
|
|
);
|
|
assert.equal(
|
|
h.events.filter((event) => event.type === 'pointer').at(-1).pointer,
|
|
null,
|
|
);
|
|
h.c.runAction = async () => {
|
|
assert.equal(pointer.activeSnapshot().entity.id, '400');
|
|
assert.equal(h.c.announcePointer(), true);
|
|
assert.equal(h.c.announcePointer(), false);
|
|
return { ok: true };
|
|
};
|
|
h.sockets[0].message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{
|
|
id: 'implicit-pointer',
|
|
name: 'get_entity_context',
|
|
args: { scope: 'pointer' },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.equal(
|
|
h.events.filter((event) => event.type === 'pointer').at(-1).pointer.label,
|
|
'Aircraft 400',
|
|
);
|
|
h.sockets[0].message({ serverContent: { turnComplete: true } });
|
|
advance(5001);
|
|
h.c.sendTextCommand('what is here?');
|
|
assert.equal(pointer.activeSnapshot(), null);
|
|
parts = h.sockets[0].sent.at(-1).clientContent.turns[0].parts;
|
|
assert.deepEqual(JSON.parse(parts[0].text), {
|
|
type: 'pointer_context',
|
|
at: 'text',
|
|
target: 'none',
|
|
});
|
|
assert.equal(parts.at(-1).text, 'what is here?');
|
|
h.c.stop();
|
|
pointer.destroy();
|
|
});
|
|
|
|
test('B48 pointer: automatic speech snapshots once per onset and late input transcript keeps it', async () => {
|
|
const { pointer } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
await h.c.start();
|
|
pointer.moveTo(450, 200);
|
|
const ws = h.sockets[0];
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
|
const turn = h.c.turn;
|
|
assert.equal(pointer.activeSnapshot().entity.id, '450');
|
|
assert.deepEqual(JSON.parse(ws.sent.at(-1).realtimeInput.text), {
|
|
type: 'pointer_context',
|
|
at: 'speech_start',
|
|
pointing: 'aircraft',
|
|
});
|
|
pointer.moveTo(750, 200);
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
|
ws.message({
|
|
serverContent: { inputTranscription: { text: 'what is this?' } },
|
|
});
|
|
assert.equal(
|
|
h.c.turn,
|
|
turn,
|
|
'duplicate onset does not revoke the current turn',
|
|
);
|
|
assert.equal(pointer.activeSnapshot().entity.id, '450');
|
|
assert.equal(
|
|
ws.sent.some((value) => value.clientContent),
|
|
false,
|
|
);
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
|
ws.message({ serverContent: { turnComplete: true } });
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
|
assert.equal(pointer.activeSnapshot().entity.id, '750');
|
|
h.c.stop();
|
|
pointer.destroy();
|
|
});
|
|
|
|
for (const route of ['push-to-talk', 'user-text']) {
|
|
for (const idle of ['first', 'completed']) {
|
|
test(`B48 fence: ${route} permits the ${idle} idle reply without an interruption marker`, async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: route === 'push-to-talk' });
|
|
const ws = h.sockets[0];
|
|
const audio = h.captures[0];
|
|
if (idle === 'completed') {
|
|
ws.message({
|
|
serverContent: { ...replyAudio('previous'), turnComplete: true },
|
|
});
|
|
audio.drain();
|
|
}
|
|
if (route === 'user-text') h.c.sendTextCommand('first live request');
|
|
else {
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
ws.message({
|
|
serverContent: { inputTranscription: { text: 'what is in view' } },
|
|
});
|
|
}
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'valid-first-call', name: 'get_entity_context', args: {} },
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
ws.message({
|
|
serverContent: { ...replyAudio('legitimate'), turnComplete: true },
|
|
});
|
|
assert.equal(h.calls.length, 1);
|
|
assert.equal(audio.played.at(-1).data, 'legitimate');
|
|
h.c.stop();
|
|
});
|
|
}
|
|
}
|
|
|
|
test('B48 fence: an empty PTT hold creates no retired boundary for the next command', async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
h.captures[0].callbacks.onAudio(
|
|
{
|
|
data: 'AAAA',
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
h.captures[0].metadata,
|
|
);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
h.sockets[0].message({
|
|
serverContent: { ...replyAudio('first-real'), turnComplete: true },
|
|
});
|
|
assert.equal(h.captures[0].played.at(-1).data, 'first-real');
|
|
h.c.stop();
|
|
});
|
|
|
|
test('B48 fence: a coalesced interrupted boundary keeps successor speech and its pointer', async () => {
|
|
const { pointer } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.message({ serverContent: replyAudio('old') });
|
|
pointer.moveTo(600, 200);
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
|
const turn = h.c.turn;
|
|
ws.message({
|
|
voiceActivity: { type: 'ACTIVITY_START' },
|
|
serverContent: {
|
|
...replyAudio('boundary-old'),
|
|
interrupted: true,
|
|
turnComplete: true,
|
|
inputTranscription: { text: 'new pointed question' },
|
|
outputTranscription: { text: 'old answer' },
|
|
},
|
|
});
|
|
assert.equal(h.c.turn, turn);
|
|
assert.equal(pointer.activeSnapshot().entity.id, '600');
|
|
assert.equal(h.captures[0].played.length, 1);
|
|
assert.equal(
|
|
h.events.some((event) => event.type === 'completion'),
|
|
false,
|
|
);
|
|
assert.equal(
|
|
h.events.some(
|
|
(event) => event.role === 'user' && event.text === 'new pointed question',
|
|
),
|
|
true,
|
|
);
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
|
ws.message({
|
|
serverContent: { ...replyAudio('successor'), turnComplete: true },
|
|
});
|
|
assert.equal(h.captures[0].played.at(-1).data, 'successor');
|
|
h.c.stop();
|
|
pointer.destroy();
|
|
});
|
|
|
|
for (const route of ['push-to-talk', 'user-text']) {
|
|
test(`B48 queue: rapid ${route} keeps only the latest pointer/input and serializes reply boundaries`, async () => {
|
|
const { pointer } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
const audio = h.captures[0];
|
|
ws.message({ serverContent: replyAudio('old') });
|
|
const submit = (x) => {
|
|
pointer.moveTo(x, 200);
|
|
if (route === 'user-text') return h.c.sendTextCommand(`request ${x}`);
|
|
h.c.input.holdPointer();
|
|
h.c.input.bargeIn();
|
|
h.c.input.beginPointerTurn('keydown');
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
audio.callbacks.onAudio(
|
|
{
|
|
data: String(x),
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
audio.metadata,
|
|
);
|
|
h.c.input.releasePointerHold();
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
};
|
|
submit(300); // This start/command must reach the provider to cause interruption.
|
|
const sentFirst = ws.sent.length;
|
|
submit(500);
|
|
submit(700); // Newer intent replaces the still-unsent second turn.
|
|
assert.equal(
|
|
ws.sent.length,
|
|
sentFirst,
|
|
'later input waits without opening another server generation',
|
|
);
|
|
assert.equal(pointer.activeSnapshot().entity.id, '700');
|
|
const latestTurn = h.c.turn;
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
const flushed = ws.sent.slice(sentFirst);
|
|
if (route === 'user-text') {
|
|
assert.equal(flushed.length, 1);
|
|
assert.equal(
|
|
flushed[0].clientContent.turns[0].parts.at(-1).text,
|
|
'request 700',
|
|
);
|
|
} else {
|
|
assert.deepEqual(
|
|
flushed.map((value) =>
|
|
value.clientContent ? ['context'] : Object.keys(value.realtimeInput),
|
|
),
|
|
[['context'], ['activityStart'], ['audio'], ['activityEnd']],
|
|
);
|
|
assert.equal(
|
|
JSON.parse(flushed[0].clientContent.turns[0].parts[0].text).target
|
|
.label,
|
|
'Aircraft 700',
|
|
);
|
|
assert.equal(flushed[0].clientContent.turnComplete, false);
|
|
assert.equal(flushed[2].realtimeInput.audio.data, '700');
|
|
}
|
|
ws.message({
|
|
serverContent: {
|
|
...replyAudio('first-successor-retired'),
|
|
outputTranscription: { text: 'stale second answer' },
|
|
},
|
|
});
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{
|
|
id: 'queued-old-call',
|
|
name: 'fly_to_location',
|
|
args: { query: 'stale' },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
assert.equal(audio.played.length, 1);
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
assert.equal(h.c.turn, latestTurn);
|
|
ws.message({
|
|
serverContent: { inputTranscription: { text: 'what is this' } },
|
|
});
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{
|
|
id: 'latest-call',
|
|
name: 'get_entity_context',
|
|
args: { scope: 'pointer' },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
if (route === 'push-to-talk') {
|
|
await tick();
|
|
h.advance(5_000);
|
|
}
|
|
await h.c.toolQueue;
|
|
assert.equal(h.calls.length, 1);
|
|
assert.equal(pointer.activeSnapshot().entity.id, '700');
|
|
ws.message({
|
|
serverContent: { ...replyAudio('latest'), turnComplete: true },
|
|
});
|
|
assert.equal(audio.played.at(-1).data, 'latest');
|
|
h.c.stop();
|
|
pointer.destroy();
|
|
});
|
|
}
|
|
|
|
test('B48 queue: wait timeout fails closed and Stop releases queued input without replay', async () => {
|
|
for (const exit of ['timeout', 'stop']) {
|
|
const h = harness({ inputBoundaryTimeoutMs: 10 });
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.message({ serverContent: replyAudio('old') });
|
|
h.c.sendTextCommand('sent successor');
|
|
h.c.sendTextCommand('queued latest');
|
|
const sent = ws.sent.length;
|
|
if (exit === 'stop') h.c.stop();
|
|
else h.advance(25);
|
|
assert.equal(ws.closed, true);
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(h.c.status, exit === 'stop' ? 'idle' : 'error');
|
|
assert.equal(h.c.deferredInput, null);
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
assert.equal(
|
|
ws.sent.length,
|
|
sent,
|
|
'released queue cannot replay on a late boundary',
|
|
);
|
|
h.c.stop();
|
|
}
|
|
});
|
|
|
|
test('B48 queue: 256 KiB input bound fails closed without flushing partial queued speech', async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
ws.message({ serverContent: replyAudio('old') });
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
h.captures[0].callbacks.onAudio(
|
|
{
|
|
data: 'first',
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
h.captures[0].metadata,
|
|
);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
const sent = ws.sent.length;
|
|
h.captures[0].callbacks.onAudio(
|
|
{
|
|
data: 'A'.repeat(256 * 1024),
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
h.captures[0].metadata,
|
|
);
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(ws.closed, true);
|
|
assert.equal(h.c.deferredInput, null);
|
|
assert.equal(ws.sent.length, sent);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('B48 fence: duplicate automatic activity end cannot retire an idle next command', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
|
ws.message({
|
|
serverContent: { ...replyAudio('finished'), turnComplete: true },
|
|
});
|
|
h.captures[0].drain();
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
|
h.c.sendTextCommand('new idle turn');
|
|
ws.message({
|
|
serverContent: { ...replyAudio('new idle answer'), turnComplete: true },
|
|
});
|
|
assert.equal(h.captures[0].played.at(-1).data, 'new idle answer');
|
|
h.c.stop();
|
|
});
|
|
|
|
test('B48 queue: a boundary before deferred PTT worklet start still fences the submitted successor', async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
ws.message({ serverContent: replyAudio('old') });
|
|
h.c.sendTextCommand('first successor');
|
|
h.c.input.bargeIn(); // Audio-start callback is deliberately delayed.
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
ws.message({ serverContent: replyAudio('first-successor-stale') });
|
|
assert.equal(h.captures[0].played.length, 1);
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
h.captures[0].callbacks.onAudio(
|
|
{
|
|
data: 'latest',
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
h.captures[0].metadata,
|
|
);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
ws.message({
|
|
serverContent: { ...replyAudio('latest'), turnComplete: true },
|
|
});
|
|
assert.equal(h.captures[0].played.at(-1).data, 'latest');
|
|
h.c.stop();
|
|
});
|
|
|
|
for (const latest of ['push-to-talk', 'user-text']) {
|
|
test(`B48 queue: ${latest} can replace the other unsent input mode without stale pointer context`, async () => {
|
|
const { pointer, advance } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
const audio = h.captures[0];
|
|
ws.message({ serverContent: replyAudio('old') });
|
|
pointer.moveTo(300, 200);
|
|
h.c.sendTextCommand('first successor');
|
|
const speak = (x) => {
|
|
pointer.moveTo(x, 200);
|
|
h.c.input.holdPointer();
|
|
h.c.input.bargeIn();
|
|
h.c.input.beginPointerTurn('keydown');
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
audio.callbacks.onAudio(
|
|
{
|
|
data: String(x),
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
audio.metadata,
|
|
);
|
|
h.c.input.releasePointerHold();
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
};
|
|
if (latest === 'push-to-talk') {
|
|
pointer.moveTo(500, 200);
|
|
h.c.sendTextCommand('discard queued text');
|
|
speak(700);
|
|
} else {
|
|
speak(500);
|
|
advance(5001);
|
|
h.c.sendTextCommand('latest text without pointer');
|
|
}
|
|
const sent = ws.sent.length;
|
|
ws.message({ serverContent: { turnComplete: true } }); // Natural completion races the claim.
|
|
const frames = ws.sent.slice(sent);
|
|
if (latest === 'push-to-talk') {
|
|
assert.equal(
|
|
frames.some((message) => message.clientContent?.turnComplete === true),
|
|
false,
|
|
);
|
|
assert.equal(frames[0].clientContent.turnComplete, false);
|
|
assert.deepEqual(frames[1], { realtimeInput: { activityStart: {} } });
|
|
assert.equal(
|
|
JSON.parse(frames[0].clientContent.turns[0].parts[0].text).target.label,
|
|
'Aircraft 700',
|
|
);
|
|
} else {
|
|
assert.equal(frames.length, 1);
|
|
assert.deepEqual(
|
|
JSON.parse(frames[0].clientContent.turns[0].parts[0].text),
|
|
{ type: 'pointer_context', at: 'text', target: 'none' },
|
|
);
|
|
assert.equal(pointer.activeSnapshot(), null);
|
|
}
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
ws.message({
|
|
serverContent: { ...replyAudio('latest'), turnComplete: true },
|
|
});
|
|
assert.equal(audio.played.at(-1).data, 'latest');
|
|
h.c.stop();
|
|
pointer.destroy();
|
|
});
|
|
}
|
|
|
|
test('R51: typed override closes forwarded PTT before its queued text and suppresses the old flush', async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
const audio = h.captures[0];
|
|
ws.message({ serverContent: replyAudio('retired A') });
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
audio.callbacks.onAudio(
|
|
{
|
|
data: 'spoken B',
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
audio.metadata,
|
|
);
|
|
const before = ws.sent.length;
|
|
h.c.sendTextCommand('latest typed C');
|
|
assert.equal(
|
|
ws.sent.at(-1).realtimeInput?.activityEnd !== undefined,
|
|
true,
|
|
'the forwarded speech must end when typed intent supersedes it',
|
|
);
|
|
assert.equal(ws.sent.length, before + 1);
|
|
audio.callbacks.onAudio(
|
|
{
|
|
data: 'retired buffered B',
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
audio.metadata,
|
|
);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
assert.equal(
|
|
ws.sent.length,
|
|
before + 1,
|
|
'late old PCM/end cannot extend or duplicate closed activity',
|
|
);
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
const flushed = ws.sent.slice(before);
|
|
assert.equal(flushed.length, 2);
|
|
assert.equal(flushed[0].realtimeInput?.activityEnd !== undefined, true);
|
|
assert.equal(
|
|
flushed[1].clientContent.turns[0].parts.at(-1).text,
|
|
'latest typed C',
|
|
);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('R51: a delayed old worklet start cannot reclaim a typed turn; a fresh hold still works', async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
const audio = h.captures[0];
|
|
h.c.sendTextCommand('current text');
|
|
const before = ws.sent.length;
|
|
// This is the prior gesture's callback, delayed until after typed intent.
|
|
audio.callbacks.onAudioStart(audio.metadata);
|
|
audio.callbacks.onAudio(
|
|
{
|
|
data: 'old gesture',
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
audio.metadata,
|
|
);
|
|
audio.callbacks.onAudioEnd(audio.metadata);
|
|
assert.equal(ws.sent.length, before);
|
|
ws.message({
|
|
serverContent: { ...replyAudio('text completed'), turnComplete: true },
|
|
});
|
|
audio.drain();
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
audio.callbacks.onAudio(
|
|
{
|
|
data: 'fresh hold',
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
audio.metadata,
|
|
);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
const frames = ws.sent.slice(before);
|
|
assert.equal(frames[0].realtimeInput.activityStart !== undefined, true);
|
|
assert.equal(frames[1].realtimeInput.audio.data, 'fresh hold');
|
|
assert.equal(frames[2].realtimeInput.activityEnd !== undefined, true);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('R51: replaced unsent PTT contributes no unmatched end to the forwarded stream', async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
const audio = h.captures[0];
|
|
ws.message({ serverContent: replyAudio('retired A') });
|
|
h.c.sendTextCommand('submitted B');
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
audio.callbacks.onAudio(
|
|
{
|
|
data: 'unsent C',
|
|
mimeType: 'audio/pcm;rate=16000',
|
|
},
|
|
audio.metadata,
|
|
);
|
|
h.c.sendTextCommand('newest typed D');
|
|
const before = ws.sent.length;
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
assert.equal(ws.sent.length, before);
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
const flushed = ws.sent.slice(before);
|
|
assert.equal(flushed.length, 1);
|
|
assert.equal(
|
|
flushed[0].clientContent.turns[0].parts.at(-1).text,
|
|
'newest typed D',
|
|
);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('R51-2: FIFO old capture messages stay retired after typed replacement and a fresh claim', async () => {
|
|
const { pointer } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
const audio = h.captures[0];
|
|
// Delay only the worklet message port; gate/gesture callbacks still run now.
|
|
const port = [];
|
|
const callbacks = audio.callbacks;
|
|
const enable = audio.setMicrophoneEnabled.bind(audio);
|
|
audio.setMicrophoneEnabled = (value, metadata) => {
|
|
if (value && !audio.enabled)
|
|
port.push(() => callbacks.onAudioStart(metadata));
|
|
if (!value && audio.enabled) {
|
|
const ending = audio.metadata;
|
|
port.push(() => callbacks.onAudioEnd(ending));
|
|
}
|
|
audio.enabled = value;
|
|
if (value) audio.metadata = metadata;
|
|
};
|
|
ws.message({ serverContent: replyAudio('outstanding A') });
|
|
const claim = (x) => {
|
|
pointer.moveTo(x, 200);
|
|
h.c.input.holdPointer();
|
|
h.c.input.bargeIn();
|
|
h.c.input.beginPointerTurn('keydown');
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
return audio.metadata;
|
|
};
|
|
const b = claim(300);
|
|
port.push(() =>
|
|
callbacks.onAudio(
|
|
{ data: 'retired B', mimeType: 'audio/pcm;rate=16000' },
|
|
b,
|
|
),
|
|
);
|
|
h.c.sendTextCommand('typed C');
|
|
h.c.input.releasePointerHold();
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
const d = claim(700);
|
|
port.push(() =>
|
|
callbacks.onAudio(
|
|
{ data: 'current D', mimeType: 'audio/pcm;rate=16000' },
|
|
d,
|
|
),
|
|
);
|
|
h.c.input.releasePointerHold();
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
const before = ws.sent.length;
|
|
// Allowed port FIFO: start B / PCM B / end B / start D / PCM D / end D.
|
|
for (const deliver of port) deliver();
|
|
assert.equal(ws.sent.length, before);
|
|
const queued = h.c.deferredInput.messages;
|
|
assert.deepEqual(
|
|
queued.map((message) =>
|
|
message.clientContent
|
|
? 'context'
|
|
: message.realtimeInput.audio?.data ||
|
|
(message.realtimeInput.activityStart ? 'start' : 'end'),
|
|
),
|
|
['context', 'start', 'current D', 'end'],
|
|
);
|
|
assert.equal(
|
|
JSON.parse(queued[0].clientContent.turns[0].parts[0].text).target.label,
|
|
'Aircraft 700',
|
|
);
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
assert.deepEqual(ws.sent.slice(before), queued);
|
|
assert.equal(JSON.stringify(ws.sent).includes('captureId'), false);
|
|
audio.setMicrophoneEnabled = enable;
|
|
h.c.stop();
|
|
});
|
|
|
|
test('R51-2: a normal release keeps its delayed final PCM and end, without relabelling duplicate gates or tool aborts', async () => {
|
|
const h = harness();
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
await h.c.start({ pushToTalk: true });
|
|
const audio = h.captures[0];
|
|
const metadata = audio.metadata;
|
|
const ws = h.sockets[0];
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
assert.equal(audio.metadata.captureId, metadata.captureId);
|
|
h.c.abortTools(); // Radio cleanup is not a new physical capture.
|
|
const end = audio.callbacks.onAudioEnd;
|
|
let released;
|
|
audio.callbacks.onAudioEnd = (identity) => {
|
|
released = identity;
|
|
};
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
assert.equal(released.captureId, metadata.captureId);
|
|
const before = ws.sent.length;
|
|
audio.callbacks.onAudio(
|
|
{ data: 'released tail', mimeType: 'audio/pcm;rate=16000' },
|
|
metadata,
|
|
);
|
|
end(released);
|
|
assert.deepEqual(ws.sent.slice(before), [
|
|
{
|
|
realtimeInput: {
|
|
audio: { data: 'released tail', mimeType: 'audio/pcm;rate=16000' },
|
|
},
|
|
},
|
|
{ realtimeInput: { activityEnd: {} } },
|
|
]);
|
|
audio.callbacks.onAudioEnd = end;
|
|
h.c.stop();
|
|
});
|
|
|
|
test('R51-2: a forwarded old capture closes once before fresh FIFO start and cannot close the new activity', async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
const audio = h.captures[0];
|
|
const ws = h.sockets[0];
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
const b = audio.metadata;
|
|
audio.callbacks.onAudio(
|
|
{ data: 'forwarded B', mimeType: 'audio/pcm;rate=16000' },
|
|
b,
|
|
);
|
|
const start = audio.callbacks.onAudioStart;
|
|
const end = audio.callbacks.onAudioEnd;
|
|
const port = [];
|
|
audio.callbacks.onAudioStart = (identity) => port.push(() => start(identity));
|
|
audio.callbacks.onAudioEnd = (identity) => port.push(() => end(identity));
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
const d = audio.metadata;
|
|
const before = ws.sent.length;
|
|
for (const deliver of port) deliver(); // end B, start D in port FIFO.
|
|
assert.deepEqual(ws.sent.slice(before), [
|
|
{ realtimeInput: { activityEnd: {} } },
|
|
{ realtimeInput: { activityStart: {} } },
|
|
]);
|
|
assert.equal(h.c.forwardedCaptureId, d.captureId);
|
|
end(b); // A duplicate old end cannot close D.
|
|
assert.equal(ws.sent.length, before + 2);
|
|
assert.equal(h.c.forwardedActivityOpen, true);
|
|
audio.callbacks.onAudio(
|
|
{ data: 'current D', mimeType: 'audio/pcm;rate=16000' },
|
|
d,
|
|
);
|
|
audio.callbacks.onAudioStart = start;
|
|
audio.callbacks.onAudioEnd = end;
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
assert.equal(ws.sent.at(-1).realtimeInput.activityEnd !== undefined, true);
|
|
assert.equal(h.c.forwardedActivityOpen, false);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('R51-2: absent metadata fails closed and Stop/error restart never reuse capture identity', async () => {
|
|
const h = harness();
|
|
let previous = 0;
|
|
let old;
|
|
let oldMetadata;
|
|
for (const terminate of ['stop', 'error', 'stop']) {
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
await h.c.start({ pushToTalk: true });
|
|
const audio = h.captures.at(-1);
|
|
const ws = h.sockets.at(-1);
|
|
assert.ok(audio.metadata.captureId > previous);
|
|
previous = audio.metadata.captureId;
|
|
const before = ws.sent.length;
|
|
for (const captureId of [undefined, null, 0, -1, 1.5, '1', Infinity]) {
|
|
const metadata = captureId === undefined ? undefined : { captureId };
|
|
audio.callbacks.onAudioStart(metadata);
|
|
audio.callbacks.onAudio(
|
|
{ data: 'must reject', mimeType: 'audio/pcm;rate=16000' },
|
|
metadata,
|
|
);
|
|
audio.callbacks.onAudioEnd(metadata);
|
|
}
|
|
if (old) {
|
|
old.onAudioStart(oldMetadata);
|
|
old.onAudio(
|
|
{ data: 'old epoch', mimeType: 'audio/pcm;rate=16000' },
|
|
oldMetadata,
|
|
);
|
|
old.onAudioEnd(oldMetadata);
|
|
}
|
|
assert.equal(ws.sent.length, before);
|
|
assert.equal(h.c.forwardedActivityOpen, true);
|
|
old = audio.callbacks;
|
|
oldMetadata = audio.metadata;
|
|
if (terminate === 'error') h.c.fail(new Error('capture failed'));
|
|
else h.c.stop();
|
|
assert.equal(h.c.captureOwner, null);
|
|
assert.equal(h.c.captureEnabled, false);
|
|
assert.equal(h.c.forwardedCaptureId, null);
|
|
}
|
|
});
|
|
|
|
for (const queued of ['empty', 'text']) {
|
|
test(`R52: ${queued} flush keeps the original retirement wait bound`, async () => {
|
|
const h = harness({ inputBoundaryTimeoutMs: 40 });
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
ws.message({ serverContent: replyAudio('old') });
|
|
h.c.sendTextCommand('first submitted successor');
|
|
if (queued === 'empty') h.c.input.bargeIn();
|
|
else h.c.sendTextCommand('newest queued text');
|
|
// A physical empty hold no longer submits/retires a successor. The first
|
|
// submitted text still owns the reply when the old native boundary arrives.
|
|
if (queued === 'empty') {
|
|
assert.equal(h.c.deferredInput, null);
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
assert.equal(h.c.retiredReplyPending, false);
|
|
assert.equal(h.c.replyPending, true);
|
|
h.advance(60);
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(ws.closed, undefined);
|
|
h.c.stop();
|
|
return;
|
|
}
|
|
const deadline = h.c.deferredInput.deadline;
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
assert.equal(h.c.deferredInput, null);
|
|
assert.equal(h.c.retiredReplyPending, true);
|
|
h.advance(deadline + 20);
|
|
assert.equal(
|
|
h.c.status,
|
|
'error',
|
|
'missing genuine boundary cannot leave voice waiting indefinitely',
|
|
);
|
|
assert.equal(ws.closed, true);
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
h.c.stop();
|
|
});
|
|
}
|
|
|
|
test('R52: replacement cannot extend a flushed retirement deadline and Stop cancels it', async () => {
|
|
for (const exit of ['timeout', 'stop', 'boundary']) {
|
|
const h = harness({ inputBoundaryTimeoutMs: 60 });
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.message({ serverContent: replyAudio('old') });
|
|
h.c.sendTextCommand('first successor');
|
|
h.c.sendTextCommand('queued successor');
|
|
const deadline = h.c.deferredInput.deadline;
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
h.advance(15);
|
|
h.c.sendTextCommand('replacement while successor retired');
|
|
assert.equal(h.c.deferredInput.deadline, deadline);
|
|
if (exit === 'stop') h.c.stop();
|
|
if (exit === 'boundary') {
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
ws.message({
|
|
serverContent: { ...replyAudio('latest'), turnComplete: true },
|
|
});
|
|
h.captures[0].drain();
|
|
}
|
|
h.advance(deadline + 20);
|
|
assert.equal(
|
|
h.c.status,
|
|
exit === 'timeout' ? 'error' : exit === 'stop' ? 'idle' : 'listening',
|
|
);
|
|
assert.equal(Boolean(ws.closed), exit !== 'boundary');
|
|
h.c.stop();
|
|
}
|
|
});
|
|
|
|
test('PTT pointer context is quiet, ordered once, and only PCM creates a pending reply', async () => {
|
|
const { pointer } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
pointer.moveTo(300, 200);
|
|
h.c.input.holdPointer();
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
h.c.input.spaceKeyHeld = true;
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0],
|
|
audio = h.captures[0];
|
|
const [context, start] = ws.sent.slice(1);
|
|
assert.deepEqual(Object.keys(context), ['clientContent']);
|
|
assert.equal(context.clientContent.turnComplete, false);
|
|
assert.equal(context.clientContent.turns.length, 1);
|
|
assert.equal(context.clientContent.turns[0].role, 'user');
|
|
assert.equal(
|
|
JSON.parse(context.clientContent.turns[0].parts[0].text).target.label,
|
|
'Aircraft 300',
|
|
);
|
|
assert.deepEqual(start, { realtimeInput: { activityStart: {} } });
|
|
assert.equal(h.c.replyPending, false);
|
|
assert.equal(audio.played.length, 0);
|
|
assert.equal(h.calls.length, 0);
|
|
audio.callbacks.onAudioStart(audio.metadata);
|
|
assert.equal(
|
|
ws.sent.length,
|
|
3,
|
|
'duplicate start does not resend quiet context',
|
|
);
|
|
audio.callbacks.onAudio(
|
|
{ data: 'AAABAA==', mimeType: 'audio/pcm;rate=16000' },
|
|
audio.metadata,
|
|
);
|
|
assert.equal(h.c.replyPending, true);
|
|
assert.equal(
|
|
ws.sent.some((message) => message.realtimeInput?.text),
|
|
false,
|
|
);
|
|
h.c.stop();
|
|
pointer.destroy();
|
|
});
|
|
|
|
test('PTT pointer context rejected by the channel fails closed without opening activity', async () => {
|
|
const { pointer } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
await h.c.start({ pushToTalk: true });
|
|
pointer.moveTo(300, 200);
|
|
h.c.input.holdPointer();
|
|
const ws = h.sockets[0],
|
|
before = ws.sent.length;
|
|
h.c.channel.readyState = 'connecting';
|
|
// The adapter emits one admitted start, not a retryable channel-open event.
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
assert.equal(ws.sent.length, before);
|
|
assert.equal(h.c.forwardedActivityOpen, false);
|
|
assert.equal(h.c.pointerSentKey, null);
|
|
assert.equal(h.c.captureOwner, null);
|
|
assert.equal(h.c.userTurnPending, false);
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(ws.closed, true);
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
h.c.stop();
|
|
pointer.destroy();
|
|
});
|
|
|
|
test('quiet pointer context rejects stale or absent capture metadata without touching the new pointer', async () => {
|
|
const { pointer } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
await h.c.start({ pushToTalk: true });
|
|
pointer.moveTo(300, 200);
|
|
h.c.input.holdPointer();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
const old = h.captures[0].metadata,
|
|
context = h.c.captureOwner.context;
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
pointer.moveTo(700, 200);
|
|
h.c.input.holdPointer();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
const before = h.sockets[0].sent.length,
|
|
key = h.c.pointerSentKey;
|
|
assert.equal(h.c.sendPointerContext(context, old), false);
|
|
assert.equal(h.c.sendPointerContext(context), false);
|
|
assert.equal(h.c.sendPointerContext(context, { captureId: 0 }), false);
|
|
assert.equal(h.sockets[0].sent.length, before);
|
|
assert.equal(h.c.pointerSentKey, key);
|
|
assert.equal(pointer.activeSnapshot().entity.id, '700');
|
|
h.c.stop();
|
|
pointer.destroy();
|
|
});
|
|
|
|
test('replacing a deferred quiet PTT context preserves its deadline and only flushes the latest owner', async () => {
|
|
const { pointer } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0],
|
|
audio = h.captures[0];
|
|
ws.message({ serverContent: replyAudio('old A') });
|
|
const claim = (x, pcm) => {
|
|
pointer.moveTo(x, 200);
|
|
h.c.input.holdPointer();
|
|
h.c.input.bargeIn();
|
|
h.c.input.beginPointerTurn('keydown');
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
if (pcm)
|
|
audio.callbacks.onAudio(
|
|
{ data: pcm, mimeType: 'audio/pcm;rate=16000' },
|
|
audio.metadata,
|
|
);
|
|
h.c.input.releasePointerHold();
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
};
|
|
claim(300, 'submitted B');
|
|
claim(500);
|
|
const deadline = h.c.deferredInput.deadline;
|
|
const sent = ws.sent.length;
|
|
claim(700);
|
|
assert.equal(h.c.deferredInput.deadline, deadline);
|
|
assert.equal(h.c.deferredInput.kind, 'push-to-talk');
|
|
assert.equal(h.c.deferredInput.captureId, h.c.captureOwner.captureId);
|
|
assert.equal(h.c.deferredInput.messages.length, 3);
|
|
assert.equal(
|
|
JSON.parse(
|
|
h.c.deferredInput.messages[0].clientContent.turns[0].parts[0].text,
|
|
).target.label,
|
|
'Aircraft 700',
|
|
);
|
|
assert.equal(
|
|
JSON.stringify(h.c.deferredInput.messages).includes('Aircraft 500'),
|
|
false,
|
|
);
|
|
assert.equal(ws.sent.length, sent);
|
|
// The genuine retired boundary may coalesce content; it must discard it
|
|
// while releasing only the latest quiet context and its activity markers.
|
|
ws.message({
|
|
serverContent: {
|
|
...replyAudio('retired coalesced A'),
|
|
interrupted: true,
|
|
turnComplete: true,
|
|
},
|
|
});
|
|
const flushed = ws.sent.slice(sent);
|
|
assert.equal(flushed.length, 3);
|
|
assert.equal(flushed[0].clientContent.turnComplete, false);
|
|
assert.deepEqual(flushed[1], { realtimeInput: { activityStart: {} } });
|
|
assert.deepEqual(flushed[2], { realtimeInput: { activityEnd: {} } });
|
|
assert.equal(audio.played.at(-1).data, 'old A');
|
|
assert.equal(
|
|
h.c.replyPending,
|
|
false,
|
|
'flushed context without PCM asks for no reply',
|
|
);
|
|
assert.equal(h.c.retiredBoundaryWait.deadline, deadline);
|
|
assert.equal(h.c.retiredReplyPending, true);
|
|
assert.equal(h.c.pointerSentKey, h.c.captureOwner.context.key);
|
|
h.c.stop();
|
|
pointer.destroy();
|
|
});
|
|
|
|
test('projected PCM bytes and optional viewport frames drop while reliable tool results wait', async () => {
|
|
const h = harness({
|
|
captureViewport: async () => 'data:image/jpeg;base64,aW1hZ2U=',
|
|
});
|
|
await h.c.start();
|
|
establishToolInput(h);
|
|
const ws = h.sockets[0],
|
|
audio = h.captures[0],
|
|
before = ws.sent.length;
|
|
ws.bufferedAmount = 262140;
|
|
audio.callbacks.onAudio(
|
|
{ data: 'AA==', mimeType: 'audio/pcm;rate=16000' },
|
|
audio.metadata,
|
|
);
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'image-pressure', name: 'get_entity_context', args: {} },
|
|
],
|
|
},
|
|
});
|
|
h.c.runAction = async () => ({
|
|
ok: true,
|
|
action: 'get_entity_context',
|
|
scene: { basemap: { viewScale: 'local' } },
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1);
|
|
assert.equal(h.c.getDiagnostics().droppedImages, 1);
|
|
assert.equal(ws.sent.length, before);
|
|
assert.equal(h.c.controlQueue.length, 1);
|
|
ws.bufferedAmount = 0;
|
|
h.advance(25);
|
|
assert.equal(
|
|
ws.sent.at(-1).toolResponse.functionResponses[0].id,
|
|
'image-pressure',
|
|
);
|
|
assert.match(
|
|
ws.sent.at(-1).toolResponse.functionResponses[0].response.responseGuidance,
|
|
/briefly|request|read|view/i,
|
|
);
|
|
assert.equal(h.c.controlQueue.length, 0);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('congested PTT markers remain ordered and forwarding flags commit only at wire delivery', async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0],
|
|
audio = h.captures[0],
|
|
before = ws.sent.length;
|
|
ws.bufferedAmount = 300000;
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
const capture = audio.metadata;
|
|
audio.callbacks.onAudio(
|
|
{ data: 'discarded PCM', mimeType: 'audio/pcm;rate=16000' },
|
|
capture,
|
|
);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
assert.equal(h.c.forwardedActivityOpen, false);
|
|
assert.equal(h.c.controlQueue.length, 2);
|
|
assert.equal(ws.sent.length, before);
|
|
assert.equal(h.c.replyPending, false);
|
|
ws.bufferedAmount = 0;
|
|
h.advance(25);
|
|
assert.deepEqual(ws.sent.slice(before), [
|
|
{ realtimeInput: { activityStart: {} } },
|
|
{ realtimeInput: { activityEnd: {} } },
|
|
]);
|
|
assert.equal(h.c.forwardedActivityOpen, false);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('saturation then typed override and a fresh hold cannot flush retired pointer or capture markers', async () => {
|
|
const { pointer } = pointerHarness();
|
|
const h = harness({ pointer });
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0],
|
|
audio = h.captures[0],
|
|
before = ws.sent.length;
|
|
ws.bufferedAmount = 300000;
|
|
pointer.moveTo(300, 200);
|
|
h.c.input.holdPointer();
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
const old = audio.metadata;
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
h.c.sendTextCommand('unsent typed override');
|
|
assert.equal(
|
|
h.c.replyPending,
|
|
false,
|
|
'queued text creates no phantom retired reply',
|
|
);
|
|
pointer.moveTo(700, 200);
|
|
h.c.input.holdPointer();
|
|
h.c.input.bargeIn();
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
audio.callbacks.onAudioStart(old);
|
|
audio.callbacks.onAudioEnd(old);
|
|
assert.equal(
|
|
h.c.pointerSentKey,
|
|
null,
|
|
'queued context is not consumed before wire delivery',
|
|
);
|
|
ws.bufferedAmount = 0;
|
|
h.advance(25);
|
|
const frames = ws.sent.slice(before);
|
|
assert.equal(JSON.stringify(frames).includes('Aircraft 300'), false);
|
|
assert.equal(JSON.stringify(frames).includes('unsent typed override'), false);
|
|
assert.equal(
|
|
JSON.parse(frames[0].clientContent.turns[0].parts[0].text).target.label,
|
|
'Aircraft 700',
|
|
);
|
|
assert.deepEqual(frames.slice(1), [
|
|
{ realtimeInput: { activityStart: {} } },
|
|
{ realtimeInput: { activityEnd: {} } },
|
|
]);
|
|
assert.equal(h.c.forwardedActivityOpen, false);
|
|
h.c.stop();
|
|
pointer.destroy();
|
|
});
|
|
|
|
for (const cancellation of ['new-turn', 'server-call']) {
|
|
test(`a queued tool response is revoked by ${cancellation} before pressure clears`, async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const ws = h.sockets[0],
|
|
before = ws.sent.length;
|
|
ws.bufferedAmount = 300000;
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'queued-old', name: 'get_current_view_state', args: {} },
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.equal(h.c.controlQueue.length, 1);
|
|
if (cancellation === 'new-turn') h.c.sendTextCommand('new owned input');
|
|
else ws.message({ toolCallCancellation: { ids: ['queued-old'] } });
|
|
ws.bufferedAmount = 0;
|
|
h.advance(25);
|
|
assert.equal(
|
|
ws.sent.slice(before).some((frame) => frame.toolResponse),
|
|
false,
|
|
);
|
|
h.c.stop();
|
|
});
|
|
}
|
|
|
|
test('reliable pressure has an absolute deadline and Stop releases its poll without successor effects', async () => {
|
|
for (const mode of ['deadline', 'stop']) {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.bufferedAmount = 300000;
|
|
h.c.send({ realtimeInput: { text: 'reliable authored context' } });
|
|
h.advance(4975);
|
|
assert.equal(h.c.status, 'listening');
|
|
if (mode === 'stop') {
|
|
h.c.stop();
|
|
await h.c.start();
|
|
}
|
|
h.advance(25);
|
|
assert.equal(h.c.status, mode === 'deadline' ? 'error' : 'listening');
|
|
assert.equal(h.c.controlQueue.length, 0);
|
|
assert.equal(h.timers.size, 0);
|
|
h.c.stop();
|
|
}
|
|
});
|
|
|
|
test('reliable queue is byte and message bounded without partially sending oversized frames', async () => {
|
|
for (const kind of ['bytes', 'messages']) {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const ws = h.sockets[0],
|
|
before = ws.sent.length;
|
|
ws.bufferedAmount = 300000;
|
|
if (kind === 'bytes') {
|
|
assert.throws(
|
|
() => h.c.send({ realtimeInput: { text: 'x'.repeat(1024 * 1024) } }),
|
|
/queue limit/,
|
|
);
|
|
} else {
|
|
for (let i = 0; i < 64; i++)
|
|
assert.equal(h.c.send({ realtimeInput: { text: String(i) } }), true);
|
|
assert.throws(
|
|
() => h.c.send({ realtimeInput: { text: 'overflow' } }),
|
|
/queue limit/,
|
|
);
|
|
}
|
|
assert.equal(ws.sent.length, before);
|
|
h.c.stop();
|
|
assert.equal(h.timers.size, 0);
|
|
}
|
|
});
|
|
|
|
test('goAway permits the current tool/reply/audio drain and denies successor input', async () => {
|
|
const tool = deferred();
|
|
const h = harness({ runAction: () => tool.promise });
|
|
await h.c.start();
|
|
const ws = h.sockets[0],
|
|
audio = h.captures[0];
|
|
h.c.sendTextCommand('read the view');
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [{ id: 'current', name: 'get_entity_context', args: {} }],
|
|
},
|
|
});
|
|
await tick();
|
|
ws.message({ goAway: { timeLeft: '5s' } });
|
|
assert.equal(h.c.goAway.deadline, 5000);
|
|
assert.equal(h.tracks[0].enabled, false);
|
|
assert.equal(h.c.sendTextCommand('successor is disallowed'), false);
|
|
assert.equal(h.c.bargeIn(), false);
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
assert.equal(h.tracks[0].enabled, false);
|
|
tool.resolve({ ok: true, action: 'get_entity_context' });
|
|
await h.c.toolQueue;
|
|
assert.equal(ws.sent.at(-1).toolResponse.functionResponses[0].id, 'current');
|
|
ws.message({
|
|
serverContent: {
|
|
...replyAudio('accepted reply'),
|
|
outputTranscription: { text: 'Current result' },
|
|
turnComplete: true,
|
|
},
|
|
});
|
|
assert.equal(h.c.isActive(), true);
|
|
assert.equal(audio.pending, true);
|
|
audio.drain();
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(ws.closed, true);
|
|
assert.equal(h.timers.size, 0);
|
|
assert.equal(
|
|
h.events.findLast((event) => event.type === 'completion').audioDrained,
|
|
true,
|
|
);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('goAway chooses the earliest bounded deadline and Stop cancels it for a fresh epoch', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.message({ serverContent: replyAudio('playing') });
|
|
ws.message({ goAway: { timeLeft: '1.250s' } });
|
|
h.advance(250);
|
|
ws.message({ goAway: { timeLeft: '500s' } });
|
|
assert.equal(h.c.goAway.deadline, 1250);
|
|
ws.message({ goAway: { timeLeft: '0.5s' } });
|
|
assert.equal(h.c.goAway.deadline, 750);
|
|
h.c.stop();
|
|
await h.c.start();
|
|
h.advance(1000);
|
|
assert.equal(h.c.status, 'listening');
|
|
assert.equal(h.sockets.length, 2);
|
|
assert.equal(h.c.goAway, null);
|
|
h.c.stop();
|
|
});
|
|
|
|
for (const duration of ['bad', '-1s', 'Infinitys', '10000000s', null]) {
|
|
test(`malformed goAway timeLeft ${duration} closes safely without an unbounded timer`, async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
h.sockets[0].message({ serverContent: replyAudio('playing') });
|
|
h.sockets[0].message({ goAway: { timeLeft: duration } });
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(h.timers.size, 0);
|
|
});
|
|
}
|
|
|
|
test('close diagnostics preserve bounded code/category but never provider reason or credential text', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
h.sockets[0].onclose({
|
|
code: 1008,
|
|
reason: 'auth_tokens/private-secret user-private-text',
|
|
});
|
|
assert.deepEqual(h.c.getDiagnostics().close, {
|
|
code: 1008,
|
|
category: 'policy',
|
|
});
|
|
assert.doesNotMatch(
|
|
JSON.stringify(h.c.getDiagnostics()),
|
|
/private-secret|user-private-text/,
|
|
);
|
|
await h.c.start();
|
|
h.sockets[1].onclose({ code: Infinity, reason: 'private' });
|
|
assert.deepEqual(h.c.getDiagnostics().close, {
|
|
code: null,
|
|
category: 'connection',
|
|
});
|
|
h.c.stop();
|
|
});
|
|
|
|
test('goAway never flushes an unsent successor at the retired response boundary', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.message({ serverContent: replyAudio('retired previous') });
|
|
h.c.sendTextCommand('already forwarded successor');
|
|
h.c.sendTextCommand('unsent newest successor');
|
|
assert.ok(h.c.deferredInput);
|
|
const before = ws.sent.length;
|
|
ws.message({ goAway: { timeLeft: '5s' } });
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
assert.equal(
|
|
ws.sent.slice(before).some((message) => message.clientContent),
|
|
false,
|
|
);
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(h.c.deferredInput, null);
|
|
assert.equal(h.timers.size, 0);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('permission rejection opens no token/socket and initialized audio is released', async () => {
|
|
const h = harness({
|
|
getUserMedia: async () => {
|
|
throw new DOMException('not allowed', 'NotAllowedError');
|
|
},
|
|
});
|
|
await h.c.start();
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(h.requests.length, 0);
|
|
assert.equal(h.sockets.length, 0);
|
|
assert.equal(h.captures[0].closed, true);
|
|
assert.match(h.c.lastError, /permission was denied/);
|
|
});
|
|
|
|
test('consented microphone stays muted through the network phase and its deadline releases tracks', async () => {
|
|
const pending = deferred();
|
|
const h = harness({ startupTimeoutMs: 20, fetchImpl: () => pending.promise });
|
|
const start = h.c.start();
|
|
await tick();
|
|
assert.equal(h.tracks[0].enabled, false);
|
|
await Promise.all([start, new Promise((resolve) => setTimeout(resolve, 40))]);
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(h.tracks[0].readyState, 'ended');
|
|
assert.equal(h.captures[0].closed, true);
|
|
pending.resolve(Response.json(credential));
|
|
await tick();
|
|
assert.equal(h.sockets.length, 0);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('valid huge goAway duration is clamped to a bounded timer', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
h.sockets[0].message({ serverContent: replyAudio('playing') });
|
|
h.sockets[0].message({ goAway: { timeLeft: '999999s' } });
|
|
assert.equal(h.c.goAway.deadline, 600000);
|
|
h.c.stop();
|
|
assert.equal(h.timers.size, 0);
|
|
});
|
|
|
|
test('cancelling a pressure-queued response retains passive map context for the next owned text', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
h.c.notifyMapEvent({ type: 'map_annotation_outline', label: 'Austin' });
|
|
ws.bufferedAmount = 300000;
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'old-map-envelope', name: 'get_current_view_state', args: {} },
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.equal(
|
|
h.c.pendingMapEvents.length,
|
|
1,
|
|
'unsent data remains owned by the context queue',
|
|
);
|
|
ws.message({ toolCallCancellation: { ids: ['old-map-envelope'] } });
|
|
h.c.sendTextCommand('use the map context');
|
|
ws.bufferedAmount = 0;
|
|
h.advance(25);
|
|
const frame = ws.sent.at(-1);
|
|
assert.equal(
|
|
JSON.parse(frame.clientContent.turns[0].parts[0].text).events[0].label,
|
|
'Austin',
|
|
);
|
|
assert.equal(h.c.pendingMapEvents.length, 0);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('an expired goAway deadline rejects late output before the timer callback can run', async () => {
|
|
let now = 0;
|
|
const h = harness({ now: () => now });
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
h.c.sendTextCommand('current request');
|
|
ws.message({ serverContent: replyAudio('initial output') });
|
|
ws.message({ goAway: { timeLeft: '1s' } });
|
|
now = 1000;
|
|
ws.message({
|
|
serverContent: { ...replyAudio('too late'), turnComplete: true },
|
|
});
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(h.captures[0].played.length, 1);
|
|
assert.equal(
|
|
h.events.some((event) => event.type === 'completion'),
|
|
false,
|
|
);
|
|
assert.equal(h.timers.size, 0);
|
|
});
|
|
|
|
function guardedNativeHarness(overrides = {}) {
|
|
const mutations = [];
|
|
const observed = [];
|
|
const guarded = withToolCatalog(
|
|
async (name, args, options) => {
|
|
mutations.push({ name, args, options });
|
|
return { ok: true, action: name };
|
|
},
|
|
async () => null,
|
|
);
|
|
const h = harness({
|
|
...overrides,
|
|
runAction: async (name, args, options) => {
|
|
observed.push({ name, args, options });
|
|
return guarded(name, args, options);
|
|
},
|
|
});
|
|
return { ...h, mutations, observed };
|
|
}
|
|
|
|
for (const stage of ['deferred', 'flushed']) {
|
|
test(`Sameh P3-1: ${stage} retirement deadline uses the injected clock and releases timers`, async () => {
|
|
const h = harness({ inputBoundaryTimeoutMs: 60 });
|
|
try {
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.message({ serverContent: replyAudio('old') });
|
|
h.c.sendTextCommand('first successor');
|
|
h.c.sendTextCommand('queued successor');
|
|
assert.equal(h.c.deferredInput.deadline, 60);
|
|
h.advance(15);
|
|
h.c.sendTextCommand('replacement');
|
|
assert.equal(h.c.deferredInput.deadline, 60);
|
|
if (stage === 'flushed') {
|
|
ws.message({
|
|
serverContent: { interrupted: true, turnComplete: true },
|
|
});
|
|
assert.equal(h.c.retiredBoundaryWait.deadline, 60);
|
|
}
|
|
h.advance(44);
|
|
assert.notEqual(h.c.status, 'error');
|
|
h.advance(1);
|
|
assert.equal(h.c.status, 'error');
|
|
assert.equal(ws.closed, true);
|
|
assert.equal(h.timers.size, 0);
|
|
} finally {
|
|
h.c.stop();
|
|
}
|
|
});
|
|
}
|
|
|
|
for (const exit of ['stop', 'boundary']) {
|
|
test(`Sameh P3-1: ${exit} clears an injected retirement deadline before replacement`, async () => {
|
|
const h = harness({ inputBoundaryTimeoutMs: 60 });
|
|
try {
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.message({ serverContent: replyAudio('old') });
|
|
h.c.sendTextCommand('first successor');
|
|
h.c.sendTextCommand('queued successor');
|
|
const deadline = h.c.deferredInput.deadline;
|
|
h.advance(15);
|
|
h.c.sendTextCommand('newest successor');
|
|
assert.equal(h.c.deferredInput.deadline, deadline);
|
|
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
|
assert.equal(h.c.retiredBoundaryWait.deadline, deadline);
|
|
assert.equal(
|
|
[...h.timers.values()].some((timer) => timer.at === 60),
|
|
true,
|
|
'retirement wait must belong to the injected timer registry',
|
|
);
|
|
if (exit === 'stop') h.c.stop();
|
|
else {
|
|
ws.message({
|
|
serverContent: { interrupted: true, turnComplete: true },
|
|
});
|
|
ws.message({ serverContent: { turnComplete: true } });
|
|
}
|
|
assert.equal(
|
|
[...h.timers.values()].some((timer) => timer.at === 60),
|
|
false,
|
|
);
|
|
h.advance(100);
|
|
assert.equal(h.c.status, exit === 'stop' ? 'idle' : 'listening');
|
|
assert.equal(Boolean(ws.closed), exit === 'stop');
|
|
} finally {
|
|
h.c.stop();
|
|
}
|
|
});
|
|
}
|
|
|
|
for (const segments of [['use foo as my voice provider', 'and zoom out']]) {
|
|
test(`native advisory caption text ${JSON.stringify(segments)} releases tools`, async () => {
|
|
const h = guardedNativeHarness();
|
|
try {
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
|
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
|
for (const text of segments)
|
|
ws.message({ serverContent: { inputTranscription: { text } } });
|
|
requestTool(ws, 'ordinary-port', 'zoom_to_globe', {});
|
|
await h.c.toolQueue;
|
|
assert.equal(h.mutations.length, 1);
|
|
} finally {
|
|
h.c.stop();
|
|
}
|
|
assert.equal(h.timers.size, 0);
|
|
});
|
|
}
|
|
|
|
const zoomCall = (id) => ({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id, name: 'adjust_camera_zoom', args: { direction: 'in' } },
|
|
],
|
|
},
|
|
});
|
|
const toolResponses = (ws, from = 0) =>
|
|
ws.sent
|
|
.slice(from)
|
|
.flatMap((message) => message.toolResponse?.functionResponses || []);
|
|
|
|
async function completedPttTurn() {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
establishToolInput(h);
|
|
ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } });
|
|
h.captures[0].drain();
|
|
await tick();
|
|
return { h, ws };
|
|
}
|
|
|
|
test('push-to-talk refuses an unrequested tool call and every call in its reply', async () => {
|
|
const { h, ws } = await completedPttTurn();
|
|
const sent = ws.sent.length;
|
|
ws.message(zoomCall('unrequested'));
|
|
ws.message(zoomCall('follow-up'));
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(h.calls, []);
|
|
assert.deepEqual(
|
|
toolResponses(ws, sent).map(({ id, response }) => [id, response.ok]),
|
|
[
|
|
['unrequested', false],
|
|
['follow-up', false],
|
|
],
|
|
);
|
|
ws.message({ serverContent: { turnComplete: true } });
|
|
establishToolInput(h, 'Zoom in');
|
|
// A refused ID stays spent; only the newly requested call runs.
|
|
ws.message(zoomCall('unrequested'));
|
|
ws.message(zoomCall('requested'));
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(
|
|
h.calls.map((call) => call.name),
|
|
['adjust_camera_zoom'],
|
|
);
|
|
assert.equal(toolResponses(ws, sent).length, 3);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('push-to-talk refuses a call after unprompted speech that started the reply', async () => {
|
|
const { h, ws } = await completedPttTurn();
|
|
const sent = ws.sent.length;
|
|
ws.message({
|
|
serverContent: { outputTranscription: { text: 'Let me zoom in' } },
|
|
});
|
|
ws.message(zoomCall('after-speech'));
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(h.calls, []);
|
|
assert.deepEqual(
|
|
toolResponses(ws, sent).map(({ id, response }) => [id, response.ok]),
|
|
[['after-speech', false]],
|
|
);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('a refusal does not hold back the completion of the reply still playing', async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
establishToolInput(h);
|
|
ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } });
|
|
ws.message(zoomCall('unrequested'));
|
|
h.captures[0].drain();
|
|
await tick();
|
|
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
|
assert.deepEqual(h.calls, []);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('open microphone still runs a call Gemini makes without a local boundary', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
h.sockets[0].message(zoomCall('server-detected'));
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(
|
|
h.calls.map((call) => call.name),
|
|
['adjust_camera_zoom'],
|
|
);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('a late caption after a completed push-to-talk turn cannot retire the next request', async () => {
|
|
const { h, ws } = await completedPttTurn();
|
|
ws.message({ serverContent: { outputTranscription: { text: ' done.' } } });
|
|
establishToolInput(h, 'Zoom in');
|
|
ws.message(zoomCall('requested'));
|
|
await h.c.toolQueue;
|
|
h.advance(5_000);
|
|
assert.deepEqual(
|
|
h.calls.map((call) => call.name),
|
|
['adjust_camera_zoom'],
|
|
);
|
|
assert.equal(h.c.lastError, null);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('an open-mic reply that starts with a caption is retired by newer typed input', async () => {
|
|
const h = harness();
|
|
await h.c.start();
|
|
const ws = h.sockets[0];
|
|
establishToolInput(h);
|
|
ws.message({ serverContent: { ...replyAudio('answer'), turnComplete: true } });
|
|
h.captures[0].drain();
|
|
await tick();
|
|
ws.message({ serverContent: { outputTranscription: { text: 'Zooming' } } });
|
|
establishToolInput(h, 'Replacement request');
|
|
ws.message(zoomCall('earlier-reply'));
|
|
ws.message({ serverContent: { turnComplete: true } });
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(h.calls, []);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('a refused reply boundary that arrives after new push-to-talk input cannot close it', async () => {
|
|
const { h, ws } = await completedPttTurn();
|
|
const audio = h.captures[0];
|
|
ws.message(zoomCall('unrequested'));
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
audio.callbacks.onAudio(
|
|
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
|
|
audio.metadata,
|
|
);
|
|
h.c.input.pushToTalkKeyHeld = false;
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
const completions = h.events.filter((e) => e.type === 'completion').length;
|
|
ws.message({ serverContent: { turnComplete: true } });
|
|
assert.equal(
|
|
h.events.filter((e) => e.type === 'completion').length,
|
|
completions,
|
|
);
|
|
ws.message(zoomCall('requested'));
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(
|
|
h.calls.map((call) => call.name),
|
|
['adjust_camera_zoom'],
|
|
);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('push-to-talk admits a call made while an earlier requested tool is still running', async () => {
|
|
const running = deferred();
|
|
const h = harness({
|
|
runAction: async (name) => {
|
|
h.calls.push({ name });
|
|
if (h.calls.length === 1) await running.promise;
|
|
return { ok: true, action: name };
|
|
},
|
|
});
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
establishToolInput(h, 'Zoom in');
|
|
ws.message(zoomCall('first'));
|
|
await tick();
|
|
ws.message({ serverContent: { turnComplete: true } });
|
|
ws.message(zoomCall('second'));
|
|
running.resolve();
|
|
await h.c.toolQueue;
|
|
assert.equal(h.calls.length, 2);
|
|
assert.equal(
|
|
toolResponses(ws).some(({ response }) => response.ok === false),
|
|
false,
|
|
);
|
|
h.c.stop();
|
|
});
|
|
|
|
test('a narration callback that submits a successor keeps the old packet tools from running', async () => {
|
|
const h = harness();
|
|
await h.c.start({ pushToTalk: true });
|
|
const ws = h.sockets[0];
|
|
establishToolInput(h, 'Zoom in');
|
|
const audible = h.c.narration.audible.bind(h.c.narration);
|
|
let submitted = false;
|
|
h.c.narration.audible = () => {
|
|
audible();
|
|
if (submitted) return;
|
|
submitted = true;
|
|
assert.equal(h.c.sendTextCommand('New request'), true);
|
|
};
|
|
ws.message({
|
|
serverContent: replyAudio('old'),
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'old', name: 'adjust_camera_zoom', args: { direction: 'in' } },
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.equal(submitted, true);
|
|
assert.deepEqual(h.calls, []);
|
|
h.c.stop();
|
|
});
|
|
|
|
for (const reply of ['typed', 'push-to-talk', 'open-mic']) {
|
|
test(`globe refusal blocks queued and later tools until fresh ${reply} input`, async () => {
|
|
const names = [];
|
|
const h = harness({
|
|
runAction: async (name) => {
|
|
names.push(name);
|
|
return name === 'move_camera'
|
|
? { ok: false, action: 'move_camera', reason: 'globe-view-required' }
|
|
: { ok: true, action: name };
|
|
},
|
|
});
|
|
try {
|
|
await h.c.start({ pushToTalk: reply === 'push-to-talk' });
|
|
establishToolInput(h, 'rotate the globe');
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{
|
|
id: 'rotate',
|
|
name: 'move_camera',
|
|
args: { motion: 'rotate_globe' },
|
|
},
|
|
{ id: 'queued-reset', name: 'zoom_to_globe', args: {} },
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(names, ['move_camera']);
|
|
const blocked = ws.sent
|
|
.flatMap((entry) => entry.toolResponse?.functionResponses || [])
|
|
.find((call) => call.id === 'queued-reset');
|
|
assert.equal(blocked.response.ok, false);
|
|
assert.match(blocked.response.error, /fresh user input/i);
|
|
assert.equal(blocked.response.skipped, true);
|
|
assert.match(blocked.response.responseGuidance, /did not run/i);
|
|
assert.doesNotMatch(
|
|
blocked.response.responseGuidance,
|
|
/confirm.*completed/i,
|
|
);
|
|
requestTool(ws, 'repeated-rotation', 'move_camera', {
|
|
motion: 'rotate_globe',
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(names, ['move_camera']);
|
|
const repeated = toolResponses(ws).find(
|
|
(call) => call.id === 'repeated-rotation',
|
|
).response;
|
|
assert.equal(repeated.ok, false);
|
|
assert.equal(repeated.skipped, true);
|
|
assert.match(repeated.responseGuidance, /did not run/i);
|
|
assert.doesNotMatch(
|
|
repeated.responseGuidance,
|
|
/confirm.*completed/i,
|
|
);
|
|
|
|
if (reply === 'typed') {
|
|
const stimuli = [
|
|
() => {},
|
|
() => ws.message({ serverContent: { interrupted: true } }),
|
|
() =>
|
|
ws.message({
|
|
serverContent: {
|
|
inputTranscription: { text: 'reset to globe view' },
|
|
},
|
|
}),
|
|
() => {
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
},
|
|
];
|
|
for (const [i, stimulus] of stimuli.entries()) {
|
|
stimulus();
|
|
requestTool(ws, `unrequested-reset-${i}`, 'zoom_to_globe', {});
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(names, ['move_camera']);
|
|
}
|
|
}
|
|
// Finish the refusal reply before admitting the actual next user input.
|
|
ws.message({ serverContent: { turnComplete: true } });
|
|
if (reply === 'typed')
|
|
assert.equal(h.c.sendTextCommand('reset to globe view'), true);
|
|
else if (reply === 'push-to-talk') {
|
|
h.c.input.pushToTalkKeyHeld = true;
|
|
h.c.input.spaceKeyHeld = true;
|
|
h.c.input.setMicrophoneEnabled(true);
|
|
h.c.input.setMicrophoneEnabled(false);
|
|
} else ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
|
requestTool(ws, 'accepted-reset', 'zoom_to_globe', {});
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(names, ['move_camera', 'zoom_to_globe']);
|
|
} finally {
|
|
h.c.stop();
|
|
}
|
|
});
|
|
}
|
|
|
|
test('a cancelled globe reset skips queued rotation, preserves Stop and recovers on fresh input', async () => {
|
|
const calls = [];
|
|
let resets = 0;
|
|
const h = harness({
|
|
runAction: async (name, args) => {
|
|
calls.push(name === 'move_camera' ? args.motion : name);
|
|
return name === 'zoom_to_globe' && resets++ === 0
|
|
? { ok: false, action: name, cancelled: true }
|
|
: { ok: true, action: name };
|
|
},
|
|
});
|
|
try {
|
|
await h.c.start();
|
|
establishToolInput(h, 'reset to globe view and rotate');
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'reset-failed', name: 'zoom_to_globe', args: {} },
|
|
{
|
|
id: 'rotate-skipped',
|
|
name: 'move_camera',
|
|
args: { motion: 'rotate_globe' },
|
|
},
|
|
{ id: 'stop-allowed', name: 'move_camera', args: { motion: 'stop' } },
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(calls, ['zoom_to_globe', 'stop']);
|
|
const skipped = toolResponses(ws).find(
|
|
(call) => call.id === 'rotate-skipped',
|
|
).response;
|
|
assert.equal(skipped.ok, false);
|
|
assert.equal(skipped.reason, 'globe-reset-failed');
|
|
assert.equal(skipped.skipped, true);
|
|
assert.equal(
|
|
toolResponses(ws).find((call) => call.id === 'stop-allowed').response.ok,
|
|
true,
|
|
);
|
|
|
|
ws.message({ serverContent: { turnComplete: true } });
|
|
establishToolInput(h, 'reset to globe view and rotate');
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{ id: 'reset-success', name: 'zoom_to_globe', args: {} },
|
|
{
|
|
id: 'rotate-success',
|
|
name: 'move_camera',
|
|
args: { motion: 'rotate_globe' },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(calls, [
|
|
'zoom_to_globe',
|
|
'stop',
|
|
'zoom_to_globe',
|
|
'rotate_globe',
|
|
]);
|
|
} finally {
|
|
h.c.stop();
|
|
}
|
|
});
|
|
|
|
test('cancelling a queued globe reset also skips its dependent rotation', async () => {
|
|
const blocker = deferred();
|
|
const calls = [];
|
|
const h = harness({
|
|
runAction: async (name, args) => {
|
|
calls.push(name === 'move_camera' ? args.motion : name);
|
|
return args.motion === 'pan'
|
|
? blocker.promise
|
|
: { ok: true, action: name };
|
|
},
|
|
});
|
|
try {
|
|
await h.c.start();
|
|
establishToolInput(h, 'pan, reset to globe view and rotate');
|
|
const ws = h.sockets[0];
|
|
ws.message({
|
|
toolCall: {
|
|
functionCalls: [
|
|
{
|
|
id: 'blocker',
|
|
name: 'move_camera',
|
|
args: { motion: 'pan', direction: 'left' },
|
|
},
|
|
{ id: 'reset-cancelled', name: 'zoom_to_globe', args: {} },
|
|
{
|
|
id: 'rotate-after-cancel',
|
|
name: 'move_camera',
|
|
args: { motion: 'rotate_globe' },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
await tick();
|
|
ws.message({ toolCallCancellation: { ids: ['reset-cancelled'] } });
|
|
blocker.resolve({ ok: true, action: 'move_camera' });
|
|
await h.c.toolQueue;
|
|
assert.deepEqual(calls, ['pan']);
|
|
const skipped = toolResponses(ws).find(
|
|
(call) => call.id === 'rotate-after-cancel',
|
|
).response;
|
|
assert.equal(skipped.reason, 'globe-reset-failed');
|
|
assert.equal(skipped.skipped, true);
|
|
} finally {
|
|
blocker.resolve({ ok: true });
|
|
h.c.stop();
|
|
}
|
|
});
|