mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 10:53:59 +00:00
refactor(voice): isolate gemini live provider core
This commit is contained in:
manjunath committed
commit
10916227cb
655 files changed
+45946
-26483
No files matched your search
@@ -1,13 +1,11 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { harness, tick, deferred } from './geminiNativeHarness.mjs';
|
||||
import { createGevActionRunner } from './gevActions.js';
|
||||
import {
|
||||
initialVoiceCardState,
|
||||
reduceVoiceCard,
|
||||
voiceCardView,
|
||||
} from './voiceCardPresentation.js';
|
||||
import { readVoiceUserTurnText } from './userTurnIntent.js';
|
||||
harness,
|
||||
tick,
|
||||
deferred,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
import { createGevActionRunner } from './gevActions.js';
|
||||
|
||||
const tool = (ws, id) =>
|
||||
ws.message({
|
||||
@@ -280,322 +278,3 @@ for (const [id, exit] of [
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function captionFixture() {
|
||||
let card = initialVoiceCardState();
|
||||
const events = [];
|
||||
const h = harness({
|
||||
emit(event) {
|
||||
events.push(event);
|
||||
card = reduceVoiceCard(card, event);
|
||||
},
|
||||
});
|
||||
return {
|
||||
...h,
|
||||
events,
|
||||
view: () => voiceCardView(card),
|
||||
close() {
|
||||
h.c.stop();
|
||||
assert.equal(h.timers.size, 0);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const startSpeech = (ws) =>
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
const endSpeech = (ws) =>
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
const speak = (ws, text) =>
|
||||
ws.message({
|
||||
serverContent: {
|
||||
outputTranscription: { text },
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{
|
||||
inlineData: {
|
||||
data: 'AAAA',
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
const zoom = (ws, id) =>
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id,
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'in', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
test('native captions: interrupted request cannot overwrite fragmented successor preview at completion', async () => {
|
||||
const h = captionFixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
startSpeech(ws);
|
||||
caption(ws, 'Can you explain the data layers in GEV?');
|
||||
endSpeech(ws);
|
||||
speak(ws, 'The app has several data layers.');
|
||||
|
||||
startSpeech(ws);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
complete(ws);
|
||||
assert.equal(h.view().user.text, '…');
|
||||
assert.equal(h.view().assistant.text, '');
|
||||
|
||||
for (const text of ['Zoom ', 'in ', 'a little.']) caption(ws, text);
|
||||
endSpeech(ws);
|
||||
assert.equal(h.view().user.text, 'Zoom in a little.');
|
||||
zoom(ws, 'successor-zoom');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.deepEqual(h.calls[0].args, { direction: 'in', amount: 'little' });
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(
|
||||
ws.sent.some((m) =>
|
||||
m.toolResponse?.functionResponses?.some(
|
||||
(call) => call.id === 'successor-zoom' && call.response.ok,
|
||||
),
|
||||
),
|
||||
true,
|
||||
);
|
||||
speak(ws, 'Zooming in ');
|
||||
speak(ws, 'slightly.');
|
||||
complete(ws);
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.view().user.text, 'Zoom in a little.');
|
||||
assert.equal(h.view().assistant.text, 'Zooming in slightly.');
|
||||
assert.equal(
|
||||
h.events.some(
|
||||
(e) => e.type === 'transcript' && e.role === 'user' && e.final,
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
h.events.some(
|
||||
(e) =>
|
||||
e.type === 'transcript' &&
|
||||
e.role === 'assistant' &&
|
||||
e.final &&
|
||||
e.text === 'Zooming in slightly.',
|
||||
),
|
||||
true,
|
||||
);
|
||||
const segments = h.events.filter((e) => e.type === 'user-input-segment');
|
||||
assert.deepEqual(
|
||||
segments.map((e) => e.text),
|
||||
['Can you explain the data layers in GEV?', 'Zoom ', 'in ', 'a little.'],
|
||||
);
|
||||
assert.equal(
|
||||
segments.every(
|
||||
(e) => e.inputId === null && e.inputScope === 'unattributed-segment',
|
||||
),
|
||||
true,
|
||||
);
|
||||
const captions = h.events.filter(
|
||||
(e) => e.type === 'transcript' && e.role === 'user',
|
||||
);
|
||||
assert.equal(
|
||||
captions.every(
|
||||
(e) =>
|
||||
e.delta === true &&
|
||||
e.final === false &&
|
||||
e.attribution === 'unverified' &&
|
||||
e.responseId === null,
|
||||
),
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('native captions: duplicate activity and interrupted acknowledgments preserve received fragments', async () => {
|
||||
const h = captionFixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speak(ws, 'Old reply');
|
||||
startSpeech(ws);
|
||||
caption(ws, 'Zoom ');
|
||||
startSpeech(ws);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
assert.equal(h.view().user.text, 'Zoom');
|
||||
caption(ws, 'in ');
|
||||
complete(ws);
|
||||
caption(ws, 'a little.');
|
||||
endSpeech(ws);
|
||||
assert.equal(h.view().user.text, 'Zoom in a little.');
|
||||
complete(ws);
|
||||
assert.equal(h.view().user.text, 'Zoom in a little.');
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('native captions: no new caption means successor completion cannot replay old request', async () => {
|
||||
const h = captionFixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
caption(ws, 'Explain the data layers.');
|
||||
speak(ws, 'Several layers are available.');
|
||||
startSpeech(ws);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
complete(ws);
|
||||
zoom(ws, 'uncaptioned-successor');
|
||||
await h.c.toolQueue;
|
||||
complete(ws);
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(h.view().user.text, '…');
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('native captions: advisory arrival order is displayed without synthesizing final speech ownership', async () => {
|
||||
const h = captionFixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
// Captions have no input id: presentation does not guess their semantic order.
|
||||
caption(ws, 'little. ');
|
||||
caption(ws, 'Zoom in a ');
|
||||
assert.equal(h.view().user.text, 'little. Zoom in a');
|
||||
complete(ws);
|
||||
assert.equal(h.view().user.text, 'little. Zoom in a');
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'user-input-final'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'response-start'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
h.events.some((e) => e.role === 'user' && e.final),
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('native captions: local PTT and typed boundaries clear preview without changing typed authority', async () => {
|
||||
const h = captionFixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
caption(ws, 'Older speech.');
|
||||
assert.equal(h.c.bargeIn(), true);
|
||||
assert.equal(h.view().user.text, '…');
|
||||
caption(ws, 'Zoom ');
|
||||
caption(ws, 'in a little.');
|
||||
assert.equal(h.view().user.text, 'Zoom in a little.');
|
||||
complete(ws);
|
||||
assert.equal(h.view().user.text, 'Zoom in a little.');
|
||||
|
||||
assert.equal(h.c.sendTextCommand('Zoom in a little.'), true);
|
||||
assert.equal(h.view().user.text, '…');
|
||||
caption(ws, 'unrelated advisory speech');
|
||||
zoom(ws, 'typed-zoom');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(
|
||||
readVoiceUserTurnText(h.calls[0].options.userTurn),
|
||||
'Zoom in a little.',
|
||||
);
|
||||
complete(ws);
|
||||
assert.equal(h.view().user.text, 'unrelated advisory speech');
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('native captions: Stop and restart do not retain old display fragments or accept old socket captions', async () => {
|
||||
const h = captionFixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const old = h.sockets[0];
|
||||
caption(old, 'Old partial ');
|
||||
h.c.stop();
|
||||
assert.equal(h.view().visible, false);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[1];
|
||||
caption(ws, 'Zoom ');
|
||||
caption(old, 'stale caption');
|
||||
caption(ws, 'in a little.');
|
||||
complete(ws);
|
||||
assert.equal(h.view().user.text, 'Zoom in a little.');
|
||||
assert.equal(h.tracks[0].readyState, 'ended');
|
||||
assert.equal(h.captures[0].closed, true);
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
|
||||
for (const split of [false, true]) {
|
||||
test(`native captions: server-only interruption retains ${split ? 'a leading fragment alongside its successor' : 'a complete pre-marker caption'} as recent speech`, async () => {
|
||||
const h = captionFixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speak(ws, 'An earlier assistant reply.');
|
||||
caption(ws, split ? 'Zoom ' : 'Zoom in a little.');
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
assert.equal(h.view().user.text, '…');
|
||||
assert.equal(h.view().recentSpeech.text, split ? 'Zoom' : 'Zoom in a little.');
|
||||
assert.equal(h.view().recentSpeech.visible, true);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
assert.equal(h.view().recentSpeech.text, split ? 'Zoom' : 'Zoom in a little.');
|
||||
if (split) caption(ws, 'in a little.');
|
||||
complete(ws);
|
||||
zoom(ws, 'server-only-caption');
|
||||
await h.c.toolQueue;
|
||||
speak(ws, 'Zooming in slightly.');
|
||||
complete(ws);
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.view().recentSpeech.text, split ? 'Zoom' : 'Zoom in a little.');
|
||||
assert.equal(h.view().user.text, split ? 'in a little.' : '…');
|
||||
assert.equal(h.view().assistant.text, 'Zooming in slightly.');
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(h.events.some((e) => e.role === 'user' && e.final), false);
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (const boundary of ['native-onset', 'push-to-talk', 'typed', 'stop']) {
|
||||
test(`native captions: ${boundary} clears recent speech retained by a server-only acknowledgment`, async () => {
|
||||
const h = captionFixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speak(ws, 'Old response');
|
||||
caption(ws, 'Earlier advisory speech');
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
assert.equal(h.view().recentSpeech.visible, true);
|
||||
// The native onset must clear presentation even while retirement is
|
||||
// already pending and therefore does not emit another interruption.
|
||||
if (boundary === 'native-onset') startSpeech(ws);
|
||||
else if (boundary === 'push-to-talk') h.c.bargeIn();
|
||||
else if (boundary === 'typed') h.c.sendTextCommand('Zoom in a little.');
|
||||
else h.c.stop();
|
||||
assert.equal(h.view().recentSpeech.visible, false);
|
||||
assert.equal(h.view().recentSpeech.text, '');
|
||||
assert.equal(h.view().user.text, '…');
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user