mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 10:53:59 +00:00
refactor(voice): isolate gemini live provider core
This commit is contained in:
manjunath committed
commit
10916227cb
655 files changed
+45946
-26483
No files matched your search
@@ -2,7 +2,6 @@ import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import vm from 'node:vm';
|
||||
import { harness as sessionHarness, tick } from './geminiNativeHarness.mjs';
|
||||
import {
|
||||
createGeminiAudio,
|
||||
decodeGeminiPcm,
|
||||
@@ -1133,351 +1132,3 @@ test('PTT admission leaves open-mic start, silence, PCM and end unchanged', asyn
|
||||
);
|
||||
assert.deepEqual(emittedPcm(h.events), new Array(1024).fill(0));
|
||||
});
|
||||
|
||||
async function realAdapterSession(t, overrides = {}) {
|
||||
const a = admissionHarness();
|
||||
await a.audio.close(); // Only the controller-owned adapter remains live below.
|
||||
const h = sessionHarness({ createAudio: a.createAudio, ...overrides });
|
||||
t.after(() => h.c.stop());
|
||||
await h.c.start({ pushToTalk: true });
|
||||
return { ...h, worklet: a.worklets[0] };
|
||||
}
|
||||
|
||||
test('silent PTT actual-controller integration sends no activity/context/PCM and releases pending state', async (t) => {
|
||||
const h = await realAdapterSession(t);
|
||||
const ws = h.sockets[0];
|
||||
const before = ws.sent.length;
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(new Float32Array(1024));
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.equal(ws.sent.length, before);
|
||||
assert.equal(h.c.userTurnPending, false);
|
||||
assert.equal(h.c.replyPending, false);
|
||||
assert.equal(h.c.forwardedActivityOpen, false);
|
||||
assert.equal(h.c.audio.diagnostics().suppressedPttCaptures, 1);
|
||||
assert.equal(h.c.status, 'listening');
|
||||
});
|
||||
|
||||
test('silent PTT during native playback retains local interruption without creating a successor', async (t) => {
|
||||
const h = await realAdapterSession(t);
|
||||
const ws = h.sockets[0];
|
||||
ws.message({
|
||||
serverContent: {
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{ inlineData: { data: 'AAA=', mimeType: 'audio/pcm;rate=24000' } },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal(h.c.audio.pending, true);
|
||||
const before = ws.sent.length;
|
||||
assert.equal(h.c.input.bargeIn(), true);
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(new Float32Array(1024));
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.equal(ws.sent.length, before);
|
||||
assert.equal(h.c.audio.pending, false);
|
||||
assert.equal(h.c.userTurnPending, false);
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
h.advance(5001);
|
||||
await tick();
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
assert.equal(h.c.status, 'listening');
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('voiced PTT actual-controller integration submits one ordered activity and recovers after silence', async (t) => {
|
||||
const h = await realAdapterSession(t);
|
||||
const ws = h.sockets[0];
|
||||
for (const [samples, admitted] of [
|
||||
[new Float32Array(640), false],
|
||||
[tone(640, 0.005), true],
|
||||
]) {
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(samples);
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
if (!admitted) assert.equal(h.c.userTurnPending, false);
|
||||
}
|
||||
const frames = ws.sent
|
||||
.filter((m) => m.realtimeInput)
|
||||
.map((m) => Object.keys(m.realtimeInput)[0]);
|
||||
assert.deepEqual(frames, ['activityStart', 'audio', 'activityEnd']);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: 'after-silent',
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'out', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(h.c.userTurnPending, false);
|
||||
});
|
||||
|
||||
test('silent PTT discard cannot clear a newer typed request', async (t) => {
|
||||
const h = await realAdapterSession(t);
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(new Float32Array(1024));
|
||||
h.c.sendTextCommand('zoom out a little');
|
||||
const typed = h.c.readTypedTurnInput();
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
assert.equal(h.c.readTypedTurnInput(), typed);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
});
|
||||
|
||||
// A physical hold alone cannot retire a provider tool request: without admitted
|
||||
// speech there is no native activity boundary to cancel that request.
|
||||
test('silent physical PTT preserves old tool completion without a retired-boundary timeout', async (t) => {
|
||||
const h = await realAdapterSession(t),
|
||||
ws = h.sockets[0];
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(tone(640, 0.005));
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
const before = ws.sent.length,
|
||||
turn = h.c.turn;
|
||||
assert.equal(h.c.input.bargeIn(), true);
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(new Float32Array(1024));
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: 'after-empty-hold',
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'out', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.c.turn, turn);
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(ws.sent.slice(before).filter((m) => m.toolResponse).length, 1);
|
||||
assert.equal(
|
||||
ws.sent.slice(before).some((m) => m.realtimeInput || m.clientContent),
|
||||
false,
|
||||
);
|
||||
h.advance(5001);
|
||||
await tick();
|
||||
assert.equal(h.c.status, 'listening');
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(h.c.replyPending, false);
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('silent physical PTT does not supersede an earlier typed request', async (t) => {
|
||||
const h = await realAdapterSession(t);
|
||||
h.c.sendTextCommand('zoom out a little');
|
||||
const typed = h.c.readTypedTurnInput(),
|
||||
turn = h.c.turn;
|
||||
h.c.input.bargeIn();
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(new Float32Array(640));
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.equal(h.c.turn, turn);
|
||||
assert.equal(h.c.readTypedTurnInput(), typed);
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
h.advance(5001);
|
||||
await tick();
|
||||
assert.equal(h.c.status, 'listening');
|
||||
});
|
||||
|
||||
test('admitted physical PTT retires older input only at sustained energy', async (t) => {
|
||||
const h = await realAdapterSession(t),
|
||||
ws = h.sockets[0];
|
||||
h.c.sendTextCommand('zoom out a little');
|
||||
const turn = h.c.turn;
|
||||
h.c.input.bargeIn();
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(new Float32Array(1024));
|
||||
assert.equal(h.c.turn, turn);
|
||||
h.worklet.feed(tone(1024, 0.005));
|
||||
assert.equal(h.c.turn, turn + 1);
|
||||
assert.equal(h.c.readTypedTurnInput(), undefined);
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.equal(ws.sent.filter((m) => m.realtimeInput?.activityStart).length, 1);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
});
|
||||
|
||||
test('rejected admitted PTT start fails closed instead of stranding a pending turn', async (t) => {
|
||||
const h = await realAdapterSession(t),
|
||||
ws = h.sockets[0];
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
ws.readyState = 0;
|
||||
h.worklet.feed(tone(1024, 0.005));
|
||||
await tick();
|
||||
assert.equal(h.c.status, 'error');
|
||||
assert.equal(h.c.userTurnPending, false);
|
||||
assert.equal(h.c.replyPending, false);
|
||||
assert.equal(ws.closed, true);
|
||||
});
|
||||
|
||||
test('a silent physical hold cannot replace the current tool pointer; admitted released speech uses its owned snapshot', async (t) => {
|
||||
const { createPointerTracker } = await import('./pointerContext.js');
|
||||
let clock = 0;
|
||||
const pointer = createPointerTracker({
|
||||
element: { clientWidth: 1000, clientHeight: 500, addEventListener() {} },
|
||||
pick: (x, y) => ({ ground: { lat: x / 10, lon: y / 10, heightM: 0 } }),
|
||||
now: () => clock,
|
||||
setTimer: () => 0,
|
||||
clearTimer() {},
|
||||
});
|
||||
const h = await realAdapterSession(t, { pointer });
|
||||
pointer.moveTo(100, 100);
|
||||
const old = pointer.beginTurn('text');
|
||||
pointer.moveTo(200, 200);
|
||||
h.c.input.holdPointer();
|
||||
h.c.input.beginPointerTurn('keydown');
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
assert.equal(pointer.activeSnapshot(), old);
|
||||
h.worklet.feed(new Float32Array(640));
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
h.c.input.releasePointerHold();
|
||||
assert.equal(pointer.activeSnapshot(), old);
|
||||
pointer.moveTo(300, 250);
|
||||
h.c.input.holdPointer();
|
||||
h.c.input.beginPointerTurn('keydown');
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(tone(640, 0.005));
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.releasePointerHold();
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.deepEqual(pointer.activeSnapshot().screenPx, { x: 300, y: 250 });
|
||||
const contexts = h.sockets[0].sent.filter((m) => m.clientContent);
|
||||
assert.equal(contexts.length, 1);
|
||||
assert.match(contexts[0].clientContent.turns[0].parts[0].text, /30/);
|
||||
});
|
||||
|
||||
test('a typed reply completes while its superseded silent PTT hold is still open', async (t) => {
|
||||
const h = await realAdapterSession(t),
|
||||
ws = h.sockets[0];
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(new Float32Array(640));
|
||||
h.c.sendTextCommand('zoom out a little');
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
assert.equal(h.c.pendingCompletionStatus, null);
|
||||
});
|
||||
|
||||
test('typed voice-off completion is not held behind a superseded silent capture', async (t) => {
|
||||
const h = await realAdapterSession(t),
|
||||
ws = h.sockets[0];
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(new Float32Array(640));
|
||||
h.c.sendTextCommand('turn voice off');
|
||||
h.c.stopAfterReply = true;
|
||||
h.c.postToolOutput = true;
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(h.c.status, 'idle');
|
||||
assert.equal(ws.closed, true);
|
||||
});
|
||||
|
||||
test('silent PTT release waits for asynchronous discard before presentation or Radio handoff', async (t) => {
|
||||
const h = await realAdapterSession(t),
|
||||
ws = h.sockets[0];
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(new Float32Array(640));
|
||||
const deliver = h.worklet.port.onmessage,
|
||||
queued = [];
|
||||
h.worklet.port.onmessage = (e) => queued.push(e);
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
let handoffs = 0;
|
||||
h.c.radio.startPendingRadioHandoff = async () => {
|
||||
handoffs++;
|
||||
};
|
||||
h.c.radio.pendingRadioPlaybackResult = {
|
||||
ok: true,
|
||||
radioPlaybackRequested: true,
|
||||
};
|
||||
h.c.postToolOutput = true;
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 0);
|
||||
assert.equal(handoffs, 0);
|
||||
for (const e of queued) deliver(e);
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
assert.equal(handoffs, 0);
|
||||
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
||||
});
|
||||
|
||||
test('expiry cannot admit a voiced partial worklet tail or replace its pointer', async (t) => {
|
||||
const h = await realAdapterSession(t),
|
||||
ws = h.sockets[0];
|
||||
h.c.sendTextCommand('zoom out a little');
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(tone(320, 0.005));
|
||||
const before = ws.sent.length;
|
||||
ws.message({ goAway: { timeLeft: '10s' } });
|
||||
assert.equal(ws.sent.length, before);
|
||||
assert.equal(h.c.forwardedActivityOpen, false);
|
||||
assert.equal(h.c.captureOwner.settled, true);
|
||||
assert.equal(h.c.captureOwner.admitted, false);
|
||||
});
|
||||
|
||||
test('suppressed old native PCM is not reported as spoken after a silent claim', async (t) => {
|
||||
const h = await realAdapterSession(t),
|
||||
ws = h.sockets[0];
|
||||
h.c.sendTextCommand('zoom out a little');
|
||||
h.c.input.bargeIn();
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.worklet.feed(new Float32Array(640));
|
||||
h.c.input.pushToTalkKeyHeld = false;
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
ws.message({
|
||||
serverContent: {
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{ inlineData: { data: 'AAA=', mimeType: 'audio/pcm;rate=24000' } },
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal(h.c.audio.diagnostics().outputChunks, 0);
|
||||
assert.equal(h.c.modelSpoke, false);
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(
|
||||
h.events.filter((e) => e.type === 'completion').at(-1).spoken,
|
||||
false,
|
||||
);
|
||||
});
|
||||
Reference in new issue
Block a user