mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 08:12:24 +00:00
fix(voice): recover idle timers and retire panel audio observers
This commit is contained in:
1 parent
aeee3652ce
commit
074e272356
9 files changed
+429
-16
No files matched your search
@@ -36,6 +36,33 @@ export function observeNativeActivity({
|
||||
let audio = null,
|
||||
version = 0;
|
||||
const restores = [];
|
||||
const audioRestores = [];
|
||||
const pendingOutput = new Map();
|
||||
let untrackedOutput = !outputKnown;
|
||||
function retireAudio() {
|
||||
audio = null;
|
||||
for (const restore of audioRestores.splice(0).reverse()) restore();
|
||||
}
|
||||
function refreshOutputKnowledge() {
|
||||
outputKnown = !untrackedOutput && pendingOutput.size === 0;
|
||||
}
|
||||
function hasNoAudioOutput(response) {
|
||||
return (
|
||||
['completed', 'cancelled', 'failed', 'incomplete'].includes(
|
||||
response?.status,
|
||||
) &&
|
||||
Array.isArray(response.output) &&
|
||||
response.output.every(
|
||||
(item) =>
|
||||
item?.type === 'function_call' ||
|
||||
(item?.type === 'message' &&
|
||||
Array.isArray(item.content) &&
|
||||
item.content.every((part) =>
|
||||
['text', 'output_text', 'refusal'].includes(part?.type),
|
||||
)),
|
||||
)
|
||||
);
|
||||
}
|
||||
function snapshot() {
|
||||
const turns = nativeProvider === 'openai' ? controller?._turns : controller;
|
||||
return {
|
||||
@@ -72,7 +99,7 @@ export function observeNativeActivity({
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
function wrap(object, key, after) {
|
||||
function wrap(object, key, after, ownedRestores = restores) {
|
||||
const original = object?.[key];
|
||||
if (typeof original !== 'function') return;
|
||||
function wrapped(...args) {
|
||||
@@ -86,7 +113,7 @@ export function observeNativeActivity({
|
||||
return result;
|
||||
}
|
||||
object[key] = wrapped;
|
||||
restores.push(() => {
|
||||
ownedRestores.push(() => {
|
||||
if (object[key] === wrapped) object[key] = original;
|
||||
});
|
||||
}
|
||||
@@ -104,9 +131,11 @@ export function observeNativeActivity({
|
||||
(state === 'connecting' && wasConnected)
|
||||
) {
|
||||
version++;
|
||||
audio = null;
|
||||
retireAudio();
|
||||
inputBusy = false;
|
||||
outputKnown = cleanNativeOutput();
|
||||
untrackedOutput = !outputKnown;
|
||||
pendingOutput.clear();
|
||||
}
|
||||
}
|
||||
notify();
|
||||
@@ -127,13 +156,41 @@ export function observeNativeActivity({
|
||||
inputBusy = true;
|
||||
if (event?.type === 'input_audio_buffer.speech_stopped')
|
||||
inputBusy = false;
|
||||
// A previous turn's drain does not prove the next transport emits output events.
|
||||
if (event?.type === 'response.created') outputKnown = false;
|
||||
const type = event?.type;
|
||||
const id = event?.response_id || event?.response?.id;
|
||||
// Keep each unresolved response separate: a later silent reply cannot
|
||||
// establish that an earlier audio reply has drained.
|
||||
if (type === 'response.created') {
|
||||
if (id && pendingOutput.size < 64)
|
||||
pendingOutput.set(id, { audioSeen: false });
|
||||
else untrackedOutput = true;
|
||||
}
|
||||
const pending = pendingOutput.get(id);
|
||||
if (
|
||||
event?.type === 'output_audio_buffer.started' ||
|
||||
event?.type === 'output_audio_buffer.cleared'
|
||||
pending &&
|
||||
(type?.startsWith('response.output_audio') ||
|
||||
type?.startsWith('response.audio') ||
|
||||
['audio', 'output_audio'].includes(event?.part?.type))
|
||||
)
|
||||
outputKnown = true;
|
||||
pending.audioSeen = true;
|
||||
if (
|
||||
type === 'response.done' &&
|
||||
pending &&
|
||||
!pending.audioSeen &&
|
||||
hasNoAudioOutput(event.response)
|
||||
)
|
||||
pendingOutput.delete(id);
|
||||
if (
|
||||
type === 'output_audio_buffer.started' ||
|
||||
type === 'output_audio_buffer.cleared'
|
||||
) {
|
||||
// Native playback events establish the transport's output lifecycle.
|
||||
// Generation completion alone never establishes audio drainage.
|
||||
untrackedOutput = false;
|
||||
if (id) pendingOutput.delete(id);
|
||||
else pendingOutput.clear();
|
||||
}
|
||||
refreshOutputKnowledge();
|
||||
notify();
|
||||
});
|
||||
} else {
|
||||
@@ -183,12 +240,23 @@ export function observeNativeActivity({
|
||||
forwarded[0] = decorated;
|
||||
current = Reflect.apply(original, this, forwarded);
|
||||
if (!dead && stamp === version) {
|
||||
retireAudio();
|
||||
audio = current;
|
||||
inputBusy = false;
|
||||
for (const key of ['play', 'clearPlayback', 'close'])
|
||||
wrap(current, key, () => {
|
||||
if (audio === current && controller.audio === current) notify();
|
||||
});
|
||||
wrap(
|
||||
current,
|
||||
key,
|
||||
() => {
|
||||
if (audio !== current) return;
|
||||
if (key === 'close') {
|
||||
retireAudio();
|
||||
inputBusy = false;
|
||||
}
|
||||
if (controller.audio === current) notify();
|
||||
},
|
||||
audioRestores,
|
||||
);
|
||||
}
|
||||
return current;
|
||||
}
|
||||
@@ -203,7 +271,9 @@ export function observeNativeActivity({
|
||||
if (dead) return;
|
||||
dead = true;
|
||||
version++;
|
||||
for (const restore of restores.reverse()) restore();
|
||||
retireAudio();
|
||||
pendingOutput.clear();
|
||||
for (const restore of restores.splice(0).reverse()) restore();
|
||||
signal?.removeEventListener('abort', destroy);
|
||||
}
|
||||
signal?.addEventListener('abort', destroy, { once: true });
|
||||
|
||||
@@ -471,3 +471,202 @@ test('native typed input observation preserves receiver/result, refuses failed s
|
||||
old.call(controller, 'late');
|
||||
assert.equal(inputs, 1);
|
||||
});
|
||||
|
||||
test('explicit silent OpenAI completion rearms expiry, including cancelled and failed responses', () => {
|
||||
for (const status of ['completed', 'cancelled', 'failed', 'incomplete']) {
|
||||
const turns = new RealtimeTurns({});
|
||||
const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
|
||||
let idle;
|
||||
let timer;
|
||||
let expired = 0;
|
||||
const observer = observeNativeActivity({
|
||||
adapter: { controller },
|
||||
nativeProvider: 'openai',
|
||||
onChange: () => idle?.update(),
|
||||
});
|
||||
idle = createVoiceInactivity({
|
||||
readActivity: observer.snapshot,
|
||||
readPending: () => false,
|
||||
setTimer(callback) {
|
||||
timer = callback;
|
||||
return 1;
|
||||
},
|
||||
clearTimer() {
|
||||
timer = null;
|
||||
},
|
||||
onExpire: () => expired++,
|
||||
});
|
||||
controller.status = 'listening';
|
||||
controller.dc = {};
|
||||
observer.handle({ type: 'state', state: 'listening' });
|
||||
assert.equal(typeof timer, 'function');
|
||||
turns.responseActive = true;
|
||||
turns.observeSpeech({
|
||||
type: 'response.created',
|
||||
response: { id: 'silent' },
|
||||
});
|
||||
assert.equal(timer, null);
|
||||
turns.observeSpeech({
|
||||
type: 'response.done',
|
||||
response: {
|
||||
id: 'silent',
|
||||
status,
|
||||
output:
|
||||
status === 'completed'
|
||||
? [{ type: 'function_call', name: 'set_panel_open' }]
|
||||
: [],
|
||||
},
|
||||
});
|
||||
// The provider updates generation state after forwarding the raw event.
|
||||
assert.equal(timer, null);
|
||||
turns.responseActive = false;
|
||||
observer.handle({ type: 'completion' });
|
||||
assert.equal(observer.snapshot().known, true);
|
||||
assert.equal(typeof timer, 'function');
|
||||
timer();
|
||||
assert.equal(expired, 1);
|
||||
idle.destroy();
|
||||
observer.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('silent OpenAI replies cannot clear unresolved older audio or incomplete output metadata', () => {
|
||||
const turns = new RealtimeTurns({});
|
||||
const observer = observeNativeActivity({
|
||||
adapter: {
|
||||
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
|
||||
},
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
const created = (id) =>
|
||||
turns.observeSpeech({ type: 'response.created', response: { id } });
|
||||
const done = (id, output) =>
|
||||
turns.observeSpeech({
|
||||
type: 'response.done',
|
||||
response: { id, status: 'completed', output },
|
||||
});
|
||||
created('audio');
|
||||
turns.observeSpeech({
|
||||
type: 'response.output_audio.delta',
|
||||
response_id: 'audio',
|
||||
});
|
||||
created('silent');
|
||||
done('silent', []);
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
done('audio', []); // Even contradictory completion metadata cannot erase audio evidence.
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
turns.observeSpeech({
|
||||
type: 'output_audio_buffer.started',
|
||||
response_id: 'audio',
|
||||
});
|
||||
assert.equal(observer.snapshot().outputBusy, true);
|
||||
done('silent', []);
|
||||
assert.equal(observer.snapshot().outputBusy, true);
|
||||
turns.observeSpeech({
|
||||
type: 'output_audio_buffer.stopped',
|
||||
response_id: 'audio',
|
||||
});
|
||||
assert.equal(observer.snapshot().outputBusy, false);
|
||||
created('unknown');
|
||||
done('unknown', undefined);
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
done('unobserved', []);
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
observer.destroy();
|
||||
});
|
||||
|
||||
test('overlapping silent replies resolve independently and audio output items remain unknown until playback', () => {
|
||||
const turns = new RealtimeTurns({});
|
||||
const observer = observeNativeActivity({
|
||||
adapter: {
|
||||
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
|
||||
},
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
for (const id of ['a', 'b'])
|
||||
turns.observeSpeech({ type: 'response.created', response: { id } });
|
||||
for (const id of ['a', 'b']) {
|
||||
turns.observeSpeech({
|
||||
type: 'response.done',
|
||||
response: {
|
||||
id,
|
||||
status: 'completed',
|
||||
output: [
|
||||
{ type: 'message', content: [{ type: 'text', text: 'Done' }] },
|
||||
],
|
||||
},
|
||||
});
|
||||
assert.equal(observer.snapshot().known, id === 'b');
|
||||
}
|
||||
turns.observeSpeech({ type: 'response.created', response: { id: 'audio' } });
|
||||
turns.observeSpeech({
|
||||
type: 'response.done',
|
||||
response: {
|
||||
id: 'audio',
|
||||
status: 'completed',
|
||||
output: [{ type: 'message', content: [{ type: 'audio' }] }],
|
||||
},
|
||||
});
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
observer.destroy();
|
||||
});
|
||||
|
||||
test('silent completion cannot establish output knowledge when observer attached mid-session', () => {
|
||||
const turns = new RealtimeTurns({});
|
||||
const observer = observeNativeActivity({
|
||||
adapter: {
|
||||
controller: { status: 'listening', dc: {}, _turns: turns, _input: {} },
|
||||
},
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
turns.observeSpeech({ type: 'response.created', response: { id: 'new' } });
|
||||
turns.observeSpeech({
|
||||
type: 'response.done',
|
||||
response: { id: 'new', status: 'completed', output: [] },
|
||||
});
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
observer.destroy();
|
||||
});
|
||||
|
||||
test('Gemini restores each retired audio immediately across repeated same-mount restarts', () => {
|
||||
const originals = new Map();
|
||||
const controller = {
|
||||
input: {},
|
||||
createAudio() {
|
||||
const audio = { pending: 0, play() {}, clearPlayback() {}, close() {} };
|
||||
originals.set(audio, { ...audio });
|
||||
return audio;
|
||||
},
|
||||
handleMessage() {},
|
||||
};
|
||||
const observer = observeNativeActivity({
|
||||
adapter: { controller },
|
||||
nativeProvider: 'gemini',
|
||||
});
|
||||
const assertRestored = (audio) => {
|
||||
for (const key of ['play', 'clearPlayback', 'close'])
|
||||
assert.equal(audio[key], originals.get(audio)[key]);
|
||||
};
|
||||
for (let i = 0; i < 5; i++) {
|
||||
observer.handle({ type: 'state', state: 'connecting' });
|
||||
controller.audio = controller.createAudio({});
|
||||
const audio = controller.audio;
|
||||
observer.handle({ type: 'state', state: 'listening' });
|
||||
assert.notEqual(audio.close, originals.get(audio).close);
|
||||
audio.close();
|
||||
assertRestored(audio);
|
||||
controller.audio = null;
|
||||
observer.handle({ type: 'state', state: 'idle' });
|
||||
}
|
||||
controller.audio = controller.createAudio({});
|
||||
const replaced = controller.audio;
|
||||
controller.audio = controller.createAudio({});
|
||||
assertRestored(replaced);
|
||||
const current = controller.audio;
|
||||
const replacement = () => {};
|
||||
current.play = replacement;
|
||||
observer.handle({ type: 'state', state: 'error' });
|
||||
assert.equal(current.play, replacement);
|
||||
assert.equal(current.close, originals.get(current).close);
|
||||
observer.destroy();
|
||||
});
|
||||
Reference in new issue
Block a user