Files
gods-eye-view/src/voice/panel/nativeActivity.js
T

293 lines
9.4 KiB
JavaScript

/** Inspected native capabilities only; scheduled/reported output is not acoustics. */
export function observeNativeActivity({
adapter,
nativeProvider,
signal,
onChange,
onInput,
}) {
const native = nativeProvider === 'openai' || nativeProvider === 'gemini';
// Do not even inspect custom controller properties.
const controller = native ? adapter.controller : null;
const input = native
? nativeProvider === 'gemini'
? controller.input
: controller._input
: null;
let dead = false,
connected = false,
inputBusy = false,
recoverableError = false;
function cleanNativeOutput() {
if (nativeProvider === 'gemini') return true;
if (nativeProvider !== 'openai') return false;
const turns = controller._turns;
return (
['idle', 'connecting', 'error'].includes(controller.status) &&
controller.dc == null &&
!turns.assistantAudioPlaying &&
!turns.responseActive &&
!turns.responseCreatePending &&
!turns.userTurnPending &&
!turns.activeResponseId &&
!turns.progressResponse
);
}
let outputKnown = cleanNativeOutput();
let audio = null,
version = 0;
const restores = [];
const audioRestores = [];
const pendingOutput = new Map();
let untrackedOutput = !outputKnown;
function retireAudio() {
audio = null;
for (const restore of audioRestores.splice(0).reverse()) restore();
}
function refreshOutputKnowledge() {
outputKnown = !untrackedOutput && pendingOutput.size === 0;
}
function hasNoAudioOutput(response) {
return (
['completed', 'cancelled', 'failed', 'incomplete'].includes(
response?.status,
) &&
Array.isArray(response.output) &&
response.output.every(
(item) =>
item?.type === 'function_call' ||
(item?.type === 'message' &&
Array.isArray(item.content) &&
item.content.every((part) =>
['text', 'output_text', 'refusal'].includes(part?.type),
)),
)
);
}
function snapshot() {
const turns = nativeProvider === 'openai' ? controller?._turns : controller;
return {
known: native && outputKnown && !dead,
connected: connected && !dead,
held: Boolean(input?.pushToTalkKeyHeld),
inputBusy: Boolean(
inputBusy ||
(nativeProvider === 'gemini' && controller?.userActivityOpen),
),
outputBusy: Boolean(
nativeProvider === 'gemini'
? audio && audio === controller.audio && audio.pending
: turns?.assistantAudioPlaying,
),
generationBusy: Boolean(
turns?.responseActive ||
turns?.responseCreatePending ||
(nativeProvider === 'openai' && turns?.userTurnPending) ||
(nativeProvider === 'gemini' &&
(turns?.replyPending ||
turns?.retiredReplyPending ||
turns?.deferredInput ||
turns?.automaticTurnOpen ||
turns?.pendingTools ||
turns?.controlQueue?.length)),
),
};
}
function notify() {
if (!dead) {
try {
onChange?.(snapshot());
} catch {}
}
}
function wrap(object, key, after, ownedRestores = restores) {
const original = object?.[key];
if (typeof original !== 'function') return;
function wrapped(...args) {
const stamp = version;
const result = Reflect.apply(original, this, args);
if (!dead && stamp === version) {
try {
after?.(args, result);
} catch {}
}
return result;
}
object[key] = wrapped;
ownedRestores.push(() => {
if (object[key] === wrapped) object[key] = original;
});
}
function handle(event) {
if (dead || !native) return;
if (event?.type === 'state') {
const state = event.state;
const wasConnected = connected;
const liveError =
state === 'error' &&
nativeProvider === 'openai' &&
controller.dc?.readyState === 'open';
const restartingAfterError = state === 'connecting' && recoverableError;
recoverableError = liveError;
connected = !['idle', 'connecting', 'error'].includes(state);
// Connecting has multiple native phases around audio allocation. Retire
// a previous connected generation once, not each phase of this start.
// A failed reply can briefly report error on a live OpenAI transport.
// Preserve its output evidence through recovery to listening; a later
// reconnect still retires the failed connection's generation.
if (
state === 'idle' ||
(state === 'error' && !liveError) ||
(state === 'connecting' && (wasConnected || restartingAfterError))
) {
version++;
retireAudio();
inputBusy = false;
outputKnown = cleanNativeOutput();
untrackedOutput = !outputKnown;
pendingOutput.clear();
}
}
notify();
}
if (native) {
wrap(controller, 'sendTextCommand', ([text], result) => {
if (String(text || '').trim() && result !== false) onInput?.();
});
for (const key of [
'setMicrophoneEnabled',
'releasePushToTalkKey',
'resetSession',
])
wrap(input, key, notify);
if (nativeProvider === 'openai') {
wrap(controller._turns, 'observeSpeech', ([event]) => {
if (event?.type === 'input_audio_buffer.speech_started')
inputBusy = true;
if (event?.type === 'input_audio_buffer.speech_stopped')
inputBusy = false;
const type = event?.type;
const id = event?.response_id || event?.response?.id;
// Keep each unresolved response separate: a later silent reply cannot
// establish that an earlier audio reply has drained.
if (type === 'response.created') {
if (id && pendingOutput.size < 64)
pendingOutput.set(id, { audioSeen: false });
else untrackedOutput = true;
}
const pending = pendingOutput.get(id);
if (
pending &&
(type?.startsWith('response.output_audio') ||
type?.startsWith('response.audio') ||
['audio', 'output_audio'].includes(event?.part?.type))
)
pending.audioSeen = true;
if (
type === 'response.done' &&
pending &&
!pending.audioSeen &&
hasNoAudioOutput(event.response)
)
pendingOutput.delete(id);
if (
type === 'output_audio_buffer.started' ||
type === 'output_audio_buffer.cleared'
) {
// Native playback events establish the transport's output lifecycle.
// Generation completion alone never establishes audio drainage.
untrackedOutput = false;
if (id) pendingOutput.delete(id);
else pendingOutput.clear();
}
refreshOutputKnowledge();
notify();
});
} else {
wrap(controller, 'handleMessage', notify);
const original = controller.createAudio;
function createAudio(...args) {
const stamp = version;
const options = args[0];
let current;
const decorated = { ...options };
for (const key of [
'onAudioStart',
'onAudioEnd',
'onAudioDiscard',
'onDrain',
]) {
const callback = options?.[key];
decorated[key] = function (...values) {
const result =
typeof callback === 'function'
? Reflect.apply(
callback,
this === decorated ? options : this,
values,
)
: undefined;
if (
!dead &&
stamp === version &&
audio === current &&
controller.audio === current
) {
if (
key === 'onAudioStart' &&
options.pushToTalk &&
controller.captureOwner?.admitted
)
inputBusy = true;
if (key === 'onAudioEnd' || key === 'onAudioDiscard')
inputBusy = false;
notify();
}
return result;
};
}
const forwarded = args.slice();
forwarded[0] = decorated;
current = Reflect.apply(original, this, forwarded);
if (!dead && stamp === version) {
retireAudio();
audio = current;
inputBusy = false;
for (const key of ['play', 'clearPlayback', 'close'])
wrap(
current,
key,
() => {
if (audio !== current) return;
if (key === 'close') {
retireAudio();
inputBusy = false;
}
if (controller.audio === current) notify();
},
audioRestores,
);
}
return current;
}
controller.createAudio = createAudio;
restores.push(() => {
if (controller.createAudio === createAudio)
controller.createAudio = original;
});
}
}
function destroy() {
if (dead) return;
dead = true;
version++;
retireAudio();
pendingOutput.clear();
for (const restore of restores.splice(0).reverse()) restore();
signal?.removeEventListener('abort', destroy);
}
signal?.addEventListener('abort', destroy, { once: true });
if (signal?.aborted) destroy();
return { snapshot, handle, destroy };
}