mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 10:53:59 +00:00
feat(voice): harden Gemini Live provider integration
This commit is contained in:
1 parent
d4492d8f8c
commit
4fde920665
102 files changed
+28528
-1581
No files matched your search
@@ -3,6 +3,9 @@ import { createVoiceSession } from './session.js';
|
||||
import { VoiceCardControls } from './voiceCard.js';
|
||||
import { PointerReticle } from './pointerReticle.js';
|
||||
|
||||
const DEFAULT_ERROR_HINT =
|
||||
'Check microphone permission and network access, then try again.';
|
||||
|
||||
function createDefaultReticle(pointer) {
|
||||
const element = pointer?.element;
|
||||
const doc = element?.ownerDocument;
|
||||
@@ -24,15 +27,37 @@ export function createVoiceCommands({
|
||||
signal,
|
||||
debugSink,
|
||||
createControl = createVoiceControl,
|
||||
resetExisting = true,
|
||||
provider,
|
||||
supportedProviders,
|
||||
onProviderChange,
|
||||
onSessionEvent,
|
||||
disposeRunner = true,
|
||||
createCard = (options) => new VoiceCardControls(options),
|
||||
pointer = null,
|
||||
referents = null,
|
||||
createReticle = createDefaultReticle,
|
||||
}) {
|
||||
window.__gevVoiceCommands?.stop?.({ removeUi: true });
|
||||
if (resetExisting) window.__gevVoiceCommands?.stop?.({ removeUi: true });
|
||||
const ui = createControl({ reset: true });
|
||||
const hasProviderChoice = () =>
|
||||
new Set(
|
||||
Array.from(ui.providerSelect?.options || [], (option) =>
|
||||
option.disabled ? null : option.value,
|
||||
).filter(Boolean),
|
||||
).size > 1;
|
||||
const session = createVoiceSession({
|
||||
runner,
|
||||
providerDropdown: () =>
|
||||
Boolean(
|
||||
onProviderChange &&
|
||||
ui.providerSelect &&
|
||||
!ui.providerSelect.disabled &&
|
||||
!ui.providerSelect.hidden &&
|
||||
ui.providerField &&
|
||||
hasProviderChoice() &&
|
||||
!ui.providerField.hidden,
|
||||
),
|
||||
signal,
|
||||
createAdapter: (hooks) =>
|
||||
createSession({
|
||||
@@ -49,6 +74,27 @@ export function createVoiceCommands({
|
||||
});
|
||||
const adapter = session.adapter;
|
||||
const capabilities = adapter.capabilities || {};
|
||||
const providerHandler = () => onProviderChange?.(ui.providerSelect.value);
|
||||
if (ui.providerSelect && onProviderChange) {
|
||||
if (supportedProviders && ui.providerSelect.replaceChildren) {
|
||||
const doc = ui.providerSelect.ownerDocument;
|
||||
ui.providerSelect.replaceChildren(
|
||||
...supportedProviders.map((name) => {
|
||||
const option = doc.createElement('option');
|
||||
option.value = name;
|
||||
option.textContent = name === 'openai' ? 'OpenAI' : 'Gemini';
|
||||
return option;
|
||||
}),
|
||||
);
|
||||
}
|
||||
ui.providerSelect.value = provider;
|
||||
if (capabilities.advisoryLimit)
|
||||
ui.providerSelect.title = `Changing provider reconnects voice if it is on. ${capabilities.advisoryLimit}`;
|
||||
ui.root.dataset.provider = provider;
|
||||
ui.providerSelect.addEventListener('change', providerHandler);
|
||||
}
|
||||
if (ui.providerField)
|
||||
ui.providerField.hidden = !onProviderChange || !hasProviderChoice();
|
||||
if (ui.tierButton) ui.tierButton.hidden = !capabilities.costControls;
|
||||
if (ui.costValue) ui.costValue.hidden = !capabilities.costControls;
|
||||
if (!capabilities.pushToTalk) {
|
||||
@@ -58,7 +104,9 @@ export function createVoiceCommands({
|
||||
// Retain the existing controller's inspection surface for browser tools.
|
||||
const controls = adapter.controller || session;
|
||||
controls.session = session;
|
||||
controls.ui = ui;
|
||||
const updateStatus = session.subscribe((event) => {
|
||||
onSessionEvent?.(event);
|
||||
if (event.type !== 'state') return;
|
||||
ui.root.dataset.status = event.state;
|
||||
ui.status.textContent =
|
||||
@@ -71,6 +119,11 @@ export function createVoiceCommands({
|
||||
event.state === 'error'
|
||||
? event.detail || 'Voice could not be started.'
|
||||
: '';
|
||||
if (ui.errorHint)
|
||||
ui.errorHint.textContent =
|
||||
event.state === 'error' && event.recovery
|
||||
? event.recovery
|
||||
: DEFAULT_ERROR_HINT;
|
||||
if (event.state === 'error') ui.root.classList?.remove('error-dismissed');
|
||||
});
|
||||
// Captions, plan and result card nested in the control; it only reads
|
||||
@@ -109,26 +162,34 @@ export function createVoiceCommands({
|
||||
'abort',
|
||||
() => {
|
||||
ui.button.removeEventListener('click', buttonHandler);
|
||||
ui.providerSelect?.removeEventListener('change', providerHandler);
|
||||
annotationUnsubscribe?.();
|
||||
updateStatus();
|
||||
updateCard?.();
|
||||
updatePointer();
|
||||
reticle?.destroy();
|
||||
card?.destroy();
|
||||
runner.dispose?.();
|
||||
pointer?.clear();
|
||||
referents?.clear();
|
||||
runner.resetConversation?.();
|
||||
if (disposeRunner) runner.dispose?.();
|
||||
ui.root.remove();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
if (session.disposed) {
|
||||
ui.button.removeEventListener('click', buttonHandler);
|
||||
ui.providerSelect?.removeEventListener('change', providerHandler);
|
||||
annotationUnsubscribe?.();
|
||||
updateStatus();
|
||||
updateCard?.();
|
||||
updatePointer();
|
||||
reticle?.destroy();
|
||||
card?.destroy();
|
||||
runner.dispose?.();
|
||||
pointer?.clear();
|
||||
referents?.clear();
|
||||
runner.resetConversation?.();
|
||||
if (disposeRunner) runner.dispose?.();
|
||||
ui.root.remove();
|
||||
} else adapter.bindControls?.();
|
||||
window.__gevVoiceCommands = controls;
|
||||
|
||||
Reference in new issue
Block a user