Files
gods-eye-view/src/voice/realtimeInput.js
T
Sameh Khamis 874f83ba48 fix(voice): messages worded before reaching the status also follow a language change (#1661)
The radio and push-to-talk helpers shared by both voice providers now pass
English catalog text, and the OpenAI controller words its detail when shown,
like the Gemini session. A token failure without a server message carries
its status as sourceParams, so it is worded again with the number. The docs
say sourceDetail is the message as given, catalog text when it is the app's
own.
2026-11-10 18:11:07 -08:00

422 lines
15 KiB
JavaScript

import { msg, t } from '../i18n/index.js';
import {
PUSH_TO_TALK_HOLD_DELAY_MS,
MICROPHONE_VISUALIZER_GATE,
ASSISTANT_VISUALIZER_GATE,
isPushToTalkKey,
SPACE_INTERACTIVE_SELECTOR,
isInteractiveSpaceTarget,
shouldHandlePushToTalkKeyDown,
selectVoiceVisualizerSignal,
resolveVoiceVisualizerSpeaker,
resolveVoiceControlHint,
gateVoiceVisualizerLevel,
} from './realtimeInputPolicy.js';
import { VOICE_VISUALIZER_BARS } from './control.js';
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
/** Own physical push-to-talk gestures, microphone controls and audio meters. */
export class RealtimeInput {
constructor({ readUi, readStream, readStatus, operations }) {
Object.assign(this, { readUi, readStream, readStatus }, operations);
this.visualizerAudioContext = null;
this.visualizerAnalyser = null;
this.visualizerSource = null;
this.visualizerFrame = null;
this.visualizerData = null;
this.visualizerOutputSource = null;
this.visualizerOutputAnalyser = null;
this.visualizerOutputData = null;
this.visualizerGeneration = 0;
this.visualizerSpeaker = 'idle';
this.pushToTalkMode = false;
this.pushToTalkKeyHeld = false;
this.spaceKeyHeld = false;
this.pushToTalkHoldTimer = null;
this.pushToTalkHoldGeneration = 0;
this.pushToTalkHoldFocusOwner = null;
this.pushToTalkHoldControl = null;
this.pushToTalkHoldPreservesNative = false;
this.shortcutKeyDownHandler = null;
this.shortcutKeyUpHandler = null;
this.shortcutBlurHandler = null;
this.shortcutVisibilityHandler = null;
}
get ui() {
return this.readUi();
}
get stream() {
return this.readStream();
}
get status() {
return this.readStatus();
}
/** Registers delayed hold-Space push-to-talk while preserving short control taps. */
bindPushToTalkShortcut() {
if (this.shortcutKeyDownHandler) return;
this.shortcutKeyDownHandler = (event) => {
// Space types in chat; it never talks while chat is open.
if (
this.ui.state?.chat?.open === true ||
!shouldHandlePushToTalkKeyDown(event)
)
return;
if (event.repeat) {
if (this.spaceKeyHeld && !this.pushToTalkHoldPreservesNative)
event.preventDefault();
if (this.pushToTalkKeyHeld) event.preventDefault();
return;
}
this.spaceKeyHeld = true;
this.pushToTalkHoldFocusOwner = document.activeElement;
this.pushToTalkHoldControl = isInteractiveSpaceTarget(
document.activeElement,
)
? document.activeElement
: isInteractiveSpaceTarget(event.target)
? event.target.closest?.(SPACE_INTERACTIVE_SELECTOR) || event.target
: null;
this.pushToTalkHoldPreservesNative = Boolean(this.pushToTalkHoldControl);
// Background Space is reserved immediately to avoid scrolling. A focused
// control keeps its native keydown and release unless the hold is claimed.
if (!this.pushToTalkHoldPreservesNative) event.preventDefault();
// A click-started session is intentionally open-mic. Space only claims an
// idle session (or a session it already started) so releasing the key can
// never surprise the user by muting a click-started conversation.
if (this.isActive() && !this.pushToTalkMode) return;
this.cancelPushToTalkHold();
// Point-and-ask: the user decided to ask at keydown, so that is where
// the pointer is read. A short tap releases it below.
this.holdPointer?.();
const holdGeneration = ++this.pushToTalkHoldGeneration;
this.pushToTalkHoldTimer = setTimeout(() => {
this.pushToTalkHoldTimer = null;
if (holdGeneration !== this.pushToTalkHoldGeneration) return;
if (!this.spaceKeyHeld || this.pushToTalkKeyHeld) return;
// A pointer or Tab focus change during the delay cancels the original
// gesture rather than letting voice claim a key held for another owner.
if (document.activeElement !== this.pushToTalkHoldFocusOwner) return;
if (
document.visibilityState === 'hidden' ||
(typeof document.hasFocus === 'function' && !document.hasFocus())
)
return;
if (this.isActive() && !this.pushToTalkMode) return;
// Blur before voice starts. This removes the focused state and ensures
// the eventual Space release cannot activate the old control.
if (
this.pushToTalkHoldControl &&
document.activeElement === this.pushToTalkHoldControl &&
typeof this.pushToTalkHoldControl.blur === 'function'
) {
this.pushToTalkHoldControl.blur();
}
// Read Radio's claim before pausing it: pausing can change the very
// handoff state that decides whether voice may interrupt.
const mayInterrupt = this.mayClaimSpeaker?.() ?? true;
this.pauseRadioForVoice();
this.pushToTalkKeyHeld = true;
if (this.isActive()) {
// A claimed hold is an explicit request to speak: cut the assistant
// off if it is talking. Short taps never reach this point.
if (mayInterrupt) this.bargeIn?.();
this.beginPointerTurn?.('keydown');
this.ui.update('root', { pushToTalk: 'held' });
this.setMicrophoneEnabled(true);
if (this.status === 'listening')
this.setStatus('listening', msg('Release Space to send'));
} else {
this.start({ pushToTalk: true });
// start() performs a controlled stop() before connecting. Restore the
// marker it clears so the UI reflects this still-held gesture.
if (this.pushToTalkKeyHeld)
this.ui.update('root', { pushToTalk: 'held' });
}
}, PUSH_TO_TALK_HOLD_DELAY_MS);
};
this.shortcutKeyUpHandler = (event) => {
if (!isPushToTalkKey(event)) return;
const wasHoldingSpace = this.spaceKeyHeld;
const preservedNativeActivation = this.pushToTalkHoldPreservesNative;
this.spaceKeyHeld = false;
this.cancelPushToTalkHold();
if (!this.pushToTalkKeyHeld) {
if (wasHoldingSpace) this.releasePointerHold?.();
if (wasHoldingSpace && !preservedNativeActivation)
event.preventDefault();
this.resetPushToTalkGesture();
return;
}
event.preventDefault();
this.releasePushToTalkKey();
this.resetPushToTalkGesture();
};
this.shortcutBlurHandler = () => {
this.spaceKeyHeld = false;
this.cancelPushToTalkHold();
this.releasePushToTalkKey();
this.resetPushToTalkGesture();
};
this.shortcutVisibilityHandler = () => {
if (document.visibilityState === 'hidden') this.shortcutBlurHandler();
};
// Observe before custom controls call preventDefault for their own Space
// behavior. This lets the same physical hold cross the 500ms voice
// threshold while short taps still reach the control unchanged.
document.addEventListener('keydown', this.shortcutKeyDownHandler, true);
document.addEventListener('keyup', this.shortcutKeyUpHandler, true);
window.addEventListener('blur', this.shortcutBlurHandler);
document.addEventListener(
'visibilitychange',
this.shortcutVisibilityHandler,
);
}
/** Cancels an unclaimed Space hold before it starts voice. */
cancelPushToTalkHold() {
this.pushToTalkHoldGeneration++;
if (this.pushToTalkHoldTimer === null) return;
clearTimeout(this.pushToTalkHoldTimer);
this.pushToTalkHoldTimer = null;
}
/** Clears the owner metadata for the current physical Space gesture. */
resetPushToTalkGesture() {
this.pushToTalkHoldFocusOwner = null;
this.pushToTalkHoldControl = null;
this.pushToTalkHoldPreservesNative = false;
}
/**
* Mutes a keyboard-started microphone while leaving WebRTC alive for the reply.
* @returns {void}
*/
releasePushToTalkKey() {
this.cancelPushToTalkHold();
// The pointer snapshot belongs to this gesture; release or blur ends it.
this.releasePointerHold?.();
if (!this.pushToTalkKeyHeld) return;
this.pushToTalkKeyHeld = false;
this.ui.update('root', { pushToTalk: null });
if (!this.pushToTalkMode) return;
this.setMicrophoneEnabled(false);
if (this.status === 'listening')
this.setStatus('listening', msg('Hold Space to talk'));
else this.updateVoiceButtonLabel();
}
/**
* Enables or mutes only the outbound microphone tracks.
* @param {boolean} enabled
* @returns {void}
*/
setMicrophoneEnabled(enabled) {
this.ui?.update('root', { microphone: enabled ? 'active' : 'muted' });
this.stream?.getAudioTracks?.().forEach((track) => {
track.enabled = Boolean(enabled);
});
}
/**
* Drives the dock waveform from live microphone energy while voice is active.
* @param {MediaStream} stream
* @returns {void}
*/
startVoiceVisualizer(stream) {
this.stopVoiceVisualizer();
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
if (!AudioContextClass || !stream || !this.ui) return;
try {
const context = new AudioContextClass();
this.visualizerAudioContext = context;
context.resume().catch(() => {});
const analyser = context.createAnalyser();
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.72;
const source = context.createMediaStreamSource(stream);
source.connect(analyser);
this.visualizerAnalyser = analyser;
this.visualizerSource = source;
this.visualizerData = new Uint8Array(analyser.frequencyBinCount);
const generation = this.visualizerGeneration;
const render = () => {
if (generation !== this.visualizerGeneration) return;
const signal = selectVoiceVisualizerSignal(
this.visualizerSpeaker,
{
analyser: this.visualizerAnalyser,
data: this.visualizerData,
},
{
analyser: this.visualizerOutputAnalyser,
data: this.visualizerOutputData,
},
);
if (!signal) {
this.restVoiceVisualizer();
this.visualizerFrame = requestAnimationFrame(render);
return;
}
signal.analyser.getByteFrequencyData(signal.data);
const binCount = signal.data.length;
const count = VOICE_VISUALIZER_BARS;
const bars = Array.from({ length: count }, (_, index) => {
const start = Math.floor((index / count) * binCount);
const end = Math.max(
start + 1,
Math.floor(((index + 1) / count) * binCount),
);
let energy = 0;
for (let bin = start; bin < end; bin++) energy += signal.data[bin];
const normalized = Math.min(1, energy / (end - start) / 190);
const gate =
this.visualizerSpeaker === 'ai'
? ASSISTANT_VISUALIZER_GATE
: MICROPHONE_VISUALIZER_GATE;
const shaped = Math.pow(
gateVoiceVisualizerLevel(normalized, gate),
0.72,
);
return {
level: `${Math.round(5 + shaped * 29)}px`,
opacity: `${(0.5 + shaped * 0.5).toFixed(2)}`,
};
});
this.ui?.setBars(bars);
this.visualizerFrame = requestAnimationFrame(render);
};
render();
} catch {
this.stopVoiceVisualizer();
}
}
/**
* Adds the incoming assistant audio stream to the existing Web Audio meter.
* The audio element remains responsible for playback; this branch only reads
* its frequency energy for the visualizer.
* @param {MediaStream} stream
* @returns {void}
*/
startAssistantVoiceVisualizer(stream) {
const context = this.visualizerAudioContext;
if (!context || !stream) return;
try {
try {
this.visualizerOutputSource?.disconnect();
} catch {
/* no-op */
}
const analyser = context.createAnalyser();
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.72;
const source = context.createMediaStreamSource(stream);
source.connect(analyser);
this.visualizerOutputSource = source;
this.visualizerOutputAnalyser = analyser;
this.visualizerOutputData = new Uint8Array(analyser.frequencyBinCount);
} catch {
// Playback continues through audioEl even if a browser declines analysis.
this.visualizerOutputSource = null;
this.visualizerOutputAnalyser = null;
this.visualizerOutputData = null;
}
}
/**
* Releases the microphone meter and restores its five-pixel baseline.
* @returns {void}
*/
stopVoiceVisualizer() {
this.visualizerGeneration++;
if (this.visualizerFrame) cancelAnimationFrame(this.visualizerFrame);
this.visualizerFrame = null;
try {
this.visualizerSource?.disconnect();
} catch {
/* no-op */
}
try {
this.visualizerOutputSource?.disconnect();
} catch {
/* no-op */
}
this.visualizerSource = null;
this.visualizerAnalyser = null;
this.visualizerData = null;
this.visualizerOutputSource = null;
this.visualizerOutputAnalyser = null;
this.visualizerOutputData = null;
this.visualizerSpeaker = 'idle';
if (this.visualizerAudioContext) {
this.visualizerAudioContext.close().catch(() => {});
this.visualizerAudioContext = null;
}
this.restVoiceVisualizer();
}
/** Return the waveform to its CSS standby baseline. */
restVoiceVisualizer() {
this.ui?.setBars(null);
}
/**
* Keeps the microphone caption in sync with click and hold-to-talk modes.
* @returns {void}
*/
updateVoiceButtonLabel() {
if (!this.ui) return;
// The mic button's caption; the first-run tip names it the same way.
this.ui.update('mic', { text: () => t('MIC') });
const { pushToTalkMode, pushToTalkKeyHeld } = this;
this.ui.update('help', {
text: () => resolveVoiceControlHint(pushToTalkMode, pushToTalkKeyHeld),
});
}
setVoiceSpeaker(speaker, { keepVisualizerSpeaker = false } = {}) {
const nextSpeaker =
speaker === 'user' || speaker === 'ai' ? speaker : 'idle';
this.visualizerSpeaker = resolveVoiceVisualizerSpeaker(
this.visualizerSpeaker,
nextSpeaker,
keepVisualizerSpeaker,
);
this.ui.update('root', { speaker: nextSpeaker });
if (shouldPauseRadioForVoice({ speaker: nextSpeaker }))
this.pauseRadioForVoice();
}
resetSession() {
this.pushToTalkMode = false;
this.pushToTalkKeyHeld = false;
this.spaceKeyHeld = false;
this.resetPushToTalkGesture();
this.ui?.update('root', { pushToTalk: null, microphone: null });
}
detachBindings() {
if (this.shortcutKeyDownHandler)
document.removeEventListener(
'keydown',
this.shortcutKeyDownHandler,
true,
);
if (this.shortcutKeyUpHandler)
document.removeEventListener('keyup', this.shortcutKeyUpHandler, true);
if (this.shortcutBlurHandler)
window.removeEventListener('blur', this.shortcutBlurHandler);
if (this.shortcutVisibilityHandler) {
document.removeEventListener(
'visibilitychange',
this.shortcutVisibilityHandler,
);
}
this.shortcutKeyDownHandler = null;
this.shortcutKeyUpHandler = null;
this.shortcutBlurHandler = null;
this.shortcutVisibilityHandler = null;
}
}