mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 08:12:24 +00:00
Chat sits next to the mic in the command dock. Replies are silent and never request the microphone; the first message connects with the selected provider and runs the same map actions as voice through each provider's text API (OpenAI OPENAI_CHAT_MODEL, Gemini GEMINI_CHAT_MODEL via generateContent, which never starts Live). Only one mode runs at a time. - Chat panel rendered by the voice control presenter: conversation, recent and quick-command suggestions, Up/Down history, attachments (map-layer references or small UTF-8 text files, sent only with the message) - Chat and voice panels sit above the dock or dock into the right rail's column; minimized bars sit centred above the dock, and height animates on swaps - A centred panel steps down to its bar for the Location and Visual Presets trays without moving focus, rests above an open tray, and docks right while expanded over one; it returns to the centre once the trays close - Minimized voice previews the last sentence from each side; the empty Voice Preview suggests things to say - Settings opened from chat keep chat's frame and show the shared AI provider and quick commands; voice Settings show Back to voice - Closing chat ends a typed session; the voice panel's close turns voice off - The dock widens its voice control for the chat button and restyles its wings, which show icons only at 900px and below; assistant panels share the left panels' glass, and Cyber keeps its own surfaces - Server text-chat handlers with body, time and per-IP limits (GEV_RATELIMIT_*_CHAT_PER_MIN), documented in SECURITY.md; tests and all eleven locales Co-authored-by: Darren Dignam <[email protected]>
422 lines
15 KiB
JavaScript
422 lines
15 KiB
JavaScript
import { t } from '../i18n/index.js';
|
|
import {
|
|
PUSH_TO_TALK_HOLD_DELAY_MS,
|
|
MICROPHONE_VISUALIZER_GATE,
|
|
ASSISTANT_VISUALIZER_GATE,
|
|
isPushToTalkKey,
|
|
SPACE_INTERACTIVE_SELECTOR,
|
|
isInteractiveSpaceTarget,
|
|
shouldHandlePushToTalkKeyDown,
|
|
selectVoiceVisualizerSignal,
|
|
resolveVoiceVisualizerSpeaker,
|
|
resolveVoiceControlHint,
|
|
gateVoiceVisualizerLevel,
|
|
} from './realtimeInputPolicy.js';
|
|
import { VOICE_VISUALIZER_BARS } from './control.js';
|
|
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
|
|
|
|
/** Own physical push-to-talk gestures, microphone controls and audio meters. */
|
|
export class RealtimeInput {
|
|
constructor({ readUi, readStream, readStatus, operations }) {
|
|
Object.assign(this, { readUi, readStream, readStatus }, operations);
|
|
this.visualizerAudioContext = null;
|
|
this.visualizerAnalyser = null;
|
|
this.visualizerSource = null;
|
|
this.visualizerFrame = null;
|
|
this.visualizerData = null;
|
|
this.visualizerOutputSource = null;
|
|
this.visualizerOutputAnalyser = null;
|
|
this.visualizerOutputData = null;
|
|
this.visualizerGeneration = 0;
|
|
this.visualizerSpeaker = 'idle';
|
|
this.pushToTalkMode = false;
|
|
this.pushToTalkKeyHeld = false;
|
|
this.spaceKeyHeld = false;
|
|
this.pushToTalkHoldTimer = null;
|
|
this.pushToTalkHoldGeneration = 0;
|
|
this.pushToTalkHoldFocusOwner = null;
|
|
this.pushToTalkHoldControl = null;
|
|
this.pushToTalkHoldPreservesNative = false;
|
|
this.shortcutKeyDownHandler = null;
|
|
this.shortcutKeyUpHandler = null;
|
|
this.shortcutBlurHandler = null;
|
|
this.shortcutVisibilityHandler = null;
|
|
}
|
|
get ui() {
|
|
return this.readUi();
|
|
}
|
|
get stream() {
|
|
return this.readStream();
|
|
}
|
|
get status() {
|
|
return this.readStatus();
|
|
}
|
|
/** Registers delayed hold-Space push-to-talk while preserving short control taps. */
|
|
bindPushToTalkShortcut() {
|
|
if (this.shortcutKeyDownHandler) return;
|
|
this.shortcutKeyDownHandler = (event) => {
|
|
// Space types in chat; it never talks while chat is open.
|
|
if (
|
|
this.ui.state?.chat?.open === true ||
|
|
!shouldHandlePushToTalkKeyDown(event)
|
|
)
|
|
return;
|
|
if (event.repeat) {
|
|
if (this.spaceKeyHeld && !this.pushToTalkHoldPreservesNative)
|
|
event.preventDefault();
|
|
if (this.pushToTalkKeyHeld) event.preventDefault();
|
|
return;
|
|
}
|
|
this.spaceKeyHeld = true;
|
|
this.pushToTalkHoldFocusOwner = document.activeElement;
|
|
this.pushToTalkHoldControl = isInteractiveSpaceTarget(
|
|
document.activeElement,
|
|
)
|
|
? document.activeElement
|
|
: isInteractiveSpaceTarget(event.target)
|
|
? event.target.closest?.(SPACE_INTERACTIVE_SELECTOR) || event.target
|
|
: null;
|
|
this.pushToTalkHoldPreservesNative = Boolean(this.pushToTalkHoldControl);
|
|
// Background Space is reserved immediately to avoid scrolling. A focused
|
|
// control keeps its native keydown and release unless the hold is claimed.
|
|
if (!this.pushToTalkHoldPreservesNative) event.preventDefault();
|
|
// A click-started session is intentionally open-mic. Space only claims an
|
|
// idle session (or a session it already started) so releasing the key can
|
|
// never surprise the user by muting a click-started conversation.
|
|
if (this.isActive() && !this.pushToTalkMode) return;
|
|
this.cancelPushToTalkHold();
|
|
// Point-and-ask: the user decided to ask at keydown, so that is where
|
|
// the pointer is read. A short tap releases it below.
|
|
this.holdPointer?.();
|
|
const holdGeneration = ++this.pushToTalkHoldGeneration;
|
|
this.pushToTalkHoldTimer = setTimeout(() => {
|
|
this.pushToTalkHoldTimer = null;
|
|
if (holdGeneration !== this.pushToTalkHoldGeneration) return;
|
|
if (!this.spaceKeyHeld || this.pushToTalkKeyHeld) return;
|
|
// A pointer or Tab focus change during the delay cancels the original
|
|
// gesture rather than letting voice claim a key held for another owner.
|
|
if (document.activeElement !== this.pushToTalkHoldFocusOwner) return;
|
|
if (
|
|
document.visibilityState === 'hidden' ||
|
|
(typeof document.hasFocus === 'function' && !document.hasFocus())
|
|
)
|
|
return;
|
|
if (this.isActive() && !this.pushToTalkMode) return;
|
|
// Blur before voice starts. This removes the focused state and ensures
|
|
// the eventual Space release cannot activate the old control.
|
|
if (
|
|
this.pushToTalkHoldControl &&
|
|
document.activeElement === this.pushToTalkHoldControl &&
|
|
typeof this.pushToTalkHoldControl.blur === 'function'
|
|
) {
|
|
this.pushToTalkHoldControl.blur();
|
|
}
|
|
// Read Radio's claim before pausing it: pausing can change the very
|
|
// handoff state that decides whether voice may interrupt.
|
|
const mayInterrupt = this.mayClaimSpeaker?.() ?? true;
|
|
this.pauseRadioForVoice();
|
|
this.pushToTalkKeyHeld = true;
|
|
if (this.isActive()) {
|
|
// A claimed hold is an explicit request to speak: cut the assistant
|
|
// off if it is talking. Short taps never reach this point.
|
|
if (mayInterrupt) this.bargeIn?.();
|
|
this.beginPointerTurn?.('keydown');
|
|
this.ui.update('root', { pushToTalk: 'held' });
|
|
this.setMicrophoneEnabled(true);
|
|
if (this.status === 'listening')
|
|
this.setStatus('listening', t('Release Space to send'));
|
|
} else {
|
|
this.start({ pushToTalk: true });
|
|
// start() performs a controlled stop() before connecting. Restore the
|
|
// marker it clears so the UI reflects this still-held gesture.
|
|
if (this.pushToTalkKeyHeld)
|
|
this.ui.update('root', { pushToTalk: 'held' });
|
|
}
|
|
}, PUSH_TO_TALK_HOLD_DELAY_MS);
|
|
};
|
|
this.shortcutKeyUpHandler = (event) => {
|
|
if (!isPushToTalkKey(event)) return;
|
|
const wasHoldingSpace = this.spaceKeyHeld;
|
|
const preservedNativeActivation = this.pushToTalkHoldPreservesNative;
|
|
this.spaceKeyHeld = false;
|
|
this.cancelPushToTalkHold();
|
|
if (!this.pushToTalkKeyHeld) {
|
|
if (wasHoldingSpace) this.releasePointerHold?.();
|
|
if (wasHoldingSpace && !preservedNativeActivation)
|
|
event.preventDefault();
|
|
this.resetPushToTalkGesture();
|
|
return;
|
|
}
|
|
event.preventDefault();
|
|
this.releasePushToTalkKey();
|
|
this.resetPushToTalkGesture();
|
|
};
|
|
this.shortcutBlurHandler = () => {
|
|
this.spaceKeyHeld = false;
|
|
this.cancelPushToTalkHold();
|
|
this.releasePushToTalkKey();
|
|
this.resetPushToTalkGesture();
|
|
};
|
|
this.shortcutVisibilityHandler = () => {
|
|
if (document.visibilityState === 'hidden') this.shortcutBlurHandler();
|
|
};
|
|
// Observe before custom controls call preventDefault for their own Space
|
|
// behavior. This lets the same physical hold cross the 500ms voice
|
|
// threshold while short taps still reach the control unchanged.
|
|
document.addEventListener('keydown', this.shortcutKeyDownHandler, true);
|
|
document.addEventListener('keyup', this.shortcutKeyUpHandler, true);
|
|
window.addEventListener('blur', this.shortcutBlurHandler);
|
|
document.addEventListener(
|
|
'visibilitychange',
|
|
this.shortcutVisibilityHandler,
|
|
);
|
|
}
|
|
|
|
/** Cancels an unclaimed Space hold before it starts voice. */
|
|
cancelPushToTalkHold() {
|
|
this.pushToTalkHoldGeneration++;
|
|
if (this.pushToTalkHoldTimer === null) return;
|
|
clearTimeout(this.pushToTalkHoldTimer);
|
|
this.pushToTalkHoldTimer = null;
|
|
}
|
|
|
|
/** Clears the owner metadata for the current physical Space gesture. */
|
|
resetPushToTalkGesture() {
|
|
this.pushToTalkHoldFocusOwner = null;
|
|
this.pushToTalkHoldControl = null;
|
|
this.pushToTalkHoldPreservesNative = false;
|
|
}
|
|
|
|
/**
|
|
* Mutes a keyboard-started microphone while leaving WebRTC alive for the reply.
|
|
* @returns {void}
|
|
*/
|
|
releasePushToTalkKey() {
|
|
this.cancelPushToTalkHold();
|
|
// The pointer snapshot belongs to this gesture; release or blur ends it.
|
|
this.releasePointerHold?.();
|
|
if (!this.pushToTalkKeyHeld) return;
|
|
this.pushToTalkKeyHeld = false;
|
|
this.ui.update('root', { pushToTalk: null });
|
|
if (!this.pushToTalkMode) return;
|
|
this.setMicrophoneEnabled(false);
|
|
if (this.status === 'listening')
|
|
this.setStatus('listening', t('Hold Space to talk'));
|
|
else this.updateVoiceButtonLabel();
|
|
}
|
|
|
|
/**
|
|
* Enables or mutes only the outbound microphone tracks.
|
|
* @param {boolean} enabled
|
|
* @returns {void}
|
|
*/
|
|
setMicrophoneEnabled(enabled) {
|
|
this.ui?.update('root', { microphone: enabled ? 'active' : 'muted' });
|
|
this.stream?.getAudioTracks?.().forEach((track) => {
|
|
track.enabled = Boolean(enabled);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Drives the dock waveform from live microphone energy while voice is active.
|
|
* @param {MediaStream} stream
|
|
* @returns {void}
|
|
*/
|
|
startVoiceVisualizer(stream) {
|
|
this.stopVoiceVisualizer();
|
|
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
|
|
if (!AudioContextClass || !stream || !this.ui) return;
|
|
try {
|
|
const context = new AudioContextClass();
|
|
this.visualizerAudioContext = context;
|
|
context.resume().catch(() => {});
|
|
const analyser = context.createAnalyser();
|
|
analyser.fftSize = 64;
|
|
analyser.smoothingTimeConstant = 0.72;
|
|
const source = context.createMediaStreamSource(stream);
|
|
source.connect(analyser);
|
|
this.visualizerAnalyser = analyser;
|
|
this.visualizerSource = source;
|
|
this.visualizerData = new Uint8Array(analyser.frequencyBinCount);
|
|
|
|
const generation = this.visualizerGeneration;
|
|
const render = () => {
|
|
if (generation !== this.visualizerGeneration) return;
|
|
const signal = selectVoiceVisualizerSignal(
|
|
this.visualizerSpeaker,
|
|
{
|
|
analyser: this.visualizerAnalyser,
|
|
data: this.visualizerData,
|
|
},
|
|
{
|
|
analyser: this.visualizerOutputAnalyser,
|
|
data: this.visualizerOutputData,
|
|
},
|
|
);
|
|
if (!signal) {
|
|
this.restVoiceVisualizer();
|
|
this.visualizerFrame = requestAnimationFrame(render);
|
|
return;
|
|
}
|
|
signal.analyser.getByteFrequencyData(signal.data);
|
|
const binCount = signal.data.length;
|
|
const count = VOICE_VISUALIZER_BARS;
|
|
const bars = Array.from({ length: count }, (_, index) => {
|
|
const start = Math.floor((index / count) * binCount);
|
|
const end = Math.max(
|
|
start + 1,
|
|
Math.floor(((index + 1) / count) * binCount),
|
|
);
|
|
let energy = 0;
|
|
for (let bin = start; bin < end; bin++) energy += signal.data[bin];
|
|
const normalized = Math.min(1, energy / (end - start) / 190);
|
|
const gate =
|
|
this.visualizerSpeaker === 'ai'
|
|
? ASSISTANT_VISUALIZER_GATE
|
|
: MICROPHONE_VISUALIZER_GATE;
|
|
const shaped = Math.pow(
|
|
gateVoiceVisualizerLevel(normalized, gate),
|
|
0.72,
|
|
);
|
|
return {
|
|
level: `${Math.round(5 + shaped * 29)}px`,
|
|
opacity: `${(0.5 + shaped * 0.5).toFixed(2)}`,
|
|
};
|
|
});
|
|
this.ui?.setBars(bars);
|
|
this.visualizerFrame = requestAnimationFrame(render);
|
|
};
|
|
render();
|
|
} catch {
|
|
this.stopVoiceVisualizer();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Adds the incoming assistant audio stream to the existing Web Audio meter.
|
|
* The audio element remains responsible for playback; this branch only reads
|
|
* its frequency energy for the visualizer.
|
|
* @param {MediaStream} stream
|
|
* @returns {void}
|
|
*/
|
|
startAssistantVoiceVisualizer(stream) {
|
|
const context = this.visualizerAudioContext;
|
|
if (!context || !stream) return;
|
|
try {
|
|
try {
|
|
this.visualizerOutputSource?.disconnect();
|
|
} catch {
|
|
/* no-op */
|
|
}
|
|
const analyser = context.createAnalyser();
|
|
analyser.fftSize = 64;
|
|
analyser.smoothingTimeConstant = 0.72;
|
|
const source = context.createMediaStreamSource(stream);
|
|
source.connect(analyser);
|
|
this.visualizerOutputSource = source;
|
|
this.visualizerOutputAnalyser = analyser;
|
|
this.visualizerOutputData = new Uint8Array(analyser.frequencyBinCount);
|
|
} catch {
|
|
// Playback continues through audioEl even if a browser declines analysis.
|
|
this.visualizerOutputSource = null;
|
|
this.visualizerOutputAnalyser = null;
|
|
this.visualizerOutputData = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Releases the microphone meter and restores its five-pixel baseline.
|
|
* @returns {void}
|
|
*/
|
|
stopVoiceVisualizer() {
|
|
this.visualizerGeneration++;
|
|
if (this.visualizerFrame) cancelAnimationFrame(this.visualizerFrame);
|
|
this.visualizerFrame = null;
|
|
try {
|
|
this.visualizerSource?.disconnect();
|
|
} catch {
|
|
/* no-op */
|
|
}
|
|
try {
|
|
this.visualizerOutputSource?.disconnect();
|
|
} catch {
|
|
/* no-op */
|
|
}
|
|
this.visualizerSource = null;
|
|
this.visualizerAnalyser = null;
|
|
this.visualizerData = null;
|
|
this.visualizerOutputSource = null;
|
|
this.visualizerOutputAnalyser = null;
|
|
this.visualizerOutputData = null;
|
|
this.visualizerSpeaker = 'idle';
|
|
if (this.visualizerAudioContext) {
|
|
this.visualizerAudioContext.close().catch(() => {});
|
|
this.visualizerAudioContext = null;
|
|
}
|
|
this.restVoiceVisualizer();
|
|
}
|
|
|
|
/** Return the waveform to its CSS standby baseline. */
|
|
restVoiceVisualizer() {
|
|
this.ui?.setBars(null);
|
|
}
|
|
|
|
/**
|
|
* Keeps the microphone caption in sync with click and hold-to-talk modes.
|
|
* @returns {void}
|
|
*/
|
|
updateVoiceButtonLabel() {
|
|
if (!this.ui) return;
|
|
// The mic button's caption; the first-run tip names it the same way.
|
|
this.ui.update('mic', { text: () => t('MIC') });
|
|
const { pushToTalkMode, pushToTalkKeyHeld } = this;
|
|
this.ui.update('help', {
|
|
text: () => resolveVoiceControlHint(pushToTalkMode, pushToTalkKeyHeld),
|
|
});
|
|
}
|
|
|
|
setVoiceSpeaker(speaker, { keepVisualizerSpeaker = false } = {}) {
|
|
const nextSpeaker =
|
|
speaker === 'user' || speaker === 'ai' ? speaker : 'idle';
|
|
this.visualizerSpeaker = resolveVoiceVisualizerSpeaker(
|
|
this.visualizerSpeaker,
|
|
nextSpeaker,
|
|
keepVisualizerSpeaker,
|
|
);
|
|
this.ui.update('root', { speaker: nextSpeaker });
|
|
if (shouldPauseRadioForVoice({ speaker: nextSpeaker }))
|
|
this.pauseRadioForVoice();
|
|
}
|
|
|
|
resetSession() {
|
|
this.pushToTalkMode = false;
|
|
this.pushToTalkKeyHeld = false;
|
|
this.spaceKeyHeld = false;
|
|
this.resetPushToTalkGesture();
|
|
this.ui?.update('root', { pushToTalk: null, microphone: null });
|
|
}
|
|
|
|
detachBindings() {
|
|
if (this.shortcutKeyDownHandler)
|
|
document.removeEventListener(
|
|
'keydown',
|
|
this.shortcutKeyDownHandler,
|
|
true,
|
|
);
|
|
if (this.shortcutKeyUpHandler)
|
|
document.removeEventListener('keyup', this.shortcutKeyUpHandler, true);
|
|
if (this.shortcutBlurHandler)
|
|
window.removeEventListener('blur', this.shortcutBlurHandler);
|
|
if (this.shortcutVisibilityHandler) {
|
|
document.removeEventListener(
|
|
'visibilitychange',
|
|
this.shortcutVisibilityHandler,
|
|
);
|
|
}
|
|
this.shortcutKeyDownHandler = null;
|
|
this.shortcutKeyUpHandler = null;
|
|
this.shortcutBlurHandler = null;
|
|
this.shortcutVisibilityHandler = null;
|
|
}
|
|
}
|