Files
gods-eye-view/src/voice/realtimeInput.js
T
manjunath22466andDarren Dignam af461febae feat(voice): add typed AI chat beside voice
Chat sits next to the mic in the command dock. Replies are silent and never
request the microphone; the first message connects with the selected provider
and runs the same map actions as voice through each provider's text API
(OpenAI OPENAI_CHAT_MODEL, Gemini GEMINI_CHAT_MODEL via generateContent, which
never starts Live). Only one mode runs at a time.

- Chat panel rendered by the voice control presenter: conversation, recent and
  quick-command suggestions, Up/Down history, attachments (map-layer
  references or small UTF-8 text files, sent only with the message)
- Chat and voice panels sit above the dock or dock into the right rail's
  column; minimized bars sit centred above the dock, and height animates on
  swaps
- A centred panel steps down to its bar for the Location and Visual Presets
  trays without moving focus, rests above an open tray, and docks right while
  expanded over one; it returns to the centre once the trays close
- Minimized voice previews the last sentence from each side; the empty Voice
  Preview suggests things to say
- Settings opened from chat keep chat's frame and show the shared AI provider
  and quick commands; voice Settings show Back to voice
- Closing chat ends a typed session; the voice panel's close turns voice off
- The dock widens its voice control for the chat button and restyles its
  wings, which show icons only at 900px and below; assistant panels share the
  left panels' glass, and Cyber keeps its own surfaces
- Server text-chat handlers with body, time and per-IP limits
  (GEV_RATELIMIT_*_CHAT_PER_MIN), documented in SECURITY.md; tests and all
  eleven locales

Co-authored-by: Darren Dignam <[email protected]>
2026-11-02 20:56:39 +05:30

422 lines
15 KiB
JavaScript

import { t } from '../i18n/index.js';
import {
PUSH_TO_TALK_HOLD_DELAY_MS,
MICROPHONE_VISUALIZER_GATE,
ASSISTANT_VISUALIZER_GATE,
isPushToTalkKey,
SPACE_INTERACTIVE_SELECTOR,
isInteractiveSpaceTarget,
shouldHandlePushToTalkKeyDown,
selectVoiceVisualizerSignal,
resolveVoiceVisualizerSpeaker,
resolveVoiceControlHint,
gateVoiceVisualizerLevel,
} from './realtimeInputPolicy.js';
import { VOICE_VISUALIZER_BARS } from './control.js';
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
/** Own physical push-to-talk gestures, microphone controls and audio meters. */
export class RealtimeInput {
constructor({ readUi, readStream, readStatus, operations }) {
Object.assign(this, { readUi, readStream, readStatus }, operations);
this.visualizerAudioContext = null;
this.visualizerAnalyser = null;
this.visualizerSource = null;
this.visualizerFrame = null;
this.visualizerData = null;
this.visualizerOutputSource = null;
this.visualizerOutputAnalyser = null;
this.visualizerOutputData = null;
this.visualizerGeneration = 0;
this.visualizerSpeaker = 'idle';
this.pushToTalkMode = false;
this.pushToTalkKeyHeld = false;
this.spaceKeyHeld = false;
this.pushToTalkHoldTimer = null;
this.pushToTalkHoldGeneration = 0;
this.pushToTalkHoldFocusOwner = null;
this.pushToTalkHoldControl = null;
this.pushToTalkHoldPreservesNative = false;
this.shortcutKeyDownHandler = null;
this.shortcutKeyUpHandler = null;
this.shortcutBlurHandler = null;
this.shortcutVisibilityHandler = null;
}
get ui() {
return this.readUi();
}
get stream() {
return this.readStream();
}
get status() {
return this.readStatus();
}
/** Registers delayed hold-Space push-to-talk while preserving short control taps. */
bindPushToTalkShortcut() {
if (this.shortcutKeyDownHandler) return;
this.shortcutKeyDownHandler = (event) => {
// Space types in chat; it never talks while chat is open.
if (
this.ui.state?.chat?.open === true ||
!shouldHandlePushToTalkKeyDown(event)
)
return;
if (event.repeat) {
if (this.spaceKeyHeld && !this.pushToTalkHoldPreservesNative)
event.preventDefault();
if (this.pushToTalkKeyHeld) event.preventDefault();
return;
}
this.spaceKeyHeld = true;
this.pushToTalkHoldFocusOwner = document.activeElement;
this.pushToTalkHoldControl = isInteractiveSpaceTarget(
document.activeElement,
)
? document.activeElement
: isInteractiveSpaceTarget(event.target)
? event.target.closest?.(SPACE_INTERACTIVE_SELECTOR) || event.target
: null;
this.pushToTalkHoldPreservesNative = Boolean(this.pushToTalkHoldControl);
// Background Space is reserved immediately to avoid scrolling. A focused
// control keeps its native keydown and release unless the hold is claimed.
if (!this.pushToTalkHoldPreservesNative) event.preventDefault();
// A click-started session is intentionally open-mic. Space only claims an
// idle session (or a session it already started) so releasing the key can
// never surprise the user by muting a click-started conversation.
if (this.isActive() && !this.pushToTalkMode) return;
this.cancelPushToTalkHold();
// Point-and-ask: the user decided to ask at keydown, so that is where
// the pointer is read. A short tap releases it below.
this.holdPointer?.();
const holdGeneration = ++this.pushToTalkHoldGeneration;
this.pushToTalkHoldTimer = setTimeout(() => {
this.pushToTalkHoldTimer = null;
if (holdGeneration !== this.pushToTalkHoldGeneration) return;
if (!this.spaceKeyHeld || this.pushToTalkKeyHeld) return;
// A pointer or Tab focus change during the delay cancels the original
// gesture rather than letting voice claim a key held for another owner.
if (document.activeElement !== this.pushToTalkHoldFocusOwner) return;
if (
document.visibilityState === 'hidden' ||
(typeof document.hasFocus === 'function' && !document.hasFocus())
)
return;
if (this.isActive() && !this.pushToTalkMode) return;
// Blur before voice starts. This removes the focused state and ensures
// the eventual Space release cannot activate the old control.
if (
this.pushToTalkHoldControl &&
document.activeElement === this.pushToTalkHoldControl &&
typeof this.pushToTalkHoldControl.blur === 'function'
) {
this.pushToTalkHoldControl.blur();
}
// Read Radio's claim before pausing it: pausing can change the very
// handoff state that decides whether voice may interrupt.
const mayInterrupt = this.mayClaimSpeaker?.() ?? true;
this.pauseRadioForVoice();
this.pushToTalkKeyHeld = true;
if (this.isActive()) {
// A claimed hold is an explicit request to speak: cut the assistant
// off if it is talking. Short taps never reach this point.
if (mayInterrupt) this.bargeIn?.();
this.beginPointerTurn?.('keydown');
this.ui.update('root', { pushToTalk: 'held' });
this.setMicrophoneEnabled(true);
if (this.status === 'listening')
this.setStatus('listening', t('Release Space to send'));
} else {
this.start({ pushToTalk: true });
// start() performs a controlled stop() before connecting. Restore the
// marker it clears so the UI reflects this still-held gesture.
if (this.pushToTalkKeyHeld)
this.ui.update('root', { pushToTalk: 'held' });
}
}, PUSH_TO_TALK_HOLD_DELAY_MS);
};
this.shortcutKeyUpHandler = (event) => {
if (!isPushToTalkKey(event)) return;
const wasHoldingSpace = this.spaceKeyHeld;
const preservedNativeActivation = this.pushToTalkHoldPreservesNative;
this.spaceKeyHeld = false;
this.cancelPushToTalkHold();
if (!this.pushToTalkKeyHeld) {
if (wasHoldingSpace) this.releasePointerHold?.();
if (wasHoldingSpace && !preservedNativeActivation)
event.preventDefault();
this.resetPushToTalkGesture();
return;
}
event.preventDefault();
this.releasePushToTalkKey();
this.resetPushToTalkGesture();
};
this.shortcutBlurHandler = () => {
this.spaceKeyHeld = false;
this.cancelPushToTalkHold();
this.releasePushToTalkKey();
this.resetPushToTalkGesture();
};
this.shortcutVisibilityHandler = () => {
if (document.visibilityState === 'hidden') this.shortcutBlurHandler();
};
// Observe before custom controls call preventDefault for their own Space
// behavior. This lets the same physical hold cross the 500ms voice
// threshold while short taps still reach the control unchanged.
document.addEventListener('keydown', this.shortcutKeyDownHandler, true);
document.addEventListener('keyup', this.shortcutKeyUpHandler, true);
window.addEventListener('blur', this.shortcutBlurHandler);
document.addEventListener(
'visibilitychange',
this.shortcutVisibilityHandler,
);
}
/** Cancels an unclaimed Space hold before it starts voice. */
cancelPushToTalkHold() {
this.pushToTalkHoldGeneration++;
if (this.pushToTalkHoldTimer === null) return;
clearTimeout(this.pushToTalkHoldTimer);
this.pushToTalkHoldTimer = null;
}
/** Clears the owner metadata for the current physical Space gesture. */
resetPushToTalkGesture() {
this.pushToTalkHoldFocusOwner = null;
this.pushToTalkHoldControl = null;
this.pushToTalkHoldPreservesNative = false;
}
/**
* Mutes a keyboard-started microphone while leaving WebRTC alive for the reply.
* @returns {void}
*/
releasePushToTalkKey() {
this.cancelPushToTalkHold();
// The pointer snapshot belongs to this gesture; release or blur ends it.
this.releasePointerHold?.();
if (!this.pushToTalkKeyHeld) return;
this.pushToTalkKeyHeld = false;
this.ui.update('root', { pushToTalk: null });
if (!this.pushToTalkMode) return;
this.setMicrophoneEnabled(false);
if (this.status === 'listening')
this.setStatus('listening', t('Hold Space to talk'));
else this.updateVoiceButtonLabel();
}
/**
* Enables or mutes only the outbound microphone tracks.
* @param {boolean} enabled
* @returns {void}
*/
setMicrophoneEnabled(enabled) {
this.ui?.update('root', { microphone: enabled ? 'active' : 'muted' });
this.stream?.getAudioTracks?.().forEach((track) => {
track.enabled = Boolean(enabled);
});
}
/**
* Drives the dock waveform from live microphone energy while voice is active.
* @param {MediaStream} stream
* @returns {void}
*/
startVoiceVisualizer(stream) {
this.stopVoiceVisualizer();
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
if (!AudioContextClass || !stream || !this.ui) return;
try {
const context = new AudioContextClass();
this.visualizerAudioContext = context;
context.resume().catch(() => {});
const analyser = context.createAnalyser();
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.72;
const source = context.createMediaStreamSource(stream);
source.connect(analyser);
this.visualizerAnalyser = analyser;
this.visualizerSource = source;
this.visualizerData = new Uint8Array(analyser.frequencyBinCount);
const generation = this.visualizerGeneration;
const render = () => {
if (generation !== this.visualizerGeneration) return;
const signal = selectVoiceVisualizerSignal(
this.visualizerSpeaker,
{
analyser: this.visualizerAnalyser,
data: this.visualizerData,
},
{
analyser: this.visualizerOutputAnalyser,
data: this.visualizerOutputData,
},
);
if (!signal) {
this.restVoiceVisualizer();
this.visualizerFrame = requestAnimationFrame(render);
return;
}
signal.analyser.getByteFrequencyData(signal.data);
const binCount = signal.data.length;
const count = VOICE_VISUALIZER_BARS;
const bars = Array.from({ length: count }, (_, index) => {
const start = Math.floor((index / count) * binCount);
const end = Math.max(
start + 1,
Math.floor(((index + 1) / count) * binCount),
);
let energy = 0;
for (let bin = start; bin < end; bin++) energy += signal.data[bin];
const normalized = Math.min(1, energy / (end - start) / 190);
const gate =
this.visualizerSpeaker === 'ai'
? ASSISTANT_VISUALIZER_GATE
: MICROPHONE_VISUALIZER_GATE;
const shaped = Math.pow(
gateVoiceVisualizerLevel(normalized, gate),
0.72,
);
return {
level: `${Math.round(5 + shaped * 29)}px`,
opacity: `${(0.5 + shaped * 0.5).toFixed(2)}`,
};
});
this.ui?.setBars(bars);
this.visualizerFrame = requestAnimationFrame(render);
};
render();
} catch {
this.stopVoiceVisualizer();
}
}
/**
* Adds the incoming assistant audio stream to the existing Web Audio meter.
* The audio element remains responsible for playback; this branch only reads
* its frequency energy for the visualizer.
* @param {MediaStream} stream
* @returns {void}
*/
startAssistantVoiceVisualizer(stream) {
const context = this.visualizerAudioContext;
if (!context || !stream) return;
try {
try {
this.visualizerOutputSource?.disconnect();
} catch {
/* no-op */
}
const analyser = context.createAnalyser();
analyser.fftSize = 64;
analyser.smoothingTimeConstant = 0.72;
const source = context.createMediaStreamSource(stream);
source.connect(analyser);
this.visualizerOutputSource = source;
this.visualizerOutputAnalyser = analyser;
this.visualizerOutputData = new Uint8Array(analyser.frequencyBinCount);
} catch {
// Playback continues through audioEl even if a browser declines analysis.
this.visualizerOutputSource = null;
this.visualizerOutputAnalyser = null;
this.visualizerOutputData = null;
}
}
/**
* Releases the microphone meter and restores its five-pixel baseline.
* @returns {void}
*/
stopVoiceVisualizer() {
this.visualizerGeneration++;
if (this.visualizerFrame) cancelAnimationFrame(this.visualizerFrame);
this.visualizerFrame = null;
try {
this.visualizerSource?.disconnect();
} catch {
/* no-op */
}
try {
this.visualizerOutputSource?.disconnect();
} catch {
/* no-op */
}
this.visualizerSource = null;
this.visualizerAnalyser = null;
this.visualizerData = null;
this.visualizerOutputSource = null;
this.visualizerOutputAnalyser = null;
this.visualizerOutputData = null;
this.visualizerSpeaker = 'idle';
if (this.visualizerAudioContext) {
this.visualizerAudioContext.close().catch(() => {});
this.visualizerAudioContext = null;
}
this.restVoiceVisualizer();
}
/** Return the waveform to its CSS standby baseline. */
restVoiceVisualizer() {
this.ui?.setBars(null);
}
/**
* Keeps the microphone caption in sync with click and hold-to-talk modes.
* @returns {void}
*/
updateVoiceButtonLabel() {
if (!this.ui) return;
// The mic button's caption; the first-run tip names it the same way.
this.ui.update('mic', { text: () => t('MIC') });
const { pushToTalkMode, pushToTalkKeyHeld } = this;
this.ui.update('help', {
text: () => resolveVoiceControlHint(pushToTalkMode, pushToTalkKeyHeld),
});
}
setVoiceSpeaker(speaker, { keepVisualizerSpeaker = false } = {}) {
const nextSpeaker =
speaker === 'user' || speaker === 'ai' ? speaker : 'idle';
this.visualizerSpeaker = resolveVoiceVisualizerSpeaker(
this.visualizerSpeaker,
nextSpeaker,
keepVisualizerSpeaker,
);
this.ui.update('root', { speaker: nextSpeaker });
if (shouldPauseRadioForVoice({ speaker: nextSpeaker }))
this.pauseRadioForVoice();
}
resetSession() {
this.pushToTalkMode = false;
this.pushToTalkKeyHeld = false;
this.spaceKeyHeld = false;
this.resetPushToTalkGesture();
this.ui?.update('root', { pushToTalk: null, microphone: null });
}
detachBindings() {
if (this.shortcutKeyDownHandler)
document.removeEventListener(
'keydown',
this.shortcutKeyDownHandler,
true,
);
if (this.shortcutKeyUpHandler)
document.removeEventListener('keyup', this.shortcutKeyUpHandler, true);
if (this.shortcutBlurHandler)
window.removeEventListener('blur', this.shortcutBlurHandler);
if (this.shortcutVisibilityHandler) {
document.removeEventListener(
'visibilitychange',
this.shortcutVisibilityHandler,
);
}
this.shortcutKeyDownHandler = null;
this.shortcutKeyUpHandler = null;
this.shortcutBlurHandler = null;
this.shortcutVisibilityHandler = null;
}
}