Files
gods-eye-view/src/voice/realtimeController.js
T
Sameh Khamis b20c80e590 Switch language in place, without a reload
Choosing a language in the picker now applies it at once. The choice is
stored and ?lang is kept in step through replaceState, and the picker keeps
focus.

Code that writes translated text itself re-presents it on a switch:
onLocaleChange() lets an owner listen, removed at teardown, and status
writers hand over a function of the locale (voice cards, scene director,
SDR, draw hints, imagery split, map sources, Bhote Koshi media) so the
words are rebuilt rather than re-sent. The HUD, cockpit, radio, CCTV,
launch, awareness and credit texts follow the same hook, and
applyDocumentTranslations can be applied again without undoing text code
wrote since.

The unsaved-work registry and persistLocaleAndReload existed only to guard
the reload, so they go, with their catalog entries.
2026-11-01 16:47:39 -08:00

484 lines
18 KiB
JavaScript

import { RealtimeConnection } from './realtimeConnection.js';
import { RealtimeTurns } from './realtimeTurns.js';
import { RealtimeViewport } from './realtimeViewport.js';
import { RealtimeDiagnostics } from './realtimeDiagnostics.js';
import { RealtimeRadio } from './realtimeRadio.js';
import { RealtimeFacade } from './realtimeFacade.js';
import { RealtimeCost } from './realtimeCost.js';
import { RealtimeInput } from './realtimeInput.js';
import { TurnMetrics } from './turnMetrics.js';
import { createNarrationScheduler } from './narration.js';
import { createEarcon } from './earcon.js';
import {
POINTER_NONE_KEY,
POINTER_OPEN_MIC_RECENT_MS,
pointerAnnouncement,
pointerTurnContext,
pointerTurnEvent,
} from './pointerContext.js';
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
import { postDebugLog } from './realtimeDiagnostics.js';
export {
readStoredVoiceTier,
writeStoredVoiceTier,
readStoredVoiceLimits,
writeStoredVoiceLimits,
} from './realtimePreferences.js';
export {
shouldPauseRadioForVoice,
shouldStopVoiceAfterRadioTool,
startPreparedRadioAfterPlaybackReady,
silenceRadioForVoice,
} from './realtimeProtocol.js';
export {
computeDownscale,
estimateDataUrlBytes,
renderFreshCesiumFrame,
isBenignViewportDeleteError,
} from './realtimeViewport.js';
export {
PUSH_TO_TALK_HOLD_DELAY_MS,
isPushToTalkKey,
isPushToTalkSurface,
isInteractiveSpaceTarget,
isEditingSpaceTarget,
shouldHandlePushToTalkKeyDown,
shouldIgnoreVoiceButtonClick,
selectVoiceVisualizerSignal,
resolveVoiceVisualizerSpeaker,
resolveVoiceControlHint,
gateVoiceVisualizerLevel,
} from './realtimeInputPolicy.js';
import { createRealtimeBackend } from './realtimeBackend.js';
import { t } from '../i18n/index.js';
import { voiceStatusLabel } from './control.js';
// Machine states (dataset.status, CSS); voiceStatusLabel() translates labels.
const STATUS_STATES = new Set([
'idle',
'connecting',
'listening',
'executing',
'error',
]);
/** Compose voice state owners and coordinate ordered session startup/teardown. */
export class GevRealtimeController extends RealtimeFacade {
constructor({
runner,
ui,
radioLayer = null,
dataManager = null,
backend = createRealtimeBackend(),
signal,
debugSink = postDebugLog,
actionExecutor,
onSessionEvent,
pointer = null,
}) {
super();
this.pointer = pointer;
this.pointerSentKey = null;
this.pointerAnnounced = null;
this.actionExecutor = actionExecutor;
this.onSessionEvent = onSessionEvent;
this.backend = backend;
this.lifetimeSignal = signal;
this.lifetimeAbort = () => this.stop({ removeUi: true });
signal?.addEventListener('abort', this.lifetimeAbort, { once: true });
this.runner = runner;
this.ui = ui;
this.radioLayer = radioLayer;
this.dataManager = dataManager;
this._viewport = new RealtimeViewport({
readChannel: () => this.dc,
readPointer: () => this.pointer?.activeSnapshot?.() || null,
operations: {
sendRealtimeEvent: (...args) => this.sendRealtimeEvent(...args),
},
});
this._diagnostics = new RealtimeDiagnostics({
readStatus: () => this.status,
readChannel: () => this.dc,
readPeer: () => this.pc,
readCostTracker: () => this.costTracker,
debugSink,
operations: {
setStatus: (...args) => this.setStatus(...args),
},
});
this._radio = new RealtimeRadio({
readRadioLayer: () => this.radioLayer,
readDataManager: () => this.dataManager,
readChannel: () => this.dc,
readUserTurnPending: () => this.userTurnPending,
readSessionId: () => this.sessionId,
operations: {
abortTools: () => this._turns.abortTools(),
isActive: (...args) => this.isActive(...args),
stop: (...args) => this.stop(...args),
setStatus: (...args) => this.setStatus(...args),
queueResponseCreate: (...args) => this.queueResponseCreate(...args),
debugLog: (...args) => this.debugLog(...args),
},
});
this._cost = new RealtimeCost({
readUi: () => this.ui,
readStatus: () => this.status,
operations: {
isActive: (...args) => this.isActive(...args),
isVoiceSessionSettled: (...args) => this.isVoiceSessionSettled(...args),
setStatus: (...args) => this.setStatus(...args),
debugLog: (...args) => this.debugLog(...args),
stop: (...args) => this.stop(...args),
},
});
this._input = new RealtimeInput({
readUi: () => this.ui,
readStream: () => this.stream,
readStatus: () => this.status,
operations: {
isActive: (...args) => this.isActive(...args),
setStatus: (...args) => this.setStatus(...args),
start: (...args) => this.start(...args),
pauseRadioForVoice: (...args) => this.pauseRadioForVoice(...args),
bargeIn: () => this._turns.bargeIn(),
holdPointer: () => this.pointer?.hold('keydown'),
releasePointerHold: () => this.pointer?.releaseHold(),
beginPointerTurn: (at) => this.beginPointerTurn(at),
mayClaimSpeaker: () => this._radio.mayVoiceClaimSpeaker(),
},
});
this.tierHandler = null;
this.annotationEventUnsubscribe = null;
this.status = 'idle';
this._metrics = new TurnMetrics({
emit: (span) => {
this.debugLog('turn.span', span);
this.emitSessionEvent({ type: 'turn-metrics', span });
},
});
this._turns = new RealtimeTurns({
readActionExecutor: () => this.actionExecutor,
readRunner: () => this.runner,
readChannel: () => this.dc,
readDataManager: () => this.dataManager,
readRadioLayer: () => this.radioLayer,
radio: this._radio,
viewport: this._viewport,
metrics: this._metrics,
operations: {
cancelRadioHandoff: (...args) => this.cancelRadioHandoff(...args),
beginPointerTurn: (...args) => this.beginPointerTurn(...args),
connectionDiagnostics: (...args) => this.connectionDiagnostics(...args),
debugLog: (...args) => this.debugLog(...args),
emitSessionEvent: (...args) => this.emitSessionEvent(...args),
isRadioHandoffReserved: (...args) =>
this.isRadioHandoffReserved(...args),
isSessionEnding: (...args) => this.isSessionEnding(...args),
pauseRadioForVoice: (...args) => this.pauseRadioForVoice(...args),
recordUsage: (...args) => this.recordUsage(...args),
recordTranscriptionUsage: (...args) =>
this.recordTranscriptionUsage(...args),
reportError: (...args) => this.reportError(...args),
reserveRadioToolHandoff: (...args) =>
this.reserveRadioToolHandoff(...args),
sendRealtimeEvent: (...args) => this.sendRealtimeEvent(...args),
sendVisualContextIfUseful: (...args) =>
this.sendVisualContextIfUseful(...args),
setStatus: (...args) => this.setStatus(...args),
setVoiceSpeaker: (...args) => this.setVoiceSpeaker(...args),
settleRadioToolHandoffReservation: (...args) =>
this.settleRadioToolHandoffReservation(...args),
startPendingRadioHandoff: (...args) =>
this.startPendingRadioHandoff(...args),
stop: (...args) => this.stop(...args),
},
});
// Progress narration: spoken lines go out of band through the turns
// owner; the earcon plays only while push-to-talk has the mic muted.
this._earcon = createEarcon({
isMicrophoneMuted: () =>
Boolean(this.pushToTalkMode && !this.pushToTalkKeyHeld && this.stream),
});
this._turns.narration = createNarrationScheduler({
speak: (line) => this._turns.speakProgressLine(line),
retract: () => this._turns.retractProgressLine(),
stop: () => this._turns.stopProgressLine(),
earcon: this._earcon,
canSpeak: () => !this.pushToTalkKeyHeld && this._turns.canNarrate(),
onNarration: (kind) => this._metrics.narration(kind),
});
this._connection = new RealtimeConnection({
readLifetimeSignal: () => this.lifetimeSignal,
readBackend: () => this.backend,
readStatus: () => this.status,
input: this._input,
cost: this._cost,
operations: {
isActive: (...args) => this.isActive(...args),
pauseRadioForVoice: (...args) => this.pauseRadioForVoice(...args),
stop: (...args) => this.stop(...args),
syncCostUi: (...args) => this.syncCostUi(...args),
setStatus: (...args) => this.setStatus(...args),
debugLog: (...args) => this.debugLog(...args),
connectionDiagnostics: (...args) => this.connectionDiagnostics(...args),
setMicrophoneEnabled: (...args) => this.setMicrophoneEnabled(...args),
startVoiceVisualizer: (...args) => this.startVoiceVisualizer(...args),
startAssistantVoiceVisualizer: (...args) =>
this.startAssistantVoiceVisualizer(...args),
fatalError: (...args) => this.fatalError(...args),
reportError: (...args) => this.reportError(...args),
handleRealtimeEvent: (...args) => this.handleRealtimeEvent(...args),
},
});
this._radio.observe();
this.debugLog('controller.created', { status: this.status });
}
isActive() {
return this.status !== 'idle' && this.status !== 'error';
}
/**
* Point-and-ask: read the pointer as a user turn starts and tell the model.
*
* A Space hold is an explicit pointing gesture: its keydown snapshot is
* shown at once (reticle and chip) and sent as a pointer_context item with
* the target. Open-mic speech and typed turns carry no gesture, so they
* count only a pointer that moved on the map in the last few seconds, and
* the model is told only that one is available; the target is revealed
* when a tool resolves 'pointer' (see announcePointer). Losing the pointer
* after one was sent appends a clearing item. Unchanged items are not
* repeated.
* @param {'keydown'|'speech_start'|'text'} at
*/
beginPointerTurn(at) {
if (!this.pointer) return null;
this._viewport.beginTurn();
// A speech start after the Space release belongs to the gesture that
// just ended; it must not re-read a pointer that has since moved.
if (
at === 'speech_start' &&
this.pushToTalkMode &&
!this.pointer.isHeld?.()
)
return this.pointer.activeSnapshot?.() || null;
const gesture =
at === 'keydown' || (at === 'speech_start' && this.pushToTalkMode);
let snapshot = null;
try {
snapshot = this.pointer.beginTurn(
at,
gesture ? {} : { recentMs: POINTER_OPEN_MIC_RECENT_MS },
);
} catch {
snapshot = null;
}
const event = pointerTurnEvent(snapshot, gesture);
this.pointerAnnounced = event.announced;
this.emitSessionEvent({ type: 'pointer', pointer: event.pointer });
const context = pointerTurnContext(snapshot, {
at,
gesture,
sentKey: this.pointerSentKey,
});
if (!context) return snapshot;
if (this._turns.notifyMapEvent(context.item)) {
this.pointerSentKey = context.key;
const chip = context.key !== POINTER_NONE_KEY;
this.debugLog('pointer.context', {
at,
target: chip ? snapshot.target : 'none',
detail: chip ? (gesture ? 'target' : 'available') : 'cleared',
});
}
return snapshot;
}
/**
* A tool just resolved 'pointer' for a turn that had no gesture: show the
* reticle and chip now, once per snapshot.
*/
announcePointer() {
const snapshot = this.pointer?.activeSnapshot?.() || null;
const pointer = pointerAnnouncement(snapshot, this.pointerAnnounced);
if (!pointer) return false;
this.pointerAnnounced = snapshot;
this.emitSessionEvent({ type: 'pointer', pointer });
return true;
}
/** What the retained context image shows, for pixel targeting. */
retainedImageFrame() {
return this._viewport.retainedFrame;
}
// Fatal error path: tear the session down (stop tracks, close pc/dc, kill the
// mic) BEFORE flipping the UI to ERROR, so we never sit in an ERROR state with
// a live hot mic behind it (H8). stop() itself bumps the epoch and clears the
// grace timer; preserveStatus lets reportError own the final 'error' status.
fatalError(source, error = null, extra = {}) {
this.stop({ preserveStatus: true });
return this.reportError(source, error, extra);
}
stop(options = {}) {
const {
removeUi = false,
preserveStatus = false,
preserveRadioPlayback = false,
} = options;
// Bump the epoch so any start() awaiting a token/getUserMedia/SDP bails and
// releases its own resources instead of promoting them onto a stopped
// controller (H7).
this._connection.invalidate();
if (removeUi)
this.lifetimeSignal?.removeEventListener('abort', this.lifetimeAbort);
this.cancelPushToTalkHold();
this._radio.invalidateHandoff();
this._turns.abortTools();
this._radio.stopHandoff({ preserveRadioPlayback });
this.clearDisconnectGrace();
// Guard against the dc.close() below re-entering our own error handlers while
// we're intentionally tearing down (the close/error listeners bail on this
// flag) — H8.
this._connection.beginTeardown();
this.debugLog('session.stop', {
removeUi,
preserveStatus,
status: this.status,
connection: this.connectionDiagnostics(),
});
if (this.dc && this.responseActive) {
this.costTracker.markIncomplete();
this.syncCostUi();
}
this._metrics.flush('stop');
this._connection.closeTransport();
this.stopVoiceVisualizer();
this._connection.releaseMedia();
this._turns.reset();
// A Space hold that is starting a session keeps its keydown snapshot.
this.pointer?.clear({ keepHold: this.pushToTalkKeyHeld });
this.pointerSentKey = null;
this.pointerAnnounced = null;
this._earcon.stop();
if (removeUi) this._earcon.dispose();
this._radio.clearPendingPlayback();
this._viewport.reset();
this._input.resetSession();
if (removeUi && this.tierHandler) {
this.tierHandler();
this.tierHandler = null;
}
if (removeUi) this._input.detachBindings();
if (removeUi && this.annotationEventUnsubscribe) {
// Full teardown (re-init path): stop listening to the long-lived annotation
// engine so a replaced controller can't keep receiving outline events.
this.annotationEventUnsubscribe();
this.annotationEventUnsubscribe = null;
}
if (removeUi) this._radio.detachObservers();
if (removeUi) this.ui?.remove();
if (!preserveStatus && !removeUi) {
this.setStatus('idle', t('Voice off'));
}
this.setRadioVoiceDucking(false);
if (removeUi) this.emitSessionEvent({ type: 'disposed' });
}
emitSessionEvent(event) {
try {
this.onSessionEvent?.(event);
} catch {
/* Observers cannot interrupt voice. */
}
}
setStatus(status, detail) {
this.status = status;
this.emitSessionEvent({ type: 'state', state: status, detail });
this.ui.update('root', {
status,
...(status === 'error' && { errorDismissed: false }),
});
this.updateVoiceButtonLabel();
const shownStatus = STATUS_STATES.has(status) ? status : 'idle';
this.ui.update('status', { text: () => voiceStatusLabel(shownStatus) });
// Words are given as functions, so a locale change words them again.
const holdToTalk = status === 'listening' && this.pushToTalkMode;
const held = this.pushToTalkKeyHeld;
const resolvedDetail = () =>
holdToTalk
? held
? t('Release Space to send')
: t('Hold Space to talk')
: detail;
const primaryDetail = () =>
status === 'error'
? t('VOICE UNAVAILABLE')
: resolvedDetail() ||
(status === 'idle' ? t('VOICE STANDBY') : t('VOICE ACTIVE'));
this.ui.update('detail', { text: primaryDetail, title: primaryDetail });
this.ui.update('error', {
text: () =>
status === 'error'
? resolvedDetail() || t('Voice session could not be started.')
: '',
});
if (status === 'idle' || status === 'connecting' || status === 'error') {
this.setVoiceSpeaker('idle');
}
if (
shouldPauseRadioForVoice({
status,
pushToTalkKeyHeld: this.pushToTalkKeyHeld,
})
) {
this.pauseRadioForVoice();
}
}
/* ---------------- voice cost control ---------------- */
/**
* Is this session terminating (spend cap reached)? Latched — never clears
* until the next start().
*
* IN-FLIGHT TOOLS RUN TO COMPLETION, AND ARE NOT ROLLED BACK. A tool already
* executing when the cap trips may finish its map mutation (a camera flight,
* a layer toggle, an annotation). That is deliberate: unwinding a partially
* applied map change has no safe general implementation — a half-reverted
* camera/layer/annotation state is worse than a completed one, and the tool
* abort signal is advisory (most actions do not check it). What the latch DOES
* guarantee is that no NEW tool is dispatched once the cap has tripped.
*/
isSessionEnding() {
return this.costCapStopped === true;
}
/**
* Is the voice session FULLY settled — no live session and no transport left?
*
* Replacing the cost tracker is only legal here. `!isActive()` alone is not
* enough: the 'error' status reports inactive while the data/peer connection
* may still be open and delivering a late `response.done`. Rebuilding on that
* signal would send late usage to a fresh preview tracker instead of the one
* that owns the session's spend.
*/
isVoiceSessionSettled() {
return !this.isActive() && !this.dc && !this.pc;
}
}