feat(voice): voice core — honest layer reach, spoken summary with on-screen card, narration, point-and-ask

- Voice layer manifest (src/voice/layerManifest.js) generates the layer enums,
  aliases and analyst field hints; a test fails when a catalog layer has no entry.
- analyst_query validates fields, operators, units, values and scopes, counts up
  to 250,000 loaded records per layer, reports complete/partial/unanswered, tells
  LAYER_OFF / NOT_READY / FEED_UNAVAILABLE apart from zero, refuses mismatched
  follow-ups and never lets a superseded query replace follow-up memory. Seven
  more layers expose analyst records.
- Tool results carry a code-built say line and a display card; the voice card in
  the mic control shows captions, plan steps and the result
  (src/voice/resultDisplay.js is the one adapter). Slow tools narrate their
  current step out of band; holding Space while the assistant talks cuts it off.
  Turn spans go to the debug log; caption transcription is metered; the log
  omits what was said unless GEV_VOICE_LOG_CONTENT=1.
- Point and ask: the Space-down pointer snapshot resolves "this", "that one" and
  "around here" through 'pointer' targets, and "the second one" through
  referent:n from the last numbered list.
- Prompt states each policy once (instructions + tools 48.5k -> 46.7k chars).
- Voice benchmark harness (scripts/qa-voice-bench.mjs, scripts/voice-bench/)
  shares its phrase table and grading with scripts/qa-voice-routing.mjs.
This commit is contained in:
Bilawal Sidhu committed 2026-09-28 21:19:23 -05:00
1 parent e7707d9a0f
commit a19897ad15
97 files changed
+14343 -755

No files matched your search

+142
View File
@@ -6,6 +6,17 @@ import { RealtimeRadio } from './realtimeRadio.js';
import { RealtimeFacade } from './realtimeFacade.js';
import { RealtimeCost } from './realtimeCost.js';
import { RealtimeInput } from './realtimeInput.js';
import { TurnMetrics } from './turnMetrics.js';
import { createNarrationScheduler } from './narration.js';
import { createEarcon } from './earcon.js';
import {
POINTER_OPEN_MIC_RECENT_MS,
pointerChip,
pointerContextItem,
} from './pointerContext.js';
/** Dedupe key of the clearing pointer_context item. */
const POINTER_NONE_KEY = 'none';
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
import { postDebugLog } from './realtimeDiagnostics.js';
@@ -64,8 +75,12 @@ export class GevRealtimeController extends RealtimeFacade {
debugSink = postDebugLog,
actionExecutor,
onSessionEvent,
pointer = null,
}) {
super();
this.pointer = pointer;
this.pointerSentKey = null;
this.pointerAnnounced = null;
this.actionExecutor = actionExecutor;
this.onSessionEvent = onSessionEvent;
this.backend = backend;
@@ -79,6 +94,7 @@ export class GevRealtimeController extends RealtimeFacade {
this.dataManager = dataManager;
this._viewport = new RealtimeViewport({
readChannel: () => this.dc,
readPointer: () => this.pointer?.activeSnapshot?.() || null,
operations: {
sendRealtimeEvent: (...args) => this.sendRealtimeEvent(...args),
@@ -130,6 +146,11 @@ export class GevRealtimeController extends RealtimeFacade {
setStatus: (...args) => this.setStatus(...args),
start: (...args) => this.start(...args),
pauseRadioForVoice: (...args) => this.pauseRadioForVoice(...args),
bargeIn: () => this._turns.bargeIn(),
holdPointer: () => this.pointer?.hold('keydown'),
releasePointerHold: () => this.pointer?.releaseHold(),
beginPointerTurn: (at) => this.beginPointerTurn(at),
mayClaimSpeaker: () => this._radio.mayVoiceClaimSpeaker(),
},
});
@@ -139,6 +160,12 @@ export class GevRealtimeController extends RealtimeFacade {
this.status = 'idle';
this._metrics = new TurnMetrics({
emit: (span) => {
this.debugLog('turn.span', span);
this.emitSessionEvent({ type: 'turn-metrics', span });
},
});
this._turns = new RealtimeTurns({
readActionExecutor: () => this.actionExecutor,
readRunner: () => this.runner,
@@ -147,8 +174,10 @@ export class GevRealtimeController extends RealtimeFacade {
readRadioLayer: () => this.radioLayer,
radio: this._radio,
viewport: this._viewport,
metrics: this._metrics,
operations: {
cancelRadioHandoff: (...args) => this.cancelRadioHandoff(...args),
beginPointerTurn: (...args) => this.beginPointerTurn(...args),
connectionDiagnostics: (...args) => this.connectionDiagnostics(...args),
debugLog: (...args) => this.debugLog(...args),
emitSessionEvent: (...args) => this.emitSessionEvent(...args),
@@ -157,6 +186,8 @@ export class GevRealtimeController extends RealtimeFacade {
isSessionEnding: (...args) => this.isSessionEnding(...args),
pauseRadioForVoice: (...args) => this.pauseRadioForVoice(...args),
recordUsage: (...args) => this.recordUsage(...args),
recordTranscriptionUsage: (...args) =>
this.recordTranscriptionUsage(...args),
reportError: (...args) => this.reportError(...args),
reserveRadioToolHandoff: (...args) =>
this.reserveRadioToolHandoff(...args),
@@ -172,6 +203,20 @@ export class GevRealtimeController extends RealtimeFacade {
stop: (...args) => this.stop(...args),
},
});
// Progress narration: spoken lines go out of band through the turns
// owner; the earcon plays only while push-to-talk has the mic muted.
this._earcon = createEarcon({
isMicrophoneMuted: () =>
Boolean(this.pushToTalkMode && !this.pushToTalkKeyHeld && this.stream),
});
this._turns.narration = createNarrationScheduler({
speak: (line) => this._turns.speakProgressLine(line),
retract: () => this._turns.retractProgressLine(),
stop: () => this._turns.stopProgressLine(),
earcon: this._earcon,
canSpeak: () => !this.pushToTalkKeyHeld && this._turns.canNarrate(),
onNarration: (kind) => this._metrics.narration(kind),
});
this._connection = new RealtimeConnection({
readLifetimeSignal: () => this.lifetimeSignal,
readBackend: () => this.backend,
@@ -203,6 +248,96 @@ export class GevRealtimeController extends RealtimeFacade {
return this.status !== 'idle' && this.status !== 'error';
}
/**
* Point-and-ask: read the pointer as a user turn starts and tell the model.
*
* A Space hold is an explicit pointing gesture: its keydown snapshot is
* shown at once (reticle and chip) and sent as a pointer_context item with
* the target. Open-mic speech and typed turns carry no gesture, so they
* count only a pointer that moved on the map in the last few seconds, and
* the model is told only that one is available; the target is revealed
* when a tool resolves 'pointer' (see announcePointer). Losing the pointer
* after one was sent appends a clearing item. Unchanged items are not
* repeated.
* @param {'keydown'|'speech_start'|'text'} at
*/
beginPointerTurn(at) {
if (!this.pointer) return null;
this._viewport.beginTurn();
// A speech start after the Space release belongs to the gesture that
// just ended; it must not re-read a pointer that has since moved.
if (
at === 'speech_start' &&
this.pushToTalkMode &&
!this.pointer.isHeld?.()
)
return this.pointer.activeSnapshot?.() || null;
const gesture =
at === 'keydown' || (at === 'speech_start' && this.pushToTalkMode);
let snapshot = null;
try {
snapshot = this.pointer.beginTurn(
at,
gesture ? {} : { recentMs: POINTER_OPEN_MIC_RECENT_MS },
);
} catch {
snapshot = null;
}
const chip = pointerChip(snapshot);
this.pointerAnnounced = gesture && chip ? snapshot : null;
this.emitSessionEvent({
type: 'pointer',
pointer:
gesture && chip ? { ...chip, screenPx: snapshot.screenPx } : null,
});
const item = !chip
? this.pointerSentKey && this.pointerSentKey !== POINTER_NONE_KEY
? { type: 'pointer_context', at, target: 'none' }
: null
: gesture
? pointerContextItem(snapshot)
: // What kind of thing, never which: no label or coordinates.
{
type: 'pointer_context',
at,
pointing: snapshot.entity?.kind || 'ground',
};
if (!item) return snapshot;
const { at: _at, ...content } = item;
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
if (key === this.pointerSentKey) return snapshot;
if (this._turns.notifyMapEvent(item)) {
this.pointerSentKey = key;
this.debugLog('pointer.context', {
at,
target: chip ? snapshot.target : 'none',
detail: chip ? (gesture ? 'target' : 'available') : 'cleared',
});
}
return snapshot;
}
/**
* A tool just resolved 'pointer' for a turn that had no gesture: show the
* reticle and chip now, once per snapshot.
*/
announcePointer() {
const snapshot = this.pointer?.activeSnapshot?.() || null;
const chip = pointerChip(snapshot);
if (!chip || snapshot === this.pointerAnnounced) return false;
this.pointerAnnounced = snapshot;
this.emitSessionEvent({
type: 'pointer',
pointer: { ...chip, screenPx: snapshot.screenPx },
});
return true;
}
/** What the retained context image shows, for pixel targeting. */
retainedImageFrame() {
return this._viewport.retainedFrame;
}
// Fatal error path: tear the session down (stop tracks, close pc/dc, kill the
// mic) BEFORE flipping the UI to ERROR, so we never sit in an ERROR state with
// a live hot mic behind it (H8). stop() itself bumps the epoch and clears the
@@ -240,10 +375,17 @@ export class GevRealtimeController extends RealtimeFacade {
connection: this.connectionDiagnostics(),
});
if (this.dc && this.responseActive) this.costTracker.markIncomplete();
this._metrics.flush('stop');
this._connection.closeTransport();
this.stopVoiceVisualizer();
this._connection.releaseMedia();
this._turns.reset();
// A Space hold that is starting a session keeps its keydown snapshot.
this.pointer?.clear({ keepHold: this.pushToTalkKeyHeld });
this.pointerSentKey = null;
this.pointerAnnounced = null;
this._earcon.stop();
if (removeUi) this._earcon.dispose();
this._radio.clearPendingPlayback();
this._viewport.reset();
this._input.resetSession();