feat(voice): voice core — honest layer reach, spoken summary with on-screen card, narration, point-and-ask

- Voice layer manifest (src/voice/layerManifest.js) generates the layer enums,
  aliases and analyst field hints; a test fails when a catalog layer has no entry.
- analyst_query validates fields, operators, units, values and scopes, counts up
  to 250,000 loaded records per layer, reports complete/partial/unanswered, tells
  LAYER_OFF / NOT_READY / FEED_UNAVAILABLE apart from zero, refuses mismatched
  follow-ups and never lets a superseded query replace follow-up memory. Seven
  more layers expose analyst records.
- Tool results carry a code-built say line and a display card; the voice card in
  the mic control shows captions, plan steps and the result
  (src/voice/resultDisplay.js is the one adapter). Slow tools narrate their
  current step out of band; holding Space while the assistant talks cuts it off.
  Turn spans go to the debug log; caption transcription is metered; the log
  omits what was said unless GEV_VOICE_LOG_CONTENT=1.
- Point and ask: the Space-down pointer snapshot resolves "this", "that one" and
  "around here" through 'pointer' targets, and "the second one" through
  referent:n from the last numbered list.
- Prompt states each policy once (instructions + tools 48.5k -> 46.7k chars).
- Voice benchmark harness (scripts/qa-voice-bench.mjs, scripts/voice-bench/)
  shares its phrase table and grading with scripts/qa-voice-routing.mjs.
This commit is contained in:
Bilawal Sidhu committed 2026-09-28 21:19:23 -05:00
1 parent e7707d9a0f
commit a19897ad15
97 files changed
+14343 -755

No files matched your search

+13
View File
@@ -79,6 +79,9 @@ export class RealtimeInput {
// never surprise the user by muting a click-started conversation.
if (this.isActive() && !this.pushToTalkMode) return;
this.cancelPushToTalkHold();
// Point-and-ask: the user decided to ask at keydown, so that is where
// the pointer is read. A short tap releases it below.
this.holdPointer?.();
const holdGeneration = ++this.pushToTalkHoldGeneration;
this.pushToTalkHoldTimer = setTimeout(() => {
this.pushToTalkHoldTimer = null;
@@ -102,9 +105,16 @@ export class RealtimeInput {
) {
this.pushToTalkHoldControl.blur();
}
// Read Radio's claim before pausing it: pausing can change the very
// handoff state that decides whether voice may interrupt.
const mayInterrupt = this.mayClaimSpeaker?.() ?? true;
this.pauseRadioForVoice();
this.pushToTalkKeyHeld = true;
if (this.isActive()) {
// A claimed hold is an explicit request to speak: cut the assistant
// off if it is talking. Short taps never reach this point.
if (mayInterrupt) this.bargeIn?.();
this.beginPointerTurn?.('keydown');
this.ui.root.dataset.pushToTalk = 'held';
this.setMicrophoneEnabled(true);
if (this.status === 'listening')
@@ -124,6 +134,7 @@ export class RealtimeInput {
this.spaceKeyHeld = false;
this.cancelPushToTalkHold();
if (!this.pushToTalkKeyHeld) {
if (wasHoldingSpace) this.releasePointerHold?.();
if (wasHoldingSpace && !preservedNativeActivation)
event.preventDefault();
this.resetPushToTalkGesture();
@@ -175,6 +186,8 @@ export class RealtimeInput {
*/
releasePushToTalkKey() {
this.cancelPushToTalkHold();
// The pointer snapshot belongs to this gesture; release or blur ends it.
this.releasePointerHold?.();
if (!this.pushToTalkKeyHeld) return;
this.pushToTalkKeyHeld = false;
delete this.ui.root.dataset.pushToTalk;