feat(voice): voice core — honest layer reach, spoken summary with on-screen card, narration, point-and-ask

- Voice layer manifest (src/voice/layerManifest.js) generates the layer enums,
  aliases and analyst field hints; a test fails when a catalog layer has no entry.
- analyst_query validates fields, operators, units, values and scopes, counts up
  to 250,000 loaded records per layer, reports complete/partial/unanswered, tells
  LAYER_OFF / NOT_READY / FEED_UNAVAILABLE apart from zero, refuses mismatched
  follow-ups and never lets a superseded query replace follow-up memory. Seven
  more layers expose analyst records.
- Tool results carry a code-built say line and a display card; the voice card in
  the mic control shows captions, plan steps and the result
  (src/voice/resultDisplay.js is the one adapter). Slow tools narrate their
  current step out of band; holding Space while the assistant talks cuts it off.
  Turn spans go to the debug log; caption transcription is metered; the log
  omits what was said unless GEV_VOICE_LOG_CONTENT=1.
- Point and ask: the Space-down pointer snapshot resolves "this", "that one" and
  "around here" through 'pointer' targets, and "the second one" through
  referent:n from the last numbered list.
- Prompt states each policy once (instructions + tools 48.5k -> 46.7k chars).
- Voice benchmark harness (scripts/qa-voice-bench.mjs, scripts/voice-bench/)
  shares its phrase table and grading with scripts/qa-voice-routing.mjs.
This commit is contained in:
Bilawal Sidhu committed 2026-09-28 21:19:23 -05:00
1 parent e7707d9a0f
commit a19897ad15
97 files changed
+14343 -755

No files matched your search

+33 -2
View File
@@ -1,11 +1,42 @@
import { createGevActionRunner } from './gevActions.js';
import { createVoiceCommands } from './commands.js';
import { cameraPoseKey, createPointerTracker } from './pointerContext.js';
import { createPointerPicker } from './pointerPick.js';
import { createReferentRegistry } from './referents.js';
export * from './realtimeController.js';
/** Compose the standalone action runner with the voice controls. */
export function initGevVoiceCommands(options) {
return createVoiceCommands({
const canvas = options.viewer?.scene?.canvas || null;
const cameraKey = () => cameraPoseKey(options.viewer);
// Point-and-ask: the tracker and referents are shared by the runner (which
// resolves 'pointer' and referent:n) and the controls (which snapshot the
// pointer at turn start and clear both when the session ends).
const pointer = canvas
? createPointerTracker({
element: canvas,
cameraKey,
pick: createPointerPicker({
viewer: options.viewer,
dataManager: options.dataManager,
}),
})
: null;
const referents = createReferentRegistry();
// The controller exists only after the runner, so its hooks bind late.
const deixis = { pointer, referents, cameraKey };
const commands = createVoiceCommands({
...options,
runner: createGevActionRunner(options),
pointer,
referents,
runner: createGevActionRunner({ ...options, deixis }),
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();
commands.session?.signal?.addEventListener(
'abort',
() => pointer?.destroy(),
{ once: true },
);
return commands;
}