mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-06 11:24:00 +00:00
feat(voice): voice core — honest layer reach, spoken summary with on-screen card, narration, point-and-ask
- Voice layer manifest (src/voice/layerManifest.js) generates the layer enums, aliases and analyst field hints; a test fails when a catalog layer has no entry. - analyst_query validates fields, operators, units, values and scopes, counts up to 250,000 loaded records per layer, reports complete/partial/unanswered, tells LAYER_OFF / NOT_READY / FEED_UNAVAILABLE apart from zero, refuses mismatched follow-ups and never lets a superseded query replace follow-up memory. Seven more layers expose analyst records. - Tool results carry a code-built say line and a display card; the voice card in the mic control shows captions, plan steps and the result (src/voice/resultDisplay.js is the one adapter). Slow tools narrate their current step out of band; holding Space while the assistant talks cuts it off. Turn spans go to the debug log; caption transcription is metered; the log omits what was said unless GEV_VOICE_LOG_CONTENT=1. - Point and ask: the Space-down pointer snapshot resolves "this", "that one" and "around here" through 'pointer' targets, and "the second one" through referent:n from the last numbered list. - Prompt states each policy once (instructions + tools 48.5k -> 46.7k chars). - Voice benchmark harness (scripts/qa-voice-bench.mjs, scripts/voice-bench/) shares its phrase table and grading with scripts/qa-voice-routing.mjs.
This commit is contained in:
1 parent
e7707d9a0f
commit
a19897ad15
97 files changed
+14343
-755
No files matched your search
@@ -1,11 +1,42 @@
|
||||
import { createGevActionRunner } from './gevActions.js';
|
||||
import { createVoiceCommands } from './commands.js';
|
||||
import { cameraPoseKey, createPointerTracker } from './pointerContext.js';
|
||||
import { createPointerPicker } from './pointerPick.js';
|
||||
import { createReferentRegistry } from './referents.js';
|
||||
export * from './realtimeController.js';
|
||||
|
||||
/** Compose the standalone action runner with the voice controls. */
|
||||
export function initGevVoiceCommands(options) {
|
||||
return createVoiceCommands({
|
||||
const canvas = options.viewer?.scene?.canvas || null;
|
||||
const cameraKey = () => cameraPoseKey(options.viewer);
|
||||
// Point-and-ask: the tracker and referents are shared by the runner (which
|
||||
// resolves 'pointer' and referent:n) and the controls (which snapshot the
|
||||
// pointer at turn start and clear both when the session ends).
|
||||
const pointer = canvas
|
||||
? createPointerTracker({
|
||||
element: canvas,
|
||||
cameraKey,
|
||||
pick: createPointerPicker({
|
||||
viewer: options.viewer,
|
||||
dataManager: options.dataManager,
|
||||
}),
|
||||
})
|
||||
: null;
|
||||
const referents = createReferentRegistry();
|
||||
// The controller exists only after the runner, so its hooks bind late.
|
||||
const deixis = { pointer, referents, cameraKey };
|
||||
const commands = createVoiceCommands({
|
||||
...options,
|
||||
runner: createGevActionRunner(options),
|
||||
pointer,
|
||||
referents,
|
||||
runner: createGevActionRunner({ ...options, deixis }),
|
||||
});
|
||||
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
|
||||
deixis.onPointerUsed = () => commands.announcePointer?.();
|
||||
commands.session?.signal?.addEventListener(
|
||||
'abort',
|
||||
() => pointer?.destroy(),
|
||||
{ once: true },
|
||||
);
|
||||
return commands;
|
||||
}
|
||||
Reference in new issue
Block a user