mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-06 03:14:00 +00:00
feat(voice): voice core — honest layer reach, spoken summary with on-screen card, narration, point-and-ask
- Voice layer manifest (src/voice/layerManifest.js) generates the layer enums, aliases and analyst field hints; a test fails when a catalog layer has no entry. - analyst_query validates fields, operators, units, values and scopes, counts up to 250,000 loaded records per layer, reports complete/partial/unanswered, tells LAYER_OFF / NOT_READY / FEED_UNAVAILABLE apart from zero, refuses mismatched follow-ups and never lets a superseded query replace follow-up memory. Seven more layers expose analyst records. - Tool results carry a code-built say line and a display card; the voice card in the mic control shows captions, plan steps and the result (src/voice/resultDisplay.js is the one adapter). Slow tools narrate their current step out of band; holding Space while the assistant talks cuts it off. Turn spans go to the debug log; caption transcription is metered; the log omits what was said unless GEV_VOICE_LOG_CONTENT=1. - Point and ask: the Space-down pointer snapshot resolves "this", "that one" and "around here" through 'pointer' targets, and "the second one" through referent:n from the last numbered list. - Prompt states each policy once (instructions + tools 48.5k -> 46.7k chars). - Voice benchmark harness (scripts/qa-voice-bench.mjs, scripts/voice-bench/) shares its phrase table and grading with scripts/qa-voice-routing.mjs.
This commit is contained in:
1 parent
e7707d9a0f
commit
a19897ad15
97 files changed
+14343
-755
No files matched your search
@@ -1,5 +1,17 @@
|
||||
import { createVoiceControl } from './control.js';
|
||||
import { createVoiceSession } from './session.js';
|
||||
import { VoiceCardControls } from './voiceCard.js';
|
||||
import { PointerReticle } from './pointerReticle.js';
|
||||
|
||||
function createDefaultReticle(pointer) {
|
||||
const element = pointer?.element;
|
||||
const doc = element?.ownerDocument;
|
||||
if (!doc) return null;
|
||||
return new PointerReticle({
|
||||
doc,
|
||||
rectOf: () => element.getBoundingClientRect?.() || null,
|
||||
});
|
||||
}
|
||||
|
||||
/** Bind common controls to a supplied voice-session adapter. */
|
||||
export function createVoiceCommands({
|
||||
@@ -12,6 +24,10 @@ export function createVoiceCommands({
|
||||
signal,
|
||||
debugSink,
|
||||
createControl = createVoiceControl,
|
||||
createCard = (options) => new VoiceCardControls(options),
|
||||
pointer = null,
|
||||
referents = null,
|
||||
createReticle = createDefaultReticle,
|
||||
}) {
|
||||
window.__gevVoiceCommands?.stop?.({ removeUi: true });
|
||||
const ui = createControl({ reset: true });
|
||||
@@ -27,6 +43,7 @@ export function createVoiceCommands({
|
||||
backend,
|
||||
debugSink,
|
||||
createController,
|
||||
pointer,
|
||||
radioLayer: dataManager?.layers?.get('radio')?.module || null,
|
||||
}),
|
||||
});
|
||||
@@ -56,7 +73,30 @@ export function createVoiceCommands({
|
||||
: '';
|
||||
if (event.state === 'error') ui.root.classList?.remove('error-dismissed');
|
||||
});
|
||||
// Captions, plan and result card nested in the control; it only reads
|
||||
// provider-neutral session events.
|
||||
const card = ui.card ? createCard({ root: ui.root }) : null;
|
||||
const updateCard = card
|
||||
? session.subscribe((event) => card.handle(event))
|
||||
: null;
|
||||
controls.voiceCard = card;
|
||||
// Point-and-ask: the reticle follows turn events; pointer and referents are
|
||||
// session-scoped, so a session that ends forgets both.
|
||||
const reticle = pointer ? createReticle(pointer) : null;
|
||||
const updatePointer = session.subscribe((event) => {
|
||||
reticle?.handle(event);
|
||||
if (event.type === 'state' && ['idle', 'error'].includes(event.state)) {
|
||||
pointer?.clear();
|
||||
referents?.clear();
|
||||
// Conversation memory ("those", "the last answer") ends with the
|
||||
// session.
|
||||
runner.resetConversation?.();
|
||||
}
|
||||
});
|
||||
controls.pointer = pointer;
|
||||
controls.referents = referents;
|
||||
const annotationUnsubscribe = annotations?.onOutlineEvent?.((event) => {
|
||||
card?.outline(event);
|
||||
session.sendMapEvent({ type: 'map_annotation_outline', ...event });
|
||||
});
|
||||
const buttonHandler = () => {
|
||||
@@ -71,6 +111,11 @@ export function createVoiceCommands({
|
||||
ui.button.removeEventListener('click', buttonHandler);
|
||||
annotationUnsubscribe?.();
|
||||
updateStatus();
|
||||
updateCard?.();
|
||||
updatePointer();
|
||||
reticle?.destroy();
|
||||
card?.destroy();
|
||||
runner.dispose?.();
|
||||
ui.root.remove();
|
||||
},
|
||||
{ once: true },
|
||||
@@ -79,6 +124,11 @@ export function createVoiceCommands({
|
||||
ui.button.removeEventListener('click', buttonHandler);
|
||||
annotationUnsubscribe?.();
|
||||
updateStatus();
|
||||
updateCard?.();
|
||||
updatePointer();
|
||||
reticle?.destroy();
|
||||
card?.destroy();
|
||||
runner.dispose?.();
|
||||
ui.root.remove();
|
||||
} else adapter.bindControls?.();
|
||||
window.__gevVoiceCommands = controls;
|
||||
|
||||
Reference in new issue
Block a user