mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-16 16:22:20 +00:00
feat(voice): voice core — honest layer reach, spoken summary with on-screen card, narration, point-and-ask
- Voice layer manifest (src/voice/layerManifest.js) generates the layer enums, aliases and analyst field hints; a test fails when a catalog layer has no entry. - analyst_query validates fields, operators, units, values and scopes, counts up to 250,000 loaded records per layer, reports complete/partial/unanswered, tells LAYER_OFF / NOT_READY / FEED_UNAVAILABLE apart from zero, refuses mismatched follow-ups and never lets a superseded query replace follow-up memory. Seven more layers expose analyst records. - Tool results carry a code-built say line and a display card; the voice card in the mic control shows captions, plan steps and the result (src/voice/resultDisplay.js is the one adapter). Slow tools narrate their current step out of band; holding Space while the assistant talks cuts it off. Turn spans go to the debug log; caption transcription is metered; the log omits what was said unless GEV_VOICE_LOG_CONTENT=1. - Point and ask: the Space-down pointer snapshot resolves "this", "that one" and "around here" through 'pointer' targets, and "the second one" through referent:n from the last numbered list. - Prompt states each policy once (instructions + tools 48.5k -> 46.7k chars). - Voice benchmark harness (scripts/qa-voice-bench.mjs, scripts/voice-bench/) shares its phrase table and grading with scripts/qa-voice-routing.mjs.
This commit is contained in:
1 parent
e7707d9a0f
commit
a19897ad15
97 files changed
+14343
-755
No files matched your search
@@ -0,0 +1,287 @@
|
||||
import { planStepLabel, progressStepLabel, spokenLabel } from './speech.js';
|
||||
import { presentResult } from './resultDisplay.js';
|
||||
import { normalizeReferents } from './referents.js';
|
||||
|
||||
/**
|
||||
* Pure presentation of the voice card nested in the voice control: captions,
|
||||
* the current plan, the latest result and its numbered referents.
|
||||
*
|
||||
* `reduceVoiceCard` folds provider-neutral session events (`state`,
|
||||
* `interruption`, `transcript`, `action-call`, `progress`, `action-result`)
|
||||
* plus the card's own `outline` and `dismiss` events into a snapshot;
|
||||
* `voiceCardView` derives what each element shows.
|
||||
*/
|
||||
|
||||
const MAX_STEPS = 6;
|
||||
const MAX_LINES = 6;
|
||||
const MAX_REFERENTS = 5;
|
||||
const MAX_NOTES = 6;
|
||||
const MAX_CAPTION = 220;
|
||||
|
||||
const STATUS_TEXT = Object.freeze({
|
||||
running: 'in progress',
|
||||
done: 'done',
|
||||
failed: 'failed',
|
||||
cancelled: 'cancelled',
|
||||
});
|
||||
|
||||
/** An empty, hidden card. */
|
||||
export function initialVoiceCardState() {
|
||||
return {
|
||||
visible: false,
|
||||
dismissed: false,
|
||||
user: null,
|
||||
pointer: null,
|
||||
assistant: null,
|
||||
assistantResponse: null,
|
||||
steps: [],
|
||||
stepSeq: 0,
|
||||
result: null,
|
||||
announce: '',
|
||||
};
|
||||
}
|
||||
|
||||
function caption(text) {
|
||||
const clean = String(text || '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
return clean.length > MAX_CAPTION
|
||||
? `…${clean.slice(-(MAX_CAPTION - 1))}`
|
||||
: clean;
|
||||
}
|
||||
|
||||
function newTurn(state) {
|
||||
return {
|
||||
...state,
|
||||
visible: true,
|
||||
dismissed: false,
|
||||
user: '',
|
||||
pointer: null,
|
||||
assistant: null,
|
||||
assistantResponse: null,
|
||||
steps: [],
|
||||
result: null,
|
||||
};
|
||||
}
|
||||
|
||||
function reveal(state) {
|
||||
return state.dismissed ? state : { ...state, visible: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Patch the step an event belongs to: the step with the event's call id, or,
|
||||
* for adapters without call ids, the latest matching step of that tool.
|
||||
* Returns the same snapshot when nothing matches, so stale or unmatched
|
||||
* events never re-render or reveal the card.
|
||||
*/
|
||||
function patchStep(state, event, match, patch) {
|
||||
for (let index = state.steps.length - 1; index >= 0; index--) {
|
||||
const step = state.steps[index];
|
||||
const sameCall = event.callId
|
||||
? step.callId === event.callId
|
||||
: !step.callId && step.name === event.name;
|
||||
if (!sameCall || !match(step)) continue;
|
||||
const steps = state.steps.slice();
|
||||
steps[index] = { ...step, ...patch(step) };
|
||||
return { ...state, steps };
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
function annotationIds(result) {
|
||||
return (Array.isArray(result.items) ? result.items : [])
|
||||
.filter((item) => item?.ok && item.outlinePending && item.id)
|
||||
.map((item) => String(item.id));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold one event into the card snapshot. Unknown, stale or unmatched events
|
||||
* return the same snapshot, so callers can skip rendering.
|
||||
*/
|
||||
export function reduceVoiceCard(state, event) {
|
||||
if (!event?.type) return state;
|
||||
switch (event.type) {
|
||||
case 'state':
|
||||
return ['idle', 'error'].includes(event.state)
|
||||
? initialVoiceCardState()
|
||||
: state;
|
||||
case 'interruption':
|
||||
return newTurn(state);
|
||||
case 'dismiss':
|
||||
return { ...state, visible: false, dismissed: true };
|
||||
case 'idle':
|
||||
return { ...state, visible: false };
|
||||
case 'pointer': {
|
||||
// What "this"/"here" resolved to; a stale pointer clears the chip.
|
||||
const pointer =
|
||||
event.pointer?.label && ['this', 'here'].includes(event.pointer.kind)
|
||||
? { kind: event.pointer.kind, label: event.pointer.label }
|
||||
: null;
|
||||
if (!pointer && !state.pointer) return state;
|
||||
if (!pointer) return { ...state, pointer };
|
||||
// The live region says what "this" meant, as the chip does.
|
||||
const kind = pointer.kind === 'here' ? 'Here' : 'This';
|
||||
return reveal({
|
||||
...state,
|
||||
pointer,
|
||||
announce: `${kind}: ${spokenLabel(pointer.label, 32)}`,
|
||||
});
|
||||
}
|
||||
case 'transcript': {
|
||||
const text = event.text || '';
|
||||
if (event.role === 'user') {
|
||||
return text ? reveal({ ...state, user: caption(text) }) : state;
|
||||
}
|
||||
if (event.role !== 'assistant' || !text) return state;
|
||||
const sameResponse =
|
||||
(event.responseId || null) === state.assistantResponse &&
|
||||
state.assistant !== null;
|
||||
// Deltas accumulate raw (their spacing is significant); only a final
|
||||
// transcript is normalized.
|
||||
const running = `${sameResponse ? state.assistant : ''}${text}`;
|
||||
const assistant = event.final
|
||||
? caption(text)
|
||||
: running.length > MAX_CAPTION
|
||||
? `…${running.slice(-(MAX_CAPTION - 1))}`
|
||||
: running;
|
||||
return reveal({
|
||||
...state,
|
||||
assistant,
|
||||
assistantResponse: event.responseId || null,
|
||||
announce: event.final ? caption(text) : state.announce,
|
||||
});
|
||||
}
|
||||
case 'action-call': {
|
||||
const stepSeq = state.stepSeq + 1;
|
||||
return reveal({
|
||||
...state,
|
||||
stepSeq,
|
||||
steps: [
|
||||
...state.steps,
|
||||
{
|
||||
id: stepSeq,
|
||||
callId: event.callId || null,
|
||||
name: event.name,
|
||||
label: planStepLabel(event.name, event.arguments || {}),
|
||||
status: 'running',
|
||||
detail: '',
|
||||
annotationIds: [],
|
||||
},
|
||||
].slice(-MAX_STEPS),
|
||||
});
|
||||
}
|
||||
case 'progress': {
|
||||
const next = patchStep(
|
||||
state,
|
||||
event,
|
||||
(step) => step.status === 'running',
|
||||
() => ({ detail: progressStepLabel(event.step, event.label) }),
|
||||
);
|
||||
return next === state ? state : reveal(next);
|
||||
}
|
||||
case 'action-result': {
|
||||
const result = event.result || {};
|
||||
const next = patchStep(
|
||||
state,
|
||||
event,
|
||||
(step) => step.status === 'running',
|
||||
() => ({
|
||||
status: result.cancelled
|
||||
? 'cancelled'
|
||||
: result.ok
|
||||
? 'done'
|
||||
: 'failed',
|
||||
detail: result.outlinePending ? 'Tracing outline' : '',
|
||||
annotationIds: result.outlinePending ? annotationIds(result) : [],
|
||||
}),
|
||||
);
|
||||
// A result for a call this turn never showed is stale.
|
||||
if (next === state) return state;
|
||||
// One adapter decides what a result shows; a silent lookup or a
|
||||
// refusal informs the model and leaves the card.
|
||||
const shown = presentResult(event.name, result);
|
||||
if (!shown) return reveal(next);
|
||||
return reveal({
|
||||
...next,
|
||||
result: {
|
||||
display: shown.display,
|
||||
referents: normalizeReferents(shown.referents),
|
||||
},
|
||||
announce: spokenLabel(shown.display.title, 64),
|
||||
});
|
||||
}
|
||||
case 'outline': {
|
||||
const id = event.id == null ? null : String(event.id);
|
||||
if (!id) return state;
|
||||
for (let index = state.steps.length - 1; index >= 0; index--) {
|
||||
const step = state.steps[index];
|
||||
if (!step.annotationIds?.includes(id)) continue;
|
||||
const remaining = step.annotationIds.filter((other) => other !== id);
|
||||
const steps = state.steps.slice();
|
||||
steps[index] = {
|
||||
...step,
|
||||
annotationIds: remaining,
|
||||
detail: remaining.length
|
||||
? 'Tracing outline'
|
||||
: event.status === 'resolved'
|
||||
? 'Outline traced'
|
||||
: 'Outline unavailable',
|
||||
};
|
||||
return reveal({ ...state, steps });
|
||||
}
|
||||
return state;
|
||||
}
|
||||
default:
|
||||
return state;
|
||||
}
|
||||
}
|
||||
|
||||
/** Derive element content from a card snapshot. */
|
||||
export function voiceCardView(state) {
|
||||
const running = state.steps.some((step) => step.status === 'running');
|
||||
const display = state.result?.display || null;
|
||||
const notes = [
|
||||
...(display?.notes || []),
|
||||
...(display?.sources || []).map((source) => `Source: ${source.label}`),
|
||||
]
|
||||
.map((note) => spokenLabel(note, 96))
|
||||
.slice(0, MAX_NOTES);
|
||||
return {
|
||||
visible: Boolean(state.visible),
|
||||
busy: running,
|
||||
phase: running ? 'WORKING' : state.steps.length ? 'DONE' : '',
|
||||
user: {
|
||||
visible: state.user !== null,
|
||||
text: state.user || '…',
|
||||
},
|
||||
pointer: {
|
||||
visible: Boolean(state.pointer),
|
||||
kind: state.pointer?.kind === 'here' ? 'HERE' : 'THIS',
|
||||
text: spokenLabel(state.pointer?.label || '', 32),
|
||||
},
|
||||
assistant: {
|
||||
visible: Boolean(state.assistant),
|
||||
text: state.assistant || '',
|
||||
},
|
||||
steps: state.steps.map((step) => ({
|
||||
key: String(step.id),
|
||||
label: step.label,
|
||||
status: step.status,
|
||||
statusText: STATUS_TEXT[step.status] || '',
|
||||
detail: step.detail || '',
|
||||
})),
|
||||
result: {
|
||||
visible: Boolean(display?.title),
|
||||
title: spokenLabel(display?.title || '', 64),
|
||||
chips: (display?.chips || []).map((chip) => spokenLabel(chip.label, 24)),
|
||||
lines: (display?.lines || [])
|
||||
.map((line) => spokenLabel(line, 80))
|
||||
.slice(0, MAX_LINES),
|
||||
referents: (state.result?.referents || [])
|
||||
.slice(0, MAX_REFERENTS)
|
||||
.map((ref) => ({ n: ref.n, label: spokenLabel(ref.label, 48) })),
|
||||
notes,
|
||||
},
|
||||
announce: state.announce || '',
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user