feat(voice): voice core — honest layer reach, spoken summary with on-screen card, narration, point-and-ask

- Voice layer manifest (src/voice/layerManifest.js) generates the layer enums,
  aliases and analyst field hints; a test fails when a catalog layer has no entry.
- analyst_query validates fields, operators, units, values and scopes, counts up
  to 250,000 loaded records per layer, reports complete/partial/unanswered, tells
  LAYER_OFF / NOT_READY / FEED_UNAVAILABLE apart from zero, refuses mismatched
  follow-ups and never lets a superseded query replace follow-up memory. Seven
  more layers expose analyst records.
- Tool results carry a code-built say line and a display card; the voice card in
  the mic control shows captions, plan steps and the result
  (src/voice/resultDisplay.js is the one adapter). Slow tools narrate their
  current step out of band; holding Space while the assistant talks cuts it off.
  Turn spans go to the debug log; caption transcription is metered; the log
  omits what was said unless GEV_VOICE_LOG_CONTENT=1.
- Point and ask: the Space-down pointer snapshot resolves "this", "that one" and
  "around here" through 'pointer' targets, and "the second one" through
  referent:n from the last numbered list.
- Prompt states each policy once (instructions + tools 48.5k -> 46.7k chars).
- Voice benchmark harness (scripts/qa-voice-bench.mjs, scripts/voice-bench/)
  shares its phrase table and grading with scripts/qa-voice-routing.mjs.
This commit is contained in:
Bilawal Sidhu committed 2026-09-28 21:19:23 -05:00
1 parent e7707d9a0f
commit a19897ad15
97 files changed
+14343 -755

No files matched your search

+59
View File
@@ -21,6 +21,7 @@ import { openAiRealtimeProxy } from 'gods-eye-view/server/providers/openai';
import { keySetupEndpoint } from 'gods-eye-view/server/standalone/key-setup';
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
import { createDebugLogHandler } from '../../server/providers/openai/debug-log.js';
function install(plugin, preview = false) {
const routes = new Map();
@@ -195,6 +196,30 @@ test('Realtime handler preserves tools and default instructions, isolates suppli
assert.equal(sent[0].session.instructions, sent[2].session.instructions);
});
test('caption transcription is on by default, reported to the client meter, and can be turned off', async (t) => {
env(t, 'OPENAI_API_KEY', 'fixture-upstream-secret');
env(t, 'GEV_RATELIMIT_OPENAI_PER_MIN', undefined);
const sent = [];
t.mock.method(globalThis, 'fetch', async (url, options) => {
sent.push(JSON.parse(options.body));
return Response.json({ value: 'fixture-ephemeral' });
});
for (const [setting, expected] of [
[undefined, 'gpt-4o-mini-transcribe'],
['off', 'off'],
]) {
env(t, 'OPENAI_REALTIME_TRANSCRIBE_MODEL', setting);
const response = await request(
install(openAiRealtimeProxy()).get('/api/realtime/token'),
);
assert.equal(response.headers['x-gev-voice-transcribe-model'], expected);
assert.equal(
sent.at(-1).session.audio.input.transcription?.model ?? 'off',
expected,
);
}
});
test('debug logging resolves each supplied application directory independently', async (t) => {
const first = root(t),
second = root(t);
@@ -339,6 +364,40 @@ test('OpenAI routes answer generically when the upstream or the request fails',
assert.equal(token.body.includes('fixture-upstream-secret'), false);
});
test('the debug-log sink omits what people said unless content logging is enabled', async (t) => {
const record = {
event: 'server.event',
payload: {
type: 'conversation.item.input_audio_transcription.completed',
transcript: 'my home address is 12 Elm Street',
usage: { input_tokens: 40 },
nested: [{ type: 'input_text', text: 'call my sister' }],
},
span: { answer_text: 'Flying to Tokyo.', answer_words: 3 },
};
for (const includeContent of [false, true]) {
const sourceRoot = root(t);
const handler = createDebugLogHandler({ sourceRoot, includeContent });
const response = await request(handler, {
method: 'POST',
body: JSON.stringify(record),
});
assert.equal(response.status, 204);
const logged = readFileSync(
path.join(sourceRoot, '.gev-logs/realtime-conversations.jsonl'),
'utf8',
);
assert.equal(logged.includes('Elm Street'), includeContent);
assert.equal(logged.includes('call my sister'), includeContent);
assert.equal(logged.includes('Flying to Tokyo'), includeContent);
const parsed = JSON.parse(logged);
assert.equal(parsed.payload.usage.input_tokens, 40, 'numbers stay');
assert.equal(parsed.span.answer_words, 3);
if (!includeContent)
assert.equal(parsed.payload.transcript, '[omitted 32 chars]');
}
});
test('the debug-log sink stays bounded, rate limited, and quiet about failures', async (t) => {
const sourceRoot = root(t);
const handler = install(openAiRealtimeProxy({ sourceRoot })).get(