mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 19:04:02 +00:00
feat(voice): voice core — honest layer reach, spoken summary with on-screen card, narration, point-and-ask
- Voice layer manifest (src/voice/layerManifest.js) generates the layer enums, aliases and analyst field hints; a test fails when a catalog layer has no entry. - analyst_query validates fields, operators, units, values and scopes, counts up to 250,000 loaded records per layer, reports complete/partial/unanswered, tells LAYER_OFF / NOT_READY / FEED_UNAVAILABLE apart from zero, refuses mismatched follow-ups and never lets a superseded query replace follow-up memory. Seven more layers expose analyst records. - Tool results carry a code-built say line and a display card; the voice card in the mic control shows captions, plan steps and the result (src/voice/resultDisplay.js is the one adapter). Slow tools narrate their current step out of band; holding Space while the assistant talks cuts it off. Turn spans go to the debug log; caption transcription is metered; the log omits what was said unless GEV_VOICE_LOG_CONTENT=1. - Point and ask: the Space-down pointer snapshot resolves "this", "that one" and "around here" through 'pointer' targets, and "the second one" through referent:n from the last numbered list. - Prompt states each policy once (instructions + tools 48.5k -> 46.7k chars). - Voice benchmark harness (scripts/qa-voice-bench.mjs, scripts/voice-bench/) shares its phrase table and grading with scripts/qa-voice-routing.mjs.
This commit is contained in:
1 parent
e7707d9a0f
commit
a19897ad15
97 files changed
+14343
-755
No files matched your search
@@ -0,0 +1,232 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { RealtimeViewport, pointerCropRect } from './realtimeViewport.js';
|
||||
|
||||
test('crop: a square around the pointer, clamped inside the canvas, in buffer pixels', () => {
|
||||
assert.deepEqual(
|
||||
pointerCropRect({
|
||||
x: 750,
|
||||
y: 475,
|
||||
cssWidth: 1500,
|
||||
cssHeight: 950,
|
||||
pixelWidth: 3000,
|
||||
pixelHeight: 1900,
|
||||
}),
|
||||
{
|
||||
sx: 988,
|
||||
sy: 438,
|
||||
sw: 1024,
|
||||
sh: 1024,
|
||||
output: 512,
|
||||
pointX: 256,
|
||||
pointY: 256,
|
||||
},
|
||||
);
|
||||
const corner = pointerCropRect({
|
||||
x: 20,
|
||||
y: 20,
|
||||
cssWidth: 1500,
|
||||
cssHeight: 950,
|
||||
pixelWidth: 1500,
|
||||
pixelHeight: 950,
|
||||
});
|
||||
assert.equal(corner.sx, 0);
|
||||
assert.equal(corner.sy, 0);
|
||||
assert.equal(
|
||||
corner.pointX,
|
||||
20,
|
||||
'the ring still marks the real spot near an edge',
|
||||
);
|
||||
assert.equal(
|
||||
pointerCropRect({
|
||||
x: 1,
|
||||
y: 1,
|
||||
cssWidth: 0,
|
||||
cssHeight: 0,
|
||||
pixelWidth: 0,
|
||||
pixelHeight: 0,
|
||||
}),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
function viewportFixture({ capturePointer, capture } = {}) {
|
||||
const sent = [];
|
||||
const captures = [];
|
||||
const state = { clock: 10_000, pointer: null, camera: 'pose-A' };
|
||||
const channel = { readyState: 'open' };
|
||||
const viewport = new RealtimeViewport({
|
||||
readChannel: () => channel,
|
||||
readPointer: () => state.pointer,
|
||||
readCameraKey: () => state.camera,
|
||||
capture: capture || (async () => 'data:image/jpeg;base64,FULL'),
|
||||
capturePointer:
|
||||
capturePointer ||
|
||||
(async (px) => {
|
||||
captures.push(px);
|
||||
return {
|
||||
dataUrl: 'data:image/jpeg;base64,CROP',
|
||||
rect: { x: 0.25, y: 0.2, w: 0.3, h: 0.5 },
|
||||
};
|
||||
}),
|
||||
now: () => state.clock,
|
||||
operations: {
|
||||
sendRealtimeEvent: (event, label) => {
|
||||
sent.push({ event, label });
|
||||
return true;
|
||||
},
|
||||
},
|
||||
});
|
||||
return { viewport, sent, captures, state };
|
||||
}
|
||||
|
||||
const ground = {
|
||||
fresh: true,
|
||||
target: 'ground',
|
||||
at: 'keydown',
|
||||
screenPx: { x: 400, y: 300 },
|
||||
capturedAt: 9_000,
|
||||
cameraKey: 'pose-A',
|
||||
};
|
||||
|
||||
test('crop: sent once per snapshot, only at local/city scale, into the one retained slot', async () => {
|
||||
const { viewport, sent, captures, state } = viewportFixture();
|
||||
assert.equal(
|
||||
await viewport.sendPointerCrop(ground, { viewScale: 'regional' }),
|
||||
false,
|
||||
'the scale guard lives in the shared send path',
|
||||
);
|
||||
assert.equal(
|
||||
await viewport.sendPointerCrop(ground),
|
||||
false,
|
||||
'no scale, no crop',
|
||||
);
|
||||
assert.equal(
|
||||
await viewport.sendPointerCrop(ground, { viewScale: 'city' }),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
await viewport.sendPointerCrop(ground, { viewScale: 'city' }),
|
||||
false,
|
||||
'once per snapshot',
|
||||
);
|
||||
assert.deepEqual(captures, [{ x: 400, y: 300 }]);
|
||||
assert.equal(sent[0].label, 'client.pointer_crop');
|
||||
assert.match(sent[0].event.item.content[0].text, /ring marks the exact spot/);
|
||||
assert.equal(
|
||||
sent[0].event.item.content[1].image_url,
|
||||
'data:image/jpeg;base64,CROP',
|
||||
);
|
||||
assert.deepEqual(viewport.retainedFrame, {
|
||||
kind: 'crop',
|
||||
rect: { x: 0.25, y: 0.2, w: 0.3, h: 0.5 },
|
||||
cameraKey: 'pose-A',
|
||||
});
|
||||
assert.equal(
|
||||
await viewport.sendPointerCrop(
|
||||
{ ...ground, target: 'entity' },
|
||||
{ viewScale: 'city' },
|
||||
),
|
||||
false,
|
||||
'entities are named, not cropped',
|
||||
);
|
||||
state.clock = 60_000;
|
||||
assert.equal(
|
||||
await viewport.sendPointerCrop({ ...ground }, { viewScale: 'city' }),
|
||||
false,
|
||||
'too old',
|
||||
);
|
||||
|
||||
// Pull: a ground pointer lookup sends the crop, deleting the old image.
|
||||
state.clock = 10_000;
|
||||
state.pointer = { ...ground, at: 'speech_start' };
|
||||
const lookup = {
|
||||
action: 'get_entity_context',
|
||||
scope: 'pointer',
|
||||
selected: null,
|
||||
pointer: { latitude: 1, longitude: 2 },
|
||||
scene: { basemap: { viewScale: 'city' } },
|
||||
};
|
||||
assert.equal(await viewport.sendVisualContextIfUseful(lookup), true);
|
||||
assert.equal(
|
||||
sent.at(-2).event.type,
|
||||
'conversation.item.delete',
|
||||
'one retained image at most',
|
||||
);
|
||||
assert.equal(sent.at(-1).label, 'client.pointer_crop');
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful({
|
||||
...lookup,
|
||||
selected: { id: 'x' },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
await viewport.sendVisualContextIfUseful({
|
||||
...lookup,
|
||||
scene: { basemap: { viewScale: 'regional' } },
|
||||
}),
|
||||
false,
|
||||
);
|
||||
viewport.reset();
|
||||
assert.equal(viewport.lastPointerCrop, null);
|
||||
assert.equal(viewport.retainedFrame, null);
|
||||
});
|
||||
|
||||
test('crop: a camera move before or during capture drops it; pixels only name their own pose', async () => {
|
||||
let release;
|
||||
const { viewport, sent, state } = viewportFixture({
|
||||
capturePointer: () =>
|
||||
new Promise((resolve) => {
|
||||
release = () =>
|
||||
resolve({
|
||||
dataUrl: 'data:image/jpeg;base64,CROP',
|
||||
rect: { x: 0, y: 0, w: 1, h: 1 },
|
||||
});
|
||||
}),
|
||||
});
|
||||
state.camera = 'pose-B';
|
||||
assert.equal(
|
||||
await viewport.sendPointerCrop(ground, { viewScale: 'local' }),
|
||||
false,
|
||||
'moved before capture',
|
||||
);
|
||||
state.camera = 'pose-A';
|
||||
const pending = viewport.sendPointerCrop(
|
||||
{ ...ground },
|
||||
{ viewScale: 'local' },
|
||||
);
|
||||
state.camera = 'pose-B';
|
||||
release();
|
||||
assert.equal(await pending, false, 'moved while capturing');
|
||||
assert.equal(sent.length, 0);
|
||||
});
|
||||
|
||||
test('crop: an older capture that finishes last never replaces the newer image', async () => {
|
||||
const releases = [];
|
||||
const { viewport, sent } = viewportFixture({
|
||||
capturePointer: () =>
|
||||
new Promise((resolve) =>
|
||||
releases.push((url) => resolve({ dataUrl: url, rect: null })),
|
||||
),
|
||||
});
|
||||
const first = viewport.sendPointerCrop({ ...ground }, { viewScale: 'local' });
|
||||
const second = viewport.sendPointerCrop(
|
||||
{ ...ground },
|
||||
{ viewScale: 'local' },
|
||||
);
|
||||
releases[1]('data:image/jpeg;base64,NEW');
|
||||
assert.equal(await second, true);
|
||||
releases[0]('data:image/jpeg;base64,OLD');
|
||||
assert.equal(await first, false, 'superseded request');
|
||||
assert.deepEqual(
|
||||
sent.map((s) => s.event.item?.content?.[1]?.image_url).filter(Boolean),
|
||||
['data:image/jpeg;base64,NEW'],
|
||||
);
|
||||
|
||||
// A new user turn drops a capture still in flight from the previous one.
|
||||
const third = viewport.sendPointerCrop({ ...ground }, { viewScale: 'local' });
|
||||
viewport.beginTurn();
|
||||
releases[2]('data:image/jpeg;base64,LATE');
|
||||
assert.equal(await third, false);
|
||||
});
|
||||
Reference in new issue
Block a user