Files
gods-eye-view/src/voice/pointAndAsk.test.mjs
T
manjunath 8abc99aa0a fix(voice): preserve request ownership across callbacks
keep typed and tool-follow-up responses attached to their original requests
reject duplicate events before they affect newer work
preserve newer input and Radio handoffs during old callback cleanup
integrate the tested main snapshot without dropping existing history
2026-10-16 22:29:00 +05:30

363 lines
12 KiB
JavaScript

import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import test from 'node:test';
import { GevRealtimeController } from './realtimeController.js';
import {
POINTER_OPEN_MIC_RECENT_MS,
buildPointerSnapshot,
} from './pointerContext.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';
const aircraftSnapshot = buildPointerSnapshot({
at: 'speech_start',
x: 600,
y: 200,
width: 1000,
height: 500,
pick: {
entity: {
layerId: 'flights',
id: 'a1b2c3',
label: 'UPS793',
kind: 'aircraft',
lat: 30.2,
lon: -97.7,
},
ground: { lat: 30.19, lon: -97.71 },
},
});
function fakePointer(snapshots, { held = false } = {}) {
const calls = [];
let active = null;
const pointer = {
calls,
held,
beginTurn(at, options) {
calls.push([at, options?.recentMs ?? null]);
const snapshot = snapshots.shift() ?? { fresh: false, at };
active = snapshot.fresh ? snapshot : null;
return snapshot;
},
activeSnapshot: () => active,
isHeld: () => pointer.held,
hold: () => null,
releaseHold() {
pointer.held = false;
},
clear(options) {
calls.push(['clear', options]);
active = null;
},
};
return pointer;
}
function controllerFixture(pointer, { pushToTalk = false } = {}) {
const sent = [];
const events = [];
const controller = new GevRealtimeController({
ui: {
root: {
dataset: {},
classList: { remove() {} },
querySelectorAll: () => [],
},
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
},
runner: async () => ({ ok: true }),
debugSink: null,
pointer,
onSessionEvent: (event) => events.push(event),
});
controller.dc = {
readyState: 'open',
send: (body) => sent.push(JSON.parse(body)),
close() {},
};
controller._input.pushToTalkMode = pushToTalk;
const event = (payload) =>
controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
const items = () =>
sent
.filter(
(m) =>
m.type === 'conversation.item.create' && m.item.role === 'system',
)
.map((m) => JSON.parse(m.item.content[0].text));
return { controller, sent, events, event, items };
}
test('push-to-talk: the held gesture sends the target once, with the chip after the card resets', async () => {
const pointer = fakePointer([aircraftSnapshot, aircraftSnapshot], {
held: true,
});
const { controller, sent, events, event, items } = controllerFixture(
pointer,
{ pushToTalk: true },
);
controller.beginPointerTurn('keydown');
await event({ type: 'input_audio_buffer.speech_started' });
assert.deepEqual(
pointer.calls.map(([at]) => at),
['keydown', 'speech_start'],
);
assert.equal(
pointer.calls[0][1],
null,
'a gesture uses the normal freshness window',
);
assert.equal(items().length, 1, 'keydown and speech start share one item');
assert.deepEqual(items()[0].target, {
kind: 'aircraft',
layer: 'flights',
label: 'UPS793',
});
assert.equal(
sent.some((m) => m.type === 'response.create'),
false,
'context never asks for a reply',
);
const order = events.map((e) => e.type);
assert.ok(
order.lastIndexOf('interruption') < order.lastIndexOf('pointer'),
'the chip lands after the card resets',
);
assert.deepEqual(events.filter((e) => e.type === 'pointer').at(-1).pointer, {
kind: 'this',
label: 'UPS793',
screenPx: { x: 600, y: 200 },
});
});
test('push-to-talk: a speech start after the release keeps the gesture referent', async () => {
const pointer = fakePointer([aircraftSnapshot], { held: true });
const { controller, items, event } = controllerFixture(pointer, {
pushToTalk: true,
});
controller.beginPointerTurn('keydown');
pointer.releaseHold();
await event({ type: 'input_audio_buffer.speech_started' });
assert.equal(
pointer.calls.length,
1,
'no new pointer read after the gesture ended',
);
assert.equal(pointer.activeSnapshot(), aircraftSnapshot);
assert.equal(items().length, 1);
});
test('open mic: only a recent pointer counts, and the model learns only that one exists', async () => {
const pointer = fakePointer([aircraftSnapshot]);
const { events, event, items } = controllerFixture(pointer);
await event({ type: 'input_audio_buffer.speech_started' });
assert.equal(
pointer.calls[0][1],
POINTER_OPEN_MIC_RECENT_MS,
'open mic asks for a recent pointer',
);
assert.deepEqual(
items(),
[{ type: 'pointer_context', at: 'speech_start', pointing: 'aircraft' }],
'no label or coordinates',
);
assert.equal(
events.filter((e) => e.type === 'pointer').at(-1).pointer,
null,
'no reticle or chip until a tool uses it',
);
});
test('open mic: the chip and reticle appear when a tool resolves the pointer, once', async () => {
const pointer = fakePointer([aircraftSnapshot]);
const { controller, events, event } = controllerFixture(pointer);
await event({ type: 'input_audio_buffer.speech_started' });
assert.equal(controller.announcePointer(), true);
assert.equal(controller.announcePointer(), false);
assert.equal(
events.filter((e) => e.type === 'pointer').at(-1).pointer.label,
'UPS793',
);
});
test('losing the pointer tells the model once, then a new pointer is sent again', async () => {
const pointer = fakePointer(
[aircraftSnapshot, { fresh: false }, { fresh: false }, aircraftSnapshot],
{ held: true },
);
const { controller, events, items } = controllerFixture(pointer, {
pushToTalk: true,
});
controller.beginPointerTurn('keydown');
controller.beginPointerTurn('keydown');
controller.beginPointerTurn('keydown');
controller.beginPointerTurn('keydown');
assert.deepEqual(
items().map((item) => item.target?.label || item.target),
['UPS793', 'none', 'UPS793'],
'one clearing item, and the dedupe resets after it',
);
assert.equal(events.filter((e) => e.type === 'pointer')[1].pointer, null);
});
test('an unchanged pointer is not re-sent on the next utterance', async () => {
const again = { ...aircraftSnapshot };
const moved = { ...aircraftSnapshot, lat: 30.3, lon: -97.6 };
const pointer = fakePointer([aircraftSnapshot, again, moved], { held: true });
const { controller, events, items } = controllerFixture(pointer, {
pushToTalk: true,
});
controller.beginPointerTurn('keydown');
controller.beginPointerTurn('keydown');
assert.equal(items().length, 1);
assert.equal(
events.filter((e) => e.type === 'pointer' && e.pointer).length,
2,
'the chip still shows every turn',
);
controller.beginPointerTurn('keydown');
assert.equal(items().length, 2, 'a moved pointer is sent');
});
test('typed turns read the pointer themselves, before the text; stop clears it but keeps a starting hold', () => {
const pointer = fakePointer([aircraftSnapshot], { held: true });
const { controller, sent } = controllerFixture(pointer, { pushToTalk: true });
controller.sendTextCommand('tell me about this');
assert.deepEqual(pointer.calls[0], ['text', POINTER_OPEN_MIC_RECENT_MS]);
const created = sent.filter((m) => m.type === 'conversation.item.create');
assert.equal(created[0].item.role, 'system');
assert.equal(created[1].item.role, 'user');
controller._input.pushToTalkKeyHeld = true;
controller.stop({ preserveStatus: true });
assert.deepEqual(pointer.calls.at(-1), ['clear', { keepHold: true }]);
});
test('a claimed Space release or blur ends the pointer gesture', () => {
const pointer = fakePointer([], { held: true });
const { controller } = controllerFixture(pointer, { pushToTalk: true });
controller._input.pushToTalkKeyHeld = true;
controller._input.releasePushToTalkKey();
assert.equal(pointer.held, false, 'released with the key');
pointer.held = true;
controller._input.releasePushToTalkKey();
assert.equal(
pointer.held,
false,
'blur takes the same path even when unclaimed',
);
});
test('card: the chip says what "this" or "here" resolved to and resets per turn', () => {
let state = [
{ type: 'interruption' },
{
type: 'pointer',
pointer: { kind: 'this', label: 'UPS793', screenPx: { x: 1, y: 1 } },
},
].reduce(reduceVoiceCard, initialVoiceCardState());
let view = voiceCardView(state);
assert.equal(view.visible, true);
assert.deepEqual(view.pointer, {
visible: true,
kind: 'THIS',
text: 'UPS793',
});
assert.equal(
view.announce,
'This: UPS793',
'the live region says what "this" meant',
);
state = reduceVoiceCard(state, {
type: 'pointer',
pointer: { kind: 'here', label: '30.27, -97.74' },
});
assert.deepEqual(voiceCardView(state).pointer, {
visible: true,
kind: 'HERE',
text: '30.27, -97.74',
});
assert.equal(
voiceCardView(state).announce,
'Here: 30.27, -97.74',
'a changed target is announced',
);
state = reduceVoiceCard(state, { type: 'interruption' });
assert.equal(
voiceCardView(state).pointer.visible,
false,
'a new turn starts without a referent',
);
const unchanged = reduceVoiceCard(state, { type: 'pointer', pointer: null });
assert.equal(unchanged, state, 'a stale pointer on an empty chip is a no-op');
});
test('markup: the chip sits under the user caption with a screen-reader separator', () => {
const control = readFileSync(
new URL('./control.js', import.meta.url),
'utf8',
);
const user = control.indexOf('id="gev-voice-card-user"');
const chip = control.indexOf('id="gev-voice-card-pointer"');
const assistant = control.indexOf('id="gev-voice-card-assistant"');
assert.ok(user < chip && chip < assistant);
assert.match(
control,
/id="gev-voice-card-pointer" class="gev-voice-card-pointer" hidden/,
);
assert.match(control, /<span class="gev-voice-card-sr">: <\/span>/);
});
test('deixis prompt lint: one short rule, and every sentinel it names exists in the schema', () => {
const instructions = realtimeInstructions();
const rules = instructions
.split('\n')
.filter((line) => line.startsWith('POINTING:'));
assert.equal(rules.length, 1, 'one deixis rule');
assert.ok(
rules[0].length <= 470,
`deixis rule stays short (${rules[0].length} chars)`,
);
const tool = (name) =>
GEV_REALTIME_TOOLS.find((t) => t.name === name).parameters.properties;
assert.ok(tool('get_entity_context').scope.enum.includes('pointer'));
assert.ok(
tool('analyst_query').scope.properties.kind.enum.includes('pointer'),
);
for (const name of ['track_entity', 'fly_to_location', 'get_entity_context'])
assert.equal(
tool(name).referent.type,
'integer',
`${name} accepts referent:n`,
);
assert.match(tool('track_entity').query.description, /"pointer"/);
assert.match(tool('fly_to_location').query.description, /"pointer"/);
assert.match(
tool('annotate_map').annotations.items.properties.target.description,
/"pointer"/,
);
// Budget: the whole Realtime prompt (instructions + tools), measured in
// characters. It was 56,308 after the area/imagery/OSM merge; stating each
// policy once (WHERE precedence, AREAS routing, AFTER TOOLS speech) brought
// it to 51,646. A new tool or rule has to fit under this ceiling.
const size = instructions.length + JSON.stringify(GEV_REALTIME_TOOLS).length;
assert.ok(size <= 52000, `prompt size ${size}`);
});
test('after-tool speech allows returned items to answer requested lists and rankings', () => {
const rule = realtimeInstructions().split('\n').find(line => line.startsWith('AFTER TOOLS:'));
assert.match(rule, /requested analyst list or ranking/);
assert.match(rule, /returned items.*returned values/);
assert.match(rule, /keep their order/);
assert.doesNotMatch(rule, /never adding to it/);
assert.match(rule, /always preserve them/);
});