gods-eye-view/src/voice/pointAndAsk.test.mjs

import '../testSupport/registerDom.mjs';
import assert from 'node:assert/strict';
import test from 'node:test';
import { nextTick } from 'vue';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import { mountVoiceView } from '../testSupport/voiceView.mjs';
import { GevRealtimeController } from './realtimeController.js';
import {
  POINTER_OPEN_MIC_RECENT_MS,
  buildPointerSnapshot,
} from './pointerContext.js';
import {
  initialVoiceCardState,
  reduceVoiceCard,
  voiceCardView,
} from './voiceCardPresentation.js';
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js';

const aircraftSnapshot = buildPointerSnapshot({
  at: 'speech_start',
  x: 600,
  y: 200,
  width: 1000,
  height: 500,
  pick: {
    entity: {
      layerId: 'flights',
      id: 'a1b2c3',
      label: 'UPS793',
      kind: 'aircraft',
      lat: 30.2,
      lon: -97.7,
    },
    ground: { lat: 30.19, lon: -97.71 },
  },
});

function fakePointer(snapshots, { held = false } = {}) {
  const calls = [];
  let active = null;
  const pointer = {
    calls,
    held,
    beginTurn(at, options) {
      calls.push([at, options?.recentMs ?? null]);
      const snapshot = snapshots.shift() ?? { fresh: false, at };
      active = snapshot.fresh ? snapshot : null;
      return snapshot;
    },
    activeSnapshot: () => active,
    isHeld: () => pointer.held,
    hold: () => null,
    releaseHold() {
      pointer.held = false;
    },
    clear(options) {
      calls.push(['clear', options]);
      active = null;
    },
  };
  return pointer;
}

function controllerFixture(pointer, { pushToTalk = false } = {}) {
  const sent = [];
  const events = [];
  const controller = new GevRealtimeController({
    ui: createTestVoiceUi(),
    runner: async () => ({ ok: true }),
    debugSink: null,
    pointer,
    onSessionEvent: (event) => events.push(event),
  });
  controller.dc = {
    readyState: 'open',
    send: (body) => sent.push(JSON.parse(body)),
    close() {},
  };
  controller._input.pushToTalkMode = pushToTalk;
  const event = (payload) =>
    controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
  const items = () =>
    sent
      .filter(
        (m) =>
          m.type === 'conversation.item.create' && m.item.role === 'system',
      )
      .map((m) => JSON.parse(m.item.content[0].text));
  return { controller, sent, events, event, items };
}

test('push-to-talk: the held gesture sends the target once, with the chip after the card resets', async () => {
  const pointer = fakePointer([aircraftSnapshot, aircraftSnapshot], {
    held: true,
  });
  const { controller, sent, events, event, items } = controllerFixture(
    pointer,
    { pushToTalk: true },
  );
  controller.beginPointerTurn('keydown');
  await event({ type: 'input_audio_buffer.speech_started' });
  assert.deepEqual(
    pointer.calls.map(([at]) => at),
    ['keydown', 'speech_start'],
  );
  assert.equal(
    pointer.calls[0][1],
    null,
    'a gesture uses the normal freshness window',
  );
  assert.equal(items().length, 1, 'keydown and speech start share one item');
  assert.deepEqual(items()[0].target, {
    kind: 'aircraft',
    layer: 'flights',
    label: 'UPS793',
  });
  assert.equal(
    sent.some((m) => m.type === 'response.create'),
    false,
    'context never asks for a reply',
  );
  const order = events.map((e) => e.type);
  assert.ok(
    order.lastIndexOf('interruption') < order.lastIndexOf('pointer'),
    'the chip lands after the card resets',
  );
  assert.deepEqual(events.filter((e) => e.type === 'pointer').at(-1).pointer, {
    kind: 'this',
    label: 'UPS793',
    screenPx: { x: 600, y: 200 },
  });
});

test('push-to-talk: a speech start after the release keeps the gesture referent', async () => {
  const pointer = fakePointer([aircraftSnapshot], { held: true });
  const { controller, items, event } = controllerFixture(pointer, {
    pushToTalk: true,
  });
  controller.beginPointerTurn('keydown');
  pointer.releaseHold();
  await event({ type: 'input_audio_buffer.speech_started' });
  assert.equal(
    pointer.calls.length,
    1,
    'no new pointer read after the gesture ended',
  );
  assert.equal(pointer.activeSnapshot(), aircraftSnapshot);
  assert.equal(items().length, 1);
});

test('open mic: only a recent pointer counts, and the model learns only that one exists', async () => {
  const pointer = fakePointer([aircraftSnapshot]);
  const { events, event, items } = controllerFixture(pointer);
  await event({ type: 'input_audio_buffer.speech_started' });
  assert.equal(
    pointer.calls[0][1],
    POINTER_OPEN_MIC_RECENT_MS,
    'open mic asks for a recent pointer',
  );
  assert.deepEqual(
    items(),
    [{ type: 'pointer_context', at: 'speech_start', pointing: 'aircraft' }],
    'no label or coordinates',
  );
  assert.equal(
    events.filter((e) => e.type === 'pointer').at(-1).pointer,
    null,
    'no reticle or chip until a tool uses it',
  );
});

test('open mic: the chip and reticle appear when a tool resolves the pointer, once', async () => {
  const pointer = fakePointer([aircraftSnapshot]);
  const { controller, events, event } = controllerFixture(pointer);
  await event({ type: 'input_audio_buffer.speech_started' });
  assert.equal(controller.announcePointer(), true);
  assert.equal(controller.announcePointer(), false);
  assert.equal(
    events.filter((e) => e.type === 'pointer').at(-1).pointer.label,
    'UPS793',
  );
});

test('losing the pointer tells the model once, then a new pointer is sent again', async () => {
  const pointer = fakePointer(
    [aircraftSnapshot, { fresh: false }, { fresh: false }, aircraftSnapshot],
    { held: true },
  );
  const { controller, events, items } = controllerFixture(pointer, {
    pushToTalk: true,
  });
  controller.beginPointerTurn('keydown');
  controller.beginPointerTurn('keydown');
  controller.beginPointerTurn('keydown');
  controller.beginPointerTurn('keydown');
  assert.deepEqual(
    items().map((item) => item.target?.label || item.target),
    ['UPS793', 'none', 'UPS793'],
    'one clearing item, and the dedupe resets after it',
  );
  assert.equal(events.filter((e) => e.type === 'pointer')[1].pointer, null);
});

test('an unchanged pointer is not re-sent on the next utterance', async () => {
  const again = { ...aircraftSnapshot };
  const moved = { ...aircraftSnapshot, lat: 30.3, lon: -97.6 };
  const pointer = fakePointer([aircraftSnapshot, again, moved], { held: true });
  const { controller, events, items } = controllerFixture(pointer, {
    pushToTalk: true,
  });
  controller.beginPointerTurn('keydown');
  controller.beginPointerTurn('keydown');
  assert.equal(items().length, 1);
  assert.equal(
    events.filter((e) => e.type === 'pointer' && e.pointer).length,
    2,
    'the chip still shows every turn',
  );
  controller.beginPointerTurn('keydown');
  assert.equal(items().length, 2, 'a moved pointer is sent');
});

test('typed turns read the pointer themselves, before the text; stop clears it but keeps a starting hold', () => {
  const pointer = fakePointer([aircraftSnapshot], { held: true });
  const { controller, sent } = controllerFixture(pointer, { pushToTalk: true });
  controller.sendTextCommand('tell me about this');
  assert.deepEqual(pointer.calls[0], ['text', POINTER_OPEN_MIC_RECENT_MS]);
  const created = sent.filter((m) => m.type === 'conversation.item.create');
  assert.equal(created[0].item.role, 'system');
  assert.equal(created[1].item.role, 'user');
  controller._input.pushToTalkKeyHeld = true;
  controller.stop({ preserveStatus: true });
  assert.deepEqual(pointer.calls.at(-1), ['clear', { keepHold: true }]);
});

test('a claimed Space release or blur ends the pointer gesture', () => {
  const pointer = fakePointer([], { held: true });
  const { controller } = controllerFixture(pointer, { pushToTalk: true });
  controller._input.pushToTalkKeyHeld = true;
  controller._input.releasePushToTalkKey();
  assert.equal(pointer.held, false, 'released with the key');
  pointer.held = true;
  controller._input.releasePushToTalkKey();
  assert.equal(
    pointer.held,
    false,
    'blur takes the same path even when unclaimed',
  );
});

test('card: the chip says what "this" or "here" resolved to and resets per turn', () => {
  let state = [
    { type: 'interruption' },
    {
      type: 'pointer',
      pointer: { kind: 'this', label: 'UPS793', screenPx: { x: 1, y: 1 } },
    },
  ].reduce(reduceVoiceCard, initialVoiceCardState());
  let view = voiceCardView(state);
  assert.equal(view.visible, true);
  assert.deepEqual(view.pointer, {
    visible: true,
    kind: 'THIS',
    text: 'UPS793',
  });
  assert.equal(
    view.announce,
    'This: UPS793',
    'the live region says what "this" meant',
  );
  state = reduceVoiceCard(state, {
    type: 'pointer',
    pointer: { kind: 'here', label: '30.27, -97.74' },
  });
  assert.deepEqual(voiceCardView(state).pointer, {
    visible: true,
    kind: 'HERE',
    text: '30.27, -97.74',
  });
  assert.equal(
    voiceCardView(state).announce,
    'Here: 30.27, -97.74',
    'a changed target is announced',
  );
  state = reduceVoiceCard(state, { type: 'interruption' });
  assert.equal(
    voiceCardView(state).pointer.visible,
    false,
    'a new turn starts without a referent',
  );
  const unchanged = reduceVoiceCard(state, { type: 'pointer', pointer: null });
  assert.equal(unchanged, state, 'a stale pointer on an empty chip is a no-op');
});

test('markup: the chip sits under the user caption with a screen-reader separator', async (t) => {
  const { pinia } = mountVoiceView(t);
  const ui = createTestVoiceUi({ pinia });
  t.after(() => ui.remove());
  await nextTick();
  const card = document.getElementById('gev-voice-card');
  const order = [...card.children].map((node) => node.id);
  const user = order.indexOf('gev-voice-card-user');
  const chip = order.indexOf('gev-voice-card-pointer');
  const assistant = order.indexOf('gev-voice-card-assistant');
  assert.ok(user >= 0 && user < chip && chip < assistant);
  const pointer = document.getElementById('gev-voice-card-pointer');
  assert.equal(pointer.className, 'gev-voice-card-pointer');
  assert.equal(pointer.hidden, true);
  const separator = pointer.querySelector('.gev-voice-card-sr');
  assert.equal(separator.textContent, ': ');
  assert.equal(separator.previousElementSibling.id, 'gev-voice-card-pointer-kind');
});

test('deixis prompt lint: one short rule, and every sentinel it names exists in the schema', () => {
  const instructions = realtimeInstructions();
  const rules = instructions
    .split('\n')
    .filter((line) => line.startsWith('POINTING:'));
  assert.equal(rules.length, 1, 'one deixis rule');
  assert.ok(
    rules[0].length <= 470,
    `deixis rule stays short (${rules[0].length} chars)`,
  );
  const tool = (name) =>
    GEV_REALTIME_TOOLS.find((t) => t.name === name).parameters.properties;
  assert.ok(tool('get_entity_context').scope.enum.includes('pointer'));
  assert.ok(
    tool('analyst_query').scope.properties.kind.enum.includes('pointer'),
  );
  for (const name of ['track_entity', 'fly_to_location', 'get_entity_context'])
    assert.equal(
      tool(name).referent.type,
      'integer',
      `${name} accepts referent:n`,
    );
  assert.match(tool('track_entity').query.description, /"pointer"/);
  assert.match(tool('fly_to_location').query.description, /"pointer"/);
  assert.match(
    tool('annotate_map').annotations.items.properties.target.description,
    /"pointer"/,
  );
  // Budget: the whole Realtime prompt (instructions + tools), measured in
  // characters. It was 56,308 after the area/imagery/OSM merge; stating each
  // policy once (WHERE precedence, AREAS routing, AFTER TOOLS speech) brought
  // it to 51,646. Voice geometry (resolve_area, find_imagery, osm_query and
  // the analyst area scopes) raised the ceiling to 55,500; tightening the
  // AREAS rule and the area tool descriptions left it at 55,103 as measured
  // below (every tool, osm_query included); main's Sea Marks and Ski Pistes
  // layers bring it to 55,210, ~290 under. A new tool or rule has to fit
  // under this ceiling.
  const size = instructions.length + JSON.stringify(GEV_REALTIME_TOOLS).length;
  assert.ok(size <= 55_500, `prompt size ${size}`);
});


test('after-tool speech allows returned items to answer requested lists and rankings', () => {
  const rule = realtimeInstructions().split('\n').find(line => line.startsWith('AFTER TOOLS:'));
  assert.match(rule, /requested analyst list or ranking/);
  assert.match(rule, /returned items.*returned values/);
  assert.match(rule, /keep their order/);
  assert.doesNotMatch(rule, /never adding to it/);
  assert.match(rule, /always preserve them/);
});