fix(voice): resolve follow-up review edge cases

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+- Voice revisits numbered analyst records through current layer snapshots and
+  preserves resolved pin coordinates. Requested analyst lists and rankings can
+  speak returned items alongside the exact count and coverage caveats. Failed
+  actions settle the voice card without reviving cancelled turns. Coverage
+  benchmarks reject invalid calls before accepting semantic matches.
+
 - Street Level is included in the voice layer manifest for visibility toggles;
   imagery is not exposed as countable analyst records.
 

docs/CURRENT-STATE.md

@@ -387,6 +387,12 @@ the session cost and cap. The debug log omits transcripts, text, tool arguments
 and complete tool-result bodies (including serialized function outputs) unless
 `GEV_VOICE_LOG_CONTENT=1`.
 Tool names, call IDs, protocol event metadata and usage remain available.
+Numbered analyst targets resolve from current layer records, including layers
+without pick/context lookup; numbered annotations retain their resolved coordinates.
+Requested analyst lists and rankings may speak up to three returned items and
+values in result order alongside the count and its caveats. A rejected current
+action emits a failure result to settle its card; cancelled turns remain silent.
+Coverage grading validates calls before accepting an expected capability match.
 The voice manifest includes Street Level visibility toggles; its imagery has no
 analyst query or entity-context records.
 

scripts/voice-bench/grade.mjs

@@ -239,14 +239,14 @@ export const REFUSAL_RE =
  * Grade one coverage-probe turn.
  *
  * Order matters:
- *   1. a call pattern in `correct`                  → correct
+ *   1. a schema-invalid call (invented arg/tool)     → hallucinated-capability
- *   2. a schema-invalid call (invented arg/tool),
+ *   2. a call pattern in `correct`                  → correct
- *      or a pattern in `hallucinated`               → hallucinated-capability
+ *   3. a pattern in `hallucinated`                  → hallucinated-capability
- *   3. any other acting call (not lookup/neutral)   → wrong-tool
+ *   4. any other acting call (not lookup/neutral)    → wrong-tool
- *   4. no acting call: text matches `claimText`
+ *   5. no acting call: text matches `claimText`
  *      and no refusal wording                        → hallucinated-capability
- *   5. no acting call: refusal wording or a question → honest-refusal
+ *   6. no acting call: refusal wording or a question → honest-refusal
- *   6. no acting call, no refusal                    → hallucinated-capability
+ *   7. no acting call, no refusal                    → hallucinated-capability
  *      (it answered as if it had the data)
  *
  * `acceptable` = correct, or honest-refusal on an unsupported item.
@@ -259,17 +259,17 @@ export function gradeCoverage(item, calls, text, tools = []) {
     reason,
     acceptable: v === 'correct' || (v === 'honest-refusal' && !item.supported),
   });
-  for (const alternative of item.correct || []) {
-    if (alternative.every((m) => calls.some((c) => m(c)))) {
-      return verdict('correct', `matched ${alternative.map((m) => m.toolName || 'pattern').join('+')}`);
-    }
-  }
   if (tools.length) {
     for (const c of calls) {
       const errors = validateCall(tools, c);
       if (errors.length) return verdict('hallucinated-capability', `invalid ${c.name}: ${errors[0]}`);
     }
   }
+  for (const alternative of item.correct || []) {
+    if (alternative.every((m) => calls.some((c) => m(c)))) {
+      return verdict('correct', `matched ${alternative.map((m) => m.toolName || 'pattern').join('+')}`);
+    }
+  }
   for (const m of item.hallucinated || []) {
     const hit = calls.find((c) => m(c));
     if (hit) return verdict('hallucinated-capability', `faked capability via ${hit.name}(${JSON.stringify(hit.args).slice(0, 80)})`);

server/providers/openai/instructions.js

@@ -30,7 +30,7 @@ function realtimeInstructions(
     'POINTING: pointer_context says what the cursor is on (pointing:<kind>: resolved by the tools; target "none": not pointing). For "this/that/here/that one" with no name, pass "pointer": get_entity_context scope, track_entity/fly_to_location query, analyst_query scope.kind ("around here" is pointer, not view), annotate_map target. "The second/last one" is referent:n from the last numbered list. Without a pointer, ask or follow WHERE.',
     'PREAMBLES: for instant commands (layers, styles, HUD, detection, zoom, panels, basemap, radio controls, camera nudges, and lookups such as get_entity_context) call the tool without speaking first. Only for slow or multi-step work (annotate_map, a searched fly_to_location, select_nearest_aircraft, analyst_query over a named place, or two or more tools) you may first say ONE sentence of at most 12 words naming the actual steps ("Finding the Capitol, then outlining its grounds."), then call the tools at once in the same response.',
     'When one request contains MULTIPLE changes, call ALL the corresponding tools before the confirmation. Never confirm a partial subset; if one fails, say which.',
-    'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never adding to it and never repeating your preamble; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
+    'AFTER TOOLS: speak the result\'s say line once, lightly rephrased at most, never repeating your preamble. For a requested analyst list or ranking, also name up to three returned items with the relevant returned values; keep their order. A count alone does not answer that request; if say is null, finish any remaining requested tools before confirming. Never add a count, list or action claim that the executed tool results do not support. Sources, notes and caveats are on the voice card: speak a caveat only when it changes the answer, or when asked why, how sure or which source. Lower bounds ("at least 500"), partial answers and stale, degraded or unavailable feed states in say are part of the answer — always preserve them. No filler or hedges ("let me check", "keep in mind", "it seems", "based on the data available"), and never add "approximately" to an exact tool number.',
     'For "show/open/turn on" layer requests, enable the matching layer. For "hide/close/turn off", disable it.',
     // INSTRUCTION-ONLY mapping for the two globe-scale named views.
     //

src/tooling/voiceBench.test.mjs

@@ -308,3 +308,16 @@ test('coverage v2: a count for the removed drawn-area scope is never graded corr
   assert.equal(honest.verdict, 'honest-refusal');
   assert.equal(honest.acceptable, true);
 });
+
+
+test('coverage validates all calls before accepting a matching capability', () => {
+  const item = coverageRubric(coverage('transit-on'), 'v2');
+  const valid = { name: 'set_layer_visibility', args: { layerId: 'transit', enabled: true } };
+  const invalid = { name: 'set_layer_visibility', args: { layerId: 'transit-invented', enabled: true } };
+  for (const calls of [[invalid], [valid, invalid]]) {
+    const result = gradeCoverage(item, calls, 'Transit on.', GEV_REALTIME_TOOLS);
+    assert.equal(result.verdict, 'hallucinated-capability');
+    assert.equal(result.acceptable, false);
+  }
+  assert.equal(gradeCoverage(item, [valid], 'Transit on.', GEV_REALTIME_TOOLS).verdict, 'correct');
+});

src/voice/deixis.js

@@ -137,7 +137,14 @@ function resolveFly(args, { pointer, referents }) {
   if (target.layerId && target.id && source === 'referent') {
     // Contacts move: locate them when the flight starts, not from a list.
     return {
-      args: { ...rest, entity: { layerId: target.layerId, id: target.id } },
+      args: {
+        ...rest,
+        entity: {
+          layerId: target.layerId,
+          id: target.id,
+          ...(hasPosition(target) ? { lat: target.lat, lon: target.lon } : {}),
+        },
+      },
       used,
     };
   }

src/voice/deixis.test.mjs

@@ -1,5 +1,6 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
+import { attachVoiceResult } from './speech.js';
 import { resolveDeicticArgs } from './deixis.js';
 import { createReferentRegistry, referentsFromResult } from './referents.js';
 
@@ -167,3 +168,15 @@ test('registry: hidden card rows never resolve and last means displayed item fiv
   assert.equal(referents.get(6), null);
   assert.equal(referents.get(-1)?.id, 'F5');
 });
+
+
+test('numbered custom-labeled pins retain their resolved coordinates', () => {
+  const referents = createReferentRegistry();
+  const result = attachVoiceResult('annotate_map', {
+    ok: true,
+    items: [{ ok: true, id: 'pin-1', label: 'Meeting point', latitude: 30.27, longitude: -97.74 }],
+  });
+  referents.recordResult('annotate_map', result);
+  assert.deepEqual(resolveDeicticArgs('fly_to_location', { referent: 1 }, { referents }).args,
+    { latitude: 30.27, longitude: -97.74 });
+});

src/voice/gevActions.js

@@ -3318,7 +3318,7 @@ async function getEntityContext(
 const AIRCRAFT_LAYERS = new Set(['flights', 'military']);
 
 /** Live position of a layer record by id, or null when it is gone. */
-function locateLayerEntity(dataManager, { layerId, id }) {
+function locateLayerEntity(dataManager, { layerId, id, lat, lon }) {
   if (!layerId || !id) return null;
   if (dataManager?.isEnabled && !dataManager.isEnabled(layerId)) return null;
   let found = null;
@@ -3338,6 +3338,33 @@ function locateLayerEntity(dataManager, { layerId, id }) {
     Number.isFinite(record.longitude)
   )
     return { lat: record.latitude, lon: record.longitude, record };
+  // Analyst-only layers (e.g. bikeshare) have no pick/context lookup. Read
+  // their current snapshot, so a removed or moved record is not resurrected.
+  try {
+    const matches =
+      dataManager?.layers
+        ?.get(layerId)
+        ?.module?.getAnalystRecords?.()
+        .filter(
+          (item) =>
+            String(item.icao24 ?? item.mmsi ?? item.noradId ?? item.id) ===
+            String(id),
+        ) || [];
+    // Bikeshare uses station names as IDs: retain the coordinate identity
+    // even if a namesake disappears. Moving records use their fresh fix.
+    const row =
+      matches.length === 1 && layerId !== 'bikeshare'
+        ? matches[0]
+        : matches.find((item) => item.lat === lat && item.lon === lon);
+    if (Number.isFinite(row?.lat) && Number.isFinite(row?.lon))
+      return {
+        lat: row.lat,
+        lon: row.lon,
+        found: { ...row, latitude: row.lat, longitude: row.lon },
+      };
+  } catch {
+    // A missing or unavailable snapshot cannot establish a live position.
+  }
   return null;
 }
 

src/voice/gevActions.test.mjs

@@ -3822,3 +3822,66 @@ test('point-and-ask: only accepted, current results become the referent list', a
   assert.equal(used, 1, 'a current pointer resolution announces the chip');
   assert.equal(tracked, 1);
 });
+
+
+test('numbered analyst records navigate using the current layer accessor', async () => {
+  globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
+  const { viewer, styleManager } = createVoiceNavigationHarness();
+  const referents = createReferentRegistry();
+  let enabled = true;
+  let records = [{ id: 'congress', name: 'Congress station', lat: 30.27, lon: -97.74 }];
+  const runner = createActionRunner({
+    viewer, styleManager,
+    dataManager: {
+      layers: new Map([['bikeshare', { module: { getAnalystRecords: () => records } }]]),
+      isEnabled: () => enabled,
+      getAll: () => [],
+    },
+    deixis: { referents },
+  });
+  referents.recordResult('analyst_query', { ok: true, items: [{ ...records[0], layerKey: 'bikeshare' }] });
+  const result = await runner('fly_to_location', { referent: 1 });
+  assert.equal(result.ok, true);
+  assert.equal(result.latitude, 30.27);
+  assert.equal(result.longitude, -97.74);
+  enabled = false;
+  assert.equal((await runner('fly_to_location', { referent: 1 })).ok, false, 'disabled records cannot be revisited');
+  enabled = true;
+  records = [];
+  assert.equal((await runner('fly_to_location', { referent: 1 })).ok, false, 'removed records are not resurrected');
+});
+
+
+test('analyst-only lookup distinguishes duplicate names and uses canonical aircraft IDs', async () => {
+  globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
+  const { viewer, styleManager } = createVoiceNavigationHarness();
+  const referents = createReferentRegistry();
+  const stations = [
+    { id: 'Central Station', name: 'Central Station', lat: 30, lon: -97 },
+    { id: 'Central Station', name: 'Central Station', lat: 40, lon: -74 },
+  ];
+  const aircraft = [{ id: 'UAL1', icao24: 'abc123', lat: 35, lon: -100 }];
+  const runner = createActionRunner({
+    viewer, styleManager,
+    dataManager: {
+      layers: new Map([
+        ['bikeshare', { module: { getAnalystRecords: () => stations } }],
+        ['local-adsb', { module: { getAnalystRecords: () => aircraft } }],
+      ]),
+      isEnabled: () => true, getAll: () => [],
+    },
+    deixis: { referents },
+  });
+  referents.recordResult('analyst_query', { ok: true, items: stations.map(row => ({ ...row, layerKey: 'bikeshare' })) });
+  const station = await runner('fly_to_location', { referent: 2 });
+  assert.equal(station.ok, true);
+  assert.equal(station.latitude, 40);
+  assert.equal(station.longitude, -74);
+  stations.pop();
+  assert.equal((await runner('fly_to_location', { referent: 2 })).ok, false, 'a removed station must not resolve to its remaining namesake');
+  referents.recordResult('analyst_query', { ok: true, items: aircraft.map(row => ({ ...row, layerKey: 'local-adsb' })) });
+  aircraft[0].lat = 36;
+  const plane = await runner('fly_to_location', { referent: 1 });
+  assert.equal(plane.ok, true);
+  assert.equal(plane.latitude, 36, 'canonical ID resolves the current position, not old coordinates');
+});

src/voice/pointAndAsk.test.mjs

@@ -350,3 +350,13 @@ test('deixis prompt lint: one short rule, and every sentinel it names exists in
   const size = instructions.length + JSON.stringify(GEV_REALTIME_TOOLS).length;
   assert.ok(size <= 52000, `prompt size ${size}`);
 });
+
+
+test('after-tool speech allows returned items to answer requested lists and rankings', () => {
+  const rule = realtimeInstructions().split('\n').find(line => line.startsWith('AFTER TOOLS:'));
+  assert.match(rule, /requested analyst list or ranking/);
+  assert.match(rule, /returned items.*returned values/);
+  assert.match(rule, /keep their order/);
+  assert.doesNotMatch(rule, /never adding to it/);
+  assert.match(rule, /always preserve them/);
+});

src/voice/session.js

@@ -62,11 +62,11 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
       !actionSignal.aborted &&
       epoch === generation &&
       (!options.isCurrent || options.isCurrent());
+    const callId = options.callId ?? null;
     try {
       if (!isCurrent()) throw abortError();
       // Calls carry the adapter's call id so observers can match progress
       // and results to the right call when a tool runs twice at once.
-      const callId = options.callId ?? null;
       emit({ type: 'action-call', name, callId, arguments: args });
       if (!isCurrent()) throw abortError();
       // Step reports reach observers only while the action is current, so a
@@ -91,6 +91,15 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
       if (!isCurrent()) throw abortError();
       emit({ type: 'action-result', name, callId, result });
       return result;
+    } catch (error) {
+      if (isCurrent())
+        emit({
+          type: 'action-result',
+          name,
+          callId,
+          result: { ok: false, error: error?.message || String(error) },
+        });
+      throw error;
     } finally {
       actions.delete(action);
     }

src/voice/session.test.mjs

@@ -1,5 +1,10 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
+import {
+  initialVoiceCardState,
+  reduceVoiceCard,
+  voiceCardView,
+} from './voiceCardPresentation.js';
 import { createVoiceSession } from './session.js';
 import { realtimeSessionEvent } from './realtimeEvents.js';
 import { createVoiceCommands } from './sessionCommands.js';
@@ -373,3 +378,58 @@ test('action events carry the call id; progress after cancellation is dropped',
   report({ step: 'outline' });
   assert.equal(events.filter((e) => e.type === 'progress').length, 1);
 });
+
+test('a rejected current action settles its card and preserves the rejection', async () => {
+  const failure = new Error('Unknown panel');
+  const f = fixture(async () => {
+    throw failure;
+  });
+  let card = initialVoiceCardState();
+  const events = [];
+  f.session.subscribe((event) => {
+    events.push(event);
+    card = reduceVoiceCard(card, event);
+  });
+  await f.session.start();
+  await assert.rejects(
+    f.hooks.runAction(
+      'set_panel_open',
+      { panelId: 'missing' },
+      { callId: 'failed-call' },
+    ),
+    (error) => error === failure,
+  );
+  f.hooks.emit({ type: 'completion', status: 'completed' });
+  assert.equal(voiceCardView(card).busy, false);
+  assert.equal(card.steps[0].status, 'failed');
+  const result = events.find((event) => event.type === 'action-result');
+  assert.equal(result.callId, 'failed-call');
+  assert.equal(result.result.ok, false);
+  f.session.destroy();
+});
+
+test('a late rejection cannot settle a newer turn card', async () => {
+  let reject;
+  const f = fixture(
+    () =>
+      new Promise((_, fail) => {
+        reject = fail;
+      }),
+  );
+  const events = [];
+  f.session.subscribe((event) => events.push(event));
+  await f.session.start();
+  const pending = f.hooks.runAction(
+    'set_panel_open',
+    {},
+    { callId: 'old-call' },
+  );
+  f.hooks.emit({ type: 'interruption', reason: 'user-speech' });
+  reject(new Error('late failure'));
+  await assert.rejects(pending, /late failure/);
+  assert.equal(
+    events.some((event) => event.type === 'action-result'),
+    false,
+  );
+  f.session.destroy();
+});

src/voice/speech.js

@@ -373,6 +373,8 @@ function annotateMap(result) {
     n: index + 1,
     id: item.id || null,
     label: spokenLabel(item.label || item.target || `Mark ${index + 1}`),
+    latitude: item.latitude,
+    longitude: item.longitude,
   }));
   const parts = [];
   if (failed.length)