Refuse a motion after its reply's fly-to found nothing, also on Gemini

Gemini runs one turn's calls one after another. By the time "pan left" ran beside "fly to Xyzzyville", the lookup had already found nothing, so there was no flight to wait on and the pan moved the old view with ok: true. OpenAI's parallel calls refused the same reply.

The runner now remembers, for the latest reply that sent a voice fly-to, whether one moved the camera and whether one found nothing. A tilt, orbit, pan or rotate of that reply with no flight left to wait on is refused when its fly-to found nothing and none moved. The reply is Gemini's turn, passed as a per-turn reply object, or OpenAI's authorityGroup. A motion in a later reply, or beside a fly-to of the same reply that did land, runs as before.

src/voice/geminiSession.js

@@ -184,6 +184,9 @@ export class GeminiSessionController {
     this.model = null;
     this.epoch = 0;
     this.turn = 0;
+    // Identifies one turn's tool calls to the runner as one reply; a new
+    // object each turn, so no two sessions or turns share it.
+    this.turnReply = {};
     this.globeToolsBlocked = false;
     this.globeResetFailed = false;
     this.inputGeneration = 0;
@@ -971,6 +974,7 @@ export class GeminiSessionController {
     const tools = [...this.tools.values()];
     this.turnAbort = new AbortController();
     this.turn++;
+    this.turnReply = {};
     this.typedIntent = null;
     this.radioStrongerCommit = null;
     this.pruneControls();
@@ -1544,6 +1548,7 @@ export class GeminiSessionController {
     )
       throw new Error(msg('Gemini requested too many simultaneous actions'));
     const turn = this.turn;
+    const reply = this.turnReply;
     this.userTurnPending = false;
     this.modelComplete = false;
     // Admit each valid native call ID once per connection.
@@ -1565,6 +1570,7 @@ export class GeminiSessionController {
       const action = new AbortController();
       action.call = { id: call.id, name: call.name };
       action.sequence = ++this.toolSequence;
+      action.reply = reply;
       this.tools.set(call.id, action);
       const priorTools = this.toolQueue;
       const args =
@@ -1674,6 +1680,9 @@ export class GeminiSessionController {
         this.runAction(call.name, args, {
           signal: toolSignal,
           callId: call.id,
+          // Calls run one by one, so a motion can follow a fly-to of the
+          // same reply that already found nothing.
+          reply: action.reply,
           progress: (update) => {
             if (actionCurrent()) this.narration.progress(call.id, update);
           },

src/voice/geminiSession.test.mjs

@@ -1241,6 +1241,34 @@ test('successful stronger Radio stop still invalidates the prepared group', asyn
   h.c.stop();
 });
 
+test('Gemini tools of one turn share a reply identity, and a new turn gets another', async () => {
+  const replies = [];
+  const h = harness({
+    runAction: (name, _args, next) => {
+      replies.push([name, next.reply]);
+      return { ok: true, action: name };
+    },
+  });
+  await h.c.start({ pushToTalk: true });
+  h.c.sendTextCommand('Fly to Xyzzyville and pan left');
+  const ws = h.sockets[0];
+  requestTool(ws, 'fly', 'fly_to_location', { query: 'Xyzzyville' });
+  requestTool(ws, 'pan', 'move_camera', { motion: 'pan', direction: 'left' });
+  await h.c.toolQueue;
+  assert.equal(replies.length, 2);
+  assert.ok(replies[0][1], 'a sequentially run call carries its reply');
+  assert.equal(replies[1][1], replies[0][1]);
+  drainRadioReply(h);
+  await tick();
+  assert.equal(h.c.sendTextCommand('Pan left'), true);
+  requestTool(ws, 'pan-2', 'move_camera', { motion: 'pan', direction: 'left' });
+  await h.c.toolQueue;
+  assert.equal(replies.length, 3);
+  assert.ok(replies[2][1]);
+  assert.notEqual(replies[2][1], replies[0][1]);
+  h.c.stop();
+});
+
 test('slow Gemini tools reuse shared progress timing without injecting a conversation turn', async () => {
   const pending = deferred();
   let options;

src/voice/gevActions.js

@@ -299,6 +299,11 @@ export function createGevActionRunner({
   // Camera handoffs (navigation stamps) seen, and how many of them voice
   // fly-tos made. A search, share link or import that stamps during a flight
   // leaves it flying; the motion queued behind it then yields to that one.
+  // The voice fly-tos of the latest reply that sent one: whether any moved
+  // the camera, and whether one found nothing. A motion of that reply run
+  // after its fly-to found nothing (Gemini runs a reply's calls one by one)
+  // has no flight to wait on, and must not move the old view instead.
+  let replyFlights = null;
   let cameraHandoffs = 0;
   let voiceFlightHandoffs = 0;
   const otherHandoffs = () => cameraHandoffs - voiceFlightHandoffs;
@@ -1168,6 +1173,10 @@ export function createGevActionRunner({
       // Navigation by anything but a voice fly-to since this one began.
       flight.handoffs = otherHandoffs();
       voiceFlight = flight;
+      const reply = voiceReplyOf(runOptions);
+      if (reply !== null && replyFlights?.reply !== reply)
+        replyFlights = { reply, moved: false, foundNothing: false };
+      const replyRecord = reply !== null ? replyFlights : null;
       for (const notify of [...voiceFlightWatchers]) notify();
       // A motion already running (one the model sent before this fly-to) is
       // stopped when the flight takes navigation; the result says so.
@@ -1239,6 +1248,13 @@ export function createGevActionRunner({
           flight.settle('failed');
         else notStarted();
       }, notStarted);
+      // After the outcome above, which marks a lookup that moved nothing.
+      const noteReply = (result) => {
+        if (!replyRecord) return;
+        if (result?.ok === true) replyRecord.moved = true;
+        else if (flight.movedNothing) replyRecord.foundNothing = true;
+      };
+      flown.then(noteReply, () => noteReply(null));
       return flown.then((result) =>
         stoppedMotion && result && typeof result === 'object'
           ? { ...result, stoppedMotion }
@@ -1349,6 +1365,21 @@ export function createGevActionRunner({
       // A lookup that already found nothing does not hide the last flight
       // that moved: running now would cancel that flight mid-air.
       const pending = voiceFlightTarget(voiceFlight);
+      const reply = voiceReplyOf(runOptions);
+      if (
+        VOICE_FLIGHT_WAITING_MOTIONS.has(motion) &&
+        !(pending && !pending.done) &&
+        reply !== null &&
+        replyFlights?.reply === reply &&
+        replyFlights.foundNothing &&
+        !replyFlights.moved
+      )
+        return {
+          ok: false,
+          action: 'move_camera',
+          motion,
+          error: `Not run: the fly-to in this reply found nothing, so the ${motion} was not run at the old view.`,
+        };
       if (
         VOICE_FLIGHT_WAITING_MOTIONS.has(motion) &&
         pending &&
@@ -1724,6 +1755,7 @@ export function createGevActionRunner({
    * forget the analyst's "last answer". Area handles stay (page lifetime).
    */
   runVoiceAction.resetConversation = () => {
+    replyFlights = null;
     conversationAbort.abort();
     conversationAbort = new AbortController();
     conversation++;
@@ -3012,6 +3044,15 @@ async function frameOverhead(viewer, dataManager, styleManager, args = {}) {
 
 let voiceFlightCount = 0;
 
+/**
+ * The model reply a voice call belongs to: Gemini's turn (`reply`), or the
+ * OpenAI response its sibling calls share (`authorityGroup`). Null when the
+ * caller names none.
+ */
+function voiceReplyOf(options) {
+  return options?.reply ?? options?.authorityGroup ?? null;
+}
+
 /**
  * A voice flight's end: 'arrived', 'cancelled' or 'failed', settled once.
  * `replaced` is the last flight that moved when this one began (past lookups

src/voice/gevActions.test.mjs

@@ -938,6 +938,80 @@ test('a motion follows a newer fly-to that lands while its own lookup stalls', a
 // flies: the search stamps a deferred navigation without stopping the flight.
 // Run where the flight landed, the orbit took navigation and the search's
 // reassert failed, so the search was silently dropped.
+// Gemini runs one reply's calls one after another: by the time "pan left"
+// runs, "fly to Xyzzyville" has already found nothing, so there is no flight
+// to wait on, and the pan moved the old view with ok: true.
+test('a motion run after its reply\'s fly-to found nothing is not run', async (t) => {
+  t.after(() => interruptCameraMotion('test-cleanup'));
+  const { viewer, styleManager } = tiltAfterFlightHarness();
+  const runner = createGevActionRunner({
+    viewer,
+    styleManager,
+    dataManager: { layers: new Map(), getAll: () => [] },
+    searchNavigation: async () => null,
+  });
+  const reply = { turn: 1 };
+  const fly = await runner('fly_to_location', { query: 'Xyzzyville' }, { reply });
+  assert.equal(fly.ok, false);
+  const pan = await runner(
+    'move_camera',
+    { motion: 'pan', direction: 'left' },
+    { reply },
+  );
+  assert.equal(pan.ok, false);
+  assert.match(pan.error, /Not run: the fly-to in this reply found nothing/);
+  assert.equal(getActiveCameraMotion(), null);
+  // The same holds for OpenAI's reply identity.
+  const group = 'response-1';
+  await runner('fly_to_location', { query: 'Xyzzyville' }, { authorityGroup: group });
+  const tilt = await runner(
+    'move_camera',
+    { motion: 'tilt', pitchDeg: -45 },
+    { authorityGroup: group },
+  );
+  assert.equal(tilt.ok, false);
+  assert.match(tilt.error, /found nothing/);
+});
+
+test('a motion in a later reply runs after a fly-to that found nothing', async (t) => {
+  t.after(() => interruptCameraMotion('test-cleanup'));
+  const { viewer, styleManager } = tiltAfterFlightHarness();
+  const runner = createGevActionRunner({
+    viewer,
+    styleManager,
+    dataManager: { layers: new Map(), getAll: () => [] },
+    searchNavigation: async () => null,
+  });
+  await runner('fly_to_location', { query: 'Xyzzyville' }, { reply: { turn: 1 } });
+  const pan = await runner(
+    'move_camera',
+    { motion: 'pan', direction: 'left' },
+    { reply: { turn: 2 } },
+  );
+  assert.equal(pan.ok, true, pan.error);
+});
+
+test('a motion runs at a flight of its reply that landed after another lookup found nothing', async (t) => {
+  t.after(() => interruptCameraMotion('test-cleanup'));
+  const { viewer, styleManager, flights } = tiltAfterFlightHarness();
+  const runner = createGevActionRunner({
+    viewer,
+    styleManager,
+    dataManager: { layers: new Map(), getAll: () => [] },
+    searchNavigation: async () => null,
+  });
+  const reply = { turn: 1 };
+  await runner('fly_to_location', { query: 'Xyzzyville' }, { reply });
+  assert.equal((await runner('fly_to_location', { locationId: 'austin' }, { reply })).ok, true);
+  flights[0].complete();
+  const pan = await runner(
+    'move_camera',
+    { motion: 'pan', direction: 'left' },
+    { reply },
+  );
+  assert.equal(pan.ok, true, pan.error);
+});
+
 test('a motion queued behind a fly-to yields to a search started during the flight', async (t) => {
   t.after(() => interruptCameraMotion('test-cleanup'));
   const { viewer, styleManager, flights, panelSearch, currentGeneration } =