From 0c2e5727a5c1748323986eb901bc2cf31006b5a2 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:52:44 -0500 Subject: [PATCH 01/17] feat(voice): reason-code viewport context escalation --- src/voice/realtimeViewport.js | 59 ++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/src/voice/realtimeViewport.js b/src/voice/realtimeViewport.js index fd93996af..c41049d4c 100644 --- a/src/voice/realtimeViewport.js +++ b/src/voice/realtimeViewport.js @@ -11,6 +11,31 @@ export const VIEWPORT_MAX_PIXELS = 1200 * 900; // ~1.08 MP, matches the old 1200px-wide landscape budget export const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024; +/** + * Explain whether the existing Realtime path should escalate structured entity + * context to a viewport image. This is instrumentation vocabulary only; it + * preserves the current send policy exactly. + */ +export function viewportContextDecision(result, { channelOpen = true } = {}) { + if (result?.action !== 'get_entity_context') + return { send: false, reason: 'not-entity-context' }; + if (!channelOpen) return { send: false, reason: 'channel-closed' }; + const viewScale = result.scene?.basemap?.viewScale ?? null; + if (!shouldSendViewportImage(viewScale)) + return { send: false, reason: 'view-scale-not-local', viewScale }; + if (hasStructuredViewIdentity(result)) + return { + send: false, + reason: 'structured-identity-sufficient', + viewScale, + }; + return { + send: true, + reason: 'structured-identity-missing', + viewScale, + }; +} + export async function captureViewportImage() { const viewer = window.__godsEyeView?.viewer; const source = @@ -179,25 +204,31 @@ export class RealtimeViewport { } async sendVisualContextIfUseful(result) { - if ( - result?.action !== 'get_entity_context' || - !this.dc || - this.dc.readyState !== 'open' - ) - return false; - const viewScale = result.scene?.basemap?.viewScale; - if (!shouldSendViewportImage(viewScale)) return false; - if (hasStructuredViewIdentity(result)) return false; - const generation = this.generation; const channel = this.dc; + const decision = viewportContextDecision(result, { + channelOpen: channel?.readyState === 'open', + }); + this.debugLog?.('viewport_context.decision', decision); + if (!decision.send) return false; + + const generation = this.generation; const imageUrl = await this.capture(); + if (!imageUrl) { + this.debugLog?.('viewport_context.skipped', { + reason: 'capture-unavailable', + }); + return false; + } if ( - !imageUrl || generation !== this.generation || this.dc !== channel || channel.readyState !== 'open' - ) + ) { + this.debugLog?.('viewport_context.skipped', { + reason: 'session-changed-during-capture', + }); return false; + } // Keep at most one viewport screenshot in context. Images are the single // most expensive item (re-billed every turn they linger), so we proactively @@ -257,6 +288,10 @@ export class RealtimeViewport { 'client.viewport_context', ); this.lastViewportItemId = sent ? newItemId : null; + this.debugLog?.('viewport_context.result', { + sent, + reason: sent ? 'sent' : 'data-channel-send-failed', + }); return sent; } From 073ffedc1fbccfe64ed1b9bad6ce636dad23c859 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:52:54 -0500 Subject: [PATCH 02/17] feat(voice): route viewport decisions to diagnostics --- src/voice/realtimeController.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/voice/realtimeController.js b/src/voice/realtimeController.js index d0c9f1bae..285d0451b 100644 --- a/src/voice/realtimeController.js +++ b/src/voice/realtimeController.js @@ -82,6 +82,7 @@ export class GevRealtimeController extends RealtimeFacade { operations: { sendRealtimeEvent: (...args) => this.sendRealtimeEvent(...args), + debugLog: (...args) => this.debugLog(...args), }, }); this._diagnostics = new RealtimeDiagnostics({ From 3a40117e0619f477e7451cf57475670f9ccdcf99 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:53:12 -0500 Subject: [PATCH 03/17] test(voice): pin viewport escalation reasons --- src/voice/realtimeViewportPolicy.test.mjs | 72 +++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/voice/realtimeViewportPolicy.test.mjs diff --git a/src/voice/realtimeViewportPolicy.test.mjs b/src/voice/realtimeViewportPolicy.test.mjs new file mode 100644 index 000000000..b2d135597 --- /dev/null +++ b/src/voice/realtimeViewportPolicy.test.mjs @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { viewportContextDecision } from './realtimeViewport.js'; + +const localContext = (overrides = {}) => ({ + action: 'get_entity_context', + scene: { basemap: { viewScale: 'local' } }, + ...overrides, +}); + +test('viewport context reasons preserve the existing escalation gate', () => { + assert.deepEqual(viewportContextDecision({ action: 'get_current_view_state' }), { + send: false, + reason: 'not-entity-context', + }); + assert.deepEqual( + viewportContextDecision(localContext(), { channelOpen: false }), + { send: false, reason: 'channel-closed' }, + ); + assert.deepEqual( + viewportContextDecision({ + action: 'get_entity_context', + scene: { basemap: { viewScale: 'regional' } }, + }), + { + send: false, + reason: 'view-scale-not-local', + viewScale: 'regional', + }, + ); + assert.deepEqual( + viewportContextDecision( + localContext({ + selected: { + id: 'earthquake:1', + layerId: 'earthquakes', + }, + }), + ), + { + send: false, + reason: 'structured-identity-sufficient', + viewScale: 'local', + }, + ); + assert.deepEqual(viewportContextDecision(localContext()), { + send: true, + reason: 'structured-identity-missing', + viewScale: 'local', + }); +}); + +test('nearby structured place identity avoids visual escalation too', () => { + assert.deepEqual( + viewportContextDecision( + localContext({ + scene: { + basemap: { + viewScale: 'local', + nearbyPlaces: [{ name: 'Austin' }], + }, + }, + }), + ), + { + send: false, + reason: 'structured-identity-sufficient', + viewScale: 'local', + }, + ); +}); From 86969f8a11812be9f8a2133cd4d6bbf74c365614 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:53:15 -0500 Subject: [PATCH 04/17] chore(format): adopt viewport policy tests --- scripts/format-scope.json | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/format-scope.json b/scripts/format-scope.json index cd836a5cb..16041e033 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -603,6 +603,7 @@ "src/voice/control.js", "src/voice/realtimeBackend.js", "src/voice/realtimeBackend.test.mjs", + "src/voice/realtimeViewportPolicy.test.mjs", "src/voice/layerSummary.js", "build/application-html.js", "src/app/sourceSlot.js", From 5195012c7f3ea322c1717d2cf9078d898ae54936 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:55:51 -0500 Subject: [PATCH 05/17] debug(format): print viewport policy canonical output --- scripts/format.mjs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/format.mjs b/scripts/format.mjs index 25311ba29..e1f2d4ca6 100644 --- a/scripts/format.mjs +++ b/scripts/format.mjs @@ -120,6 +120,15 @@ export async function formatAdoptedFiles(root, mode) { } const changed = files.filter(({ source, formatted }) => source !== formatted); + if (mode === '--check') { + const target = changed.find( + ({ name }) => name === 'src/voice/realtimeViewportPolicy.test.mjs', + ); + if (target) + console.error( + `FORMAT_EXPECTED BEGIN\n${target.formatted}FORMAT_EXPECTED END`, + ); + } if (mode === '--write') { for (const { file, formatted } of changed) await writeFile(file, formatted); } From 989e740d6efb9276f22f80a638d402776029a0c2 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:57:43 -0500 Subject: [PATCH 06/17] style(voice): format viewport policy tests --- src/voice/realtimeViewportPolicy.test.mjs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/voice/realtimeViewportPolicy.test.mjs b/src/voice/realtimeViewportPolicy.test.mjs index b2d135597..cd49cd3f0 100644 --- a/src/voice/realtimeViewportPolicy.test.mjs +++ b/src/voice/realtimeViewportPolicy.test.mjs @@ -10,10 +10,13 @@ const localContext = (overrides = {}) => ({ }); test('viewport context reasons preserve the existing escalation gate', () => { - assert.deepEqual(viewportContextDecision({ action: 'get_current_view_state' }), { - send: false, - reason: 'not-entity-context', - }); + assert.deepEqual( + viewportContextDecision({ action: 'get_current_view_state' }), + { + send: false, + reason: 'not-entity-context', + }, + ); assert.deepEqual( viewportContextDecision(localContext(), { channelOpen: false }), { send: false, reason: 'channel-closed' }, From cacbbc972bf8ada8314a2a62b7cdae1abefa6c0c Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:57:48 -0500 Subject: [PATCH 07/17] chore: remove temporary formatter diagnostics --- scripts/format.mjs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/scripts/format.mjs b/scripts/format.mjs index e1f2d4ca6..25311ba29 100644 --- a/scripts/format.mjs +++ b/scripts/format.mjs @@ -120,15 +120,6 @@ export async function formatAdoptedFiles(root, mode) { } const changed = files.filter(({ source, formatted }) => source !== formatted); - if (mode === '--check') { - const target = changed.find( - ({ name }) => name === 'src/voice/realtimeViewportPolicy.test.mjs', - ); - if (target) - console.error( - `FORMAT_EXPECTED BEGIN\n${target.formatted}FORMAT_EXPECTED END`, - ); - } if (mode === '--write') { for (const { file, formatted } of changed) await writeFile(file, formatted); } From 050f68ac03ea65a27627517f8f72b4e814398568 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:37:22 -0500 Subject: [PATCH 08/17] test(voice): classify current-frame send and cancellation outcomes Keep viewport and pointer-crop capture unchanged, but expose static reason codes for unavailable frames, invalidated turn captures and retained image sends. No image or user text in diagnostics. --- src/voice/realtimeViewport.js | 16 +++++- src/voice/realtimeViewportPolicy.test.mjs | 61 +++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/src/voice/realtimeViewport.js b/src/voice/realtimeViewport.js index 9ab1b07a6..8ba50c73b 100644 --- a/src/voice/realtimeViewport.js +++ b/src/voice/realtimeViewport.js @@ -437,15 +437,24 @@ export class RealtimeViewport { const captured = await captureImage(); const imageUrl = typeof captured === 'string' ? captured : captured?.dataUrl || null; + if (!imageUrl) { + this.debugLog?.('viewport_context.skipped', { + reason: 'capture-unavailable', + }); + return false; + } if ( - !imageUrl || generation !== this.generation || request !== this.captureSeq || this.dc !== channel || channel.readyState !== 'open' || (typeof frame.valid === 'function' && !frame.valid()) - ) + ) { + this.debugLog?.('viewport_context.skipped', { + reason: 'capture-invalidated', + }); return false; + } // Keep at most one viewport screenshot in context. Images are the single // most expensive item (re-billed every turn they linger), so we proactively @@ -506,6 +515,9 @@ export class RealtimeViewport { cameraKey: this.readCameraKey(), } : null; + this.debugLog?.('viewport_context.result', { + reason: sent ? 'sent' : 'data-channel-send-failed', + }); return sent; } diff --git a/src/voice/realtimeViewportPolicy.test.mjs b/src/voice/realtimeViewportPolicy.test.mjs index 5a361bcb5..3398f2a09 100644 --- a/src/voice/realtimeViewportPolicy.test.mjs +++ b/src/voice/realtimeViewportPolicy.test.mjs @@ -179,3 +179,64 @@ test('closed channel never captures pointer or full viewport', async () => { ); assert.equal(await viewport.sendVisualContextIfUseful(localContext()), false); }); + +test('capture failures and invalidated turns are reason-coded without sending', async () => { + const channel = { readyState: 'open' }; + const diagnostics = []; + const viewport = new RealtimeViewport({ + readChannel: () => channel, + operations: { + sendRealtimeEvent: () => { + throw new Error('should not send unavailable image'); + }, + debugLog: (event, detail) => diagnostics.push({ event, detail }), + }, + capture: async () => null, + }); + assert.equal(await viewport.sendVisualContextIfUseful(localContext()), false); + assert.ok( + diagnostics.some( + (item) => + item.event === 'viewport_context.skipped' && + item.detail.reason === 'capture-unavailable', + ), + ); + let release; + viewport.capture = () => + new Promise((resolve) => { + release = resolve; + }); + const pending = viewport.sendVisualContextIfUseful(localContext()); + viewport.reset(); + release('data:image/jpeg;base64,AA=='); + assert.equal(await pending, false); + assert.ok( + diagnostics.some( + (item) => + item.event === 'viewport_context.skipped' && + item.detail.reason === 'capture-invalidated', + ), + ); +}); + +test('successful retained send is logged without recording image contents', async () => { + const channel = { readyState: 'open' }; + const diagnostics = []; + const viewport = new RealtimeViewport({ + readChannel: () => channel, + readCameraKey: () => 'camera:fixture', + operations: { + sendRealtimeEvent: () => true, + debugLog: (event, detail) => diagnostics.push({ event, detail }), + }, + capture: async () => 'data:image/jpeg;base64,AA==', + }); + assert.equal(await viewport.sendVisualContextIfUseful(localContext()), true); + assert.ok(diagnostics.some((item) => + item.event === 'viewport_context.result' && item.detail.reason === 'sent' + )); + assert.equal( + JSON.stringify(diagnostics).includes('data:image/jpeg'), + false, + ); +}); From d4980b1b77e2b2d00686dcdc7214df02bb99c779 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:39:42 -0500 Subject: [PATCH 09/17] test(voice): extract pinned formatting for #835 review repair (temporary) --- scripts/format.mjs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/format.mjs b/scripts/format.mjs index 25311ba29..1828bd84e 100644 --- a/scripts/format.mjs +++ b/scripts/format.mjs @@ -141,6 +141,18 @@ if (import.meta.url === invoked) { if (mode === '--check' && result.changed.length) { for (const name of result.changed) console.error(`Needs formatting: ${name}`); + // Temporary CI-only formatter diagnostic. This commit is never promoted. + for (const name of result.changed.filter((file) => + ['src/voice/realtimeViewport.js', 'src/voice/realtimeViewportPolicy.test.mjs'].includes(file) + )) { + const file = path.join(fileURLToPath(new URL('../', import.meta.url)), name); + const output = await prettier.format(await readFile(file, 'utf8'), { + ...(await prettier.resolveConfig(file)), filepath: file, + }); + const encoded = Buffer.from(output).toString('base64'); + for (let pos = 0; pos < encoded.length; pos += 1000) + console.error(`PRETTIER-DATA|${name}|${pos}|${encoded.slice(pos, pos + 1000)}`); + } process.exitCode = 1; } else { console.log( From cb3b66e38b4c1f1d8ce2d00576efd2641d487b08 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:41:12 -0500 Subject: [PATCH 10/17] style(voice): apply exact Prettier 3.9.6 output to #835 Restore unchanged formatter script after ephemeral CI verification. Keep current-main-compatible reason instrumentation, pointer crop and capture cancellation tests. --- scripts/format.mjs | 12 ------- src/voice/realtimeViewport.js | 1 - src/voice/realtimeViewportPolicy.test.mjs | 44 +++++++++++++++-------- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/scripts/format.mjs b/scripts/format.mjs index 1828bd84e..25311ba29 100644 --- a/scripts/format.mjs +++ b/scripts/format.mjs @@ -141,18 +141,6 @@ if (import.meta.url === invoked) { if (mode === '--check' && result.changed.length) { for (const name of result.changed) console.error(`Needs formatting: ${name}`); - // Temporary CI-only formatter diagnostic. This commit is never promoted. - for (const name of result.changed.filter((file) => - ['src/voice/realtimeViewport.js', 'src/voice/realtimeViewportPolicy.test.mjs'].includes(file) - )) { - const file = path.join(fileURLToPath(new URL('../', import.meta.url)), name); - const output = await prettier.format(await readFile(file, 'utf8'), { - ...(await prettier.resolveConfig(file)), filepath: file, - }); - const encoded = Buffer.from(output).toString('base64'); - for (let pos = 0; pos < encoded.length; pos += 1000) - console.error(`PRETTIER-DATA|${name}|${pos}|${encoded.slice(pos, pos + 1000)}`); - } process.exitCode = 1; } else { console.log( diff --git a/src/voice/realtimeViewport.js b/src/voice/realtimeViewport.js index 8ba50c73b..18d06093f 100644 --- a/src/voice/realtimeViewport.js +++ b/src/voice/realtimeViewport.js @@ -112,7 +112,6 @@ export function viewportContextDecision(result, { channelOpen = true } = {}) { }; } - /** * Square source rectangle around a canvas point, clamped inside the canvas, * in drawing-buffer pixels, plus where the point lands in the output image. diff --git a/src/voice/realtimeViewportPolicy.test.mjs b/src/voice/realtimeViewportPolicy.test.mjs index 3398f2a09..0d679559f 100644 --- a/src/voice/realtimeViewportPolicy.test.mjs +++ b/src/voice/realtimeViewportPolicy.test.mjs @@ -1,7 +1,10 @@ import assert from 'node:assert/strict'; import test from 'node:test'; -import { RealtimeViewport, viewportContextDecision } from './realtimeViewport.js'; +import { + RealtimeViewport, + viewportContextDecision, +} from './realtimeViewport.js'; const localContext = (overrides = {}) => ({ action: 'get_entity_context', @@ -99,7 +102,10 @@ test('pointer ground selects the #837 crop path, not a full viewport', () => { }, ); assert.deepEqual( - viewportContextDecision({ ...pointerContext, selected: { id: 'building' } }), + viewportContextDecision({ + ...pointerContext, + selected: { id: 'building' }, + }), { send: false, reason: 'pointer-identity-present', @@ -144,8 +150,14 @@ test('runtime dispatch preserves pointer crop and full image separation', async true, ); assert.equal(calls.filter((call) => call.kind === 'crop').length, 1); - assert.deepEqual(calls.find((call) => call.kind === 'crop').snapshot, pointer); - assert.equal(calls.find((call) => call.kind === 'crop').options.viewScale, 'city'); + assert.deepEqual( + calls.find((call) => call.kind === 'crop').snapshot, + pointer, + ); + assert.equal( + calls.find((call) => call.kind === 'crop').options.viewScale, + 'city', + ); assert.equal(calls.filter((call) => call.kind === 'viewport').length, 0); assert.equal(await viewport.sendVisualContextIfUseful(localContext()), true); @@ -157,10 +169,13 @@ test('runtime dispatch preserves pointer crop and full image separation', async false, ); assert.equal(calls.filter((call) => call.kind === 'viewport').length, 1); - assert.ok(calls.some((call) => - call.event === 'viewport_context.decision' && - call.detail.reason === 'pointer-ground-crop' - )); + assert.ok( + calls.some( + (call) => + call.event === 'viewport_context.decision' && + call.detail.reason === 'pointer-ground-crop', + ), + ); }); test('closed channel never captures pointer or full viewport', async () => { @@ -232,11 +247,12 @@ test('successful retained send is logged without recording image contents', asyn capture: async () => 'data:image/jpeg;base64,AA==', }); assert.equal(await viewport.sendVisualContextIfUseful(localContext()), true); - assert.ok(diagnostics.some((item) => - item.event === 'viewport_context.result' && item.detail.reason === 'sent' - )); - assert.equal( - JSON.stringify(diagnostics).includes('data:image/jpeg'), - false, + assert.ok( + diagnostics.some( + (item) => + item.event === 'viewport_context.result' && + item.detail.reason === 'sent', + ), ); + assert.equal(JSON.stringify(diagnostics).includes('data:image/jpeg'), false); }); From 27823d7d05c8bfbe40fbf3ef759835226c227bd8 Mon Sep 17 00:00:00 2001 From: manjunath22466 Date: Fri, 6 Nov 2026 22:37:41 +0530 Subject: [PATCH 11/17] fix(voice): make viewport escalation diagnostics truthful and paired Review follow-up for #835. What is captured, sent and kept is unchanged. - Log nothing for tool results other than get_entity_context, so other tools stop spending the shared debug-log budget. - Give every decision and outcome the same contextId and image kind, so overlapping requests pair up. - Log why a crop decision did not send (stale, off ground, off screen, expired, pose changed, already requested) instead of leaving a lone "pointer-ground-crop" that reads as sent. - Split capture-invalidated into session-reset, superseded, channel-lost and frame-stale. - Close the pair with "error" when a step throws, then rethrow. - Document the events in CURRENT-STATE and reword the changelog fragment. --- .../835-viewport-escalation-reasons.md | 12 +- docs/CURRENT-STATE.md | 1 + src/voice/realtimeViewport.js | 137 ++++++--- src/voice/realtimeViewportPolicy.test.mjs | 282 +++++++++++++++++- 4 files changed, 386 insertions(+), 46 deletions(-) diff --git a/changelog.d/835-viewport-escalation-reasons.md b/changelog.d/835-viewport-escalation-reasons.md index a5246d585..0fe8a1fbd 100644 --- a/changelog.d/835-viewport-escalation-reasons.md +++ b/changelog.d/835-viewport-escalation-reasons.md @@ -1,5 +1,7 @@ -- Voice viewport escalation now emits compact reason-code diagnostics without - recording tool payloads. The policy still prefers structured entity identity, - preserves the live point-and-ask crop added in #837, and does not add image - requests, change image retention or automatically widen visual scope. - This follows asturksever's #835 review; credit the #837 pointer-crop authors. +- The voice debug log now says why OpenAI Realtime did or did not get a + viewport screenshot or pointer crop. Each entity-context answer logs a + reason-coded decision and, when an image was due, one outcome under the same + id, so a crop that was dropped because the pointer went stale or the camera + moved is no longer counted as sent. What is captured, sent and kept is + unchanged, and no image data or user text is logged (kvnloo, manjunath22466, + #835). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 0c5d7e278..246993256 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5774,6 +5774,7 @@ and unreachable upstream (502/504) separately from road geometry. - **Reliability**: tool-call dedupe (2.5s window across call/item/args keys); response-create queueing that respects active responses and defers follow-ups when the user starts speaking; per-tool follow-up instructions so the agent confirms only what actually happened (zoom confirms only on `ok=true`). - **Aircraft identity honesty:** “What is this aircraft?” reads callsign, operator, registration, type, and route only from the selected contact context. Missing operator, route, or type enrichment is named explicitly rather than silently omitted or inferred from the callsign. - **Diagnostics**: every client/server event is posted to `/api/realtime/debug-log` and appended to `.gev-logs/realtime-conversations.jsonl` (gitignored) with secret/image redaction; last 30 errors persist in `localStorage` (`gev-realtime-errors`); `window.__gevVoiceCommands.getDiagnostics()` in the console. + - **Visual-context escalation** (OpenAI Realtime): when a `get_entity_context` answer reaches the image step (its tool output was sent and the data channel is open), it logs one `viewport_context.decision` (`contextId`, `send`, `kind` `viewport` or `crop`, `reason`, `viewScale`). A send decision is followed by exactly one outcome with the same `contextId` and `kind`: `viewport_context.result` (`sent`, `data-channel-send-failed`) or `viewport_context.skipped`. Pointer-crop skips are `pointer-not-fresh`, `pointer-not-ground`, `pointer-off-screen`, `pointer-already-requested` (the same snapshot was already sent or is still being captured), `pointer-expired`, `pointer-pose-unknown` and `pose-changed`; dropped captures are `capture-unavailable`, `session-reset`, `superseded` (a newer request or turn), `channel-lost` and `frame-stale`; a step that throws closes the pair with `error` before the caller logs `viewport_context.failed`. `channel-closed` and the crop's `pointer-scale-not-useful` are defensive and not expected from the current caller. Payloads are static codes only, never image data, tool payloads or user text, and other tool results log nothing here. Gemini Live uses a different gate in `geminiSession.js` (no pointer crop, a 1 s throttle), which is not instrumented. - **Counting semantics ("near")** — a CONTRACT; new count-bearing tool work inherits it. Three honest numbers exist for one question: the Contacts cohort (250 km around the subject, what the panel shows), `analyst_query`'s count of _currently-loaded_ records for the requested scope, and the layer-wide loaded total in `coverage.layersQueried[].total`. They diverge legitimately — the flights layer loads by viewport, so after a camera dive the loaded set can hold a fraction of the cohort (field case: panel 42, analyst 8). The contract: 1. **Contacts ACTIVE** → "near / nearby / how many aircraft" means the **Contacts window** — the panel's numbers, spoken verbatim. Mechanism: `contactsWindow` (`{centeredOn, radiusKm, flights, military, vessels, complete}`), carried by both `analyst_query` and `get_current_view_state`, derived by `contactsWindowFromSnapshot()` from the same snapshot the panel renders so the two cannot drift. Voice filtering and follow-up memory use that immutable snapshot too. A cohort whose feed cannot answer reports `'unknown'`, never a confident zero; a cohort at the 20,000-row retention bound reports a lower bound. 2. **Contacts OFF** → "nearby" means **in view**; "near \" means a radius around that place. A radius query with Contacts active and no explicit centre is centred on the **active contact**, not the camera. diff --git a/src/voice/realtimeViewport.js b/src/voice/realtimeViewport.js index e35adefd2..89b9051f7 100644 --- a/src/voice/realtimeViewport.js +++ b/src/voice/realtimeViewport.js @@ -339,6 +339,16 @@ export function isBenignViewportDeleteError(payload, pendingDeleteIds = null) { return code === 'item_not_found'; } +/** Pairs one visual-context request's outcome with its decision. */ +function diagnosticsRecord(contextId = null) { + return { contextId, logged: false }; +} + +function reportOutcome(owner, diag, event, kind, reason) { + diag.logged = true; + owner.debugLog?.(event, { contextId: diag.contextId, kind, reason }); +} + /** Own the one retained viewport image and bounded pending deletion identities. */ export class RealtimeViewport { constructor({ @@ -359,6 +369,8 @@ export class RealtimeViewport { // Bumped by every capture request and every new user turn: only the // newest request of the current turn may publish. this.captureSeq = 0; + // Diagnostics only: pairs each visual-context decision with its outcome. + this.contextSeq = 0; this.lastPointerCrop = null; // What the retained image shows: `{kind:'viewport'|'crop', rect, cameraKey}`. // Pixel targeting (screenX/screenY) is read against it. @@ -370,24 +382,47 @@ export class RealtimeViewport { return this.readChannel(); } + /** + * Escalate an entity-context answer to an image when the gate allows it. + * Logs one `viewport_context.decision`, then, for a send decision, exactly + * one `.skipped` or `.result` outcome, even when a step throws. Every + * event carries the same `contextId` and the image `kind`, so overlapping + * requests pair up. + */ async sendVisualContextIfUseful(result) { - const channel = this.dc; + // Runs after every tool output. Only entity context can escalate, so + // other tools log nothing and leave the shared debug-log budget alone. + if (result?.action !== 'get_entity_context') return false; + const contextId = ++this.contextSeq; const decision = viewportContextDecision(result, { - channelOpen: channel?.readyState === 'open', + channelOpen: this.dc?.readyState === 'open', }); - this.debugLog?.('viewport_context.decision', decision); + this.debugLog?.('viewport_context.decision', { contextId, ...decision }); if (!decision.send) return false; - // #837's pointer crop is separate from the full viewport: preserve it. - if (decision.kind === 'crop') - return this.sendPointerCrop(this.readPointer(), { - viewScale: decision.viewScale, - }); - return this.sendRetainedImage( - () => this.capture(), - "Current God's Eye View viewport screenshot. Read any clearly visible street, building, and place labels in the image and combine them with the structured nearbyPlaces, streetLabels, and scene context. Do not invent labels that are not legible.", - 'client.viewport_context', - { kind: 'viewport' }, - ); + const diag = diagnosticsRecord(contextId); + try { + // #837's pointer crop is separate from the full viewport: preserve it. + if (decision.kind === 'crop') + return await this.sendPointerCrop(this.readPointer(), { + viewScale: decision.viewScale, + diag, + }); + return await this.sendRetainedImage( + () => this.capture(), + "Current God's Eye View viewport screenshot. Read any clearly visible street, building, and place labels in the image and combine them with the structured nearbyPlaces, streetLabels, and scene context. Do not invent labels that are not legible.", + 'client.viewport_context', + { kind: 'viewport', diag }, + ); + } catch (error) { + // The caller logs the error itself; close the pair, then rethrow. + if (!diag.logged) + this.debugLog?.('viewport_context.skipped', { + contextId, + kind: decision.kind, + reason: 'error', + }); + throw error; + } } /** A new user turn: captures still in flight from the last one are dropped. */ @@ -401,27 +436,39 @@ export class RealtimeViewport { * @param {object|null} snapshot Fresh pointer snapshot on bare ground. * @returns {Promise} */ - async sendPointerCrop(snapshot, { viewScale } = {}) { - if (!snapshot?.fresh || snapshot.target !== 'ground' || !snapshot.screenPx) + async sendPointerCrop( + snapshot, + { viewScale, diag = diagnosticsRecord() } = {}, + ) { + const skip = (reason) => { + reportOutcome(this, diag, 'viewport_context.skipped', 'crop', reason); return false; + }; + if (!snapshot?.fresh) return skip('pointer-not-fresh'); + if (snapshot.target !== 'ground') return skip('pointer-not-ground'); + if (!snapshot.screenPx) return skip('pointer-off-screen'); // Too far out, the crop is a smear: every path shares this guard. - if (!POINTER_CROP_SCALES.has(viewScale)) return false; - if (snapshot === this.lastPointerCrop) return false; + if (!POINTER_CROP_SCALES.has(viewScale)) + return skip('pointer-scale-not-useful'); + // Set before the capture finishes, so this also covers one in flight. + if (snapshot === this.lastPointerCrop) + return skip('pointer-already-requested'); const capturedAt = Number(snapshot.capturedAt) || 0; - if (this.now() - capturedAt > POINTER_CROP_MAX_AGE_MS) return false; + if (this.now() - capturedAt > POINTER_CROP_MAX_AGE_MS) + return skip('pointer-expired'); // The snapshot's pixels only name its own camera pose. const snapshotCameraKey = snapshot.cameraKey; - if (!snapshotCameraKey) return false; + if (!snapshotCameraKey) return skip('pointer-pose-unknown'); const samePose = () => snapshotCameraKey === this.readCameraKey() && this.now() - capturedAt <= POINTER_CROP_MAX_AGE_MS; - if (!samePose()) return false; + if (!samePose()) return skip('pose-changed'); this.lastPointerCrop = snapshot; const sent = await this.sendRetainedImage( () => this.capturePointer(snapshot.screenPx), "Crop of the map around the user's pointer; the ring marks the exact spot they mean by this or here. Describe only what is clearly visible.", 'client.pointer_crop', - { kind: 'crop', valid: samePose }, + { kind: 'crop', valid: samePose, diag }, ); if (!sent && this.lastPointerCrop === snapshot) this.lastPointerCrop = null; return sent; @@ -429,7 +476,14 @@ export class RealtimeViewport { /** Capture one image and make it the single retained image in context. */ async sendRetainedImage(captureImage, text, label, frame = {}) { - if (!this.dc || this.dc.readyState !== 'open') return false; + const kind = frame.kind || 'viewport'; + const diag = frame.diag || diagnosticsRecord(); + const report = (event, reason) => + reportOutcome(this, diag, event, kind, reason); + if (!this.dc || this.dc.readyState !== 'open') { + report('viewport_context.skipped', 'channel-closed'); + return false; + } const generation = this.generation; const request = ++this.captureSeq; const channel = this.dc; @@ -437,21 +491,23 @@ export class RealtimeViewport { const imageUrl = typeof captured === 'string' ? captured : captured?.dataUrl || null; if (!imageUrl) { - this.debugLog?.('viewport_context.skipped', { - reason: 'capture-unavailable', - }); + report('viewport_context.skipped', 'capture-unavailable'); return false; } - if ( - generation !== this.generation || - request !== this.captureSeq || - this.dc !== channel || - channel.readyState !== 'open' || - (typeof frame.valid === 'function' && !frame.valid()) - ) { - this.debugLog?.('viewport_context.skipped', { - reason: 'capture-invalidated', - }); + // Same guards as before, checked in order so each drop names its cause: + // reset() bumps both counters, so a session reset is tested first. + const invalidated = + generation !== this.generation + ? 'session-reset' + : request !== this.captureSeq + ? 'superseded' + : this.dc !== channel || channel.readyState !== 'open' + ? 'channel-lost' + : typeof frame.valid === 'function' && !frame.valid() + ? 'frame-stale' + : null; + if (invalidated) { + report('viewport_context.skipped', invalidated); return false; } @@ -509,14 +565,15 @@ export class RealtimeViewport { this.lastViewportItemId = sent ? newItemId : null; this.retainedFrame = sent ? { - kind: frame.kind || 'viewport', + kind, rect: captured?.rect || null, cameraKey: this.readCameraKey(), } : null; - this.debugLog?.('viewport_context.result', { - reason: sent ? 'sent' : 'data-channel-send-failed', - }); + report( + 'viewport_context.result', + sent ? 'sent' : 'data-channel-send-failed', + ); return sent; } diff --git a/src/voice/realtimeViewportPolicy.test.mjs b/src/voice/realtimeViewportPolicy.test.mjs index 0d679559f..57a60a13d 100644 --- a/src/voice/realtimeViewportPolicy.test.mjs +++ b/src/voice/realtimeViewportPolicy.test.mjs @@ -229,7 +229,7 @@ test('capture failures and invalidated turns are reason-coded without sending', diagnostics.some( (item) => item.event === 'viewport_context.skipped' && - item.detail.reason === 'capture-invalidated', + item.detail.reason === 'session-reset', ), ); }); @@ -256,3 +256,283 @@ test('successful retained send is logged without recording image contents', asyn ); assert.equal(JSON.stringify(diagnostics).includes('data:image/jpeg'), false); }); + +const outcomeEvents = new Set([ + 'viewport_context.skipped', + 'viewport_context.result', +]); + +function recordingViewport(options = {}) { + const events = []; + const channel = options.channel ?? { readyState: 'open' }; + const viewport = new RealtimeViewport({ + readChannel: () => channel, + readCameraKey: () => 'camera:fixture', + now: () => 1000, + capturePointer: async () => ({ + dataUrl: 'data:image/jpeg;base64,AA==', + rect: { x: 0, y: 0, width: 1, height: 1 }, + }), + capture: async () => 'data:image/jpeg;base64,AA==', + ...options, + operations: { + sendRealtimeEvent: () => true, + debugLog: (event, detail) => events.push({ event, detail }), + ...options.operations, + }, + }); + return { viewport, events }; +} + +const pointerGround = (overrides = {}) => + localContext({ + scope: 'pointer', + pointer: { target: 'ground' }, + scene: { basemap: { viewScale: 'city' } }, + ...overrides, + }); + +const groundSnapshot = (overrides = {}) => ({ + fresh: true, + target: 'ground', + screenPx: { x: 10, y: 10 }, + capturedAt: 900, + cameraKey: 'camera:fixture', + ...overrides, +}); + +test('tool results other than entity context log nothing', async () => { + const { viewport, events } = recordingViewport(); + assert.equal( + await viewport.sendVisualContextIfUseful({ action: 'fly_to' }), + false, + ); + assert.equal(await viewport.sendVisualContextIfUseful(null), false); + assert.deepEqual(events, []); +}); + +test('a crop decision that cannot send logs why, under the same id', async () => { + const cases = [ + [{ fresh: false }, 'pointer-not-fresh'], + [{ target: 'building' }, 'pointer-not-ground'], + [{ screenPx: null }, 'pointer-off-screen'], + [{ capturedAt: -50000 }, 'pointer-expired'], + [{ cameraKey: null }, 'pointer-pose-unknown'], + [{ cameraKey: 'camera:moved' }, 'pose-changed'], + ]; + for (const [override, reason] of cases) { + const snapshot = groundSnapshot(override); + const { viewport, events } = recordingViewport({ + readPointer: () => snapshot, + }); + assert.equal( + await viewport.sendVisualContextIfUseful(pointerGround()), + false, + reason, + ); + assert.equal(events.length, 2, reason); + const [decision, outcome] = events; + assert.equal(decision.event, 'viewport_context.decision'); + assert.equal(decision.detail.reason, 'pointer-ground-crop'); + assert.deepEqual(outcome, { + event: 'viewport_context.skipped', + detail: { contextId: decision.detail.contextId, kind: 'crop', reason }, + }); + } +}); + +test('a repeated pointer snapshot is logged as already requested', async () => { + const snapshot = groundSnapshot(); + const { viewport, events } = recordingViewport({ + readPointer: () => snapshot, + }); + assert.equal(await viewport.sendVisualContextIfUseful(pointerGround()), true); + assert.equal( + await viewport.sendVisualContextIfUseful(pointerGround()), + false, + ); + const outcomes = events.filter((item) => outcomeEvents.has(item.event)); + assert.deepEqual( + outcomes.map((item) => [item.detail.kind, item.detail.reason]), + [ + ['crop', 'sent'], + ['crop', 'pointer-already-requested'], + ], + ); +}); + +test('overlapping requests pair each outcome with its own decision', async () => { + const releases = []; + const { viewport, events } = recordingViewport({ + capture: () => new Promise((resolve) => releases.push(resolve)), + }); + const first = viewport.sendVisualContextIfUseful(localContext()); + const second = viewport.sendVisualContextIfUseful(localContext()); + await Promise.resolve(); + releases[1]('data:image/jpeg;base64,AA=='); + releases[0]('data:image/jpeg;base64,AA=='); + assert.equal(await first, false); + assert.equal(await second, true); + const [firstId, secondId] = events + .filter((item) => item.event === 'viewport_context.decision') + .map((item) => item.detail.contextId); + assert.notEqual(firstId, secondId); + const outcomeFor = (id) => + events.filter( + (item) => outcomeEvents.has(item.event) && item.detail.contextId === id, + ); + assert.deepEqual( + outcomeFor(firstId).map((item) => item.detail), + [{ contextId: firstId, kind: 'viewport', reason: 'superseded' }], + ); + assert.deepEqual( + outcomeFor(secondId).map((item) => item.detail), + [{ contextId: secondId, kind: 'viewport', reason: 'sent' }], + ); +}); + +test('dropped captures name their cause', async () => { + const reasonFor = async (context, interrupt, extra = {}) => { + let release; + const pending = () => new Promise((resolve) => (release = resolve)); + const { viewport, events } = recordingViewport({ + capture: pending, + capturePointer: pending, + ...extra, + }); + const sent = viewport.sendVisualContextIfUseful(context); + await Promise.resolve(); + interrupt(viewport); + release('data:image/jpeg;base64,AA=='); + assert.equal(await sent, false); + return events + .filter((item) => outcomeEvents.has(item.event)) + .map((item) => item.detail.reason); + }; + + assert.deepEqual( + await reasonFor(localContext(), (viewport) => viewport.beginTurn()), + ['superseded'], + ); + assert.deepEqual( + await reasonFor(localContext(), (viewport) => viewport.reset()), + ['session-reset'], + ); + const channel = { readyState: 'open' }; + assert.deepEqual( + await reasonFor(localContext(), () => (channel.readyState = 'closed'), { + channel, + }), + ['channel-lost'], + ); + let cameraKey = 'camera:fixture'; + const snapshot = groundSnapshot(); + assert.deepEqual( + await reasonFor(pointerGround(), () => (cameraKey = 'camera:moved'), { + readPointer: () => snapshot, + readCameraKey: () => cameraKey, + }), + ['frame-stale'], + ); +}); + +test('every send decision is followed by exactly one outcome', async () => { + const snapshots = [ + null, + groundSnapshot(), + groundSnapshot({ fresh: false }), + groundSnapshot({ cameraKey: 'camera:moved' }), + groundSnapshot({ capturedAt: -50000 }), + ]; + const contexts = [ + localContext(), + localContext({ selected: { id: 'x', layerId: 'y' } }), + pointerGround(), + pointerGround({ scene: { basemap: { viewScale: 'global' } } }), + pointerGround({ pointer: null }), + ]; + const captures = [ + async () => 'data:image/jpeg;base64,AA==', + async () => null, + ]; + for (const snapshot of snapshots) + for (const context of contexts) + for (const capture of captures) + for (const sendResult of [true, false]) { + const { viewport, events } = recordingViewport({ + readPointer: () => snapshot, + capture, + capturePointer: capture, + operations: { sendRealtimeEvent: () => sendResult }, + }); + await viewport.sendVisualContextIfUseful(context); + const [decision, ...rest] = events; + assert.equal(decision.event, 'viewport_context.decision'); + const outcomes = rest.filter((item) => outcomeEvents.has(item.event)); + assert.equal(outcomes.length, decision.detail.send ? 1 : 0); + for (const outcome of outcomes) { + assert.equal(outcome.detail.contextId, decision.detail.contextId); + assert.equal(outcome.detail.kind, decision.detail.kind); + if (outcome.event === 'viewport_context.result') + assert.equal( + outcome.detail.reason, + sendResult ? 'sent' : 'data-channel-send-failed', + ); + } + } +}); + +test('a failed data-channel send is never logged as sent', async () => { + for (const [sendResult, reason] of [ + [true, 'sent'], + [false, 'data-channel-send-failed'], + ]) { + const { viewport, events } = recordingViewport({ + operations: { sendRealtimeEvent: () => sendResult }, + }); + assert.equal( + await viewport.sendVisualContextIfUseful(localContext()), + sendResult, + ); + const outcomes = events.filter((item) => outcomeEvents.has(item.event)); + assert.deepEqual( + outcomes.map((item) => [item.event, item.detail.reason]), + [['viewport_context.result', reason]], + ); + } +}); + +test('a step that throws still closes its decision, then rethrows', async () => { + for (const [context, extra] of [ + [ + localContext(), + { + capture: async () => { + throw new Error('capture boom'); + }, + }, + ], + [ + pointerGround(), + { + readPointer: () => { + throw new Error('pointer boom'); + }, + }, + ], + ]) { + const { viewport, events } = recordingViewport(extra); + await assert.rejects(viewport.sendVisualContextIfUseful(context), /boom/); + const [decision, ...rest] = events; + assert.deepEqual(rest, [ + { + event: 'viewport_context.skipped', + detail: { + contextId: decision.detail.contextId, + kind: decision.detail.kind, + reason: 'error', + }, + }, + ]); + } +}); From d0a612786a17e50fdcbcb591ce3142827afabdf9 Mon Sep 17 00:00:00 2001 From: manjunath22466 Date: Fri, 6 Nov 2026 22:47:09 +0530 Subject: [PATCH 12/17] fix(voice): log a real send before bookkeeping; document decision reasons Second review pass on #835's diagnostics. Behaviour is unchanged. - Report viewport_context.result as soon as the send returns, so a throw in the retained-frame bookkeeping cannot log a real send as "error". - List the decision reasons and when each field is present in CURRENT-STATE, note that the rate limit can split a pair, and say which codes are defensive. - Changelog: the decision is logged when an entity-context answer reaches the image step, not for every answer. --- .../835-viewport-escalation-reasons.md | 12 +++++------ docs/CURRENT-STATE.md | 2 +- src/voice/realtimeViewport.js | 10 +++++---- src/voice/realtimeViewportPolicy.test.mjs | 21 +++++++++++++++++++ 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/changelog.d/835-viewport-escalation-reasons.md b/changelog.d/835-viewport-escalation-reasons.md index 0fe8a1fbd..cdd909bba 100644 --- a/changelog.d/835-viewport-escalation-reasons.md +++ b/changelog.d/835-viewport-escalation-reasons.md @@ -1,7 +1,7 @@ - The voice debug log now says why OpenAI Realtime did or did not get a - viewport screenshot or pointer crop. Each entity-context answer logs a - reason-coded decision and, when an image was due, one outcome under the same - id, so a crop that was dropped because the pointer went stale or the camera - moved is no longer counted as sent. What is captured, sent and kept is - unchanged, and no image data or user text is logged (kvnloo, manjunath22466, - #835). + viewport screenshot or pointer crop. When an entity-context answer reaches + the image step, the log records a reason-coded decision and, when an image + was due, one outcome under the same id, so a crop that was dropped because + the pointer went stale or the camera moved is no longer counted as sent. + What is captured, sent and kept is unchanged, and no image data or user text + is logged (kvnloo, manjunath22466, #835). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index b875296b5..76da8b2f0 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5779,7 +5779,7 @@ and unreachable upstream (502/504) separately from road geometry. - **Reliability**: tool-call dedupe (2.5s window across call/item/args keys); response-create queueing that respects active responses and defers follow-ups when the user starts speaking; per-tool follow-up instructions so the agent confirms only what actually happened (zoom confirms only on `ok=true`). - **Aircraft identity honesty:** “What is this aircraft?” reads callsign, operator, registration, type, and route only from the selected contact context. Missing operator, route, or type enrichment is named explicitly rather than silently omitted or inferred from the callsign. - **Diagnostics**: every client/server event is posted to `/api/realtime/debug-log` and appended to `.gev-logs/realtime-conversations.jsonl` (gitignored) with secret/image redaction; last 30 errors persist in `localStorage` (`gev-realtime-errors`); `window.__gevVoiceCommands.getDiagnostics()` in the console. - - **Visual-context escalation** (OpenAI Realtime): when a `get_entity_context` answer reaches the image step (its tool output was sent and the data channel is open), it logs one `viewport_context.decision` (`contextId`, `send`, `kind` `viewport` or `crop`, `reason`, `viewScale`). A send decision is followed by exactly one outcome with the same `contextId` and `kind`: `viewport_context.result` (`sent`, `data-channel-send-failed`) or `viewport_context.skipped`. Pointer-crop skips are `pointer-not-fresh`, `pointer-not-ground`, `pointer-off-screen`, `pointer-already-requested` (the same snapshot was already sent or is still being captured), `pointer-expired`, `pointer-pose-unknown` and `pose-changed`; dropped captures are `capture-unavailable`, `session-reset`, `superseded` (a newer request or turn), `channel-lost` and `frame-stale`; a step that throws closes the pair with `error` before the caller logs `viewport_context.failed`. `channel-closed` and the crop's `pointer-scale-not-useful` are defensive and not expected from the current caller. Payloads are static codes only, never image data, tool payloads or user text, and other tool results log nothing here. Gemini Live uses a different gate in `geminiSession.js` (no pointer crop, a 1 s throttle), which is not instrumented. + - **Visual-context escalation** (OpenAI Realtime): when a response's last tool result is a `get_entity_context` answer and its tool output was sent with the data channel open, it logs one `viewport_context.decision` with `contextId`, `send` and `reason`, plus `viewScale` (absent only for `channel-closed`) and, when `send` is true, `kind` (`viewport` or `crop`). Decision reasons: `structured-identity-sufficient` (structured context made an image unnecessary), `view-scale-not-local`, `structured-identity-missing` (sends a viewport), `pointer-ground-crop` (sends a crop), `pointer-identity-present`, `pointer-missing`, `pointer-scale-not-useful` and `channel-closed`. A send decision is followed by exactly one outcome with the same `contextId` and `kind`: `viewport_context.result` (`sent`, `data-channel-send-failed`), logged as soon as the send returns, or `viewport_context.skipped`. Pointer-crop skips are `pointer-not-fresh`, `pointer-not-ground`, `pointer-off-screen`, `pointer-already-requested` (the same snapshot was already sent, is still being captured, or an earlier capture of it failed), `pointer-expired`, `pointer-pose-unknown` and `pose-changed`; dropped captures are `capture-unavailable`, `session-reset`, `superseded` (a newer request or turn), `channel-lost` and `frame-stale`; a step that throws before an outcome closes the pair with `error`, and the caller then logs `viewport_context.failed`. The `channel-closed` decision and the `channel-closed` and `pointer-scale-not-useful` skip outcomes are defensive and not expected from the current caller. Decision and outcome are separate records, so the debug-log rate limit can drop one half of a pair. Payloads are static codes only, never image data, tool payloads or user text, and other tool results log nothing here. Gemini Live uses a different gate in `geminiSession.js` (no pointer crop, a 1 s throttle), which is not instrumented. - **Counting semantics ("near")** — a CONTRACT; new count-bearing tool work inherits it. Three honest numbers exist for one question: the Contacts cohort (250 km around the subject, what the panel shows), `analyst_query`'s count of _currently-loaded_ records for the requested scope, and the layer-wide loaded total in `coverage.layersQueried[].total`. They diverge legitimately — the flights layer loads by viewport, so after a camera dive the loaded set can hold a fraction of the cohort (field case: panel 42, analyst 8). The contract: 1. **Contacts ACTIVE** → "near / nearby / how many aircraft" means the **Contacts window** — the panel's numbers, spoken verbatim. Mechanism: `contactsWindow` (`{centeredOn, radiusKm, flights, military, vessels, complete}`), carried by both `analyst_query` and `get_current_view_state`, derived by `contactsWindowFromSnapshot()` from the same snapshot the panel renders so the two cannot drift. Voice filtering and follow-up memory use that immutable snapshot too. A cohort whose feed cannot answer reports `'unknown'`, never a confident zero; a cohort at the 20,000-row retention bound reports a lower bound. 2. **Contacts OFF** → "nearby" means **in view**; "near \" means a radius around that place. A radius query with Contacts active and no explicit centre is centred on the **active contact**, not the camera. diff --git a/src/voice/realtimeViewport.js b/src/voice/realtimeViewport.js index 89b9051f7..dc845da31 100644 --- a/src/voice/realtimeViewport.js +++ b/src/voice/realtimeViewport.js @@ -562,6 +562,12 @@ export class RealtimeViewport { // the item we just deleted as null and fall through so the caller still // issues queueResponseCreate WITHOUT the image, instead of stranding the turn. const sent = this.sendRealtimeEvent(contextEvent, label); + // Report before the bookkeeping below, so a later throw cannot turn a + // real send into a logged "error". + report( + 'viewport_context.result', + sent ? 'sent' : 'data-channel-send-failed', + ); this.lastViewportItemId = sent ? newItemId : null; this.retainedFrame = sent ? { @@ -570,10 +576,6 @@ export class RealtimeViewport { cameraKey: this.readCameraKey(), } : null; - report( - 'viewport_context.result', - sent ? 'sent' : 'data-channel-send-failed', - ); return sent; } diff --git a/src/voice/realtimeViewportPolicy.test.mjs b/src/voice/realtimeViewportPolicy.test.mjs index 57a60a13d..9c7ddfb74 100644 --- a/src/voice/realtimeViewportPolicy.test.mjs +++ b/src/voice/realtimeViewportPolicy.test.mjs @@ -536,3 +536,24 @@ test('a step that throws still closes its decision, then rethrows', async () => ]); } }); + +test('a throw after a real send still logs it as sent', async () => { + let reads = 0; + const { viewport, events } = recordingViewport({ + // The first read happens only after the image was handed to the channel. + readCameraKey: () => { + reads += 1; + throw new Error('camera gone'); + }, + }); + await assert.rejects( + viewport.sendVisualContextIfUseful(localContext()), + /camera gone/, + ); + assert.equal(reads, 1); + const outcomes = events.filter((item) => outcomeEvents.has(item.event)); + assert.deepEqual( + outcomes.map((item) => [item.event, item.detail.reason]), + [['viewport_context.result', 'sent']], + ); +}); From df37c9360dcfc54ea80915e603e1955a67fc68bb Mon Sep 17 00:00:00 2001 From: manjunath22466 Date: Fri, 6 Nov 2026 23:12:20 +0530 Subject: [PATCH 13/17] test(voice): pin diagnostics wiring and in-flight crop dedupe; tighten docs Third review pass on #835. No runtime change. - Controller test: viewport decisions reach the controller's debug log. - Policy test: a second request for a snapshot still being captured is logged pointer-already-requested and is not captured twice. - CURRENT-STATE: pointer-not-fresh also covers a missing snapshot, already-requested covers an earlier attempt that threw, and the image step runs when any tool output of the response was sent. - Changelog: describe what is recorded rather than a fixed miscount. --- .../835-viewport-escalation-reasons.md | 8 +++--- docs/CURRENT-STATE.md | 2 +- src/voice/gevRealtime.test.mjs | 26 +++++++++++++++++++ src/voice/realtimeViewportPolicy.test.mjs | 24 +++++++++++++++++ 4 files changed, 55 insertions(+), 5 deletions(-) diff --git a/changelog.d/835-viewport-escalation-reasons.md b/changelog.d/835-viewport-escalation-reasons.md index cdd909bba..b10444e8e 100644 --- a/changelog.d/835-viewport-escalation-reasons.md +++ b/changelog.d/835-viewport-escalation-reasons.md @@ -1,7 +1,7 @@ - The voice debug log now says why OpenAI Realtime did or did not get a viewport screenshot or pointer crop. When an entity-context answer reaches the image step, the log records a reason-coded decision and, when an image - was due, one outcome under the same id, so a crop that was dropped because - the pointer went stale or the camera moved is no longer counted as sent. - What is captured, sent and kept is unchanged, and no image data or user text - is logged (kvnloo, manjunath22466, #835). + was due, one outcome under the same id: sent, or why it was dropped (a stale + pointer, a camera move, a newer turn). What is captured, sent and kept is + unchanged, and no image data or user text is logged (kvnloo, manjunath22466, + #835). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 76da8b2f0..48a113873 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5779,7 +5779,7 @@ and unreachable upstream (502/504) separately from road geometry. - **Reliability**: tool-call dedupe (2.5s window across call/item/args keys); response-create queueing that respects active responses and defers follow-ups when the user starts speaking; per-tool follow-up instructions so the agent confirms only what actually happened (zoom confirms only on `ok=true`). - **Aircraft identity honesty:** “What is this aircraft?” reads callsign, operator, registration, type, and route only from the selected contact context. Missing operator, route, or type enrichment is named explicitly rather than silently omitted or inferred from the callsign. - **Diagnostics**: every client/server event is posted to `/api/realtime/debug-log` and appended to `.gev-logs/realtime-conversations.jsonl` (gitignored) with secret/image redaction; last 30 errors persist in `localStorage` (`gev-realtime-errors`); `window.__gevVoiceCommands.getDiagnostics()` in the console. - - **Visual-context escalation** (OpenAI Realtime): when a response's last tool result is a `get_entity_context` answer and its tool output was sent with the data channel open, it logs one `viewport_context.decision` with `contextId`, `send` and `reason`, plus `viewScale` (absent only for `channel-closed`) and, when `send` is true, `kind` (`viewport` or `crop`). Decision reasons: `structured-identity-sufficient` (structured context made an image unnecessary), `view-scale-not-local`, `structured-identity-missing` (sends a viewport), `pointer-ground-crop` (sends a crop), `pointer-identity-present`, `pointer-missing`, `pointer-scale-not-useful` and `channel-closed`. A send decision is followed by exactly one outcome with the same `contextId` and `kind`: `viewport_context.result` (`sent`, `data-channel-send-failed`), logged as soon as the send returns, or `viewport_context.skipped`. Pointer-crop skips are `pointer-not-fresh`, `pointer-not-ground`, `pointer-off-screen`, `pointer-already-requested` (the same snapshot was already sent, is still being captured, or an earlier capture of it failed), `pointer-expired`, `pointer-pose-unknown` and `pose-changed`; dropped captures are `capture-unavailable`, `session-reset`, `superseded` (a newer request or turn), `channel-lost` and `frame-stale`; a step that throws before an outcome closes the pair with `error`, and the caller then logs `viewport_context.failed`. The `channel-closed` decision and the `channel-closed` and `pointer-scale-not-useful` skip outcomes are defensive and not expected from the current caller. Decision and outcome are separate records, so the debug-log rate limit can drop one half of a pair. Payloads are static codes only, never image data, tool payloads or user text, and other tool results log nothing here. Gemini Live uses a different gate in `geminiSession.js` (no pointer crop, a 1 s throttle), which is not instrumented. + - **Visual-context escalation** (OpenAI Realtime): when a response's last tool result is a `get_entity_context` answer, a tool output from that response was sent and the data channel is open, it logs one `viewport_context.decision` with `contextId`, `send` and `reason`, plus `viewScale` (absent only for `channel-closed`) and, when `send` is true, `kind` (`viewport` or `crop`). Decision reasons: `structured-identity-sufficient` (structured context made an image unnecessary), `view-scale-not-local`, `structured-identity-missing` (sends a viewport), `pointer-ground-crop` (sends a crop), `pointer-identity-present`, `pointer-missing`, `pointer-scale-not-useful` and `channel-closed`. A send decision is followed by exactly one outcome with the same `contextId` and `kind`: `viewport_context.result` (`sent`, `data-channel-send-failed`), logged as soon as the send returns, or `viewport_context.skipped`. Pointer-crop skips are `pointer-not-fresh` (no snapshot, or a stale one), `pointer-not-ground`, `pointer-off-screen`, `pointer-already-requested` (the same snapshot was already sent, is still being captured, or an earlier attempt for it threw), `pointer-expired`, `pointer-pose-unknown` and `pose-changed`; dropped captures are `capture-unavailable`, `session-reset`, `superseded` (a newer request or turn), `channel-lost` and `frame-stale`; a step that throws before an outcome closes the pair with `error`, and the caller then logs `viewport_context.failed`. The `channel-closed` decision and the `channel-closed` and `pointer-scale-not-useful` skip outcomes are defensive and not expected from the current caller. Decision and outcome are separate records, so the debug-log rate limit can drop one half of a pair. Payloads are static codes only, never image data, tool payloads or user text, and other tool results log nothing here. Gemini Live uses a different gate in `geminiSession.js` (no pointer crop, a 1 s throttle), which is not instrumented. - **Counting semantics ("near")** — a CONTRACT; new count-bearing tool work inherits it. Three honest numbers exist for one question: the Contacts cohort (250 km around the subject, what the panel shows), `analyst_query`'s count of _currently-loaded_ records for the requested scope, and the layer-wide loaded total in `coverage.layersQueried[].total`. They diverge legitimately — the flights layer loads by viewport, so after a camera dive the loaded set can hold a fraction of the cohort (field case: panel 42, analyst 8). The contract: 1. **Contacts ACTIVE** → "near / nearby / how many aircraft" means the **Contacts window** — the panel's numbers, spoken verbatim. Mechanism: `contactsWindow` (`{centeredOn, radiusKm, flights, military, vessels, complete}`), carried by both `analyst_query` and `get_current_view_state`, derived by `contactsWindowFromSnapshot()` from the same snapshot the panel renders so the two cannot drift. Voice filtering and follow-up memory use that immutable snapshot too. A cohort whose feed cannot answer reports `'unknown'`, never a confident zero; a cohort at the 20,000-row retention bound reports a lower bound. 2. **Contacts OFF** → "nearby" means **in view**; "near \" means a radius around that place. A radius query with Contacts active and no explicit centre is centred on the **active contact**, not the camera. diff --git a/src/voice/gevRealtime.test.mjs b/src/voice/gevRealtime.test.mjs index 954f4f993..859932e08 100644 --- a/src/voice/gevRealtime.test.mjs +++ b/src/voice/gevRealtime.test.mjs @@ -3708,3 +3708,29 @@ test('voice runs app actions itself and other tools through the catalog', async assert.deepEqual(disposals, [['removed']]); assert.equal(withToolCatalog(runner, undefined), runner); }); + +test('the controller forwards viewport escalation diagnostics to its debug log', async () => { + const records = []; + const controller = new GevRealtimeController({ + ui: createTestVoiceUi(), + runner: async () => ({ ok: true }), + radioLayer: { subscribePlaybackControls: () => () => {}, setVoiceDucked() {} }, + }); + controller.debugLog = (event, detail) => records.push({ event, detail }); + controller.dc = { readyState: 'open', send() {}, close() {} }; + + assert.equal( + await controller.sendVisualContextIfUseful({ + action: 'get_entity_context', + scene: { basemap: { viewScale: 'regional' } }, + }), + false, + ); + assert.deepEqual( + records.filter((record) => record.event.startsWith('viewport_context.')), + [{ + event: 'viewport_context.decision', + detail: { contextId: 1, send: false, reason: 'view-scale-not-local', viewScale: 'regional' }, + }], + ); +}); diff --git a/src/voice/realtimeViewportPolicy.test.mjs b/src/voice/realtimeViewportPolicy.test.mjs index 9c7ddfb74..4ed1c0531 100644 --- a/src/voice/realtimeViewportPolicy.test.mjs +++ b/src/voice/realtimeViewportPolicy.test.mjs @@ -557,3 +557,27 @@ test('a throw after a real send still logs it as sent', async () => { [['viewport_context.result', 'sent']], ); }); + +test('a second request for a snapshot still being captured is not sent twice', async () => { + const releases = []; + const snapshot = groundSnapshot(); + const { viewport, events } = recordingViewport({ + readPointer: () => snapshot, + capturePointer: () => new Promise((resolve) => releases.push(resolve)), + }); + const first = viewport.sendVisualContextIfUseful(pointerGround()); + const second = viewport.sendVisualContextIfUseful(pointerGround()); + await Promise.resolve(); + for (const release of releases) + release({ dataUrl: 'data:image/jpeg;base64,AA==', rect: {} }); + assert.deepEqual([await first, await second], [true, false]); + assert.equal(releases.length, 1); + const outcomes = events.filter((item) => outcomeEvents.has(item.event)); + assert.deepEqual( + outcomes.map((item) => [item.detail.contextId, item.detail.reason]), + [ + [2, 'pointer-already-requested'], + [1, 'sent'], + ], + ); +}); From 4384e0d402b122006d4635700acb2e4fd86067e8 Mon Sep 17 00:00:00 2001 From: manjunath22466 Date: Sat, 7 Nov 2026 00:39:26 +0530 Subject: [PATCH 14/17] test(voice): pin crop-capture throws and defensive skips; tighten reason glosses A throw from the crop capture now has a test that it closes its decision with an error outcome. Direct calls to the crop and image senders pin the channel-closed and pointer-scale-not-useful skips. The docs say a send decision requests an image, and when a pointer snapshot counts as already requested. --- docs/CURRENT-STATE.md | 2 +- src/voice/realtimeViewportPolicy.test.mjs | 42 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 48a113873..39230c3e5 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -5779,7 +5779,7 @@ and unreachable upstream (502/504) separately from road geometry. - **Reliability**: tool-call dedupe (2.5s window across call/item/args keys); response-create queueing that respects active responses and defers follow-ups when the user starts speaking; per-tool follow-up instructions so the agent confirms only what actually happened (zoom confirms only on `ok=true`). - **Aircraft identity honesty:** “What is this aircraft?” reads callsign, operator, registration, type, and route only from the selected contact context. Missing operator, route, or type enrichment is named explicitly rather than silently omitted or inferred from the callsign. - **Diagnostics**: every client/server event is posted to `/api/realtime/debug-log` and appended to `.gev-logs/realtime-conversations.jsonl` (gitignored) with secret/image redaction; last 30 errors persist in `localStorage` (`gev-realtime-errors`); `window.__gevVoiceCommands.getDiagnostics()` in the console. - - **Visual-context escalation** (OpenAI Realtime): when a response's last tool result is a `get_entity_context` answer, a tool output from that response was sent and the data channel is open, it logs one `viewport_context.decision` with `contextId`, `send` and `reason`, plus `viewScale` (absent only for `channel-closed`) and, when `send` is true, `kind` (`viewport` or `crop`). Decision reasons: `structured-identity-sufficient` (structured context made an image unnecessary), `view-scale-not-local`, `structured-identity-missing` (sends a viewport), `pointer-ground-crop` (sends a crop), `pointer-identity-present`, `pointer-missing`, `pointer-scale-not-useful` and `channel-closed`. A send decision is followed by exactly one outcome with the same `contextId` and `kind`: `viewport_context.result` (`sent`, `data-channel-send-failed`), logged as soon as the send returns, or `viewport_context.skipped`. Pointer-crop skips are `pointer-not-fresh` (no snapshot, or a stale one), `pointer-not-ground`, `pointer-off-screen`, `pointer-already-requested` (the same snapshot was already sent, is still being captured, or an earlier attempt for it threw), `pointer-expired`, `pointer-pose-unknown` and `pose-changed`; dropped captures are `capture-unavailable`, `session-reset`, `superseded` (a newer request or turn), `channel-lost` and `frame-stale`; a step that throws before an outcome closes the pair with `error`, and the caller then logs `viewport_context.failed`. The `channel-closed` decision and the `channel-closed` and `pointer-scale-not-useful` skip outcomes are defensive and not expected from the current caller. Decision and outcome are separate records, so the debug-log rate limit can drop one half of a pair. Payloads are static codes only, never image data, tool payloads or user text, and other tool results log nothing here. Gemini Live uses a different gate in `geminiSession.js` (no pointer crop, a 1 s throttle), which is not instrumented. + - **Visual-context escalation** (OpenAI Realtime): when a response's last tool result is a `get_entity_context` answer, a tool output from that response was sent and the data channel is open, it logs one `viewport_context.decision` with `contextId`, `send` and `reason`, plus `viewScale` (absent only for `channel-closed`) and, when `send` is true, `kind` (`viewport` or `crop`). Decision reasons: `structured-identity-sufficient` (structured context made an image unnecessary), `view-scale-not-local`, `structured-identity-missing` (requests a viewport), `pointer-ground-crop` (a pointer answer with no identity at `local` or `city` scale; requests a crop, and the snapshot checks below confirm the target is ground), `pointer-identity-present`, `pointer-missing`, `pointer-scale-not-useful` and `channel-closed`. A send decision is followed by exactly one outcome with the same `contextId` and `kind`: `viewport_context.result` (`sent`, `data-channel-send-failed`), logged as soon as the send returns, or `viewport_context.skipped`. Pointer-crop skips are `pointer-not-fresh` (no snapshot, or a stale one), `pointer-not-ground`, `pointer-off-screen`, `pointer-already-requested` (the most recently requested snapshot again: already sent, still being captured, or an earlier attempt for it threw), `pointer-expired`, `pointer-pose-unknown` and `pose-changed`; dropped captures are `capture-unavailable`, `session-reset`, `superseded` (a newer request or turn), `channel-lost` and `frame-stale`; a step that throws before an outcome closes the pair with `error`, and the caller then logs `viewport_context.failed`. The `channel-closed` decision and the `channel-closed` and `pointer-scale-not-useful` skip outcomes are defensive and not expected from the current caller; direct calls to the crop or image senders log those skips with a null `contextId`. Decision and outcome are separate records, so the debug-log rate limit can drop one half of a pair. Payloads are static codes only, never image data, tool payloads or user text, and other tool results log nothing here. Gemini Live uses a different gate in `geminiSession.js` (no pointer crop, a 1 s throttle), which is not instrumented. - **Counting semantics ("near")** — a CONTRACT; new count-bearing tool work inherits it. Three honest numbers exist for one question: the Contacts cohort (250 km around the subject, what the panel shows), `analyst_query`'s count of _currently-loaded_ records for the requested scope, and the layer-wide loaded total in `coverage.layersQueried[].total`. They diverge legitimately — the flights layer loads by viewport, so after a camera dive the loaded set can hold a fraction of the cohort (field case: panel 42, analyst 8). The contract: 1. **Contacts ACTIVE** → "near / nearby / how many aircraft" means the **Contacts window** — the panel's numbers, spoken verbatim. Mechanism: `contactsWindow` (`{centeredOn, radiusKm, flights, military, vessels, complete}`), carried by both `analyst_query` and `get_current_view_state`, derived by `contactsWindowFromSnapshot()` from the same snapshot the panel renders so the two cannot drift. Voice filtering and follow-up memory use that immutable snapshot too. A cohort whose feed cannot answer reports `'unknown'`, never a confident zero; a cohort at the 20,000-row retention bound reports a lower bound. 2. **Contacts OFF** → "nearby" means **in view**; "near \" means a radius around that place. A radius query with Contacts active and no explicit centre is centred on the **active contact**, not the camera. diff --git a/src/voice/realtimeViewportPolicy.test.mjs b/src/voice/realtimeViewportPolicy.test.mjs index 4ed1c0531..2a90b3795 100644 --- a/src/voice/realtimeViewportPolicy.test.mjs +++ b/src/voice/realtimeViewportPolicy.test.mjs @@ -520,6 +520,15 @@ test('a step that throws still closes its decision, then rethrows', async () => }, }, ], + [ + pointerGround(), + { + readPointer: () => groundSnapshot(), + capturePointer: async () => { + throw new Error('crop boom'); + }, + }, + ], ]) { const { viewport, events } = recordingViewport(extra); await assert.rejects(viewport.sendVisualContextIfUseful(context), /boom/); @@ -537,6 +546,39 @@ test('a step that throws still closes its decision, then rethrows', async () => } }); +test('direct calls log their defensive skips without a decision id', async () => { + const open = recordingViewport(); + assert.equal( + await open.viewport.sendPointerCrop(groundSnapshot(), { + viewScale: 'global', + }), + false, + ); + const closed = recordingViewport({ channel: { readyState: 'closed' } }); + assert.equal( + await closed.viewport.sendPointerCrop(groundSnapshot(), { + viewScale: 'city', + }), + false, + ); + assert.equal( + await closed.viewport.sendRetainedImage( + async () => 'data:image/jpeg;base64,AA==', + 'text', + 'client.viewport_context', + ), + false, + ); + assert.deepEqual( + [...open.events, ...closed.events].map((item) => item.detail), + [ + { contextId: null, kind: 'crop', reason: 'pointer-scale-not-useful' }, + { contextId: null, kind: 'crop', reason: 'channel-closed' }, + { contextId: null, kind: 'viewport', reason: 'channel-closed' }, + ], + ); +}); + test('a throw after a real send still logs it as sent', async () => { let reads = 0; const { viewport, events } = recordingViewport({ From fd2200effaa853508c36edb93fbf989f445190d8 Mon Sep 17 00:00:00 2001 From: manjunath22466 Date: Tue, 10 Nov 2026 12:36:22 +0530 Subject: [PATCH 15/17] docs: credit viewport diagnostics contributors (#835) --- changelog.d/835-viewport-escalation-reasons.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/835-viewport-escalation-reasons.md b/changelog.d/835-viewport-escalation-reasons.md index b10444e8e..ed2c7b6b9 100644 --- a/changelog.d/835-viewport-escalation-reasons.md +++ b/changelog.d/835-viewport-escalation-reasons.md @@ -3,5 +3,5 @@ the image step, the log records a reason-coded decision and, when an image was due, one outcome under the same id: sent, or why it was dropped (a stale pointer, a camera move, a newer turn). What is captured, sent and kept is - unchanged, and no image data or user text is logged (kvnloo, manjunath22466, + unchanged, and no image data or user text is logged (Kevin Rajan, Manjunath, #835). From e7325685e55268cf8ceab8d2326641e3c6e326cc Mon Sep 17 00:00:00 2001 From: manjunath22466 Date: Tue, 10 Nov 2026 13:30:41 +0530 Subject: [PATCH 16/17] docs: restore GitHub handles in the #835 changelog credit Recent fragments credit as name, handle and PR number; keep that form. --- changelog.d/835-viewport-escalation-reasons.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/835-viewport-escalation-reasons.md b/changelog.d/835-viewport-escalation-reasons.md index ed2c7b6b9..b533533eb 100644 --- a/changelog.d/835-viewport-escalation-reasons.md +++ b/changelog.d/835-viewport-escalation-reasons.md @@ -3,5 +3,5 @@ the image step, the log records a reason-coded decision and, when an image was due, one outcome under the same id: sent, or why it was dropped (a stale pointer, a camera move, a newer turn). What is captured, sent and kept is - unchanged, and no image data or user text is logged (Kevin Rajan, Manjunath, - #835). + unchanged, and no image data or user text is logged (Kevin Rajan, @kvnloo, + Manjunath, @manjunath22466, #835). From 9234c1b3d2d95b814059d814c36be54bf3acb1a5 Mon Sep 17 00:00:00 2001 From: Sameh Khamis Date: Tue, 10 Nov 2026 15:44:17 -0800 Subject: [PATCH 17/17] fix(voice): drop viewport decision codes the caller can never log, name the crop decision eligible (#835) The decision no longer reports not-entity-context or channel-closed, which its only caller rules out first, nor pointer-scale-not-useful, which the crop path already checks and logs. A closed channel returns before any record, as on main. The crop decision is pointer-crop-eligible, since the crop checks after it decide whether the pointer is on ground, and a sender called on a closed channel logs channel-lost. --- docs/CURRENT-STATE.md | 2 +- src/voice/realtimeViewport.js | 35 ++++++++++++----------- src/voice/realtimeViewportPolicy.test.mjs | 26 ++++++----------- 3 files changed, 27 insertions(+), 36 deletions(-) diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index 5c6e847fe..291d365d2 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -6147,7 +6147,7 @@ and unreachable upstream (502/504) separately from road geometry. - **Reliability**: tool-call dedupe (2.5s window across call/item/args keys); response-create queueing that respects active responses and defers follow-ups when the user starts speaking; per-tool follow-up instructions so the agent confirms only what actually happened (zoom confirms only on `ok=true`). - **Aircraft identity honesty:** “What is this aircraft?” reads callsign, operator, registration, type, and route only from the selected contact context. Missing operator, route, or type enrichment is named explicitly rather than silently omitted or inferred from the callsign. - **Diagnostics**: every client/server event is posted to `/api/realtime/debug-log` and appended to `.gev-logs/realtime-conversations.jsonl` (gitignored) with secret/image redaction; last 30 errors persist in `localStorage` (`gev-realtime-errors`); `window.__gevVoiceCommands.getDiagnostics()` in the console. - - **Visual-context escalation** (OpenAI Realtime): when a response's last tool result is a `get_entity_context` answer, a tool output from that response was sent and the data channel is open, it logs one `viewport_context.decision` with `contextId`, `send` and `reason`, plus `viewScale` (absent only for `channel-closed`) and, when `send` is true, `kind` (`viewport` or `crop`). Decision reasons: `structured-identity-sufficient` (structured context made an image unnecessary), `view-scale-not-local`, `structured-identity-missing` (requests a viewport), `pointer-ground-crop` (a pointer answer with no identity at `local` or `city` scale; requests a crop, and the snapshot checks below confirm the target is ground), `pointer-identity-present`, `pointer-missing`, `pointer-scale-not-useful` and `channel-closed`. A send decision is followed by exactly one outcome with the same `contextId` and `kind`: `viewport_context.result` (`sent`, `data-channel-send-failed`), logged as soon as the send returns, or `viewport_context.skipped`. Pointer-crop skips are `pointer-not-fresh` (no snapshot, or a stale one), `pointer-not-ground`, `pointer-off-screen`, `pointer-already-requested` (the most recently requested snapshot again: already sent, still being captured, or an earlier attempt for it threw), `pointer-expired`, `pointer-pose-unknown` and `pose-changed`; dropped captures are `capture-unavailable`, `session-reset`, `superseded` (a newer request or turn), `channel-lost` and `frame-stale`; a step that throws before an outcome closes the pair with `error`, and the caller then logs `viewport_context.failed`. The `channel-closed` decision and the `channel-closed` and `pointer-scale-not-useful` skip outcomes are defensive and not expected from the current caller; direct calls to the crop or image senders log those skips with a null `contextId`. Decision and outcome are separate records, so the debug-log rate limit can drop one half of a pair. Payloads are static codes only, never image data, tool payloads or user text, and other tool results log nothing here. Gemini Live uses a different gate in `geminiSession.js` (no pointer crop, a 1 s throttle), which is not instrumented. + - **Visual-context escalation** (OpenAI Realtime): when a response's last tool result is a `get_entity_context` answer, a tool output from that response was sent and the data channel is open, it logs one `viewport_context.decision` with `contextId`, `send` and `reason`, plus `viewScale` and, when `send` is true, `kind` (`viewport` or `crop`). Decision reasons: `structured-identity-sufficient` (structured context made an image unnecessary), `view-scale-not-local`, `structured-identity-missing` (requests a viewport), `pointer-crop-eligible` (a pointer answer with no identity; requests a crop, and the crop checks below decide whether one is sent), `pointer-identity-present` and `pointer-missing`. A send decision is followed by exactly one outcome with the same `contextId` and `kind`: `viewport_context.result` (`sent`, `data-channel-send-failed`), logged as soon as the send returns, or `viewport_context.skipped`. Pointer-crop skips are `pointer-not-fresh` (no snapshot, or a stale one), `pointer-not-ground`, `pointer-off-screen`, `pointer-scale-not-useful` (zoomed out past `local` or `city`), `pointer-already-requested` (the most recently requested snapshot again: already sent, still being captured, or an earlier attempt for it threw), `pointer-expired`, `pointer-pose-unknown` and `pose-changed`; dropped captures are `capture-unavailable`, `session-reset`, `superseded` (a newer request or turn), `channel-lost` and `frame-stale`; a step that throws before an outcome closes the pair with `error`, and the caller then logs `viewport_context.failed`. Direct calls to the crop or image senders log their skips with a null `contextId`. Decision and outcome are separate records, so the debug-log rate limit can drop one half of a pair. Payloads are static codes only, never image data, tool payloads or user text, and other tool results log nothing here. Gemini Live uses a different gate in `geminiSession.js` (no pointer crop, a 1 s throttle), which is not instrumented. - **Counting semantics ("near")** — a CONTRACT; new count-bearing tool work inherits it. Three honest numbers exist for one question: the Contacts cohort (250 km around the subject, what the panel shows), `analyst_query`'s count of _currently-loaded_ records for the requested scope, and the layer-wide loaded total in `coverage.layersQueried[].total`. They diverge legitimately — the flights layer loads by viewport, so after a camera dive the loaded set can hold a fraction of the cohort (field case: panel 42, analyst 8). The contract: 1. **Contacts ACTIVE** → "near / nearby / how many aircraft" means the **Contacts window** — the panel's numbers, spoken verbatim. Mechanism: `contactsWindow` (`{centeredOn, radiusKm, flights, military, vessels, complete}`), carried by both `analyst_query` and `get_current_view_state`, derived by `contactsWindowFromSnapshot()` from the same snapshot the panel renders so the two cannot drift. Voice filtering and follow-up memory use that immutable snapshot too. A cohort whose feed cannot answer reports `'unknown'`, never a confident zero; a cohort at the 20,000-row retention bound reports a lower bound. 2. **Contacts OFF** → "nearby" means **in view**; "near \" means a radius around that place. A radius query with Contacts active and no explicit centre is centred on the **active contact**, not the camera. diff --git a/src/voice/realtimeViewport.js b/src/voice/realtimeViewport.js index dc845da31..edfe9ab11 100644 --- a/src/voice/realtimeViewport.js +++ b/src/voice/realtimeViewport.js @@ -73,26 +73,23 @@ const POINTER_CROP_MAX_AGE_MS = 20000; const POINTER_CROP_SCALES = new Set(['local', 'city']); /** - * Reason-coded explanation of the existing visual escalation policy. + * Reason-coded explanation of the existing visual escalation policy, for an + * entity-context answer on an open channel (the caller checks both). * Pointer crops remain a separate path from full-frame screenshots. * These reasons are static diagnostics, never model-generated claims. */ -export function viewportContextDecision(result, { channelOpen = true } = {}) { - if (result?.action !== 'get_entity_context') - return { send: false, reason: 'not-entity-context' }; - if (!channelOpen) return { send: false, reason: 'channel-closed' }; - const viewScale = result.scene?.basemap?.viewScale ?? null; - if (result.scope === 'pointer') { +export function viewportContextDecision(result) { + const viewScale = result?.scene?.basemap?.viewScale ?? null; + if (result?.scope === 'pointer') { if (result.selected) return { send: false, reason: 'pointer-identity-present', viewScale }; if (!result.pointer) return { send: false, reason: 'pointer-missing', viewScale }; - if (!POINTER_CROP_SCALES.has(viewScale)) - return { send: false, reason: 'pointer-scale-not-useful', viewScale }; + // The crop path checks ground, freshness and scale itself, and logs why. return { send: true, kind: 'crop', - reason: 'pointer-ground-crop', + reason: 'pointer-crop-eligible', viewScale, }; } @@ -390,13 +387,17 @@ export class RealtimeViewport { * requests pair up. */ async sendVisualContextIfUseful(result) { - // Runs after every tool output. Only entity context can escalate, so - // other tools log nothing and leave the shared debug-log budget alone. - if (result?.action !== 'get_entity_context') return false; + // Runs after every tool output. Only entity context on an open channel + // can escalate, so anything else logs nothing and leaves the shared + // debug-log budget alone. + if ( + result?.action !== 'get_entity_context' || + !this.dc || + this.dc.readyState !== 'open' + ) + return false; const contextId = ++this.contextSeq; - const decision = viewportContextDecision(result, { - channelOpen: this.dc?.readyState === 'open', - }); + const decision = viewportContextDecision(result); this.debugLog?.('viewport_context.decision', { contextId, ...decision }); if (!decision.send) return false; const diag = diagnosticsRecord(contextId); @@ -481,7 +482,7 @@ export class RealtimeViewport { const report = (event, reason) => reportOutcome(this, diag, event, kind, reason); if (!this.dc || this.dc.readyState !== 'open') { - report('viewport_context.skipped', 'channel-closed'); + report('viewport_context.skipped', 'channel-lost'); return false; } const generation = this.generation; diff --git a/src/voice/realtimeViewportPolicy.test.mjs b/src/voice/realtimeViewportPolicy.test.mjs index 2a90b3795..94f8c9379 100644 --- a/src/voice/realtimeViewportPolicy.test.mjs +++ b/src/voice/realtimeViewportPolicy.test.mjs @@ -13,17 +13,6 @@ const localContext = (overrides = {}) => ({ }); test('viewport context reasons preserve the existing escalation gate', () => { - assert.deepEqual( - viewportContextDecision({ action: 'get_current_view_state' }), - { - send: false, - reason: 'not-entity-context', - }, - ); - assert.deepEqual( - viewportContextDecision(localContext(), { channelOpen: false }), - { send: false, reason: 'channel-closed' }, - ); assert.deepEqual( viewportContextDecision({ action: 'get_entity_context', @@ -87,7 +76,7 @@ test('pointer ground selects the #837 crop path, not a full viewport', () => { assert.deepEqual(viewportContextDecision(pointerContext), { send: true, kind: 'crop', - reason: 'pointer-ground-crop', + reason: 'pointer-crop-eligible', viewScale: 'city', }); assert.deepEqual( @@ -96,8 +85,9 @@ test('pointer ground selects the #837 crop path, not a full viewport', () => { scene: { basemap: { viewScale: 'global' } }, }), { - send: false, - reason: 'pointer-scale-not-useful', + send: true, + kind: 'crop', + reason: 'pointer-crop-eligible', viewScale: 'global', }, ); @@ -173,7 +163,7 @@ test('runtime dispatch preserves pointer crop and full image separation', async calls.some( (call) => call.event === 'viewport_context.decision' && - call.detail.reason === 'pointer-ground-crop', + call.detail.reason === 'pointer-crop-eligible', ), ); }); @@ -333,7 +323,7 @@ test('a crop decision that cannot send logs why, under the same id', async () => assert.equal(events.length, 2, reason); const [decision, outcome] = events; assert.equal(decision.event, 'viewport_context.decision'); - assert.equal(decision.detail.reason, 'pointer-ground-crop'); + assert.equal(decision.detail.reason, 'pointer-crop-eligible'); assert.deepEqual(outcome, { event: 'viewport_context.skipped', detail: { contextId: decision.detail.contextId, kind: 'crop', reason }, @@ -573,8 +563,8 @@ test('direct calls log their defensive skips without a decision id', async () => [...open.events, ...closed.events].map((item) => item.detail), [ { contextId: null, kind: 'crop', reason: 'pointer-scale-not-useful' }, - { contextId: null, kind: 'crop', reason: 'channel-closed' }, - { contextId: null, kind: 'viewport', reason: 'channel-closed' }, + { contextId: null, kind: 'crop', reason: 'channel-lost' }, + { contextId: null, kind: 'viewport', reason: 'channel-lost' }, ], ); });