fix the remaining voice review gaps

keep analyst counts, speech, cards, cancellation, pointer capture, tool schema boundaries, and session lifetime consistent
add the exact Alps fallback and focused regression coverage
This commit is contained in:
manjunath22466 committed 2026-10-05 04:37:28 +05:30
1 parent 6858976b01
commit 0343b4495e
21 files changed
+1745 -441

No files matched your search

+3 -2
View File
@@ -34,8 +34,9 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
// Re-derived for the voice layer manifest (generated layer enums and
// aliases), point-and-ask (pointer sentinels, referent args) and the
// consolidated tool wording (each policy stated once); model-facing tools
// omit top-level anyOf (OpenAI rejects it).
'66c955a41e36c6e7d2d255a950993128528288be13946fe4f7dacb05ad2ba291',
// omit top-level anyOf (OpenAI rejects it). Contacts list routing preserves
// requested radius and center authority in descriptions only.
'71c66bb725e6987e56780db1e1a05e17359df6f2fcc4884b7ad46f326eae5f5d',
);
});
+63 -15
View File
@@ -3478,6 +3478,9 @@ async function aircraftProximityWindowForQuery(
result,
isCurrent = () => true,
) {
// Follow-ups re-filter the engine's retained answer. Re-projecting a fresh
// Contacts snapshot here would restore rows a previous filter removed.
if (result?.coverage?.followUp) return null;
const spec = normalizeAnalystSpec(args, args?.layers || []);
if (!spec) return null;
const scope = args?.scope;
@@ -3552,8 +3555,10 @@ async function aircraftProximityWindowForQuery(
lat,
lon,
};
// Rank the retained cohort at its original precision; display rounding
// must not turn nearby contacts into artificial ID-sorted ties.
if (Number.isFinite(item.distance))
distanceKm.set(row, Math.round(item.distance / 100) / 10);
distanceKm.set(row, item.distance / 1000);
return row;
});
return { key, rows };
@@ -3659,6 +3664,9 @@ async function aircraftProximityWindowForQuery(
answer,
memory: {
matched: selection.matched,
// Follow-up ranking belongs to this retained Contacts evaluation, not
// the camera's current position. Keep the accessor with its row owners.
distanceOf: (row) => distanceKm.get(row),
layerKeys: wanted,
coverage,
unanswered: [...(result?.unanswered || [])],
@@ -4791,6 +4799,46 @@ const ANALYST_REFUSAL_FIELDS = Object.freeze([
'cancelled',
]);
const VIEWPORT_COVERAGE_NOTE = 'the flights layer loads by viewport';
/** Resolve omitted follow-up scope from the engine's retained coverage. */
function effectiveAnalystScopeKind(args, coverage) {
const explicit = String(args.scope?.kind || '').toLowerCase();
if (explicit) return explicit;
const retained = String(coverage?.scope || '')
.split(':', 1)[0]
.toLowerCase();
if (retained === 'window') return 'radius';
if (['anywhere', 'radius', 'region', 'view'].includes(retained))
return retained;
return args.followUp ? 'anywhere' : 'view';
}
/** Add the viewport qualification without replacing retained provenance. */
function withViewportCoverageNote(coverage) {
if (!coverage) return coverage;
const current = typeof coverage.note === 'string' ? coverage.note : '';
if (current.includes(VIEWPORT_COVERAGE_NOTE)) return coverage;
const separator = current ? '; ' : '';
return {
...coverage,
note: current
? `${current}${separator}${VIEWPORT_COVERAGE_NOTE}`
: VIEWPORT_COVERAGE_NOTE,
};
}
/** Whether this query counts records from a viewport-loaded layer. */
function isViewportScopedAnalystResult(args, coverage) {
const scopeKind = effectiveAnalystScopeKind(args, coverage);
return (
(scopeKind === 'radius' || scopeKind === 'view') &&
(coverage?.layersQueried || []).some(
(layer) => voiceLayer(layer.layerKey)?.query?.viewportLoaded,
)
);
}
async function runAnalystQuery(
analystEngine,
dataManager,
@@ -4820,6 +4868,15 @@ async function runAnalystQuery(
candidate,
isCurrent,
);
if (
projected?.answer &&
projected?.memory &&
isViewportScopedAnalystResult(args, projected.answer.coverage)
) {
const coverage = withViewportCoverageNote(projected.answer.coverage);
projected.answer.coverage = coverage;
projected.memory.coverage = coverage;
}
entityWindow = projected?.answer || null;
return projected?.memory || null;
},
@@ -4855,14 +4912,12 @@ async function runAnalystQuery(
// A radius/view count over a viewport-loaded layer counts what is LOADED, and
// the flights layer reloads as the camera moves — so this number can sit well
// under the Contacts cohort without either being wrong.
const scopeKind = String(args.scope?.kind || 'view').toLowerCase();
const viewportScoped =
(scopeKind === 'radius' || scopeKind === 'view') &&
(result.coverage?.layersQueried || []).some(
(l) => voiceLayer(l.layerKey)?.query?.viewportLoaded,
);
const scopeKind = effectiveAnalystScopeKind(args, result.coverage);
const viewportScoped = isViewportScopedAnalystResult(args, result.coverage);
if (viewportScoped && result.coverage) {
result.coverage.note = 'the flights layer loads by viewport';
// The engine's follow-up memory retains this same object, so update its
// note in place after deriving the qualified copy.
result.coverage.note = withViewportCoverageNote(result.coverage).note;
}
// ENTITY-CENTRED NEARBY: answered by the SAME engine that fills the Contacts
// panel, so the spoken number and the panel readout for one centre cannot
@@ -4870,13 +4925,6 @@ async function runAnalystQuery(
// arbitrary points — only "how many aircraft around <this contact>" is
// unified, because that is the question the panel is already answering.
if (entityWindow) {
// The detached Contacts snapshot keeps its own coverage; carry the same
// viewport note so both answer shapes say what they measured.
if (viewportScoped && entityWindow.coverage)
entityWindow.coverage = {
...entityWindow.coverage,
note: result.coverage?.note,
};
return entityWindow;
}
+5 -1
View File
@@ -15,7 +15,7 @@ const ACTION_NAMES = new Set(GEV_ACTION_SCHEMAS.map((schema) => schema.name));
*/
export function withToolCatalog(runner, loadCatalog) {
if (typeof loadCatalog !== 'function') return runner;
return async function runGevTool(name, args, options = {}) {
const runGevTool = async function runGevTool(name, args, options = {}) {
if (ACTION_NAMES.has(name)) return runner(name, args, options);
const catalog = await loadCatalog();
if (!catalog?.get(name)) return runner(name, args, options);
@@ -24,6 +24,10 @@ export function withToolCatalog(runner, loadCatalog) {
});
return toFunctionOutput(name, result);
};
runGevTool.resetConversation = (...args) =>
runner.resetConversation?.(...args);
runGevTool.dispose = (...args) => runner.dispose?.(...args);
return runGevTool;
}
/** Compose the standalone action runner with the voice controls. */
+14
View File
@@ -3733,6 +3733,16 @@ test('voice runs app actions itself and other tools through the catalog', async
actions.push([name, args]);
return { ok: true, action: name };
};
const resets = [];
const disposals = [];
runner.resetConversation = (...args) => {
resets.push(args);
return 'reset';
};
runner.dispose = (...args) => {
disposals.push(args);
return 'disposed';
};
const calls = [];
let loads = 0;
const catalog = {
@@ -3764,5 +3774,9 @@ test('voice runs app actions itself and other tools through the catalog', async
actions.map(([name]) => name),
['zoom_to_globe', 'not_a_tool'],
);
assert.equal(run.resetConversation('stopped'), 'reset');
assert.equal(run.dispose('removed'), 'disposed');
assert.deepEqual(resets, [['stopped']]);
assert.deepEqual(disposals, [['removed']]);
assert.equal(withToolCatalog(runner, undefined), runner);
});
+118 -1
View File
@@ -1,6 +1,10 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { RealtimeViewport, pointerCropRect } from './realtimeViewport.js';
import {
RealtimeViewport,
pointerCropRect,
renderFreshPointerFrame,
} from './realtimeViewport.js';
test('crop: a square around the pointer, clamped inside the canvas, in buffer pixels', () => {
assert.deepEqual(
@@ -48,6 +52,88 @@ test('crop: a square around the pointer, clamped inside the canvas, in buffer pi
}),
null,
);
assert.equal(
pointerCropRect({
x: Number.NaN,
y: 1,
cssWidth: 100,
cssHeight: 100,
pixelWidth: 100,
pixelHeight: 100,
}),
null,
'non-finite pointer geometry fails closed',
);
assert.equal(
pointerCropRect({
x: 101,
y: 1,
cssWidth: 100,
cssHeight: 100,
pixelWidth: 100,
pixelHeight: 100,
}),
null,
'out-of-canvas pointer geometry fails closed',
);
assert.equal(
pointerCropRect({
x: 120,
y: 100,
cssWidth: 320,
cssHeight: 240,
pixelWidth: 320,
pixelHeight: 240,
}).output,
512,
'the provider image stays 512 px even when the source crop is smaller',
);
});
test('crop: a slow render gets one bounded fresh-frame recovery request', async () => {
const originalDocument = globalThis.document;
let requests = 0;
let listener = null;
try {
globalThis.document = { hidden: false };
const scene = {
postRender: {
addEventListener(next) {
listener = next;
return () => {
if (listener === next) listener = null;
};
},
},
requestRender() {
requests += 1;
if (requests === 2) queueMicrotask(() => listener?.());
},
};
assert.equal(await renderFreshPointerFrame({ scene }), true);
assert.equal(requests, 2, 'only the pointer path retries once');
assert.equal(listener, null, 'the successful listener is removed');
} finally {
globalThis.document = originalDocument;
}
});
test('crop: a hidden document never starts the recovery request', async () => {
const originalDocument = globalThis.document;
let requests = 0;
try {
globalThis.document = { hidden: true };
const scene = {
postRender: { addEventListener: () => () => {} },
requestRender() {
requests += 1;
},
};
assert.equal(await renderFreshPointerFrame({ scene }), false);
assert.equal(requests, 0);
} finally {
globalThis.document = originalDocument;
}
});
function viewportFixture({ capturePointer, capture } = {}) {
@@ -191,6 +277,15 @@ test('crop: a camera move before or during capture drops it; pixels only name th
false,
'moved before capture',
);
state.camera = null;
assert.equal(
await viewport.sendPointerCrop(
{ ...ground, cameraKey: null },
{ viewScale: 'local' },
),
false,
'a missing pose identity fails closed',
);
state.camera = 'pose-A';
const pending = viewport.sendPointerCrop(
{ ...ground },
@@ -202,6 +297,28 @@ test('crop: a camera move before or during capture drops it; pixels only name th
assert.equal(sent.length, 0);
});
test('crop: a snapshot that expires during capture is not published', async () => {
let release;
const { viewport, sent, state } = viewportFixture({
capturePointer: () =>
new Promise((resolve) => {
release = () =>
resolve({
dataUrl: 'data:image/jpeg;base64,CROP',
rect: { x: 0, y: 0, w: 1, h: 1 },
});
}),
});
const pending = viewport.sendPointerCrop(
{ ...ground },
{ viewScale: 'local' },
);
state.clock = 40_000;
release();
assert.equal(await pending, false);
assert.equal(sent.length, 0);
});
test('crop: an older capture that finishes last never replaces the newer image', async () => {
const releases = [];
const { viewport, sent } = viewportFixture({
+8 -1
View File
@@ -143,7 +143,14 @@ export function responseInstructionForToolResult(result) {
return 'The map annotations are cleared. Continue naturally; do not announce the clear.';
}
if (typeof result?.say === 'string' && result.say) {
return "Speak the say line of this turn's tool results once, in order, lightly rephrased at most; add no qualifiers and don't repeat your preamble.";
return "Speak the say line of this turn's tool results once, in order, lightly rephrased at most. Preserve its lower bounds, partial answers and stale, degraded or unavailable feed states. Add no counts, lists, qualifiers or action claims beyond the executed tool results, and don't repeat your preamble. If a requested list or another action is still unfinished, call the required tool before the final answer.";
}
if (
result?.ok &&
result.action === 'set_context_mode' &&
result.contactsWindow
) {
return 'Contacts activation and its window count do not fulfill a requested list or ranking. Continue any unfinished request with analyst_query: around the active subject use radius with the actual requested km and omit center; preserve explicit view, place, pointer and explicit center scopes. Use the returned items, never invent aircraft, counts or qualifications. Otherwise briefly confirm the completed action from its result.';
}
return 'Briefly confirm the completed GEV action once. Do not repeat yourself.';
}
+35 -6
View File
@@ -86,13 +86,29 @@ export function pointerCropRect({
pixelHeight,
size = POINTER_CROP_SIZE,
}) {
if (!cssWidth || !cssHeight || !pixelWidth || !pixelHeight) return null;
const values = [x, y, cssWidth, cssHeight, pixelWidth, pixelHeight, size];
if (
values.some((value) => !Number.isFinite(value)) ||
cssWidth <= 0 ||
cssHeight <= 0 ||
pixelWidth <= 0 ||
pixelHeight <= 0 ||
size <= 0 ||
x < 0 ||
x > cssWidth ||
y < 0 ||
y > cssHeight
)
return null;
const side = Math.min(size, cssWidth, cssHeight);
const left = Math.min(Math.max(0, x - side / 2), cssWidth - side);
const top = Math.min(Math.max(0, y - side / 2), cssHeight - side);
const scaleX = pixelWidth / cssWidth;
const scaleY = pixelHeight / cssHeight;
const output = Math.round(Math.min(size, side * scaleX));
// Keep the provider-facing image contract stable even on a narrow canvas.
// drawImage may upscale a smaller source crop, but the retained image and
// pointer-ring coordinates are always expressed in one 512 px square.
const output = Math.round(size);
return {
sx: Math.round(left * scaleX),
sy: Math.round(top * scaleY),
@@ -116,7 +132,7 @@ export async function capturePointerCrop(screenPx) {
const viewer = window.__godsEyeView?.viewer;
const source = viewer?.scene?.canvas;
if (!source?.width || !source?.height || !screenPx) return null;
const fresh = await renderFreshCesiumFrame(viewer);
const fresh = await renderFreshPointerFrame(viewer);
if (!fresh) return null;
const rect = pointerCropRect({
x: screenPx.x,
@@ -166,6 +182,16 @@ export async function capturePointerCrop(screenPx) {
}
}
/**
* Pointer crops get one bounded recovery request for a slow render loop.
* Full-viewport captures intentionally keep their single 400 ms attempt.
*/
export async function renderFreshPointerFrame(viewer) {
if (await renderFreshCesiumFrame(viewer)) return true;
if (typeof document !== 'undefined' && document.hidden) return false;
return renderFreshCesiumFrame(viewer);
}
// Scale (w, h) down so w*h <= maxPixels while preserving aspect ratio. Never
// upscales. Both dimensions shrink together, so portrait and landscape are
// treated equally (M13). Pure + deterministic → unit-tested (exported below).
@@ -345,11 +371,14 @@ export class RealtimeViewport {
// Too far out, the crop is a smear: every path shares this guard.
if (!POINTER_CROP_SCALES.has(viewScale)) return false;
if (snapshot === this.lastPointerCrop) return false;
if (this.now() - (snapshot.capturedAt || 0) > POINTER_CROP_MAX_AGE_MS)
return false;
const capturedAt = Number(snapshot.capturedAt) || 0;
if (this.now() - capturedAt > POINTER_CROP_MAX_AGE_MS) return false;
// The snapshot's pixels only name its own camera pose.
const snapshotCameraKey = snapshot.cameraKey;
if (!snapshotCameraKey) return false;
const samePose = () =>
!snapshot.cameraKey || snapshot.cameraKey === this.readCameraKey();
snapshotCameraKey === this.readCameraKey() &&
this.now() - capturedAt <= POINTER_CROP_MAX_AGE_MS;
if (!samePose()) return false;
this.lastPointerCrop = snapshot;
const sent = await this.sendRetainedImage(
+31 -5
View File
@@ -28,10 +28,19 @@ const LAYER_NOUNS = Object.freeze({
/** Split "a; b" caveat text into separate notes. */
function notesOf(...values) {
return values
.flatMap((value) => String(value || '').split(/;\s*/))
.map((text) => text.trim())
.filter(Boolean);
return [
...new Set(
values
.flatMap((value) => String(value || '').split(/;\s*/))
.map((text) =>
text
.replace(/[\p{Cc}\p{Cf}]+/gu, ' ')
.replace(/\s+/g, ' ')
.trim(),
)
.filter(Boolean),
),
];
}
export function analystNounFor(layerKeys) {
@@ -67,6 +76,19 @@ function analystCard(result) {
if (!result?.ok) return null;
const display = result.display || {};
const unanswered = Array.isArray(result.unanswered) ? result.unanswered : [];
const provenance = result.feedProvenance || result.coverage?.feedProvenance;
const sources = [
...new Set(
[...(provenance?.layers || []), ...(result.coverage?.layersQueried || [])]
.map((layer) =>
String(layer?.source || '')
.replace(/[\p{Cc}\p{Cf}]+/gu, ' ')
.replace(/\s+/g, ' ')
.trim(),
)
.filter(Boolean),
),
].map((label) => ({ label }));
return {
display: {
title: analystHeadline(result),
@@ -81,12 +103,16 @@ function analystCard(result) {
: []),
...(result.partial ? [{ label: 'partial' }] : []),
],
// Coverage qualifications lead the feed detail. The bounded card groups
// overflow in its existing Notes disclosure rather than dropping them.
preserveNotes: true,
notes: [
...notesOf(display.caveat, result.coverage?.note),
...notesOf(result.coverage?.note, display.caveat),
...(unanswered.length
? [`Not answered: ${unanswered.join(', ')}`]
: []),
],
sources,
},
referents: Array.isArray(result.items) ? result.items : [],
resultSet: true,
+13 -1
View File
@@ -401,6 +401,18 @@ function annotateMap(result) {
function analystQuery(result) {
if (!result?.ok || result.cancelled) return null;
const headline = spokenLabel(analystHeadline(result), 96);
const state =
result.feedState ||
result.feedProvenance?.overall ||
result.coverage?.feedProvenance?.overall;
const tag = feedTag(state);
const scope = spokenLabel(result.scopeLabel || '', 96);
const answer =
tag === 'feed unavailable'
? sentence(
`Feed unavailable${scope ? ` ${scope}` : ''}; no authoritative count${result.complete === false ? ', retained coverage is a lower bound' : ''}`,
)
: sentence(`${headline}${tag ? `, ${tag}` : ''}`);
const unanswered = (Array.isArray(result.unanswered) ? result.unanswered : [])
.map((layer) => spokenLabel(layer, 24))
.filter(Boolean);
@@ -408,7 +420,7 @@ function analystQuery(result) {
? `Partial; ${listPhrase(unanswered.slice(0, 2))}${unanswered.length > 2 ? ` and ${unanswered.length - 2} more` : ''} not answered.`
: 'Partial answer.';
return {
say: `${sentence(headline)}${result.partial ? ` ${missing}` : ''}`,
say: `${answer}${result.partial ? ` ${missing}` : ''}`,
};
}
+20 -4
View File
@@ -242,12 +242,28 @@ export function reduceVoiceCard(state, event) {
export function voiceCardView(state) {
const running = state.steps.some((step) => step.status === 'running');
const display = state.result?.display || null;
const notes = [
const allNotes = [
...(display?.notes || []),
...(display?.sources || []).map((source) => `Source: ${source.label}`),
]
.map((note) => spokenLabel(note, 96))
.slice(0, MAX_NOTES);
];
// Analyst notes carry count/scope/provenance qualifications. Preserve their
// full sanitized text in at most six existing rows, joining overflow into
// the last row. Other tool cards retain their compact note contract.
const completeNotes = display?.preserveNotes
? [
...new Set(
allNotes.map((note) => spokenLabel(note, Infinity)).filter(Boolean),
),
]
: null;
const notes = completeNotes
? completeNotes.length > MAX_NOTES
? [
...completeNotes.slice(0, MAX_NOTES - 1),
completeNotes.slice(MAX_NOTES - 1).join('; '),
]
: completeNotes
: allNotes.map((note) => spokenLabel(note, 96)).slice(0, MAX_NOTES);
return {
visible: Boolean(state.visible),
busy: running,