Merge current main into S2 GhostMaps and preserve updated import and voice behavior

This commit is contained in:
Mazeyar Moeini Feizabadi committed 2026-10-26 16:25:49 +01:00
commit 4da002bd42
515 files changed
+17681 -2202

No files matched your search

+1 -1
View File
@@ -729,7 +729,7 @@ const schemas = [
properties: {
motion: {
type: 'string',
enum: ['orbit', 'pan', 'tilt', 'rotate', 'stop'],
enum: ['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe', 'stop'],
},
direction: {
type: 'string',
+47 -2
View File
@@ -1,3 +1,4 @@
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
import test from 'node:test';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
@@ -38,8 +39,11 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
// requested radius and center authority in descriptions only.
// Street Level and GNSS extend the manifest; tracked and analyst aircraft describe broadcasts.
// Air quality (ECCC AQHI) extends the manifest.
// S2 GhostMaps extends the manifest.
'64a7ae9719cbf53f2c5b57739c1e84e8b60dc4ede040decd48a9c06d1ccbcbca',
// Globe rotation adds one motion; visual keyhole fit and explicit
// reset guidance/defaults update only move_camera and mode descriptions.
// Restore base descriptions; append only globe-rotation defaults and fit guidance.
// S2 GhostMaps adds its layer id to the generated layer enums.
'1f95d78074559217872ccbdfa57d4363264ea88c1a22b259e6ebba3c8189b806',
);
});
@@ -107,6 +111,17 @@ test('all legacy action arguments are byte-identical after removing the delibera
delete property('show_data_layers_menu').layerId.enum;
delete property('get_entity_context').layerId.enum;
delete property('analyst_query').layers.items.enum;
// Earth-centered rotation is additive; local camera yaw keeps its contract.
const motion = property('move_camera').motion;
assert.deepEqual(motion.enum, [
'orbit',
'pan',
'tilt',
'rotate',
'rotate_globe',
'stop',
]);
motion.enum = motion.enum.filter((value) => value !== 'rotate_globe');
const panels = property('set_panel_open').panelId;
panels.enum = panels.enum.filter(
(id) => !['weather-panel', 'recent-imagery-panel'].includes(id),
@@ -146,6 +161,24 @@ test('all legacy action arguments are byte-identical after removing the delibera
);
});
test('globe rotation is an additive camera motion with the existing control arguments', () => {
const schema = GEV_ACTION_SCHEMAS.find(
(tool) => tool.name === 'move_camera',
).parameters;
for (const motion of ['rotate_globe', 'rotate', 'orbit', 'pan', 'tilt']) {
assert.deepEqual(
validateArgs(schema, {
motion,
direction: 'right',
speed: 'slow',
mode: 'continuous',
}),
[],
);
}
assert.ok(validateArgs(schema, { motion: 'spin_camera' }).length > 0);
});
test('track_entity accepts a referent alone but rejects an empty target', () => {
const schema = GEV_ACTION_SCHEMAS.find(
(tool) => tool.name === 'track_entity',
@@ -184,3 +217,15 @@ test('the benchmark still rejects an empty track_entity although the model-facin
[],
);
});
test('globe reset guidance requires a separate explicit command after refusal', () => {
const paragraph = realtimeInstructions()
.split('\n')
.find((line) => line.includes('globe-view-required'));
assert.ok(paragraph);
assert.match(paragraph, /calls only move_camera motion rotate_globe/);
assert.match(paragraph, /explicit user Reset\/Show Globe View/);
assert.match(paragraph, /never from your own plan/);
assert.match(paragraph, /bare yes\/ok/);
assert.match(paragraph, /Do not ask a question/);
});
+3 -3
View File
@@ -1,5 +1,5 @@
export const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']);
export const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key';
const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']);
const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key';
export const CLOUD_VOICE_AUTH_STORAGE_KEY = 'godsEyeView.voice.cloudAuth';
function storageRef(storage) {
@@ -11,7 +11,7 @@ function storageRef(storage) {
}
}
export function normalizeCloudVoiceAuthMode(mode) {
function normalizeCloudVoiceAuthMode(mode) {
const normalized = String(mode || '')
.trim()
.toLowerCase();
+1 -1
View File
@@ -18,7 +18,7 @@
import { finiteOrNull } from './pointerContext.js';
export const POINTER_SENTINEL = 'pointer';
const POINTER_SENTINEL = 'pointer';
const TRACKABLE_LAYERS = new Set([
'flights',
+48 -2
View File
@@ -21,6 +21,7 @@ import {
shouldSendViewportImage,
shouldStopVoiceAfterRadioTool,
responseInstructionForToolResult,
isGlobeViewRefusal,
} from './realtimeProtocol.js';
const SOCKET_PATH =
@@ -182,6 +183,8 @@ export class GeminiSessionController {
this.model = null;
this.epoch = 0;
this.turn = 0;
this.globeToolsBlocked = false;
this.globeResetFailed = false;
this.inputGeneration = 0;
this.turnAbort = new AbortController();
this.disposed = false;
@@ -659,6 +662,8 @@ export class GeminiSessionController {
return;
if (!pushToTalk || activityCaptureId === metadata.captureId) return;
this.captureOwner.admitted = true;
this.globeToolsBlocked = false;
this.globeResetFailed = false;
if (
!this.interrupt('push-to-talk', {
keepPlayback: !this.pttPlaybackSuppressed,
@@ -1294,6 +1299,8 @@ export class GeminiSessionController {
this.replyPending = false;
if (!this.interrupt()) return;
} else if (!this.retiredReplyPending && !this.interrupt()) return;
this.globeToolsBlocked = false;
this.globeResetFailed = false;
// Local activity ownership is only for expiry liveness, not tools.
// Assign after retiring observed output so its boundary cannot clear
// a newer detected activity that has not received a reply yet.
@@ -1573,6 +1580,7 @@ export class GeminiSessionController {
.then(async () => {
if (!this.owns(epoch) || turn !== this.turn) return;
if (this.cancelledCalls.has(call.id)) {
if (call.name === 'zoom_to_globe') this.globeResetFailed = true;
if (this.tools.get(call.id) === action) this.tools.delete(call.id);
this.pendingTools--;
if (!this.pendingTools) {
@@ -1618,8 +1626,28 @@ export class GeminiSessionController {
(authority !== 'playback' ||
handoffEpoch === this.radio.radioHandoffEpoch);
let result;
let skippedGlobeRotation = false;
let blockedGlobeTool = false;
try {
if (!actionCurrent()) throw cancelled();
// Live has no response-scoped tool_choice. Recheck here so queued siblings
// cannot reset after a rotation refusal.
if (this.globeToolsBlocked) {
blockedGlobeTool = true;
throw new Error(
'Wait for fresh user input before calling another tool.',
);
}
if (
this.globeResetFailed &&
call.name === 'move_camera' &&
String(args.motion || '').toLowerCase() === 'rotate_globe'
) {
skippedGlobeRotation = true;
throw new Error(
'Not run: the Globe View reset/rotation sequence failed or was cancelled.',
);
}
if (stronger)
reservation = this.radio.reserveRadioToolHandoff({ abortScope });
handoffEpoch = this.radio.radioHandoffEpoch;
@@ -1689,7 +1717,12 @@ export class GeminiSessionController {
} catch (error) {
result = {
ok: false,
tool: call.name,
...(skippedGlobeRotation
? { action: call.name, reason: 'globe-reset-failed', skipped: true }
: {
tool: call.name,
...(blockedGlobeTool ? { skipped: true } : {}),
}),
...(radioVisibility ? { layerId: 'radio' } : {}),
error:
error?.name === 'AbortError'
@@ -1716,6 +1749,11 @@ export class GeminiSessionController {
this.noteRadioStrongerCommit(action, abortScope, turn);
}
if (!this.owns(epoch) || turn !== this.turn) return;
if (
call.name === 'zoom_to_globe' &&
(result?.ok !== true || result.cancelled)
)
this.globeResetFailed = true;
const playbackObsolete =
authority === 'playback' && handoffEpoch !== this.radio.radioHandoffEpoch;
if (radioControl || radioVisibility) {
@@ -1744,6 +1782,8 @@ export class GeminiSessionController {
this.stopAfterReply = true;
// Native call cancellation suppresses its output, without canceling siblings.
if (!this.cancelledCalls.has(call.id)) {
if (isGlobeViewRefusal(result) && !skippedGlobeRotation)
this.globeToolsBlocked = true;
this.postToolOutput = false;
this.modelComplete = false;
this.replyPending = true;
@@ -1751,7 +1791,9 @@ export class GeminiSessionController {
const mapEvents = mapSnapshot.map((text) => JSON.parse(text));
const response = {
...(result ?? { ok: false, error: 'Command returned no result' }),
responseGuidance: responseInstructionForToolResult(result),
responseGuidance: blockedGlobeTool
? 'That action did not run. Wait for fresh user input; do not claim any change or call another tool.'
: responseInstructionForToolResult(result),
};
this.send(
{
@@ -1902,6 +1944,8 @@ export class GeminiSessionController {
{
onSent: () => {
if (!owned() || this.typedIntent !== typedIntent) return;
this.globeToolsBlocked = false;
this.globeResetFailed = false;
this.replyPending = true;
this.bindTextIntent(typedIntent);
if (!owned() || this.typedIntent !== typedIntent) return;
@@ -2455,6 +2499,8 @@ export class GeminiSessionController {
this.cancelledCalls.clear();
this.pendingMapEvents = [];
this.userTurnPending = false;
this.globeToolsBlocked = false;
this.globeResetFailed = false;
this.replyPending = false;
this.unrequestedReply = false;
this.automaticTurnOpen = false;
+212
View File
@@ -3398,3 +3398,215 @@ test('a narration callback that submits a successor keeps the old packet tools f
assert.deepEqual(h.calls, []);
h.c.stop();
});
for (const reply of ['typed', 'push-to-talk', 'open-mic']) {
test(`globe refusal blocks queued and later tools until fresh ${reply} input`, async () => {
const names = [];
const h = harness({
runAction: async (name) => {
names.push(name);
return name === 'move_camera'
? { ok: false, action: 'move_camera', reason: 'globe-view-required' }
: { ok: true, action: name };
},
});
try {
await h.c.start({ pushToTalk: reply === 'push-to-talk' });
establishToolInput(h, 'rotate the globe');
const ws = h.sockets[0];
ws.message({
toolCall: {
functionCalls: [
{
id: 'rotate',
name: 'move_camera',
args: { motion: 'rotate_globe' },
},
{ id: 'queued-reset', name: 'zoom_to_globe', args: {} },
],
},
});
await h.c.toolQueue;
assert.deepEqual(names, ['move_camera']);
const blocked = ws.sent
.flatMap((entry) => entry.toolResponse?.functionResponses || [])
.find((call) => call.id === 'queued-reset');
assert.equal(blocked.response.ok, false);
assert.match(blocked.response.error, /fresh user input/i);
assert.equal(blocked.response.skipped, true);
assert.match(blocked.response.responseGuidance, /did not run/i);
assert.doesNotMatch(
blocked.response.responseGuidance,
/confirm.*completed/i,
);
requestTool(ws, 'repeated-rotation', 'move_camera', {
motion: 'rotate_globe',
});
await h.c.toolQueue;
assert.deepEqual(names, ['move_camera']);
const repeated = toolResponses(ws).find(
(call) => call.id === 'repeated-rotation',
).response;
assert.equal(repeated.ok, false);
assert.equal(repeated.skipped, true);
assert.match(repeated.responseGuidance, /did not run/i);
assert.doesNotMatch(
repeated.responseGuidance,
/confirm.*completed/i,
);
if (reply === 'typed') {
const stimuli = [
() => {},
() => ws.message({ serverContent: { interrupted: true } }),
() =>
ws.message({
serverContent: {
inputTranscription: { text: 'reset to globe view' },
},
}),
() => {
h.c.input.setMicrophoneEnabled(false);
h.c.input.setMicrophoneEnabled(true);
},
];
for (const [i, stimulus] of stimuli.entries()) {
stimulus();
requestTool(ws, `unrequested-reset-${i}`, 'zoom_to_globe', {});
await h.c.toolQueue;
assert.deepEqual(names, ['move_camera']);
}
}
// Finish the refusal reply before admitting the actual next user input.
ws.message({ serverContent: { turnComplete: true } });
if (reply === 'typed')
assert.equal(h.c.sendTextCommand('reset to globe view'), true);
else if (reply === 'push-to-talk') {
h.c.input.pushToTalkKeyHeld = true;
h.c.input.spaceKeyHeld = true;
h.c.input.setMicrophoneEnabled(true);
h.c.input.setMicrophoneEnabled(false);
} else ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
requestTool(ws, 'accepted-reset', 'zoom_to_globe', {});
await h.c.toolQueue;
assert.deepEqual(names, ['move_camera', 'zoom_to_globe']);
} finally {
h.c.stop();
}
});
}
test('a cancelled globe reset skips queued rotation, preserves Stop and recovers on fresh input', async () => {
const calls = [];
let resets = 0;
const h = harness({
runAction: async (name, args) => {
calls.push(name === 'move_camera' ? args.motion : name);
return name === 'zoom_to_globe' && resets++ === 0
? { ok: false, action: name, cancelled: true }
: { ok: true, action: name };
},
});
try {
await h.c.start();
establishToolInput(h, 'reset to globe view and rotate');
const ws = h.sockets[0];
ws.message({
toolCall: {
functionCalls: [
{ id: 'reset-failed', name: 'zoom_to_globe', args: {} },
{
id: 'rotate-skipped',
name: 'move_camera',
args: { motion: 'rotate_globe' },
},
{ id: 'stop-allowed', name: 'move_camera', args: { motion: 'stop' } },
],
},
});
await h.c.toolQueue;
assert.deepEqual(calls, ['zoom_to_globe', 'stop']);
const skipped = toolResponses(ws).find(
(call) => call.id === 'rotate-skipped',
).response;
assert.equal(skipped.ok, false);
assert.equal(skipped.reason, 'globe-reset-failed');
assert.equal(skipped.skipped, true);
assert.equal(
toolResponses(ws).find((call) => call.id === 'stop-allowed').response.ok,
true,
);
ws.message({ serverContent: { turnComplete: true } });
establishToolInput(h, 'reset to globe view and rotate');
ws.message({
toolCall: {
functionCalls: [
{ id: 'reset-success', name: 'zoom_to_globe', args: {} },
{
id: 'rotate-success',
name: 'move_camera',
args: { motion: 'rotate_globe' },
},
],
},
});
await h.c.toolQueue;
assert.deepEqual(calls, [
'zoom_to_globe',
'stop',
'zoom_to_globe',
'rotate_globe',
]);
} finally {
h.c.stop();
}
});
test('cancelling a queued globe reset also skips its dependent rotation', async () => {
const blocker = deferred();
const calls = [];
const h = harness({
runAction: async (name, args) => {
calls.push(name === 'move_camera' ? args.motion : name);
return args.motion === 'pan'
? blocker.promise
: { ok: true, action: name };
},
});
try {
await h.c.start();
establishToolInput(h, 'pan, reset to globe view and rotate');
const ws = h.sockets[0];
ws.message({
toolCall: {
functionCalls: [
{
id: 'blocker',
name: 'move_camera',
args: { motion: 'pan', direction: 'left' },
},
{ id: 'reset-cancelled', name: 'zoom_to_globe', args: {} },
{
id: 'rotate-after-cancel',
name: 'move_camera',
args: { motion: 'rotate_globe' },
},
],
},
});
await tick();
ws.message({ toolCallCancellation: { ids: ['reset-cancelled'] } });
blocker.resolve({ ok: true, action: 'move_camera' });
await h.c.toolQueue;
assert.deepEqual(calls, ['pan']);
const skipped = toolResponses(ws).find(
(call) => call.id === 'rotate-after-cancel',
).response;
assert.equal(skipped.reason, 'globe-reset-failed');
assert.equal(skipped.skipped, true);
} finally {
blocker.resolve({ ok: true });
h.c.stop();
}
});
+5 -66
View File
@@ -1,3 +1,4 @@
import { interruptGlobeRotation } from '../cameraVerbs.js';
import {
layerSnapshot,
feedProvenanceEnvelope,
@@ -262,7 +263,9 @@ export function createGevActionRunner({
}),
));
installViewTargetPrewarm(viewer);
initCameraVerbs(viewer, getViewTargetCartesian);
initCameraVerbs(viewer, getViewTargetCartesian, {
isCockpitActive: () => Boolean(styleManager?.getCockpitState?.()?.active),
});
// Nested calls (select_nearest_aircraft composing layer + flight) use the
// raw runner; only the outer call gets a voice envelope.
const runGevAction = async function runGevAction(
@@ -298,6 +301,7 @@ export function createGevActionRunner({
// Zoom during an active orbit adjusts the orbit RADIUS (spiral in/out) —
// a straight camera move would be snapped back by the per-frame lookAt.
if (name === 'adjust_camera_zoom') {
interruptGlobeRotation('voice-zoom-intent');
const zoomOut = String(args.direction || '').toLowerCase() === 'out';
const amt = String(args.amount || 'medium').toLowerCase();
const factor = { little: 1.25, medium: 1.6, lot: 2.4 }[amt] || 1.6;
@@ -4369,46 +4373,6 @@ function dominantValue(values) {
};
}
function summarizeEntity(viewer, entity, { includeProperties = false } = {}) {
const now = Cesium.JulianDate.now();
if (entity.__gevContextId) {
const store = window.__gevContextStore;
const record = store?.entities?.get(entity.__gevContextId);
if (record) return summarizeContextRecord(record, { includeProperties });
}
const props = propertyObject(entity);
const layerId = entity.__localLayerId || props.layerId || null;
const tags = props.tags || {};
const label = cleanText(
props.name ||
tags.name ||
tags['name:en'] ||
tags.official_name ||
tags.operator ||
props.operator ||
entity.name ||
layerTitle(layerId),
);
const position =
entity.__localBaseCartesian ||
entity.position?.getValue?.(now) ||
polygonCenter(entity, now);
const carto = position ? Cesium.Cartographic.fromCartesian(position) : null;
return {
id: String(entity.id || ''),
name: label || layerTitle(layerId),
layerId,
layerName: layerTitle(layerId),
latitude: carto
? Number(Cesium.Math.toDegrees(carto.latitude).toFixed(6))
: null,
longitude: carto
? Number(Cesium.Math.toDegrees(carto.longitude).toFixed(6))
: null,
properties: includeProperties ? compactProperties(props) : undefined,
};
}
function summarizeContextRecord(record, { includeProperties = false } = {}) {
return {
id: String(record.id || ''),
@@ -4427,31 +4391,6 @@ function summarizeContextRecord(record, { includeProperties = false } = {}) {
};
}
function polygonCenter(entity, now) {
const hierarchy = entity.polygon?.hierarchy?.getValue?.(now);
const positions = hierarchy?.positions;
if (!positions?.length) return null;
return Cesium.BoundingSphere.fromPoints(positions).center;
}
function propertyObject(entity) {
const raw = entity?.properties?.getValue?.(Cesium.JulianDate.now()) || {};
return unwrapProperties(raw);
}
function unwrapProperties(value) {
if (!value || typeof value !== 'object') return value;
if (Array.isArray(value)) return value.map(unwrapProperties);
const out = {};
for (const [key, entry] of Object.entries(value)) {
out[key] =
entry && typeof entry.getValue === 'function'
? unwrapProperties(entry.getValue(Cesium.JulianDate.now()))
: unwrapProperties(entry);
}
return out;
}
function compactProperties(props) {
const preferredKeys = [
'name',
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -4,7 +4,7 @@ import { progressLine } from './speech.js';
* Timing policy for filling silence while tools run. Tuned against the turn
* spans' `max_silence_ms`.
*/
export const NARRATION_TIMING = Object.freeze({
const NARRATION_TIMING = Object.freeze({
/** Start the earcon when nothing is audible this long after end of speech. */
earconAfterMs: 700,
/** First spoken progress line when a tool is still running after this. */
+4 -4
View File
@@ -20,15 +20,15 @@
*/
/** Movement below this many CSS pixels is still the same dwell. */
export const POINTER_DWELL_PX = 12;
const POINTER_DWELL_PX = 12;
/** Stillness that makes a dwell point. */
export const POINTER_DWELL_MS = 250;
/** A resting pointer stays a referent this long after it last moved. */
export const POINTER_FRESH_MS = 30000;
/** Pixels from the canvas edge that count as "parked", never a referent. */
export const POINTER_EDGE_PX = 8;
const POINTER_EDGE_PX = 8;
/** A dwell pick is reused by a snapshot only while this recent (contacts move). */
export const POINTER_DWELL_REUSE_MS = 1000;
const POINTER_DWELL_REUSE_MS = 1000;
/**
* Open mic has no gesture, so the pointer only counts when it moved or
* settled on the map this recently: a cursor resting there for minutes is
@@ -76,7 +76,7 @@ export function cameraPoseKey(viewer) {
}
/** Bound and de-control a label before it goes to the model or the card. */
export function pointerLabel(value, max = LABEL_MAX) {
function pointerLabel(value, max = LABEL_MAX) {
const clean = String(value ?? '')
.replace(/[\p{Cc}\p{Cf}]+/gu, ' ')
.replace(/\s+/g, ' ')
+1 -1
View File
@@ -4,7 +4,7 @@ import { cameraPoseKey } from './pointerContext.js';
import * as defaultAnnotationResolver from '../annotations/annotationResolver.js';
/** "Around here" radius as a share of camera height, clamped to sane bounds. */
export function pointerRadiusKm(cameraHeightM) {
function pointerRadiusKm(cameraHeightM) {
const heightKm = Number(cameraHeightM) / 1000;
if (!Number.isFinite(heightKm) || heightKm <= 0) return 50;
return Math.round(Math.min(400, Math.max(5, heightKm * 0.75)));
+1 -1
View File
@@ -14,7 +14,7 @@ export const RETICLE_MAX_MS = 10000;
export const RETICLE_LINGER_MS = 1500;
/** Tools that move the camera away from the pointed spot. */
export const CAMERA_TOOLS = new Set([
const CAMERA_TOOLS = new Set([
'fly_to_location',
'track_entity',
'zoom_to_globe',
+11 -18
View File
@@ -1,10 +1,10 @@
export const ERROR_LOG_LIMIT = 30;
const ERROR_LOG_LIMIT = 30;
export const ERROR_STORAGE_KEY = 'gev-realtime-errors';
const ERROR_STORAGE_KEY = 'gev-realtime-errors';
export const DEBUG_LOG_URL = '/api/realtime/debug-log';
const DEBUG_LOG_URL = '/api/realtime/debug-log';
export function createDebugSessionId() {
function createDebugSessionId() {
const randomPart = Math.random().toString(36).slice(2, 10);
return `gev-${Date.now().toString(36)}-${randomPart}`;
}
@@ -47,7 +47,7 @@ export function sanitizeDebugValue(value, depth = 0) {
return output;
}
export function sanitizeDebugString(value) {
function sanitizeDebugString(value) {
if (value.startsWith('data:image/')) {
return `[Redacted image data URL, ${value.length} chars]`;
}
@@ -65,13 +65,13 @@ export function sanitizeDebugString(value) {
: redacted;
}
export function isSecretLikeKey(key) {
function isSecretLikeKey(key) {
return /(?:api[_-]?key|authorization|bearer|client[_-]?secret|token|secret|password)/i.test(
key,
);
}
export function createErrorRecord(source, error, extra = {}) {
function createErrorRecord(source, error, extra = {}) {
const rtcError = error?.error || error;
return {
timestamp: new Date().toISOString(),
@@ -90,7 +90,7 @@ export function createErrorRecord(source, error, extra = {}) {
};
}
export function formatErrorForDisplay(record) {
function formatErrorForDisplay(record) {
const primary = [record.source, record.message].filter(Boolean).join(': ');
const state = [
record.errorDetail && `detail=${record.errorDetail}`,
@@ -105,7 +105,7 @@ export function formatErrorForDisplay(record) {
return state ? `${primary}\n${state}` : primary;
}
export function removeEmptyValues(value) {
function removeEmptyValues(value) {
return Object.fromEntries(
Object.entries(value || {}).filter(
([, item]) => item !== null && item !== undefined && item !== '',
@@ -113,14 +113,7 @@ export function removeEmptyValues(value) {
);
}
export function compactText(value, maxLength) {
const text = String(value || '')
.replace(/\s+/g, ' ')
.trim();
return text.length <= maxLength ? text : `${text.slice(0, maxLength - 1)}…`;
}
export function loadStoredErrors() {
function loadStoredErrors() {
try {
const value = JSON.parse(localStorage.getItem(ERROR_STORAGE_KEY) || '[]');
return Array.isArray(value) ? value.slice(0, ERROR_LOG_LIMIT) : [];
@@ -129,7 +122,7 @@ export function loadStoredErrors() {
}
}
export function storeErrors(errors) {
function storeErrors(errors) {
try {
localStorage.setItem(
ERROR_STORAGE_KEY,
+1 -1
View File
@@ -135,7 +135,7 @@ test('an old progress completion leaves the active in-band response alone', asyn
});
test('user speech stops an audible progress line: cancel and clear its audio', async () => {
const { controller, sent, event, progressLine } = controllerFixture();
const { sent, event, progressLine } = controllerFixture();
await progressLine('resp_p');
await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' });
sent.length = 0;
+3 -3
View File
@@ -8,12 +8,12 @@ import {
// Voice cost control (repo-wide `godsEyeView.<feature>.<field>` convention;
// the neighbouring ERROR_STORAGE_KEY predates it).
export const VOICE_TIER_STORAGE_KEY = 'godsEyeView.voiceCost.tier';
const VOICE_TIER_STORAGE_KEY = 'godsEyeView.voiceCost.tier';
export const VOICE_LIMITS_STORAGE_KEY = 'godsEyeView.voiceCost.limits';
const VOICE_LIMITS_STORAGE_KEY = 'godsEyeView.voiceCost.limits';
/** Best-effort localStorage handle; absent in tests and locked-down browsers. */
export function voiceStorage(storage) {
function voiceStorage(storage) {
if (storage) return storage;
try {
return typeof localStorage === 'undefined' ? null : localStorage;
+26
View File
@@ -99,6 +99,18 @@ export function hasStructuredViewIdentity(result) {
);
}
export function isGlobeViewRefusal(result) {
return (
result?.action === 'move_camera' &&
result.ok === false &&
[
'globe-view-required',
'globe-reset-unavailable',
'globe-reset-failed',
].includes(result.reason)
);
}
export function responseInstructionForToolResult(result) {
if (result?.action === 'control_radio' && result.radioPlaybackSuppressed) {
if (result.audioState === 'paused') {
@@ -123,6 +135,20 @@ export function responseInstructionForToolResult(result) {
: 'Be concise: useful names, place, layer and one or two notable properties.',
].join(' ');
}
if (isGlobeViewRefusal(result)) {
// A refusal is not a completed action. Keep tool text out of instructions.
if (result.reason === 'globe-reset-failed')
return 'The globe did not rotate after the Globe View reset attempt. Say so briefly. Do not retry, offer or perform another reset for this request.';
if (result.reason === 'globe-reset-unavailable')
return 'The globe did not rotate. Ask the user to zoom out further; the Globe View preset does not fit this viewport. Do not offer or perform a reset.';
return 'The globe did not rotate. Say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" The suggested command is not a user request. Do not ask a question, perform a reset, or call tools in this reply.';
}
if (
(result?.action ?? result?.tool) === 'zoom_to_globe' &&
(result.ok !== true || result.cancelled)
) {
return 'The Globe View reset did not finish. Say so briefly; do not claim Globe View was reached. Do not retry the reset or rotate the globe for this request.';
}
if (result?.action === 'adjust_camera_zoom') {
return result.ok
? `Confirm once that the camera zoomed ${result.direction}. Do not claim any other change.`
+230 -11
View File
@@ -6,6 +6,7 @@ import {
SUPERSEDED_RESPONSE_MEMORY,
shouldStopVoiceAfterRadioTool,
responseInstructionForToolResult,
isGlobeViewRefusal,
extractFunctionCalls,
callDedupeKeys,
parseArguments,
@@ -13,7 +14,7 @@ import {
import { narrationLabel } from './speech.js';
/** Out-of-band progress responses carry this metadata tag. */
export const PROGRESS_RESPONSE_TAG = 'progress';
const PROGRESS_RESPONSE_TAG = 'progress';
/**
* Speaks one code-authored progress line out of band. The line travels as
@@ -66,6 +67,11 @@ export class RealtimeTurns {
this.pendingUserTextResponse = false;
this.activeResponseId = null;
this.supersededResponseIds = new Set();
this.globeRefusals = new Map();
this.globeAdmissionGates = new Map();
this.globeResets = new Map();
this.globeResponseRequests = new Map();
this.globeRequest = { failed: false, pending: new Set() };
this.activeToolAbortControllers = new Set();
this.narration = null;
this.narrationSequence = 0;
@@ -138,6 +144,7 @@ export class RealtimeTurns {
},
};
if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return;
this.globeRequest = { failed: false, pending: new Set() };
this.requestUserTextResponse();
}
@@ -330,6 +337,7 @@ export class RealtimeTurns {
this.userTurnPending = true;
this.pendingResponseInstructions = null;
this.cancelRadioHandoff({ abortTools: true });
this.globeRequest = { failed: false, pending: new Set() };
this.setVoiceSpeaker('user');
// Point-and-ask: read the pointer as speech starts (open mic), or reuse
// the push-to-talk keydown snapshot, before the user audio commits.
@@ -384,6 +392,27 @@ export class RealtimeTurns {
}
const toolResponseId = payload.response_id || payload.response?.id || null;
// Follow-up responses retain reset ownership until fresh admitted input.
const arrivalRequest = this.globeRequest;
const globeRequest =
this.globeResponseRequests.get(toolResponseId) ?? arrivalRequest;
// Later siblings must wait for globe admission before they can mutate state.
// Other tool responses keep their existing concurrency.
let heldForAdmission = false;
while (toolResponseId && this.globeAdmissionGates.has(toolResponseId)) {
// Drop claimed duplicates before every wait, before their keys can expire.
if (
calls.every((call) =>
callDedupeKeys(call).some((key) => this.processedCalls.has(key)),
)
)
return;
heldForAdmission = true;
await this.globeAdmissionGates.get(toolResponseId).promise;
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
return;
}
// A newer typed command superseded the response these calls belong to.
// They are stale intent — dispatching one would let the old turn mutate
// the map after the operator asked for something else.
@@ -394,7 +423,10 @@ export class RealtimeTurns {
// hazard `callDedupeKeys` is written to avoid). So each refused call gets
// a terminal output saying plainly that the turn moved on. No
// `response.create` follows — the deferred typed turn is the single answer.
if (this.isSupersededResponse(toolResponseId)) {
if (
this.isSupersededResponse(toolResponseId) ||
(heldForAdmission && arrivalRequest !== this.globeRequest)
) {
this.pruneProcessedCalls();
for (const call of calls) {
const keys = callDedupeKeys(call);
@@ -429,7 +461,19 @@ export class RealtimeTurns {
if (keys.some((key) => this.processedCalls.has(key))) continue;
keys.forEach((key) => this.processedCalls.set(key, performance.now()));
if (this.globeRefusals.get(toolResponseId)?.blocking) {
this.sendToolOutput(call.call_id || call.id, {
ok: false,
action: call.name,
skipped: true,
error: 'Not run: this response must wait for the Globe View reply.',
});
continue;
}
let result;
let globeAdmissionGate = null;
let globeResetGate = null;
const resultChannel = this.dc;
let radioHandoffEpochAtStart = this.radio.radioHandoffEpoch;
let toolController = null;
@@ -443,6 +487,80 @@ export class RealtimeTurns {
this.narration?.preamble();
try {
const parsedArguments = parseArguments(call.arguments);
if (
toolResponseId &&
call.name === 'move_camera' &&
String(parsedArguments.motion || '').toLowerCase() === 'rotate_globe'
) {
let resolve;
const promise = new Promise((settle) => {
resolve = settle;
});
globeAdmissionGate = { promise, resolve };
this.globeAdmissionGates.set(toolResponseId, globeAdmissionGate);
// OpenAI delivers sibling calls concurrently. A reset dispatched
// first must finish before rotation evaluates the camera's fit.
// Hold later siblings at admission while this rotation waits.
const resets = this.globeResets.get(toolResponseId);
if (
resets ||
globeRequest.failed ||
globeRequest.pending.size ||
arrivalRequest !== this.globeRequest
) {
while (resets?.pending.size || globeRequest.pending.size)
await Promise.all(
[
...new Set([
...(resets?.pending || []),
...globeRequest.pending,
]),
].map((gate) => gate.promise),
);
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
return;
keys.forEach((key) =>
this.processedCalls.set(key, performance.now()),
);
const superseded =
this.isSupersededResponse(toolResponseId) ||
arrivalRequest !== this.globeRequest;
if (superseded || resets?.failed || globeRequest.failed) {
result = {
ok: false,
action: call.name,
reason: 'globe-reset-failed',
skipped: true,
...(superseded ? { superseded: true } : {}),
error: superseded
? 'Superseded by a newer command from the operator — this call was not run.'
: 'Not run: the Globe View reset/rotation sequence failed or was cancelled.',
};
}
}
}
if (toolResponseId && call.name === 'zoom_to_globe') {
let resolve;
const promise = new Promise((settle) => {
resolve = settle;
});
let resets = this.globeResets.get(toolResponseId);
if (!resets) {
resets = { pending: new Set(), failed: false };
this.globeResets.set(toolResponseId, resets);
// Retain settled outcomes for later siblings, without evicting an
// active reset that another response still needs to await.
for (const [id, record] of this.globeResets) {
if (this.globeResets.size <= SUPERSEDED_RESPONSE_MEMORY) break;
if (!record.pending.size && id !== toolResponseId)
this.globeResets.delete(id);
}
}
globeResetGate = { promise, resolve, resets, request: globeRequest };
resets.pending.add(globeResetGate);
globeRequest.pending.add(globeResetGate);
}
const isRadioControlCall = call.name === 'control_radio';
const isRadioVisibilityCall =
call.name === 'set_layer_visibility' &&
@@ -499,7 +617,7 @@ export class RealtimeTurns {
authorityDomain: radioAuthorityDomain,
});
}
result = await (this.actionExecutor || this.runner)(
result ??= await (this.actionExecutor || this.runner)(
call.name,
parsedArguments,
{
@@ -515,7 +633,28 @@ export class RealtimeTurns {
radioHandoffEpochAtStart === this.radio.radioHandoffEpoch),
},
);
if (
globeAdmissionGate &&
this.globeResets.has(toolResponseId) &&
result?.reason === 'globe-view-required'
)
result = { ...result, reason: 'globe-reset-failed' };
if (!this.ownsConversation(eventChannel)) return;
if (
isGlobeViewRefusal(result) &&
toolResponseId &&
arrivalRequest === this.globeRequest &&
!this.isSupersededResponse(toolResponseId)
) {
this.globeRefusals.set(toolResponseId, {
result,
followupQueued: false,
blocking: !result.skipped,
});
while (this.globeRefusals.size > SUPERSEDED_RESPONSE_MEMORY) {
this.globeRefusals.delete(this.globeRefusals.keys().next().value);
}
}
if (result?.ok && result.radioPlaybackRequested) {
const sessionIsCurrent =
this.activeToolAbortControllers.has(toolController) &&
@@ -595,6 +734,23 @@ export class RealtimeTurns {
: {}),
};
} finally {
if (globeResetGate) {
const { resets, request } = globeResetGate;
if (result?.ok !== true || result.cancelled) {
resets.failed = true;
request.failed = true;
}
resets.pending.delete(globeResetGate);
request.pending.delete(globeResetGate);
globeResetGate.resolve();
}
if (globeAdmissionGate) {
if (
this.globeAdmissionGates.get(toolResponseId) === globeAdmissionGate
)
this.globeAdmissionGates.delete(toolResponseId);
globeAdmissionGate.resolve();
}
if (toolController) {
this.activeToolAbortControllers.delete(toolController);
this.radio.releaseTool(toolController);
@@ -671,13 +827,38 @@ export class RealtimeTurns {
});
}
if (!this.ownsConversation(eventChannel)) return;
// Keep the Radio handoff wording authoritative even when another tool
// result follows Radio in the same multi-intent response.
this.queueResponseCreate(
responseInstructionForToolResult(
this.radio.pendingRadioPlaybackResult || lastResult,
),
);
const refusal = this.globeRefusals.get(toolResponseId);
// A newer turn may arrive during admission or the viewport-image await.
// Preserve its reply; neither the refusal map nor lastResult may revive
// an old refusal follow-up after supersession.
if (
(this.isSupersededResponse(toolResponseId) ||
arrivalRequest !== this.globeRequest) &&
(refusal || isGlobeViewRefusal(lastResult) || lastResult?.superseded)
) {
// Complete an idle turn below, but leave a newer running tool's status
// authoritative while suppressing this obsolete refusal follow-up.
if (this.activeToolAbortControllers.size) return;
} else if (refusal) {
// Even an earlier sibling finishing late cannot replace the reset instruction.
if (!refusal.followupQueued) {
refusal.followupQueued = true;
this.queueResponseCreate({
instructions: responseInstructionForToolResult(refusal.result),
tool_choice: 'none',
});
}
} else {
// Keep Radio handoff wording authoritative for ordinary multi-tool replies.
const followupResult =
this.radio.pendingRadioPlaybackResult || lastResult;
const instructions = responseInstructionForToolResult(followupResult);
this.queueResponseCreate(
isGlobeViewRefusal(followupResult)
? { instructions, tool_choice: 'none' }
: instructions,
);
}
}
this.setStatus('listening', t('Ask or command'));
}
@@ -966,6 +1147,22 @@ export class RealtimeTurns {
this.userTurnPending = false;
this.activeResponseId =
payload.response?.id || payload.response_id || null;
// Late calls retain only their globe reset/request ownership. Ordinary
// tool admission and concurrency continue to use their existing policy.
if (
this.activeResponseId &&
!this.globeResponseRequests.has(this.activeResponseId)
) {
this.globeResponseRequests.set(
this.activeResponseId,
this.globeRequest,
);
while (this.globeResponseRequests.size > SUPERSEDED_RESPONSE_MEMORY)
this.globeResponseRequests.delete(
this.globeResponseRequests.keys().next().value,
);
}
this.setVoiceSpeaker('ai');
return;
}
@@ -1045,7 +1242,8 @@ export class RealtimeTurns {
const sent = this.sendRealtimeEvent(
{
type: 'response.create',
response: { instructions },
response:
typeof instructions === 'string' ? { instructions } : instructions,
},
'client.response_create.tool_followup',
);
@@ -1057,6 +1255,20 @@ export class RealtimeTurns {
controller.abort();
this.activeToolAbortControllers.clear();
this.radio.clearTools();
this.releaseGlobeResets();
}
releaseGlobeResets() {
for (const resets of this.globeResets.values()) {
if (!resets.pending.size) continue;
resets.failed = true;
for (const gate of resets.pending) {
gate.request.failed = true;
gate.request.pending.delete(gate);
gate.resolve();
}
resets.pending.clear();
}
}
reset() {
@@ -1068,6 +1280,13 @@ export class RealtimeTurns {
this.pendingUserTextResponse = false;
this.activeResponseId = null;
this.supersededResponseIds.clear();
this.globeRefusals.clear();
for (const gate of this.globeAdmissionGates.values()) gate.resolve();
this.globeAdmissionGates.clear();
this.releaseGlobeResets();
this.globeResets.clear();
this.globeResponseRequests.clear();
this.globeRequest = { failed: false, pending: new Set() };
this.narration?.cancel();
this.assistantAudioPlaying = false;
this.speechResponseIds.clear();
+8 -8
View File
@@ -5,17 +5,17 @@ import {
import { cameraPoseKey } from './pointerContext.js';
/** The live viewer's camera pose identity (Cesium-free read). */
export function readViewerCameraKey() {
function readViewerCameraKey() {
return cameraPoseKey(globalThis.window?.__godsEyeView?.viewer);
}
// Viewport-screenshot size guards (M13). The old code clamped WIDTH only, so a
// tall portrait window produced an oversized capture whose dc.send could throw.
// Cap total pixels (clamps both dimensions) and drop the image entirely if the
// encoded data URL is still too big for the data channel.
export const VIEWPORT_MAX_PIXELS = 1200 * 900;
const VIEWPORT_MAX_PIXELS = 1200 * 900;
// ~1.08 MP, matches the old 1200px-wide landscape budget
export const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
export async function captureViewportImage() {
const viewer = window.__godsEyeView?.viewer;
@@ -66,11 +66,11 @@ export async function captureViewportImage() {
}
/** Side of the point-and-ask crop, in CSS pixels, and its output size. */
export const POINTER_CROP_SIZE = 512;
const POINTER_CROP_SIZE = 512;
/** A pointer snapshot older than this no longer matches the frame. */
export const POINTER_CROP_MAX_AGE_MS = 20000;
const POINTER_CROP_MAX_AGE_MS = 20000;
/** View scales where a crop shows buildings and terrain, not a smear. */
export const POINTER_CROP_SCALES = new Set(['local', 'city']);
const POINTER_CROP_SCALES = new Set(['local', 'city']);
/**
* Square source rectangle around a canvas point, clamped inside the canvas,
@@ -128,7 +128,7 @@ export function pointerCropRect({
* @returns {Promise<{dataUrl: string, rect: object}|null>} JPEG data URL and
* the crop's normalized bounds in the full canvas, or null.
*/
export async function capturePointerCrop(screenPx) {
async function capturePointerCrop(screenPx) {
const viewer = window.__godsEyeView?.viewer;
const source = viewer?.scene?.canvas;
if (!source?.width || !source?.height || !screenPx) return null;
@@ -259,7 +259,7 @@ export async function renderFreshCesiumFrame(viewer) {
}
}
export function isNearlyBlackFrame(ctx, width, height) {
function isNearlyBlackFrame(ctx, width, height) {
const sampleWidth = Math.min(48, width);
const sampleHeight = Math.min(32, height);
if (!sampleWidth || !sampleHeight) return true;
+1 -1
View File
@@ -59,7 +59,7 @@ function notesOf(...values) {
];
}
export function analystNounFor(layerKeys) {
function analystNounFor(layerKeys) {
const keys = [...new Set(layerKeys.filter(Boolean))];
if (keys.length !== 1) return 'results';
const key = keys[0];
+13 -1
View File
@@ -115,6 +115,18 @@ function shownSources(source) {
return source ? [{ label: sourceLabel(source) }] : [];
}
// Entity labels that layers register as English data (firms/, alpr/); keep in
// step with the producers. Other names (callsigns, places) show as they came.
const ENTITY_LABELS = new Set([msg('ALPR camera')]);
const FIRE_LABEL = /^Fire · FRP (\S+) MW$/;
/** An entity label as the card shows it, in the active language. */
export function shownEntityLabel(label) {
if (ENTITY_LABELS.has(label)) return t(label);
const fire = FIRE_LABEL.exec(label ?? '');
return fire ? t('Fire · FRP {frp} MW', { frp: fire[1] }) : label;
}
/*
* Tool errors stay English in the result (the model reads them). The card
* translates the known ones; the English must match gevActions exactly.
@@ -592,7 +604,7 @@ function getEntityContext(result) {
return {
say: null,
display: {
title,
title: spokenLabel(shownEntityLabel(title)),
lines: [
selected.layerName
? shownLayerName(selected.layerName, selected.layerId)
+1 -1
View File
@@ -7,7 +7,7 @@
* Phrases that add length without adding an answer. Tool numbers are spoken
* verbatim, so approximations are hedges too.
*/
export const HEDGE_PHRASES = Object.freeze([
const HEDGE_PHRASES = Object.freeze([
'keep in mind',
"it's worth noting",
'it is worth noting',
+1 -1
View File
@@ -1,7 +1,7 @@
import { countWords, findHedges } from './speechLint.js';
/** Quiet period after the last audio or response before a turn is closed. */
export const TURN_SETTLE_MS = 2500;
const TURN_SETTLE_MS = 2500;
/**
* Per-turn latency and speech spans for one voice session.
+1 -1
View File
@@ -5,7 +5,7 @@ import {
} from './voiceCardPresentation.js';
/** Hide an idle card this long after its last update. */
export const VOICE_CARD_IDLE_MS = 14000;
const VOICE_CARD_IDLE_MS = 14000;
const ELEMENT_IDS = Object.freeze({
card: 'gev-voice-card',
+56 -2
View File
@@ -3,8 +3,16 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { setLocale } from '../i18n/index.js';
import { attachVoiceResult, planStepLabel } from './speech.js';
import { voiceCardView } from './voiceCardPresentation.js';
import {
attachVoiceResult,
planStepLabel,
shownEntityLabel,
} from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
for (const locale of ['ru', 'es']) {
test(`voice card is in ${locale} for a layer switch`, (t) => {
@@ -86,3 +94,49 @@ test('en: layer switch, distance and plan labels in English', () => {
'Earthquakes (24h) on',
);
});
// FIRMS and ALPR register English entity labels: the model reads them as data
// (get_entity_context, pointer_context), the card shows them translated.
const ENTITY_LABELS = {
ru: { fire: 'Пожар · FRP 12 МВт', alpr: 'Камера ALPR' },
es: { fire: 'Incendio · FRP 12 MW', alpr: 'Cámara ALPR' },
};
for (const [locale, shown] of Object.entries(ENTITY_LABELS)) {
test(`entity labels stay English in the result and show in ${locale}`, (t) => {
t.after(() => setLocale('en'));
setLocale(locale);
const out = attachVoiceResult('get_entity_context', {
ok: true,
selected: {
id: 'fire-1',
name: 'Fire · FRP 12 MW',
layerId: 'local-firms',
layerName: 'NASA FIRMS',
},
});
assert.equal(out.selected.name, 'Fire · FRP 12 MW');
assert.equal(out.display.title, shown.fire);
const state = reduceVoiceCard(initialVoiceCardState(), {
type: 'pointer',
pointer: { kind: 'this', label: 'ALPR camera' },
});
assert.equal(state.pointer.label, 'ALPR camera');
assert.equal(voiceCardView(state).pointer.text, shown.alpr);
assert.equal(state.announce, `This: ${shown.alpr}`);
});
}
test('only the registered entity labels are translated', () => {
assert.equal(shownEntityLabel('Fire · FRP 4.9 MW'), 'Fire · FRP 4.9 MW');
assert.equal(shownEntityLabel('ALPR camera'), 'ALPR camera');
setLocale('ru');
try {
assert.equal(shownEntityLabel('Fire · FRP 4.9 MW'), 'Пожар · FRP 4.9 МВт');
assert.equal(shownEntityLabel('UPS793'), 'UPS793');
assert.equal(shownEntityLabel('Fire Station 5'), 'Fire Station 5');
} finally {
setLocale('en');
}
});
+8 -3
View File
@@ -1,4 +1,9 @@
import { planStepLabel, progressStepLabel, spokenLabel } from './speech.js';
import {
planStepLabel,
progressStepLabel,
shownEntityLabel,
spokenLabel,
} from './speech.js';
import { presentResult } from './resultDisplay.js';
import { MAX_DISPLAYED_REFERENTS, normalizeReferents } from './referents.js';
@@ -132,7 +137,7 @@ export function reduceVoiceCard(state, event) {
return reveal({
...state,
pointer,
announce: `${kind}: ${spokenLabel(pointer.label, 32)}`,
announce: `${kind}: ${spokenLabel(shownEntityLabel(pointer.label), 32)}`,
});
}
case 'transcript': {
@@ -288,7 +293,7 @@ export function voiceCardView(state) {
pointer: {
visible: Boolean(state.pointer),
kind: state.pointer?.kind === 'here' ? t('HERE') : t('THIS'),
text: spokenLabel(state.pointer?.label || '', 32),
text: spokenLabel(shownEntityLabel(state.pointer?.label || ''), 32),
},
assistant: {
visible: Boolean(state.assistant),