mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-05 19:04:02 +00:00
Merge current main into S2 GhostMaps and preserve updated import and voice behavior
This commit is contained in:
commit
4da002bd42
515 files changed
+17681
-2202
No files matched your search
@@ -729,7 +729,7 @@ const schemas = [
|
||||
properties: {
|
||||
motion: {
|
||||
type: 'string',
|
||||
enum: ['orbit', 'pan', 'tilt', 'rotate', 'stop'],
|
||||
enum: ['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe', 'stop'],
|
||||
},
|
||||
direction: {
|
||||
type: 'string',
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
@@ -38,8 +39,11 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
|
||||
// requested radius and center authority in descriptions only.
|
||||
// Street Level and GNSS extend the manifest; tracked and analyst aircraft describe broadcasts.
|
||||
// Air quality (ECCC AQHI) extends the manifest.
|
||||
// S2 GhostMaps extends the manifest.
|
||||
'64a7ae9719cbf53f2c5b57739c1e84e8b60dc4ede040decd48a9c06d1ccbcbca',
|
||||
// Globe rotation adds one motion; visual keyhole fit and explicit
|
||||
// reset guidance/defaults update only move_camera and mode descriptions.
|
||||
// Restore base descriptions; append only globe-rotation defaults and fit guidance.
|
||||
// S2 GhostMaps adds its layer id to the generated layer enums.
|
||||
'1f95d78074559217872ccbdfa57d4363264ea88c1a22b259e6ebba3c8189b806',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -107,6 +111,17 @@ test('all legacy action arguments are byte-identical after removing the delibera
|
||||
delete property('show_data_layers_menu').layerId.enum;
|
||||
delete property('get_entity_context').layerId.enum;
|
||||
delete property('analyst_query').layers.items.enum;
|
||||
// Earth-centered rotation is additive; local camera yaw keeps its contract.
|
||||
const motion = property('move_camera').motion;
|
||||
assert.deepEqual(motion.enum, [
|
||||
'orbit',
|
||||
'pan',
|
||||
'tilt',
|
||||
'rotate',
|
||||
'rotate_globe',
|
||||
'stop',
|
||||
]);
|
||||
motion.enum = motion.enum.filter((value) => value !== 'rotate_globe');
|
||||
const panels = property('set_panel_open').panelId;
|
||||
panels.enum = panels.enum.filter(
|
||||
(id) => !['weather-panel', 'recent-imagery-panel'].includes(id),
|
||||
@@ -146,6 +161,24 @@ test('all legacy action arguments are byte-identical after removing the delibera
|
||||
);
|
||||
});
|
||||
|
||||
test('globe rotation is an additive camera motion with the existing control arguments', () => {
|
||||
const schema = GEV_ACTION_SCHEMAS.find(
|
||||
(tool) => tool.name === 'move_camera',
|
||||
).parameters;
|
||||
for (const motion of ['rotate_globe', 'rotate', 'orbit', 'pan', 'tilt']) {
|
||||
assert.deepEqual(
|
||||
validateArgs(schema, {
|
||||
motion,
|
||||
direction: 'right',
|
||||
speed: 'slow',
|
||||
mode: 'continuous',
|
||||
}),
|
||||
[],
|
||||
);
|
||||
}
|
||||
assert.ok(validateArgs(schema, { motion: 'spin_camera' }).length > 0);
|
||||
});
|
||||
|
||||
test('track_entity accepts a referent alone but rejects an empty target', () => {
|
||||
const schema = GEV_ACTION_SCHEMAS.find(
|
||||
(tool) => tool.name === 'track_entity',
|
||||
@@ -184,3 +217,15 @@ test('the benchmark still rejects an empty track_entity although the model-facin
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test('globe reset guidance requires a separate explicit command after refusal', () => {
|
||||
const paragraph = realtimeInstructions()
|
||||
.split('\n')
|
||||
.find((line) => line.includes('globe-view-required'));
|
||||
assert.ok(paragraph);
|
||||
assert.match(paragraph, /calls only move_camera motion rotate_globe/);
|
||||
assert.match(paragraph, /explicit user Reset\/Show Globe View/);
|
||||
assert.match(paragraph, /never from your own plan/);
|
||||
assert.match(paragraph, /bare yes\/ok/);
|
||||
assert.match(paragraph, /Do not ask a question/);
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
export const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']);
|
||||
export const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key';
|
||||
const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']);
|
||||
const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key';
|
||||
export const CLOUD_VOICE_AUTH_STORAGE_KEY = 'godsEyeView.voice.cloudAuth';
|
||||
|
||||
function storageRef(storage) {
|
||||
@@ -11,7 +11,7 @@ function storageRef(storage) {
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeCloudVoiceAuthMode(mode) {
|
||||
function normalizeCloudVoiceAuthMode(mode) {
|
||||
const normalized = String(mode || '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@
|
||||
|
||||
import { finiteOrNull } from './pointerContext.js';
|
||||
|
||||
export const POINTER_SENTINEL = 'pointer';
|
||||
const POINTER_SENTINEL = 'pointer';
|
||||
|
||||
const TRACKABLE_LAYERS = new Set([
|
||||
'flights',
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
shouldSendViewportImage,
|
||||
shouldStopVoiceAfterRadioTool,
|
||||
responseInstructionForToolResult,
|
||||
isGlobeViewRefusal,
|
||||
} from './realtimeProtocol.js';
|
||||
|
||||
const SOCKET_PATH =
|
||||
@@ -182,6 +183,8 @@ export class GeminiSessionController {
|
||||
this.model = null;
|
||||
this.epoch = 0;
|
||||
this.turn = 0;
|
||||
this.globeToolsBlocked = false;
|
||||
this.globeResetFailed = false;
|
||||
this.inputGeneration = 0;
|
||||
this.turnAbort = new AbortController();
|
||||
this.disposed = false;
|
||||
@@ -659,6 +662,8 @@ export class GeminiSessionController {
|
||||
return;
|
||||
if (!pushToTalk || activityCaptureId === metadata.captureId) return;
|
||||
this.captureOwner.admitted = true;
|
||||
this.globeToolsBlocked = false;
|
||||
this.globeResetFailed = false;
|
||||
if (
|
||||
!this.interrupt('push-to-talk', {
|
||||
keepPlayback: !this.pttPlaybackSuppressed,
|
||||
@@ -1294,6 +1299,8 @@ export class GeminiSessionController {
|
||||
this.replyPending = false;
|
||||
if (!this.interrupt()) return;
|
||||
} else if (!this.retiredReplyPending && !this.interrupt()) return;
|
||||
this.globeToolsBlocked = false;
|
||||
this.globeResetFailed = false;
|
||||
// Local activity ownership is only for expiry liveness, not tools.
|
||||
// Assign after retiring observed output so its boundary cannot clear
|
||||
// a newer detected activity that has not received a reply yet.
|
||||
@@ -1573,6 +1580,7 @@ export class GeminiSessionController {
|
||||
.then(async () => {
|
||||
if (!this.owns(epoch) || turn !== this.turn) return;
|
||||
if (this.cancelledCalls.has(call.id)) {
|
||||
if (call.name === 'zoom_to_globe') this.globeResetFailed = true;
|
||||
if (this.tools.get(call.id) === action) this.tools.delete(call.id);
|
||||
this.pendingTools--;
|
||||
if (!this.pendingTools) {
|
||||
@@ -1618,8 +1626,28 @@ export class GeminiSessionController {
|
||||
(authority !== 'playback' ||
|
||||
handoffEpoch === this.radio.radioHandoffEpoch);
|
||||
let result;
|
||||
let skippedGlobeRotation = false;
|
||||
let blockedGlobeTool = false;
|
||||
try {
|
||||
if (!actionCurrent()) throw cancelled();
|
||||
// Live has no response-scoped tool_choice. Recheck here so queued siblings
|
||||
// cannot reset after a rotation refusal.
|
||||
if (this.globeToolsBlocked) {
|
||||
blockedGlobeTool = true;
|
||||
throw new Error(
|
||||
'Wait for fresh user input before calling another tool.',
|
||||
);
|
||||
}
|
||||
if (
|
||||
this.globeResetFailed &&
|
||||
call.name === 'move_camera' &&
|
||||
String(args.motion || '').toLowerCase() === 'rotate_globe'
|
||||
) {
|
||||
skippedGlobeRotation = true;
|
||||
throw new Error(
|
||||
'Not run: the Globe View reset/rotation sequence failed or was cancelled.',
|
||||
);
|
||||
}
|
||||
if (stronger)
|
||||
reservation = this.radio.reserveRadioToolHandoff({ abortScope });
|
||||
handoffEpoch = this.radio.radioHandoffEpoch;
|
||||
@@ -1689,7 +1717,12 @@ export class GeminiSessionController {
|
||||
} catch (error) {
|
||||
result = {
|
||||
ok: false,
|
||||
tool: call.name,
|
||||
...(skippedGlobeRotation
|
||||
? { action: call.name, reason: 'globe-reset-failed', skipped: true }
|
||||
: {
|
||||
tool: call.name,
|
||||
...(blockedGlobeTool ? { skipped: true } : {}),
|
||||
}),
|
||||
...(radioVisibility ? { layerId: 'radio' } : {}),
|
||||
error:
|
||||
error?.name === 'AbortError'
|
||||
@@ -1716,6 +1749,11 @@ export class GeminiSessionController {
|
||||
this.noteRadioStrongerCommit(action, abortScope, turn);
|
||||
}
|
||||
if (!this.owns(epoch) || turn !== this.turn) return;
|
||||
if (
|
||||
call.name === 'zoom_to_globe' &&
|
||||
(result?.ok !== true || result.cancelled)
|
||||
)
|
||||
this.globeResetFailed = true;
|
||||
const playbackObsolete =
|
||||
authority === 'playback' && handoffEpoch !== this.radio.radioHandoffEpoch;
|
||||
if (radioControl || radioVisibility) {
|
||||
@@ -1744,6 +1782,8 @@ export class GeminiSessionController {
|
||||
this.stopAfterReply = true;
|
||||
// Native call cancellation suppresses its output, without canceling siblings.
|
||||
if (!this.cancelledCalls.has(call.id)) {
|
||||
if (isGlobeViewRefusal(result) && !skippedGlobeRotation)
|
||||
this.globeToolsBlocked = true;
|
||||
this.postToolOutput = false;
|
||||
this.modelComplete = false;
|
||||
this.replyPending = true;
|
||||
@@ -1751,7 +1791,9 @@ export class GeminiSessionController {
|
||||
const mapEvents = mapSnapshot.map((text) => JSON.parse(text));
|
||||
const response = {
|
||||
...(result ?? { ok: false, error: 'Command returned no result' }),
|
||||
responseGuidance: responseInstructionForToolResult(result),
|
||||
responseGuidance: blockedGlobeTool
|
||||
? 'That action did not run. Wait for fresh user input; do not claim any change or call another tool.'
|
||||
: responseInstructionForToolResult(result),
|
||||
};
|
||||
this.send(
|
||||
{
|
||||
@@ -1902,6 +1944,8 @@ export class GeminiSessionController {
|
||||
{
|
||||
onSent: () => {
|
||||
if (!owned() || this.typedIntent !== typedIntent) return;
|
||||
this.globeToolsBlocked = false;
|
||||
this.globeResetFailed = false;
|
||||
this.replyPending = true;
|
||||
this.bindTextIntent(typedIntent);
|
||||
if (!owned() || this.typedIntent !== typedIntent) return;
|
||||
@@ -2455,6 +2499,8 @@ export class GeminiSessionController {
|
||||
this.cancelledCalls.clear();
|
||||
this.pendingMapEvents = [];
|
||||
this.userTurnPending = false;
|
||||
this.globeToolsBlocked = false;
|
||||
this.globeResetFailed = false;
|
||||
this.replyPending = false;
|
||||
this.unrequestedReply = false;
|
||||
this.automaticTurnOpen = false;
|
||||
|
||||
@@ -3398,3 +3398,215 @@ test('a narration callback that submits a successor keeps the old packet tools f
|
||||
assert.deepEqual(h.calls, []);
|
||||
h.c.stop();
|
||||
});
|
||||
|
||||
for (const reply of ['typed', 'push-to-talk', 'open-mic']) {
|
||||
test(`globe refusal blocks queued and later tools until fresh ${reply} input`, async () => {
|
||||
const names = [];
|
||||
const h = harness({
|
||||
runAction: async (name) => {
|
||||
names.push(name);
|
||||
return name === 'move_camera'
|
||||
? { ok: false, action: 'move_camera', reason: 'globe-view-required' }
|
||||
: { ok: true, action: name };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start({ pushToTalk: reply === 'push-to-talk' });
|
||||
establishToolInput(h, 'rotate the globe');
|
||||
const ws = h.sockets[0];
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: 'rotate',
|
||||
name: 'move_camera',
|
||||
args: { motion: 'rotate_globe' },
|
||||
},
|
||||
{ id: 'queued-reset', name: 'zoom_to_globe', args: {} },
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(names, ['move_camera']);
|
||||
const blocked = ws.sent
|
||||
.flatMap((entry) => entry.toolResponse?.functionResponses || [])
|
||||
.find((call) => call.id === 'queued-reset');
|
||||
assert.equal(blocked.response.ok, false);
|
||||
assert.match(blocked.response.error, /fresh user input/i);
|
||||
assert.equal(blocked.response.skipped, true);
|
||||
assert.match(blocked.response.responseGuidance, /did not run/i);
|
||||
assert.doesNotMatch(
|
||||
blocked.response.responseGuidance,
|
||||
/confirm.*completed/i,
|
||||
);
|
||||
requestTool(ws, 'repeated-rotation', 'move_camera', {
|
||||
motion: 'rotate_globe',
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(names, ['move_camera']);
|
||||
const repeated = toolResponses(ws).find(
|
||||
(call) => call.id === 'repeated-rotation',
|
||||
).response;
|
||||
assert.equal(repeated.ok, false);
|
||||
assert.equal(repeated.skipped, true);
|
||||
assert.match(repeated.responseGuidance, /did not run/i);
|
||||
assert.doesNotMatch(
|
||||
repeated.responseGuidance,
|
||||
/confirm.*completed/i,
|
||||
);
|
||||
|
||||
if (reply === 'typed') {
|
||||
const stimuli = [
|
||||
() => {},
|
||||
() => ws.message({ serverContent: { interrupted: true } }),
|
||||
() =>
|
||||
ws.message({
|
||||
serverContent: {
|
||||
inputTranscription: { text: 'reset to globe view' },
|
||||
},
|
||||
}),
|
||||
() => {
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
},
|
||||
];
|
||||
for (const [i, stimulus] of stimuli.entries()) {
|
||||
stimulus();
|
||||
requestTool(ws, `unrequested-reset-${i}`, 'zoom_to_globe', {});
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(names, ['move_camera']);
|
||||
}
|
||||
}
|
||||
// Finish the refusal reply before admitting the actual next user input.
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
if (reply === 'typed')
|
||||
assert.equal(h.c.sendTextCommand('reset to globe view'), true);
|
||||
else if (reply === 'push-to-talk') {
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
h.c.input.spaceKeyHeld = true;
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
} else ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
requestTool(ws, 'accepted-reset', 'zoom_to_globe', {});
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(names, ['move_camera', 'zoom_to_globe']);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('a cancelled globe reset skips queued rotation, preserves Stop and recovers on fresh input', async () => {
|
||||
const calls = [];
|
||||
let resets = 0;
|
||||
const h = harness({
|
||||
runAction: async (name, args) => {
|
||||
calls.push(name === 'move_camera' ? args.motion : name);
|
||||
return name === 'zoom_to_globe' && resets++ === 0
|
||||
? { ok: false, action: name, cancelled: true }
|
||||
: { ok: true, action: name };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
establishToolInput(h, 'reset to globe view and rotate');
|
||||
const ws = h.sockets[0];
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{ id: 'reset-failed', name: 'zoom_to_globe', args: {} },
|
||||
{
|
||||
id: 'rotate-skipped',
|
||||
name: 'move_camera',
|
||||
args: { motion: 'rotate_globe' },
|
||||
},
|
||||
{ id: 'stop-allowed', name: 'move_camera', args: { motion: 'stop' } },
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(calls, ['zoom_to_globe', 'stop']);
|
||||
const skipped = toolResponses(ws).find(
|
||||
(call) => call.id === 'rotate-skipped',
|
||||
).response;
|
||||
assert.equal(skipped.ok, false);
|
||||
assert.equal(skipped.reason, 'globe-reset-failed');
|
||||
assert.equal(skipped.skipped, true);
|
||||
assert.equal(
|
||||
toolResponses(ws).find((call) => call.id === 'stop-allowed').response.ok,
|
||||
true,
|
||||
);
|
||||
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
establishToolInput(h, 'reset to globe view and rotate');
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{ id: 'reset-success', name: 'zoom_to_globe', args: {} },
|
||||
{
|
||||
id: 'rotate-success',
|
||||
name: 'move_camera',
|
||||
args: { motion: 'rotate_globe' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(calls, [
|
||||
'zoom_to_globe',
|
||||
'stop',
|
||||
'zoom_to_globe',
|
||||
'rotate_globe',
|
||||
]);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('cancelling a queued globe reset also skips its dependent rotation', async () => {
|
||||
const blocker = deferred();
|
||||
const calls = [];
|
||||
const h = harness({
|
||||
runAction: async (name, args) => {
|
||||
calls.push(name === 'move_camera' ? args.motion : name);
|
||||
return args.motion === 'pan'
|
||||
? blocker.promise
|
||||
: { ok: true, action: name };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
establishToolInput(h, 'pan, reset to globe view and rotate');
|
||||
const ws = h.sockets[0];
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: 'blocker',
|
||||
name: 'move_camera',
|
||||
args: { motion: 'pan', direction: 'left' },
|
||||
},
|
||||
{ id: 'reset-cancelled', name: 'zoom_to_globe', args: {} },
|
||||
{
|
||||
id: 'rotate-after-cancel',
|
||||
name: 'move_camera',
|
||||
args: { motion: 'rotate_globe' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
await tick();
|
||||
ws.message({ toolCallCancellation: { ids: ['reset-cancelled'] } });
|
||||
blocker.resolve({ ok: true, action: 'move_camera' });
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(calls, ['pan']);
|
||||
const skipped = toolResponses(ws).find(
|
||||
(call) => call.id === 'rotate-after-cancel',
|
||||
).response;
|
||||
assert.equal(skipped.reason, 'globe-reset-failed');
|
||||
assert.equal(skipped.skipped, true);
|
||||
} finally {
|
||||
blocker.resolve({ ok: true });
|
||||
h.c.stop();
|
||||
}
|
||||
});
|
||||
+5
-66
@@ -1,3 +1,4 @@
|
||||
import { interruptGlobeRotation } from '../cameraVerbs.js';
|
||||
import {
|
||||
layerSnapshot,
|
||||
feedProvenanceEnvelope,
|
||||
@@ -262,7 +263,9 @@ export function createGevActionRunner({
|
||||
}),
|
||||
));
|
||||
installViewTargetPrewarm(viewer);
|
||||
initCameraVerbs(viewer, getViewTargetCartesian);
|
||||
initCameraVerbs(viewer, getViewTargetCartesian, {
|
||||
isCockpitActive: () => Boolean(styleManager?.getCockpitState?.()?.active),
|
||||
});
|
||||
// Nested calls (select_nearest_aircraft composing layer + flight) use the
|
||||
// raw runner; only the outer call gets a voice envelope.
|
||||
const runGevAction = async function runGevAction(
|
||||
@@ -298,6 +301,7 @@ export function createGevActionRunner({
|
||||
// Zoom during an active orbit adjusts the orbit RADIUS (spiral in/out) —
|
||||
// a straight camera move would be snapped back by the per-frame lookAt.
|
||||
if (name === 'adjust_camera_zoom') {
|
||||
interruptGlobeRotation('voice-zoom-intent');
|
||||
const zoomOut = String(args.direction || '').toLowerCase() === 'out';
|
||||
const amt = String(args.amount || 'medium').toLowerCase();
|
||||
const factor = { little: 1.25, medium: 1.6, lot: 2.4 }[amt] || 1.6;
|
||||
@@ -4369,46 +4373,6 @@ function dominantValue(values) {
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeEntity(viewer, entity, { includeProperties = false } = {}) {
|
||||
const now = Cesium.JulianDate.now();
|
||||
if (entity.__gevContextId) {
|
||||
const store = window.__gevContextStore;
|
||||
const record = store?.entities?.get(entity.__gevContextId);
|
||||
if (record) return summarizeContextRecord(record, { includeProperties });
|
||||
}
|
||||
const props = propertyObject(entity);
|
||||
const layerId = entity.__localLayerId || props.layerId || null;
|
||||
const tags = props.tags || {};
|
||||
const label = cleanText(
|
||||
props.name ||
|
||||
tags.name ||
|
||||
tags['name:en'] ||
|
||||
tags.official_name ||
|
||||
tags.operator ||
|
||||
props.operator ||
|
||||
entity.name ||
|
||||
layerTitle(layerId),
|
||||
);
|
||||
const position =
|
||||
entity.__localBaseCartesian ||
|
||||
entity.position?.getValue?.(now) ||
|
||||
polygonCenter(entity, now);
|
||||
const carto = position ? Cesium.Cartographic.fromCartesian(position) : null;
|
||||
return {
|
||||
id: String(entity.id || ''),
|
||||
name: label || layerTitle(layerId),
|
||||
layerId,
|
||||
layerName: layerTitle(layerId),
|
||||
latitude: carto
|
||||
? Number(Cesium.Math.toDegrees(carto.latitude).toFixed(6))
|
||||
: null,
|
||||
longitude: carto
|
||||
? Number(Cesium.Math.toDegrees(carto.longitude).toFixed(6))
|
||||
: null,
|
||||
properties: includeProperties ? compactProperties(props) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeContextRecord(record, { includeProperties = false } = {}) {
|
||||
return {
|
||||
id: String(record.id || ''),
|
||||
@@ -4427,31 +4391,6 @@ function summarizeContextRecord(record, { includeProperties = false } = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function polygonCenter(entity, now) {
|
||||
const hierarchy = entity.polygon?.hierarchy?.getValue?.(now);
|
||||
const positions = hierarchy?.positions;
|
||||
if (!positions?.length) return null;
|
||||
return Cesium.BoundingSphere.fromPoints(positions).center;
|
||||
}
|
||||
|
||||
function propertyObject(entity) {
|
||||
const raw = entity?.properties?.getValue?.(Cesium.JulianDate.now()) || {};
|
||||
return unwrapProperties(raw);
|
||||
}
|
||||
|
||||
function unwrapProperties(value) {
|
||||
if (!value || typeof value !== 'object') return value;
|
||||
if (Array.isArray(value)) return value.map(unwrapProperties);
|
||||
const out = {};
|
||||
for (const [key, entry] of Object.entries(value)) {
|
||||
out[key] =
|
||||
entry && typeof entry.getValue === 'function'
|
||||
? unwrapProperties(entry.getValue(Cesium.JulianDate.now()))
|
||||
: unwrapProperties(entry);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function compactProperties(props) {
|
||||
const preferredKeys = [
|
||||
'name',
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -4,7 +4,7 @@ import { progressLine } from './speech.js';
|
||||
* Timing policy for filling silence while tools run. Tuned against the turn
|
||||
* spans' `max_silence_ms`.
|
||||
*/
|
||||
export const NARRATION_TIMING = Object.freeze({
|
||||
const NARRATION_TIMING = Object.freeze({
|
||||
/** Start the earcon when nothing is audible this long after end of speech. */
|
||||
earconAfterMs: 700,
|
||||
/** First spoken progress line when a tool is still running after this. */
|
||||
|
||||
@@ -20,15 +20,15 @@
|
||||
*/
|
||||
|
||||
/** Movement below this many CSS pixels is still the same dwell. */
|
||||
export const POINTER_DWELL_PX = 12;
|
||||
const POINTER_DWELL_PX = 12;
|
||||
/** Stillness that makes a dwell point. */
|
||||
export const POINTER_DWELL_MS = 250;
|
||||
/** A resting pointer stays a referent this long after it last moved. */
|
||||
export const POINTER_FRESH_MS = 30000;
|
||||
/** Pixels from the canvas edge that count as "parked", never a referent. */
|
||||
export const POINTER_EDGE_PX = 8;
|
||||
const POINTER_EDGE_PX = 8;
|
||||
/** A dwell pick is reused by a snapshot only while this recent (contacts move). */
|
||||
export const POINTER_DWELL_REUSE_MS = 1000;
|
||||
const POINTER_DWELL_REUSE_MS = 1000;
|
||||
/**
|
||||
* Open mic has no gesture, so the pointer only counts when it moved or
|
||||
* settled on the map this recently: a cursor resting there for minutes is
|
||||
@@ -76,7 +76,7 @@ export function cameraPoseKey(viewer) {
|
||||
}
|
||||
|
||||
/** Bound and de-control a label before it goes to the model or the card. */
|
||||
export function pointerLabel(value, max = LABEL_MAX) {
|
||||
function pointerLabel(value, max = LABEL_MAX) {
|
||||
const clean = String(value ?? '')
|
||||
.replace(/[\p{Cc}\p{Cf}]+/gu, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
|
||||
@@ -4,7 +4,7 @@ import { cameraPoseKey } from './pointerContext.js';
|
||||
import * as defaultAnnotationResolver from '../annotations/annotationResolver.js';
|
||||
|
||||
/** "Around here" radius as a share of camera height, clamped to sane bounds. */
|
||||
export function pointerRadiusKm(cameraHeightM) {
|
||||
function pointerRadiusKm(cameraHeightM) {
|
||||
const heightKm = Number(cameraHeightM) / 1000;
|
||||
if (!Number.isFinite(heightKm) || heightKm <= 0) return 50;
|
||||
return Math.round(Math.min(400, Math.max(5, heightKm * 0.75)));
|
||||
|
||||
@@ -14,7 +14,7 @@ export const RETICLE_MAX_MS = 10000;
|
||||
export const RETICLE_LINGER_MS = 1500;
|
||||
|
||||
/** Tools that move the camera away from the pointed spot. */
|
||||
export const CAMERA_TOOLS = new Set([
|
||||
const CAMERA_TOOLS = new Set([
|
||||
'fly_to_location',
|
||||
'track_entity',
|
||||
'zoom_to_globe',
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
export const ERROR_LOG_LIMIT = 30;
|
||||
const ERROR_LOG_LIMIT = 30;
|
||||
|
||||
export const ERROR_STORAGE_KEY = 'gev-realtime-errors';
|
||||
const ERROR_STORAGE_KEY = 'gev-realtime-errors';
|
||||
|
||||
export const DEBUG_LOG_URL = '/api/realtime/debug-log';
|
||||
const DEBUG_LOG_URL = '/api/realtime/debug-log';
|
||||
|
||||
export function createDebugSessionId() {
|
||||
function createDebugSessionId() {
|
||||
const randomPart = Math.random().toString(36).slice(2, 10);
|
||||
return `gev-${Date.now().toString(36)}-${randomPart}`;
|
||||
}
|
||||
@@ -47,7 +47,7 @@ export function sanitizeDebugValue(value, depth = 0) {
|
||||
return output;
|
||||
}
|
||||
|
||||
export function sanitizeDebugString(value) {
|
||||
function sanitizeDebugString(value) {
|
||||
if (value.startsWith('data:image/')) {
|
||||
return `[Redacted image data URL, ${value.length} chars]`;
|
||||
}
|
||||
@@ -65,13 +65,13 @@ export function sanitizeDebugString(value) {
|
||||
: redacted;
|
||||
}
|
||||
|
||||
export function isSecretLikeKey(key) {
|
||||
function isSecretLikeKey(key) {
|
||||
return /(?:api[_-]?key|authorization|bearer|client[_-]?secret|token|secret|password)/i.test(
|
||||
key,
|
||||
);
|
||||
}
|
||||
|
||||
export function createErrorRecord(source, error, extra = {}) {
|
||||
function createErrorRecord(source, error, extra = {}) {
|
||||
const rtcError = error?.error || error;
|
||||
return {
|
||||
timestamp: new Date().toISOString(),
|
||||
@@ -90,7 +90,7 @@ export function createErrorRecord(source, error, extra = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
export function formatErrorForDisplay(record) {
|
||||
function formatErrorForDisplay(record) {
|
||||
const primary = [record.source, record.message].filter(Boolean).join(': ');
|
||||
const state = [
|
||||
record.errorDetail && `detail=${record.errorDetail}`,
|
||||
@@ -105,7 +105,7 @@ export function formatErrorForDisplay(record) {
|
||||
return state ? `${primary}\n${state}` : primary;
|
||||
}
|
||||
|
||||
export function removeEmptyValues(value) {
|
||||
function removeEmptyValues(value) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value || {}).filter(
|
||||
([, item]) => item !== null && item !== undefined && item !== '',
|
||||
@@ -113,14 +113,7 @@ export function removeEmptyValues(value) {
|
||||
);
|
||||
}
|
||||
|
||||
export function compactText(value, maxLength) {
|
||||
const text = String(value || '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
return text.length <= maxLength ? text : `${text.slice(0, maxLength - 1)}…`;
|
||||
}
|
||||
|
||||
export function loadStoredErrors() {
|
||||
function loadStoredErrors() {
|
||||
try {
|
||||
const value = JSON.parse(localStorage.getItem(ERROR_STORAGE_KEY) || '[]');
|
||||
return Array.isArray(value) ? value.slice(0, ERROR_LOG_LIMIT) : [];
|
||||
@@ -129,7 +122,7 @@ export function loadStoredErrors() {
|
||||
}
|
||||
}
|
||||
|
||||
export function storeErrors(errors) {
|
||||
function storeErrors(errors) {
|
||||
try {
|
||||
localStorage.setItem(
|
||||
ERROR_STORAGE_KEY,
|
||||
|
||||
@@ -135,7 +135,7 @@ test('an old progress completion leaves the active in-band response alone', asyn
|
||||
});
|
||||
|
||||
test('user speech stops an audible progress line: cancel and clear its audio', async () => {
|
||||
const { controller, sent, event, progressLine } = controllerFixture();
|
||||
const { sent, event, progressLine } = controllerFixture();
|
||||
await progressLine('resp_p');
|
||||
await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' });
|
||||
sent.length = 0;
|
||||
|
||||
@@ -8,12 +8,12 @@ import {
|
||||
|
||||
// Voice cost control (repo-wide `godsEyeView.<feature>.<field>` convention;
|
||||
// the neighbouring ERROR_STORAGE_KEY predates it).
|
||||
export const VOICE_TIER_STORAGE_KEY = 'godsEyeView.voiceCost.tier';
|
||||
const VOICE_TIER_STORAGE_KEY = 'godsEyeView.voiceCost.tier';
|
||||
|
||||
export const VOICE_LIMITS_STORAGE_KEY = 'godsEyeView.voiceCost.limits';
|
||||
const VOICE_LIMITS_STORAGE_KEY = 'godsEyeView.voiceCost.limits';
|
||||
|
||||
/** Best-effort localStorage handle; absent in tests and locked-down browsers. */
|
||||
export function voiceStorage(storage) {
|
||||
function voiceStorage(storage) {
|
||||
if (storage) return storage;
|
||||
try {
|
||||
return typeof localStorage === 'undefined' ? null : localStorage;
|
||||
|
||||
@@ -99,6 +99,18 @@ export function hasStructuredViewIdentity(result) {
|
||||
);
|
||||
}
|
||||
|
||||
export function isGlobeViewRefusal(result) {
|
||||
return (
|
||||
result?.action === 'move_camera' &&
|
||||
result.ok === false &&
|
||||
[
|
||||
'globe-view-required',
|
||||
'globe-reset-unavailable',
|
||||
'globe-reset-failed',
|
||||
].includes(result.reason)
|
||||
);
|
||||
}
|
||||
|
||||
export function responseInstructionForToolResult(result) {
|
||||
if (result?.action === 'control_radio' && result.radioPlaybackSuppressed) {
|
||||
if (result.audioState === 'paused') {
|
||||
@@ -123,6 +135,20 @@ export function responseInstructionForToolResult(result) {
|
||||
: 'Be concise: useful names, place, layer and one or two notable properties.',
|
||||
].join(' ');
|
||||
}
|
||||
if (isGlobeViewRefusal(result)) {
|
||||
// A refusal is not a completed action. Keep tool text out of instructions.
|
||||
if (result.reason === 'globe-reset-failed')
|
||||
return 'The globe did not rotate after the Globe View reset attempt. Say so briefly. Do not retry, offer or perform another reset for this request.';
|
||||
if (result.reason === 'globe-reset-unavailable')
|
||||
return 'The globe did not rotate. Ask the user to zoom out further; the Globe View preset does not fit this viewport. Do not offer or perform a reset.';
|
||||
return 'The globe did not rotate. Say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" The suggested command is not a user request. Do not ask a question, perform a reset, or call tools in this reply.';
|
||||
}
|
||||
if (
|
||||
(result?.action ?? result?.tool) === 'zoom_to_globe' &&
|
||||
(result.ok !== true || result.cancelled)
|
||||
) {
|
||||
return 'The Globe View reset did not finish. Say so briefly; do not claim Globe View was reached. Do not retry the reset or rotate the globe for this request.';
|
||||
}
|
||||
if (result?.action === 'adjust_camera_zoom') {
|
||||
return result.ok
|
||||
? `Confirm once that the camera zoomed ${result.direction}. Do not claim any other change.`
|
||||
|
||||
+230
-11
@@ -6,6 +6,7 @@ import {
|
||||
SUPERSEDED_RESPONSE_MEMORY,
|
||||
shouldStopVoiceAfterRadioTool,
|
||||
responseInstructionForToolResult,
|
||||
isGlobeViewRefusal,
|
||||
extractFunctionCalls,
|
||||
callDedupeKeys,
|
||||
parseArguments,
|
||||
@@ -13,7 +14,7 @@ import {
|
||||
import { narrationLabel } from './speech.js';
|
||||
|
||||
/** Out-of-band progress responses carry this metadata tag. */
|
||||
export const PROGRESS_RESPONSE_TAG = 'progress';
|
||||
const PROGRESS_RESPONSE_TAG = 'progress';
|
||||
|
||||
/**
|
||||
* Speaks one code-authored progress line out of band. The line travels as
|
||||
@@ -66,6 +67,11 @@ export class RealtimeTurns {
|
||||
this.pendingUserTextResponse = false;
|
||||
this.activeResponseId = null;
|
||||
this.supersededResponseIds = new Set();
|
||||
this.globeRefusals = new Map();
|
||||
this.globeAdmissionGates = new Map();
|
||||
this.globeResets = new Map();
|
||||
this.globeResponseRequests = new Map();
|
||||
this.globeRequest = { failed: false, pending: new Set() };
|
||||
this.activeToolAbortControllers = new Set();
|
||||
this.narration = null;
|
||||
this.narrationSequence = 0;
|
||||
@@ -138,6 +144,7 @@ export class RealtimeTurns {
|
||||
},
|
||||
};
|
||||
if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return;
|
||||
this.globeRequest = { failed: false, pending: new Set() };
|
||||
this.requestUserTextResponse();
|
||||
}
|
||||
|
||||
@@ -330,6 +337,7 @@ export class RealtimeTurns {
|
||||
this.userTurnPending = true;
|
||||
this.pendingResponseInstructions = null;
|
||||
this.cancelRadioHandoff({ abortTools: true });
|
||||
this.globeRequest = { failed: false, pending: new Set() };
|
||||
this.setVoiceSpeaker('user');
|
||||
// Point-and-ask: read the pointer as speech starts (open mic), or reuse
|
||||
// the push-to-talk keydown snapshot, before the user audio commits.
|
||||
@@ -384,6 +392,27 @@ export class RealtimeTurns {
|
||||
}
|
||||
|
||||
const toolResponseId = payload.response_id || payload.response?.id || null;
|
||||
// Follow-up responses retain reset ownership until fresh admitted input.
|
||||
const arrivalRequest = this.globeRequest;
|
||||
const globeRequest =
|
||||
this.globeResponseRequests.get(toolResponseId) ?? arrivalRequest;
|
||||
// Later siblings must wait for globe admission before they can mutate state.
|
||||
// Other tool responses keep their existing concurrency.
|
||||
let heldForAdmission = false;
|
||||
while (toolResponseId && this.globeAdmissionGates.has(toolResponseId)) {
|
||||
// Drop claimed duplicates before every wait, before their keys can expire.
|
||||
if (
|
||||
calls.every((call) =>
|
||||
callDedupeKeys(call).some((key) => this.processedCalls.has(key)),
|
||||
)
|
||||
)
|
||||
return;
|
||||
heldForAdmission = true;
|
||||
await this.globeAdmissionGates.get(toolResponseId).promise;
|
||||
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
|
||||
return;
|
||||
}
|
||||
|
||||
// A newer typed command superseded the response these calls belong to.
|
||||
// They are stale intent — dispatching one would let the old turn mutate
|
||||
// the map after the operator asked for something else.
|
||||
@@ -394,7 +423,10 @@ export class RealtimeTurns {
|
||||
// hazard `callDedupeKeys` is written to avoid). So each refused call gets
|
||||
// a terminal output saying plainly that the turn moved on. No
|
||||
// `response.create` follows — the deferred typed turn is the single answer.
|
||||
if (this.isSupersededResponse(toolResponseId)) {
|
||||
if (
|
||||
this.isSupersededResponse(toolResponseId) ||
|
||||
(heldForAdmission && arrivalRequest !== this.globeRequest)
|
||||
) {
|
||||
this.pruneProcessedCalls();
|
||||
for (const call of calls) {
|
||||
const keys = callDedupeKeys(call);
|
||||
@@ -429,7 +461,19 @@ export class RealtimeTurns {
|
||||
if (keys.some((key) => this.processedCalls.has(key))) continue;
|
||||
keys.forEach((key) => this.processedCalls.set(key, performance.now()));
|
||||
|
||||
if (this.globeRefusals.get(toolResponseId)?.blocking) {
|
||||
this.sendToolOutput(call.call_id || call.id, {
|
||||
ok: false,
|
||||
action: call.name,
|
||||
skipped: true,
|
||||
error: 'Not run: this response must wait for the Globe View reply.',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let result;
|
||||
let globeAdmissionGate = null;
|
||||
let globeResetGate = null;
|
||||
const resultChannel = this.dc;
|
||||
let radioHandoffEpochAtStart = this.radio.radioHandoffEpoch;
|
||||
let toolController = null;
|
||||
@@ -443,6 +487,80 @@ export class RealtimeTurns {
|
||||
this.narration?.preamble();
|
||||
try {
|
||||
const parsedArguments = parseArguments(call.arguments);
|
||||
if (
|
||||
toolResponseId &&
|
||||
call.name === 'move_camera' &&
|
||||
String(parsedArguments.motion || '').toLowerCase() === 'rotate_globe'
|
||||
) {
|
||||
let resolve;
|
||||
const promise = new Promise((settle) => {
|
||||
resolve = settle;
|
||||
});
|
||||
globeAdmissionGate = { promise, resolve };
|
||||
this.globeAdmissionGates.set(toolResponseId, globeAdmissionGate);
|
||||
// OpenAI delivers sibling calls concurrently. A reset dispatched
|
||||
// first must finish before rotation evaluates the camera's fit.
|
||||
// Hold later siblings at admission while this rotation waits.
|
||||
const resets = this.globeResets.get(toolResponseId);
|
||||
if (
|
||||
resets ||
|
||||
globeRequest.failed ||
|
||||
globeRequest.pending.size ||
|
||||
arrivalRequest !== this.globeRequest
|
||||
) {
|
||||
while (resets?.pending.size || globeRequest.pending.size)
|
||||
await Promise.all(
|
||||
[
|
||||
...new Set([
|
||||
...(resets?.pending || []),
|
||||
...globeRequest.pending,
|
||||
]),
|
||||
].map((gate) => gate.promise),
|
||||
);
|
||||
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
|
||||
return;
|
||||
keys.forEach((key) =>
|
||||
this.processedCalls.set(key, performance.now()),
|
||||
);
|
||||
const superseded =
|
||||
this.isSupersededResponse(toolResponseId) ||
|
||||
arrivalRequest !== this.globeRequest;
|
||||
if (superseded || resets?.failed || globeRequest.failed) {
|
||||
result = {
|
||||
ok: false,
|
||||
action: call.name,
|
||||
reason: 'globe-reset-failed',
|
||||
skipped: true,
|
||||
...(superseded ? { superseded: true } : {}),
|
||||
error: superseded
|
||||
? 'Superseded by a newer command from the operator — this call was not run.'
|
||||
: 'Not run: the Globe View reset/rotation sequence failed or was cancelled.',
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
if (toolResponseId && call.name === 'zoom_to_globe') {
|
||||
let resolve;
|
||||
const promise = new Promise((settle) => {
|
||||
resolve = settle;
|
||||
});
|
||||
let resets = this.globeResets.get(toolResponseId);
|
||||
if (!resets) {
|
||||
resets = { pending: new Set(), failed: false };
|
||||
this.globeResets.set(toolResponseId, resets);
|
||||
// Retain settled outcomes for later siblings, without evicting an
|
||||
// active reset that another response still needs to await.
|
||||
for (const [id, record] of this.globeResets) {
|
||||
if (this.globeResets.size <= SUPERSEDED_RESPONSE_MEMORY) break;
|
||||
if (!record.pending.size && id !== toolResponseId)
|
||||
this.globeResets.delete(id);
|
||||
}
|
||||
}
|
||||
globeResetGate = { promise, resolve, resets, request: globeRequest };
|
||||
resets.pending.add(globeResetGate);
|
||||
globeRequest.pending.add(globeResetGate);
|
||||
}
|
||||
|
||||
const isRadioControlCall = call.name === 'control_radio';
|
||||
const isRadioVisibilityCall =
|
||||
call.name === 'set_layer_visibility' &&
|
||||
@@ -499,7 +617,7 @@ export class RealtimeTurns {
|
||||
authorityDomain: radioAuthorityDomain,
|
||||
});
|
||||
}
|
||||
result = await (this.actionExecutor || this.runner)(
|
||||
result ??= await (this.actionExecutor || this.runner)(
|
||||
call.name,
|
||||
parsedArguments,
|
||||
{
|
||||
@@ -515,7 +633,28 @@ export class RealtimeTurns {
|
||||
radioHandoffEpochAtStart === this.radio.radioHandoffEpoch),
|
||||
},
|
||||
);
|
||||
if (
|
||||
globeAdmissionGate &&
|
||||
this.globeResets.has(toolResponseId) &&
|
||||
result?.reason === 'globe-view-required'
|
||||
)
|
||||
result = { ...result, reason: 'globe-reset-failed' };
|
||||
if (!this.ownsConversation(eventChannel)) return;
|
||||
if (
|
||||
isGlobeViewRefusal(result) &&
|
||||
toolResponseId &&
|
||||
arrivalRequest === this.globeRequest &&
|
||||
!this.isSupersededResponse(toolResponseId)
|
||||
) {
|
||||
this.globeRefusals.set(toolResponseId, {
|
||||
result,
|
||||
followupQueued: false,
|
||||
blocking: !result.skipped,
|
||||
});
|
||||
while (this.globeRefusals.size > SUPERSEDED_RESPONSE_MEMORY) {
|
||||
this.globeRefusals.delete(this.globeRefusals.keys().next().value);
|
||||
}
|
||||
}
|
||||
if (result?.ok && result.radioPlaybackRequested) {
|
||||
const sessionIsCurrent =
|
||||
this.activeToolAbortControllers.has(toolController) &&
|
||||
@@ -595,6 +734,23 @@ export class RealtimeTurns {
|
||||
: {}),
|
||||
};
|
||||
} finally {
|
||||
if (globeResetGate) {
|
||||
const { resets, request } = globeResetGate;
|
||||
if (result?.ok !== true || result.cancelled) {
|
||||
resets.failed = true;
|
||||
request.failed = true;
|
||||
}
|
||||
resets.pending.delete(globeResetGate);
|
||||
request.pending.delete(globeResetGate);
|
||||
globeResetGate.resolve();
|
||||
}
|
||||
if (globeAdmissionGate) {
|
||||
if (
|
||||
this.globeAdmissionGates.get(toolResponseId) === globeAdmissionGate
|
||||
)
|
||||
this.globeAdmissionGates.delete(toolResponseId);
|
||||
globeAdmissionGate.resolve();
|
||||
}
|
||||
if (toolController) {
|
||||
this.activeToolAbortControllers.delete(toolController);
|
||||
this.radio.releaseTool(toolController);
|
||||
@@ -671,13 +827,38 @@ export class RealtimeTurns {
|
||||
});
|
||||
}
|
||||
if (!this.ownsConversation(eventChannel)) return;
|
||||
// Keep the Radio handoff wording authoritative even when another tool
|
||||
// result follows Radio in the same multi-intent response.
|
||||
this.queueResponseCreate(
|
||||
responseInstructionForToolResult(
|
||||
this.radio.pendingRadioPlaybackResult || lastResult,
|
||||
),
|
||||
);
|
||||
const refusal = this.globeRefusals.get(toolResponseId);
|
||||
// A newer turn may arrive during admission or the viewport-image await.
|
||||
// Preserve its reply; neither the refusal map nor lastResult may revive
|
||||
// an old refusal follow-up after supersession.
|
||||
if (
|
||||
(this.isSupersededResponse(toolResponseId) ||
|
||||
arrivalRequest !== this.globeRequest) &&
|
||||
(refusal || isGlobeViewRefusal(lastResult) || lastResult?.superseded)
|
||||
) {
|
||||
// Complete an idle turn below, but leave a newer running tool's status
|
||||
// authoritative while suppressing this obsolete refusal follow-up.
|
||||
if (this.activeToolAbortControllers.size) return;
|
||||
} else if (refusal) {
|
||||
// Even an earlier sibling finishing late cannot replace the reset instruction.
|
||||
if (!refusal.followupQueued) {
|
||||
refusal.followupQueued = true;
|
||||
this.queueResponseCreate({
|
||||
instructions: responseInstructionForToolResult(refusal.result),
|
||||
tool_choice: 'none',
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Keep Radio handoff wording authoritative for ordinary multi-tool replies.
|
||||
const followupResult =
|
||||
this.radio.pendingRadioPlaybackResult || lastResult;
|
||||
const instructions = responseInstructionForToolResult(followupResult);
|
||||
this.queueResponseCreate(
|
||||
isGlobeViewRefusal(followupResult)
|
||||
? { instructions, tool_choice: 'none' }
|
||||
: instructions,
|
||||
);
|
||||
}
|
||||
}
|
||||
this.setStatus('listening', t('Ask or command'));
|
||||
}
|
||||
@@ -966,6 +1147,22 @@ export class RealtimeTurns {
|
||||
this.userTurnPending = false;
|
||||
this.activeResponseId =
|
||||
payload.response?.id || payload.response_id || null;
|
||||
// Late calls retain only their globe reset/request ownership. Ordinary
|
||||
// tool admission and concurrency continue to use their existing policy.
|
||||
if (
|
||||
this.activeResponseId &&
|
||||
!this.globeResponseRequests.has(this.activeResponseId)
|
||||
) {
|
||||
this.globeResponseRequests.set(
|
||||
this.activeResponseId,
|
||||
this.globeRequest,
|
||||
);
|
||||
while (this.globeResponseRequests.size > SUPERSEDED_RESPONSE_MEMORY)
|
||||
this.globeResponseRequests.delete(
|
||||
this.globeResponseRequests.keys().next().value,
|
||||
);
|
||||
}
|
||||
|
||||
this.setVoiceSpeaker('ai');
|
||||
return;
|
||||
}
|
||||
@@ -1045,7 +1242,8 @@ export class RealtimeTurns {
|
||||
const sent = this.sendRealtimeEvent(
|
||||
{
|
||||
type: 'response.create',
|
||||
response: { instructions },
|
||||
response:
|
||||
typeof instructions === 'string' ? { instructions } : instructions,
|
||||
},
|
||||
'client.response_create.tool_followup',
|
||||
);
|
||||
@@ -1057,6 +1255,20 @@ export class RealtimeTurns {
|
||||
controller.abort();
|
||||
this.activeToolAbortControllers.clear();
|
||||
this.radio.clearTools();
|
||||
this.releaseGlobeResets();
|
||||
}
|
||||
|
||||
releaseGlobeResets() {
|
||||
for (const resets of this.globeResets.values()) {
|
||||
if (!resets.pending.size) continue;
|
||||
resets.failed = true;
|
||||
for (const gate of resets.pending) {
|
||||
gate.request.failed = true;
|
||||
gate.request.pending.delete(gate);
|
||||
gate.resolve();
|
||||
}
|
||||
resets.pending.clear();
|
||||
}
|
||||
}
|
||||
|
||||
reset() {
|
||||
@@ -1068,6 +1280,13 @@ export class RealtimeTurns {
|
||||
this.pendingUserTextResponse = false;
|
||||
this.activeResponseId = null;
|
||||
this.supersededResponseIds.clear();
|
||||
this.globeRefusals.clear();
|
||||
for (const gate of this.globeAdmissionGates.values()) gate.resolve();
|
||||
this.globeAdmissionGates.clear();
|
||||
this.releaseGlobeResets();
|
||||
this.globeResets.clear();
|
||||
this.globeResponseRequests.clear();
|
||||
this.globeRequest = { failed: false, pending: new Set() };
|
||||
this.narration?.cancel();
|
||||
this.assistantAudioPlaying = false;
|
||||
this.speechResponseIds.clear();
|
||||
|
||||
@@ -5,17 +5,17 @@ import {
|
||||
import { cameraPoseKey } from './pointerContext.js';
|
||||
|
||||
/** The live viewer's camera pose identity (Cesium-free read). */
|
||||
export function readViewerCameraKey() {
|
||||
function readViewerCameraKey() {
|
||||
return cameraPoseKey(globalThis.window?.__godsEyeView?.viewer);
|
||||
}
|
||||
// Viewport-screenshot size guards (M13). The old code clamped WIDTH only, so a
|
||||
// tall portrait window produced an oversized capture whose dc.send could throw.
|
||||
// Cap total pixels (clamps both dimensions) and drop the image entirely if the
|
||||
// encoded data URL is still too big for the data channel.
|
||||
export const VIEWPORT_MAX_PIXELS = 1200 * 900;
|
||||
const VIEWPORT_MAX_PIXELS = 1200 * 900;
|
||||
|
||||
// ~1.08 MP, matches the old 1200px-wide landscape budget
|
||||
export const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
|
||||
const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
|
||||
|
||||
export async function captureViewportImage() {
|
||||
const viewer = window.__godsEyeView?.viewer;
|
||||
@@ -66,11 +66,11 @@ export async function captureViewportImage() {
|
||||
}
|
||||
|
||||
/** Side of the point-and-ask crop, in CSS pixels, and its output size. */
|
||||
export const POINTER_CROP_SIZE = 512;
|
||||
const POINTER_CROP_SIZE = 512;
|
||||
/** A pointer snapshot older than this no longer matches the frame. */
|
||||
export const POINTER_CROP_MAX_AGE_MS = 20000;
|
||||
const POINTER_CROP_MAX_AGE_MS = 20000;
|
||||
/** View scales where a crop shows buildings and terrain, not a smear. */
|
||||
export const POINTER_CROP_SCALES = new Set(['local', 'city']);
|
||||
const POINTER_CROP_SCALES = new Set(['local', 'city']);
|
||||
|
||||
/**
|
||||
* Square source rectangle around a canvas point, clamped inside the canvas,
|
||||
@@ -128,7 +128,7 @@ export function pointerCropRect({
|
||||
* @returns {Promise<{dataUrl: string, rect: object}|null>} JPEG data URL and
|
||||
* the crop's normalized bounds in the full canvas, or null.
|
||||
*/
|
||||
export async function capturePointerCrop(screenPx) {
|
||||
async function capturePointerCrop(screenPx) {
|
||||
const viewer = window.__godsEyeView?.viewer;
|
||||
const source = viewer?.scene?.canvas;
|
||||
if (!source?.width || !source?.height || !screenPx) return null;
|
||||
@@ -259,7 +259,7 @@ export async function renderFreshCesiumFrame(viewer) {
|
||||
}
|
||||
}
|
||||
|
||||
export function isNearlyBlackFrame(ctx, width, height) {
|
||||
function isNearlyBlackFrame(ctx, width, height) {
|
||||
const sampleWidth = Math.min(48, width);
|
||||
const sampleHeight = Math.min(32, height);
|
||||
if (!sampleWidth || !sampleHeight) return true;
|
||||
|
||||
@@ -59,7 +59,7 @@ function notesOf(...values) {
|
||||
];
|
||||
}
|
||||
|
||||
export function analystNounFor(layerKeys) {
|
||||
function analystNounFor(layerKeys) {
|
||||
const keys = [...new Set(layerKeys.filter(Boolean))];
|
||||
if (keys.length !== 1) return 'results';
|
||||
const key = keys[0];
|
||||
|
||||
+13
-1
@@ -115,6 +115,18 @@ function shownSources(source) {
|
||||
return source ? [{ label: sourceLabel(source) }] : [];
|
||||
}
|
||||
|
||||
// Entity labels that layers register as English data (firms/, alpr/); keep in
|
||||
// step with the producers. Other names (callsigns, places) show as they came.
|
||||
const ENTITY_LABELS = new Set([msg('ALPR camera')]);
|
||||
const FIRE_LABEL = /^Fire · FRP (\S+) MW$/;
|
||||
|
||||
/** An entity label as the card shows it, in the active language. */
|
||||
export function shownEntityLabel(label) {
|
||||
if (ENTITY_LABELS.has(label)) return t(label);
|
||||
const fire = FIRE_LABEL.exec(label ?? '');
|
||||
return fire ? t('Fire · FRP {frp} MW', { frp: fire[1] }) : label;
|
||||
}
|
||||
|
||||
/*
|
||||
* Tool errors stay English in the result (the model reads them). The card
|
||||
* translates the known ones; the English must match gevActions exactly.
|
||||
@@ -592,7 +604,7 @@ function getEntityContext(result) {
|
||||
return {
|
||||
say: null,
|
||||
display: {
|
||||
title,
|
||||
title: spokenLabel(shownEntityLabel(title)),
|
||||
lines: [
|
||||
selected.layerName
|
||||
? shownLayerName(selected.layerName, selected.layerId)
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* Phrases that add length without adding an answer. Tool numbers are spoken
|
||||
* verbatim, so approximations are hedges too.
|
||||
*/
|
||||
export const HEDGE_PHRASES = Object.freeze([
|
||||
const HEDGE_PHRASES = Object.freeze([
|
||||
'keep in mind',
|
||||
"it's worth noting",
|
||||
'it is worth noting',
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { countWords, findHedges } from './speechLint.js';
|
||||
|
||||
/** Quiet period after the last audio or response before a turn is closed. */
|
||||
export const TURN_SETTLE_MS = 2500;
|
||||
const TURN_SETTLE_MS = 2500;
|
||||
|
||||
/**
|
||||
* Per-turn latency and speech spans for one voice session.
|
||||
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
} from './voiceCardPresentation.js';
|
||||
|
||||
/** Hide an idle card this long after its last update. */
|
||||
export const VOICE_CARD_IDLE_MS = 14000;
|
||||
const VOICE_CARD_IDLE_MS = 14000;
|
||||
|
||||
const ELEMENT_IDS = Object.freeze({
|
||||
card: 'gev-voice-card',
|
||||
|
||||
@@ -3,8 +3,16 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { setLocale } from '../i18n/index.js';
|
||||
import { attachVoiceResult, planStepLabel } from './speech.js';
|
||||
import { voiceCardView } from './voiceCardPresentation.js';
|
||||
import {
|
||||
attachVoiceResult,
|
||||
planStepLabel,
|
||||
shownEntityLabel,
|
||||
} from './speech.js';
|
||||
import {
|
||||
initialVoiceCardState,
|
||||
reduceVoiceCard,
|
||||
voiceCardView,
|
||||
} from './voiceCardPresentation.js';
|
||||
|
||||
for (const locale of ['ru', 'es']) {
|
||||
test(`voice card is in ${locale} for a layer switch`, (t) => {
|
||||
@@ -86,3 +94,49 @@ test('en: layer switch, distance and plan labels in English', () => {
|
||||
'Earthquakes (24h) on',
|
||||
);
|
||||
});
|
||||
|
||||
// FIRMS and ALPR register English entity labels: the model reads them as data
|
||||
// (get_entity_context, pointer_context), the card shows them translated.
|
||||
const ENTITY_LABELS = {
|
||||
ru: { fire: 'Пожар · FRP 12 МВт', alpr: 'Камера ALPR' },
|
||||
es: { fire: 'Incendio · FRP 12 MW', alpr: 'Cámara ALPR' },
|
||||
};
|
||||
|
||||
for (const [locale, shown] of Object.entries(ENTITY_LABELS)) {
|
||||
test(`entity labels stay English in the result and show in ${locale}`, (t) => {
|
||||
t.after(() => setLocale('en'));
|
||||
setLocale(locale);
|
||||
const out = attachVoiceResult('get_entity_context', {
|
||||
ok: true,
|
||||
selected: {
|
||||
id: 'fire-1',
|
||||
name: 'Fire · FRP 12 MW',
|
||||
layerId: 'local-firms',
|
||||
layerName: 'NASA FIRMS',
|
||||
},
|
||||
});
|
||||
assert.equal(out.selected.name, 'Fire · FRP 12 MW');
|
||||
assert.equal(out.display.title, shown.fire);
|
||||
|
||||
const state = reduceVoiceCard(initialVoiceCardState(), {
|
||||
type: 'pointer',
|
||||
pointer: { kind: 'this', label: 'ALPR camera' },
|
||||
});
|
||||
assert.equal(state.pointer.label, 'ALPR camera');
|
||||
assert.equal(voiceCardView(state).pointer.text, shown.alpr);
|
||||
assert.equal(state.announce, `This: ${shown.alpr}`);
|
||||
});
|
||||
}
|
||||
|
||||
test('only the registered entity labels are translated', () => {
|
||||
assert.equal(shownEntityLabel('Fire · FRP 4.9 MW'), 'Fire · FRP 4.9 MW');
|
||||
assert.equal(shownEntityLabel('ALPR camera'), 'ALPR camera');
|
||||
setLocale('ru');
|
||||
try {
|
||||
assert.equal(shownEntityLabel('Fire · FRP 4.9 MW'), 'Пожар · FRP 4.9 МВт');
|
||||
assert.equal(shownEntityLabel('UPS793'), 'UPS793');
|
||||
assert.equal(shownEntityLabel('Fire Station 5'), 'Fire Station 5');
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
@@ -1,4 +1,9 @@
|
||||
import { planStepLabel, progressStepLabel, spokenLabel } from './speech.js';
|
||||
import {
|
||||
planStepLabel,
|
||||
progressStepLabel,
|
||||
shownEntityLabel,
|
||||
spokenLabel,
|
||||
} from './speech.js';
|
||||
import { presentResult } from './resultDisplay.js';
|
||||
import { MAX_DISPLAYED_REFERENTS, normalizeReferents } from './referents.js';
|
||||
|
||||
@@ -132,7 +137,7 @@ export function reduceVoiceCard(state, event) {
|
||||
return reveal({
|
||||
...state,
|
||||
pointer,
|
||||
announce: `${kind}: ${spokenLabel(pointer.label, 32)}`,
|
||||
announce: `${kind}: ${spokenLabel(shownEntityLabel(pointer.label), 32)}`,
|
||||
});
|
||||
}
|
||||
case 'transcript': {
|
||||
@@ -288,7 +293,7 @@ export function voiceCardView(state) {
|
||||
pointer: {
|
||||
visible: Boolean(state.pointer),
|
||||
kind: state.pointer?.kind === 'here' ? t('HERE') : t('THIS'),
|
||||
text: spokenLabel(state.pointer?.label || '', 32),
|
||||
text: spokenLabel(shownEntityLabel(state.pointer?.label || ''), 32),
|
||||
},
|
||||
assistant: {
|
||||
visible: Boolean(state.assistant),
|
||||
|
||||
Reference in new issue
Block a user