feat(camera): add globe rotation (#1225)

Add continuous globe rotation with direction and speed controls, viewport fit checks, camera ownership handoff, and voice refusal guards.
This commit is contained in:
manjunath22466 authored and GitHub committed 2026-10-23 14:51:42 -07:00
1 parent 8cdd184a90
commit b2fb4a261e
25 files changed
+3407 -36

No files matched your search

File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -223,7 +223,7 @@ test('validated voice camera destinations share the UI navigation authority faca
'move camera',
);
ordered(move, [
"if (!['orbit', 'pan', 'tilt', 'rotate'].includes(motion))",
"if (!['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe'].includes(motion))",
'const start = () => {',
/const preserveCameraFlight =\s*motion === 'orbit'/,
'runNavigation(start, { preserveCameraFlight })',
+300 -12
View File
@@ -4,7 +4,7 @@
*
* One motion at a time, driven per clock tick. `once` = bounded eased nudge;
* `continuous` runs until move_camera{stop}, ANY manual camera input on the
* canvas (pointerdown/wheel — the cancelFlight reflex), or a navigation tool
* canvas (pointerdown/wheel/touchstart — the cancelFlight reflex), or a navigation tool
* starts (the shared UI navigation policy releases this owner before the new
* destination mutates the camera). fly_route is a
* continuous motion in the same slot: a cinematic dolly along an existing
@@ -13,14 +13,24 @@
* Orbit circles the current screen-center ground target holding range and
* pitch while advancing heading (per-frame camera.lookAt; the lookAt
* transform is cleared on every stop so manual control returns cleanly).
* Globe rotation turns the world-frame camera about its screen-up axis through
* Earth's center, preserving range and framing. Left/right name the visible
* globe motion, opposite to the camera's travel around Earth.
*/
import * as Cesium from 'cesium';
import { GLOBE_VIEW } from './locations.js';
import {
projectEarthDiscToViewport,
earthDiscScreenRadius,
getKeyholeGeometry,
} from './celestialRing.js';
import {
holdContinuousRender,
releaseContinuousRender,
} from './renderGovernor.js';
let _isCockpitActive = null;
/** °/s by speed word — orbit; pan uses fractions of view height/s. */
const ORBIT_DEG_S = { slow: 2, normal: 6, fast: 15 };
const TILT_DEG_S = { slow: 4, normal: 10, fast: 20 };
@@ -952,6 +962,7 @@ let _viewer = null;
let _getTarget = null;
let _active = null; // { kind, motion, direction, speed, mode, motionId, ...state }
let _tickRemover = null;
let _trackingRemover = null;
let _inputRemovers = [];
/** Monotonic id for the motion currently running; 0 means "no motion". */
let _motionSeq = 0;
@@ -1162,7 +1173,11 @@ function onTick() {
else if (m.direction === 'right') cam.moveRight(step);
else if (m.direction === 'up') cam.moveUp(step);
else cam.moveDown(step);
} else if (m.kind === 'tilt' || m.kind === 'rotate') {
} else if (
m.kind === 'tilt' ||
m.kind === 'rotate' ||
m.kind === 'rotate_globe'
) {
const degS = TILT_DEG_S[m.speed];
const stepRad =
m.mode === 'once'
@@ -1172,7 +1187,38 @@ function onTick() {
(scale - (m.lastScale || 0))
: Cesium.Math.toRadians(degS) * dt;
m.lastScale = scale;
if (m.kind === 'tilt') {
if (m.kind === 'rotate_globe') {
// Tracking can acquire the camera without going through navigation.
// Yield without resetting its pose/frame: the world-space arc proof
// only describes Cesium's rotation while the frame is identity.
if (
_viewer.scene.mode !== Cesium.SceneMode.SCENE3D ||
_viewer.trackedEntity ||
_isCockpitActive?.() ||
!Cesium.Matrix4.equals(cam.transform, Cesium.Matrix4.IDENTITY)
) {
interruptCameraMotion('globe-camera-owner-changed');
return;
}
// Cesium rotates both position and basis about the reference-frame
// origin. The captured screen-up axis keeps left/right horizontal even
// in a tilted or rolled view; a positive angle moves the globe right.
const angle = m.direction === 'left' ? -stepRad : stepRad;
if (globeRotationFitsKeyhole() !== true) {
interruptCameraMotion('globe-framing-changed');
return;
}
if (
!globeRotationHasClearance(cam.positionWC) ||
Math.abs(
Cesium.Cartesian3.magnitude(cam.positionWC) - m.rotationRadius,
) > 0.01
) {
interruptCameraMotion('globe-surface-limit');
return;
}
cam.rotate(m.rotationAxis, angle);
} else if (m.kind === 'tilt') {
const up = m.direction === 'up';
// Clamp pitch to [-89°, -5°]: never through the ground, never at the sky.
const next = cam.pitch + (up ? stepRad : -stepRad);
@@ -1206,6 +1252,133 @@ function onTick() {
}
}
/** Rigid origin-centred rotation stays outside the actual ellipsoid's bounding sphere. */
function globeRotationHasClearance(position) {
const ellipsoid = _viewer.scene.globe?.ellipsoid ?? Cesium.Ellipsoid.WGS84;
const { minimumRadius, maximumRadius } = ellipsoid;
const radius = Cesium.Cartesian3.magnitude(position);
return (
Number.isFinite(minimumRadius) &&
minimumRadius > 0 &&
Number.isFinite(maximumRadius) &&
maximumRadius >= minimumRadius &&
Number.isFinite(radius) &&
radius > maximumRadius * (1 + 1e-12)
);
}
/** Use the existing WGS84/CSS-pixel projection; null means unsupported, not close-up. */
function globeRotationFitsKeyhole(camera = _viewer.camera) {
const ellipsoid = _viewer.scene.globe?.ellipsoid ?? Cesium.Ellipsoid.WGS84;
if (!Cesium.Cartesian3.equals(ellipsoid.radii, Cesium.Ellipsoid.WGS84.radii))
return null;
const canvas = _viewer.scene.canvas;
const width = canvas.clientWidth;
const height = canvas.clientHeight;
const frustum = camera.frustum;
if (
!Number.isFinite(width) ||
!Number.isFinite(height) ||
width <= 0 ||
height <= 0 ||
!(frustum instanceof Cesium.PerspectiveFrustum) ||
!Number.isFinite(frustum.near) ||
frustum.near <= 0 ||
!Number.isFinite(frustum.far) ||
frustum.far <= frustum.near ||
frustum.xOffset !== 0 ||
frustum.yOffset !== 0
)
return null;
try {
const cam = camera;
const distance = Cesium.Cartesian3.magnitude(cam.positionWC);
const sphereRadius = ellipsoid.maximumRadius;
if (!Number.isFinite(distance) || distance <= 0) return null;
if (distance <= sphereRadius) return false; // Near-surface polar views are close-up too.
const radiusPx = earthDiscScreenRadius(distance, height, frustum.fovy);
if (!radiusPx) return null;
const keyhole = getKeyholeGeometry(width, height);
// The known nadir reset probe uses the same exact cone below; projecting
// through SceneTransforms would read the live scene.camera instead.
if (
camera === _viewer.camera &&
!projectEarthDiscToViewport(_viewer, width, height)
) {
// A near-surface horizon view cannot fit even when pointed at Earth;
// an orbital camera looking away is unsupported, not a close-up offer.
return radiusPx > Math.min(keyhole.radius, width / 2, height / 2)
? false
: null;
}
// The shared disc is an overlay approximation off-axis. Bound the sphere
// against the existing keyhole cone. Canvas-edge clipping does not change
// framing during rigid globe rotation and is not a clearance condition,
// without changing the UI projection or defining another keyhole/mask.
const toCenter = Cesium.Cartesian3.negate(
cam.positionWC,
new Cesium.Cartesian3(),
);
const depth = Cesium.Cartesian3.dot(toCenter, cam.directionWC);
const right = Math.abs(Cesium.Cartesian3.dot(toCenter, cam.rightWC));
const up = Math.abs(Cesium.Cartesian3.dot(toCenter, cam.upWC));
const focalY = height / (2 * Math.tan(frustum.fovy / 2));
const focalX =
width / (2 * Math.tan(frustum.fovy / 2) * frustum.aspectRatio);
if (
![depth, right, up, focalX, focalY].every(Number.isFinite) ||
focalX <= 0 ||
focalY <= 0
)
return null;
const halfAngle = Math.asin(sphereRadius / distance);
const centerAngle = Math.atan2(Math.hypot(right, up), depth);
const keyholeAngle = Math.atan(keyhole.radius / Math.max(focalX, focalY));
// No overlay visibility hysteresis: a fully contained smaller globe needs no reset.
const framed = centerAngle + halfAngle <= keyholeAngle;
if (!framed) return false;
// Projection alone does not prove that a tiny globe is within depth planes.
if (
depth - sphereRadius < frustum.near ||
depth + sphereRadius > frustum.far
)
return null;
return true;
} catch {
return null;
}
}
/** Check the unchanged reset destination without moving the live camera. */
function globeResetFitsKeyhole() {
try {
const camera = Cesium.Camera.clone(_viewer.camera);
const { longitude, latitude } = _viewer.camera.positionCartographic;
camera.setView({
destination: Cesium.Cartesian3.fromRadians(
longitude,
latitude,
GLOBE_VIEW.heightM,
),
orientation: {
heading: 0,
pitch: Cesium.Math.toRadians(GLOBE_VIEW.pitchDeg),
roll: 0,
},
endTransform: Cesium.Matrix4.IDENTITY,
});
return globeRotationFitsKeyhole(camera) === true;
} catch {
return false;
}
}
/** Release only globe rotation when a new camera intent takes ownership. */
export function interruptGlobeRotation(reason = 'camera-intent') {
if (_active?.kind === 'rotate_globe') return interruptCameraMotion(reason);
return { wasActive: false, reason, leveled: false };
}
/**
* Wire the module to the viewer once (idempotent). The view-target getter is
* optional so a non-voice caller (the Directions FLY chip) can arm route
@@ -1218,24 +1391,42 @@ function onTick() {
* call brings none, so a callback from the old viewer can never answer for the
* new one.
*/
export function initCameraVerbs(viewer, getViewTargetCartesian = null) {
export function initCameraVerbs(
viewer,
getViewTargetCartesian = null,
{ isCockpitActive } = {},
) {
const nextTarget =
typeof getViewTargetCartesian === 'function'
? getViewTargetCartesian
: null;
if (_viewer === viewer) {
if (nextTarget) _getTarget = nextTarget;
if (typeof isCockpitActive === 'function')
_isCockpitActive = isCockpitActive;
return;
}
// A globe motion belongs to the viewer whose world pose it captured.
if (_active?.kind === 'rotate_globe')
interruptCameraMotion('viewer-replaced');
_getTarget = nextTarget;
_isCockpitActive =
typeof isCockpitActive === 'function' ? isCockpitActive : null;
_viewer = viewer;
if (_tickRemover) _tickRemover();
_tickRemover = viewer.clock.onTick.addEventListener(onTick);
if (_trackingRemover) _trackingRemover();
// Cockpit can acquire then clear tracking before the next clock tick.
// Yield on acquisition itself without changing legacy camera motions.
_trackingRemover = viewer.trackedEntityChanged?.addEventListener((entity) => {
if (entity && _active?.kind === 'rotate_globe')
interruptCameraMotion('globe-camera-owner-changed');
});
for (const rm of _inputRemovers) rm();
_inputRemovers = [];
const canvas = viewer.scene.canvas;
// ANY manual camera input reclaims control — the cancelFlight reflex.
for (const evt of ['pointerdown', 'wheel']) {
for (const evt of ['pointerdown', 'touchstart', 'wheel']) {
const h = () => interruptCameraMotion('manual-input');
canvas.addEventListener(evt, h, { passive: true });
_inputRemovers.push(() => canvas.removeEventListener(evt, h));
@@ -1256,7 +1447,11 @@ export function moveCamera(args = {}, runNavigation = null) {
const speed = ORBIT_DEG_S[String(args.speed || 'normal').toLowerCase()]
? String(args.speed || 'normal').toLowerCase()
: 'normal';
const mode = args.mode === 'continuous' ? 'continuous' : 'once';
const mode =
args.mode === 'continuous' ||
(motion === 'rotate_globe' && args.mode === undefined)
? 'continuous'
: 'once';
if (motion === 'stop') {
const wasActiveBeforeHandoff = Boolean(_active);
const stop = () => {
@@ -1270,15 +1465,16 @@ export function moveCamera(args = {}, runNavigation = null) {
};
return typeof runNavigation === 'function' ? runNavigation(stop) : stop();
}
if (!['orbit', 'pan', 'tilt', 'rotate'].includes(motion)) {
if (!['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe'].includes(motion)) {
return {
ok: false,
action: 'move_camera',
error: `Unknown motion "${args.motion}" — use orbit, pan, tilt, rotate, or stop.`,
error: `Unknown motion "${args.motion}" — use orbit, pan, tilt, rotate, rotate_globe, or stop.`,
};
}
if (
motion !== 'orbit' &&
motion !== 'rotate_globe' &&
!['left', 'right', 'up', 'down'].includes(direction)
) {
return {
@@ -1294,6 +1490,17 @@ export function moveCamera(args = {}, runNavigation = null) {
error: 'rotate needs left or right.',
};
}
if (
motion === 'rotate_globe' &&
direction !== null &&
!['left', 'right'].includes(direction)
) {
return {
ok: false,
action: 'move_camera',
error: 'rotate_globe needs left or right.',
};
}
if (motion === 'tilt') {
// Honest limits: at the clamp a tilt is a no-op — say so instead of
// reporting success and letting the model gaslight itself (field test).
@@ -1318,9 +1525,9 @@ export function moveCamera(args = {}, runNavigation = null) {
}
}
if (
motion === 'orbit' &&
_viewer?.trackedEntity &&
typeof runNavigation !== 'function'
(motion === 'rotate_globe' ||
(motion === 'orbit' && typeof runNavigation !== 'function')) &&
_viewer?.trackedEntity
) {
// The follow camera owns a tracked view; a lookAt orbit fights it frame
// by frame and stop rips the camera out (field finding). Honest refusal
@@ -1332,9 +1539,74 @@ export function moveCamera(args = {}, runNavigation = null) {
return {
ok: false,
action: 'move_camera',
error: `Already following ${label} — the follow camera owns the view while tracking. Say "stop tracking" first if you want a free orbit.`,
error: `Already following ${label} — the follow camera owns the view while tracking. Say "stop tracking" first if you want ${motion === 'rotate_globe' ? 'to rotate the globe' : 'a free orbit'}.`,
};
}
let globePose;
if (motion === 'rotate_globe') {
if (_viewer.scene.mode !== Cesium.SceneMode.SCENE3D) {
return {
ok: false,
action: 'move_camera',
error: 'Globe rotation requires a 3D view.',
};
}
const cam = _viewer.camera;
globePose = {
position: Cesium.Cartesian3.clone(cam.positionWC),
direction: Cesium.Cartesian3.clone(cam.directionWC),
up: Cesium.Cartesian3.normalize(cam.upWC, new Cesium.Cartesian3()),
right: Cesium.Cartesian3.clone(cam.rightWC),
};
if (_isCockpitActive?.())
return {
ok: false,
action: 'move_camera',
reason: 'camera-owned',
error: 'Exit Cockpit before rotating the globe.',
};
const ellipsoid = _viewer.scene.globe?.ellipsoid ?? Cesium.Ellipsoid.WGS84;
if (!(
Number.isFinite(ellipsoid.minimumRadius) &&
ellipsoid.minimumRadius > 0 &&
Number.isFinite(ellipsoid.maximumRadius) &&
ellipsoid.maximumRadius >= ellipsoid.minimumRadius &&
Number.isFinite(Cesium.Cartesian3.magnitude(globePose.position))
))
return {
ok: false,
action: 'move_camera',
reason: 'unsafe-ellipsoid',
error:
'Globe rotation requires a finite supported ellipsoid and a clear globe view.',
};
const fits = globeRotationFitsKeyhole();
if (fits === null)
return {
ok: false,
action: 'move_camera',
reason: 'unsupported-view',
error: 'Globe rotation requires a supported perspective view.',
};
if (!fits) {
const canReset = globeResetFitsKeyhole();
return {
ok: false,
action: 'move_camera',
reason: canReset ? 'globe-view-required' : 'globe-reset-unavailable',
error: canReset
? 'Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’'
: 'Zoom out further to fit the globe inside the keyhole; the Globe View preset does not fit this viewport.',
};
}
if (!globeRotationHasClearance(globePose.position))
return {
ok: false,
action: 'move_camera',
reason: 'unsafe-ellipsoid',
error: 'Globe rotation requires a clear ellipsoid bounding sphere.',
};
}
const start = () => {
interruptCameraMotion('replaced');
const state = {
@@ -1343,6 +1615,22 @@ export function moveCamera(args = {}, runNavigation = null) {
speed,
mode,
};
if (motion === 'rotate_globe') {
// Convert an inherited ground-target frame without changing the world
// pose. Rotation must pivot about Earth, never the previous local target.
const cam = _viewer.camera;
cam.cancelFlight();
cam.lookAtTransform(Cesium.Matrix4.IDENTITY);
// Handoff may level a route's bank. Keep the caller's exact world pose.
// Copy the basis directly: setView's heading/pitch conversion is singular
// at nadir and can change the captured orientation near the poles.
Cesium.Cartesian3.clone(globePose.position, cam.position);
Cesium.Cartesian3.clone(globePose.direction, cam.direction);
Cesium.Cartesian3.clone(globePose.up, cam.up);
Cesium.Cartesian3.clone(globePose.right, cam.right);
state.rotationAxis = globePose.up;
state.rotationRadius = Cesium.Cartesian3.magnitude(globePose.position);
}
if (motion === 'orbit') {
if (_viewer.scene.tweens.length > 0) {
// A camera flight is in progress ("fly to X and orbit it") — ALWAYS arm
+16 -2
View File
@@ -988,6 +988,16 @@ test('the voice TOOL SCHEMA matches the pinned contract — the mission mapping
]);
// Cyber deliberately adds one layout; first-run missions still change no tools.
hudLayout.enum = hudLayout.enum.filter((layout) => layout !== 'cyber');
const cameraMotion = legacyTools.find((tool) => tool.name === 'move_camera')
.parameters.properties.motion;
assert.deepEqual(cameraMotion.enum, [
'orbit',
'pan',
'tilt',
'rotate',
'rotate_globe',
'stop',
]);
const block = JSON.stringify(legacyTools);
// Re-derived for the voice layer manifest (generated layer enums, alias and
// field hints), point-and-ask's pointer/referent arguments and the prompt
@@ -997,10 +1007,14 @@ test('the voice TOOL SCHEMA matches the pinned contract — the mission mapping
// list descriptions; the missions still ride existing tools.
// Street Level and GNSS extend the manifest; broadcast caveats describe tracked and analyst aircraft.
// Air quality (ECCC AQHI) extends the manifest.
assert.equal(block.length, 30655, 'serialized tool schema length drifted');
// Globe rotation adds one camera motion and its descriptions; Globe View
// framing/default guidance updates only move_camera and mode descriptions.
// Radio markup and the complete production-request comparison pin locality.
// Restore the base camera/Globe View wording with the globe-rotation append.
assert.equal(block.length, 30878, 'serialized tool schema length drifted');
assert.equal(
crypto.createHash('sha256').update(block).digest('hex'),
'3b1d5e77c10a92f6543a4f9d1ff1870101a0306b65e5b8a6113111c81eda371e',
'7b95f0f4258896fe5c3316265a19f3a6602809d4e7e681ee7ecd8f2df514ef1f',
'the first-run missions must ride EXISTING tools: no schema edit, no cache bust',
);
const instructions = fs.readFileSync(
+29
View File
@@ -209,6 +209,35 @@ test('no unchanged Realtime tool definition drifts silently', () => {
const hudLayout = unchanged.find((tool) => tool.name === 'set_hud').parameters.properties.layout;
assert.deepEqual(hudLayout.enum, ['tactical', 'operator', 'minimal', 'cyber']);
hudLayout.enum = hudLayout.enum.filter((layout) => layout !== 'cyber');
// Globe rotation adds one motion and clarifies the two affected descriptions.
// Check those edits explicitly, then keep the legacy pin for every other field.
const camera = unchanged.find((tool) => tool.name === 'move_camera');
assert.deepEqual(camera.parameters.properties.motion.enum, [
'orbit',
'pan',
'tilt',
'rotate',
'rotate_globe',
'stop',
]);
assert.equal(
camera.description,
'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly"). Globe rotation (rotate_globe) spins Earth right continuously by default; explicit direction, mode and speed take precedence. It checks fit and may refuse.',
);
assert.equal(
camera.parameters.properties.direction.description,
'Required for pan/tilt/rotate; orbit/rotate_globe default right.',
);
assert.equal(camera.parameters.properties.mode.description, 'once = bounded eased nudge; continuous = until stop/manual input. rotate_globe defaults continuous; other motions default once.');
camera.parameters.properties.mode.description = 'once = bounded eased nudge (default); continuous = until stop/manual input.';
camera.parameters.properties.motion.enum =
camera.parameters.properties.motion.enum.filter(
(motion) => motion !== 'rotate_globe',
);
camera.description =
'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly").';
camera.parameters.properties.direction.description =
'Required except for orbit (defaults right/clockwise) and stop.';
assert.equal(unchanged.length, 14);
const digest = createHash('sha256')
.update(JSON.stringify(unchanged))
+13
View File
@@ -234,6 +234,19 @@ test('typed argument checks feed full-call accuracy without changing the legacy
const inMeters = gradeCore(altitude, [{ name: 'analyst_query', args: { layers: ['flights'], filters: [{ field: 'altitudeM', op: 'gt', value: 12192 }] } }]);
assert.equal(inMeters.fullCallOk, true);
assert.equal(where('set_hud', (a) => a.visible === 'off')({ name: 'set_hud', args: { visible: 'off' } }), true);
const globe = byPhrase('Keep spinning the Earth slowly to the left');
const globeArgs = { motion: 'rotate_globe', direction: 'left', speed: 'slow' };
for (const mode of [undefined, 'continuous', 'once', null]) {
assert.equal(gradeCore(globe, [{ name: 'move_camera', args: { ...globeArgs, mode } }]).fullCallOk, mode === undefined || mode === 'continuous');
}
assert.equal(gradeCore(globe, [{ name: 'move_camera', args: { ...globeArgs, direction: 'right' } }]).fullCallOk, false);
assert.equal(gradeCore(globe, [{ name: 'move_camera', args: { ...globeArgs, speed: 'normal' } }]).fullCallOk, false);
const right = byPhrase('Rotate the globe to the right');
assert.equal(gradeCore(right, [{ name: 'move_camera', args: { motion: 'rotate_globe' } }]).fullCallOk, true);
for (const direction of ['right', 'left', null]) {
assert.equal(gradeCore(right, [{ name: 'move_camera', args: { motion: 'rotate_globe', direction } }]).fullCallOk, direction === 'right');
}
assert.equal(gradeCore(right, [{ name: 'move_camera', args: { motion: 'rotate', direction: 'right' } }]).fullCallOk, false);
});
test('gradeDialogue scores turn 2 in the context of turn 1', () => {
+2
View File
@@ -147,6 +147,7 @@ export class StyleManager extends ShellFacade {
},
searchInput: this._locationSearch,
interruptCameraMotion: services.interruptCameraMotion,
interruptGlobeRotation: services.interruptGlobeRotation,
isCockpitActive: () => !!this.cockpitView?.active,
clearLocation: () => this.clearSearchedLocation(),
cancelShareSelection: () => this._shareRestoration.cancelSelection(),
@@ -322,6 +323,7 @@ export class StyleManager extends ShellFacade {
GLOBE_VIEW: services.GLOBE_VIEW,
flyToGlobeView: services.flyToGlobeView,
interruptCameraMotion: services.interruptCameraMotion,
interruptGlobeRotation: services.interruptGlobeRotation,
flightsLayer: services.flightsLayer,
militaryFlightsLayer: services.militaryFlightsLayer,
satellitesLayer: services.satellitesLayer,
+5 -1
View File
@@ -9,7 +9,10 @@ import {
flyToPOI,
searchAndFlyTo,
} from '../locations.js';
import { interruptCameraMotion } from '../cameraVerbs.js';
import {
interruptCameraMotion,
interruptGlobeRotation,
} from '../cameraVerbs.js';
import { IntelHUD } from '../hud.js';
import { ShareLinkManager } from '../sharelink.js';
import { OrbitController } from '../orbit.js';
@@ -73,6 +76,7 @@ export class StyleManager extends ApplicationShell {
flyToPOI,
searchAndFlyTo,
interruptCameraMotion,
interruptGlobeRotation,
IntelHUD,
ShareLinkManager,
OrbitController,
+1
View File
@@ -283,6 +283,7 @@ export class LocationNavigation {
return;
}
this.services.interruptGlobeRotation?.('orbit-intent');
const isActive = this.orbitController.toggle(this._currentTarget, {
radius: this._currentPoi?.alt || 500,
pitch: this._currentPoi?.pitch || -30,
+3
View File
@@ -14,6 +14,7 @@ export class NavigationController {
tracking,
searchInput,
interruptCameraMotion,
interruptGlobeRotation = () => {},
isCockpitActive,
clearLocation,
cancelShareSelection,
@@ -27,6 +28,7 @@ export class NavigationController {
tracking,
searchInput,
interruptCameraMotion,
interruptGlobeRotation,
isCockpitActive,
clearLocation,
cancelShareSelection,
@@ -46,6 +48,7 @@ export class NavigationController {
cancelPendingSelection = true,
clearSearchedLocation = true,
} = {}) {
this.interruptGlobeRotation('navigation-intent');
cancelCameraArrival(this.viewer);
this.cancelOrientation();
const { flightsLayer, militaryFlightsLayer, satellitesLayer } =
+1 -1
View File
@@ -729,7 +729,7 @@ const schemas = [
properties: {
motion: {
type: 'string',
enum: ['orbit', 'pan', 'tilt', 'rotate', 'stop'],
enum: ['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe', 'stop'],
},
direction: {
type: 'string',
+46 -1
View File
@@ -1,3 +1,4 @@
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
import test from 'node:test';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
@@ -38,7 +39,10 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
// requested radius and center authority in descriptions only.
// Street Level and GNSS extend the manifest; tracked and analyst aircraft describe broadcasts.
// Air quality (ECCC AQHI) extends the manifest.
'775ba323384b4c1b62fb8415bac52aecc3fc668e20bedcfa57e426dcd1da3969',
// Globe rotation adds one motion; visual keyhole fit and explicit
// reset guidance/defaults update only move_camera and mode descriptions.
// Restore base descriptions; append only globe-rotation defaults and fit guidance.
'967b5a043b812e2f8c1e190fc571cf12b960c9d4486df736902221aab4d99cf6',
);
});
@@ -106,6 +110,17 @@ test('all legacy action arguments are byte-identical after removing the delibera
delete property('show_data_layers_menu').layerId.enum;
delete property('get_entity_context').layerId.enum;
delete property('analyst_query').layers.items.enum;
// Earth-centered rotation is additive; local camera yaw keeps its contract.
const motion = property('move_camera').motion;
assert.deepEqual(motion.enum, [
'orbit',
'pan',
'tilt',
'rotate',
'rotate_globe',
'stop',
]);
motion.enum = motion.enum.filter((value) => value !== 'rotate_globe');
const panels = property('set_panel_open').panelId;
panels.enum = panels.enum.filter(
(id) => !['weather-panel', 'recent-imagery-panel'].includes(id),
@@ -145,6 +160,24 @@ test('all legacy action arguments are byte-identical after removing the delibera
);
});
test('globe rotation is an additive camera motion with the existing control arguments', () => {
const schema = GEV_ACTION_SCHEMAS.find(
(tool) => tool.name === 'move_camera',
).parameters;
for (const motion of ['rotate_globe', 'rotate', 'orbit', 'pan', 'tilt']) {
assert.deepEqual(
validateArgs(schema, {
motion,
direction: 'right',
speed: 'slow',
mode: 'continuous',
}),
[],
);
}
assert.ok(validateArgs(schema, { motion: 'spin_camera' }).length > 0);
});
test('track_entity accepts a referent alone but rejects an empty target', () => {
const schema = GEV_ACTION_SCHEMAS.find(
(tool) => tool.name === 'track_entity',
@@ -183,3 +216,15 @@ test('the benchmark still rejects an empty track_entity although the model-facin
[],
);
});
test('globe reset guidance requires a separate explicit command after refusal', () => {
const paragraph = realtimeInstructions()
.split('\n')
.find((line) => line.includes('globe-view-required'));
assert.ok(paragraph);
assert.match(paragraph, /calls only move_camera motion rotate_globe/);
assert.match(paragraph, /explicit user Reset\/Show Globe View/);
assert.match(paragraph, /never from your own plan/);
assert.match(paragraph, /bare yes\/ok/);
assert.match(paragraph, /Do not ask a question/);
});
+48 -2
View File
@@ -21,6 +21,7 @@ import {
shouldSendViewportImage,
shouldStopVoiceAfterRadioTool,
responseInstructionForToolResult,
isGlobeViewRefusal,
} from './realtimeProtocol.js';
const SOCKET_PATH =
@@ -182,6 +183,8 @@ export class GeminiSessionController {
this.model = null;
this.epoch = 0;
this.turn = 0;
this.globeToolsBlocked = false;
this.globeResetFailed = false;
this.inputGeneration = 0;
this.turnAbort = new AbortController();
this.disposed = false;
@@ -659,6 +662,8 @@ export class GeminiSessionController {
return;
if (!pushToTalk || activityCaptureId === metadata.captureId) return;
this.captureOwner.admitted = true;
this.globeToolsBlocked = false;
this.globeResetFailed = false;
if (
!this.interrupt('push-to-talk', {
keepPlayback: !this.pttPlaybackSuppressed,
@@ -1294,6 +1299,8 @@ export class GeminiSessionController {
this.replyPending = false;
if (!this.interrupt()) return;
} else if (!this.retiredReplyPending && !this.interrupt()) return;
this.globeToolsBlocked = false;
this.globeResetFailed = false;
// Local activity ownership is only for expiry liveness, not tools.
// Assign after retiring observed output so its boundary cannot clear
// a newer detected activity that has not received a reply yet.
@@ -1573,6 +1580,7 @@ export class GeminiSessionController {
.then(async () => {
if (!this.owns(epoch) || turn !== this.turn) return;
if (this.cancelledCalls.has(call.id)) {
if (call.name === 'zoom_to_globe') this.globeResetFailed = true;
if (this.tools.get(call.id) === action) this.tools.delete(call.id);
this.pendingTools--;
if (!this.pendingTools) {
@@ -1618,8 +1626,28 @@ export class GeminiSessionController {
(authority !== 'playback' ||
handoffEpoch === this.radio.radioHandoffEpoch);
let result;
let skippedGlobeRotation = false;
let blockedGlobeTool = false;
try {
if (!actionCurrent()) throw cancelled();
// Live has no response-scoped tool_choice. Recheck here so queued siblings
// cannot reset after a rotation refusal.
if (this.globeToolsBlocked) {
blockedGlobeTool = true;
throw new Error(
'Wait for fresh user input before calling another tool.',
);
}
if (
this.globeResetFailed &&
call.name === 'move_camera' &&
String(args.motion || '').toLowerCase() === 'rotate_globe'
) {
skippedGlobeRotation = true;
throw new Error(
'Not run: the Globe View reset/rotation sequence failed or was cancelled.',
);
}
if (stronger)
reservation = this.radio.reserveRadioToolHandoff({ abortScope });
handoffEpoch = this.radio.radioHandoffEpoch;
@@ -1689,7 +1717,12 @@ export class GeminiSessionController {
} catch (error) {
result = {
ok: false,
tool: call.name,
...(skippedGlobeRotation
? { action: call.name, reason: 'globe-reset-failed', skipped: true }
: {
tool: call.name,
...(blockedGlobeTool ? { skipped: true } : {}),
}),
...(radioVisibility ? { layerId: 'radio' } : {}),
error:
error?.name === 'AbortError'
@@ -1716,6 +1749,11 @@ export class GeminiSessionController {
this.noteRadioStrongerCommit(action, abortScope, turn);
}
if (!this.owns(epoch) || turn !== this.turn) return;
if (
call.name === 'zoom_to_globe' &&
(result?.ok !== true || result.cancelled)
)
this.globeResetFailed = true;
const playbackObsolete =
authority === 'playback' && handoffEpoch !== this.radio.radioHandoffEpoch;
if (radioControl || radioVisibility) {
@@ -1744,6 +1782,8 @@ export class GeminiSessionController {
this.stopAfterReply = true;
// Native call cancellation suppresses its output, without canceling siblings.
if (!this.cancelledCalls.has(call.id)) {
if (isGlobeViewRefusal(result) && !skippedGlobeRotation)
this.globeToolsBlocked = true;
this.postToolOutput = false;
this.modelComplete = false;
this.replyPending = true;
@@ -1751,7 +1791,9 @@ export class GeminiSessionController {
const mapEvents = mapSnapshot.map((text) => JSON.parse(text));
const response = {
...(result ?? { ok: false, error: 'Command returned no result' }),
responseGuidance: responseInstructionForToolResult(result),
responseGuidance: blockedGlobeTool
? 'That action did not run. Wait for fresh user input; do not claim any change or call another tool.'
: responseInstructionForToolResult(result),
};
this.send(
{
@@ -1902,6 +1944,8 @@ export class GeminiSessionController {
{
onSent: () => {
if (!owned() || this.typedIntent !== typedIntent) return;
this.globeToolsBlocked = false;
this.globeResetFailed = false;
this.replyPending = true;
this.bindTextIntent(typedIntent);
if (!owned() || this.typedIntent !== typedIntent) return;
@@ -2455,6 +2499,8 @@ export class GeminiSessionController {
this.cancelledCalls.clear();
this.pendingMapEvents = [];
this.userTurnPending = false;
this.globeToolsBlocked = false;
this.globeResetFailed = false;
this.replyPending = false;
this.unrequestedReply = false;
this.automaticTurnOpen = false;
+212
View File
@@ -3398,3 +3398,215 @@ test('a narration callback that submits a successor keeps the old packet tools f
assert.deepEqual(h.calls, []);
h.c.stop();
});
for (const reply of ['typed', 'push-to-talk', 'open-mic']) {
test(`globe refusal blocks queued and later tools until fresh ${reply} input`, async () => {
const names = [];
const h = harness({
runAction: async (name) => {
names.push(name);
return name === 'move_camera'
? { ok: false, action: 'move_camera', reason: 'globe-view-required' }
: { ok: true, action: name };
},
});
try {
await h.c.start({ pushToTalk: reply === 'push-to-talk' });
establishToolInput(h, 'rotate the globe');
const ws = h.sockets[0];
ws.message({
toolCall: {
functionCalls: [
{
id: 'rotate',
name: 'move_camera',
args: { motion: 'rotate_globe' },
},
{ id: 'queued-reset', name: 'zoom_to_globe', args: {} },
],
},
});
await h.c.toolQueue;
assert.deepEqual(names, ['move_camera']);
const blocked = ws.sent
.flatMap((entry) => entry.toolResponse?.functionResponses || [])
.find((call) => call.id === 'queued-reset');
assert.equal(blocked.response.ok, false);
assert.match(blocked.response.error, /fresh user input/i);
assert.equal(blocked.response.skipped, true);
assert.match(blocked.response.responseGuidance, /did not run/i);
assert.doesNotMatch(
blocked.response.responseGuidance,
/confirm.*completed/i,
);
requestTool(ws, 'repeated-rotation', 'move_camera', {
motion: 'rotate_globe',
});
await h.c.toolQueue;
assert.deepEqual(names, ['move_camera']);
const repeated = toolResponses(ws).find(
(call) => call.id === 'repeated-rotation',
).response;
assert.equal(repeated.ok, false);
assert.equal(repeated.skipped, true);
assert.match(repeated.responseGuidance, /did not run/i);
assert.doesNotMatch(
repeated.responseGuidance,
/confirm.*completed/i,
);
if (reply === 'typed') {
const stimuli = [
() => {},
() => ws.message({ serverContent: { interrupted: true } }),
() =>
ws.message({
serverContent: {
inputTranscription: { text: 'reset to globe view' },
},
}),
() => {
h.c.input.setMicrophoneEnabled(false);
h.c.input.setMicrophoneEnabled(true);
},
];
for (const [i, stimulus] of stimuli.entries()) {
stimulus();
requestTool(ws, `unrequested-reset-${i}`, 'zoom_to_globe', {});
await h.c.toolQueue;
assert.deepEqual(names, ['move_camera']);
}
}
// Finish the refusal reply before admitting the actual next user input.
ws.message({ serverContent: { turnComplete: true } });
if (reply === 'typed')
assert.equal(h.c.sendTextCommand('reset to globe view'), true);
else if (reply === 'push-to-talk') {
h.c.input.pushToTalkKeyHeld = true;
h.c.input.spaceKeyHeld = true;
h.c.input.setMicrophoneEnabled(true);
h.c.input.setMicrophoneEnabled(false);
} else ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
requestTool(ws, 'accepted-reset', 'zoom_to_globe', {});
await h.c.toolQueue;
assert.deepEqual(names, ['move_camera', 'zoom_to_globe']);
} finally {
h.c.stop();
}
});
}
test('a cancelled globe reset skips queued rotation, preserves Stop and recovers on fresh input', async () => {
const calls = [];
let resets = 0;
const h = harness({
runAction: async (name, args) => {
calls.push(name === 'move_camera' ? args.motion : name);
return name === 'zoom_to_globe' && resets++ === 0
? { ok: false, action: name, cancelled: true }
: { ok: true, action: name };
},
});
try {
await h.c.start();
establishToolInput(h, 'reset to globe view and rotate');
const ws = h.sockets[0];
ws.message({
toolCall: {
functionCalls: [
{ id: 'reset-failed', name: 'zoom_to_globe', args: {} },
{
id: 'rotate-skipped',
name: 'move_camera',
args: { motion: 'rotate_globe' },
},
{ id: 'stop-allowed', name: 'move_camera', args: { motion: 'stop' } },
],
},
});
await h.c.toolQueue;
assert.deepEqual(calls, ['zoom_to_globe', 'stop']);
const skipped = toolResponses(ws).find(
(call) => call.id === 'rotate-skipped',
).response;
assert.equal(skipped.ok, false);
assert.equal(skipped.reason, 'globe-reset-failed');
assert.equal(skipped.skipped, true);
assert.equal(
toolResponses(ws).find((call) => call.id === 'stop-allowed').response.ok,
true,
);
ws.message({ serverContent: { turnComplete: true } });
establishToolInput(h, 'reset to globe view and rotate');
ws.message({
toolCall: {
functionCalls: [
{ id: 'reset-success', name: 'zoom_to_globe', args: {} },
{
id: 'rotate-success',
name: 'move_camera',
args: { motion: 'rotate_globe' },
},
],
},
});
await h.c.toolQueue;
assert.deepEqual(calls, [
'zoom_to_globe',
'stop',
'zoom_to_globe',
'rotate_globe',
]);
} finally {
h.c.stop();
}
});
test('cancelling a queued globe reset also skips its dependent rotation', async () => {
const blocker = deferred();
const calls = [];
const h = harness({
runAction: async (name, args) => {
calls.push(name === 'move_camera' ? args.motion : name);
return args.motion === 'pan'
? blocker.promise
: { ok: true, action: name };
},
});
try {
await h.c.start();
establishToolInput(h, 'pan, reset to globe view and rotate');
const ws = h.sockets[0];
ws.message({
toolCall: {
functionCalls: [
{
id: 'blocker',
name: 'move_camera',
args: { motion: 'pan', direction: 'left' },
},
{ id: 'reset-cancelled', name: 'zoom_to_globe', args: {} },
{
id: 'rotate-after-cancel',
name: 'move_camera',
args: { motion: 'rotate_globe' },
},
],
},
});
await tick();
ws.message({ toolCallCancellation: { ids: ['reset-cancelled'] } });
blocker.resolve({ ok: true, action: 'move_camera' });
await h.c.toolQueue;
assert.deepEqual(calls, ['pan']);
const skipped = toolResponses(ws).find(
(call) => call.id === 'rotate-after-cancel',
).response;
assert.equal(skipped.reason, 'globe-reset-failed');
assert.equal(skipped.skipped, true);
} finally {
blocker.resolve({ ok: true });
h.c.stop();
}
});
+5 -1
View File
@@ -1,3 +1,4 @@
import { interruptGlobeRotation } from '../cameraVerbs.js';
import {
layerSnapshot,
feedProvenanceEnvelope,
@@ -262,7 +263,9 @@ export function createGevActionRunner({
}),
));
installViewTargetPrewarm(viewer);
initCameraVerbs(viewer, getViewTargetCartesian);
initCameraVerbs(viewer, getViewTargetCartesian, {
isCockpitActive: () => Boolean(styleManager?.getCockpitState?.()?.active),
});
// Nested calls (select_nearest_aircraft composing layer + flight) use the
// raw runner; only the outer call gets a voice envelope.
const runGevAction = async function runGevAction(
@@ -298,6 +301,7 @@ export function createGevActionRunner({
// Zoom during an active orbit adjusts the orbit RADIUS (spiral in/out) —
// a straight camera move would be snapped back by the per-frame lookAt.
if (name === 'adjust_camera_zoom') {
interruptGlobeRotation('voice-zoom-intent');
const zoomOut = String(args.direction || '').toLowerCase() === 'out';
const amt = String(args.amount || 'medium').toLowerCase();
const factor = { little: 1.25, medium: 1.6, lot: 2.4 }[amt] || 1.6;
File diff suppressed because it is too large. Load diff
+26
View File
@@ -99,6 +99,18 @@ export function hasStructuredViewIdentity(result) {
);
}
export function isGlobeViewRefusal(result) {
return (
result?.action === 'move_camera' &&
result.ok === false &&
[
'globe-view-required',
'globe-reset-unavailable',
'globe-reset-failed',
].includes(result.reason)
);
}
export function responseInstructionForToolResult(result) {
if (result?.action === 'control_radio' && result.radioPlaybackSuppressed) {
if (result.audioState === 'paused') {
@@ -123,6 +135,20 @@ export function responseInstructionForToolResult(result) {
: 'Be concise: useful names, place, layer and one or two notable properties.',
].join(' ');
}
if (isGlobeViewRefusal(result)) {
// A refusal is not a completed action. Keep tool text out of instructions.
if (result.reason === 'globe-reset-failed')
return 'The globe did not rotate after the Globe View reset attempt. Say so briefly. Do not retry, offer or perform another reset for this request.';
if (result.reason === 'globe-reset-unavailable')
return 'The globe did not rotate. Ask the user to zoom out further; the Globe View preset does not fit this viewport. Do not offer or perform a reset.';
return 'The globe did not rotate. Say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" The suggested command is not a user request. Do not ask a question, perform a reset, or call tools in this reply.';
}
if (
(result?.action ?? result?.tool) === 'zoom_to_globe' &&
(result.ok !== true || result.cancelled)
) {
return 'The Globe View reset did not finish. Say so briefly; do not claim Globe View was reached. Do not retry the reset or rotate the globe for this request.';
}
if (result?.action === 'adjust_camera_zoom') {
return result.ok
? `Confirm once that the camera zoomed ${result.direction}. Do not claim any other change.`
+229 -10
View File
@@ -6,6 +6,7 @@ import {
SUPERSEDED_RESPONSE_MEMORY,
shouldStopVoiceAfterRadioTool,
responseInstructionForToolResult,
isGlobeViewRefusal,
extractFunctionCalls,
callDedupeKeys,
parseArguments,
@@ -66,6 +67,11 @@ export class RealtimeTurns {
this.pendingUserTextResponse = false;
this.activeResponseId = null;
this.supersededResponseIds = new Set();
this.globeRefusals = new Map();
this.globeAdmissionGates = new Map();
this.globeResets = new Map();
this.globeResponseRequests = new Map();
this.globeRequest = { failed: false, pending: new Set() };
this.activeToolAbortControllers = new Set();
this.narration = null;
this.narrationSequence = 0;
@@ -138,6 +144,7 @@ export class RealtimeTurns {
},
};
if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return;
this.globeRequest = { failed: false, pending: new Set() };
this.requestUserTextResponse();
}
@@ -330,6 +337,7 @@ export class RealtimeTurns {
this.userTurnPending = true;
this.pendingResponseInstructions = null;
this.cancelRadioHandoff({ abortTools: true });
this.globeRequest = { failed: false, pending: new Set() };
this.setVoiceSpeaker('user');
// Point-and-ask: read the pointer as speech starts (open mic), or reuse
// the push-to-talk keydown snapshot, before the user audio commits.
@@ -384,6 +392,27 @@ export class RealtimeTurns {
}
const toolResponseId = payload.response_id || payload.response?.id || null;
// Follow-up responses retain reset ownership until fresh admitted input.
const arrivalRequest = this.globeRequest;
const globeRequest =
this.globeResponseRequests.get(toolResponseId) ?? arrivalRequest;
// Later siblings must wait for globe admission before they can mutate state.
// Other tool responses keep their existing concurrency.
let heldForAdmission = false;
while (toolResponseId && this.globeAdmissionGates.has(toolResponseId)) {
// Drop claimed duplicates before every wait, before their keys can expire.
if (
calls.every((call) =>
callDedupeKeys(call).some((key) => this.processedCalls.has(key)),
)
)
return;
heldForAdmission = true;
await this.globeAdmissionGates.get(toolResponseId).promise;
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
return;
}
// A newer typed command superseded the response these calls belong to.
// They are stale intent — dispatching one would let the old turn mutate
// the map after the operator asked for something else.
@@ -394,7 +423,10 @@ export class RealtimeTurns {
// hazard `callDedupeKeys` is written to avoid). So each refused call gets
// a terminal output saying plainly that the turn moved on. No
// `response.create` follows — the deferred typed turn is the single answer.
if (this.isSupersededResponse(toolResponseId)) {
if (
this.isSupersededResponse(toolResponseId) ||
(heldForAdmission && arrivalRequest !== this.globeRequest)
) {
this.pruneProcessedCalls();
for (const call of calls) {
const keys = callDedupeKeys(call);
@@ -429,7 +461,19 @@ export class RealtimeTurns {
if (keys.some((key) => this.processedCalls.has(key))) continue;
keys.forEach((key) => this.processedCalls.set(key, performance.now()));
if (this.globeRefusals.get(toolResponseId)?.blocking) {
this.sendToolOutput(call.call_id || call.id, {
ok: false,
action: call.name,
skipped: true,
error: 'Not run: this response must wait for the Globe View reply.',
});
continue;
}
let result;
let globeAdmissionGate = null;
let globeResetGate = null;
const resultChannel = this.dc;
let radioHandoffEpochAtStart = this.radio.radioHandoffEpoch;
let toolController = null;
@@ -443,6 +487,80 @@ export class RealtimeTurns {
this.narration?.preamble();
try {
const parsedArguments = parseArguments(call.arguments);
if (
toolResponseId &&
call.name === 'move_camera' &&
String(parsedArguments.motion || '').toLowerCase() === 'rotate_globe'
) {
let resolve;
const promise = new Promise((settle) => {
resolve = settle;
});
globeAdmissionGate = { promise, resolve };
this.globeAdmissionGates.set(toolResponseId, globeAdmissionGate);
// OpenAI delivers sibling calls concurrently. A reset dispatched
// first must finish before rotation evaluates the camera's fit.
// Hold later siblings at admission while this rotation waits.
const resets = this.globeResets.get(toolResponseId);
if (
resets ||
globeRequest.failed ||
globeRequest.pending.size ||
arrivalRequest !== this.globeRequest
) {
while (resets?.pending.size || globeRequest.pending.size)
await Promise.all(
[
...new Set([
...(resets?.pending || []),
...globeRequest.pending,
]),
].map((gate) => gate.promise),
);
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
return;
keys.forEach((key) =>
this.processedCalls.set(key, performance.now()),
);
const superseded =
this.isSupersededResponse(toolResponseId) ||
arrivalRequest !== this.globeRequest;
if (superseded || resets?.failed || globeRequest.failed) {
result = {
ok: false,
action: call.name,
reason: 'globe-reset-failed',
skipped: true,
...(superseded ? { superseded: true } : {}),
error: superseded
? 'Superseded by a newer command from the operator — this call was not run.'
: 'Not run: the Globe View reset/rotation sequence failed or was cancelled.',
};
}
}
}
if (toolResponseId && call.name === 'zoom_to_globe') {
let resolve;
const promise = new Promise((settle) => {
resolve = settle;
});
let resets = this.globeResets.get(toolResponseId);
if (!resets) {
resets = { pending: new Set(), failed: false };
this.globeResets.set(toolResponseId, resets);
// Retain settled outcomes for later siblings, without evicting an
// active reset that another response still needs to await.
for (const [id, record] of this.globeResets) {
if (this.globeResets.size <= SUPERSEDED_RESPONSE_MEMORY) break;
if (!record.pending.size && id !== toolResponseId)
this.globeResets.delete(id);
}
}
globeResetGate = { promise, resolve, resets, request: globeRequest };
resets.pending.add(globeResetGate);
globeRequest.pending.add(globeResetGate);
}
const isRadioControlCall = call.name === 'control_radio';
const isRadioVisibilityCall =
call.name === 'set_layer_visibility' &&
@@ -499,7 +617,7 @@ export class RealtimeTurns {
authorityDomain: radioAuthorityDomain,
});
}
result = await (this.actionExecutor || this.runner)(
result ??= await (this.actionExecutor || this.runner)(
call.name,
parsedArguments,
{
@@ -515,7 +633,28 @@ export class RealtimeTurns {
radioHandoffEpochAtStart === this.radio.radioHandoffEpoch),
},
);
if (
globeAdmissionGate &&
this.globeResets.has(toolResponseId) &&
result?.reason === 'globe-view-required'
)
result = { ...result, reason: 'globe-reset-failed' };
if (!this.ownsConversation(eventChannel)) return;
if (
isGlobeViewRefusal(result) &&
toolResponseId &&
arrivalRequest === this.globeRequest &&
!this.isSupersededResponse(toolResponseId)
) {
this.globeRefusals.set(toolResponseId, {
result,
followupQueued: false,
blocking: !result.skipped,
});
while (this.globeRefusals.size > SUPERSEDED_RESPONSE_MEMORY) {
this.globeRefusals.delete(this.globeRefusals.keys().next().value);
}
}
if (result?.ok && result.radioPlaybackRequested) {
const sessionIsCurrent =
this.activeToolAbortControllers.has(toolController) &&
@@ -595,6 +734,23 @@ export class RealtimeTurns {
: {}),
};
} finally {
if (globeResetGate) {
const { resets, request } = globeResetGate;
if (result?.ok !== true || result.cancelled) {
resets.failed = true;
request.failed = true;
}
resets.pending.delete(globeResetGate);
request.pending.delete(globeResetGate);
globeResetGate.resolve();
}
if (globeAdmissionGate) {
if (
this.globeAdmissionGates.get(toolResponseId) === globeAdmissionGate
)
this.globeAdmissionGates.delete(toolResponseId);
globeAdmissionGate.resolve();
}
if (toolController) {
this.activeToolAbortControllers.delete(toolController);
this.radio.releaseTool(toolController);
@@ -671,13 +827,38 @@ export class RealtimeTurns {
});
}
if (!this.ownsConversation(eventChannel)) return;
// Keep the Radio handoff wording authoritative even when another tool
// result follows Radio in the same multi-intent response.
this.queueResponseCreate(
responseInstructionForToolResult(
this.radio.pendingRadioPlaybackResult || lastResult,
),
);
const refusal = this.globeRefusals.get(toolResponseId);
// A newer turn may arrive during admission or the viewport-image await.
// Preserve its reply; neither the refusal map nor lastResult may revive
// an old refusal follow-up after supersession.
if (
(this.isSupersededResponse(toolResponseId) ||
arrivalRequest !== this.globeRequest) &&
(refusal || isGlobeViewRefusal(lastResult) || lastResult?.superseded)
) {
// Complete an idle turn below, but leave a newer running tool's status
// authoritative while suppressing this obsolete refusal follow-up.
if (this.activeToolAbortControllers.size) return;
} else if (refusal) {
// Even an earlier sibling finishing late cannot replace the reset instruction.
if (!refusal.followupQueued) {
refusal.followupQueued = true;
this.queueResponseCreate({
instructions: responseInstructionForToolResult(refusal.result),
tool_choice: 'none',
});
}
} else {
// Keep Radio handoff wording authoritative for ordinary multi-tool replies.
const followupResult =
this.radio.pendingRadioPlaybackResult || lastResult;
const instructions = responseInstructionForToolResult(followupResult);
this.queueResponseCreate(
isGlobeViewRefusal(followupResult)
? { instructions, tool_choice: 'none' }
: instructions,
);
}
}
this.setStatus('listening', t('Ask or command'));
}
@@ -966,6 +1147,22 @@ export class RealtimeTurns {
this.userTurnPending = false;
this.activeResponseId =
payload.response?.id || payload.response_id || null;
// Late calls retain only their globe reset/request ownership. Ordinary
// tool admission and concurrency continue to use their existing policy.
if (
this.activeResponseId &&
!this.globeResponseRequests.has(this.activeResponseId)
) {
this.globeResponseRequests.set(
this.activeResponseId,
this.globeRequest,
);
while (this.globeResponseRequests.size > SUPERSEDED_RESPONSE_MEMORY)
this.globeResponseRequests.delete(
this.globeResponseRequests.keys().next().value,
);
}
this.setVoiceSpeaker('ai');
return;
}
@@ -1045,7 +1242,8 @@ export class RealtimeTurns {
const sent = this.sendRealtimeEvent(
{
type: 'response.create',
response: { instructions },
response:
typeof instructions === 'string' ? { instructions } : instructions,
},
'client.response_create.tool_followup',
);
@@ -1057,6 +1255,20 @@ export class RealtimeTurns {
controller.abort();
this.activeToolAbortControllers.clear();
this.radio.clearTools();
this.releaseGlobeResets();
}
releaseGlobeResets() {
for (const resets of this.globeResets.values()) {
if (!resets.pending.size) continue;
resets.failed = true;
for (const gate of resets.pending) {
gate.request.failed = true;
gate.request.pending.delete(gate);
gate.resolve();
}
resets.pending.clear();
}
}
reset() {
@@ -1068,6 +1280,13 @@ export class RealtimeTurns {
this.pendingUserTextResponse = false;
this.activeResponseId = null;
this.supersededResponseIds.clear();
this.globeRefusals.clear();
for (const gate of this.globeAdmissionGates.values()) gate.resolve();
this.globeAdmissionGates.clear();
this.releaseGlobeResets();
this.globeResets.clear();
this.globeResponseRequests.clear();
this.globeRequest = { failed: false, pending: new Set() };
this.narration?.cancel();
this.assistantAudioPlaying = false;
this.speechResponseIds.clear();