From b2fb4a261e38557715c956e72ad2a3f9203b5a7a Mon Sep 17 00:00:00 2001 From: manjunath22466 Date: Sat, 24 Oct 2026 03:21:42 +0530 Subject: [PATCH] feat(camera): add globe rotation (#1225) Add continuous globe rotation with direction and speed controls, viewport fit checks, camera ownership handoff, and voice refusal guards. --- README.md | 4 + changelog.d/1225-globe-rotation.md | 2 + docs/CURRENT-STATE.md | 2 +- scripts/format-scope.json | 4 +- scripts/voice-bench/phrases.mjs | 3 + server/providers/openai/instructions.js | 1 + server/providers/openai/toolDescriptions.js | 6 +- src/cameraGlobeRotation.test.mjs | 1322 +++++++++++++++++++ src/cameraHandoff.test.mjs | 2 +- src/cameraVerbs.js | 312 ++++- src/firstRunExperience.test.mjs | 18 +- src/radioMarkup.test.mjs | 29 + src/tooling/voiceBench.test.mjs | 13 + src/ui/applicationShell.js | 2 + src/ui/composition.js | 6 +- src/ui/locationNavigation.js | 1 + src/ui/navigationController.js | 3 + src/voice/actionSchemas.js | 2 +- src/voice/actionSchemas.test.mjs | 47 +- src/voice/geminiSession.js | 50 +- src/voice/geminiSession.test.mjs | 212 +++ src/voice/gevActions.js | 6 +- src/voice/globeResetConfirmation.test.mjs | 1131 ++++++++++++++++ src/voice/realtimeProtocol.js | 26 + src/voice/realtimeTurns.js | 239 +++- 25 files changed, 3407 insertions(+), 36 deletions(-) create mode 100644 changelog.d/1225-globe-rotation.md create mode 100644 src/cameraGlobeRotation.test.mjs create mode 100644 src/voice/globeResetConfirmation.test.mjs diff --git a/README.md b/README.md index 1b1fc46a0..80f052571 100644 --- a/README.md +++ b/README.md @@ -343,6 +343,10 @@ Thirty tools, four jobs — the commands below come straight from the product's **🎥 Direct it** — drone-operator camera verbs: +“Rotate the globe” spins Earth to the right continuously; ask for left, a speed, +or stop. If it refuses, follow its reset-and-rotate or further-zoom guidance; +camera input stops rotation. + > 🗣️ _"Take me to Tokyo."_ · _"Orbit around this area slowly."_ · _"Draw the walking route from the Capitol to Zilker Park."_ → _"Fly the route we just drew."_ · _"Zoom out to a globe view."_ **🖊️ Annotate it** — a whiteboard over the real world: diff --git a/changelog.d/1225-globe-rotation.md b/changelog.d/1225-globe-rotation.md new file mode 100644 index 000000000..844966030 --- /dev/null +++ b/changelog.d/1225-globe-rotation.md @@ -0,0 +1,2 @@ +- Add continuous globe rotation with direction and speed controls and a globe-fit + check before starting (manjunath22466, #1225). diff --git a/docs/CURRENT-STATE.md b/docs/CURRENT-STATE.md index b10f43f3b..994f9a623 100644 --- a/docs/CURRENT-STATE.md +++ b/docs/CURRENT-STATE.md @@ -4982,7 +4982,7 @@ and unreachable upstream (502/504) separately from road geometry. > branch: **2,281 on Node 25.6.1** (allocation suites skipped) = 2,255 on > `main` + 26 route pins; the same tree on Node 24 reports 2,295. -- **Camera verbs** (`src/cameraVerbs.js`) — one motion slot, driven per clock tick. `move_camera` orbits/pans/tilts/rotates; `fly_route` is a cinematic dolly along an existing route annotation. +- **Camera verbs** (`src/cameraVerbs.js`) — one motion slot, driven per clock tick. `move_camera` orbits/pans/tilts/rotates; `fly_route` is a cinematic dolly along an existing route annotation. `rotate_globe` rotates the world-space camera rigidly around Earth's center using its screen-up axis, preserving geocentric range and framing. Left/right describe visible globe motion, while `rotate` turns the camera in place. Omitted globe direction/mode default to right/continuous; explicit values prevail. Other camera motions keep their once default. Admission requires 3D, no Cockpit/tracking owner and the whole globe inside the existing keyhole circle, using the shared keyhole/projection geometry and vertical camera FOV with a rotation-local bounding-sphere cone for off-axis containment. Canvas-edge clipping alone does not block a rigid rotation or its unchanged Globe View preset. There is no altitude cutoff. An already smaller, fully framed globe rotates directly; repeated nudges, direction and speed changes recheck current framing without demanding a reset merely because ellipsoid altitude varied. A close-up or overlap refuses without moving the camera. When the unchanged Globe View preset fits, voice says “Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’” without asking a permission question or automatically resetting. Explicit Reset/Show Globe View commands remain direct; an explicit reset-and-rotate request waits for reset success before requesting rotation. If OpenAI dispatches both tools together, rotation waits for earlier same-response resets and is skipped if any fails or is cancelled. OpenAI rejects later tool calls in the same response after a globe refusal, including calls arriving while globe admission is pending, and preserves response-scoped `tool_choice: none` for the refusal follow-up; Gemini rejects further tool execution, including queued sibling calls, until fresh typed input or admitted speech. Model interruption, advisory captions and microphone toggles do not release that guard. There is no pending Yes/No offer. Outside the refused response, explicit-command interpretation and avoiding repeated resets remain model-guided; a distinct model-issued reset call is not an app-enforced proof of user intent. A failed standalone reset can still be retried by the model, and a reset result does not guarantee the camera has fully settled. Before suggesting a reset, a cloned camera checks the unchanged Globe View destination against the same keyhole/depth rules without moving the live camera. If that preset cannot fit, `globe-reset-unavailable` asks for further zoom and the refusal instructs further zoom instead of a reset; this also covers ultrawide views. Unsupported view geometry or non-WGS84 projection assumptions fail closed without the closer-zoom instruction. Supported camera inputs/navigation intents cancel rotation before their action. Finite radius greater than the ellipsoid's maximum radius proves whole-arc clearance at constant centre-distance; no low-altitude swept solver is needed. The unchanged Globe View reset preset remains 18,000,000 m. Terrain and buildings remain outside the guarantee. - **The route dolly is shaped, not linear** (2026-08-20). A trapezoid speed profile (smoothstep up, cruise, smoothstep down, distance taken as the closed-form integral) eases both ends without changing the pace — duration is still `totalM / ROUTE_M_S[speed]`, with one exception: a 0.5 s minimum keeps a degenerate route from being an instant teleport, so routes under 10 m (slow) / 20 m (normal) / 45 m (fast) fly SLOWER than the speed word, never faster. Turns bank up to **10°** (a 90° street corner settles near 7.5°), measured as a triangular pulse over a 4 s window centred on the camera so the roll leads in and unwinds after. Altitude breathes ±20 m around the 260 m mean and lifts up to 26 m into turns. Pitch is LOCKED at −32°; heading comes from a gaze that leads the path by 6.5 s of travel. `prefers-reduced-motion` zeroes the roll and the altitude shaping and keeps the easing. - **Invariants — do not "fix" these.** (1) **Every** release levels the roll: completion unwinds it through the ease-out envelope, and `interruptCameraMotion` zeroes it synchronously (heading/pitch/position preserved). Cesium keeps the last up vector it was handed, so skipping this leaves the user holding a tilted horizon. (2) Heading is interpolated as an ANGLE about the local up, never as a Cartesian lerp — a lerp cannot cross an antipodal pair, so an out-and-back route looked backwards for the entire return leg; `signedTurnRad` branches the exact-180° case deterministically because the cross product's sign there is a floating-point coin toss. (3) A COLD floor cell is missing data, not flat ground: route vertices carry height 0, so trusting them flew a mountain corridor at 260 m above the ELLIPSOID. The corridor warm is fire-and-forget, so the dolly ARMS — camera untouched, no teleport onto the route — for up to 1.2 s waiting for real floor data, falls back to a rendered-mesh probe (`scene.sampleHeight`) when the DEM stays cold — latched to ONE firing per flight, for exactly the cells the cache could not answer, so a route costs at most 8 `sampleHeight` calls however long it arms — and otherwise holds the launch altitude for the whole route. A corridor is only RESOLVED when every cell is accounted for: one warm cell says nothing about the ground under the other seven, and treating it as an answer let the dolly descend to 460 m over a 1,600 m rendered surface. The probe reads the rendered surface at the current LOD (rooftops and primitives included) — a better estimate than nothing, not a guaranteed upper bound. Never descend blind. The floor is SMOOTHED in both directions for the ride while the hard clearance clamp reads the RAW sample, so a cell boundary cannot pop the eye but a cliff is still cleared on the frame it is seen. A pre-departure floor is adopted whole; one arriving mid-flight is eased onto, and the eye's descent rate is capped. - Pins: `src/routeCinematics.test.mjs` (26 tests), with `scripts/qa-flyroute-mutations.mjs` reverting each fix individually to prove they are load-bearing (19 named defects); rendered proof `scripts/qa-flyroute-cinema.mjs` (drives the real voice runner, measures the real Cesium camera every frame, writes a labelled contact sheet). diff --git a/scripts/format-scope.json b/scripts/format-scope.json index e313ade42..7b7171c68 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -1086,5 +1086,7 @@ "src/ui/statusBadge.test.mjs", "src/ui/cctvBadges.test.mjs", "scripts/qa-status-badges.mjs", - "src/dockerTrustedPeers.test.mjs" + "src/dockerTrustedPeers.test.mjs", + "src/cameraGlobeRotation.test.mjs", + "src/voice/globeResetConfirmation.test.mjs" ] diff --git a/scripts/voice-bench/phrases.mjs b/scripts/voice-bench/phrases.mjs index 2f3f73a4c..9d1bcc8ba 100644 --- a/scripts/voice-bench/phrases.mjs +++ b/scripts/voice-bench/phrases.mjs @@ -116,6 +116,9 @@ export const CORE_PHRASES = Object.freeze([ // — camera verbs (tools #23/#24) — { phrase: 'Orbit around this area slowly', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'orbit' && a.speed === 'slow')] }, { phrase: 'Pan left a bit', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'pan' && a.direction === 'left')] }, + { phrase: 'Rotate the globe to the right', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'rotate_globe' && (a.direction === undefined || a.direction === 'right'))] }, + { phrase: 'Keep spinning the Earth slowly to the left', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'rotate_globe' && a.direction === 'left' && a.speed === 'slow' && (a.mode === undefined || a.mode === 'continuous'))] }, + { phrase: 'Turn the camera to look right', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'rotate' && a.direction === 'right')] }, { phrase: 'Stop moving the camera', expect: 'move_camera', typedArgs: [where('move_camera', (a) => a.motion === 'stop')] }, { phrase: 'Fly the route we just drew', expect: 'fly_route' }, diff --git a/server/providers/openai/instructions.js b/server/providers/openai/instructions.js index 7d0e35636..9735f300c 100644 --- a/server/providers/openai/instructions.js +++ b/server/providers/openai/instructions.js @@ -16,6 +16,7 @@ function realtimeInstructions( 'For "what is this aircraft?", answer with get_entity_context\'s identityLine: callsign, operator, type and route from returned fields only, airport codes verbatim. Other questions about the selection answer from its properties. Never infer them from the callsign.', 'While a camera motion or route flight is active, a bare "stop" means move_camera{motion:stop} — NOT control_scene and NOT stop_tracking (those need explicit words like "stop the scene" / "stop tracking"). If move_camera stop returns stopped:false and an entity is being tracked, call stop_tracking next — the user means "stop whatever is moving". Flying somewhere while tracking stops the tracking.', 'For camera-motion requests — "orbit around this", "pan left", "tilt up", "stop moving" — call move_camera. For "fly the route" over a drawn route, call fly_route. Confirm with the RESULTING state ("Orbiting slowly", "Flying the route").', + 'Globe spin: plain Rotate, including a repeat after a refusal, calls only move_camera motion rotate_globe; the app checks fit. For rotation, call zoom_to_globe only for this turn\'s explicit user Reset/Show Globe View request, never from your own plan, an earlier refusal or a bare yes/ok. On globe-view-required say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" Do not ask a question or call more tools for that request. Explicit reset-and-rotate: await reset success before rotate_globe; never both together. After a failed reset, do not retry for that request. Local turn/look left/right: move_camera motion rotate in that direction, not pan.', 'analyst_query ANSWERS questions; it never moves the camera or starts tracking. For requests to FOLLOW or TRACK a specific aircraft/ship, call track_entity (get_entity_context first when the target is ambiguous), never analyst_query as the final or only action. For "follow/track the nearest aircraft", first call analyst_query with the aircraft layer(s), sortBy=distance, and limit=1, then call track_entity with the returned aircraft identity in the same turn. The lookup alone does not fulfill a follow/track command.', 'For a request to enable an aircraft layer and SELECT or FIND the nearest/closest aircraft near a named place — for example, "Turn on flights and select the closest aircraft to Austin" — call select_nearest_aircraft once. It atomically turns on the requested aircraft layer first, waits for location arrival, refreshes that layer for the destination viewport, filters out landed/on-ground records, and selects the nearest airborne result. A healthy fallback feed is valid data, never an enable failure. Do not also call fly_to_location, set_layer_visibility, analyst_query, track_entity, set_context_mode, or control_cockpit for the same request. SELECT/FIND never implies Contacts or Cockpit unless the user explicitly asks for either mode.', 'For ANALYTICAL questions about loaded layer data — how many / which / fastest / highest / biggest / nearest / next flights, ships, fires, earthquakes, satellites, launches, storms, transit, bikes, bases, datacenters or dams ("how many flights over Texas", "anything above 40,000 feet") — call analyst_query, not get_entity_context. Always set scope by WHERE: "around here" or "near this" while pointing is kind pointer, never view. Filter on the attribute the user names ("MODIS" → sensor eq MODIS). Say the count with its scope in a few words ("37 loaded flights over Texas") plus one or two examples by name. If the needed layer is off, say so and offer to turn it on. "Which of THOSE…" about the last answer → followUp=true with only the new filter/sort.', diff --git a/server/providers/openai/toolDescriptions.js b/server/providers/openai/toolDescriptions.js index 18de877db..b8121a201 100644 --- a/server/providers/openai/toolDescriptions.js +++ b/server/providers/openai/toolDescriptions.js @@ -511,18 +511,18 @@ export const ACTION_DESCRIPTIONS = { }, move_camera: { description: - 'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly").', + 'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly"). Globe rotation (rotate_globe) spins Earth right continuously by default; explicit direction, mode and speed take precedence. It checks fit and may refuse.', $position: 1, parameters: { properties: { direction: { description: - 'Required except for orbit (defaults right/clockwise) and stop.', + 'Required for pan/tilt/rotate; orbit/rotate_globe default right.', $position: 2, }, mode: { description: - 'once = bounded eased nudge (default); continuous = until stop/manual input.', + 'once = bounded eased nudge; continuous = until stop/manual input. rotate_globe defaults continuous; other motions default once.', $position: 2, }, }, diff --git a/src/cameraGlobeRotation.test.mjs b/src/cameraGlobeRotation.test.mjs new file mode 100644 index 000000000..db9cbd852 --- /dev/null +++ b/src/cameraGlobeRotation.test.mjs @@ -0,0 +1,1322 @@ +import { GLOBE_VIEW } from './locations.js'; +import { projectEarthDiscToViewport } from './celestialRing.js'; +import assert from 'node:assert/strict'; +import test from 'node:test'; +import * as Cesium from 'cesium'; +import { + activeCameraMotionId, + getActiveCameraMotion, + initCameraVerbs, + interruptCameraMotion, + interruptGlobeRotation, + interruptCameraMotionIfActive, + moveCamera, +} from './cameraVerbs.js'; +import { getRenderGovernorDiagnostics } from './renderGovernor.js'; + +const radians = Cesium.Math.toRadians; +const clone = (value) => Cesium.Cartesian3.clone(value); +const longitude = (camera) => + Cesium.Cartographic.fromCartesian(camera.positionWC).longitude; +const wrappedAngle = Cesium.Math.negativePiToPi; +const hasRenderHold = () => + getRenderGovernorDiagnostics().holds.includes('camera-verb'); + +function near(actual, expected, tolerance = 1e-10, message = '') { + assert.ok( + Math.abs(actual - expected) <= tolerance, + `${message}: expected ${expected}, received ${actual}`, + ); +} + +function pose(camera) { + return { + position: clone(camera.positionWC), + direction: clone(camera.directionWC), + up: clone(camera.upWC), + }; +} + +function samePose(camera, expected) { + const actual = pose(camera); + for (const key of ['position', 'direction', 'up']) { + near( + Cesium.Cartesian3.distance(actual[key], expected[key]), + 0, + key === 'position' ? 1e-6 : 1e-10, + key, + ); + } +} + +function orthonormalBasis(camera) { + const { directionWC: direction, upWC: up, rightWC: right } = camera; + for (const vector of [direction, up, right]) { + near(Cesium.Cartesian3.magnitude(vector), 1, 1e-12, 'unit camera basis'); + } + near(Cesium.Cartesian3.dot(direction, up), 0, 1e-12); + near(Cesium.Cartesian3.dot(direction, right), 0, 1e-12); + near(Cesium.Cartesian3.dot(up, right), 0, 1e-12); + near( + Cesium.Cartesian3.distance( + Cesium.Cartesian3.cross(direction, up, new Cesium.Cartesian3()), + right, + ), + 0, + 1e-12, + 'right-handed camera basis', + ); +} + +/** Use Cesium itself as an independent oracle for the swept camera path. */ +function arcHeights(camera, axis, angle, ellipsoid = Cesium.Ellipsoid.WGS84) { + const heights = []; + for (let i = 0; i <= 64; i += 1) { + const probe = Cesium.Camera.clone(camera); + probe.rotate(axis, (angle * i) / 64); + heights.push(ellipsoid.cartesianToCartographic(probe.positionWC).height); + } + return heights; +} + +/** Camera-space screen coordinates, before FOV/aspect scaling. */ +function projection(camera, target = Cesium.Cartesian3.ZERO) { + const offset = Cesium.Cartesian3.subtract( + target, + camera.positionWC, + new Cesium.Cartesian3(), + ); + const depth = Cesium.Cartesian3.dot(offset, camera.directionWC); + return { + x: Cesium.Cartesian3.dot(offset, camera.rightWC) / depth, + y: Cesium.Cartesian3.dot(offset, camera.upWC) / depth, + depth, + }; +} + +/** Real Cesium geometry with no renderer, GPU, provider or wall-clock waits. */ +function harness( + t, + { + latitude = 15, + longitudeDeg = 0, + height = GLOBE_VIEW.heightM, + heading = 0, + pitch = -90, + roll = 0, + ellipsoid = Cesium.Ellipsoid.WGS84, + viewportWidth = 1280, + viewportHeight = 800, + } = {}, +) { + const realPerformance = globalThis.performance; + let nowMs = 0; + globalThis.performance = { now: () => nowMs }; + const canvas = Object.assign(new EventTarget(), { + clientWidth: viewportWidth, + clientHeight: viewportHeight, + width: viewportWidth, + height: viewportHeight, + }); + const scene = { + canvas, + drawingBufferWidth: viewportWidth, + drawingBufferHeight: viewportHeight, + frameState: { mode: Cesium.SceneMode.SCENE3D }, + mapProjection: new Cesium.GeographicProjection(ellipsoid), + globe: { ellipsoid }, + mapMode2D: Cesium.MapMode2D.INFINITE_SCROLL, + mode: Cesium.SceneMode.SCENE3D, + pixelRatio: 1, + tweens: [], + requestRender() {}, + }; + const camera = new Cesium.Camera(scene); + scene.camera = camera; + camera.setView({ + destination: Cesium.Cartesian3.fromDegrees( + longitudeDeg, + latitude, + height, + ellipsoid, + ), + orientation: { + heading: radians(heading), + pitch: radians(pitch), + roll: radians(roll), + }, + }); + const viewer = { + camera, + scene, + trackedEntity: undefined, + trackedEntityChanged: new Cesium.Event(), + clock: { onTick: new Cesium.Event() }, + }; + const target = Cesium.Cartesian3.fromDegrees( + longitudeDeg, + latitude, + 0, + ellipsoid, + ); + initCameraVerbs(viewer, () => target); + t.after(() => { + interruptCameraMotion('test-cleanup'); + globalThis.performance = realPerformance; + }); + const tick = (seconds = 0.1) => { + nowMs += seconds * 1000; + viewer.clock.onTick.raiseEvent(viewer.clock); + }; + const finishOnce = () => { + for (let i = 0; i < 20 && getActiveCameraMotion(); i += 1) tick(); + assert.equal(getActiveCameraMotion(), null, 'once motion must finish'); + }; + return { viewer, camera, canvas, target, tick, finishOnce }; +} + +for (const [name, sceneMode] of [ + ['2D', Cesium.SceneMode.SCENE2D], + ['Columbus', Cesium.SceneMode.COLUMBUS_VIEW], + ['morphing', Cesium.SceneMode.MORPHING], +]) { + test(`globe rotation refuses ${name} before changing the prior owner`, (t) => { + const { viewer, camera } = harness(t); + assert.equal( + moveCamera({ motion: 'rotate', direction: 'right', mode: 'continuous' }) + .ok, + true, + ); + const priorId = activeCameraMotionId(); + const before = pose(camera); + const transform = Cesium.Matrix4.clone(camera.transform); + viewer.scene.mode = sceneMode; + let handoffs = 0; + let cancellations = 0; + camera.cancelFlight = () => { + cancellations += 1; + }; + for (const mode of ['once', 'continuous']) { + const result = moveCamera( + { motion: 'rotate_globe', direction: 'left', mode }, + () => { + handoffs += 1; + return { ok: true }; + }, + ); + assert.equal(result.ok, false); + assert.match(result.error, /3D/); + } + assert.equal(handoffs, 0); + assert.equal(cancellations, 0); + assert.equal(activeCameraMotionId(), priorId); + assert.equal(hasRenderHold(), true); + assert.ok(Cesium.Matrix4.equals(camera.transform, transform)); + samePose(camera, before); + }); + + test(`active globe rotation yields to ${name} without changing the camera`, (t) => { + const { viewer, camera, tick } = harness(t); + assert.equal( + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }).ok, + true, + ); + viewer.scene.mode = sceneMode; + const before = pose(camera); + let rotations = 0; + camera.rotate = () => { + rotations += 1; + }; + tick(); + assert.equal(rotations, 0); + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); + samePose(camera, before); + }); +} + +test('globe rotation yields to tracking acquired and cleared before the next tick', (t) => { + const { viewer, camera, tick } = harness(t); + assert.equal( + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }).ok, + true, + ); + const before = pose(camera); + const entity = { id: 'cockpit-aircraft', position: {} }; + viewer.trackedEntity = entity; + viewer.trackedEntityChanged.raiseEvent(entity); + assert.equal( + getActiveCameraMotion(), + null, + 'tracking acquisition must synchronously yield', + ); + viewer.trackedEntity = undefined; + viewer.trackedEntityChanged.raiseEvent(undefined); + tick(); + samePose(camera, before); + assert.equal(hasRenderHold(), false); +}); + +test('tracking event keeps legacy yaw and an untracked globe rotation active', (t) => { + const { viewer } = harness(t); + assert.equal( + moveCamera({ motion: 'rotate', direction: 'right', mode: 'continuous' }).ok, + true, + ); + const yawId = activeCameraMotionId(); + viewer.trackedEntityChanged.raiseEvent({ id: 'aircraft' }); + assert.equal(activeCameraMotionId(), yawId); + assert.equal( + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }).ok, + true, + ); + const globeId = activeCameraMotionId(); + viewer.trackedEntityChanged.raiseEvent(undefined); + assert.equal(activeCameraMotionId(), globeId); +}); + +test('replaced viewer releases the old tracking listener without duplicate subscriptions', (t) => { + const first = harness(t); + assert.equal(first.viewer.trackedEntityChanged.numberOfListeners, 1); + initCameraVerbs(first.viewer); + assert.equal(first.viewer.trackedEntityChanged.numberOfListeners, 1); + const next = harness(t); + assert.equal(first.viewer.trackedEntityChanged.numberOfListeners, 0); + assert.equal(next.viewer.trackedEntityChanged.numberOfListeners, 1); + assert.equal( + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }).ok, + true, + ); + const id = activeCameraMotionId(); + first.viewer.trackedEntityChanged.raiseEvent({ id: 'stale-aircraft' }); + assert.equal(activeCameraMotionId(), id); + next.viewer.trackedEntityChanged.raiseEvent({ id: 'new-aircraft' }); + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); +}); + +for (const latitude of [0, 30, -60, 89.5]) { + test(`globe rotation preserves Earth framing and radius from latitude ${latitude} degrees`, (t) => { + const { camera, tick, finishOnce } = harness(t, { latitude }); + const before = projection(camera); + const radius = Cesium.Cartesian3.magnitude(camera.positionWC); + const direction = clone(camera.directionWC); + const started = moveCamera({ motion: 'rotate_globe', mode: 'once' }); + assert.equal(started.ok, true); + assert.equal(started.direction, 'right'); + assert.equal(started.mode, 'once'); + for (let i = 0; i < 5; i += 1) { + tick(); + const current = projection(camera); + near(current.x, before.x); + near(current.y, before.y); + near(current.depth, before.depth, 1e-6); + near(Cesium.Cartesian3.magnitude(camera.positionWC), radius, 1e-6); + orthonormalBasis(camera); + } + finishOnce(); + near( + Cesium.Cartesian3.angleBetween(camera.directionWC, direction), + radians(15), + ); + assert.equal(hasRenderHold(), false); + }); +} + +for (const [direction, sign] of [ + ['left', -1], + ['right', 1], +]) { + test(`globe ${direction} means visible landmark motion ${direction}`, (t) => { + const { camera, target, finishOnce } = harness(t, { latitude: 0 }); + const originalLongitude = longitude(camera); + near(projection(camera, target).x, 0); + assert.equal( + moveCamera({ motion: 'rotate_globe', direction, mode: 'once' }).ok, + true, + ); + finishOnce(); + assert.ok(sign * projection(camera, target).x > 0.05); + near( + wrappedAngle(longitude(camera) - originalLongitude), + -sign * radians(15), + ); + }); +} + +for (const view of [ + { label: 'north-up', heading: 0 }, + { label: 'east-up', heading: 90 }, + { label: 'south-up', heading: 180 }, + { label: 'tilted east-up', heading: 90, pitch: -86 }, + { label: 'tilted and rolled', heading: 180, pitch: -86, roll: 35 }, + { label: 'near the pole', latitude: 89.5, heading: 180, roll: -25 }, +]) { + for (const [direction, sign] of [ + ['left', -1], + ['right', 1], + ]) { + test(`globe ${direction} stays screen-horizontal in a ${view.label} view`, (t) => { + const { camera, tick, finishOnce } = harness(t, view); + const ray = new Cesium.Ray( + clone(camera.positionWC), + clone(camera.directionWC), + ); + const intersection = Cesium.IntersectionTests.rayEllipsoid( + ray, + Cesium.Ellipsoid.WGS84, + ); + assert.ok(intersection, 'the original screen center must see the globe'); + const landmark = Cesium.Ray.getPoint( + ray, + intersection.start, + new Cesium.Cartesian3(), + ); + const before = projection(camera); + const radius = Cesium.Cartesian3.magnitude(camera.positionWC); + const originalLandmark = projection(camera, landmark); + near(originalLandmark.x, 0); + near(originalLandmark.y, 0); + // Exercise both bounded and continuous application of the same direction. + const mode = direction === 'left' ? 'once' : 'continuous'; + assert.equal( + moveCamera({ motion: 'rotate_globe', direction, mode }).ok, + true, + ); + if (mode === 'once') finishOnce(); + else { + tick(); + tick(0.25); + } + const after = projection(camera); + const movedLandmark = projection(camera, landmark); + assert.ok( + sign * movedLandmark.x > 0.001, + 'landmark must move in the requested screen direction', + ); + near(movedLandmark.y, originalLandmark.y); + near(after.x, before.x); + near(after.y, before.y); + near(after.depth, before.depth, 1e-6); + near(Cesium.Cartesian3.magnitude(camera.positionWC), radius, 1e-6); + orthonormalBasis(camera); + }); + } +} + +test('equal left and right globe rotations restore the complete camera pose', (t) => { + const { camera, finishOnce } = harness(t, { + latitude: 42, + longitudeDeg: 170, + height: GLOBE_VIEW.heightM, + }); + const before = pose(camera); + moveCamera({ motion: 'rotate_globe', direction: 'right', mode: 'once' }); + finishOnce(); + moveCamera({ motion: 'rotate_globe', direction: 'left', mode: 'once' }); + finishOnce(); + samePose(camera, before); +}); + +test('globe rotation converts an inherited target frame without zooming or recentering', (t) => { + const { camera, target, tick } = harness(t); + camera.lookAt( + target, + new Cesium.HeadingPitchRange(radians(35), radians(-35), 50_000_000), + ); + assert.equal( + Cesium.Matrix4.equals(camera.transform, Cesium.Matrix4.IDENTITY), + false, + ); + const before = pose(camera); + const screen = projection(camera); + const range = Cesium.Cartesian3.magnitude(camera.positionWC); + moveCamera({ + motion: 'rotate_globe', + direction: 'right', + mode: 'continuous', + }); + assert.equal( + Cesium.Matrix4.equals(camera.transform, Cesium.Matrix4.IDENTITY), + true, + ); + samePose(camera, before); + tick(); + tick(); + near(Cesium.Cartesian3.magnitude(camera.positionWC), range, 1e-6); + near(projection(camera).x, screen.x); + near(projection(camera).y, screen.y); +}); + +for (const [speed, degreesPerSecond] of [ + ['slow', 4], + ['normal', 10], + ['fast', 20], +]) { + test(`continuous globe rotation honors ${speed} speed and explicit stop`, (t) => { + const { camera, tick } = harness(t); + const result = moveCamera({ + motion: 'rotate_globe', + mode: 'continuous', + speed, + }); + assert.equal(result.ok, true); + assert.equal(hasRenderHold(), true); + tick(); // First tick initializes the motion's wall-clock anchor. + const before = clone(camera.directionWC); + tick(0.1); + near( + Cesium.Cartesian3.angleBetween(camera.directionWC, before), + radians(degreesPerSecond * 0.1), + ); + assert.equal(getActiveCameraMotion()?.kind, 'rotate_globe'); + const stopped = moveCamera({ motion: 'stop' }); + assert.equal(stopped.stopped, true); + assert.equal(hasRenderHold(), false); + const after = pose(camera); + tick(); + samePose(camera, after); + assert.equal(moveCamera({ motion: 'stop' }).stopped, false); + }); +} + +for (const event of ['pointerdown', 'touchstart', 'wheel']) { + test(`${event} immediately cancels globe rotation and releases the render hold`, (t) => { + const { camera, canvas, tick } = harness(t); + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }); + tick(); + tick(); + const before = pose(camera); + canvas.dispatchEvent(new Event(event)); + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); + tick(); + samePose(camera, before); + }); +} + +test('new navigation can cancel globe rotation before setting a new destination', (t) => { + const { camera, tick } = harness(t); + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }); + tick(); + assert.equal(interruptCameraMotion('explicit-navigation').wasActive, true); + camera.setView({ + destination: Cesium.Cartesian3.fromDegrees(110, -25, 1_000_000), + }); + const destination = pose(camera); + tick(); + samePose(camera, destination); + assert.equal(hasRenderHold(), false); +}); + +test('replacement motion rejects a stale globe owner cancellation', (t) => { + const { camera, tick } = harness(t); + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }); + const globeId = activeCameraMotionId(); + moveCamera({ motion: 'rotate', direction: 'left', mode: 'continuous' }); + const yawId = activeCameraMotionId(); + assert.notEqual(yawId, globeId); + assert.equal(interruptCameraMotionIfActive(globeId).wasActive, false); + assert.equal(getActiveCameraMotion()?.kind, 'rotate'); + const position = clone(camera.positionWC); + tick(); + tick(); + near(Cesium.Cartesian3.distance(camera.positionWC, position), 0, 1e-6); + assert.equal(interruptCameraMotionIfActive(yawId).wasActive, true); + assert.equal(hasRenderHold(), false); +}); + +test('replacing the viewer retires globe motion and its old clock listener', (t) => { + const { viewer, camera, tick } = harness(t); + assert.equal( + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }).ok, + true, + ); + tick(); + const before = pose(camera); + const nextViewer = { + ...viewer, + clock: { onTick: new Cesium.Event() }, + }; + initCameraVerbs(nextViewer); + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); + tick(); + nextViewer.clock.onTick.raiseEvent(nextViewer.clock); + samePose(camera, before); +}); + +for (const direction of ['up', 'down', 'diagonal']) { + test(`invalid globe direction ${direction} is refused before camera authority handoff`, (t) => { + const { camera } = harness(t); + const before = pose(camera); + let handoffs = 0; + const result = moveCamera({ motion: 'rotate_globe', direction }, () => { + handoffs += 1; + }); + assert.equal(result.ok, false); + assert.match(result.error, /left or right/); + assert.equal(handoffs, 0); + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); + samePose(camera, before); + }); +} + +for (const mode of ['once', 'continuous']) { + for (const acquisition of ['tracking', 'reference frame']) { + test(`globe ${mode} yields to new ${acquisition} without moving the new owner`, (t) => { + const { viewer, camera, target, tick } = harness(t); + assert.equal(moveCamera({ motion: 'rotate_globe', mode }).ok, true); + tick(); + tick(); + assert.equal(hasRenderHold(), true); + const tracked = { id: 'new-tracked-aircraft' }; + if (acquisition === 'tracking') viewer.trackedEntity = tracked; + else + camera.lookAt( + target, + new Cesium.HeadingPitchRange(radians(35), radians(-40), 1_000), + ); + const before = pose(camera); + const transform = Cesium.Matrix4.clone(camera.transform); + tick(); + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); + assert.equal( + viewer.trackedEntity, + acquisition === 'tracking' ? tracked : undefined, + ); + assert.equal(Cesium.Matrix4.equals(camera.transform, transform), true); + samePose(camera, before); + tick(); + samePose(camera, before); + }); + } +} + +test('direct globe rotation refuses a tracked camera without disturbing its reference frame', (t) => { + const { viewer, camera, target } = harness(t); + viewer.trackedEntity = { id: 'tracked-aircraft' }; + camera.lookAt(target, new Cesium.HeadingPitchRange(0, radians(-40), 1_000)); + const before = pose(camera); + const transform = Cesium.Matrix4.clone(camera.transform); + const result = moveCamera({ motion: 'rotate_globe' }); + assert.equal(result.ok, false); + assert.match(result.error, /following tracked-aircraft/); + assert.equal(Cesium.Matrix4.equals(camera.transform, transform), true); + samePose(camera, before); + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); +}); + +test('the existing Globe View preset remains unchanged', () => { + assert.deepEqual(GLOBE_VIEW, { + heightM: 18_000_000, + pitchDeg: -90, + durationS: 2.8, + }); +}); +for (const latitude of [0, 90, -90]) + for (const height of [12_500_000, 18_000_000, 30_000_000]) + test(`whole-globe admission depends on framing at latitude ${latitude}, height ${height}m`, (t) => { + const { camera, finishOnce } = harness(t, { latitude, height }); + const radius = Cesium.Cartesian3.magnitude(camera.positionWC); + assert.equal( + moveCamera({ motion: 'rotate_globe', direction: 'left', mode: 'once' }) + .ok, + true, + ); + finishOnce(); + near(Cesium.Cartesian3.magnitude(camera.positionWC), radius, 1e-6); + }); +test('polar preset admission keeps centre-distance although geodetic altitude varies', (t) => { + const { camera, tick } = harness(t, { latitude: 90 }); + const radius = Cesium.Cartesian3.magnitude(camera.positionWC); + assert.ok(Math.min(...arcHeights(camera, camera.upWC, Math.PI * 2)) > 0); + assert.equal( + moveCamera({ motion: 'rotate_globe', mode: 'continuous', speed: 'fast' }) + .ok, + true, + ); + for (let i = 0; i < 45; i++) tick(0.1); + assert.ok( + Cesium.Ellipsoid.WGS84.cartesianToCartographic(camera.positionWC).height < + GLOBE_VIEW.heightM - 1000, + ); + assert.equal(getActiveCameraMotion().kind, 'rotate_globe'); + near(Cesium.Cartesian3.magnitude(camera.positionWC), radius, 1e-6); + moveCamera({ motion: 'stop' }); + assert.equal(hasRenderHold(), false); +}); +test('close-up refusal preserves an inherited target frame', (t) => { + const { camera, target } = harness(t); + camera.lookAt(target, new Cesium.HeadingPitchRange(0, -0.5, 5000)); + const before = pose(camera), + transform = Cesium.Matrix4.clone(camera.transform); + let calls = 0; + const result = moveCamera({ motion: 'rotate_globe' }, () => { + calls++; + }); + assert.equal( + result.error, + 'Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’', + ); + assert.equal(calls, 0); + assert.ok(Cesium.Matrix4.equals(camera.transform, transform)); + samePose(camera, before); +}); +test('navigation callback never releases tracking for globe rotation', (t) => { + const { viewer, camera } = harness(t); + const tracked = (viewer.trackedEntity = { id: 'tracked-aircraft' }); + const before = pose(camera); + let calls = 0; + assert.equal( + moveCamera({ motion: 'rotate_globe' }, () => { + calls++; + viewer.trackedEntity = undefined; + }).ok, + false, + ); + assert.equal(calls, 0); + assert.equal(viewer.trackedEntity, tracked); + samePose(camera, before); +}); +test('direct globe request refuses Cockpit before navigation mutation', (t) => { + const { viewer, camera } = harness(t); + initCameraVerbs(viewer, null, { isCockpitActive: () => true }); + const before = pose(camera); + let calls = 0; + assert.equal( + moveCamera({ motion: 'rotate_globe' }, () => { + calls++; + }).ok, + false, + ); + assert.equal(calls, 0); + samePose(camera, before); + assert.equal(hasRenderHold(), false); +}); + +test('navigation handoff preserves the original world pose even when it levels a rolled camera', (t) => { + const { camera, tick } = harness(t, { + latitude: 50, + heading: 75, + pitch: -86, + roll: 35, + }); + const before = pose(camera); + const result = moveCamera( + { motion: 'rotate_globe', mode: 'continuous' }, + (start) => { + camera.setView({ + orientation: { heading: camera.heading, pitch: camera.pitch, roll: 0 }, + }); + return start(); + }, + ); + assert.equal(result.ok, true); + samePose(camera, before); + const screen = projection(camera); + tick(); + tick(); + near(projection(camera).x, screen.x); + near(projection(camera).y, screen.y); + orthonormalBasis(camera); +}); + +test('direct globe rotation cancels an existing camera flight before acquiring motion', (t) => { + const { camera } = harness(t); + let cancellations = 0; + const original = camera.cancelFlight; + camera.cancelFlight = function () { + cancellations += 1; + return original.call(this); + }; + const before = pose(camera); + assert.equal(moveCamera({ motion: 'rotate_globe' }).ok, true); + assert.equal(cancellations, 1); + samePose(camera, before); +}); + +test('legacy rotate remains local yaw at an unchanged camera position', (t) => { + const { camera, finishOnce } = harness(t, { latitude: 0 }); + const before = pose(camera); + moveCamera({ motion: 'rotate', direction: 'right' }); + finishOnce(); + near(Cesium.Cartesian3.distance(camera.positionWC, before.position), 0, 1e-6); + near( + Cesium.Cartesian3.angleBetween(camera.directionWC, before.direction), + radians(15), + ); + assert.ok( + Math.abs(projection(camera).x) > 0.2, + 'local yaw intentionally moves Earth out of the center', + ); +}); + +test('legacy orbit still revolves around the selected ground target', (t) => { + const { camera, target, tick } = harness(t); + camera.lookAt(target, new Cesium.HeadingPitchRange(0, radians(-45), 4_000)); + camera.lookAtTransform(Cesium.Matrix4.IDENTITY); + const range = Cesium.Cartesian3.distance(camera.positionWC, target); + moveCamera({ motion: 'orbit', direction: 'right', mode: 'continuous' }); + tick(); + tick(); + assert.equal(getActiveCameraMotion()?.kind, 'orbit'); + near(Cesium.Cartesian3.distance(camera.positionWC, target), range, 1e-6); + near(projection(camera, target).x, 0); + near(projection(camera, target).y, 0); + interruptCameraMotion('test-stop'); + assert.equal( + Cesium.Matrix4.equals(camera.transform, Cesium.Matrix4.IDENTITY), + true, + ); +}); + +// Actual UI/tool authority routes, not a direct call to the interrupt helper. +test('new UI navigation intent stops globe motion before its destination and stale ticks', async (t) => { + const { NavigationController } = await import('./ui/navigationController.js'); + const { viewer, camera, tick } = harness(t); + const navigation = new NavigationController({ + viewer, + tracking: { + flightsLayer: {}, + militaryFlightsLayer: {}, + satellitesLayer: {}, + }, + interruptCameraMotion(reason) { + assert.equal( + getActiveCameraMotion(), + null, + 'the intent stamp must retire rotation before release', + ); + return interruptCameraMotion(reason); + }, + interruptGlobeRotation, + isCockpitActive: () => false, + cancelOrientation() {}, + clearLocation() {}, + cancelShareSelection: () => false, + getDataManager: () => null, + stopOrbit() {}, + showToast() {}, + }); + assert.equal( + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }).ok, + true, + ); + navigation._runExplicitNavigation('camera', () => { + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); + }); + const after = pose(camera); + tick(); + samePose(camera, after); +}); +test('orbit-toggle controller stops globe motion before toggling', async (t) => { + const { LocationNavigation } = await import('./ui/locationNavigation.js'); + const { target } = harness(t); + let toggled = false; + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }); + LocationNavigation.prototype._toggleOrbit.call({ + services: { interruptGlobeRotation }, + _currentTarget: target, + _currentPoi: {}, + orbitController: { + toggle() { + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); + toggled = true; + return true; + }, + }, + _orbitIndicator: { classList: { toggle() {} } }, + }); + assert.equal(toggled, true); +}); +for (const owner of ['tracking', 'cockpit', 'close-up']) + test(`actual voice runner refuses ${owner} without releasing or repositioning`, async (t) => { + const { createGevActionRunner } = await import('./voice/gevActions.js'); + const { runExplicitNavigation } = await import('./navigationPolicy.js'); + const { viewer, camera } = harness(t, { + height: owner === 'close-up' ? 5000 : GLOBE_VIEW.heightM, + }); + const before = pose(camera); + if (owner === 'tracking') viewer.trackedEntity = { id: 'tracked-aircraft' }; + const tracked = viewer.trackedEntity; + let releases = 0, + stamps = 0; + const styleManager = { + getCockpitState: () => ({ active: owner === 'cockpit' }), + runImmediateNavigation(noun, navigate) { + return runExplicitNavigation({ + cockpitActive: owner === 'cockpit', + noun, + stamp() { + stamps++; + }, + release() { + releases++; + viewer.trackedEntity = undefined; + }, + navigate, + }); + }, + }; + const runner = createGevActionRunner({ + viewer, + styleManager, + dataManager: { + layers: new Map(), + isEnabled: () => false, + getAll: () => [], + }, + }); + const result = await runner('move_camera', { + motion: 'rotate_globe', + direction: 'left', + }); + assert.equal(result.ok, false); + assert.equal(releases, 0); + assert.equal(stamps, 0); + assert.equal(viewer.trackedEntity, tracked); + samePose(camera, before); + if (owner === 'close-up') { + assert.equal(result.reason, 'globe-view-required'); + assert.equal( + result.error, + 'Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’', + ); + } + }); + +for (const [label, ellipsoid] of [ + ['nonfinite radii', { minimumRadius: 1, maximumRadius: Infinity }], + ['nonpositive radii', { minimumRadius: 0, maximumRadius: 1 }], + ['non-WGS84 radii', new Cesium.Ellipsoid(30_000_000, 30_000_000, 1_000_000)], +]) + test(`globe rotation refuses ${label} before handoff`, (t) => { + const { viewer, camera } = harness(t, { height: 100_000_000 }); + const before = pose(camera); + viewer.scene.globe.ellipsoid = ellipsoid; + let handedOff = false; + const result = moveCamera({ motion: 'rotate_globe' }, () => { + handedOff = true; + }); + assert.equal(result.ok, false); + assert.equal( + result.reason, + label === 'non-WGS84 radii' ? 'unsupported-view' : 'unsafe-ellipsoid', + ); + assert.equal(handedOff, false); + assert.equal(hasRenderHold(), false); + samePose(camera, before); + }); + +test('an external radius change retires globe rotation without another write', (t) => { + const { camera, tick } = harness(t); + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }); + camera.moveForward(1000); + const changed = pose(camera); + tick(); + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); + samePose(camera, changed); +}); + +test('a non-WGS84 ellipsoid fails closed rather than borrowing the WGS84 silhouette', (t) => { + const ellipsoid = new Cesium.Ellipsoid(10_000_000, 10_000_000, 8_000_000); + const { camera } = harness(t, { ellipsoid, latitude: 60 }); + const before = pose(camera); + const result = moveCamera({ motion: 'rotate_globe' }); + assert.equal(result.ok, false); + assert.equal(result.reason, 'unsupported-view'); + samePose(camera, before); +}); + +test('untouched polar framing permits repeated direction and speed changes below preset altitude', (t) => { + const { camera, finishOnce, tick } = harness(t, { latitude: 90 }); + const radius = Cesium.Cartesian3.magnitude(camera.positionWC); + assert.equal( + moveCamera({ motion: 'rotate_globe', direction: 'right', mode: 'once' }).ok, + true, + ); + finishOnce(); + assert.ok( + Cesium.Cartographic.fromCartesian(camera.positionWC).height < + GLOBE_VIEW.heightM - 1, + ); + assert.equal( + moveCamera({ motion: 'rotate_globe', direction: 'right', mode: 'once' }).ok, + true, + ); + finishOnce(); + assert.equal( + moveCamera({ + motion: 'rotate_globe', + direction: 'left', + speed: 'fast', + mode: 'continuous', + }).ok, + true, + ); + tick(); + tick(); + assert.equal(getActiveCameraMotion().speed, 'fast'); + near(Cesium.Cartesian3.magnitude(camera.positionWC), radius, 1e-6); + assert.equal(moveCamera({ motion: 'stop' }).stopped, true); +}); + +function projected(h) { + return projectEarthDiscToViewport( + h.viewer, + h.canvas.clientWidth, + h.canvas.clientHeight, + ); +} + +test('a smaller globe wholly inside the keyhole needs no overlay entry margin or reset', (t) => { + const h = harness(t, { latitude: 0, height: 12_500_000 }); + const disc = projected(h); + const clearance = + disc.keyholeRadius - + Math.hypot( + disc.earthCenterX - disc.keyholeCenterX, + disc.earthCenterY - disc.keyholeCenterY, + ) - + disc.earthRadius; + assert.ok( + clearance > 0 && clearance < 24, + `near-edge inside fit: ${clearance}`, + ); + assert.equal(moveCamera({ motion: 'rotate_globe' }).ok, true); +}); + +test('keyhole overlap refuses without a camera handoff or pose change', (t) => { + const h = harness(t, { latitude: 0, height: 11_000_000 }); + const before = pose(h.camera); + const disc = projected(h); + assert.ok(disc.earthRadius > disc.keyholeRadius); + let handoffs = 0; + const result = moveCamera({ motion: 'rotate_globe' }, () => { + handoffs++; + }); + assert.equal(result.reason, 'globe-view-required'); + assert.equal( + result.error, + 'Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’', + ); + assert.equal(handoffs, 0); + samePose(h.camera, before); +}); + +for (const [angle, fits] of [ + [5, true], + [25, false], +]) + test(`actual off-centre globe at ${angle} degrees has fit ${fits}`, (t) => { + const h = harness(t, { height: 30_000_000 }); + h.camera.lookRight(radians(angle)); + const before = pose(h.camera); + assert.equal(moveCamera({ motion: 'rotate_globe' }).ok, fits); + samePose(h.camera, before); + }); + +test('wide viewport uses vertical fovy rather than horizontal fov', (t) => { + const h = harness(t, { + viewportWidth: 2000, + viewportHeight: 500, + latitude: 0, + }); + assert.ok(h.camera.frustum.fovy < h.camera.frustum.fov); + assert.ok(projected(h).earthRadius > projected(h).keyholeRadius); + assert.equal( + moveCamera({ motion: 'rotate_globe' }).reason, + 'globe-reset-unavailable', + ); +}); + +for (const invalid of [ + 'orthographic', + 'behind', + 'zero-css-width', + 'offset-frustum', +]) + test(`unsupported ${invalid} fails without a reset offer`, (t) => { + const h = harness(t); + if (invalid === 'orthographic') + h.camera.frustum = new Cesium.OrthographicFrustum({ + width: 50_000_000, + aspectRatio: 1.6, + }); + if (invalid === 'behind') h.camera.lookRight(Math.PI); + if (invalid === 'zero-css-width') h.canvas.clientWidth = 0; + if (invalid === 'offset-frustum') h.camera.frustum.xOffset = 0.1; + const result = moveCamera({ motion: 'rotate_globe' }); + assert.equal(result.ok, false); + assert.equal(result.reason, 'unsupported-view'); + }); + +test('a new request rechecks close-up framing rather than using previous eligibility', (t) => { + const h = harness(t); + assert.equal( + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }).ok, + true, + ); + h.canvas.dispatchEvent(new Event('wheel')); + h.camera.setView({ destination: Cesium.Cartesian3.fromDegrees(0, 0, 5000) }); + const before = pose(h.camera); + assert.equal( + moveCamera({ motion: 'rotate_globe' }).reason, + 'globe-view-required', + ); + samePose(h.camera, before); + assert.equal(getActiveCameraMotion(), null); +}); + +// Exact WGS84 equatorial tangent points expose the shared disc approximation. +// Keep the concrete counterexample public; an overlay approximation is not admission proof. +test('off-axis ellipsoid limb overlap must not be admitted as a fully framed globe', (t) => { + const h = harness(t, { latitude: 0, height: 18_000_000 }); + h.camera.lookRight(radians(6)); + const disc = projected(h); + assert.ok( + disc.earthRadius + + Math.hypot( + disc.earthCenterX - disc.keyholeCenterX, + disc.earthCenterY - disc.keyholeCenterY, + ) < + disc.keyholeRadius, + 'the shared approximation reports a fit', + ); + const radius = Cesium.Ellipsoid.WGS84.maximumRadius; + const x = (radius * radius) / h.camera.positionWC.x; + const y = Math.sqrt(radius * radius - x * x); + const limbRadii = [-y, y].map((value) => { + const pixel = Cesium.SceneTransforms.worldToWindowCoordinates( + h.viewer.scene, + new Cesium.Cartesian3(x, value, 0), + ); + return Math.hypot( + pixel.x - disc.keyholeCenterX, + pixel.y - disc.keyholeCenterY, + ); + }); + assert.ok( + Math.max(...limbRadii) > disc.keyholeRadius, + 'an actual ellipsoid tangent point overlaps the keyhole', + ); + assert.equal(moveCamera({ motion: 'rotate_globe' }).ok, false); +}); + +for (const roll of [0, 35]) + test(`off-axis boundary overlap is refused with roll ${roll}`, (t) => { + const h = harness(t, { latitude: 0, height: 18_000_000, roll }); + h.camera.lookRight(radians(6)); + const result = moveCamera({ motion: 'rotate_globe' }); + assert.equal(result.reason, 'globe-view-required'); + }); + +for (const latitude of [0, 90]) + for (const pitch of [0, 15]) + test(`close-up horizon/upward view offers reset at latitude ${latitude}, pitch ${pitch}`, (t) => { + const h = harness(t, { latitude, pitch, height: 5000 }); + const before = pose(h.camera); + const result = moveCamera({ motion: 'rotate_globe' }); + assert.equal(result.reason, 'globe-view-required'); + assert.equal( + result.error, + 'Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’', + ); + samePose(h.camera, before); + }); + +for (const [viewportWidth, viewportHeight, fits] of [ + [390, 844, true], + [844, 390, true], + [800, 1200, true], + [1280, 800, true], + [3440, 1440, false], +]) + test(`unchanged reset preset is rechecked at ${viewportWidth}x${viewportHeight}`, (t) => { + const h = harness(t, { + viewportWidth, + viewportHeight, + height: GLOBE_VIEW.heightM, + latitude: 0, + }); + const disc = projected(h); + assert.equal( + disc.earthRadius < disc.keyholeRadius, + fits, + 'reset preset keyhole fit', + ); + const before = pose(h.camera); + const result = moveCamera({ motion: 'rotate_globe' }); + assert.equal(result.ok, fits); + if (!fits) assert.equal(result.reason, 'globe-reset-unavailable'); + samePose(h.camera, before); + }); + +test('keyhole overlap after resize stops rotation before another pose write', (t) => { + const h = harness(t); + assert.equal( + moveCamera({ motion: 'rotate_globe', mode: 'continuous' }).ok, + true, + ); + h.canvas.clientWidth = 2000; + h.canvas.clientHeight = 500; + h.camera.frustum.aspectRatio = 4; + assert.ok(projected(h).earthRadius > projected(h).keyholeRadius); + const before = pose(h.camera); + h.tick(); + assert.equal(getActiveCameraMotion(), null); + assert.equal(hasRenderHold(), false); + samePose(h.camera, before); +}); + +for (const clipped of ['near', 'far', 'entirely-beyond-far']) + test(`depth-${clipped} globe is unsupported without a reset offer`, (t) => { + const h = harness(t, { latitude: 0 }); + const radius = Cesium.Ellipsoid.WGS84.maximumRadius; + const distance = Cesium.Cartesian3.magnitude(h.camera.positionWC); + if (clipped === 'near') h.camera.frustum.near = distance - radius + 100; + if (clipped === 'far') h.camera.frustum.far = distance + radius - 100; + if (clipped === 'entirely-beyond-far') + h.camera.frustum.far = distance - 2 * radius; + const before = pose(h.camera); + const result = moveCamera({ motion: 'rotate_globe' }); + assert.equal(result.ok, false); + assert.equal(result.reason, 'unsupported-view'); + samePose(h.camera, before); + assert.equal(hasRenderHold(), false); + }); + +test('fully depth-contained globe remains eligible at actual near/far limits', (t) => { + const h = harness(t, { latitude: 0 }); + const radius = Cesium.Ellipsoid.WGS84.maximumRadius; + const distance = Cesium.Cartesian3.magnitude(h.camera.positionWC); + h.camera.frustum.near = distance - radius - 1; + h.camera.frustum.far = distance + radius + 1; + assert.equal(moveCamera({ motion: 'rotate_globe' }).ok, true); +}); + +for (const [near, far] of [ + [0, 1e9], + [-1, 1e9], + [NaN, 1e9], + [1, Infinity], + [10, 10], + [10, 9], +]) + test(`invalid depth limits ${near}/${far} are unsupported`, (t) => { + const h = harness(t); + h.camera.frustum.near = near; + h.camera.frustum.far = far; + const result = moveCamera({ motion: 'rotate_globe' }); + assert.equal(result.ok, false); + assert.equal(result.reason, 'unsupported-view'); + }); + +test('bare globe rotation defaults right and continues beyond a nudge', (t) => { + const { camera, tick } = harness(t, { latitude: 0 }); + const before = projection(camera); + const result = moveCamera({ motion: 'rotate_globe' }); + assert.equal(result.ok, true); + assert.equal(result.direction, 'right'); + assert.equal(result.mode, 'continuous'); + assert.equal(result.speed, 'normal'); + for (let i = 0; i < 30; i++) tick(0.1); + assert.equal(getActiveCameraMotion().mode, 'continuous'); + near(projection(camera).x, before.x); + near(projection(camera).y, before.y); + assert.equal(moveCamera({ motion: 'stop' }).stopped, true); + assert.equal(hasRenderHold(), false); +}); +for (const [args, direction, mode, speed] of [ + [{ mode: 'once' }, 'right', 'once', 'normal'], + [{ direction: 'left', speed: 'slow' }, 'left', 'continuous', 'slow'], + [{ direction: 'left', mode: 'once', speed: 'fast' }, 'left', 'once', 'fast'], + [ + { direction: 'right', mode: 'continuous', speed: 'fast' }, + 'right', + 'continuous', + 'fast', + ], +]) + test(`explicit globe controls prevail: ${JSON.stringify(args)}`, (t) => { + const { finishOnce, tick } = harness(t); + const result = moveCamera({ motion: 'rotate_globe', ...args }); + assert.deepEqual( + [result.ok, result.direction, result.mode, result.speed], + [true, direction, mode, speed], + ); + if (mode === 'once') finishOnce(); + else { + tick(); + tick(); + assert.equal(getActiveCameraMotion().mode, 'continuous'); + } + }); +for (const motion of ['rotate', 'pan', 'tilt', 'orbit']) + test(`legacy ${motion} retains its once default`, (t) => { + harness(t); + const result = moveCamera({ + motion, + direction: motion === 'tilt' ? 'up' : 'left', + }); + assert.equal(result.ok, true); + assert.equal(result.mode, 'once'); + }); +for (const mode of [null, '', 'invalid', 'CONTINUOUS']) + test(`invalid globe mode ${JSON.stringify(mode)} retains legacy once fallback`, (t) => { + const { finishOnce } = harness(t); + assert.equal(moveCamera({ motion: 'rotate_globe', mode }).mode, 'once'); + finishOnce(); + }); + +test('canvas-only clipping after resize preserves a keyhole-fitting rotation', (t) => { + const h = harness(t, { latitude: 0 }); + assert.equal(moveCamera({ motion: 'rotate_globe' }).ok, true); + h.canvas.clientWidth = 390; + h.canvas.clientHeight = 844; + h.camera.frustum.aspectRatio = 390 / 844; + const disc = projected(h); + assert.ok(disc.earthRadius < disc.keyholeRadius); + assert.ok(disc.earthRadius > h.canvas.clientWidth / 2); + const before = pose(h.camera); + h.tick(); + h.tick(); + assert.notDeepEqual(pose(h.camera).position, before.position); + assert.equal(getActiveCameraMotion()?.kind, 'rotate_globe'); + assert.equal(hasRenderHold(), true); +}); + +test('an invalid camera request leaves the active globe rotation untouched', (t) => { + const h = harness(t); + assert.equal(moveCamera({ motion: 'rotate_globe' }).ok, true); + const before = pose(h.camera); + const active = getActiveCameraMotion(); + assert.equal(moveCamera({ motion: 'pan' }).ok, false); + assert.deepEqual(getActiveCameraMotion(), active); + samePose(h.camera, before); + assert.equal(hasRenderHold(), true); +}); + +test('close-up ultrawide refusal does not offer an unfit reset or move the camera', (t) => { + const h = harness(t, { + viewportWidth: 3440, + viewportHeight: 1440, + height: 5000, + }); + const before = pose(h.camera); + let handoffs = 0; + const result = moveCamera({ motion: 'rotate_globe' }, () => { + handoffs++; + }); + assert.equal(result.reason, 'globe-reset-unavailable'); + assert.equal(handoffs, 0); + samePose(h.camera, before); + assert.equal(hasRenderHold(), false); +}); diff --git a/src/cameraHandoff.test.mjs b/src/cameraHandoff.test.mjs index d91b9d8a4..1fe30ff06 100644 --- a/src/cameraHandoff.test.mjs +++ b/src/cameraHandoff.test.mjs @@ -223,7 +223,7 @@ test('validated voice camera destinations share the UI navigation authority faca 'move camera', ); ordered(move, [ - "if (!['orbit', 'pan', 'tilt', 'rotate'].includes(motion))", + "if (!['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe'].includes(motion))", 'const start = () => {', /const preserveCameraFlight =\s*motion === 'orbit'/, 'runNavigation(start, { preserveCameraFlight })', diff --git a/src/cameraVerbs.js b/src/cameraVerbs.js index 278a86206..231d9357d 100644 --- a/src/cameraVerbs.js +++ b/src/cameraVerbs.js @@ -4,7 +4,7 @@ * * One motion at a time, driven per clock tick. `once` = bounded eased nudge; * `continuous` runs until move_camera{stop}, ANY manual camera input on the - * canvas (pointerdown/wheel — the cancelFlight reflex), or a navigation tool + * canvas (pointerdown/wheel/touchstart — the cancelFlight reflex), or a navigation tool * starts (the shared UI navigation policy releases this owner before the new * destination mutates the camera). fly_route is a * continuous motion in the same slot: a cinematic dolly along an existing @@ -13,14 +13,24 @@ * Orbit circles the current screen-center ground target holding range and * pitch while advancing heading (per-frame camera.lookAt; the lookAt * transform is cleared on every stop so manual control returns cleanly). + * Globe rotation turns the world-frame camera about its screen-up axis through + * Earth's center, preserving range and framing. Left/right name the visible + * globe motion, opposite to the camera's travel around Earth. */ import * as Cesium from 'cesium'; +import { GLOBE_VIEW } from './locations.js'; +import { + projectEarthDiscToViewport, + earthDiscScreenRadius, + getKeyholeGeometry, +} from './celestialRing.js'; import { holdContinuousRender, releaseContinuousRender, } from './renderGovernor.js'; +let _isCockpitActive = null; /** °/s by speed word — orbit; pan uses fractions of view height/s. */ const ORBIT_DEG_S = { slow: 2, normal: 6, fast: 15 }; const TILT_DEG_S = { slow: 4, normal: 10, fast: 20 }; @@ -952,6 +962,7 @@ let _viewer = null; let _getTarget = null; let _active = null; // { kind, motion, direction, speed, mode, motionId, ...state } let _tickRemover = null; +let _trackingRemover = null; let _inputRemovers = []; /** Monotonic id for the motion currently running; 0 means "no motion". */ let _motionSeq = 0; @@ -1162,7 +1173,11 @@ function onTick() { else if (m.direction === 'right') cam.moveRight(step); else if (m.direction === 'up') cam.moveUp(step); else cam.moveDown(step); - } else if (m.kind === 'tilt' || m.kind === 'rotate') { + } else if ( + m.kind === 'tilt' || + m.kind === 'rotate' || + m.kind === 'rotate_globe' + ) { const degS = TILT_DEG_S[m.speed]; const stepRad = m.mode === 'once' @@ -1172,7 +1187,38 @@ function onTick() { (scale - (m.lastScale || 0)) : Cesium.Math.toRadians(degS) * dt; m.lastScale = scale; - if (m.kind === 'tilt') { + if (m.kind === 'rotate_globe') { + // Tracking can acquire the camera without going through navigation. + // Yield without resetting its pose/frame: the world-space arc proof + // only describes Cesium's rotation while the frame is identity. + if ( + _viewer.scene.mode !== Cesium.SceneMode.SCENE3D || + _viewer.trackedEntity || + _isCockpitActive?.() || + !Cesium.Matrix4.equals(cam.transform, Cesium.Matrix4.IDENTITY) + ) { + interruptCameraMotion('globe-camera-owner-changed'); + return; + } + // Cesium rotates both position and basis about the reference-frame + // origin. The captured screen-up axis keeps left/right horizontal even + // in a tilted or rolled view; a positive angle moves the globe right. + const angle = m.direction === 'left' ? -stepRad : stepRad; + if (globeRotationFitsKeyhole() !== true) { + interruptCameraMotion('globe-framing-changed'); + return; + } + if ( + !globeRotationHasClearance(cam.positionWC) || + Math.abs( + Cesium.Cartesian3.magnitude(cam.positionWC) - m.rotationRadius, + ) > 0.01 + ) { + interruptCameraMotion('globe-surface-limit'); + return; + } + cam.rotate(m.rotationAxis, angle); + } else if (m.kind === 'tilt') { const up = m.direction === 'up'; // Clamp pitch to [-89°, -5°]: never through the ground, never at the sky. const next = cam.pitch + (up ? stepRad : -stepRad); @@ -1206,6 +1252,133 @@ function onTick() { } } +/** Rigid origin-centred rotation stays outside the actual ellipsoid's bounding sphere. */ +function globeRotationHasClearance(position) { + const ellipsoid = _viewer.scene.globe?.ellipsoid ?? Cesium.Ellipsoid.WGS84; + const { minimumRadius, maximumRadius } = ellipsoid; + const radius = Cesium.Cartesian3.magnitude(position); + return ( + Number.isFinite(minimumRadius) && + minimumRadius > 0 && + Number.isFinite(maximumRadius) && + maximumRadius >= minimumRadius && + Number.isFinite(radius) && + radius > maximumRadius * (1 + 1e-12) + ); +} + +/** Use the existing WGS84/CSS-pixel projection; null means unsupported, not close-up. */ +function globeRotationFitsKeyhole(camera = _viewer.camera) { + const ellipsoid = _viewer.scene.globe?.ellipsoid ?? Cesium.Ellipsoid.WGS84; + if (!Cesium.Cartesian3.equals(ellipsoid.radii, Cesium.Ellipsoid.WGS84.radii)) + return null; + const canvas = _viewer.scene.canvas; + const width = canvas.clientWidth; + const height = canvas.clientHeight; + const frustum = camera.frustum; + if ( + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 || + !(frustum instanceof Cesium.PerspectiveFrustum) || + !Number.isFinite(frustum.near) || + frustum.near <= 0 || + !Number.isFinite(frustum.far) || + frustum.far <= frustum.near || + frustum.xOffset !== 0 || + frustum.yOffset !== 0 + ) + return null; + try { + const cam = camera; + const distance = Cesium.Cartesian3.magnitude(cam.positionWC); + const sphereRadius = ellipsoid.maximumRadius; + if (!Number.isFinite(distance) || distance <= 0) return null; + if (distance <= sphereRadius) return false; // Near-surface polar views are close-up too. + const radiusPx = earthDiscScreenRadius(distance, height, frustum.fovy); + if (!radiusPx) return null; + const keyhole = getKeyholeGeometry(width, height); + // The known nadir reset probe uses the same exact cone below; projecting + // through SceneTransforms would read the live scene.camera instead. + if ( + camera === _viewer.camera && + !projectEarthDiscToViewport(_viewer, width, height) + ) { + // A near-surface horizon view cannot fit even when pointed at Earth; + // an orbital camera looking away is unsupported, not a close-up offer. + return radiusPx > Math.min(keyhole.radius, width / 2, height / 2) + ? false + : null; + } + // The shared disc is an overlay approximation off-axis. Bound the sphere + // against the existing keyhole cone. Canvas-edge clipping does not change + // framing during rigid globe rotation and is not a clearance condition, + // without changing the UI projection or defining another keyhole/mask. + const toCenter = Cesium.Cartesian3.negate( + cam.positionWC, + new Cesium.Cartesian3(), + ); + const depth = Cesium.Cartesian3.dot(toCenter, cam.directionWC); + const right = Math.abs(Cesium.Cartesian3.dot(toCenter, cam.rightWC)); + const up = Math.abs(Cesium.Cartesian3.dot(toCenter, cam.upWC)); + const focalY = height / (2 * Math.tan(frustum.fovy / 2)); + const focalX = + width / (2 * Math.tan(frustum.fovy / 2) * frustum.aspectRatio); + if ( + ![depth, right, up, focalX, focalY].every(Number.isFinite) || + focalX <= 0 || + focalY <= 0 + ) + return null; + const halfAngle = Math.asin(sphereRadius / distance); + const centerAngle = Math.atan2(Math.hypot(right, up), depth); + const keyholeAngle = Math.atan(keyhole.radius / Math.max(focalX, focalY)); + // No overlay visibility hysteresis: a fully contained smaller globe needs no reset. + const framed = centerAngle + halfAngle <= keyholeAngle; + if (!framed) return false; + // Projection alone does not prove that a tiny globe is within depth planes. + if ( + depth - sphereRadius < frustum.near || + depth + sphereRadius > frustum.far + ) + return null; + return true; + } catch { + return null; + } +} + +/** Check the unchanged reset destination without moving the live camera. */ +function globeResetFitsKeyhole() { + try { + const camera = Cesium.Camera.clone(_viewer.camera); + const { longitude, latitude } = _viewer.camera.positionCartographic; + camera.setView({ + destination: Cesium.Cartesian3.fromRadians( + longitude, + latitude, + GLOBE_VIEW.heightM, + ), + orientation: { + heading: 0, + pitch: Cesium.Math.toRadians(GLOBE_VIEW.pitchDeg), + roll: 0, + }, + endTransform: Cesium.Matrix4.IDENTITY, + }); + return globeRotationFitsKeyhole(camera) === true; + } catch { + return false; + } +} + +/** Release only globe rotation when a new camera intent takes ownership. */ +export function interruptGlobeRotation(reason = 'camera-intent') { + if (_active?.kind === 'rotate_globe') return interruptCameraMotion(reason); + return { wasActive: false, reason, leveled: false }; +} + /** * Wire the module to the viewer once (idempotent). The view-target getter is * optional so a non-voice caller (the Directions FLY chip) can arm route @@ -1218,24 +1391,42 @@ function onTick() { * call brings none, so a callback from the old viewer can never answer for the * new one. */ -export function initCameraVerbs(viewer, getViewTargetCartesian = null) { +export function initCameraVerbs( + viewer, + getViewTargetCartesian = null, + { isCockpitActive } = {}, +) { const nextTarget = typeof getViewTargetCartesian === 'function' ? getViewTargetCartesian : null; if (_viewer === viewer) { if (nextTarget) _getTarget = nextTarget; + if (typeof isCockpitActive === 'function') + _isCockpitActive = isCockpitActive; return; } + // A globe motion belongs to the viewer whose world pose it captured. + if (_active?.kind === 'rotate_globe') + interruptCameraMotion('viewer-replaced'); _getTarget = nextTarget; + _isCockpitActive = + typeof isCockpitActive === 'function' ? isCockpitActive : null; _viewer = viewer; if (_tickRemover) _tickRemover(); _tickRemover = viewer.clock.onTick.addEventListener(onTick); + if (_trackingRemover) _trackingRemover(); + // Cockpit can acquire then clear tracking before the next clock tick. + // Yield on acquisition itself without changing legacy camera motions. + _trackingRemover = viewer.trackedEntityChanged?.addEventListener((entity) => { + if (entity && _active?.kind === 'rotate_globe') + interruptCameraMotion('globe-camera-owner-changed'); + }); for (const rm of _inputRemovers) rm(); _inputRemovers = []; const canvas = viewer.scene.canvas; // ANY manual camera input reclaims control — the cancelFlight reflex. - for (const evt of ['pointerdown', 'wheel']) { + for (const evt of ['pointerdown', 'touchstart', 'wheel']) { const h = () => interruptCameraMotion('manual-input'); canvas.addEventListener(evt, h, { passive: true }); _inputRemovers.push(() => canvas.removeEventListener(evt, h)); @@ -1256,7 +1447,11 @@ export function moveCamera(args = {}, runNavigation = null) { const speed = ORBIT_DEG_S[String(args.speed || 'normal').toLowerCase()] ? String(args.speed || 'normal').toLowerCase() : 'normal'; - const mode = args.mode === 'continuous' ? 'continuous' : 'once'; + const mode = + args.mode === 'continuous' || + (motion === 'rotate_globe' && args.mode === undefined) + ? 'continuous' + : 'once'; if (motion === 'stop') { const wasActiveBeforeHandoff = Boolean(_active); const stop = () => { @@ -1270,15 +1465,16 @@ export function moveCamera(args = {}, runNavigation = null) { }; return typeof runNavigation === 'function' ? runNavigation(stop) : stop(); } - if (!['orbit', 'pan', 'tilt', 'rotate'].includes(motion)) { + if (!['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe'].includes(motion)) { return { ok: false, action: 'move_camera', - error: `Unknown motion "${args.motion}" — use orbit, pan, tilt, rotate, or stop.`, + error: `Unknown motion "${args.motion}" — use orbit, pan, tilt, rotate, rotate_globe, or stop.`, }; } if ( motion !== 'orbit' && + motion !== 'rotate_globe' && !['left', 'right', 'up', 'down'].includes(direction) ) { return { @@ -1294,6 +1490,17 @@ export function moveCamera(args = {}, runNavigation = null) { error: 'rotate needs left or right.', }; } + if ( + motion === 'rotate_globe' && + direction !== null && + !['left', 'right'].includes(direction) + ) { + return { + ok: false, + action: 'move_camera', + error: 'rotate_globe needs left or right.', + }; + } if (motion === 'tilt') { // Honest limits: at the clamp a tilt is a no-op — say so instead of // reporting success and letting the model gaslight itself (field test). @@ -1318,9 +1525,9 @@ export function moveCamera(args = {}, runNavigation = null) { } } if ( - motion === 'orbit' && - _viewer?.trackedEntity && - typeof runNavigation !== 'function' + (motion === 'rotate_globe' || + (motion === 'orbit' && typeof runNavigation !== 'function')) && + _viewer?.trackedEntity ) { // The follow camera owns a tracked view; a lookAt orbit fights it frame // by frame and stop rips the camera out (field finding). Honest refusal @@ -1332,9 +1539,74 @@ export function moveCamera(args = {}, runNavigation = null) { return { ok: false, action: 'move_camera', - error: `Already following ${label} — the follow camera owns the view while tracking. Say "stop tracking" first if you want a free orbit.`, + error: `Already following ${label} — the follow camera owns the view while tracking. Say "stop tracking" first if you want ${motion === 'rotate_globe' ? 'to rotate the globe' : 'a free orbit'}.`, }; } + let globePose; + if (motion === 'rotate_globe') { + if (_viewer.scene.mode !== Cesium.SceneMode.SCENE3D) { + return { + ok: false, + action: 'move_camera', + error: 'Globe rotation requires a 3D view.', + }; + } + const cam = _viewer.camera; + globePose = { + position: Cesium.Cartesian3.clone(cam.positionWC), + direction: Cesium.Cartesian3.clone(cam.directionWC), + up: Cesium.Cartesian3.normalize(cam.upWC, new Cesium.Cartesian3()), + right: Cesium.Cartesian3.clone(cam.rightWC), + }; + if (_isCockpitActive?.()) + return { + ok: false, + action: 'move_camera', + reason: 'camera-owned', + error: 'Exit Cockpit before rotating the globe.', + }; + const ellipsoid = _viewer.scene.globe?.ellipsoid ?? Cesium.Ellipsoid.WGS84; + if (!( + Number.isFinite(ellipsoid.minimumRadius) && + ellipsoid.minimumRadius > 0 && + Number.isFinite(ellipsoid.maximumRadius) && + ellipsoid.maximumRadius >= ellipsoid.minimumRadius && + Number.isFinite(Cesium.Cartesian3.magnitude(globePose.position)) + )) + return { + ok: false, + action: 'move_camera', + reason: 'unsafe-ellipsoid', + error: + 'Globe rotation requires a finite supported ellipsoid and a clear globe view.', + }; + const fits = globeRotationFitsKeyhole(); + if (fits === null) + return { + ok: false, + action: 'move_camera', + reason: 'unsupported-view', + error: 'Globe rotation requires a supported perspective view.', + }; + if (!fits) { + const canReset = globeResetFitsKeyhole(); + return { + ok: false, + action: 'move_camera', + reason: canReset ? 'globe-view-required' : 'globe-reset-unavailable', + error: canReset + ? 'Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’' + : 'Zoom out further to fit the globe inside the keyhole; the Globe View preset does not fit this viewport.', + }; + } + if (!globeRotationHasClearance(globePose.position)) + return { + ok: false, + action: 'move_camera', + reason: 'unsafe-ellipsoid', + error: 'Globe rotation requires a clear ellipsoid bounding sphere.', + }; + } const start = () => { interruptCameraMotion('replaced'); const state = { @@ -1343,6 +1615,22 @@ export function moveCamera(args = {}, runNavigation = null) { speed, mode, }; + if (motion === 'rotate_globe') { + // Convert an inherited ground-target frame without changing the world + // pose. Rotation must pivot about Earth, never the previous local target. + const cam = _viewer.camera; + cam.cancelFlight(); + cam.lookAtTransform(Cesium.Matrix4.IDENTITY); + // Handoff may level a route's bank. Keep the caller's exact world pose. + // Copy the basis directly: setView's heading/pitch conversion is singular + // at nadir and can change the captured orientation near the poles. + Cesium.Cartesian3.clone(globePose.position, cam.position); + Cesium.Cartesian3.clone(globePose.direction, cam.direction); + Cesium.Cartesian3.clone(globePose.up, cam.up); + Cesium.Cartesian3.clone(globePose.right, cam.right); + state.rotationAxis = globePose.up; + state.rotationRadius = Cesium.Cartesian3.magnitude(globePose.position); + } if (motion === 'orbit') { if (_viewer.scene.tweens.length > 0) { // A camera flight is in progress ("fly to X and orbit it") — ALWAYS arm diff --git a/src/firstRunExperience.test.mjs b/src/firstRunExperience.test.mjs index 40d42befc..a02c4391f 100644 --- a/src/firstRunExperience.test.mjs +++ b/src/firstRunExperience.test.mjs @@ -988,6 +988,16 @@ test('the voice TOOL SCHEMA matches the pinned contract — the mission mapping ]); // Cyber deliberately adds one layout; first-run missions still change no tools. hudLayout.enum = hudLayout.enum.filter((layout) => layout !== 'cyber'); + const cameraMotion = legacyTools.find((tool) => tool.name === 'move_camera') + .parameters.properties.motion; + assert.deepEqual(cameraMotion.enum, [ + 'orbit', + 'pan', + 'tilt', + 'rotate', + 'rotate_globe', + 'stop', + ]); const block = JSON.stringify(legacyTools); // Re-derived for the voice layer manifest (generated layer enums, alias and // field hints), point-and-ask's pointer/referent arguments and the prompt @@ -997,10 +1007,14 @@ test('the voice TOOL SCHEMA matches the pinned contract — the mission mapping // list descriptions; the missions still ride existing tools. // Street Level and GNSS extend the manifest; broadcast caveats describe tracked and analyst aircraft. // Air quality (ECCC AQHI) extends the manifest. - assert.equal(block.length, 30655, 'serialized tool schema length drifted'); + // Globe rotation adds one camera motion and its descriptions; Globe View + // framing/default guidance updates only move_camera and mode descriptions. + // Radio markup and the complete production-request comparison pin locality. + // Restore the base camera/Globe View wording with the globe-rotation append. + assert.equal(block.length, 30878, 'serialized tool schema length drifted'); assert.equal( crypto.createHash('sha256').update(block).digest('hex'), - '3b1d5e77c10a92f6543a4f9d1ff1870101a0306b65e5b8a6113111c81eda371e', + '7b95f0f4258896fe5c3316265a19f3a6602809d4e7e681ee7ecd8f2df514ef1f', 'the first-run missions must ride EXISTING tools: no schema edit, no cache bust', ); const instructions = fs.readFileSync( diff --git a/src/radioMarkup.test.mjs b/src/radioMarkup.test.mjs index 23ad778d5..14ea0f3e0 100644 --- a/src/radioMarkup.test.mjs +++ b/src/radioMarkup.test.mjs @@ -209,6 +209,35 @@ test('no unchanged Realtime tool definition drifts silently', () => { const hudLayout = unchanged.find((tool) => tool.name === 'set_hud').parameters.properties.layout; assert.deepEqual(hudLayout.enum, ['tactical', 'operator', 'minimal', 'cyber']); hudLayout.enum = hudLayout.enum.filter((layout) => layout !== 'cyber'); + // Globe rotation adds one motion and clarifies the two affected descriptions. + // Check those edits explicitly, then keep the legacy pin for every other field. + const camera = unchanged.find((tool) => tool.name === 'move_camera'); + assert.deepEqual(camera.parameters.properties.motion.enum, [ + 'orbit', + 'pan', + 'tilt', + 'rotate', + 'rotate_globe', + 'stop', + ]); + assert.equal( + camera.description, + 'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly"). Globe rotation (rotate_globe) spins Earth right continuously by default; explicit direction, mode and speed take precedence. It checks fit and may refuse.', + ); + assert.equal( + camera.parameters.properties.direction.description, + 'Required for pan/tilt/rotate; orbit/rotate_globe default right.', + ); + assert.equal(camera.parameters.properties.mode.description, 'once = bounded eased nudge; continuous = until stop/manual input. rotate_globe defaults continuous; other motions default once.'); + camera.parameters.properties.mode.description = 'once = bounded eased nudge (default); continuous = until stop/manual input.'; + camera.parameters.properties.motion.enum = + camera.parameters.properties.motion.enum.filter( + (motion) => motion !== 'rotate_globe', + ); + camera.description = + 'Direct the camera like a drone operator: orbit the current view target, pan, tilt, or rotate — one bounded nudge (mode=once) or continuous motion until stopped (mode=continuous). Continuous motion also stops on any manual camera input or when a navigation tool runs. Say the RESULTING state when confirming ("Orbiting slowly").'; + camera.parameters.properties.direction.description = + 'Required except for orbit (defaults right/clockwise) and stop.'; assert.equal(unchanged.length, 14); const digest = createHash('sha256') .update(JSON.stringify(unchanged)) diff --git a/src/tooling/voiceBench.test.mjs b/src/tooling/voiceBench.test.mjs index 84c31f3dc..1cc69c8da 100644 --- a/src/tooling/voiceBench.test.mjs +++ b/src/tooling/voiceBench.test.mjs @@ -234,6 +234,19 @@ test('typed argument checks feed full-call accuracy without changing the legacy const inMeters = gradeCore(altitude, [{ name: 'analyst_query', args: { layers: ['flights'], filters: [{ field: 'altitudeM', op: 'gt', value: 12192 }] } }]); assert.equal(inMeters.fullCallOk, true); assert.equal(where('set_hud', (a) => a.visible === 'off')({ name: 'set_hud', args: { visible: 'off' } }), true); + const globe = byPhrase('Keep spinning the Earth slowly to the left'); + const globeArgs = { motion: 'rotate_globe', direction: 'left', speed: 'slow' }; + for (const mode of [undefined, 'continuous', 'once', null]) { + assert.equal(gradeCore(globe, [{ name: 'move_camera', args: { ...globeArgs, mode } }]).fullCallOk, mode === undefined || mode === 'continuous'); + } + assert.equal(gradeCore(globe, [{ name: 'move_camera', args: { ...globeArgs, direction: 'right' } }]).fullCallOk, false); + assert.equal(gradeCore(globe, [{ name: 'move_camera', args: { ...globeArgs, speed: 'normal' } }]).fullCallOk, false); + const right = byPhrase('Rotate the globe to the right'); + assert.equal(gradeCore(right, [{ name: 'move_camera', args: { motion: 'rotate_globe' } }]).fullCallOk, true); + for (const direction of ['right', 'left', null]) { + assert.equal(gradeCore(right, [{ name: 'move_camera', args: { motion: 'rotate_globe', direction } }]).fullCallOk, direction === 'right'); + } + assert.equal(gradeCore(right, [{ name: 'move_camera', args: { motion: 'rotate', direction: 'right' } }]).fullCallOk, false); }); test('gradeDialogue scores turn 2 in the context of turn 1', () => { diff --git a/src/ui/applicationShell.js b/src/ui/applicationShell.js index de37ab036..7c866bc36 100644 --- a/src/ui/applicationShell.js +++ b/src/ui/applicationShell.js @@ -147,6 +147,7 @@ export class StyleManager extends ShellFacade { }, searchInput: this._locationSearch, interruptCameraMotion: services.interruptCameraMotion, + interruptGlobeRotation: services.interruptGlobeRotation, isCockpitActive: () => !!this.cockpitView?.active, clearLocation: () => this.clearSearchedLocation(), cancelShareSelection: () => this._shareRestoration.cancelSelection(), @@ -322,6 +323,7 @@ export class StyleManager extends ShellFacade { GLOBE_VIEW: services.GLOBE_VIEW, flyToGlobeView: services.flyToGlobeView, interruptCameraMotion: services.interruptCameraMotion, + interruptGlobeRotation: services.interruptGlobeRotation, flightsLayer: services.flightsLayer, militaryFlightsLayer: services.militaryFlightsLayer, satellitesLayer: services.satellitesLayer, diff --git a/src/ui/composition.js b/src/ui/composition.js index 797157567..c56c6f37c 100644 --- a/src/ui/composition.js +++ b/src/ui/composition.js @@ -9,7 +9,10 @@ import { flyToPOI, searchAndFlyTo, } from '../locations.js'; -import { interruptCameraMotion } from '../cameraVerbs.js'; +import { + interruptCameraMotion, + interruptGlobeRotation, +} from '../cameraVerbs.js'; import { IntelHUD } from '../hud.js'; import { ShareLinkManager } from '../sharelink.js'; import { OrbitController } from '../orbit.js'; @@ -73,6 +76,7 @@ export class StyleManager extends ApplicationShell { flyToPOI, searchAndFlyTo, interruptCameraMotion, + interruptGlobeRotation, IntelHUD, ShareLinkManager, OrbitController, diff --git a/src/ui/locationNavigation.js b/src/ui/locationNavigation.js index 07ce43d8d..af8fb95b8 100644 --- a/src/ui/locationNavigation.js +++ b/src/ui/locationNavigation.js @@ -283,6 +283,7 @@ export class LocationNavigation { return; } + this.services.interruptGlobeRotation?.('orbit-intent'); const isActive = this.orbitController.toggle(this._currentTarget, { radius: this._currentPoi?.alt || 500, pitch: this._currentPoi?.pitch || -30, diff --git a/src/ui/navigationController.js b/src/ui/navigationController.js index f94eb46fe..437b643f3 100644 --- a/src/ui/navigationController.js +++ b/src/ui/navigationController.js @@ -14,6 +14,7 @@ export class NavigationController { tracking, searchInput, interruptCameraMotion, + interruptGlobeRotation = () => {}, isCockpitActive, clearLocation, cancelShareSelection, @@ -27,6 +28,7 @@ export class NavigationController { tracking, searchInput, interruptCameraMotion, + interruptGlobeRotation, isCockpitActive, clearLocation, cancelShareSelection, @@ -46,6 +48,7 @@ export class NavigationController { cancelPendingSelection = true, clearSearchedLocation = true, } = {}) { + this.interruptGlobeRotation('navigation-intent'); cancelCameraArrival(this.viewer); this.cancelOrientation(); const { flightsLayer, militaryFlightsLayer, satellitesLayer } = diff --git a/src/voice/actionSchemas.js b/src/voice/actionSchemas.js index d0810bf71..3f464cad1 100644 --- a/src/voice/actionSchemas.js +++ b/src/voice/actionSchemas.js @@ -729,7 +729,7 @@ const schemas = [ properties: { motion: { type: 'string', - enum: ['orbit', 'pan', 'tilt', 'rotate', 'stop'], + enum: ['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe', 'stop'], }, direction: { type: 'string', diff --git a/src/voice/actionSchemas.test.mjs b/src/voice/actionSchemas.test.mjs index 550d725e4..d65d7d8db 100644 --- a/src/voice/actionSchemas.test.mjs +++ b/src/voice/actionSchemas.test.mjs @@ -1,3 +1,4 @@ +import { realtimeInstructions } from '../../server/providers/openai/instructions.js'; import test from 'node:test'; import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; @@ -38,7 +39,10 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea // requested radius and center authority in descriptions only. // Street Level and GNSS extend the manifest; tracked and analyst aircraft describe broadcasts. // Air quality (ECCC AQHI) extends the manifest. - '775ba323384b4c1b62fb8415bac52aecc3fc668e20bedcfa57e426dcd1da3969', + // Globe rotation adds one motion; visual keyhole fit and explicit + // reset guidance/defaults update only move_camera and mode descriptions. + // Restore base descriptions; append only globe-rotation defaults and fit guidance. + '967b5a043b812e2f8c1e190fc571cf12b960c9d4486df736902221aab4d99cf6', ); }); @@ -106,6 +110,17 @@ test('all legacy action arguments are byte-identical after removing the delibera delete property('show_data_layers_menu').layerId.enum; delete property('get_entity_context').layerId.enum; delete property('analyst_query').layers.items.enum; + // Earth-centered rotation is additive; local camera yaw keeps its contract. + const motion = property('move_camera').motion; + assert.deepEqual(motion.enum, [ + 'orbit', + 'pan', + 'tilt', + 'rotate', + 'rotate_globe', + 'stop', + ]); + motion.enum = motion.enum.filter((value) => value !== 'rotate_globe'); const panels = property('set_panel_open').panelId; panels.enum = panels.enum.filter( (id) => !['weather-panel', 'recent-imagery-panel'].includes(id), @@ -145,6 +160,24 @@ test('all legacy action arguments are byte-identical after removing the delibera ); }); +test('globe rotation is an additive camera motion with the existing control arguments', () => { + const schema = GEV_ACTION_SCHEMAS.find( + (tool) => tool.name === 'move_camera', + ).parameters; + for (const motion of ['rotate_globe', 'rotate', 'orbit', 'pan', 'tilt']) { + assert.deepEqual( + validateArgs(schema, { + motion, + direction: 'right', + speed: 'slow', + mode: 'continuous', + }), + [], + ); + } + assert.ok(validateArgs(schema, { motion: 'spin_camera' }).length > 0); +}); + test('track_entity accepts a referent alone but rejects an empty target', () => { const schema = GEV_ACTION_SCHEMAS.find( (tool) => tool.name === 'track_entity', @@ -183,3 +216,15 @@ test('the benchmark still rejects an empty track_entity although the model-facin [], ); }); + +test('globe reset guidance requires a separate explicit command after refusal', () => { + const paragraph = realtimeInstructions() + .split('\n') + .find((line) => line.includes('globe-view-required')); + assert.ok(paragraph); + assert.match(paragraph, /calls only move_camera motion rotate_globe/); + assert.match(paragraph, /explicit user Reset\/Show Globe View/); + assert.match(paragraph, /never from your own plan/); + assert.match(paragraph, /bare yes\/ok/); + assert.match(paragraph, /Do not ask a question/); +}); diff --git a/src/voice/geminiSession.js b/src/voice/geminiSession.js index 04a573f76..041f1f643 100644 --- a/src/voice/geminiSession.js +++ b/src/voice/geminiSession.js @@ -21,6 +21,7 @@ import { shouldSendViewportImage, shouldStopVoiceAfterRadioTool, responseInstructionForToolResult, + isGlobeViewRefusal, } from './realtimeProtocol.js'; const SOCKET_PATH = @@ -182,6 +183,8 @@ export class GeminiSessionController { this.model = null; this.epoch = 0; this.turn = 0; + this.globeToolsBlocked = false; + this.globeResetFailed = false; this.inputGeneration = 0; this.turnAbort = new AbortController(); this.disposed = false; @@ -659,6 +662,8 @@ export class GeminiSessionController { return; if (!pushToTalk || activityCaptureId === metadata.captureId) return; this.captureOwner.admitted = true; + this.globeToolsBlocked = false; + this.globeResetFailed = false; if ( !this.interrupt('push-to-talk', { keepPlayback: !this.pttPlaybackSuppressed, @@ -1294,6 +1299,8 @@ export class GeminiSessionController { this.replyPending = false; if (!this.interrupt()) return; } else if (!this.retiredReplyPending && !this.interrupt()) return; + this.globeToolsBlocked = false; + this.globeResetFailed = false; // Local activity ownership is only for expiry liveness, not tools. // Assign after retiring observed output so its boundary cannot clear // a newer detected activity that has not received a reply yet. @@ -1573,6 +1580,7 @@ export class GeminiSessionController { .then(async () => { if (!this.owns(epoch) || turn !== this.turn) return; if (this.cancelledCalls.has(call.id)) { + if (call.name === 'zoom_to_globe') this.globeResetFailed = true; if (this.tools.get(call.id) === action) this.tools.delete(call.id); this.pendingTools--; if (!this.pendingTools) { @@ -1618,8 +1626,28 @@ export class GeminiSessionController { (authority !== 'playback' || handoffEpoch === this.radio.radioHandoffEpoch); let result; + let skippedGlobeRotation = false; + let blockedGlobeTool = false; try { if (!actionCurrent()) throw cancelled(); + // Live has no response-scoped tool_choice. Recheck here so queued siblings + // cannot reset after a rotation refusal. + if (this.globeToolsBlocked) { + blockedGlobeTool = true; + throw new Error( + 'Wait for fresh user input before calling another tool.', + ); + } + if ( + this.globeResetFailed && + call.name === 'move_camera' && + String(args.motion || '').toLowerCase() === 'rotate_globe' + ) { + skippedGlobeRotation = true; + throw new Error( + 'Not run: the Globe View reset/rotation sequence failed or was cancelled.', + ); + } if (stronger) reservation = this.radio.reserveRadioToolHandoff({ abortScope }); handoffEpoch = this.radio.radioHandoffEpoch; @@ -1689,7 +1717,12 @@ export class GeminiSessionController { } catch (error) { result = { ok: false, - tool: call.name, + ...(skippedGlobeRotation + ? { action: call.name, reason: 'globe-reset-failed', skipped: true } + : { + tool: call.name, + ...(blockedGlobeTool ? { skipped: true } : {}), + }), ...(radioVisibility ? { layerId: 'radio' } : {}), error: error?.name === 'AbortError' @@ -1716,6 +1749,11 @@ export class GeminiSessionController { this.noteRadioStrongerCommit(action, abortScope, turn); } if (!this.owns(epoch) || turn !== this.turn) return; + if ( + call.name === 'zoom_to_globe' && + (result?.ok !== true || result.cancelled) + ) + this.globeResetFailed = true; const playbackObsolete = authority === 'playback' && handoffEpoch !== this.radio.radioHandoffEpoch; if (radioControl || radioVisibility) { @@ -1744,6 +1782,8 @@ export class GeminiSessionController { this.stopAfterReply = true; // Native call cancellation suppresses its output, without canceling siblings. if (!this.cancelledCalls.has(call.id)) { + if (isGlobeViewRefusal(result) && !skippedGlobeRotation) + this.globeToolsBlocked = true; this.postToolOutput = false; this.modelComplete = false; this.replyPending = true; @@ -1751,7 +1791,9 @@ export class GeminiSessionController { const mapEvents = mapSnapshot.map((text) => JSON.parse(text)); const response = { ...(result ?? { ok: false, error: 'Command returned no result' }), - responseGuidance: responseInstructionForToolResult(result), + responseGuidance: blockedGlobeTool + ? 'That action did not run. Wait for fresh user input; do not claim any change or call another tool.' + : responseInstructionForToolResult(result), }; this.send( { @@ -1902,6 +1944,8 @@ export class GeminiSessionController { { onSent: () => { if (!owned() || this.typedIntent !== typedIntent) return; + this.globeToolsBlocked = false; + this.globeResetFailed = false; this.replyPending = true; this.bindTextIntent(typedIntent); if (!owned() || this.typedIntent !== typedIntent) return; @@ -2455,6 +2499,8 @@ export class GeminiSessionController { this.cancelledCalls.clear(); this.pendingMapEvents = []; this.userTurnPending = false; + this.globeToolsBlocked = false; + this.globeResetFailed = false; this.replyPending = false; this.unrequestedReply = false; this.automaticTurnOpen = false; diff --git a/src/voice/geminiSession.test.mjs b/src/voice/geminiSession.test.mjs index b2f380b37..9ce8d351a 100644 --- a/src/voice/geminiSession.test.mjs +++ b/src/voice/geminiSession.test.mjs @@ -3398,3 +3398,215 @@ test('a narration callback that submits a successor keeps the old packet tools f assert.deepEqual(h.calls, []); h.c.stop(); }); + +for (const reply of ['typed', 'push-to-talk', 'open-mic']) { + test(`globe refusal blocks queued and later tools until fresh ${reply} input`, async () => { + const names = []; + const h = harness({ + runAction: async (name) => { + names.push(name); + return name === 'move_camera' + ? { ok: false, action: 'move_camera', reason: 'globe-view-required' } + : { ok: true, action: name }; + }, + }); + try { + await h.c.start({ pushToTalk: reply === 'push-to-talk' }); + establishToolInput(h, 'rotate the globe'); + const ws = h.sockets[0]; + ws.message({ + toolCall: { + functionCalls: [ + { + id: 'rotate', + name: 'move_camera', + args: { motion: 'rotate_globe' }, + }, + { id: 'queued-reset', name: 'zoom_to_globe', args: {} }, + ], + }, + }); + await h.c.toolQueue; + assert.deepEqual(names, ['move_camera']); + const blocked = ws.sent + .flatMap((entry) => entry.toolResponse?.functionResponses || []) + .find((call) => call.id === 'queued-reset'); + assert.equal(blocked.response.ok, false); + assert.match(blocked.response.error, /fresh user input/i); + assert.equal(blocked.response.skipped, true); + assert.match(blocked.response.responseGuidance, /did not run/i); + assert.doesNotMatch( + blocked.response.responseGuidance, + /confirm.*completed/i, + ); + requestTool(ws, 'repeated-rotation', 'move_camera', { + motion: 'rotate_globe', + }); + await h.c.toolQueue; + assert.deepEqual(names, ['move_camera']); + const repeated = toolResponses(ws).find( + (call) => call.id === 'repeated-rotation', + ).response; + assert.equal(repeated.ok, false); + assert.equal(repeated.skipped, true); + assert.match(repeated.responseGuidance, /did not run/i); + assert.doesNotMatch( + repeated.responseGuidance, + /confirm.*completed/i, + ); + + if (reply === 'typed') { + const stimuli = [ + () => {}, + () => ws.message({ serverContent: { interrupted: true } }), + () => + ws.message({ + serverContent: { + inputTranscription: { text: 'reset to globe view' }, + }, + }), + () => { + h.c.input.setMicrophoneEnabled(false); + h.c.input.setMicrophoneEnabled(true); + }, + ]; + for (const [i, stimulus] of stimuli.entries()) { + stimulus(); + requestTool(ws, `unrequested-reset-${i}`, 'zoom_to_globe', {}); + await h.c.toolQueue; + assert.deepEqual(names, ['move_camera']); + } + } + // Finish the refusal reply before admitting the actual next user input. + ws.message({ serverContent: { turnComplete: true } }); + if (reply === 'typed') + assert.equal(h.c.sendTextCommand('reset to globe view'), true); + else if (reply === 'push-to-talk') { + h.c.input.pushToTalkKeyHeld = true; + h.c.input.spaceKeyHeld = true; + h.c.input.setMicrophoneEnabled(true); + h.c.input.setMicrophoneEnabled(false); + } else ws.message({ voiceActivity: { type: 'ACTIVITY_START' } }); + requestTool(ws, 'accepted-reset', 'zoom_to_globe', {}); + await h.c.toolQueue; + assert.deepEqual(names, ['move_camera', 'zoom_to_globe']); + } finally { + h.c.stop(); + } + }); +} + +test('a cancelled globe reset skips queued rotation, preserves Stop and recovers on fresh input', async () => { + const calls = []; + let resets = 0; + const h = harness({ + runAction: async (name, args) => { + calls.push(name === 'move_camera' ? args.motion : name); + return name === 'zoom_to_globe' && resets++ === 0 + ? { ok: false, action: name, cancelled: true } + : { ok: true, action: name }; + }, + }); + try { + await h.c.start(); + establishToolInput(h, 'reset to globe view and rotate'); + const ws = h.sockets[0]; + ws.message({ + toolCall: { + functionCalls: [ + { id: 'reset-failed', name: 'zoom_to_globe', args: {} }, + { + id: 'rotate-skipped', + name: 'move_camera', + args: { motion: 'rotate_globe' }, + }, + { id: 'stop-allowed', name: 'move_camera', args: { motion: 'stop' } }, + ], + }, + }); + await h.c.toolQueue; + assert.deepEqual(calls, ['zoom_to_globe', 'stop']); + const skipped = toolResponses(ws).find( + (call) => call.id === 'rotate-skipped', + ).response; + assert.equal(skipped.ok, false); + assert.equal(skipped.reason, 'globe-reset-failed'); + assert.equal(skipped.skipped, true); + assert.equal( + toolResponses(ws).find((call) => call.id === 'stop-allowed').response.ok, + true, + ); + + ws.message({ serverContent: { turnComplete: true } }); + establishToolInput(h, 'reset to globe view and rotate'); + ws.message({ + toolCall: { + functionCalls: [ + { id: 'reset-success', name: 'zoom_to_globe', args: {} }, + { + id: 'rotate-success', + name: 'move_camera', + args: { motion: 'rotate_globe' }, + }, + ], + }, + }); + await h.c.toolQueue; + assert.deepEqual(calls, [ + 'zoom_to_globe', + 'stop', + 'zoom_to_globe', + 'rotate_globe', + ]); + } finally { + h.c.stop(); + } +}); + +test('cancelling a queued globe reset also skips its dependent rotation', async () => { + const blocker = deferred(); + const calls = []; + const h = harness({ + runAction: async (name, args) => { + calls.push(name === 'move_camera' ? args.motion : name); + return args.motion === 'pan' + ? blocker.promise + : { ok: true, action: name }; + }, + }); + try { + await h.c.start(); + establishToolInput(h, 'pan, reset to globe view and rotate'); + const ws = h.sockets[0]; + ws.message({ + toolCall: { + functionCalls: [ + { + id: 'blocker', + name: 'move_camera', + args: { motion: 'pan', direction: 'left' }, + }, + { id: 'reset-cancelled', name: 'zoom_to_globe', args: {} }, + { + id: 'rotate-after-cancel', + name: 'move_camera', + args: { motion: 'rotate_globe' }, + }, + ], + }, + }); + await tick(); + ws.message({ toolCallCancellation: { ids: ['reset-cancelled'] } }); + blocker.resolve({ ok: true, action: 'move_camera' }); + await h.c.toolQueue; + assert.deepEqual(calls, ['pan']); + const skipped = toolResponses(ws).find( + (call) => call.id === 'rotate-after-cancel', + ).response; + assert.equal(skipped.reason, 'globe-reset-failed'); + assert.equal(skipped.skipped, true); + } finally { + blocker.resolve({ ok: true }); + h.c.stop(); + } +}); diff --git a/src/voice/gevActions.js b/src/voice/gevActions.js index 49e2ba20a..f7613aed1 100644 --- a/src/voice/gevActions.js +++ b/src/voice/gevActions.js @@ -1,3 +1,4 @@ +import { interruptGlobeRotation } from '../cameraVerbs.js'; import { layerSnapshot, feedProvenanceEnvelope, @@ -262,7 +263,9 @@ export function createGevActionRunner({ }), )); installViewTargetPrewarm(viewer); - initCameraVerbs(viewer, getViewTargetCartesian); + initCameraVerbs(viewer, getViewTargetCartesian, { + isCockpitActive: () => Boolean(styleManager?.getCockpitState?.()?.active), + }); // Nested calls (select_nearest_aircraft composing layer + flight) use the // raw runner; only the outer call gets a voice envelope. const runGevAction = async function runGevAction( @@ -298,6 +301,7 @@ export function createGevActionRunner({ // Zoom during an active orbit adjusts the orbit RADIUS (spiral in/out) — // a straight camera move would be snapped back by the per-frame lookAt. if (name === 'adjust_camera_zoom') { + interruptGlobeRotation('voice-zoom-intent'); const zoomOut = String(args.direction || '').toLowerCase() === 'out'; const amt = String(args.amount || 'medium').toLowerCase(); const factor = { little: 1.25, medium: 1.6, lot: 2.4 }[amt] || 1.6; diff --git a/src/voice/globeResetConfirmation.test.mjs b/src/voice/globeResetConfirmation.test.mjs new file mode 100644 index 000000000..be2778e6d --- /dev/null +++ b/src/voice/globeResetConfirmation.test.mjs @@ -0,0 +1,1131 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { GevRealtimeController } from './realtimeController.js'; +import { + CALL_DEDUPE_MS, + responseInstructionForToolResult, +} from './realtimeProtocol.js'; + +const refusal = { + ok: false, + action: 'move_camera', + reason: 'globe-view-required', +}; + +function harness(result) { + const sent = []; + const calls = []; + const controller = new GevRealtimeController({ + ui: {}, + runner: async (name, args, context) => { + calls.push(name); + return typeof result === 'function' + ? result(name, args, context) + : result; + }, + }); + controller.dc = { readyState: 'open' }; + controller.sendRealtimeEvent = (event) => { + sent.push(event); + return true; + }; + controller.debugLog = () => {}; + controller.setStatus = () => {}; + controller.setVoiceSpeaker = () => {}; + controller.recordUsage = () => {}; + controller.captureViewportScreenshot = async () => null; + controller.cancelRadioHandoff = () => {}; + return { controller, sent, calls }; +} + +async function invoke( + controller, + { + responseId = 'rotation-response', + callId = 'rotation-call', + name = 'move_camera', + args = { motion: 'rotate_globe' }, + } = {}, +) { + await controller.handleRealtimeEvent({ + data: JSON.stringify({ + type: 'response.function_call_arguments.done', + response_id: responseId, + item_id: `${callId}-item`, + call_id: callId, + name, + arguments: JSON.stringify(args), + }), + }); +} +/** A real user input boundary: the provider detected new speech. */ +async function speak(controller) { + await controller.handleRealtimeEvent({ + data: JSON.stringify({ type: 'input_audio_buffer.speech_started' }), + }); +} +function begin(controller, id = 'rotation-response') { + controller.updateResponseState({ + type: 'response.created', + response: { id }, + }); +} +function complete(controller, id = 'rotation-response') { + controller.updateResponseState({ + type: 'response.done', + response: { id, status: 'completed' }, + }); +} + +test('close-up rotation refusal instructs an explicit reset in a response with tools disabled', async () => { + const { controller, sent } = harness(refusal); + await invoke(controller); + complete(controller); + const response = sent.find( + (event) => event.type === 'response.create', + )?.response; + assert.equal(response?.tool_choice, 'none'); + assert.equal( + response.instructions, + responseInstructionForToolResult(refusal), + ); + assert.match(response.instructions, /do not ask a question/i); + assert.doesNotMatch(response.instructions, /confirm the completed/); +}); + +test('normal camera success retains the existing unrestricted followup', async () => { + const { controller, sent } = harness({ + ok: true, + action: 'move_camera', + motion: 'rotate', + }); + await invoke(controller); + complete(controller); + const response = sent.find( + (event) => event.type === 'response.create', + )?.response; + assert.equal(typeof response?.instructions, 'string'); + assert.equal(Object.hasOwn(response, 'tool_choice'), false); +}); + +test('a newer typed request drops the old refusal followup and its tool restriction', async () => { + const { controller, sent } = harness(refusal); + await invoke(controller); + controller.sendTextCommand('turn the camera right'); + complete(controller); + const responses = sent.filter((event) => event.type === 'response.create'); + assert.equal(responses.length, 1); + assert.equal(responses[0].response?.tool_choice, undefined); + assert.equal(controller.pendingResponseInstructions, null); +}); + +test('refusal instructions do not interpolate untrusted failure text or alter other tool results', () => { + const instruction = responseInstructionForToolResult({ + ...refusal, + error: 'UNTRUSTED_ERROR', + say: 'UNTRUSTED_SAY', + }); + assert.doesNotMatch(instruction, /UNTRUSTED/); + const unavailable = responseInstructionForToolResult({ + ...refusal, + reason: 'globe-reset-unavailable', + }); + assert.doesNotMatch(unavailable, /Reset to Globe View and rotate the globe/); + assert.match(unavailable, /do not offer or perform a reset/i); + const completedReset = responseInstructionForToolResult({ + action: 'zoom_to_globe', + ok: true, + }); + assert.equal( + completedReset, + responseInstructionForToolResult({ ok: true, action: 'move_camera' }), + ); + for (const outcome of [ + { action: 'zoom_to_globe', ok: false }, + { action: 'zoom_to_globe', ok: true, cancelled: true }, + { tool: 'zoom_to_globe', ok: false }, + ]) { + const resetFailure = responseInstructionForToolResult({ + ...outcome, + error: 'UNTRUSTED_ERROR', + say: 'UNTRUSTED_SAY', + }); + assert.notEqual(resetFailure, completedReset); + assert.match(resetFailure, /did not finish/i); + assert.doesNotMatch(resetFailure, /UNTRUSTED/); + } + assert.equal( + responseInstructionForToolResult({ + ok: true, + action: 'get_current_view_state', + }), + responseInstructionForToolResult({ ok: true, action: 'move_camera' }), + ); +}); + +test('globe refusal blocks same-response tools and preserves the reset instruction, but not the next response', async () => { + const { controller, sent, calls } = harness((name) => + name === 'move_camera' ? refusal : { ok: true, action: name }, + ); + begin(controller); + await invoke(controller); + await invoke(controller, { + callId: 'reset-call', + name: 'zoom_to_globe', + args: {}, + }); + // The second delivery of the original call must not produce another output. + await invoke(controller); + complete(controller); + assert.deepEqual(calls, ['move_camera']); + const outputs = sent.filter( + (event) => event.item?.type === 'function_call_output', + ); + assert.deepEqual( + outputs.map((event) => event.item.call_id), + ['rotation-call', 'reset-call'], + ); + assert.equal(JSON.parse(outputs[1].item.output).ok, false); + const replies = sent.filter((event) => event.type === 'response.create'); + assert.equal(replies.length, 1); + assert.equal(replies[0].response.tool_choice, 'none'); + assert.equal( + replies[0].response.instructions, + responseInstructionForToolResult(refusal), + ); + // A fresh explicit reset request is a separate user input. + await speak(controller); + begin(controller, 'fresh-response'); + await invoke(controller, { + responseId: 'fresh-response', + callId: 'fresh-reset', + name: 'zoom_to_globe', + args: {}, + }); + complete(controller, 'fresh-response'); + assert.deepEqual(calls, ['move_camera', 'zoom_to_globe']); + assert.equal( + sent.filter((event) => event.type === 'response.create').at(-1).response + .tool_choice, + undefined, + ); +}); + +test('a reset arriving during pending globe admission cannot bypass the refusal', async () => { + let finishRotation; + const pending = new Promise((resolve) => { + finishRotation = resolve; + }); + const { controller, sent, calls } = harness((name) => + name === 'move_camera' ? pending : { ok: true, action: name }, + ); + begin(controller); + const rotation = invoke(controller, { args: { motion: 'ROTATE_GLOBE' } }); + const reset = invoke(controller, { + callId: 'overlap-reset', + name: 'zoom_to_globe', + args: {}, + }); + finishRotation(refusal); + await Promise.all([rotation, reset]); + complete(controller); + assert.deepEqual(calls, ['move_camera']); + const outputs = sent.filter( + (event) => event.item?.type === 'function_call_output', + ); + assert.equal(outputs.length, 2); + assert.equal( + JSON.parse( + outputs.find((event) => event.item.call_id === 'overlap-reset').item + .output, + ).ok, + false, + ); + const replies = sent.filter((event) => event.type === 'response.create'); + assert.equal(replies.length, 1); + assert.equal(replies[0].response.tool_choice, 'none'); + assert.equal( + replies[0].response.instructions, + responseInstructionForToolResult(refusal), + ); +}); + +test('successful pending globe admission releases a queued ordinary tool', async () => { + let finishRotation; + const pending = new Promise((resolve) => { + finishRotation = resolve; + }); + const { controller, sent, calls } = harness((name) => + name === 'move_camera' ? pending : { ok: true, action: name }, + ); + begin(controller); + const rotation = invoke(controller); + const reset = invoke(controller, { + callId: 'allowed-reset', + name: 'zoom_to_globe', + args: {}, + }); + assert.deepEqual(calls, ['move_camera']); + finishRotation({ ok: true, action: 'move_camera', motion: 'rotate_globe' }); + await Promise.all([rotation, reset]); + complete(controller); + assert.deepEqual(calls, ['move_camera', 'zoom_to_globe']); + assert.equal( + sent.filter((event) => event.type === 'response.create').at(-1).response + .tool_choice, + undefined, + ); +}); + +test('rotation waits for an earlier reset before evaluating the globe', async () => { + let finishReset; + let atGlobe = false; + const pending = new Promise((resolve) => { + finishReset = resolve; + }); + const { controller, sent, calls } = harness(async (name) => { + if (name === 'zoom_to_globe') { + await pending; + atGlobe = true; + return { ok: true, action: name }; + } + return atGlobe + ? { ok: true, action: name, motion: 'rotate_globe' } + : refusal; + }); + begin(controller); + const reset = invoke(controller, { + callId: 'first-reset', + name: 'zoom_to_globe', + args: {}, + }); + const rotation = invoke(controller); + assert.deepEqual(calls, ['zoom_to_globe']); + finishReset(); + await Promise.all([reset, rotation]); + complete(controller); + assert.deepEqual(calls, ['zoom_to_globe', 'move_camera']); + const outputs = sent.filter( + (event) => event.item?.type === 'function_call_output', + ); + assert.deepEqual( + outputs.map((event) => event.item.call_id), + ['first-reset', 'rotation-call'], + ); + assert.equal(JSON.parse(outputs[1].item.output).ok, true); + const replies = sent.filter((event) => event.type === 'response.create'); + assert.equal(replies.length, 1); + assert.equal(replies[0].response.tool_choice, undefined); +}); + +for (const outcome of [ + 'cancelled', + 'thrown', + 'retry-refused', + 'settled-failure', +]) { + test(`a ${outcome} reset sequence cannot suggest another reset`, async () => { + let finishReset; + const pending = new Promise((resolve) => { + finishReset = resolve; + }); + const { controller, sent, calls } = harness(async (name) => { + if (name !== 'zoom_to_globe') return refusal; + await pending; + if (outcome === 'thrown') throw new Error('reset failed'); + return { + ok: outcome === 'retry-refused', + action: name, + cancelled: outcome === 'cancelled', + }; + }); + begin(controller); + const reset = invoke(controller, { + callId: 'failed-reset', + name: 'zoom_to_globe', + args: {}, + }); + if (outcome === 'settled-failure') { + finishReset(); + await reset; + } + const rotation = invoke(controller); + finishReset(); + await Promise.all([reset, rotation]); + complete(controller); + assert.deepEqual( + calls, + outcome === 'retry-refused' + ? ['zoom_to_globe', 'move_camera'] + : ['zoom_to_globe'], + ); + const outputs = sent.filter( + (event) => event.item?.call_id === 'rotation-call', + ); + assert.equal(outputs.length, 1); + const result = JSON.parse(outputs[0].item.output); + assert.equal(result.ok, false); + assert.equal(result.reason, 'globe-reset-failed'); + const replies = sent.filter((event) => event.type === 'response.create'); + assert.equal(replies.length, 1); + assert.equal(replies[0].response.tool_choice, 'none'); + assert.doesNotMatch( + replies[0].response.instructions, + /Reset to Globe View and rotate the globe/, + ); + }); +} + +test( + 'Stop releases a rotation waiting for reset and isolates a reused response id', + { timeout: 1000 }, + async () => { + let finishOld, finishNew; + const oldReset = new Promise((resolve) => { + finishOld = resolve; + }); + const newReset = new Promise((resolve) => { + finishNew = resolve; + }); + let resets = 0; + const { controller, sent, calls } = harness((name) => + name === 'zoom_to_globe' + ? ++resets === 1 + ? oldReset + : newReset + : { ok: true, action: name }, + ); + const oldChannel = controller.dc; + oldChannel.close = () => { + oldChannel.readyState = 'closed'; + }; + begin(controller); + const old = invoke(controller, { + callId: 'old-reset', + name: 'zoom_to_globe', + args: {}, + }); + const waiter = invoke(controller); + controller.stop(); + await waiter; + assert.deepEqual(calls, ['zoom_to_globe']); + assert.equal(sent.length, 0); + controller.dc = { readyState: 'open' }; + begin(controller); + const replacement = invoke(controller, { + callId: 'new-reset', + name: 'zoom_to_globe', + args: {}, + }); + const rotation = invoke(controller, { callId: 'new-rotation' }); + finishOld({ ok: false, action: 'zoom_to_globe' }); + await old; + assert.deepEqual(calls, ['zoom_to_globe', 'zoom_to_globe']); + assert.equal(sent.length, 0); + finishNew({ ok: true, action: 'zoom_to_globe' }); + await Promise.all([replacement, rotation]); + complete(controller); + assert.deepEqual(calls, ['zoom_to_globe', 'zoom_to_globe', 'move_camera']); + assert.deepEqual( + sent + .filter((event) => event.item?.type === 'function_call_output') + .map((event) => event.item.call_id), + ['new-reset', 'new-rotation'], + ); + }, +); + +test( + 'a typed turn releases rotation without waiting for its reset runner', + { timeout: 1000 }, + async () => { + let finishReset; + const pending = new Promise((resolve) => { + finishReset = resolve; + }); + const { controller, sent, calls } = harness(() => pending); + delete controller.cancelRadioHandoff; + begin(controller); + const reset = invoke(controller, { + callId: 'stale-first-reset', + name: 'zoom_to_globe', + args: {}, + }); + const rotation = invoke(controller); + // The tool response can finish while its reset and rotation still wait. + complete(controller); + controller.sendTextCommand('pan left'); + await rotation; + assert.deepEqual(calls, ['zoom_to_globe']); + const output = sent.find( + (event) => event.item?.call_id === 'rotation-call', + ); + assert.equal(JSON.parse(output.item.output).superseded, true); + assert.equal(controller.pendingResponseInstructions, null); + begin(controller, 'typed-response'); + complete(controller, 'typed-response'); + const replies = sent.filter((event) => event.type === 'response.create'); + assert.equal(replies.length, 1); + assert.equal(replies[0].response?.tool_choice, undefined); + finishReset({ ok: false, action: 'zoom_to_globe' }); + await reset; + // This protects rotation's cancellation only. Ordinary reset-result + // follow-ups after supersession remain the inherited sibling-tool limit. + }, +); + +test('an aborted reset still blocks a late same-response rotation', async () => { + let finishReset; + const pending = new Promise((resolve) => { + finishReset = resolve; + }); + const { controller, sent, calls } = harness((name) => + name === 'zoom_to_globe' ? pending : { ok: true, action: name }, + ); + delete controller.cancelRadioHandoff; + begin(controller); + const reset = invoke(controller, { + callId: 'aborted-reset', + name: 'zoom_to_globe', + args: {}, + }); + try { + await controller.handleRealtimeEvent({ + data: JSON.stringify({ + type: 'error', + error: { code: 'conversation_already_has_active_response' }, + }), + }); + await invoke(controller); + assert.deepEqual(calls, ['zoom_to_globe']); + const outputs = sent.filter( + (event) => event.item?.call_id === 'rotation-call', + ); + assert.equal(outputs.length, 1); + const result = JSON.parse(outputs[0].item.output); + assert.equal(result.skipped, true); + assert.equal(result.reason, 'globe-reset-failed'); + } finally { + finishReset({ ok: false, action: 'zoom_to_globe', cancelled: true }); + await reset; + } + complete(controller); + assert.equal( + sent.find((event) => event.type === 'response.create').response.tool_choice, + 'none', + ); +}); + +test('an earlier tool finishing late cannot replace the globe refusal followup', async () => { + let finishLookup; + const pending = new Promise((resolve) => { + finishLookup = resolve; + }); + const { controller, sent, calls } = harness((name) => + name === 'move_camera' ? refusal : pending, + ); + begin(controller); + const lookup = invoke(controller, { + callId: 'lookup-call', + name: 'get_current_view_state', + args: {}, + }); + await invoke(controller); + finishLookup({ ok: true, action: 'get_current_view_state' }); + await lookup; + complete(controller); + assert.deepEqual(calls, ['get_current_view_state', 'move_camera']); + assert.equal( + sent.filter((event) => event.item?.type === 'function_call_output').length, + 2, + ); + const replies = sent.filter((event) => event.type === 'response.create'); + assert.equal(replies.length, 1); + assert.equal(replies[0].response.tool_choice, 'none'); + assert.equal( + replies[0].response.instructions, + responseInstructionForToolResult(refusal), + ); +}); + +test('a duplicate queued behind successive admissions cannot outlive the dedupe window', async (t) => { + let clock = 0; + t.mock.method(performance, 'now', () => clock); + let finishFirst, finishSecond, secondStarted; + const first = new Promise((resolve) => { + finishFirst = resolve; + }); + const second = new Promise((resolve) => { + finishSecond = resolve; + }); + const enteredSecond = new Promise((resolve) => { + secondStarted = resolve; + }); + let executions = 0; + const { controller, sent, calls } = harness(() => { + if (++executions === 1) return first; + secondStarted(); + return second; + }); + begin(controller); + const original = invoke(controller); + const queued = invoke(controller, { callId: 'queued-rotation' }); + const duplicate = controller.handleRealtimeEvent({ + data: JSON.stringify({ + type: 'response.output_item.done', + response_id: 'rotation-response', + item: { + type: 'function_call', + id: 'queued-rotation-item', + call_id: 'queued-rotation', + name: 'move_camera', + arguments: JSON.stringify({ motion: 'rotate_globe' }), + }, + }), + }); + finishFirst({ ok: true, action: 'move_camera', motion: 'rotate_globe' }); + await enteredSecond; + clock += CALL_DEDUPE_MS + 1; + finishSecond({ ok: true, action: 'move_camera', motion: 'rotate_globe' }); + await Promise.all([original, queued, duplicate]); + complete(controller); + assert.deepEqual(calls, ['move_camera', 'move_camera']); + const outputs = sent.filter( + (event) => event.item?.type === 'function_call_output', + ); + assert.deepEqual( + outputs.map((event) => event.item.call_id), + ['rotation-call', 'queued-rotation'], + ); + assert.equal( + sent.filter((event) => event.type === 'response.create').length, + 1, + ); +}); + +test( + 'Stop releases an admission waiter and stale completion cannot affect its replacement', + { timeout: 1000 }, + async () => { + let finishRotation; + const pending = new Promise((resolve) => { + finishRotation = resolve; + }); + const { controller, sent, calls } = harness((name) => + name === 'move_camera' ? pending : { ok: true, action: name }, + ); + const oldChannel = controller.dc; + oldChannel.close = () => { + oldChannel.readyState = 'closed'; + }; + begin(controller); + const rotation = invoke(controller); + const queued = invoke(controller, { + callId: 'stopped-reset', + name: 'zoom_to_globe', + args: {}, + }); + controller.stop(); + await queued; // Must settle even though the original runner has not completed. + assert.equal(oldChannel.readyState, 'closed'); + assert.deepEqual(calls, ['move_camera']); + assert.equal(sent.length, 0); + controller.dc = { readyState: 'open' }; + begin(controller); // Reuse the id to expose any leaked old refusal state. + finishRotation(refusal); + await rotation; + assert.equal(sent.length, 0); + await invoke(controller, { + callId: 'replacement-reset', + name: 'zoom_to_globe', + args: {}, + }); + complete(controller); + assert.deepEqual(calls, ['move_camera', 'zoom_to_globe']); + const outputs = sent.filter( + (event) => event.item?.type === 'function_call_output', + ); + assert.deepEqual( + outputs.map((event) => event.item.call_id), + ['replacement-reset'], + ); + assert.equal(JSON.parse(outputs[0].item.output).ok, true); + assert.equal( + sent.find((event) => event.type === 'response.create').response + .tool_choice, + undefined, + ); + }, +); + +for (const completedFirst of [false, true]) { + test(`a newer typed turn retires its queued sibling after response completion: ${completedFirst}`, async () => { + let finishRotation; + let oldSignal; + let abortObserved = false; + const pending = new Promise((resolve) => { + finishRotation = resolve; + }); + const { controller, sent, calls } = harness((name, args, context) => { + if (args.motion === 'rotate_globe') { + oldSignal = context.signal; + oldSignal.addEventListener('abort', () => { + abortObserved = true; + }); + return pending; + } + return { ok: true, action: name }; + }); + // Use the actual cancellation owner rather than the lightweight harness stub. + delete controller.cancelRadioHandoff; + begin(controller); + const rotation = invoke(controller); + const sibling = invoke(controller, { + callId: 'stale-reset', + name: 'zoom_to_globe', + args: {}, + }); + if (completedFirst) complete(controller); + controller.sendTextCommand('pan left'); + assert.equal(oldSignal.aborted, true); + assert.equal(abortObserved, true); + finishRotation(refusal); + await Promise.all([rotation, sibling]); + if (!completedFirst) complete(controller); + const outputs = sent.filter( + (event) => event.item?.type === 'function_call_output', + ); + assert.deepEqual( + outputs.map((event) => event.item.call_id), + ['rotation-call', 'stale-reset'], + ); + assert.equal(JSON.parse(outputs[1].item.output).superseded, true); + assert.deepEqual(calls, ['move_camera']); + const replies = sent.filter((event) => event.type === 'response.create'); + assert.equal(replies.length, 1); + assert.equal(replies[0].response?.tool_choice, undefined); + assert.equal(controller.pendingResponseInstructions, null); + begin(controller, 'new-turn-response'); + await invoke(controller, { + responseId: 'new-turn-response', + callId: 'new-pan', + args: { motion: 'pan', direction: 'left' }, + }); + complete(controller, 'new-turn-response'); + assert.deepEqual(calls, ['move_camera', 'move_camera']); + const finalOutput = sent + .filter((event) => event.item?.type === 'function_call_output') + .at(-1); + assert.equal(finalOutput.item.call_id, 'new-pan'); + assert.equal(JSON.parse(finalOutput.item.output).ok, true); + assert.equal( + sent.filter((event) => event.type === 'response.create').at(-1).response + ?.tool_choice, + undefined, + ); + }); +} + +test('a newer typed turn during visual context cannot revive the old refusal followup', async () => { + let finishContext; + let enterContext; + const entered = new Promise((resolve) => { + enterContext = resolve; + }); + const pending = new Promise((resolve) => { + finishContext = resolve; + }); + const { controller, sent } = harness(refusal); + controller.setStatus = (status) => { + controller.status = status; + }; + delete controller.cancelRadioHandoff; + controller.sendVisualContextIfUseful = async () => { + enterContext(); + await pending; + }; + begin(controller); + const rotation = invoke(controller); + await entered; + controller.sendTextCommand('pan left'); + finishContext(); + await rotation; + complete(controller); + assert.equal(controller.status, 'listening'); + assert.equal( + sent.filter((event) => event.item?.type === 'function_call_output').length, + 1, + ); + const replies = sent.filter((event) => event.type === 'response.create'); + assert.equal(replies.length, 1); + assert.equal(replies[0].response?.tool_choice, undefined); + assert.equal(controller.pendingResponseInstructions, null); +}); + +test('stale globe completion preserves a newer tool status until that tool finishes', async () => { + let finishContext; + let enterContext; + let finishPan; + const entered = new Promise((resolve) => { + enterContext = resolve; + }); + const pendingContext = new Promise((resolve) => { + finishContext = resolve; + }); + const pendingPan = new Promise((resolve) => { + finishPan = resolve; + }); + const { controller, sent } = harness((name, args) => + args.motion === 'rotate_globe' ? refusal : pendingPan, + ); + controller.setStatus = (status) => { + controller.status = status; + }; + delete controller.cancelRadioHandoff; + controller.sendVisualContextIfUseful = async (result) => { + if (result === refusal) { + enterContext(); + await pendingContext; + } + }; + begin(controller); + const rotation = invoke(controller); + await entered; + controller.sendTextCommand('pan left'); + complete(controller); + begin(controller, 'new-pan-response'); + const pan = invoke(controller, { + responseId: 'new-pan-response', + callId: 'new-pan-call', + args: { motion: 'pan', direction: 'left' }, + }); + assert.equal(controller.status, 'executing'); + finishContext(); + await rotation; + assert.equal(controller.status, 'executing'); + assert.equal(controller.pendingResponseInstructions, null); + finishPan({ ok: true, action: 'move_camera', motion: 'pan' }); + await pan; + complete(controller, 'new-pan-response'); + assert.equal(controller.status, 'listening'); + assert.equal( + sent.filter((event) => event.item?.type === 'function_call_output').length, + 2, + ); + assert.equal( + sent + .filter((event) => event.type === 'response.create') + .every((event) => event.response?.tool_choice === undefined), + true, + ); +}); + +const rotated = { + ok: true, + action: 'move_camera', + motion: 'rotate_globe', + direction: 'right', + mode: 'continuous', +}; +/** A close-up camera that fits only after an executed reset. */ +function closeUpCamera() { + const state = { atGlobeView: false, resets: 0 }; + const result = (name) => { + if (name === 'zoom_to_globe') { + state.resets += 1; + state.atGlobeView = true; + return { ok: true, action: 'zoom_to_globe', cancelled: false }; + } + return state.atGlobeView ? rotated : refusal; + }; + return { state, result }; +} +test('OpenAI: explicit reset and reset-then-rotate stay direct', async () => { + const camera = closeUpCamera(); + const { controller, calls } = harness(camera.result); + await speak(controller); + begin(controller, 'reset'); + await invoke(controller, { + responseId: 'reset', + callId: 'explicit-reset', + name: 'zoom_to_globe', + args: {}, + }); + complete(controller, 'reset'); + assert.equal(camera.state.resets, 1); + // A fresh "Reset Globe View and rotate" resets, then rotates in order. + camera.state.atGlobeView = false; + await speak(controller); + begin(controller, 'both'); + await invoke(controller, { + responseId: 'both', + callId: 'reset-first', + name: 'zoom_to_globe', + args: {}, + }); + await invoke(controller, { responseId: 'both', callId: 'then-rotate' }); + complete(controller, 'both'); + assert.equal(camera.state.resets, 2); + assert.deepEqual(calls, ['zoom_to_globe', 'zoom_to_globe', 'move_camera']); +}); + +test('a failed reset survives real tool followups and a rejected typed send', async () => { + const { controller, sent, calls } = harness((name) => + name === 'zoom_to_globe' ? { ok: false, action: name } : rotated, + ); + begin(controller, 'reset-response'); + await invoke(controller, { + responseId: 'reset-response', + callId: 'failed-reset', + name: 'zoom_to_globe', + args: {}, + }); + complete(controller, 'reset-response'); + assert.equal( + sent.filter((event) => event.type === 'response.create').length, + 1, + ); + for (const id of ['followup', 'after-rejected-text']) { + if (id === 'after-rejected-text') { + const send = controller.sendRealtimeEvent; + controller.sendRealtimeEvent = (event, source) => + source === 'client.user_text' ? false : send(event, source); + controller.sendTextCommand('rotate the globe'); + controller.sendRealtimeEvent = send; + } + begin(controller, id); + await invoke(controller, { responseId: id, callId: `${id}-rotation` }); + complete(controller, id); + const output = sent.find( + (event) => event.item?.call_id === `${id}-rotation`, + ); + assert.equal(JSON.parse(output.item.output).reason, 'globe-reset-failed'); + assert.equal(JSON.parse(output.item.output).skipped, true); + } + assert.deepEqual(calls, ['zoom_to_globe']); + const reply = sent.filter((event) => event.type === 'response.create').at(-1); + assert.equal(reply.response.tool_choice, 'none'); + assert.doesNotMatch( + reply.response.instructions, + /Reset to Globe View and rotate the globe/, + ); +}); + +test('failed reset skips waiting rotation without blocking queued Stop or lookup', async () => { + let finishReset; + const pending = new Promise((resolve) => { + finishReset = resolve; + }); + const { controller, sent, calls } = harness((name, args) => + name === 'zoom_to_globe' + ? pending + : { ok: true, action: name, motion: args.motion }, + ); + begin(controller); + const reset = invoke(controller, { + callId: 'pending-reset', + name: 'zoom_to_globe', + args: {}, + }); + const rotation = invoke(controller); + const stop = invoke(controller, { + callId: 'queued-stop', + args: { motion: 'stop' }, + }); + const lookup = invoke(controller, { + callId: 'queued-lookup', + name: 'get_current_view_state', + args: {}, + }); + finishReset({ ok: false, action: 'zoom_to_globe', cancelled: true }); + await Promise.all([reset, rotation, stop, lookup]); + complete(controller); + assert.deepEqual(calls, [ + 'zoom_to_globe', + 'move_camera', + 'get_current_view_state', + ]); + for (const id of ['queued-stop', 'queued-lookup']) { + const output = sent.find((event) => event.item?.call_id === id); + assert.equal(JSON.parse(output.item.output).ok, true); + } + const output = sent.find((event) => event.item?.call_id === 'rotation-call'); + assert.equal(JSON.parse(output.item.output).skipped, true); + const replies = sent.filter((event) => event.type === 'response.create'); + assert.equal(replies.length, 1); + assert.equal(replies[0].response.tool_choice, 'none'); +}); + +for (const input of ['text', 'speech']) { + test(`aborted reset stays failed across followups; fresh ${input} isolates late settlement`, async () => { + let finishReset; + const pending = new Promise((resolve) => { + finishReset = resolve; + }); + const { controller, sent, calls } = harness((name) => + name === 'zoom_to_globe' ? pending : rotated, + ); + delete controller.cancelRadioHandoff; + begin(controller, 'old-reset'); + const reset = invoke(controller, { + responseId: 'old-reset', + callId: 'old-reset-call', + name: 'zoom_to_globe', + args: {}, + }); + await controller.handleRealtimeEvent({ + data: JSON.stringify({ + type: 'error', + error: { code: 'conversation_already_has_active_response' }, + }), + }); + complete(controller, 'old-reset'); + begin(controller, 'old-followup'); + await invoke(controller, { + responseId: 'old-followup', + callId: 'blocked-rotation', + }); + assert.deepEqual(calls, ['zoom_to_globe']); + const skipped = sent.find( + (event) => event.item?.call_id === 'blocked-rotation', + ); + assert.equal(JSON.parse(skipped.item.output).reason, 'globe-reset-failed'); + if (input === 'text') controller.sendTextCommand('rotate the globe'); + else await speak(controller); + begin(controller, 'fresh'); + await invoke(controller, { responseId: 'fresh', callId: 'fresh-rotation' }); + finishReset({ ok: false, action: 'zoom_to_globe', cancelled: true }); + await reset; + await invoke(controller, { + responseId: 'fresh', + callId: 'after-old-settlement', + }); + assert.deepEqual(calls, ['zoom_to_globe', 'move_camera', 'move_camera']); + }); +} + +test('a real tool followup waits for a reset still pending in the previous response', async () => { + let finishReset; + const pending = new Promise((resolve) => { + finishReset = resolve; + }); + const { controller, sent, calls } = harness((name) => + name === 'zoom_to_globe' ? pending : { ok: true, action: name }, + ); + begin(controller, 'reset-response'); + const reset = invoke(controller, { + responseId: 'reset-response', + callId: 'pending-reset', + name: 'zoom_to_globe', + args: {}, + }); + await invoke(controller, { + responseId: 'reset-response', + callId: 'fast-lookup', + name: 'get_current_view_state', + args: {}, + }); + complete(controller, 'reset-response'); + assert.equal( + sent.filter((event) => event.type === 'response.create').length, + 1, + ); + assert.equal( + JSON.parse( + sent.find((event) => event.item?.call_id === 'fast-lookup').item.output, + ).ok, + true, + ); + begin(controller, 'followup'); + const rotation = invoke(controller, { + responseId: 'followup', + callId: 'waiting-rotation', + }); + assert.deepEqual(calls, ['zoom_to_globe', 'get_current_view_state']); + finishReset({ ok: true, action: 'zoom_to_globe' }); + await Promise.all([reset, rotation]); + assert.deepEqual(calls, [ + 'zoom_to_globe', + 'get_current_view_state', + 'move_camera', + ]); + assert.equal( + sent.filter((event) => event.item?.call_id === 'waiting-rotation').length, + 1, + ); +}); + +test('a recoverable collision keeps a completed reset successful for its rotation', async () => { + const { controller, sent, calls } = harness((name) => + name === 'zoom_to_globe' + ? { ok: true, action: name } + : { ok: true, action: name, motion: 'rotate_globe' }, + ); + delete controller.cancelRadioHandoff; + begin(controller); + await invoke(controller, { + callId: 'done-reset', + name: 'zoom_to_globe', + args: {}, + }); + await controller.handleRealtimeEvent({ + data: JSON.stringify({ + type: 'error', + error: { code: 'conversation_already_has_active_response' }, + }), + }); + await invoke(controller); + complete(controller); + assert.deepEqual(calls, ['zoom_to_globe', 'move_camera']); + const output = sent.find((event) => event.item?.call_id === 'rotation-call'); + assert.equal(JSON.parse(output.item.output).ok, true); +}); + +test('an old response reset cannot poison a fresh spoken rotation request', async () => { + for (const lateReset of [false, true]) { + const { controller, calls } = harness((name) => + name === 'zoom_to_globe' + ? { ok: false, action: name } + : { ok: true, action: name, motion: 'rotate_globe' }, + ); + // Use the production speech-start cancellation wiring, not the harness stub. + delete controller.cancelRadioHandoff; + begin(controller, 'old-reset-response'); + if (lateReset) await speak(controller); + await invoke(controller, { + responseId: 'old-reset-response', + callId: 'old-reset', + name: 'zoom_to_globe', + args: {}, + }); + if (!lateReset) await speak(controller); + begin(controller, 'fresh-rotation-response'); + await invoke(controller, { + responseId: 'fresh-rotation-response', + callId: 'fresh-rotation', + }); + assert.deepEqual( + calls, + ['zoom_to_globe', 'move_camera'], + lateReset + ? 'late old reset must retain its original request' + : 'fresh speech clears prior reset failure', + ); + } +}); + +test('a late rotation without a reset retains existing arrival-time policy', async () => { + const { controller, calls } = harness({ + ok: true, + action: 'move_camera', + motion: 'rotate_globe', + }); + delete controller.cancelRadioHandoff; + begin(controller, 'old-rotation-response'); + await speak(controller); + await invoke(controller, { + responseId: 'old-rotation-response', + callId: 'late-rotation', + }); + assert.deepEqual(calls, ['move_camera']); +}); diff --git a/src/voice/realtimeProtocol.js b/src/voice/realtimeProtocol.js index 14105229c..a721bb7c9 100644 --- a/src/voice/realtimeProtocol.js +++ b/src/voice/realtimeProtocol.js @@ -99,6 +99,18 @@ export function hasStructuredViewIdentity(result) { ); } +export function isGlobeViewRefusal(result) { + return ( + result?.action === 'move_camera' && + result.ok === false && + [ + 'globe-view-required', + 'globe-reset-unavailable', + 'globe-reset-failed', + ].includes(result.reason) + ); +} + export function responseInstructionForToolResult(result) { if (result?.action === 'control_radio' && result.radioPlaybackSuppressed) { if (result.audioState === 'paused') { @@ -123,6 +135,20 @@ export function responseInstructionForToolResult(result) { : 'Be concise: useful names, place, layer and one or two notable properties.', ].join(' '); } + if (isGlobeViewRefusal(result)) { + // A refusal is not a completed action. Keep tool text out of instructions. + if (result.reason === 'globe-reset-failed') + return 'The globe did not rotate after the Globe View reset attempt. Say so briefly. Do not retry, offer or perform another reset for this request.'; + if (result.reason === 'globe-reset-unavailable') + return 'The globe did not rotate. Ask the user to zoom out further; the Globe View preset does not fit this viewport. Do not offer or perform a reset.'; + return 'The globe did not rotate. Say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" The suggested command is not a user request. Do not ask a question, perform a reset, or call tools in this reply.'; + } + if ( + (result?.action ?? result?.tool) === 'zoom_to_globe' && + (result.ok !== true || result.cancelled) + ) { + return 'The Globe View reset did not finish. Say so briefly; do not claim Globe View was reached. Do not retry the reset or rotate the globe for this request.'; + } if (result?.action === 'adjust_camera_zoom') { return result.ok ? `Confirm once that the camera zoomed ${result.direction}. Do not claim any other change.` diff --git a/src/voice/realtimeTurns.js b/src/voice/realtimeTurns.js index 2073be06d..29c4b3bc0 100644 --- a/src/voice/realtimeTurns.js +++ b/src/voice/realtimeTurns.js @@ -6,6 +6,7 @@ import { SUPERSEDED_RESPONSE_MEMORY, shouldStopVoiceAfterRadioTool, responseInstructionForToolResult, + isGlobeViewRefusal, extractFunctionCalls, callDedupeKeys, parseArguments, @@ -66,6 +67,11 @@ export class RealtimeTurns { this.pendingUserTextResponse = false; this.activeResponseId = null; this.supersededResponseIds = new Set(); + this.globeRefusals = new Map(); + this.globeAdmissionGates = new Map(); + this.globeResets = new Map(); + this.globeResponseRequests = new Map(); + this.globeRequest = { failed: false, pending: new Set() }; this.activeToolAbortControllers = new Set(); this.narration = null; this.narrationSequence = 0; @@ -138,6 +144,7 @@ export class RealtimeTurns { }, }; if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return; + this.globeRequest = { failed: false, pending: new Set() }; this.requestUserTextResponse(); } @@ -330,6 +337,7 @@ export class RealtimeTurns { this.userTurnPending = true; this.pendingResponseInstructions = null; this.cancelRadioHandoff({ abortTools: true }); + this.globeRequest = { failed: false, pending: new Set() }; this.setVoiceSpeaker('user'); // Point-and-ask: read the pointer as speech starts (open mic), or reuse // the push-to-talk keydown snapshot, before the user audio commits. @@ -384,6 +392,27 @@ export class RealtimeTurns { } const toolResponseId = payload.response_id || payload.response?.id || null; + // Follow-up responses retain reset ownership until fresh admitted input. + const arrivalRequest = this.globeRequest; + const globeRequest = + this.globeResponseRequests.get(toolResponseId) ?? arrivalRequest; + // Later siblings must wait for globe admission before they can mutate state. + // Other tool responses keep their existing concurrency. + let heldForAdmission = false; + while (toolResponseId && this.globeAdmissionGates.has(toolResponseId)) { + // Drop claimed duplicates before every wait, before their keys can expire. + if ( + calls.every((call) => + callDedupeKeys(call).some((key) => this.processedCalls.has(key)), + ) + ) + return; + heldForAdmission = true; + await this.globeAdmissionGates.get(toolResponseId).promise; + if (!this.ownsConversation(eventChannel) || this.isSessionEnding()) + return; + } + // A newer typed command superseded the response these calls belong to. // They are stale intent — dispatching one would let the old turn mutate // the map after the operator asked for something else. @@ -394,7 +423,10 @@ export class RealtimeTurns { // hazard `callDedupeKeys` is written to avoid). So each refused call gets // a terminal output saying plainly that the turn moved on. No // `response.create` follows — the deferred typed turn is the single answer. - if (this.isSupersededResponse(toolResponseId)) { + if ( + this.isSupersededResponse(toolResponseId) || + (heldForAdmission && arrivalRequest !== this.globeRequest) + ) { this.pruneProcessedCalls(); for (const call of calls) { const keys = callDedupeKeys(call); @@ -429,7 +461,19 @@ export class RealtimeTurns { if (keys.some((key) => this.processedCalls.has(key))) continue; keys.forEach((key) => this.processedCalls.set(key, performance.now())); + if (this.globeRefusals.get(toolResponseId)?.blocking) { + this.sendToolOutput(call.call_id || call.id, { + ok: false, + action: call.name, + skipped: true, + error: 'Not run: this response must wait for the Globe View reply.', + }); + continue; + } + let result; + let globeAdmissionGate = null; + let globeResetGate = null; const resultChannel = this.dc; let radioHandoffEpochAtStart = this.radio.radioHandoffEpoch; let toolController = null; @@ -443,6 +487,80 @@ export class RealtimeTurns { this.narration?.preamble(); try { const parsedArguments = parseArguments(call.arguments); + if ( + toolResponseId && + call.name === 'move_camera' && + String(parsedArguments.motion || '').toLowerCase() === 'rotate_globe' + ) { + let resolve; + const promise = new Promise((settle) => { + resolve = settle; + }); + globeAdmissionGate = { promise, resolve }; + this.globeAdmissionGates.set(toolResponseId, globeAdmissionGate); + // OpenAI delivers sibling calls concurrently. A reset dispatched + // first must finish before rotation evaluates the camera's fit. + // Hold later siblings at admission while this rotation waits. + const resets = this.globeResets.get(toolResponseId); + if ( + resets || + globeRequest.failed || + globeRequest.pending.size || + arrivalRequest !== this.globeRequest + ) { + while (resets?.pending.size || globeRequest.pending.size) + await Promise.all( + [ + ...new Set([ + ...(resets?.pending || []), + ...globeRequest.pending, + ]), + ].map((gate) => gate.promise), + ); + if (!this.ownsConversation(eventChannel) || this.isSessionEnding()) + return; + keys.forEach((key) => + this.processedCalls.set(key, performance.now()), + ); + const superseded = + this.isSupersededResponse(toolResponseId) || + arrivalRequest !== this.globeRequest; + if (superseded || resets?.failed || globeRequest.failed) { + result = { + ok: false, + action: call.name, + reason: 'globe-reset-failed', + skipped: true, + ...(superseded ? { superseded: true } : {}), + error: superseded + ? 'Superseded by a newer command from the operator — this call was not run.' + : 'Not run: the Globe View reset/rotation sequence failed or was cancelled.', + }; + } + } + } + if (toolResponseId && call.name === 'zoom_to_globe') { + let resolve; + const promise = new Promise((settle) => { + resolve = settle; + }); + let resets = this.globeResets.get(toolResponseId); + if (!resets) { + resets = { pending: new Set(), failed: false }; + this.globeResets.set(toolResponseId, resets); + // Retain settled outcomes for later siblings, without evicting an + // active reset that another response still needs to await. + for (const [id, record] of this.globeResets) { + if (this.globeResets.size <= SUPERSEDED_RESPONSE_MEMORY) break; + if (!record.pending.size && id !== toolResponseId) + this.globeResets.delete(id); + } + } + globeResetGate = { promise, resolve, resets, request: globeRequest }; + resets.pending.add(globeResetGate); + globeRequest.pending.add(globeResetGate); + } + const isRadioControlCall = call.name === 'control_radio'; const isRadioVisibilityCall = call.name === 'set_layer_visibility' && @@ -499,7 +617,7 @@ export class RealtimeTurns { authorityDomain: radioAuthorityDomain, }); } - result = await (this.actionExecutor || this.runner)( + result ??= await (this.actionExecutor || this.runner)( call.name, parsedArguments, { @@ -515,7 +633,28 @@ export class RealtimeTurns { radioHandoffEpochAtStart === this.radio.radioHandoffEpoch), }, ); + if ( + globeAdmissionGate && + this.globeResets.has(toolResponseId) && + result?.reason === 'globe-view-required' + ) + result = { ...result, reason: 'globe-reset-failed' }; if (!this.ownsConversation(eventChannel)) return; + if ( + isGlobeViewRefusal(result) && + toolResponseId && + arrivalRequest === this.globeRequest && + !this.isSupersededResponse(toolResponseId) + ) { + this.globeRefusals.set(toolResponseId, { + result, + followupQueued: false, + blocking: !result.skipped, + }); + while (this.globeRefusals.size > SUPERSEDED_RESPONSE_MEMORY) { + this.globeRefusals.delete(this.globeRefusals.keys().next().value); + } + } if (result?.ok && result.radioPlaybackRequested) { const sessionIsCurrent = this.activeToolAbortControllers.has(toolController) && @@ -595,6 +734,23 @@ export class RealtimeTurns { : {}), }; } finally { + if (globeResetGate) { + const { resets, request } = globeResetGate; + if (result?.ok !== true || result.cancelled) { + resets.failed = true; + request.failed = true; + } + resets.pending.delete(globeResetGate); + request.pending.delete(globeResetGate); + globeResetGate.resolve(); + } + if (globeAdmissionGate) { + if ( + this.globeAdmissionGates.get(toolResponseId) === globeAdmissionGate + ) + this.globeAdmissionGates.delete(toolResponseId); + globeAdmissionGate.resolve(); + } if (toolController) { this.activeToolAbortControllers.delete(toolController); this.radio.releaseTool(toolController); @@ -671,13 +827,38 @@ export class RealtimeTurns { }); } if (!this.ownsConversation(eventChannel)) return; - // Keep the Radio handoff wording authoritative even when another tool - // result follows Radio in the same multi-intent response. - this.queueResponseCreate( - responseInstructionForToolResult( - this.radio.pendingRadioPlaybackResult || lastResult, - ), - ); + const refusal = this.globeRefusals.get(toolResponseId); + // A newer turn may arrive during admission or the viewport-image await. + // Preserve its reply; neither the refusal map nor lastResult may revive + // an old refusal follow-up after supersession. + if ( + (this.isSupersededResponse(toolResponseId) || + arrivalRequest !== this.globeRequest) && + (refusal || isGlobeViewRefusal(lastResult) || lastResult?.superseded) + ) { + // Complete an idle turn below, but leave a newer running tool's status + // authoritative while suppressing this obsolete refusal follow-up. + if (this.activeToolAbortControllers.size) return; + } else if (refusal) { + // Even an earlier sibling finishing late cannot replace the reset instruction. + if (!refusal.followupQueued) { + refusal.followupQueued = true; + this.queueResponseCreate({ + instructions: responseInstructionForToolResult(refusal.result), + tool_choice: 'none', + }); + } + } else { + // Keep Radio handoff wording authoritative for ordinary multi-tool replies. + const followupResult = + this.radio.pendingRadioPlaybackResult || lastResult; + const instructions = responseInstructionForToolResult(followupResult); + this.queueResponseCreate( + isGlobeViewRefusal(followupResult) + ? { instructions, tool_choice: 'none' } + : instructions, + ); + } } this.setStatus('listening', t('Ask or command')); } @@ -966,6 +1147,22 @@ export class RealtimeTurns { this.userTurnPending = false; this.activeResponseId = payload.response?.id || payload.response_id || null; + // Late calls retain only their globe reset/request ownership. Ordinary + // tool admission and concurrency continue to use their existing policy. + if ( + this.activeResponseId && + !this.globeResponseRequests.has(this.activeResponseId) + ) { + this.globeResponseRequests.set( + this.activeResponseId, + this.globeRequest, + ); + while (this.globeResponseRequests.size > SUPERSEDED_RESPONSE_MEMORY) + this.globeResponseRequests.delete( + this.globeResponseRequests.keys().next().value, + ); + } + this.setVoiceSpeaker('ai'); return; } @@ -1045,7 +1242,8 @@ export class RealtimeTurns { const sent = this.sendRealtimeEvent( { type: 'response.create', - response: { instructions }, + response: + typeof instructions === 'string' ? { instructions } : instructions, }, 'client.response_create.tool_followup', ); @@ -1057,6 +1255,20 @@ export class RealtimeTurns { controller.abort(); this.activeToolAbortControllers.clear(); this.radio.clearTools(); + this.releaseGlobeResets(); + } + + releaseGlobeResets() { + for (const resets of this.globeResets.values()) { + if (!resets.pending.size) continue; + resets.failed = true; + for (const gate of resets.pending) { + gate.request.failed = true; + gate.request.pending.delete(gate); + gate.resolve(); + } + resets.pending.clear(); + } } reset() { @@ -1068,6 +1280,13 @@ export class RealtimeTurns { this.pendingUserTextResponse = false; this.activeResponseId = null; this.supersededResponseIds.clear(); + this.globeRefusals.clear(); + for (const gate of this.globeAdmissionGates.values()) gate.resolve(); + this.globeAdmissionGates.clear(); + this.releaseGlobeResets(); + this.globeResets.clear(); + this.globeResponseRequests.clear(); + this.globeRequest = { failed: false, pending: new Set() }; this.narration?.cancel(); this.assistantAudioPlaying = false; this.speechResponseIds.clear();