From bc281ab246ffd3d4d2c33e734cb2122760ad6c83 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Fri, 16 Oct 2026 22:34:48 +0400 Subject: [PATCH] refactor(voice): W1 actions on the engine The voice runner drives the globe engine (an `engine` option, by default the one registered for the viewer) instead of the Cesium viewer: track_entity, stop_tracking (camera.following().stop()), frame_overhead (camera.flyToSphere with the pose heading), fly_to_location, zoom_to_globe (follow released through camera.following()), adjust_camera_zoom, next_iss_pass / next_satellite_pass, select_nearest_aircraft and the analyst view context, get_current_view_state, get_entity_context (projector.worldToWindow), the view target (picker.pickPosition, then camera.pickEllipsoid, then picker.pickGlobe) and its moveEnd prewarm, and the viewport samples. getBasemapLabelContext takes the engine or a viewer with a registered engine (the HUD). Without a viewer the tools that still need one (VIEWER_ONLY_ACTIONS: control_scene, annotate_map, clear_annotations, fly_route, move_camera, control_cockpit, set_context_mode, control_radio, control_cctv) answer a structured refusal (code UNAVAILABLE_ON_ENGINE) instead of throwing. Point-and-ask picks with engine.picker and keys the camera pose from engine.camera.pose(); the viewport capture reads engine.host.canvas and waits for the engine's postRender (both engines keep their drawing buffer: Cesium's contextOptions, Navara's own WebGL2 canvas, S4). Tests: gevActionsEngine.test.mjs runs each moved action on the fake engine. gevActions.test.mjs keeps its expectations; its runner helper now registers an engine for the test viewer (the Cesium engine over it, or a fake engine at its positionCartographic) and the two harness cameras gain roll and frustum for camera.pose(). --- scripts/format-scope.json | 1 + scripts/package-boundaries.json | 3 +- src/voice/gevActions.js | 330 ++++++++++++---------- src/voice/gevActions.test.mjs | 31 +- src/voice/gevActionsEngine.test.mjs | 422 ++++++++++++++++++++++++++++ src/voice/gevRealtime.js | 16 +- src/voice/pointerContext.js | 13 +- src/voice/pointerPick.js | 26 +- src/voice/realtimeViewport.js | 49 +++- 9 files changed, 701 insertions(+), 190 deletions(-) create mode 100644 src/voice/gevActionsEngine.test.mjs diff --git a/scripts/format-scope.json b/scripts/format-scope.json index de886bb04..629480f0b 100644 --- a/scripts/format-scope.json +++ b/scripts/format-scope.json @@ -1010,6 +1010,7 @@ "scripts/qa/navaraContractPage.js", "scripts/qa/tileCache.mjs", "src/annotations/resolverEngine.test.mjs", + "src/voice/gevActionsEngine.test.mjs", "src/app/controls.test.mjs", "src/app/engineChoice.js", "src/app/engineChoice.test.mjs", diff --git a/scripts/package-boundaries.json b/scripts/package-boundaries.json index 54365eb9b..9f1a2bc3b 100644 --- a/scripts/package-boundaries.json +++ b/scripts/package-boundaries.json @@ -1705,7 +1705,8 @@ "src/voice/narration.js", "src/voice/earcon.js", "src/voice/pointerContext.js", - "src/voice/pointerReticle.js" + "src/voice/pointerReticle.js", + "src/globe/contract.js" ], "external": [] }, diff --git a/src/voice/gevActions.js b/src/voice/gevActions.js index c0b4cf42a..34e3d37f2 100644 --- a/src/voice/gevActions.js +++ b/src/voice/gevActions.js @@ -43,6 +43,7 @@ import { } from '../cameraVerbs.js'; import * as defaultFloorServices from '../data/groundFloor.js'; import { isPickedWorldPosition } from '../data/scenePick.js'; +import { engineFromViewer } from '../globe/cesium/engineFromViewer.js'; import { unavailablePlaceSearch } from '../search/placeSearch.js'; import * as defaultAnnotationResolver from '../annotations/annotationResolver.js'; import { normalizeRadioCountryInput } from '../data/radioCountry.js'; @@ -264,6 +265,38 @@ const FRAME_TARGETS = new Map([ ['ships', 'ais-live-vessels'], ]); +/** + * The tools that still need the Cesium viewer or a subsystem built on it: + * the scene director, annotations and routes over them, the camera verbs + * (orbit, pan, tilt), the cockpit, Contacts (the context mode), Radio and + * CCTV. Without a viewer (the Navara engine) each answers a structured + * refusal (UNAVAILABLE_ON_ENGINE) instead of running. + */ +export const VIEWER_ONLY_ACTIONS = Object.freeze([ + 'control_scene', + 'annotate_map', + 'clear_annotations', + 'fly_route', + 'move_camera', + 'control_cockpit', + 'set_context_mode', + 'control_radio', + 'control_cctv', +]); +/** The refusal code of a viewer-only tool on an engine without a viewer. */ +export const UNAVAILABLE_ON_ENGINE = 'UNAVAILABLE_ON_ENGINE'; + +function unavailableOnEngine(name, engine) { + return { + ok: false, + action: name, + unavailable: true, + code: UNAVAILABLE_ON_ENGINE, + engine: engine?.kind ?? null, + error: 'Unavailable on this engine', + }; +} + const serviceCaches = new WeakMap(); function cachesFor(service) { service.signal?.throwIfAborted(); @@ -289,10 +322,21 @@ function cachesFor(service) { const VISIBLE_ENTITY_SHORTLIST = 64; const BASEMAP_CONTEXT_WAIT_MS = 1500; const viewTargetCache = new WeakMap(); +const prewarmedEngines = new WeakSet(); -/** Create application actions over the supplied scene and services. */ +/** The camera's place (radians, metres) from its live position. */ +const cameraCartographic = (engine) => + geo.cartographic.fromCartesian(engine.camera.livePosition); + +/** + * Create application actions over the supplied scene and services. The + * actions drive the globe `engine` (by default the one registered for the + * viewer); `viewer` is null on an engine without one (Navara), where the + * VIEWER_ONLY_ACTIONS refuse. + */ export function createGevActionRunner({ viewer, + engine: givenEngine = null, styleManager, dataManager, sceneDirector = null, @@ -310,20 +354,22 @@ export function createGevActionRunner({ const builders = speechBuilders ? { ...SPEECH_BUILDERS, ...speechBuilders } : SPEECH_BUILDERS; + const engine = givenEngine ?? engineFromViewer(viewer); const _layerEnabledAt = new Map(); let analystEngine; const resolveRegionRing = (name, signal) => annotationResolver.resolveRegionRingForQuery(name, signal, placeSearch); const ensureAnalystEngine = () => (analystEngine ||= createAnalystEngine( - analystProviders(viewer, dataManager, { + analystProviders(engine, dataManager, { placeSearch, resolveRegionRing, isWarming: (layerKey) => layerIsWarming(_layerEnabledAt, layerKey), }), )); - installViewTargetPrewarm(viewer); - initCameraVerbs(viewer, getViewTargetCartesian); + installViewTargetPrewarm(engine); + // The camera verbs (move_camera, fly_route) still run on the viewer. + if (viewer) initCameraVerbs(viewer, () => getViewTargetCartesian(engine)); // Nested calls (select_nearest_aircraft composing layer + flight) use the // raw runner; only the outer call gets a voice envelope. const runGevAction = async function runGevAction( @@ -343,6 +389,8 @@ export function createGevActionRunner({ /* Progress is advisory. */ } }; + if (!viewer && VIEWER_ONLY_ACTIONS.includes(name)) + return unavailableOnEngine(name, engine); // Navigation tools interrupt any continuous camera motion (spec §1.1) — // checked FIRST because each handler returns. @@ -352,8 +400,8 @@ export function createGevActionRunner({ // Explicit navigation while TRACKING supersedes the follow camera — // otherwise the tracker drags the view back and "I flew there but can't // do anything" (field finding). track_entity manages its own handoff. - if (name === 'zoom_to_globe' && viewer.trackedEntity) { - stopAllTracking(viewer, dataManager); + if (name === 'zoom_to_globe' && engine?.camera.following()) { + stopAllTracking(engine, dataManager); } // Zoom during an active orbit adjusts the orbit RADIUS (spiral in/out) — @@ -587,7 +635,7 @@ export function createGevActionRunner({ }); } else if (typeof layerModule?.update === 'function') { refreshed = - (await layerModule.update(viewer, { + (await layerModule.update(viewer ?? engine, { signal: runOptions.signal, })) !== false; } @@ -620,7 +668,7 @@ export function createGevActionRunner({ reportProgress('nearest', location.label); const nearest = await createAnalystEngine( - analystProviders(viewer, dataManager, { + analystProviders(engine, dataManager, { recordLimitByLayer: { [layerId]: Number.MAX_SAFE_INTEGER }, placeSearch, resolveRegionRing, @@ -653,7 +701,7 @@ export function createGevActionRunner({ } const stableAircraftId = aircraft.icao24 || aircraft.id; - const selection = await trackEntity(viewer, dataManager, styleManager, { + const selection = await trackEntity(engine, dataManager, styleManager, { query: stableAircraftId, layerId, }); @@ -950,7 +998,7 @@ export function createGevActionRunner({ runOptions, ); } - return flyToRequestedLocation(viewer, args, { + return flyToRequestedLocation(engine, args, { onProgress: reportProgress, placeSearch, searchNavigation, @@ -975,20 +1023,20 @@ export function createGevActionRunner({ return; } interruptCameraMotion('nav:fly_to_location'); - if (viewer.trackedEntity) stopAllTracking(viewer, dataManager); + if (engine?.camera.following()) stopAllTracking(engine, dataManager); }, }); } if (name === 'adjust_camera_zoom') { - return adjustCameraZoom(viewer, args); + return adjustCameraZoom(engine, args); } if (name === 'zoom_to_globe') { if (typeof styleManager?.resetToGlobeView === 'function') { return styleManager.resetToGlobeView(); } - const result = flyToGlobeView(viewer); + const result = flyToGlobeView(engine); return { ok: true, action: 'zoom_to_globe', @@ -1001,11 +1049,11 @@ export function createGevActionRunner({ } if (name === 'next_satellite_pass') { - return nextSatellitePass(viewer, dataManager, args); + return nextSatellitePass(engine, dataManager, args); } if (name === 'next_iss_pass') { - return nextIssPass(viewer, dataManager, args); + return nextIssPass(engine, dataManager, args); } if (name === 'analyst_query') { @@ -1049,7 +1097,7 @@ export function createGevActionRunner({ if (name === 'get_entity_context') { return getEntityContext( - viewer, + engine, dataManager, styleManager, args, @@ -1059,7 +1107,7 @@ export function createGevActionRunner({ if (name === 'get_current_view_state') { return getCurrentViewState( - viewer, + engine, styleManager, dataManager, sceneDirector, @@ -1164,17 +1212,17 @@ export function createGevActionRunner({ } if (name === 'track_entity') { - return trackEntity(viewer, dataManager, styleManager, args, { + return trackEntity(engine, dataManager, styleManager, args, { isCurrent: current, }); } if (name === 'stop_tracking') { - return stopAllTracking(viewer, dataManager); + return stopAllTracking(engine, dataManager); } if (name === 'frame_overhead') { - return frameOverhead(viewer, dataManager, styleManager, args); + return frameOverhead(engine, dataManager, styleManager, args); } if (name === 'annotate_map') { @@ -2187,7 +2235,7 @@ export function formatTrackedEntityLabel(found, query = '') { /** Finds and tracks/selects an entity by spoken query across layer families. */ async function trackEntity( - viewer, + engine, dataManager, styleManager, args = {}, @@ -2240,7 +2288,7 @@ async function trackEntity( 'fire', 'track_entity', () => { - flyToLandmark(viewer, strongest.latitude, strongest.longitude, { + flyToLandmark(engine, strongest.latitude, strongest.longitude, { range: 14000, pitch: -50, heading: 0, @@ -2300,7 +2348,7 @@ async function trackEntity( let trackedOk = false; if (family.kind === 'vessel') { trackedOk = !!module.selectById?.(found.mmsi); - flyToLandmark(viewer, found.latitude, found.longitude, { + flyToLandmark(engine, found.latitude, found.longitude, { range: 6000, pitch: -45, heading: 0, @@ -2346,7 +2394,7 @@ async function trackEntity( } /** Releases tracking/selection on every entity layer family. */ -function stopAllTracking(viewer, dataManager) { +function stopAllTracking(engine, dataManager) { const released = []; const failed = new Set(); for (const family of TRACKABLE_FAMILIES) { @@ -2397,7 +2445,7 @@ function stopAllTracking(viewer, dataManager) { failed.add(layerId); } } - if (viewer) viewer.trackedEntity = undefined; + engine?.camera.following()?.stop(); if (failed.size) { const failedLayerIds = [...failed]; return { @@ -2418,7 +2466,7 @@ function stopAllTracking(viewer, dataManager) { * detection so the framed entities are labeled, and reports * detectionEnabled so the voice agent can mention labels are on. */ -async function frameOverhead(viewer, dataManager, styleManager, args = {}) { +async function frameOverhead(engine, dataManager, styleManager, args = {}) { const targetRaw = String(args.target || 'flights').toLowerCase(); const layerId = FRAME_TARGETS.get(targetRaw) || normalizeLayerId(targetRaw) || 'flights'; @@ -2445,7 +2493,9 @@ async function frameOverhead(viewer, dataManager, styleManager, args = {}) { ? 120 : 150; const radiusKm = clampNumber(args.radiusKm, 10, 20000, defaultRadiusKm); - const center = getViewTargetCartesian(viewer) || viewer.camera.positionWC; + const center = + getViewTargetCartesian(engine) || + geo.cartesian3.clone(engine.camera.livePosition); let entries = []; if (typeof module.getNearby === 'function') { @@ -2472,23 +2522,25 @@ async function frameOverhead(viewer, dataManager, styleManager, args = {}) { }; } - const sphere = Cesium.BoundingSphere.fromPoints( + // Cesium's bounding sphere (pure math, no viewer), flown by the engine. + const bounds = Cesium.BoundingSphere.fromPoints( entries.map((entry) => entry.position), ); - sphere.radius = Math.max(sphere.radius * 1.25, 8000); + const sphere = { + center: geo.cartesian3.clone(bounds.center), + radius: Math.max(bounds.radius * 1.25, 8000), + }; const pitch = geo.math.toRadians(isSatellites ? -35 : -62); return runManagedVoiceNavigation( styleManager, 'frame', 'frame_overhead', () => { - viewer.camera.flyToBoundingSphere(sphere, { + engine.camera.flyToSphere(sphere, { duration: 2.0, - offset: new Cesium.HeadingPitchRange( - viewer.camera.heading, - pitch, - sphere.radius * 2.4, - ), + heading: engine.camera.pose().heading, + pitch, + range: sphere.radius * 2.4, }); let detectionEnabled = false; @@ -2567,25 +2619,30 @@ function collectTrackedEntities(dataManager) { return tracked; } +/** + * @param {object} target The globe engine, or a Cesium viewer with a + * registered engine (the HUD). + */ export async function getBasemapLabelContext( - viewer, + target, service = defaultGeospatial, { cachedOnly = false } = {}, ) { + const engine = engineFromViewer(target); const { reverseGeocodeCache, nearbyPlacesCache } = cachesFor(service); - const samples = sampleViewportCartographics(viewer); - const cameraHeightM = viewer.camera.positionCartographic.height; - const target = getViewTargetCartographic(viewer); - if (!target) { + const viewTarget = engine ? getViewTargetCartographic(engine) : null; + if (!viewTarget) { return { placeLabels: [], streetLabels: [], nearbyPlaceLabels: [], }; } + const samples = sampleViewportCartographics(engine); + const cameraHeightM = engine.camera.height(); - const latitude = Number(geo.math.toDegrees(target.latitude).toFixed(6)); - const longitude = Number(geo.math.toDegrees(target.longitude).toFixed(6)); + const latitude = Number(geo.math.toDegrees(viewTarget.latitude).toFixed(6)); + const longitude = Number(geo.math.toDegrees(viewTarget.longitude).toFixed(6)); const cachedViewportPlaces = viewportPlacesFromCache( samples, cameraHeightM, @@ -2646,12 +2703,12 @@ export async function getBasemapLabelContext( }; } -function installViewTargetPrewarm(viewer) { - if (viewer.__gevViewTargetPrewarmInstalled) return; - viewer.__gevViewTargetPrewarmInstalled = true; +function installViewTargetPrewarm(engine) { + if (!engine || prewarmedEngines.has(engine)) return; + prewarmedEngines.add(engine); let timer = null; let reportedPrewarmFailure = false; - viewer.camera.moveEnd.addEventListener(() => { + engine.camera.moveEnd.addEventListener(() => { if (timer) window.clearTimeout(timer); timer = window.setTimeout(() => { timer = null; @@ -2659,10 +2716,10 @@ function installViewTargetPrewarm(viewer) { // because an idle callback is an UNCAUGHT context — nothing above it can // handle a surprise from the scene graph, and a red console error on a // plain camera flight is worse than a cold cache. Reported once per - // viewer so a repeating cause cannot spam the console. + // engine so a repeating cause cannot spam the console. const warm = () => { try { - getViewTargetCartographic(viewer); + getViewTargetCartographic(engine); } catch (error) { if (reportedPrewarmFailure) return; reportedPrewarmFailure = true; @@ -2681,7 +2738,7 @@ function installViewTargetPrewarm(viewer) { }); } -function adjustCameraZoom(viewer, args) { +function adjustCameraZoom(engine, args) { const direction = String(args.direction || '').toLowerCase(); if (direction !== 'in' && direction !== 'out') { throw new Error('adjust_camera_zoom direction must be "in" or "out"'); @@ -2695,10 +2752,10 @@ function adjustCameraZoom(viewer, args) { }[amount]; if (!fraction) throw new Error(`Unknown zoom amount: ${args.amount}`); - const camera = viewer.camera; - const beforePosition = geo.cartesian3.clone(camera.positionWC); - const beforeHeightM = camera.positionCartographic.height; - const target = getViewTargetCartesian(viewer); + const camera = engine.camera; + const beforePosition = geo.cartesian3.clone(camera.livePosition); + const beforeHeightM = camera.height(); + const target = getViewTargetCartesian(engine); const targetDistanceM = target ? geo.cartesian3.distance(beforePosition, target) : Math.max(100, beforeHeightM); @@ -2724,11 +2781,11 @@ function adjustCameraZoom(viewer, args) { } camera.zoomIn(safeMovementM); } - viewer.scene.requestRender(); + engine.render.request('voice-zoom'); - const afterPosition = camera.positionWC; + const afterPosition = camera.livePosition; const movedM = geo.cartesian3.distance(beforePosition, afterPosition); - const afterHeightM = camera.positionCartographic.height; + const afterHeightM = camera.height(); const moved = movedM >= 0.5; return { ok: moved, @@ -2766,11 +2823,11 @@ function compassDir(azDeg) { return COMPASS_16[Math.round((((azDeg % 360) + 360) % 360) / 22.5) % 16]; } -function nextIssPass(viewer, dataManager, args) { +function nextIssPass(engine, dataManager, args) { let latDeg = Number.isFinite(args.latitude) ? args.latitude : null; let lonDeg = Number.isFinite(args.longitude) ? args.longitude : null; if (latDeg == null || lonDeg == null) { - const carto = viewer?.camera?.positionCartographic; + const carto = engine ? cameraCartographic(engine) : null; if (!carto) throw new Error('Camera position unavailable'); latDeg = geo.math.toDegrees(carto.latitude); lonDeg = geo.math.toDegrees(carto.longitude); @@ -2816,7 +2873,7 @@ function nextIssPass(viewer, dataManager, args) { }; } -function nextSatellitePass(viewer, dataManager, args) { +function nextSatellitePass(engine, dataManager, args) { const layer = dataManager?.layers?.get('satellites')?.module; const identity = layer?.resolveSatelliteForPass?.(args.target) || { status: 'not-found', @@ -2849,7 +2906,7 @@ function nextSatellitePass(viewer, dataManager, args) { ], ]), }; - const result = nextIssPass(viewer, adapter, args); + const result = nextIssPass(engine, adapter, args); if (result.error) { result.error = result.error.replace( /ISS/g, @@ -3015,7 +3072,7 @@ function normalizeStyle(value) { } async function flyToRequestedLocation( - viewer, + engine, args, { placeSearch = unavailablePlaceSearch, @@ -3067,7 +3124,7 @@ async function flyToRequestedLocation( if (locationId) { const result = immediate(() => - flyToPresetLocation(viewer, locationId, { + flyToPresetLocation(engine, locationId, { ...(rangeM || args.viewMode === 'close' ? { range: rangeM || 250 } : { viewMode: 'overview' }), @@ -3097,7 +3154,7 @@ async function flyToRequestedLocation( const longitude = Number(args.longitude); if (Number.isFinite(latitude) && Number.isFinite(longitude)) { const result = immediate(() => - flyToLandmark(viewer, latitude, longitude, { + flyToLandmark(engine, latitude, longitude, { range: rangeM || 250, pitch: -35, heading: 0, @@ -3129,7 +3186,7 @@ async function flyToRequestedLocation( const poiMatch = findPoiByName(query); if (poiMatch) { const result = immediate(() => - flyToPOI(viewer, poiMatch.cityId, poiMatch.index, { + flyToPOI(engine, poiMatch.cityId, poiMatch.index, { duration: 2.2, onStart: immediateOnStart, ...arrivalHooks, @@ -3154,7 +3211,7 @@ async function flyToRequestedLocation( if (generation === false) return cancelled(query); onProgress('search', query); const managedDeferred = typeof reassertDeferred === 'function'; - const destination = await searchNavigation(viewer, query, { + const destination = await searchNavigation(engine, query, { placeSearch, signal, ...(rangeM ? { range: rangeM } : {}), @@ -3195,12 +3252,12 @@ function normalizeLocationId(value) { } function getCurrentViewState( - viewer, + engine, styleManager, dataManager, sceneDirector = null, ) { - const cartographic = geo.cartographic.fromCartesian(viewer.camera.positionWC); + const cartographic = cameraCartographic(engine); return { ok: true, action: 'get_current_view_state', @@ -3255,7 +3312,7 @@ function getCurrentViewState( } async function getEntityContext( - viewer, + engine, dataManager, styleManager, args = {}, @@ -3264,15 +3321,15 @@ async function getEntityContext( const startedAt = performance.now(); const scope = String(args.scope || 'auto').toLowerCase(); if (scope === 'target' && args.target) { - return getTargetContext(viewer, dataManager, styleManager, args, service); + return getTargetContext(engine, dataManager, styleManager, args, service); } const layerId = normalizeLayerId(args.layerId || args.layer); const limit = Math.round(clampNumber(args.limit, 1, 12, 5)); const selected = selectedEntityContext(dataManager); - const cameraHeightM = viewer.camera.positionCartographic.height; - const viewTarget = getViewTargetCartographic(viewer); + const cameraHeightM = engine.camera.height(); + const viewTarget = getViewTargetCartographic(engine); const scenePromise = getSceneContext( - viewer, + engine, styleManager, dataManager, viewTarget, @@ -3282,7 +3339,7 @@ async function getEntityContext( selected && (scope === 'selected' || scope === 'auto'); const visible = !selectedWillBeReturned && shouldScanVisibleEntities(cameraHeightM) - ? visibleEntityContexts(viewer, dataManager, { + ? visibleEntityContexts(engine, dataManager, { layerId, limit, target: viewTarget, @@ -3420,7 +3477,7 @@ function describeTargetEntity(dataManager, target) { /** get_entity_context for a pointed/referenced target (scope 'target'). */ async function getTargetContext( - viewer, + engine, dataManager, styleManager, args, @@ -3440,10 +3497,10 @@ async function getTargetContext( const hasPosition = Number.isFinite(lat) && Number.isFinite(lon); const viewTarget = hasPosition ? geo.cartographic.fromDegrees(lon, lat) - : getViewTargetCartographic(viewer); + : getViewTargetCartographic(engine); const selected = describeTargetEntity(dataManager, target); const scene = await getSceneContext( - viewer, + engine, styleManager, dataManager, viewTarget, @@ -3458,9 +3515,9 @@ async function getTargetContext( selected, }; } - const cameraHeightM = viewer.camera.positionCartographic.height; + const cameraHeightM = engine.camera.height(); const visible = shouldScanVisibleEntities(cameraHeightM) - ? visibleEntityContexts(viewer, dataManager, { + ? visibleEntityContexts(engine, dataManager, { layerId: normalizeLayerId(args.layerId || args.layer), limit: Math.round(clampNumber(args.limit, 1, 12, 5)), target: viewTarget, @@ -3747,14 +3804,12 @@ function selectedEntityContext(dataManager) { } function visibleEntityContexts( - viewer, + engine, dataManager, { layerId = null, limit = 5, target = null } = {}, ) { const nearbyRecords = []; - const canvas = viewer.scene.canvas; - const width = canvas.clientWidth || canvas.width || 0; - const height = canvas.clientHeight || canvas.height || 0; + const { width, height } = engine.host.size(); const centerX = width / 2; const centerY = height / 2; const targetLat = target ? geo.math.toDegrees(target.latitude) : null; @@ -3795,10 +3850,7 @@ function visibleEntityContexts( for (const { record } of nearbyRecords) { const position = record.entity?.__localBaseCartesian; if (!position) continue; - const screen = Cesium.SceneTransforms.worldToWindowCoordinates( - viewer.scene, - position, - ); + const screen = engine.projector.worldToWindow(position); if ( !screen || screen.x < 0 || @@ -3843,14 +3895,14 @@ function insertNearestRecord(records, candidate, limit) { } async function getSceneContext( - viewer, + engine, styleManager, dataManager, viewTarget = null, service = defaultGeospatial, ) { - const cartographic = geo.cartographic.fromCartesian(viewer.camera.positionWC); - const basemap = await getBasemapContext(viewer, viewTarget, service); + const cartographic = cameraCartographic(engine); + const basemap = await getBasemapContext(engine, viewTarget, service); const enabledLayers = dataManager .getAll() .filter((layer) => layer.enabled) @@ -3873,17 +3925,14 @@ async function getSceneContext( } async function getBasemapContext( - viewer, + engine, viewTarget = null, service = defaultGeospatial, ) { const { reverseGeocodeCache, nearbyPlacesCache } = cachesFor(service); const target = viewTarget; - const samples = sampleViewportCartographics(viewer); - const cameraCartographic = geo.cartographic.fromCartesian( - viewer.camera.positionWC, - ); - const cameraHeightM = cameraCartographic.height; + const samples = sampleViewportCartographics(engine); + const cameraHeightM = cameraCartographic(engine).height; const viewScale = classifyViewScale(cameraHeightM); const cachedViewportPlaces = viewportPlacesFromCache( samples, @@ -4076,23 +4125,23 @@ function coarseBasemapPlace( }; } -function getViewTargetCartographic(viewer) { - const signature = cameraViewSignature(viewer); - const cached = viewTargetCache.get(viewer); +function getViewTargetCartographic(engine) { + const signature = cameraViewSignature(engine); + const cached = viewTargetCache.get(engine); if ( cached?.signature === signature && performance.now() - cached.cachedAt < 2500 ) { return cached.target; } - const position = getViewTargetCartesian(viewer); + const position = getViewTargetCartesian(engine); // `fromCartesian` still returns undefined for a point too near the ellipsoid // center to project; normalize that to the same "no target" null the callers // already handle for a missed pick. const target = position ? geo.cartographic.fromCartesian(position) || null : null; - viewTargetCache.set(viewer, { + viewTargetCache.set(engine, { signature, target, cachedAt: performance.now(), @@ -4100,15 +4149,16 @@ function getViewTargetCartographic(viewer) { return target; } -function cameraViewSignature(viewer) { - const camera = viewer.camera; - const cartographic = camera.positionCartographic; +function cameraViewSignature(engine) { + const cartographic = cameraCartographic(engine); + const height = engine.camera.height(); + const { heading, pitch } = engine.camera.pose(); return [ geo.math.toDegrees(cartographic.latitude).toFixed(5), geo.math.toDegrees(cartographic.longitude).toFixed(5), - Math.round(cartographic.height / 2), - camera.heading.toFixed(3), - camera.pitch.toFixed(3), + Math.round(height / 2), + heading.toFixed(3), + pitch.toFixed(3), ].join(':'); } @@ -4120,55 +4170,32 @@ function cameraViewSignature(viewer) { * MISSED pick, so it falls through to the next stage rather than poisoning * every caller downstream. */ -function getViewTargetCartesian(viewer) { - const scene = viewer.scene; - const canvas = scene.canvas; - const width = canvas.clientWidth || canvas.width || 0; - const height = canvas.clientHeight || canvas.height || 0; - const center = new Cesium.Cartesian2(width / 2, height / 2); - let position = null; - - if (scene.pickPositionSupported && typeof scene.pickPosition === 'function') { +function getViewTargetCartesian(engine) { + if (!engine) return null; + const { width, height } = engine.host.size(); + const center = geo.cartesian2.create(width / 2, height / 2); + const stages = [ + () => + engine.picker.pickPositionSupported + ? engine.picker.pickPosition(center) + : undefined, + () => engine.camera.pickEllipsoid(center), + () => engine.picker.pickGlobe(center), + ]; + for (const stage of stages) { + let position = null; try { - position = scene.pickPosition(center); + position = stage(); } catch { position = null; } + if (isPickedWorldPosition(position)) return position; } - - if ( - !isPickedWorldPosition(position) && - viewer.camera && - typeof viewer.camera.pickEllipsoid === 'function' - ) { - try { - position = viewer.camera.pickEllipsoid(center, Cesium.Ellipsoid.WGS84); - } catch { - position = null; - } - } - - if ( - !isPickedWorldPosition(position) && - viewer.camera && - typeof viewer.camera.getPickRay === 'function' - ) { - try { - const ray = viewer.camera.getPickRay(center); - position = scene.globe?.pick(ray, scene) || null; - } catch { - position = null; - } - } - - return isPickedWorldPosition(position) ? position : null; + return null; } -function sampleViewportCartographics(viewer) { - const scene = viewer.scene; - const canvas = scene.canvas; - const width = canvas.clientWidth || canvas.width || 0; - const height = canvas.clientHeight || canvas.height || 0; +function sampleViewportCartographics(engine) { + const { width, height } = engine.host.size(); if (!width || !height) return []; const points = [ @@ -4183,9 +4210,8 @@ function sampleViewportCartographics(viewer) { const samples = []; for (const [x, y] of points) { - const cartesian = viewer.camera.pickEllipsoid( + const cartesian = engine.camera.pickEllipsoid( geo.cartesian2.create(width * x, height * y), - Cesium.Ellipsoid.WGS84, ); if (!isPickedWorldPosition(cartesian)) continue; const carto = geo.cartographic.fromCartesian(cartesian); @@ -4664,7 +4690,7 @@ function layerIsWarming(enabledAt, layerKey) { } function analystProviders( - viewer, + engine, dataManager, { recordLimitByLayer = null, @@ -4751,7 +4777,7 @@ function analystProviders( }; }, getViewContext() { - const carto = viewer.camera.positionCartographic; + const carto = cameraCartographic(engine); const altKm = carto.height / 1000; // View radius scales with altitude: street-level asks stay local, // country-level asks sweep wide. Clamped so "in view" is never absurd. diff --git a/src/voice/gevActions.test.mjs b/src/voice/gevActions.test.mjs index f736c8c1b..2c15dd411 100644 --- a/src/voice/gevActions.test.mjs +++ b/src/voice/gevActions.test.mjs @@ -25,7 +25,8 @@ import { GEV_REALTIME_TOOLS } from '../../server/providers/openai/tools.js'; import { presentResult } from './resultDisplay.js'; import { attachVoiceResult } from './speech.js'; import { createCesiumEngine } from '../globe/cesium/createCesiumEngine.js'; -import { registerViewerEngine } from '../globe/cesium/engineFromViewer.js'; +import { engineFromViewer, registerViewerEngine } from '../globe/cesium/engineFromViewer.js'; +import { createFakeEngine } from '../globe/testing/fakeEngine.js'; test('every live basemap is reachable by its own id — no enum value without a voice alias', () => { // B1 regression: a stack added to MAP_STACKS (and the set_map_stack enum) @@ -125,6 +126,9 @@ function createVoiceNavigationHarness({ cockpitActive = false } = {}) { positionCartographic: Cesium.Cartographic.fromCartesian(position), heading: Cesium.Math.toRadians(28), pitch: Cesium.Math.toRadians(-45), + // The rest of the pose the engine camera reads (camera.pose()). + roll: 0, + frustum: { fovy: 1, aspectRatio: 1.5 }, cancelFlight() { order.push('cancel'); }, flyToBoundingSphere() { order.push(`fly:${viewer.trackedEntity === undefined ? 'released' : 'owned'}`); @@ -2582,6 +2586,9 @@ function createPrewarmHarness({ pickPosition, positionCartographic } = {}) { positionWC: position, heading: Cesium.Math.toRadians(28), pitch: Cesium.Math.toRadians(-45), + // The rest of the pose the engine camera reads (camera.pose()). + roll: 0, + frustum: { fovy: 1, aspectRatio: 1.5 }, pickEllipsoid() { calls.pickEllipsoid += 1; return surface; }, getPickRay() { calls.getPickRay += 1; return null; }, cancelFlight() {}, @@ -3236,7 +3243,27 @@ test('voice Radio: the keyless path applies no country filter the keyed path wou }); const testPlaceSearch = () => createStandalonePlaceSearch({ resolveApiKey: () => globalThis.window?.__GOOGLE_MAPS_API_KEY__ }); -function createGevActionRunner(options) { return createActionRunner({ placeSearch: testPlaceSearch(), ...options }); } +function createGevActionRunner(options) { + registerTestEngine(options.viewer); + return createActionRunner({ placeSearch: testPlaceSearch(), ...options }); +} +/** + * The runner drives the globe engine registered for its viewer, as the app's + * scene engine is (src/app/globeEngine.js): a test viewer gets the Cesium + * engine over it or, when its camera only states where it is + * (positionCartographic), a fake engine at that place. + */ +function registerTestEngine(viewer) { + if (!viewer || engineFromViewer(viewer)) return; + const camera = viewer.camera ?? {}; + const place = Object.getOwnPropertyDescriptor(camera, 'positionCartographic')?.value; + registerViewerEngine( + viewer, + !camera.positionWC && place + ? createFakeEngine({ longitude: place.longitude, latitude: place.latitude, altitude: place.height }) + : createCesiumEngine(viewer), + ); +} function controlRadio(viewer, manager, args, options) { return runControlRadio(viewer, manager, args, { placeSearch: testPlaceSearch(), ...options }); } test('ALPR common names toggle only the registered camera layer through the normal voice action', async () => { diff --git a/src/voice/gevActionsEngine.test.mjs b/src/voice/gevActionsEngine.test.mjs new file mode 100644 index 000000000..4df8308b4 --- /dev/null +++ b/src/voice/gevActionsEngine.test.mjs @@ -0,0 +1,422 @@ +// The W1 voice actions on the globe engine (phase 3, Task 9): the same +// results as on the Cesium viewer, through engine.camera / picker / +// projector only, and a structured refusal for the actions that still need +// a viewer when there is none (the Navara engine). + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import * as Cesium from 'cesium'; +import * as geo from '../geo/index.js'; +import { createFakeEngine } from '../globe/testing/fakeEngine.js'; +import { GEV_ACTION_SCHEMAS } from './actionSchemas.js'; +import { + UNAVAILABLE_ON_ENGINE, + VIEWER_ONLY_ACTIONS, + createGevActionRunner, +} from './gevActions.js'; +import { cameraPoseKey } from './pointerContext.js'; +import { createPointerPicker } from './pointerPick.js'; +import { renderFreshFrame } from './realtimeViewport.js'; + +globalThis.window ||= { + setTimeout, + clearTimeout, + requestIdleCallback: null, +}; + +const toRad = geo.math.toRadians; + +/** A fake engine over Austin at `altitude`, looking straight down. */ +function engineOverAustin(options = {}) { + return createFakeEngine({ + longitude: toRad(-97.74), + latitude: toRad(30.27), + altitude: 20_000, + ...options, + }); +} + +/** The UI navigation seam, without its policy: runs the move. */ +const navigatingStyleManager = (extra = {}) => ({ + runImmediateNavigation: (noun, navigate) => navigate(), + ...extra, +}); + +function dataManagerWith(modules, { enabled = Object.keys(modules) } = {}) { + const on = new Set(enabled); + const params = []; + return { + params, + layers: new Map( + Object.entries(modules).map(([id, module]) => [id, { id, module }]), + ), + isEnabled: (id) => on.has(id), + getAll: () => + Object.keys(modules).map((id) => ({ + id, + name: id, + enabled: on.has(id), + stats: modules[id]?.getStats?.() || {}, + })), + setLayerParams(id, values) { + params.push({ id, values }); + return true; + }, + async setEnabled(id, value) { + if (value) on.add(id); + else on.delete(id); + return true; + }, + }; +} + +const runnerOn = (engine, options = {}) => + createGevActionRunner({ + viewer: null, + engine, + styleManager: navigatingStyleManager(), + dataManager: dataManagerWith({}), + ...options, + }); + +const follow = (engine, position) => + engine.camera.follow({ + position: () => position, + viewFrom: { x: 0, y: -2000, z: 800 }, + }); + +test('the viewer-only actions are one exported list of real tools', () => { + assert.deepEqual(VIEWER_ONLY_ACTIONS, [ + 'control_scene', + 'annotate_map', + 'clear_annotations', + 'fly_route', + 'move_camera', + 'control_cockpit', + 'set_context_mode', + 'control_radio', + 'control_cctv', + ]); + assert.ok(Object.isFrozen(VIEWER_ONLY_ACTIONS)); + const tools = new Set(GEV_ACTION_SCHEMAS.map((schema) => schema.name)); + for (const name of VIEWER_ONLY_ACTIONS) + assert.ok(tools.has(name), `${name} is a voice tool`); +}); + +test('without a viewer the viewer-only actions refuse with a structured result', async () => { + const engine = engineOverAustin({ kind: 'navara' }); + const touched = []; + const styleManager = new Proxy(navigatingStyleManager(), { + get(target, key) { + if (typeof key === 'string') touched.push(key); + return target[key]; + }, + }); + const runner = runnerOn(engine, { styleManager }); + engine.fake.resetCalls(); + for (const name of VIEWER_ONLY_ACTIONS) { + const result = await runner(name, { action: 'status' }); + assert.equal(result.ok, false, name); + assert.equal(result.action, name); + assert.equal(result.unavailable, true, name); + assert.equal(result.code, UNAVAILABLE_ON_ENGINE, name); + assert.equal(result.engine, 'navara', name); + assert.equal(result.error, 'Unavailable on this engine', name); + } + assert.deepEqual(touched, [], 'a refusal reaches no controller'); + assert.deepEqual(engine.fake.calls, [], 'a refusal moves no camera'); +}); + +test('with a viewer the viewer-only actions still run', async () => { + const runner = createGevActionRunner({ + viewer: { + clock: { onTick: { addEventListener: () => () => {} } }, + scene: { canvas: { addEventListener() {}, removeEventListener() {} } }, + camera: { moveEnd: { addEventListener() {} } }, + }, + engine: engineOverAustin(), + styleManager: {}, + dataManager: dataManagerWith({}), + }); + const radio = await runner('control_radio', { action: 'status' }); + assert.equal(radio.error, 'Radio layer unavailable'); + assert.equal(radio.unavailable, undefined); +}); + +test('the runner warms the view target on the engine camera, once per engine', () => { + const engine = engineOverAustin({ kind: 'navara' }); + runnerOn(engine); + runnerOn(engine); + assert.equal(engine.camera.moveEnd.numberOfListeners, 1); +}); + +test('stop_tracking stops the engine follow', async () => { + const engine = engineOverAustin({ kind: 'navara' }); + const target = geo.cartesian3.fromDegrees(-97.7, 30.3, 9000); + follow(engine, target); + assert.ok(engine.camera.following()); + const released = []; + const runner = runnerOn(engine, { + dataManager: dataManagerWith({ + flights: { + getTrackedInfo: () => ({ id: 'abc' }), + stopTracking: () => released.push('flights'), + }, + }), + }); + const result = await runner('stop_tracking'); + assert.equal(result.ok, true); + assert.deepEqual(result.released, ['flights']); + assert.equal(engine.camera.following(), null); +}); + +test('zoom_to_globe leaves the follow and flies the engine camera', async () => { + const engine = engineOverAustin({ kind: 'navara' }); + follow(engine, geo.cartesian3.fromDegrees(-97.7, 30.3, 9000)); + const runner = runnerOn(engine); + const result = await runner('zoom_to_globe'); + assert.equal(result.ok, true); + assert.equal(result.heightKm > 1000, true); + assert.equal(engine.camera.following(), null); + assert.equal(engine.fake.callsOf('camera.flyTo').length, 1); +}); + +test('fly_to_location to coordinates leaves the follow and flies the engine', async () => { + const engine = engineOverAustin({ kind: 'navara' }); + follow(engine, geo.cartesian3.fromDegrees(-97.7, 30.3, 9000)); + const runner = runnerOn(engine); + const result = await runner('fly_to_location', { + latitude: 51.5, + longitude: -0.12, + }); + assert.equal(result.ok, true); + assert.equal(result.navigationMode, 'close-coordinate'); + assert.equal(engine.camera.following(), null); + assert.equal(engine.fake.callsOf('camera.flyToSphere').length, 1); + engine.fake.frames(400, 16); + const place = engine.camera.pose().cartographic; + assert.ok(Math.abs(geo.math.toDegrees(place.latitude) - 51.5) < 0.05); +}); + +test('track_entity flies to a vessel through the engine', async () => { + const engine = engineOverAustin({ kind: 'navara' }); + let selected = null; + const runner = runnerOn(engine, { + dataManager: dataManagerWith({ + 'ais-live-vessels': { + findByQuery: () => ({ + mmsi: 366999999, + name: 'EVER GIVEN', + latitude: 29.7, + longitude: -95.0, + }), + selectById: (mmsi) => (selected = mmsi), + }, + }), + }); + const result = await runner('track_entity', { + query: 'ever given', + layerId: 'ais-live-vessels', + }); + assert.equal(result.ok, true); + assert.equal(result.label, 'EVER GIVEN'); + assert.equal(selected, 366999999); + assert.equal(engine.fake.callsOf('camera.flyToSphere').length, 1); +}); + +test('frame_overhead frames the nearby contacts with flyToSphere as on Cesium', async () => { + const engine = engineOverAustin({ kind: 'navara' }); + const center = engine.camera.pickEllipsoid({ x: 512, y: 384 }); + const positions = [ + geo.cartesian3.fromDegrees(-97.7, 30.3, 9000), + geo.cartesian3.fromDegrees(-97.9, 30.1, 11000), + geo.cartesian3.fromDegrees(-97.5, 30.5, 10000), + ]; + let asked = null; + const runner = runnerOn(engine, { + dataManager: dataManagerWith({ + flights: { + getNearby(at, radiusM, limit) { + asked = { at, radiusM, limit }; + return positions.map((position, index) => ({ + id: `F${index}`, + icao24: `a${index}`, + label: `F${index}`, + position, + })); + }, + }, + }), + }); + const heading = engine.camera.pose().heading; + const result = await runner('frame_overhead', { target: 'flights' }); + assert.equal(result.ok, true); + assert.equal(result.count, 3); + assert.equal(asked.radiusM, 150_000); + assert.equal(asked.limit, 80); + assert.ok(geo.cartesian3.distance(asked.at, center) < 1e-6); + // The same sphere and offset the Cesium path flew. + const sphere = Cesium.BoundingSphere.fromPoints(positions); + const radius = Math.max(sphere.radius * 1.25, 8000); + const [call] = engine.fake.callsOf('camera.flyToSphere'); + const [flown, options] = call.args; + assert.ok(geo.cartesian3.distance(flown.center, sphere.center) < 1e-6); + assert.equal(flown.radius, radius); + assert.equal(options.duration, 2); + assert.equal(options.heading, heading); + assert.equal(options.pitch, toRad(-62)); + assert.equal(options.range, radius * 2.4); +}); + +test('next_iss_pass observes from the engine camera', async () => { + const engine = engineOverAustin({ kind: 'navara' }); + let observer = null; + const runner = runnerOn(engine, { + dataManager: dataManagerWith({ + satellites: { + getNextIssPass(options) { + observer = options; + return { status: 'none' }; + }, + }, + }), + }); + const result = await runner('next_iss_pass'); + assert.equal(result.ok, false); + assert.ok(Math.abs(observer.latDeg - 30.27) < 1e-6); + assert.ok(Math.abs(observer.lonDeg + 97.74) < 1e-6); +}); + +test('get_current_view_state reads the engine camera', async () => { + const engine = engineOverAustin({ kind: 'navara' }); + const runner = runnerOn(engine, { + styleManager: navigatingStyleManager({ activeStyle: 'noir' }), + }); + const result = await runner('get_current_view_state'); + assert.equal(result.ok, true); + assert.ok(Math.abs(result.camera.latitude - 30.27) < 1e-6); + assert.ok(Math.abs(result.camera.longitude + 97.74) < 1e-6); + assert.ok(Math.abs(result.camera.heightM - 20_000) < 1e-3); +}); + +test('adjust_camera_zoom moves the engine camera toward the view target', async () => { + const engine = engineOverAustin({ kind: 'navara' }); + const runner = runnerOn(engine); + const result = await runner('adjust_camera_zoom', { + direction: 'in', + amount: 'medium', + }); + assert.equal(result.ok, true); + assert.equal(result.movementRequestedM, 11_000); + assert.equal(result.movementActualM, 11_000); + assert.equal(result.afterHeightM, 9000); + assert.equal(engine.fake.callsOf('camera.cancelFlight').length, 1); +}); + +test('get_entity_context samples the view through the engine', async () => { + // Continental: no reverse geocoding, every viewport sample on the globe. + const engine = engineOverAustin({ kind: 'navara', altitude: 4_000_000 }); + const runner = runnerOn(engine); + const result = await runner('get_entity_context', { scope: 'in_view' }); + assert.equal(result.ok, true); + assert.equal(result.scene.camera.heightM, 4_000_000); + assert.equal(result.scene.basemap.viewScale, 'continental'); + assert.equal(result.scene.basemap.viewportSamples.length, 7); + // The view centre, on the ellipsoid under the camera. + const [centre] = result.scene.basemap.viewportSamples; + assert.ok(Math.abs(centre.latitude - 30.27) < 0.1); + assert.equal(centre.longitude, -97.74); +}); + +test('select_nearest_aircraft runs on the engine end to end', async () => { + const engine = engineOverAustin({ kind: 'navara' }); + let tracked = null; + const flights = { + getStats: () => ({ count: 2, source: 'fixture' }), + getAnalystRecords: () => [ + { + id: 'FAR1', + icao24: 'f1', + callsign: 'FAR1', + lat: 30.4, + lon: -97.7, + onGround: false, + }, + { + id: 'NEAR1', + icao24: 'n1', + callsign: 'NEAR1', + lat: 30.3, + lon: -97.7, + onGround: false, + }, + ], + findByQuery: (query) => + query === 'n1' + ? { icao24: 'n1', callsign: 'NEAR1', latitude: 30.3, longitude: -97.7 } + : null, + trackById: (id) => (tracked = id), + }; + const dataManager = dataManagerWith({ flights }, { enabled: [] }); + dataManager.refreshLayer = async () => true; + const runner = runnerOn(engine, { dataManager }); + const pending = runner('select_nearest_aircraft', { + latitude: 30.27, + longitude: -97.74, + }); + for (let index = 0; index < 50 && !tracked; index++) { + engine.fake.frames(20, 16); + await new Promise((resolve) => setImmediate(resolve)); + } + const result = await pending; + assert.equal(result.ok, true, result.error); + assert.equal(result.aircraft.id, 'n1'); + assert.equal(tracked, 'n1'); +}); + +test('the camera pose key reads the engine as it reads the viewer', () => { + const engine = engineOverAustin({ kind: 'navara' }); + const pose = engine.camera.pose(); + const viewer = { + camera: { + positionWC: pose.position, + heading: pose.heading, + pitch: pose.pitch, + roll: pose.roll, + }, + }; + assert.equal(typeof cameraPoseKey(engine), 'string'); + assert.equal(cameraPoseKey(engine), cameraPoseKey(viewer)); +}); + +test('the pointer picker picks through the engine', () => { + const engine = engineOverAustin({ kind: 'navara' }); + engine.fake.script('pick', { id: { __gevContextId: 'none' } }); + const pick = createPointerPicker({ engine, dataManager: null }); + const result = pick(512, 384); + assert.equal(result.entity, null); + assert.ok(Math.abs(result.ground.lat - 30.27) < 1e-4); + assert.ok(Math.abs(result.ground.lon + 97.74) < 1e-4); + assert.equal(result.radiusKm, 15); + assert.equal(result.cameraKey, cameraPoseKey(engine)); + const [screen] = engine.fake.callsOf('picker.pick')[0].args; + assert.deepEqual([screen.x, screen.y], [512, 384]); +}); + +test('a fresh frame for capture comes from the engine render loop', async () => { + const previous = globalThis.document; + try { + globalThis.document = { hidden: false }; + const engine = engineOverAustin({ kind: 'navara' }); + engine.fake.resetCalls(); + const fresh = renderFreshFrame(engine); + assert.equal(engine.fake.renderRequests, 1); + engine.fake.frame(); + assert.equal(await fresh, true); + globalThis.document = { hidden: true }; + assert.equal(await renderFreshFrame(engine), false); + assert.equal(engine.fake.renderRequests, 1, 'no render while hidden'); + } finally { + globalThis.document = previous; + } +}); diff --git a/src/voice/gevRealtime.js b/src/voice/gevRealtime.js index 58efacad3..c6cd242b2 100644 --- a/src/voice/gevRealtime.js +++ b/src/voice/gevRealtime.js @@ -5,6 +5,7 @@ import { createVoiceCommands } from './commands.js'; import { cameraPoseKey, createPointerTracker } from './pointerContext.js'; import { createPointerPicker } from './pointerPick.js'; import { createReferentRegistry } from './referents.js'; +import { engineFromViewer } from '../globe/cesium/engineFromViewer.js'; export * from './realtimeController.js'; const ACTION_NAMES = new Set(GEV_ACTION_SCHEMAS.map((schema) => schema.name)); @@ -30,10 +31,15 @@ export function withToolCatalog(runner, loadCatalog) { return runGevTool; } -/** Compose the standalone action runner with the voice controls. */ +/** + * Compose the standalone action runner with the voice controls, on the globe + * `engine` (by default the one registered for `options.viewer`). Without a + * viewer (the Navara engine) the runner refuses the viewer-only tools. + */ export function initGevVoiceCommands(options) { - const canvas = options.viewer?.scene?.canvas || null; - const cameraKey = () => cameraPoseKey(options.viewer); + const engine = options.engine ?? engineFromViewer(options.viewer); + const canvas = engine?.host.canvas || null; + const cameraKey = () => cameraPoseKey(engine); // Point-and-ask: the tracker and referents are shared by the runner (which // resolves 'pointer' and referent:n) and the controls (which snapshot the // pointer at turn start and clear both when the session ends). @@ -42,7 +48,7 @@ export function initGevVoiceCommands(options) { element: canvas, cameraKey, pick: createPointerPicker({ - viewer: options.viewer, + engine, dataManager: options.dataManager, }), }) @@ -55,7 +61,7 @@ export function initGevVoiceCommands(options) { pointer, referents, runner: withToolCatalog( - createGevActionRunner({ ...options, deixis }), + createGevActionRunner({ ...options, engine, deixis }), options.toolCatalog, ), }); diff --git a/src/voice/pointerContext.js b/src/voice/pointerContext.js index d54c0bd64..49b9fe7d4 100644 --- a/src/voice/pointerContext.js +++ b/src/voice/pointerContext.js @@ -19,6 +19,8 @@ * Provider-neutral and Cesium-free: the pick is injected. */ +import { isGlobeEngine } from '../globe/contract.js'; + /** Movement below this many CSS pixels is still the same dwell. */ export const POINTER_DWELL_PX = 12; /** Stillness that makes a dwell point. */ @@ -57,11 +59,16 @@ const finite = finiteOrNull; * Camera pose identity: position to 1 m, orientation to about 0.06 degrees. * A pixel read under one key names nothing under another. Reads plain * numbers only, so it needs no Cesium import. - * @param {object} viewer Cesium viewer. + * @param {object} globe The globe engine (its camera pose), or a Cesium + * viewer. * @returns {string|null} */ -export function cameraPoseKey(viewer) { - const camera = viewer?.camera; +export function cameraPoseKey(globe) { + let camera = globe?.camera; + if (isGlobeEngine(globe)) { + const pose = camera.pose(); + camera = { ...pose, positionWC: pose.position }; + } const p = camera?.positionWC; if (!p || !Number.isFinite(p.x)) return null; const angle = (value) => Math.round((Number(value) || 0) * 1000); diff --git a/src/voice/pointerPick.js b/src/voice/pointerPick.js index fc6514f46..d7272e911 100644 --- a/src/voice/pointerPick.js +++ b/src/voice/pointerPick.js @@ -11,37 +11,35 @@ export function pointerRadiusKm(cameraHeightM) { } /** - * The Cesium side of point-and-ask: one entity pick and one depth-aware world - * pick at canvas CSS pixels. Called lazily by the pointer tracker (dwell and - * snapshot only), never per mouse move. + * The globe side of point-and-ask: one object pick (engine.picker.pick) and + * one depth-aware world pick at canvas CSS pixels, on either engine. Called + * lazily by the pointer tracker (dwell and snapshot only), never per mouse + * move. * @param {object} options - * @param {object} options.viewer Cesium viewer. + * @param {object} options.engine The globe engine. * @param {object} [options.dataManager] Layer lifecycle. * @param {Function} [options.pickWorldFromScreen] Normalized-pixel world pick. * @returns {(x: number, y: number) => {entity: object|null, ground: object|null, radiusKm: number}|null} */ export function createPointerPicker({ - viewer, + engine, dataManager = null, pickWorldFromScreen = defaultAnnotationResolver.pickWorldFromScreen, }) { return function pickPointer(x, y) { - const scene = viewer?.scene; - const canvas = scene?.canvas; - if (!scene || !canvas) return null; - const width = canvas.clientWidth || canvas.width || 0; - const height = canvas.clientHeight || canvas.height || 0; + if (!engine || engine.isDestroyed?.()) return null; + const { width, height } = engine.host.size(); if (!width || !height) return null; let picked = null; try { - picked = scene.pick(geo.cartesian2.create(x, y)); + picked = engine.picker.pick(geo.cartesian2.create(x, y)) ?? null; } catch { picked = null; } const entity = resolvePickedContext(picked, { dataManager }); let ground = null; try { - const hit = pickWorldFromScreen(viewer, x / width, y / height); + const hit = pickWorldFromScreen(engine, x / width, y / height); if (hit && Number.isFinite(hit.lat) && Number.isFinite(hit.lon)) ground = { lat: hit.lat, @@ -54,8 +52,8 @@ export function createPointerPicker({ return { entity, ground, - radiusKm: pointerRadiusKm(viewer.camera?.positionCartographic?.height), - cameraKey: cameraPoseKey(viewer), + radiusKm: pointerRadiusKm(engine.camera.height()), + cameraKey: cameraPoseKey(engine), }; }; } diff --git a/src/voice/realtimeViewport.js b/src/voice/realtimeViewport.js index 755913906..89bc0a834 100644 --- a/src/voice/realtimeViewport.js +++ b/src/voice/realtimeViewport.js @@ -3,10 +3,22 @@ import { hasStructuredViewIdentity, } from './realtimeProtocol.js'; import { cameraPoseKey } from './pointerContext.js'; +import { isGlobeEngine } from '../globe/contract.js'; -/** The live viewer's camera pose identity (Cesium-free read). */ +/** The page's globe engine (both engines), else its Cesium viewer. */ +function pageGlobe() { + const debug = globalThis.window?.__godsEyeView; + return debug?.engine ?? debug?.viewer ?? null; +} + +/** The map canvas: the engine's host canvas, else the viewer's. */ +function pageCanvas(globe) { + return isGlobeEngine(globe) ? globe.host.canvas : globe?.scene?.canvas; +} + +/** The live camera pose identity (Cesium-free read). */ export function readViewerCameraKey() { - return cameraPoseKey(globalThis.window?.__godsEyeView?.viewer); + return cameraPoseKey(pageGlobe()); } // Viewport-screenshot size guards (M13). The old code clamped WIDTH only, so a // tall portrait window produced an oversized capture whose dc.send could throw. @@ -18,16 +30,18 @@ export const VIEWPORT_MAX_PIXELS = 1200 * 900; export const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024; export async function captureViewportImage() { - const viewer = window.__godsEyeView?.viewer; + // Both engines draw with preserveDrawingBuffer (Cesium's contextOptions; + // Navara's own WebGL2 canvas, S4), so the canvas holds the last frame. + const globe = pageGlobe(); const source = - viewer?.scene?.canvas || + pageCanvas(globe) || document.querySelector('#cesiumContainer .cesium-widget canvas'); if (!source || !source.width || !source.height) return null; // No fresh frame (hidden, or the bounded render wait timed out) → no // capture. The caller labels this image "Current"; a stale preserved // frame would feed the model old entities as current context. (perf // wave 2 fix) - const fresh = await renderFreshCesiumFrame(viewer); + const fresh = await renderFreshFrame(globe); if (!fresh) return null; // Clamp BOTH dimensions by a total-pixel budget so tall portrait windows are @@ -129,10 +143,10 @@ export function pointerCropRect({ * the crop's normalized bounds in the full canvas, or null. */ export async function capturePointerCrop(screenPx) { - const viewer = window.__godsEyeView?.viewer; - const source = viewer?.scene?.canvas; + const globe = pageGlobe(); + const source = pageCanvas(globe); if (!source?.width || !source?.height || !screenPx) return null; - const fresh = await renderFreshPointerFrame(viewer); + const fresh = await renderFreshPointerFrame(globe); if (!fresh) return null; const rect = pointerCropRect({ x: screenPx.x, @@ -186,10 +200,10 @@ export async function capturePointerCrop(screenPx) { * Pointer crops get one bounded recovery request for a slow render loop. * Full-viewport captures intentionally keep their single 400 ms attempt. */ -export async function renderFreshPointerFrame(viewer) { - if (await renderFreshCesiumFrame(viewer)) return true; +export async function renderFreshPointerFrame(globe) { + if (await renderFreshFrame(globe)) return true; if (typeof document !== 'undefined' && document.hidden) return false; - return renderFreshCesiumFrame(viewer); + return renderFreshFrame(globe); } // Scale (w, h) down so w*h <= maxPixels while preserving aspect ratio. Never @@ -228,8 +242,14 @@ export function estimateDataUrlBytes(dataUrl) { * when the bounded wait timed out. Callers must not label a non-fresh * canvas as current. (perf wave 2) */ -export async function renderFreshCesiumFrame(viewer) { - const scene = viewer?.scene; +export async function renderFreshFrame(globe) { + // The engine's frame events and render request, else the viewer's scene. + const scene = isGlobeEngine(globe) + ? { + postRender: globe.frames.postRender, + requestRender: () => globe.render.request('voice-capture'), + } + : globe?.scene; if (!scene) return false; // While the document is hidden the render loop is suspended — don't // secretly restart rendering for an optional screenshot, and don't pass @@ -488,3 +508,6 @@ export class RealtimeViewport { return true; } } + +/** renderFreshFrame for callers that hold a Cesium viewer. */ +export const renderFreshCesiumFrame = renderFreshFrame;