From 05578c61b94b4af5934c0910b7a15156dd727ed5 Mon Sep 17 00:00:00 2001 From: German Khokhlov Date: Tue, 3 Nov 2026 17:20:40 +0400 Subject: [PATCH] Keep an active voice session through engine switches The page creates the voice once; each engine attachment only gives it that scene's action runner, pointer canvas and annotation events. Calls made during a switch wait for the incoming engine and are refused with ENGINE_SWITCHING when none attaches in time; results name the engine that answered, and the session hears engine-changed after a switch. The analyst's follow-up memory and layer enable times outlive one engine's runner. scripts/qa-voice-engine-switch.mjs drives a fake microphone and Realtime transport through the switches. --- scripts/qa-voice-engine-switch.mjs | 247 +++++++++++++++++++++++++++++ src/app/navaraMode.js | 153 +++++++++++------- src/app/sessionVoice.js | 47 ++++++ src/app/tools.js | 39 +++-- src/data/analystEngine.js | 38 +++-- src/main.js | 7 +- src/voice/gevActions.js | 16 +- src/voice/gevRealtime.js | 178 ++++++++++++++++----- src/voice/pointerContext.js | 23 ++- src/voice/sceneRunner.js | 158 ++++++++++++++++++ src/voice/sceneRunner.test.mjs | 107 +++++++++++++ src/voice/sceneVoice.test.mjs | 117 ++++++++++++++ src/voice/sessionCommands.js | 7 +- 13 files changed, 998 insertions(+), 139 deletions(-) create mode 100644 scripts/qa-voice-engine-switch.mjs create mode 100644 src/app/sessionVoice.js create mode 100644 src/voice/sceneRunner.js create mode 100644 src/voice/sceneRunner.test.mjs create mode 100644 src/voice/sceneVoice.test.mjs diff --git a/scripts/qa-voice-engine-switch.mjs b/scripts/qa-voice-engine-switch.mjs new file mode 100644 index 000000000..741d35361 --- /dev/null +++ b/scripts/qa-voice-engine-switch.mjs @@ -0,0 +1,247 @@ +// An active voice session survives engine switches. With a fake microphone +// and a fake Realtime transport (no model is reached: /api/openai, /api/realtime +// and api.openai.com are blocked or answered in the page), this gate starts +// voice on Cesium, switches Cesium→Navara→Cesium→Navara and checks that: +// - the microphone was requested once, one peer connection was made, and +// neither it nor its data channel closed; no WebSocket opened; +// - every session object kept its identity and each switch attached once; +// - a set_layer_visibility call after each switch ran on the engine shown; +// - a call made during a switch waited and ran on the incoming engine, and +// one whose wait ran out was refused with ENGINE_SWITCHING; +// - annotate_map works on Cesium, is refused on Navara, and the mark drawn +// before the switches is still on the board back on Cesium; +// - the session heard which engine it drives after each switch. +// +// node scripts/qa-voice-engine-switch.mjs http://127.0.0.1:/ +import assert from 'node:assert/strict'; +import { + idDifference, + launch, + openSessionPage, + previewBase, + readSession, + readVoiceEvidence, + requestEngine, + sessionAddress, + switchEngine, + waitReady, +} from './qa/appSessionHarness.mjs'; + +const base = previewBase(process.argv[2]); + +// Play one server function call on the fake data channel and wait for the +// client's function_call_output. +function callTool(page, name, args, { beforeCall = null } = {}) { + return page.evaluate( + async (name, args, beforeCall) => { + const evidence = window.__qaVoice; + const callId = `qa-call-${(window.__qaCalls = (window.__qaCalls ?? 0) + 1)}`; + const responseId = `qa-response-${callId}`; + const deliver = (event) => + evidence.channel.dispatchEvent( + new MessageEvent('message', { data: JSON.stringify(event) }), + ); + if (beforeCall === 'switch-navara' || beforeCall === 'switch-cesium') { + const select = document.querySelector('#globe-engine'); + select.value = beforeCall.slice('switch-'.length); + select.dispatchEvent(new Event('change', { bubbles: true })); + } + deliver({ type: 'response.created', response: { id: responseId } }); + deliver({ + type: 'response.function_call_arguments.done', + response_id: responseId, + item_id: `item-${callId}`, + call_id: callId, + name, + arguments: JSON.stringify(args), + }); + const started = performance.now(); + for (;;) { + const sent = evidence.sent.find( + (event) => + event?.type === 'conversation.item.create' && + event.item?.type === 'function_call_output' && + event.item.call_id === callId, + ); + if (sent) { + deliver({ + type: 'response.done', + response: { id: responseId, status: 'completed', output: [] }, + }); + return JSON.parse(sent.item.output); + } + if (performance.now() - started > 60_000) + throw new Error(`No output for ${name}`); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + }, + name, + args, + beforeCall, + ); +} + +const engineChanges = (page) => + page.evaluate(() => + window.__qaVoice.sent + .filter( + (event) => + event?.type === 'conversation.item.create' && + event.item?.role === 'system', + ) + .map((event) => { + try { + return JSON.parse(event.item.content[0].text); + } catch { + return null; + } + }) + .filter((event) => event?.type === 'engine-changed') + .map((event) => event.engine), + ); + +const ANNOTATION = { + annotations: [ + { + type: 'pin', + latitude: 42.3554, + longitude: -71.0605, + label: 'QA voice mark', + }, + ], +}; + +const browser = await launch(); +const results = []; +try { + const { page, errors } = await openSessionPage( + browser, + sessionAddress(base, { engine: 'cesium' }), + ); + await waitReady(page, 'cesium'); + await page.evaluate(async () => { + const close = document.querySelector('[data-key-setup-close]'); + if (close?.offsetParent) close.click(); + await window.__godsEyeView.voiceCommands.start({ pushToTalk: false }); + }); + await page.waitForFunction( + () => + window.__qaVoice.channel?.readyState === 'open' && + window.__godsEyeView.voiceCommands.state === 'listening', + { timeout: 30_000 }, + ); + const started = await readVoiceEvidence(page); + const boot = await readSession(page); + assert.equal(started.getUserMedia, 1, 'voice asks for the microphone once'); + assert.equal(started.peerConnections, 1); + + const annotated = await callTool(page, 'annotate_map', ANNOTATION); + assert.equal(annotated.ok, true, JSON.stringify(annotated)); + assert.equal(annotated.engine, 'cesium'); + const marks = await page.evaluate(() => window.__gevAnnotations.count()); + assert.ok(marks >= 1, 'the mark is on the board'); + results.push('PASS annotate_map drew a mark on Cesium'); + + const sequence = ['navara', 'cesium', 'navara']; + for (const kind of sequence) { + await switchEngine(page, kind); + const layer = await callTool(page, 'set_layer_visibility', { + layerId: 'earthquakes', + enabled: kind === 'navara', + }); + assert.equal(layer.engine, kind, JSON.stringify(layer)); + assert.equal( + await page.evaluate(() => window.__godsEyeView.engine.kind), + kind, + ); + if (kind === 'navara') { + const refused = await callTool(page, 'annotate_map', ANNOTATION); + assert.equal(refused.ok, false, JSON.stringify(refused)); + assert.equal(refused.engine, 'navara'); + } else { + const count = await page.evaluate(() => window.__gevAnnotations.count()); + assert.equal(count, marks, 'the mark survived the round trip'); + } + results.push(`PASS ${kind}: set_layer_visibility ran on ${layer.engine}`); + } + + // A call made during a switch waits for the incoming engine. + const midSwitch = await callTool( + page, + 'set_layer_visibility', + { layerId: 'earthquakes', enabled: true }, + { beforeCall: 'switch-cesium' }, + ); + await waitReady(page, 'cesium'); + assert.equal(midSwitch.ok, true, JSON.stringify(midSwitch)); + assert.equal(midSwitch.engine, 'cesium', JSON.stringify(midSwitch)); + results.push('PASS a call made during the switch ran on the incoming engine'); + assert.equal( + await page.evaluate(() => window.__gevAnnotations.count()), + marks, + 'the mark is still on the board after Cesium→Navara→Cesium→Navara→Cesium', + ); + + // A call whose wait runs out is refused with a clear code. + await page.evaluate(() => { + window.__godsEyeView.voiceCommands.runner.switchWaitMs = 1; + }); + const refused = await callTool( + page, + 'set_layer_visibility', + { layerId: 'earthquakes', enabled: false }, + { beforeCall: 'switch-navara' }, + ); + await waitReady(page, 'navara'); + await page.evaluate(() => { + window.__godsEyeView.voiceCommands.runner.switchWaitMs = 15_000; + }); + assert.equal(refused.ok, false, JSON.stringify(refused)); + assert.equal(refused.code, 'ENGINE_SWITCHING', JSON.stringify(refused)); + results.push( + 'PASS a call whose wait ran out was refused with ENGINE_SWITCHING', + ); + + await switchEngine(page, 'cesium'); + const after = await readVoiceEvidence(page); + const state = await readSession(page); + const changes = await engineChanges(page); + const switches = sequence.length + 3; + const report = { + started, + after, + ids: idDifference(boot.ids, state.ids), + counters: state.counters, + changes, + voiceState: await page.evaluate( + () => window.__godsEyeView.voiceCommands.state, + ), + }; + console.log(JSON.stringify(report, null, 2)); + assert.equal(after.getUserMedia, 1, 'no new microphone request'); + assert.equal(after.peerConnections, 1, 'no new peer connection'); + assert.equal(after.peerConnectionsClosed, 0, 'the connection stayed open'); + assert.equal(after.channelsClosed, 0, 'the data channel stayed open'); + assert.equal(after.channelState, 'open'); + assert.equal(after.webSockets, started.webSockets, 'no new WebSocket'); + assert.equal(after.tokenRequests, 1, 'no new token'); + assert.equal(report.voiceState, 'listening'); + assert.deepEqual(report.ids, {}, 'session objects kept their identity'); + assert.equal( + state.counters.attachments, + boot.counters.attachments + switches, + ); + assert.equal(changes.length, switches, JSON.stringify(changes)); + assert.equal(changes.at(-1), 'cesium'); + assert.deepEqual(errors, []); + results.push('PASS the voice session kept its microphone and transport'); + + await page.evaluate(() => window.__godsEyeView.voiceCommands.stop()); + await page.waitForFunction( + () => window.__qaVoice.peerConnectionsClosed === 1, + ); + results.push('PASS stopping voice closes the transport'); + console.log(results.join('\n')); +} finally { + await browser.close(); +} diff --git a/src/app/navaraMode.js b/src/app/navaraMode.js index b88aec6a1..6e0a406bb 100644 --- a/src/app/navaraMode.js +++ b/src/app/navaraMode.js @@ -17,7 +17,7 @@ import { installRenderGovernor, uninstallRenderGovernor, } from '../renderGovernor.js'; -import { initGevVoiceCommands } from '../voice/gevRealtime.js'; +import { attachSessionVoice } from './sessionVoice.js'; import { initNavaraImageryBoxTool } from '../ui/imageryBoxNavara.js'; import { createRecentImageryPanel } from '../ui/recentImagery.js'; @@ -38,7 +38,9 @@ export function navaraModeNotice(availability) { const names = availability.available().map(({ name }) => name); return Object.freeze({ label: t('Experimental Navara engine'), - detail: names.length ? `Data layers: ${names.join(', ')}` : 'No registered data layers', + detail: names.length + ? `Data layers: ${names.join(', ')}` + : 'No registered data layers', }); } @@ -112,56 +114,97 @@ export async function registerNavaraLayerRenderers() { bikeshare: async () => (await import('../layers/bikeshare/render/navara.js')) .createBikeshareNavaraRenderer, - ...Object.fromEntries(['weather-radar', 'weather-satellite', 'weather-lightning'].map((id) => [ - id, async () => { - const { createWeatherNavaraRenderer } = await import('../layers/weather/render/navara.js'); - return (engine, options) => createWeatherNavaraRenderer(engine, navaraViews.get(engine), options); - }, - ])), + ...Object.fromEntries( + ['weather-radar', 'weather-satellite', 'weather-lightning'].map((id) => [ + id, + async () => { + const { createWeatherNavaraRenderer } = + await import('../layers/weather/render/navara.js'); + return (engine, options) => + createWeatherNavaraRenderer( + engine, + navaraViews.get(engine), + options, + ); + }, + ]), + ), 'bhote-koshi-2026': async () => { - const { createNepalEventDisplay } = await import('../layers/bhoteKoshi/render/navara.js'); - return (engine) => createNepalEventDisplay(engine, navaraViews.get(engine)); + const { createNepalEventDisplay } = + await import('../layers/bhoteKoshi/render/navara.js'); + return (engine) => + createNepalEventDisplay(engine, navaraViews.get(engine)); }, 'bhote-koshi-locator': async () => { - const { drawStorySource } = await import('../layers/shared/render/navaraStory.js'); - return (engine, { data }) => drawStorySource(engine, navaraViews.get(engine), data); + const { drawStorySource } = + await import('../layers/shared/render/navaraStory.js'); + return (engine, { data }) => + drawStorySource(engine, navaraViews.get(engine), data); }, 'street-level': async () => { - const { createStreetMap } = await import('../layers/streetLevel/render/navara.js'); - return engine => createStreetMap(engine, navaraViews.get(engine)); + const { createStreetMap } = + await import('../layers/streetLevel/render/navara.js'); + return (engine) => createStreetMap(engine, navaraViews.get(engine)); }, traffic: async () => { - const { createTrafficNavaraRenderer } = await import('../layers/traffic/render/navara.js'); - return engine => createTrafficNavaraRenderer(engine, navaraViews.get(engine)); + const { createTrafficNavaraRenderer } = + await import('../layers/traffic/render/navara.js'); + return (engine) => + createTrafficNavaraRenderer(engine, navaraViews.get(engine)); }, directions: async () => { - const { createDirectionsNavaraRenderer } = await import('../layers/directions/render/navara.js'); - return (engine) => createDirectionsNavaraRenderer(engine, navaraViews.get(engine)); + const { createDirectionsNavaraRenderer } = + await import('../layers/directions/render/navara.js'); + return (engine) => + createDirectionsNavaraRenderer(engine, navaraViews.get(engine)); }, 'telegeography-submarine-cables': async () => { - const { createNavaraCables } = await import('../layers/submarineCables/render/navara.js'); - return (engine, { overlayHost }) => createNavaraCables(engine, navaraViews.get(engine), overlayHost); + const { createNavaraCables } = + await import('../layers/submarineCables/render/navara.js'); + return (engine, { overlayHost }) => + createNavaraCables(engine, navaraViews.get(engine), overlayHost); }, wind: async () => { - const { createWindNavaraRenderer } = await import('../layers/wind/render/navara.js'); - return (engine) => createWindNavaraRenderer(engine, navaraViews.get(engine)); + const { createWindNavaraRenderer } = + await import('../layers/wind/render/navara.js'); + return (engine) => + createWindNavaraRenderer(engine, navaraViews.get(engine)); }, 'weather-cyclones': async () => { - const {createCyclonesNavaraRenderer} = await import('../layers/cyclones/render/navara.js'); - return (engine,options) => createCyclonesNavaraRenderer(engine,navaraViews.get(engine).view,options); + const { createCyclonesNavaraRenderer } = + await import('../layers/cyclones/render/navara.js'); + return (engine, options) => + createCyclonesNavaraRenderer( + engine, + navaraViews.get(engine).view, + options, + ); }, - ...Object.fromEntries(['local-datacenters','local-dams'].map((id) => [ - id, async () => { - const {createInfrastructureNavaraRenderer} = await import('../layers/infrastructure/render/navara.js'); - return (engine,{config,services}) => createInfrastructureNavaraRenderer(engine,navaraViews.get(engine),config,services); - }, - ])), + ...Object.fromEntries( + ['local-datacenters', 'local-dams'].map((id) => [ + id, + async () => { + const { createInfrastructureNavaraRenderer } = + await import('../layers/infrastructure/render/navara.js'); + return (engine, { config, services }) => + createInfrastructureNavaraRenderer( + engine, + navaraViews.get(engine), + config, + services, + ); + }, + ]), + ), 's2-ghostmaps': async () => { - const { createGhostMapsNavaraRenderer } = await import('../layers/ghostMaps/render/navara.js'); - return (engine) => createGhostMapsNavaraRenderer(engine, navaraViews.get(engine).view); + const { createGhostMapsNavaraRenderer } = + await import('../layers/ghostMaps/render/navara.js'); + return (engine) => + createGhostMapsNavaraRenderer(engine, navaraViews.get(engine).view); }, 'military-awareness': async () => - (await import('../layers/awareness/render/navara.js')).createAwarenessNavaraRenderer, + (await import('../layers/awareness/render/navara.js')) + .createAwarenessNavaraRenderer, 'military-installations': async () => { const { createInstallationsNavaraRenderer } = await import('../layers/installations/render/navara.js'); @@ -181,17 +224,22 @@ export async function registerNavaraLayerRenderers() { }); }, 'fire-perimeters': async () => { - const { createPerimetersNavaraRenderer } = await import('../layers/perimeters/render/navara.js'); - return (engine) => createPerimetersNavaraRenderer(engine, navaraViews.get(engine)); + const { createPerimetersNavaraRenderer } = + await import('../layers/perimeters/render/navara.js'); + return (engine) => + createPerimetersNavaraRenderer(engine, navaraViews.get(engine)); }, 'gnss-integrity': async () => { - const { createGnssNavaraRenderer } = await import('../layers/gnss/render/navara.js'); - return (engine) => createGnssNavaraRenderer(engine, navaraViews.get(engine)); + const { createGnssNavaraRenderer } = + await import('../layers/gnss/render/navara.js'); + return (engine) => + createGnssNavaraRenderer(engine, navaraViews.get(engine)); }, 'air-quality': async () => { const { createAirQualityNavaraRenderer } = await import('../layers/airQuality/render/navara.js'); - return (engine) => createAirQualityNavaraRenderer(engine, navaraViews.get(engine)); + return (engine) => + createAirQualityNavaraRenderer(engine, navaraViews.get(engine)); }, earthquakes: async () => { const { createEarthquakesNavaraRenderer } = @@ -355,6 +403,7 @@ export function createViewerlessTools({ voice = {}, startChrome, sceneDirector, + layerSession = null, signal, defer, }) { @@ -442,23 +491,21 @@ export function createViewerlessTools({ }); // Voice on the engine (phase 3, Task 9): the W1 tools drive the engine; // the Director uses the shared session; viewer-only tools declare availability. - const voiceCommands = initGevVoiceCommands({ - ...voice, - annotationResolver: operations.annotationResolver, - searchNavigation: operations.searchAndFlyTo, - signal, + const voiceCommands = attachSessionVoice({ + layerSession, + voice, + operations, placeSearch, - viewer: null, - engine, - styleManager, dataManager, - sceneDirector, - annotations: null, - }); - defer(() => { - voiceCommands.stop({ removeUi: true }); - if (window.__gevVoiceCommands === voiceCommands) - delete window.__gevVoiceCommands; + scene: { + viewer: null, + engine, + styleManager, + sceneDirector, + annotations: null, + }, + signal, + defer, }); debug.voiceCommands = voiceCommands; return { sceneDirector, annotations: null, voiceCommands }; diff --git a/src/app/sessionVoice.js b/src/app/sessionVoice.js new file mode 100644 index 000000000..bc94ff246 --- /dev/null +++ b/src/app/sessionVoice.js @@ -0,0 +1,47 @@ +import { createSceneVoice } from '../voice/gevRealtime.js'; + +/** + * Attach the page's voice to this engine's scene. With a layer session the + * voice is created once, for the page lifetime, and every engine attachment + * only gives it the scene (runner, pointer canvas, annotation events): an + * active voice session keeps its microphone and transport through a switch. + * Without one (a host that never switches) the voice lives with this + * application. Returns the voice commands. + */ +export function attachSessionVoice({ + layerSession = null, + voice = {}, + operations, + placeSearch, + dataManager, + scene, + signal, + defer, +}) { + const options = { + ...voice, + floorServices: operations.surface?.groundFloor, + annotationResolver: operations.annotationResolver, + searchNavigation: operations.searchAndFlyTo, + placeSearch, + dataManager, + }; + let sceneVoice; + if (layerSession) { + sceneVoice = layerSession.voice ??= createSceneVoice({ + ...options, + signal: layerSession.lifetime.signal, + isSwitching: () => layerSession.switching?.() === true, + }); + } else { + sceneVoice = createSceneVoice({ ...options, signal }); + defer(() => { + sceneVoice.commands.stop({ removeUi: true }); + if (window.__gevVoiceCommands === sceneVoice.commands) + delete window.__gevVoiceCommands; + }); + } + sceneVoice.attachScene({ ...scene, dataManager }); + defer(() => sceneVoice.detachScene()); + return sceneVoice.commands; +} diff --git a/src/app/tools.js b/src/app/tools.js index 679b7e05c..e9d687e81 100644 --- a/src/app/tools.js +++ b/src/app/tools.js @@ -6,7 +6,7 @@ import { useDisplayStore } from '../ui/stores/display.js'; import { applicationPinia } from '../ui/vueRuntime.js'; import { initImageryBoxTool } from '../ui/imageryBoxTool.js'; import { createRecentImageryPanel } from '../ui/recentImagery.js'; -import { initGevVoiceCommands } from '../voice/gevRealtime.js'; +import { attachSessionVoice } from './sessionVoice.js'; import { installViews, isEmbeddedInline } from './embed.js'; import { installScopeMask, destroyScopeMask } from '../scopeMask.js'; import { installLayerReleaseTrim } from '../sceneCommandRelease.js'; @@ -42,10 +42,15 @@ export function createApplicationTools({ const { viewer, engine, tileset, mapStackController, operations } = scene; const { styleManager, weatherEffects, cockpitCloudEffects } = controls; const { dataManager } = data; - const isMapStackAvailable = id => mapStackController?.isStackAvailable(id) === true; - const sceneDirector = layerSession?.director ?? new SceneDirector(viewer, styleManager, dataManager, { - engine, dataPacks: {...sceneDataPacks,native:scene.packRenderer}, isMapStackAvailable, - }); + const isMapStackAvailable = (id) => + mapStackController?.isStackAvailable(id) === true; + const sceneDirector = + layerSession?.director ?? + new SceneDirector(viewer, styleManager, dataManager, { + engine, + dataPacks: { ...sceneDataPacks, native: scene.packRenderer }, + isMapStackAvailable, + }); if (layerSession?.director) sceneDirector.attachScene( viewer, @@ -79,6 +84,7 @@ export function createApplicationTools({ voice, sceneDirector, startChrome, + layerSession, signal, defer, }), @@ -239,24 +245,15 @@ export function createApplicationTools({ defer(() => { if (window.__godsEyeView === debug) delete window.__godsEyeView; }); - const voiceCommands = initGevVoiceCommands({ - ...voice, - floorServices: operations.surface.groundFloor, - annotationResolver: operations.annotationResolver, - searchNavigation: operations.searchAndFlyTo, - signal, + const voiceCommands = attachSessionVoice({ + layerSession, + voice, + operations, placeSearch, - viewer, - engine, - styleManager, dataManager, - sceneDirector, - annotations, - }); - defer(() => { - voiceCommands.stop({ removeUi: true }); - if (window.__gevVoiceCommands === voiceCommands) - delete window.__gevVoiceCommands; + scene: { viewer, engine, styleManager, sceneDirector, annotations }, + signal, + defer, }); debug.voiceCommands = voiceCommands; defer( diff --git a/src/data/analystEngine.js b/src/data/analystEngine.js index 5175fec48..734b29f91 100644 --- a/src/data/analystEngine.js +++ b/src/data/analystEngine.js @@ -644,8 +644,11 @@ export async function selectAnalystRows( * Create an engine bound to live providers. All spatial/text/number logic is * in the pure helpers above; this closure only sequences and remembers. */ -export function createAnalystEngine(providers) { - let lastResult = null; +export function createAnalystEngine(providers, { memory = null } = {}) { + // The follow-up memory. A caller may own it (`memory`, an object whose + // `lastResult` this engine reads and writes) so a replacement engine over + // another globe keeps the last answer; otherwise it is this engine's. + memory ??= { lastResult: null }; /** * Run one query. @@ -666,7 +669,7 @@ export function createAnalystEngine(providers) { ) { const cancelled = () => refusal('CANCELLED', 'This query was superseded.', { cancelled: true }); - if (spec.followUp && !lastResult) + if (spec.followUp && !memory.lastResult) return refusal( 'NO_RESULT_CONTEXT', 'There is no previous answer to follow up on. Ask the full question.', @@ -677,7 +680,7 @@ export function createAnalystEngine(providers) { // must be that set: "those hospitals" after an aircraft answer is a new // question, never a filter over the old aircraft. if (followUp && Array.isArray(spec.layers) && spec.layers.length) { - const remembered = new Set(lastResult.layerKeys); + const remembered = new Set(memory.lastResult.layerKeys); const named = new Set(spec.layers); const same = remembered.size === named.size && @@ -685,16 +688,16 @@ export function createAnalystEngine(providers) { if (!same) return refusal( 'FOLLOW_UP_MISMATCH', - `The last answer was about ${lastResult.layerKeys.join(', ')}, not ${spec.layers.join(', ')}. Ask again without followUp.`, + `The last answer was about ${memory.lastResult.layerKeys.join(', ')}, not ${spec.layers.join(', ')}. Ask again without followUp.`, { layers: [...spec.layers], - remembered: [...lastResult.layerKeys], + remembered: [...memory.lastResult.layerKeys], coverage: { layersQueried: [], scope: 'none' }, }, ); } const layers = followUp - ? lastResult.layerKeys + ? memory.lastResult.layerKeys : Array.isArray(spec.layers) && spec.layers.length ? spec.layers : ['flights']; @@ -729,10 +732,11 @@ export function createAnalystEngine(providers) { let queriedSnapshots; let unanswered = []; if (followUp) { - groups = lastResult.matched; - layersQueried = lastResult.coverage.layersQueried; - queriedSnapshots = lastResult.coverage.feedProvenance?.layers || []; - unanswered = [...(lastResult.unanswered || [])]; + groups = memory.lastResult.matched; + layersQueried = memory.lastResult.coverage.layersQueried; + queriedSnapshots = + memory.lastResult.coverage.feedProvenance?.layers || []; + unanswered = [...(memory.lastResult.unanswered || [])]; } else { groups = []; layersQueried = []; @@ -816,9 +820,11 @@ export function createAnalystEngine(providers) { // 2) Spatial scope const rememberedScope = - followUp && !spec.scope ? lastResult.scopePresentation || null : null; + followUp && !spec.scope + ? memory.lastResult.scopePresentation || null + : null; const rememberedDistanceOf = - followUp && !spec.scope ? lastResult.distanceOf || null : null; + followUp && !spec.scope ? memory.lastResult.distanceOf || null : null; let resolvedScope = null; let scopeNote = rememberedScope?.coverage || 'anywhere'; let scopeSource = rememberedScope?.source || null; @@ -1018,7 +1024,7 @@ export function createAnalystEngine(providers) { : null; if (!isCurrent()) return cancelled(); const committedCoverage = memoryOverride?.coverage || result.coverage; - lastResult = { + memory.lastResult = { matched: memoryOverride?.matched || selection.matched, distanceOf: memoryOverride?.distanceOf || rememberedDistanceOf, layerKeys: [...(memoryOverride?.layerKeys || layers)], @@ -1042,10 +1048,10 @@ export function createAnalystEngine(providers) { return { query, reset() { - lastResult = null; + memory.lastResult = null; }, hasMemory() { - return Boolean(lastResult); + return Boolean(memory.lastResult); }, }; } diff --git a/src/main.js b/src/main.js index 62311182a..dfadaa0ff 100644 --- a/src/main.js +++ b/src/main.js @@ -44,7 +44,7 @@ const session = createSessionDiagnostics(() => { navigation: style?._navigation, placeSearch: style?.placeSearch, dataManager: components.data?.dataManager, - voice: components.tools?.voiceCommands, + voice: layerSession.voice?.commands ?? components.tools?.voiceCommands, annotations: components.tools?.annotations, geoImport: window.__gevGeoImport, drawTool: window.__gevDrawTool, @@ -171,8 +171,13 @@ async function settle() { components.controls.styleManager.shareLinkManager?.resumeNavigation?.(); } +// Voice calls made while the engines switch wait for the next engine +// (src/voice/sceneRunner.js); a settled switch lets them look again. +layerSession.switching = () => engineSwitch?.state().busy === true; + function present() { const { kind, busy } = engineSwitch.state(); + if (!busy) layerSession.voice?.runner.wake(); display.presentGlobeEngine({ kind, busy, select }); for (const button of loading.querySelectorAll('.engine-switch-retry button')) button.disabled = busy; diff --git a/src/voice/gevActions.js b/src/voice/gevActions.js index dbf875094..04be469c7 100644 --- a/src/voice/gevActions.js +++ b/src/voice/gevActions.js @@ -349,6 +349,8 @@ export function createGevActionRunner({ speechBuilders = null, deixis = null, readVoicePanelOperation = null, + analystMemory = null, + layerEnabledAt = null, }) { // Voice enable times and analyst follow-up memory belong to this runner. // Speech builders turn results into say/display envelopes; callers extend @@ -357,7 +359,10 @@ export function createGevActionRunner({ ? { ...SPEECH_BUILDERS, ...speechBuilders } : SPEECH_BUILDERS; const engine = givenEngine ?? engineFromViewer(viewer); - const _layerEnabledAt = new Map(); + // A host that replaces this runner when the globe engine changes passes + // the analyst's follow-up memory and the layer enable times, so the next + // runner keeps them. + const _layerEnabledAt = layerEnabledAt ?? new Map(); let analystEngine; const resolveRegionRing = (name, signal) => annotationResolver.resolveRegionRingForQuery(name, signal, placeSearch); @@ -368,6 +373,7 @@ export function createGevActionRunner({ resolveRegionRing, isWarming: (layerKey) => layerIsWarming(_layerEnabledAt, layerKey), }), + { memory: analystMemory }, )); installViewTargetPrewarm(engine); // The camera verbs (move_camera, fly_route) still run on the viewer. @@ -471,7 +477,9 @@ export function createGevActionRunner({ }; if ( !viewer && - ((name === 'control_scene' && !sceneDirector) || VIEWER_ONLY_ACTIONS.includes(name) || CESIUM_LOOK_ACTIONS.includes(name)) + ((name === 'control_scene' && !sceneDirector) || + VIEWER_ONLY_ACTIONS.includes(name) || + CESIUM_LOOK_ACTIONS.includes(name)) ) return unavailableOnEngine(name, engine); @@ -1077,7 +1085,9 @@ export function createGevActionRunner({ contextState?.mode === 'flights' && contextState?.active !== false && contextState?.changing !== true; - const directEntry = !targetLayer && !aircraftClass && + const directEntry = + !targetLayer && + !aircraftClass && styleManager.getCockpitState?.()?.entryAllowed === true; if (!contactsReady && !directEntry) { const contextResult = await styleManager.setContextMode('flights', { diff --git a/src/voice/gevRealtime.js b/src/voice/gevRealtime.js index 93e7e59cf..8fad7a368 100644 --- a/src/voice/gevRealtime.js +++ b/src/voice/gevRealtime.js @@ -5,6 +5,7 @@ import { createVoiceCommands } from './commands.js'; import { cameraPoseKey, createPointerTracker } from './pointerContext.js'; import { createPointerPicker } from './pointerPick.js'; import { createReferentRegistry } from './referents.js'; +import { createSceneRunner } from './sceneRunner.js'; import { engineFromViewer } from '../globe/cesium/engineFromViewer.js'; export * from './realtimeController.js'; @@ -31,49 +32,148 @@ export function withToolCatalog(runner, loadCatalog) { return runGevTool; } +/** + * Voice for a page whose globe engine can change while it runs: the + * microphone, transport, controls and conversation belong to the page + * session; the action runner, the pointer's canvas and picker and the + * annotation events belong to one engine attachment (`attachScene`), and a + * switch only replaces those. Returns `{ commands, runner, attachScene, + * detachScene }`; `commands` is what initGevVoiceCommands returns. + * + * `isSwitching()` tells the runner that the page has started a switch, so a + * call made before the outgoing scene detaches waits for the next one too. + * `createRunner` replaces createGevActionRunner (tests). + */ +export function createSceneVoice(options) { + const scene = { engine: null }; + const cameraKey = () => cameraPoseKey(scene.engine); + let pick = null; + // Point-and-ask: the tracker and referents are shared by the runner (which + // resolves 'pointer' and referent:n) and the controls (which snapshot the + // pointer at turn start and clear both when the session ends). + const pointer = createPointerTracker({ + element: null, + cameraKey, + pick: (x, y) => pick?.(x, y) ?? null, + }); + const referents = createReferentRegistry(); + // The controller exists only after the runner, so its hooks bind late. + const deixis = { pointer, referents, cameraKey }; + // The analyst's follow-up memory and the layer enable times outlive one + // engine's runner. + const analystMemory = { lastResult: null }; + const layerEnabledAt = new Map(); + let commands; + const runner = createSceneRunner({ + isSwitching: options.isSwitching, + switchWaitMs: options.switchWaitMs, + build: (attached) => + withToolCatalog( + (options.createRunner ?? createGevActionRunner)({ + ...options, + ...attached, + deixis, + analystMemory, + layerEnabledAt, + readVoicePanelOperation: () => + commands?.captureVoicePanelOperation?.(), + }), + options.toolCatalog, + ), + }); + const resetConversation = runner.resetConversation; + runner.resetConversation = () => { + analystMemory.lastResult = null; + resetConversation(); + }; + // Annotation outline events of whichever engine is attached. + const outlineListeners = new Set(); + let offOutline = null; + const annotations = { + onOutlineEvent(listener) { + outlineListeners.add(listener); + return () => outlineListeners.delete(listener); + }, + }; + // The panel reads the cockpit of the attached shell. + let shell = null; + const styleManager = { + getCockpitState: () => shell?.getCockpitState?.() ?? null, + }; + commands = createVoiceCommands({ + ...options, + styleManager, + annotations, + pointer, + referents, + runner, + }); + deixis.imageFrame = () => commands.retainedImageFrame?.() || null; + deixis.onPointerUsed = () => commands.announcePointer?.(); + const lifetime = commands.signal ?? commands.session?.signal; + let attachedOnce = false; + const detachScene = () => { + runner.detach(); + offOutline?.(); + offOutline = null; + pointer.setElement(null); + pick = null; + scene.engine = null; + shell = null; + }; + lifetime?.addEventListener( + 'abort', + () => { + detachScene(); + pointer.destroy(); + }, + { once: true }, + ); + if (lifetime?.aborted) pointer.destroy(); + return { + commands, + runner, + /** + * Attach the voice to one engine: `{ viewer, engine, styleManager, + * dataManager, sceneDirector, annotations }` (viewer and annotations + * null on an engine without them). + */ + attachScene(attached) { + if (lifetime?.aborted) return; + detachScene(); + const engine = attached.engine ?? engineFromViewer(attached.viewer); + scene.engine = engine; + shell = attached.styleManager ?? null; + pick = createPointerPicker({ + engine, + dataManager: attached.dataManager ?? options.dataManager, + }); + pointer.setElement(engine?.host.canvas || null); + offOutline = + attached.annotations?.onOutlineEvent?.((event) => { + for (const listener of [...outlineListeners]) listener(event); + }) ?? null; + runner.attach({ ...attached, engine }); + // A session that outlived the switch hears which globe it now drives. + if (attachedOnce && commands.isActive?.()) { + const event = { type: 'engine-changed', engine: engine?.kind ?? null }; + if (typeof commands.notifyMapEvent === 'function') + commands.notifyMapEvent(event); + else commands.session?.sendMapEvent?.(event); + } + attachedOnce = true; + }, + detachScene, + }; +} + /** * Compose the standalone action runner with the voice controls, on the globe * `engine` (by default the one registered for `options.viewer`). Without a * viewer (the Navara engine) the runner refuses the viewer-only tools. */ export function initGevVoiceCommands(options) { - const engine = options.engine ?? engineFromViewer(options.viewer); - const canvas = engine?.host.canvas || null; - const cameraKey = () => cameraPoseKey(engine); - // Point-and-ask: the tracker and referents are shared by the runner (which - // resolves 'pointer' and referent:n) and the controls (which snapshot the - // pointer at turn start and clear both when the session ends). - const pointer = canvas - ? createPointerTracker({ - element: canvas, - cameraKey, - pick: createPointerPicker({ - engine, - dataManager: options.dataManager, - }), - }) - : null; - const referents = createReferentRegistry(); - // The controller exists only after the runner, so its hooks bind late. - const deixis = { pointer, referents, cameraKey }; - let commands; - commands = createVoiceCommands({ - ...options, - pointer, - referents, - runner: withToolCatalog( - createGevActionRunner({ - ...options, - deixis, - readVoicePanelOperation: () => commands?.captureVoicePanelOperation?.(), - }), - options.toolCatalog, - ), - }); - deixis.imageFrame = () => commands.retainedImageFrame?.() || null; - deixis.onPointerUsed = () => commands.announcePointer?.(); - const lifetime = commands.signal ?? commands.session?.signal; - lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true }); - if (lifetime?.aborted) pointer?.destroy(); - return commands; + const voice = createSceneVoice(options); + voice.attachScene(options); + return voice.commands; } diff --git a/src/voice/pointerContext.js b/src/voice/pointerContext.js index 762e5141b..911a61ef2 100644 --- a/src/voice/pointerContext.js +++ b/src/voice/pointerContext.js @@ -276,7 +276,8 @@ export function createPointerTracker({ let held = null; let active = null; let destroyed = false; - const listeners = new AbortController(); + // The element's own listeners; replaced with the element (setElement). + let listeners = new AbortController(); function size() { return { @@ -328,12 +329,14 @@ export function createPointerTracker({ cancelDwell(); } - if (element?.addEventListener) { + function listen() { + if (!element?.addEventListener) return; const options = { passive: true, signal: listeners.signal }; element.addEventListener('pointermove', onMove, options); element.addEventListener('pointerdown', onMove, options); element.addEventListener('pointerleave', onLeave, options); } + listen(); /** Is the pointer a referent right now? */ function isFresh(recentMs = POINTER_FRESH_MS) { @@ -372,7 +375,21 @@ export function createPointerTracker({ } return { - element, + get element() { + return element; + }, + /** + * Follow another element (the next globe's canvas after an engine + * switch), or none. The pointer starts outside it. + */ + setElement(next) { + if (destroyed || next === element) return; + listeners.abort(); + listeners = new AbortController(); + element = next ?? null; + onLeave(); + listen(); + }, capture, isFresh, /** Push-to-talk keydown: snapshot now, keep it for the hold. */ diff --git a/src/voice/sceneRunner.js b/src/voice/sceneRunner.js new file mode 100644 index 000000000..9f2d7fd95 --- /dev/null +++ b/src/voice/sceneRunner.js @@ -0,0 +1,158 @@ +// The voice action runner for a page whose globe engine can change while a +// voice session runs. The session keeps one runner; each engine attachment +// builds the runner of that scene (createGevActionRunner over its viewer, +// engine, shell and annotations) and the forwarder sends every call to the +// current one. A call made while the engines switch waits for the next +// attachment, up to a bound, and then runs on the engine that attached; when +// none attaches in time it is refused with ENGINE_SWITCHING. + +/** How long a call waits for the next engine before it is refused. */ +export const ENGINE_SWITCH_WAIT_MS = 15_000; + +/** The refusal for a call that found no engine within the wait. */ +export function engineSwitchingRefusal(name) { + return { + ok: false, + action: name, + applied: false, + code: 'ENGINE_SWITCHING', + error: 'The globe is switching engines; this action was not run.', + say: 'The globe is switching engines. Ask again in a moment.', + }; +} + +/** + * @param {object} options + * @param {(scene: object) => Function} options.build The runner for one + * attached scene; its optional `dispose()` runs when the scene detaches. + * @param {() => boolean} [options.isSwitching] True while the page switches + * engines, even before the outgoing scene has detached. + * @param {number} [options.switchWaitMs] + */ +export function createSceneRunner({ + build, + isSwitching = () => false, + switchWaitMs = ENGINE_SWITCH_WAIT_MS, +}) { + if (typeof build !== 'function') + throw new TypeError('A scene runner needs a build function'); + let inner = null; + let kind = null; + let attachment = null; + // Bumped by every attachment, so a waiting call knows one happened. + let attachments = 0; + let conversation = 0; + let disposed = false; + const waiters = new Set(); + + const wake = () => { + for (const resolve of [...waiters]) resolve(); + waiters.clear(); + }; + + // Resolves true once a scene is ready (one attached since the wait began, + // or the switch settled with a scene), false when the wait is cut short. + async function waitForScene(signal, timeoutMs) { + const since = attachments; + const deadline = Date.now() + Math.max(0, timeoutMs); + for (;;) { + if (disposed || signal?.aborted) return false; + if (inner && (attachments !== since || !isSwitching())) return true; + const remaining = deadline - Date.now(); + if (remaining <= 0) return false; + await new Promise((resolve) => { + let timer = null; + const done = () => { + clearTimeout(timer); + signal?.removeEventListener('abort', done); + waiters.delete(done); + resolve(); + }; + waiters.add(done); + signal?.addEventListener('abort', done, { once: true }); + timer = setTimeout(done, remaining); + }); + } + } + + async function run(name, args, options = {}) { + if (disposed) + return { + ok: false, + action: name, + cancelled: true, + error: 'Voice ended.', + }; + if (!inner || isSwitching()) { + const found = await waitForScene(options.signal, run.switchWaitMs); + if (disposed) + return { + ok: false, + action: name, + cancelled: true, + error: 'Voice ended.', + }; + if (!found) return engineSwitchingRefusal(name); + } + const current = inner; + const currentKind = kind; + const epoch = conversation; + const signal = options.signal + ? AbortSignal.any([options.signal, attachment.signal]) + : attachment.signal; + const result = await current(name, args, { + ...options, + signal, + isCurrent: () => + epoch === conversation && + !disposed && + current === inner && + (typeof options.isCurrent !== 'function' || options.isCurrent()), + }); + // Say which engine answered: after a switch the model reads the result + // of the globe the person now sees. + return result && + typeof result === 'object' && + !Array.isArray(result) && + result.engine === undefined + ? { ...result, engine: currentKind } + : result; + } + + run.switchWaitMs = switchWaitMs; + /** The engine kind of the attached scene, or null. */ + run.engineKind = () => kind; + /** Build the runner for `scene`; waiting calls run on it. */ + run.attach = (scene) => { + if (disposed) return; + run.detach(); + attachment = new AbortController(); + inner = build(scene); + kind = scene?.engine?.kind ?? (scene?.viewer ? 'cesium' : null); + attachments++; + wake(); + }; + /** Stop the attached runner: its calls in flight see their signal abort. */ + run.detach = () => { + attachment?.abort(); + attachment = null; + const previous = inner; + inner = null; + kind = null; + previous?.dispose?.(); + }; + /** The page's switch settled: waiting calls look again. */ + run.wake = wake; + run.resetConversation = () => { + conversation++; + inner?.resetConversation?.(); + }; + run.dispose = () => { + if (disposed) return; + run.resetConversation(); + disposed = true; + run.detach(); + wake(); + }; + return run; +} diff --git a/src/voice/sceneRunner.test.mjs b/src/voice/sceneRunner.test.mjs new file mode 100644 index 000000000..003365129 --- /dev/null +++ b/src/voice/sceneRunner.test.mjs @@ -0,0 +1,107 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createSceneRunner, engineSwitchingRefusal } from './sceneRunner.js'; + +const engine = (kind) => ({ kind }); + +function runnerFor(log) { + return (scene) => { + const run = async (name, args, options) => { + log.push({ name, kind: scene.engine.kind, signal: options.signal }); + return { ok: true, action: name }; + }; + run.dispose = () => log.push({ disposed: scene.engine.kind }); + run.resetConversation = () => log.push({ reset: scene.engine.kind }); + return run; + }; +} + +test('calls run on the attached engine and say which engine answered', async () => { + const log = []; + const run = createSceneRunner({ build: runnerFor(log) }); + run.attach({ engine: engine('cesium') }); + assert.deepEqual(await run('set_layer', {}), { + ok: true, + action: 'set_layer', + engine: 'cesium', + }); + run.attach({ engine: engine('navara') }); + assert.equal((await run('set_layer', {})).engine, 'navara'); + assert.deepEqual(log[1], { disposed: 'cesium' }); + assert.equal(run.engineKind(), 'navara'); +}); + +test('a call during a switch waits for the next engine and runs there', async () => { + const log = []; + let switching = false; + const run = createSceneRunner({ + build: runnerFor(log), + isSwitching: () => switching, + }); + run.attach({ engine: engine('cesium') }); + switching = true; + const pending = run('set_layer', {}); + run.detach(); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.equal(log.filter((entry) => entry.name).length, 0); + run.attach({ engine: engine('navara') }); + const result = await pending; + assert.equal(result.engine, 'navara'); + switching = false; +}); + +test('a call refused when no engine attaches in time names ENGINE_SWITCHING', async () => { + const run = createSceneRunner({ + build: runnerFor([]), + isSwitching: () => true, + switchWaitMs: 10, + }); + run.attach({ engine: engine('cesium') }); + run.detach(); + const result = await run('annotate_map', {}); + assert.deepEqual(result, engineSwitchingRefusal('annotate_map')); + assert.equal(result.code, 'ENGINE_SWITCHING'); +}); + +test('a switch that settles without a new engine lets waiting calls run', async () => { + const log = []; + let switching = true; + const run = createSceneRunner({ + build: runnerFor(log), + isSwitching: () => switching, + }); + run.attach({ engine: engine('cesium') }); + const pending = run('set_layer', {}); + switching = false; + run.wake(); + assert.equal((await pending).engine, 'cesium'); +}); + +test('the attachment signal aborts calls in flight when the engine detaches', async () => { + let seen; + const run = createSceneRunner({ + build: () => async (name, args, options) => { + seen = options.signal; + await new Promise((resolve) => setTimeout(resolve, 5)); + return { ok: !options.signal.aborted }; + }, + }); + run.attach({ engine: engine('cesium') }); + const pending = run('fly_to_location', {}); + await new Promise((resolve) => setTimeout(resolve, 0)); + run.detach(); + assert.equal(seen.aborted, true); + assert.equal((await pending).ok, false); +}); + +test('a disposed runner refuses and wakes its waiting calls', async () => { + const run = createSceneRunner({ + build: runnerFor([]), + isSwitching: () => true, + }); + const pending = run('set_layer', {}); + run.dispose(); + const result = await pending; + assert.equal(result.cancelled, true); + assert.equal((await run('set_layer', {})).cancelled, true); +}); diff --git a/src/voice/sceneVoice.test.mjs b/src/voice/sceneVoice.test.mjs new file mode 100644 index 000000000..e2015bc1d --- /dev/null +++ b/src/voice/sceneVoice.test.mjs @@ -0,0 +1,117 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createSceneVoice } from './gevRealtime.js'; +import { createTestVoiceUi } from '../testSupport/voiceUi.mjs'; + +function fakeEngine(kind) { + const canvas = new EventTarget(); + canvas.getBoundingClientRect = () => ({ left: 0, top: 0 }); + return { + kind, + host: { canvas, size: () => ({ width: 0, height: 0 }) }, + camera: { pose: () => ({}) }, + }; +} + +function fixture() { + const previous = globalThis.window; + globalThis.window = {}; + const lifetime = new AbortController(); + const adapter = { starts: 0, stops: [], events: [] }; + let runAction; + const built = []; + const voice = createSceneVoice({ + signal: lifetime.signal, + createControl: () => createTestVoiceUi(), + createCard: () => null, + createReticle: () => null, + createRunner: (scene) => { + built.push(scene); + return async (name) => ({ ok: true, action: name }); + }, + createSession(hooks) { + runAction = hooks.runAction; + return { + async start() { + adapter.starts++; + hooks.emit({ type: 'state', state: 'listening' }); + }, + stop(options) { + adapter.stops.push(options); + }, + sendText() {}, + sendMapEvent(event) { + adapter.events.push(event); + return true; + }, + }; + }, + }); + return { + voice, + adapter, + built, + lifetime, + run: (...args) => runAction(...args), + restore: () => { + globalThis.window = previous; + }, + }; +} + +test('an engine switch keeps the voice session and its transport', async () => { + const f = fixture(); + try { + const cesium = fakeEngine('cesium'); + f.voice.attachScene({ engine: cesium, viewer: {} }); + await f.voice.commands.start(); + assert.equal(f.adapter.starts, 1); + assert.equal(f.voice.commands.pointer.element, cesium.host.canvas); + assert.equal((await f.run('set_layer', {})).engine, 'cesium'); + + f.voice.detachScene(); + assert.deepEqual(f.adapter.stops, [], 'a detach never stops the adapter'); + assert.equal(f.voice.commands.isActive(), true); + assert.equal(f.voice.commands.pointer.element, null); + // A call made between the engines waits for the next one. + const pending = f.run('set_layer', {}); + const navara = fakeEngine('navara'); + f.voice.attachScene({ engine: navara, viewer: null }); + assert.equal((await pending).engine, 'navara'); + assert.equal(f.voice.commands.pointer.element, navara.host.canvas); + assert.deepEqual(f.adapter.events, [ + { type: 'engine-changed', engine: 'navara' }, + ]); + assert.equal(f.adapter.starts, 1); + assert.equal(f.built.length, 2); + assert.equal(f.built[1].engine, navara); + + f.lifetime.abort(); + assert.equal(f.adapter.stops.at(-1)?.removeUi, true); + } finally { + f.restore(); + } +}); + +test('annotation outline events reach the session from the attached engine only', async () => { + const f = fixture(); + try { + const listeners = new Set(); + const annotations = { + onOutlineEvent(listener) { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }; + f.voice.attachScene({ engine: fakeEngine('cesium'), annotations }); + await f.voice.commands.start(); + for (const listener of listeners) listener({ status: 'resolved' }); + assert.deepEqual(f.adapter.events, [ + { type: 'map_annotation_outline', status: 'resolved' }, + ]); + f.voice.detachScene(); + assert.equal(listeners.size, 0); + } finally { + f.restore(); + } +}); diff --git a/src/voice/sessionCommands.js b/src/voice/sessionCommands.js index 2c761bd42..82964b1e2 100644 --- a/src/voice/sessionCommands.js +++ b/src/voice/sessionCommands.js @@ -6,10 +6,11 @@ import { VOICE_PROVIDERS } from './providerCapabilities.js'; import { t } from '../i18n/index.js'; function createDefaultReticle(pointer) { - const element = pointer?.element; - if (!element?.ownerDocument) return null; + // A pointer may follow another element later (an engine switch), so the + // reticle reads it each time; a page without a document has none. + if (!pointer || !globalThis.document) return null; return new PointerReticle({ - rectOf: () => element.getBoundingClientRect?.() || null, + rectOf: () => pointer.element?.getBoundingClientRect?.() || null, }); }