Keep an active voice session through engine switches

The page creates the voice once; each engine attachment only gives it that scene's action runner, pointer canvas and annotation events. Calls made during a switch wait for the incoming engine and are refused with ENGINE_SWITCHING when none attaches in time; results name the engine that answered, and the session hears engine-changed after a switch. The analyst's follow-up memory and layer enable times outlive one engine's runner. scripts/qa-voice-engine-switch.mjs drives a fake microphone and Realtime transport through the switches.
This commit is contained in:
German Khokhlov committed 2026-11-03 17:20:40 +04:00
1 parent 35ed499ae4
commit 05578c61b9
13 files changed
+998 -139

No files matched your search

+247
View File
@@ -0,0 +1,247 @@
// An active voice session survives engine switches. With a fake microphone
// and a fake Realtime transport (no model is reached: /api/openai, /api/realtime
// and api.openai.com are blocked or answered in the page), this gate starts
// voice on Cesium, switches Cesium→Navara→Cesium→Navara and checks that:
// - the microphone was requested once, one peer connection was made, and
// neither it nor its data channel closed; no WebSocket opened;
// - every session object kept its identity and each switch attached once;
// - a set_layer_visibility call after each switch ran on the engine shown;
// - a call made during a switch waited and ran on the incoming engine, and
// one whose wait ran out was refused with ENGINE_SWITCHING;
// - annotate_map works on Cesium, is refused on Navara, and the mark drawn
// before the switches is still on the board back on Cesium;
// - the session heard which engine it drives after each switch.
//
// node scripts/qa-voice-engine-switch.mjs http://127.0.0.1:<port>/
import assert from 'node:assert/strict';
import {
idDifference,
launch,
openSessionPage,
previewBase,
readSession,
readVoiceEvidence,
requestEngine,
sessionAddress,
switchEngine,
waitReady,
} from './qa/appSessionHarness.mjs';
const base = previewBase(process.argv[2]);
// Play one server function call on the fake data channel and wait for the
// client's function_call_output.
function callTool(page, name, args, { beforeCall = null } = {}) {
return page.evaluate(
async (name, args, beforeCall) => {
const evidence = window.__qaVoice;
const callId = `qa-call-${(window.__qaCalls = (window.__qaCalls ?? 0) + 1)}`;
const responseId = `qa-response-${callId}`;
const deliver = (event) =>
evidence.channel.dispatchEvent(
new MessageEvent('message', { data: JSON.stringify(event) }),
);
if (beforeCall === 'switch-navara' || beforeCall === 'switch-cesium') {
const select = document.querySelector('#globe-engine');
select.value = beforeCall.slice('switch-'.length);
select.dispatchEvent(new Event('change', { bubbles: true }));
}
deliver({ type: 'response.created', response: { id: responseId } });
deliver({
type: 'response.function_call_arguments.done',
response_id: responseId,
item_id: `item-${callId}`,
call_id: callId,
name,
arguments: JSON.stringify(args),
});
const started = performance.now();
for (;;) {
const sent = evidence.sent.find(
(event) =>
event?.type === 'conversation.item.create' &&
event.item?.type === 'function_call_output' &&
event.item.call_id === callId,
);
if (sent) {
deliver({
type: 'response.done',
response: { id: responseId, status: 'completed', output: [] },
});
return JSON.parse(sent.item.output);
}
if (performance.now() - started > 60_000)
throw new Error(`No output for ${name}`);
await new Promise((resolve) => setTimeout(resolve, 50));
}
},
name,
args,
beforeCall,
);
}
const engineChanges = (page) =>
page.evaluate(() =>
window.__qaVoice.sent
.filter(
(event) =>
event?.type === 'conversation.item.create' &&
event.item?.role === 'system',
)
.map((event) => {
try {
return JSON.parse(event.item.content[0].text);
} catch {
return null;
}
})
.filter((event) => event?.type === 'engine-changed')
.map((event) => event.engine),
);
const ANNOTATION = {
annotations: [
{
type: 'pin',
latitude: 42.3554,
longitude: -71.0605,
label: 'QA voice mark',
},
],
};
const browser = await launch();
const results = [];
try {
const { page, errors } = await openSessionPage(
browser,
sessionAddress(base, { engine: 'cesium' }),
);
await waitReady(page, 'cesium');
await page.evaluate(async () => {
const close = document.querySelector('[data-key-setup-close]');
if (close?.offsetParent) close.click();
await window.__godsEyeView.voiceCommands.start({ pushToTalk: false });
});
await page.waitForFunction(
() =>
window.__qaVoice.channel?.readyState === 'open' &&
window.__godsEyeView.voiceCommands.state === 'listening',
{ timeout: 30_000 },
);
const started = await readVoiceEvidence(page);
const boot = await readSession(page);
assert.equal(started.getUserMedia, 1, 'voice asks for the microphone once');
assert.equal(started.peerConnections, 1);
const annotated = await callTool(page, 'annotate_map', ANNOTATION);
assert.equal(annotated.ok, true, JSON.stringify(annotated));
assert.equal(annotated.engine, 'cesium');
const marks = await page.evaluate(() => window.__gevAnnotations.count());
assert.ok(marks >= 1, 'the mark is on the board');
results.push('PASS annotate_map drew a mark on Cesium');
const sequence = ['navara', 'cesium', 'navara'];
for (const kind of sequence) {
await switchEngine(page, kind);
const layer = await callTool(page, 'set_layer_visibility', {
layerId: 'earthquakes',
enabled: kind === 'navara',
});
assert.equal(layer.engine, kind, JSON.stringify(layer));
assert.equal(
await page.evaluate(() => window.__godsEyeView.engine.kind),
kind,
);
if (kind === 'navara') {
const refused = await callTool(page, 'annotate_map', ANNOTATION);
assert.equal(refused.ok, false, JSON.stringify(refused));
assert.equal(refused.engine, 'navara');
} else {
const count = await page.evaluate(() => window.__gevAnnotations.count());
assert.equal(count, marks, 'the mark survived the round trip');
}
results.push(`PASS ${kind}: set_layer_visibility ran on ${layer.engine}`);
}
// A call made during a switch waits for the incoming engine.
const midSwitch = await callTool(
page,
'set_layer_visibility',
{ layerId: 'earthquakes', enabled: true },
{ beforeCall: 'switch-cesium' },
);
await waitReady(page, 'cesium');
assert.equal(midSwitch.ok, true, JSON.stringify(midSwitch));
assert.equal(midSwitch.engine, 'cesium', JSON.stringify(midSwitch));
results.push('PASS a call made during the switch ran on the incoming engine');
assert.equal(
await page.evaluate(() => window.__gevAnnotations.count()),
marks,
'the mark is still on the board after Cesium→Navara→Cesium→Navara→Cesium',
);
// A call whose wait runs out is refused with a clear code.
await page.evaluate(() => {
window.__godsEyeView.voiceCommands.runner.switchWaitMs = 1;
});
const refused = await callTool(
page,
'set_layer_visibility',
{ layerId: 'earthquakes', enabled: false },
{ beforeCall: 'switch-navara' },
);
await waitReady(page, 'navara');
await page.evaluate(() => {
window.__godsEyeView.voiceCommands.runner.switchWaitMs = 15_000;
});
assert.equal(refused.ok, false, JSON.stringify(refused));
assert.equal(refused.code, 'ENGINE_SWITCHING', JSON.stringify(refused));
results.push(
'PASS a call whose wait ran out was refused with ENGINE_SWITCHING',
);
await switchEngine(page, 'cesium');
const after = await readVoiceEvidence(page);
const state = await readSession(page);
const changes = await engineChanges(page);
const switches = sequence.length + 3;
const report = {
started,
after,
ids: idDifference(boot.ids, state.ids),
counters: state.counters,
changes,
voiceState: await page.evaluate(
() => window.__godsEyeView.voiceCommands.state,
),
};
console.log(JSON.stringify(report, null, 2));
assert.equal(after.getUserMedia, 1, 'no new microphone request');
assert.equal(after.peerConnections, 1, 'no new peer connection');
assert.equal(after.peerConnectionsClosed, 0, 'the connection stayed open');
assert.equal(after.channelsClosed, 0, 'the data channel stayed open');
assert.equal(after.channelState, 'open');
assert.equal(after.webSockets, started.webSockets, 'no new WebSocket');
assert.equal(after.tokenRequests, 1, 'no new token');
assert.equal(report.voiceState, 'listening');
assert.deepEqual(report.ids, {}, 'session objects kept their identity');
assert.equal(
state.counters.attachments,
boot.counters.attachments + switches,
);
assert.equal(changes.length, switches, JSON.stringify(changes));
assert.equal(changes.at(-1), 'cesium');
assert.deepEqual(errors, []);
results.push('PASS the voice session kept its microphone and transport');
await page.evaluate(() => window.__godsEyeView.voiceCommands.stop());
await page.waitForFunction(
() => window.__qaVoice.peerConnectionsClosed === 1,
);
results.push('PASS stopping voice closes the transport');
console.log(results.join('\n'));
} finally {
await browser.close();
}
+100 -53
View File
@@ -17,7 +17,7 @@ import {
installRenderGovernor,
uninstallRenderGovernor,
} from '../renderGovernor.js';
import { initGevVoiceCommands } from '../voice/gevRealtime.js';
import { attachSessionVoice } from './sessionVoice.js';
import { initNavaraImageryBoxTool } from '../ui/imageryBoxNavara.js';
import { createRecentImageryPanel } from '../ui/recentImagery.js';
@@ -38,7 +38,9 @@ export function navaraModeNotice(availability) {
const names = availability.available().map(({ name }) => name);
return Object.freeze({
label: t('Experimental Navara engine'),
detail: names.length ? `Data layers: ${names.join(', ')}` : 'No registered data layers',
detail: names.length
? `Data layers: ${names.join(', ')}`
: 'No registered data layers',
});
}
@@ -112,56 +114,97 @@ export async function registerNavaraLayerRenderers() {
bikeshare: async () =>
(await import('../layers/bikeshare/render/navara.js'))
.createBikeshareNavaraRenderer,
...Object.fromEntries(['weather-radar', 'weather-satellite', 'weather-lightning'].map((id) => [
id, async () => {
const { createWeatherNavaraRenderer } = await import('../layers/weather/render/navara.js');
return (engine, options) => createWeatherNavaraRenderer(engine, navaraViews.get(engine), options);
},
])),
...Object.fromEntries(
['weather-radar', 'weather-satellite', 'weather-lightning'].map((id) => [
id,
async () => {
const { createWeatherNavaraRenderer } =
await import('../layers/weather/render/navara.js');
return (engine, options) =>
createWeatherNavaraRenderer(
engine,
navaraViews.get(engine),
options,
);
},
]),
),
'bhote-koshi-2026': async () => {
const { createNepalEventDisplay } = await import('../layers/bhoteKoshi/render/navara.js');
return (engine) => createNepalEventDisplay(engine, navaraViews.get(engine));
const { createNepalEventDisplay } =
await import('../layers/bhoteKoshi/render/navara.js');
return (engine) =>
createNepalEventDisplay(engine, navaraViews.get(engine));
},
'bhote-koshi-locator': async () => {
const { drawStorySource } = await import('../layers/shared/render/navaraStory.js');
return (engine, { data }) => drawStorySource(engine, navaraViews.get(engine), data);
const { drawStorySource } =
await import('../layers/shared/render/navaraStory.js');
return (engine, { data }) =>
drawStorySource(engine, navaraViews.get(engine), data);
},
'street-level': async () => {
const { createStreetMap } = await import('../layers/streetLevel/render/navara.js');
return engine => createStreetMap(engine, navaraViews.get(engine));
const { createStreetMap } =
await import('../layers/streetLevel/render/navara.js');
return (engine) => createStreetMap(engine, navaraViews.get(engine));
},
traffic: async () => {
const { createTrafficNavaraRenderer } = await import('../layers/traffic/render/navara.js');
return engine => createTrafficNavaraRenderer(engine, navaraViews.get(engine));
const { createTrafficNavaraRenderer } =
await import('../layers/traffic/render/navara.js');
return (engine) =>
createTrafficNavaraRenderer(engine, navaraViews.get(engine));
},
directions: async () => {
const { createDirectionsNavaraRenderer } = await import('../layers/directions/render/navara.js');
return (engine) => createDirectionsNavaraRenderer(engine, navaraViews.get(engine));
const { createDirectionsNavaraRenderer } =
await import('../layers/directions/render/navara.js');
return (engine) =>
createDirectionsNavaraRenderer(engine, navaraViews.get(engine));
},
'telegeography-submarine-cables': async () => {
const { createNavaraCables } = await import('../layers/submarineCables/render/navara.js');
return (engine, { overlayHost }) => createNavaraCables(engine, navaraViews.get(engine), overlayHost);
const { createNavaraCables } =
await import('../layers/submarineCables/render/navara.js');
return (engine, { overlayHost }) =>
createNavaraCables(engine, navaraViews.get(engine), overlayHost);
},
wind: async () => {
const { createWindNavaraRenderer } = await import('../layers/wind/render/navara.js');
return (engine) => createWindNavaraRenderer(engine, navaraViews.get(engine));
const { createWindNavaraRenderer } =
await import('../layers/wind/render/navara.js');
return (engine) =>
createWindNavaraRenderer(engine, navaraViews.get(engine));
},
'weather-cyclones': async () => {
const {createCyclonesNavaraRenderer} = await import('../layers/cyclones/render/navara.js');
return (engine,options) => createCyclonesNavaraRenderer(engine,navaraViews.get(engine).view,options);
const { createCyclonesNavaraRenderer } =
await import('../layers/cyclones/render/navara.js');
return (engine, options) =>
createCyclonesNavaraRenderer(
engine,
navaraViews.get(engine).view,
options,
);
},
...Object.fromEntries(['local-datacenters','local-dams'].map((id) => [
id, async () => {
const {createInfrastructureNavaraRenderer} = await import('../layers/infrastructure/render/navara.js');
return (engine,{config,services}) => createInfrastructureNavaraRenderer(engine,navaraViews.get(engine),config,services);
},
])),
...Object.fromEntries(
['local-datacenters', 'local-dams'].map((id) => [
id,
async () => {
const { createInfrastructureNavaraRenderer } =
await import('../layers/infrastructure/render/navara.js');
return (engine, { config, services }) =>
createInfrastructureNavaraRenderer(
engine,
navaraViews.get(engine),
config,
services,
);
},
]),
),
's2-ghostmaps': async () => {
const { createGhostMapsNavaraRenderer } = await import('../layers/ghostMaps/render/navara.js');
return (engine) => createGhostMapsNavaraRenderer(engine, navaraViews.get(engine).view);
const { createGhostMapsNavaraRenderer } =
await import('../layers/ghostMaps/render/navara.js');
return (engine) =>
createGhostMapsNavaraRenderer(engine, navaraViews.get(engine).view);
},
'military-awareness': async () =>
(await import('../layers/awareness/render/navara.js')).createAwarenessNavaraRenderer,
(await import('../layers/awareness/render/navara.js'))
.createAwarenessNavaraRenderer,
'military-installations': async () => {
const { createInstallationsNavaraRenderer } =
await import('../layers/installations/render/navara.js');
@@ -181,17 +224,22 @@ export async function registerNavaraLayerRenderers() {
});
},
'fire-perimeters': async () => {
const { createPerimetersNavaraRenderer } = await import('../layers/perimeters/render/navara.js');
return (engine) => createPerimetersNavaraRenderer(engine, navaraViews.get(engine));
const { createPerimetersNavaraRenderer } =
await import('../layers/perimeters/render/navara.js');
return (engine) =>
createPerimetersNavaraRenderer(engine, navaraViews.get(engine));
},
'gnss-integrity': async () => {
const { createGnssNavaraRenderer } = await import('../layers/gnss/render/navara.js');
return (engine) => createGnssNavaraRenderer(engine, navaraViews.get(engine));
const { createGnssNavaraRenderer } =
await import('../layers/gnss/render/navara.js');
return (engine) =>
createGnssNavaraRenderer(engine, navaraViews.get(engine));
},
'air-quality': async () => {
const { createAirQualityNavaraRenderer } =
await import('../layers/airQuality/render/navara.js');
return (engine) => createAirQualityNavaraRenderer(engine, navaraViews.get(engine));
return (engine) =>
createAirQualityNavaraRenderer(engine, navaraViews.get(engine));
},
earthquakes: async () => {
const { createEarthquakesNavaraRenderer } =
@@ -355,6 +403,7 @@ export function createViewerlessTools({
voice = {},
startChrome,
sceneDirector,
layerSession = null,
signal,
defer,
}) {
@@ -442,23 +491,21 @@ export function createViewerlessTools({
});
// Voice on the engine (phase 3, Task 9): the W1 tools drive the engine;
// the Director uses the shared session; viewer-only tools declare availability.
const voiceCommands = initGevVoiceCommands({
...voice,
annotationResolver: operations.annotationResolver,
searchNavigation: operations.searchAndFlyTo,
signal,
const voiceCommands = attachSessionVoice({
layerSession,
voice,
operations,
placeSearch,
viewer: null,
engine,
styleManager,
dataManager,
sceneDirector,
annotations: null,
});
defer(() => {
voiceCommands.stop({ removeUi: true });
if (window.__gevVoiceCommands === voiceCommands)
delete window.__gevVoiceCommands;
scene: {
viewer: null,
engine,
styleManager,
sceneDirector,
annotations: null,
},
signal,
defer,
});
debug.voiceCommands = voiceCommands;
return { sceneDirector, annotations: null, voiceCommands };
+47
View File
@@ -0,0 +1,47 @@
import { createSceneVoice } from '../voice/gevRealtime.js';
/**
* Attach the page's voice to this engine's scene. With a layer session the
* voice is created once, for the page lifetime, and every engine attachment
* only gives it the scene (runner, pointer canvas, annotation events): an
* active voice session keeps its microphone and transport through a switch.
* Without one (a host that never switches) the voice lives with this
* application. Returns the voice commands.
*/
export function attachSessionVoice({
layerSession = null,
voice = {},
operations,
placeSearch,
dataManager,
scene,
signal,
defer,
}) {
const options = {
...voice,
floorServices: operations.surface?.groundFloor,
annotationResolver: operations.annotationResolver,
searchNavigation: operations.searchAndFlyTo,
placeSearch,
dataManager,
};
let sceneVoice;
if (layerSession) {
sceneVoice = layerSession.voice ??= createSceneVoice({
...options,
signal: layerSession.lifetime.signal,
isSwitching: () => layerSession.switching?.() === true,
});
} else {
sceneVoice = createSceneVoice({ ...options, signal });
defer(() => {
sceneVoice.commands.stop({ removeUi: true });
if (window.__gevVoiceCommands === sceneVoice.commands)
delete window.__gevVoiceCommands;
});
}
sceneVoice.attachScene({ ...scene, dataManager });
defer(() => sceneVoice.detachScene());
return sceneVoice.commands;
}
+18 -21
View File
@@ -6,7 +6,7 @@ import { useDisplayStore } from '../ui/stores/display.js';
import { applicationPinia } from '../ui/vueRuntime.js';
import { initImageryBoxTool } from '../ui/imageryBoxTool.js';
import { createRecentImageryPanel } from '../ui/recentImagery.js';
import { initGevVoiceCommands } from '../voice/gevRealtime.js';
import { attachSessionVoice } from './sessionVoice.js';
import { installViews, isEmbeddedInline } from './embed.js';
import { installScopeMask, destroyScopeMask } from '../scopeMask.js';
import { installLayerReleaseTrim } from '../sceneCommandRelease.js';
@@ -42,10 +42,15 @@ export function createApplicationTools({
const { viewer, engine, tileset, mapStackController, operations } = scene;
const { styleManager, weatherEffects, cockpitCloudEffects } = controls;
const { dataManager } = data;
const isMapStackAvailable = id => mapStackController?.isStackAvailable(id) === true;
const sceneDirector = layerSession?.director ?? new SceneDirector(viewer, styleManager, dataManager, {
engine, dataPacks: {...sceneDataPacks,native:scene.packRenderer}, isMapStackAvailable,
});
const isMapStackAvailable = (id) =>
mapStackController?.isStackAvailable(id) === true;
const sceneDirector =
layerSession?.director ??
new SceneDirector(viewer, styleManager, dataManager, {
engine,
dataPacks: { ...sceneDataPacks, native: scene.packRenderer },
isMapStackAvailable,
});
if (layerSession?.director)
sceneDirector.attachScene(
viewer,
@@ -79,6 +84,7 @@ export function createApplicationTools({
voice,
sceneDirector,
startChrome,
layerSession,
signal,
defer,
}),
@@ -239,24 +245,15 @@ export function createApplicationTools({
defer(() => {
if (window.__godsEyeView === debug) delete window.__godsEyeView;
});
const voiceCommands = initGevVoiceCommands({
...voice,
floorServices: operations.surface.groundFloor,
annotationResolver: operations.annotationResolver,
searchNavigation: operations.searchAndFlyTo,
signal,
const voiceCommands = attachSessionVoice({
layerSession,
voice,
operations,
placeSearch,
viewer,
engine,
styleManager,
dataManager,
sceneDirector,
annotations,
});
defer(() => {
voiceCommands.stop({ removeUi: true });
if (window.__gevVoiceCommands === voiceCommands)
delete window.__gevVoiceCommands;
scene: { viewer, engine, styleManager, sceneDirector, annotations },
signal,
defer,
});
debug.voiceCommands = voiceCommands;
defer(
+22 -16
View File
@@ -644,8 +644,11 @@ export async function selectAnalystRows(
* Create an engine bound to live providers. All spatial/text/number logic is
* in the pure helpers above; this closure only sequences and remembers.
*/
export function createAnalystEngine(providers) {
let lastResult = null;
export function createAnalystEngine(providers, { memory = null } = {}) {
// The follow-up memory. A caller may own it (`memory`, an object whose
// `lastResult` this engine reads and writes) so a replacement engine over
// another globe keeps the last answer; otherwise it is this engine's.
memory ??= { lastResult: null };
/**
* Run one query.
@@ -666,7 +669,7 @@ export function createAnalystEngine(providers) {
) {
const cancelled = () =>
refusal('CANCELLED', 'This query was superseded.', { cancelled: true });
if (spec.followUp && !lastResult)
if (spec.followUp && !memory.lastResult)
return refusal(
'NO_RESULT_CONTEXT',
'There is no previous answer to follow up on. Ask the full question.',
@@ -677,7 +680,7 @@ export function createAnalystEngine(providers) {
// must be that set: "those hospitals" after an aircraft answer is a new
// question, never a filter over the old aircraft.
if (followUp && Array.isArray(spec.layers) && spec.layers.length) {
const remembered = new Set(lastResult.layerKeys);
const remembered = new Set(memory.lastResult.layerKeys);
const named = new Set(spec.layers);
const same =
remembered.size === named.size &&
@@ -685,16 +688,16 @@ export function createAnalystEngine(providers) {
if (!same)
return refusal(
'FOLLOW_UP_MISMATCH',
`The last answer was about ${lastResult.layerKeys.join(', ')}, not ${spec.layers.join(', ')}. Ask again without followUp.`,
`The last answer was about ${memory.lastResult.layerKeys.join(', ')}, not ${spec.layers.join(', ')}. Ask again without followUp.`,
{
layers: [...spec.layers],
remembered: [...lastResult.layerKeys],
remembered: [...memory.lastResult.layerKeys],
coverage: { layersQueried: [], scope: 'none' },
},
);
}
const layers = followUp
? lastResult.layerKeys
? memory.lastResult.layerKeys
: Array.isArray(spec.layers) && spec.layers.length
? spec.layers
: ['flights'];
@@ -729,10 +732,11 @@ export function createAnalystEngine(providers) {
let queriedSnapshots;
let unanswered = [];
if (followUp) {
groups = lastResult.matched;
layersQueried = lastResult.coverage.layersQueried;
queriedSnapshots = lastResult.coverage.feedProvenance?.layers || [];
unanswered = [...(lastResult.unanswered || [])];
groups = memory.lastResult.matched;
layersQueried = memory.lastResult.coverage.layersQueried;
queriedSnapshots =
memory.lastResult.coverage.feedProvenance?.layers || [];
unanswered = [...(memory.lastResult.unanswered || [])];
} else {
groups = [];
layersQueried = [];
@@ -816,9 +820,11 @@ export function createAnalystEngine(providers) {
// 2) Spatial scope
const rememberedScope =
followUp && !spec.scope ? lastResult.scopePresentation || null : null;
followUp && !spec.scope
? memory.lastResult.scopePresentation || null
: null;
const rememberedDistanceOf =
followUp && !spec.scope ? lastResult.distanceOf || null : null;
followUp && !spec.scope ? memory.lastResult.distanceOf || null : null;
let resolvedScope = null;
let scopeNote = rememberedScope?.coverage || 'anywhere';
let scopeSource = rememberedScope?.source || null;
@@ -1018,7 +1024,7 @@ export function createAnalystEngine(providers) {
: null;
if (!isCurrent()) return cancelled();
const committedCoverage = memoryOverride?.coverage || result.coverage;
lastResult = {
memory.lastResult = {
matched: memoryOverride?.matched || selection.matched,
distanceOf: memoryOverride?.distanceOf || rememberedDistanceOf,
layerKeys: [...(memoryOverride?.layerKeys || layers)],
@@ -1042,10 +1048,10 @@ export function createAnalystEngine(providers) {
return {
query,
reset() {
lastResult = null;
memory.lastResult = null;
},
hasMemory() {
return Boolean(lastResult);
return Boolean(memory.lastResult);
},
};
}
+6 -1
View File
@@ -44,7 +44,7 @@ const session = createSessionDiagnostics(() => {
navigation: style?._navigation,
placeSearch: style?.placeSearch,
dataManager: components.data?.dataManager,
voice: components.tools?.voiceCommands,
voice: layerSession.voice?.commands ?? components.tools?.voiceCommands,
annotations: components.tools?.annotations,
geoImport: window.__gevGeoImport,
drawTool: window.__gevDrawTool,
@@ -171,8 +171,13 @@ async function settle() {
components.controls.styleManager.shareLinkManager?.resumeNavigation?.();
}
// Voice calls made while the engines switch wait for the next engine
// (src/voice/sceneRunner.js); a settled switch lets them look again.
layerSession.switching = () => engineSwitch?.state().busy === true;
function present() {
const { kind, busy } = engineSwitch.state();
if (!busy) layerSession.voice?.runner.wake();
display.presentGlobeEngine({ kind, busy, select });
for (const button of loading.querySelectorAll('.engine-switch-retry button'))
button.disabled = busy;
+13 -3
View File
@@ -349,6 +349,8 @@ export function createGevActionRunner({
speechBuilders = null,
deixis = null,
readVoicePanelOperation = null,
analystMemory = null,
layerEnabledAt = null,
}) {
// Voice enable times and analyst follow-up memory belong to this runner.
// Speech builders turn results into say/display envelopes; callers extend
@@ -357,7 +359,10 @@ export function createGevActionRunner({
? { ...SPEECH_BUILDERS, ...speechBuilders }
: SPEECH_BUILDERS;
const engine = givenEngine ?? engineFromViewer(viewer);
const _layerEnabledAt = new Map();
// A host that replaces this runner when the globe engine changes passes
// the analyst's follow-up memory and the layer enable times, so the next
// runner keeps them.
const _layerEnabledAt = layerEnabledAt ?? new Map();
let analystEngine;
const resolveRegionRing = (name, signal) =>
annotationResolver.resolveRegionRingForQuery(name, signal, placeSearch);
@@ -368,6 +373,7 @@ export function createGevActionRunner({
resolveRegionRing,
isWarming: (layerKey) => layerIsWarming(_layerEnabledAt, layerKey),
}),
{ memory: analystMemory },
));
installViewTargetPrewarm(engine);
// The camera verbs (move_camera, fly_route) still run on the viewer.
@@ -471,7 +477,9 @@ export function createGevActionRunner({
};
if (
!viewer &&
((name === 'control_scene' && !sceneDirector) || VIEWER_ONLY_ACTIONS.includes(name) || CESIUM_LOOK_ACTIONS.includes(name))
((name === 'control_scene' && !sceneDirector) ||
VIEWER_ONLY_ACTIONS.includes(name) ||
CESIUM_LOOK_ACTIONS.includes(name))
)
return unavailableOnEngine(name, engine);
@@ -1077,7 +1085,9 @@ export function createGevActionRunner({
contextState?.mode === 'flights' &&
contextState?.active !== false &&
contextState?.changing !== true;
const directEntry = !targetLayer && !aircraftClass &&
const directEntry =
!targetLayer &&
!aircraftClass &&
styleManager.getCockpitState?.()?.entryAllowed === true;
if (!contactsReady && !directEntry) {
const contextResult = await styleManager.setContextMode('flights', {
+139 -39
View File
@@ -5,6 +5,7 @@ import { createVoiceCommands } from './commands.js';
import { cameraPoseKey, createPointerTracker } from './pointerContext.js';
import { createPointerPicker } from './pointerPick.js';
import { createReferentRegistry } from './referents.js';
import { createSceneRunner } from './sceneRunner.js';
import { engineFromViewer } from '../globe/cesium/engineFromViewer.js';
export * from './realtimeController.js';
@@ -31,49 +32,148 @@ export function withToolCatalog(runner, loadCatalog) {
return runGevTool;
}
/**
* Voice for a page whose globe engine can change while it runs: the
* microphone, transport, controls and conversation belong to the page
* session; the action runner, the pointer's canvas and picker and the
* annotation events belong to one engine attachment (`attachScene`), and a
* switch only replaces those. Returns `{ commands, runner, attachScene,
* detachScene }`; `commands` is what initGevVoiceCommands returns.
*
* `isSwitching()` tells the runner that the page has started a switch, so a
* call made before the outgoing scene detaches waits for the next one too.
* `createRunner` replaces createGevActionRunner (tests).
*/
export function createSceneVoice(options) {
const scene = { engine: null };
const cameraKey = () => cameraPoseKey(scene.engine);
let pick = null;
// Point-and-ask: the tracker and referents are shared by the runner (which
// resolves 'pointer' and referent:n) and the controls (which snapshot the
// pointer at turn start and clear both when the session ends).
const pointer = createPointerTracker({
element: null,
cameraKey,
pick: (x, y) => pick?.(x, y) ?? null,
});
const referents = createReferentRegistry();
// The controller exists only after the runner, so its hooks bind late.
const deixis = { pointer, referents, cameraKey };
// The analyst's follow-up memory and the layer enable times outlive one
// engine's runner.
const analystMemory = { lastResult: null };
const layerEnabledAt = new Map();
let commands;
const runner = createSceneRunner({
isSwitching: options.isSwitching,
switchWaitMs: options.switchWaitMs,
build: (attached) =>
withToolCatalog(
(options.createRunner ?? createGevActionRunner)({
...options,
...attached,
deixis,
analystMemory,
layerEnabledAt,
readVoicePanelOperation: () =>
commands?.captureVoicePanelOperation?.(),
}),
options.toolCatalog,
),
});
const resetConversation = runner.resetConversation;
runner.resetConversation = () => {
analystMemory.lastResult = null;
resetConversation();
};
// Annotation outline events of whichever engine is attached.
const outlineListeners = new Set();
let offOutline = null;
const annotations = {
onOutlineEvent(listener) {
outlineListeners.add(listener);
return () => outlineListeners.delete(listener);
},
};
// The panel reads the cockpit of the attached shell.
let shell = null;
const styleManager = {
getCockpitState: () => shell?.getCockpitState?.() ?? null,
};
commands = createVoiceCommands({
...options,
styleManager,
annotations,
pointer,
referents,
runner,
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();
const lifetime = commands.signal ?? commands.session?.signal;
let attachedOnce = false;
const detachScene = () => {
runner.detach();
offOutline?.();
offOutline = null;
pointer.setElement(null);
pick = null;
scene.engine = null;
shell = null;
};
lifetime?.addEventListener(
'abort',
() => {
detachScene();
pointer.destroy();
},
{ once: true },
);
if (lifetime?.aborted) pointer.destroy();
return {
commands,
runner,
/**
* Attach the voice to one engine: `{ viewer, engine, styleManager,
* dataManager, sceneDirector, annotations }` (viewer and annotations
* null on an engine without them).
*/
attachScene(attached) {
if (lifetime?.aborted) return;
detachScene();
const engine = attached.engine ?? engineFromViewer(attached.viewer);
scene.engine = engine;
shell = attached.styleManager ?? null;
pick = createPointerPicker({
engine,
dataManager: attached.dataManager ?? options.dataManager,
});
pointer.setElement(engine?.host.canvas || null);
offOutline =
attached.annotations?.onOutlineEvent?.((event) => {
for (const listener of [...outlineListeners]) listener(event);
}) ?? null;
runner.attach({ ...attached, engine });
// A session that outlived the switch hears which globe it now drives.
if (attachedOnce && commands.isActive?.()) {
const event = { type: 'engine-changed', engine: engine?.kind ?? null };
if (typeof commands.notifyMapEvent === 'function')
commands.notifyMapEvent(event);
else commands.session?.sendMapEvent?.(event);
}
attachedOnce = true;
},
detachScene,
};
}
/**
* Compose the standalone action runner with the voice controls, on the globe
* `engine` (by default the one registered for `options.viewer`). Without a
* viewer (the Navara engine) the runner refuses the viewer-only tools.
*/
export function initGevVoiceCommands(options) {
const engine = options.engine ?? engineFromViewer(options.viewer);
const canvas = engine?.host.canvas || null;
const cameraKey = () => cameraPoseKey(engine);
// Point-and-ask: the tracker and referents are shared by the runner (which
// resolves 'pointer' and referent:n) and the controls (which snapshot the
// pointer at turn start and clear both when the session ends).
const pointer = canvas
? createPointerTracker({
element: canvas,
cameraKey,
pick: createPointerPicker({
engine,
dataManager: options.dataManager,
}),
})
: null;
const referents = createReferentRegistry();
// The controller exists only after the runner, so its hooks bind late.
const deixis = { pointer, referents, cameraKey };
let commands;
commands = createVoiceCommands({
...options,
pointer,
referents,
runner: withToolCatalog(
createGevActionRunner({
...options,
deixis,
readVoicePanelOperation: () => commands?.captureVoicePanelOperation?.(),
}),
options.toolCatalog,
),
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();
const lifetime = commands.signal ?? commands.session?.signal;
lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true });
if (lifetime?.aborted) pointer?.destroy();
return commands;
const voice = createSceneVoice(options);
voice.attachScene(options);
return voice.commands;
}
+20 -3
View File
@@ -276,7 +276,8 @@ export function createPointerTracker({
let held = null;
let active = null;
let destroyed = false;
const listeners = new AbortController();
// The element's own listeners; replaced with the element (setElement).
let listeners = new AbortController();
function size() {
return {
@@ -328,12 +329,14 @@ export function createPointerTracker({
cancelDwell();
}
if (element?.addEventListener) {
function listen() {
if (!element?.addEventListener) return;
const options = { passive: true, signal: listeners.signal };
element.addEventListener('pointermove', onMove, options);
element.addEventListener('pointerdown', onMove, options);
element.addEventListener('pointerleave', onLeave, options);
}
listen();
/** Is the pointer a referent right now? */
function isFresh(recentMs = POINTER_FRESH_MS) {
@@ -372,7 +375,21 @@ export function createPointerTracker({
}
return {
element,
get element() {
return element;
},
/**
* Follow another element (the next globe's canvas after an engine
* switch), or none. The pointer starts outside it.
*/
setElement(next) {
if (destroyed || next === element) return;
listeners.abort();
listeners = new AbortController();
element = next ?? null;
onLeave();
listen();
},
capture,
isFresh,
/** Push-to-talk keydown: snapshot now, keep it for the hold. */
+158
View File
@@ -0,0 +1,158 @@
// The voice action runner for a page whose globe engine can change while a
// voice session runs. The session keeps one runner; each engine attachment
// builds the runner of that scene (createGevActionRunner over its viewer,
// engine, shell and annotations) and the forwarder sends every call to the
// current one. A call made while the engines switch waits for the next
// attachment, up to a bound, and then runs on the engine that attached; when
// none attaches in time it is refused with ENGINE_SWITCHING.
/** How long a call waits for the next engine before it is refused. */
export const ENGINE_SWITCH_WAIT_MS = 15_000;
/** The refusal for a call that found no engine within the wait. */
export function engineSwitchingRefusal(name) {
return {
ok: false,
action: name,
applied: false,
code: 'ENGINE_SWITCHING',
error: 'The globe is switching engines; this action was not run.',
say: 'The globe is switching engines. Ask again in a moment.',
};
}
/**
* @param {object} options
* @param {(scene: object) => Function} options.build The runner for one
* attached scene; its optional `dispose()` runs when the scene detaches.
* @param {() => boolean} [options.isSwitching] True while the page switches
* engines, even before the outgoing scene has detached.
* @param {number} [options.switchWaitMs]
*/
export function createSceneRunner({
build,
isSwitching = () => false,
switchWaitMs = ENGINE_SWITCH_WAIT_MS,
}) {
if (typeof build !== 'function')
throw new TypeError('A scene runner needs a build function');
let inner = null;
let kind = null;
let attachment = null;
// Bumped by every attachment, so a waiting call knows one happened.
let attachments = 0;
let conversation = 0;
let disposed = false;
const waiters = new Set();
const wake = () => {
for (const resolve of [...waiters]) resolve();
waiters.clear();
};
// Resolves true once a scene is ready (one attached since the wait began,
// or the switch settled with a scene), false when the wait is cut short.
async function waitForScene(signal, timeoutMs) {
const since = attachments;
const deadline = Date.now() + Math.max(0, timeoutMs);
for (;;) {
if (disposed || signal?.aborted) return false;
if (inner && (attachments !== since || !isSwitching())) return true;
const remaining = deadline - Date.now();
if (remaining <= 0) return false;
await new Promise((resolve) => {
let timer = null;
const done = () => {
clearTimeout(timer);
signal?.removeEventListener('abort', done);
waiters.delete(done);
resolve();
};
waiters.add(done);
signal?.addEventListener('abort', done, { once: true });
timer = setTimeout(done, remaining);
});
}
}
async function run(name, args, options = {}) {
if (disposed)
return {
ok: false,
action: name,
cancelled: true,
error: 'Voice ended.',
};
if (!inner || isSwitching()) {
const found = await waitForScene(options.signal, run.switchWaitMs);
if (disposed)
return {
ok: false,
action: name,
cancelled: true,
error: 'Voice ended.',
};
if (!found) return engineSwitchingRefusal(name);
}
const current = inner;
const currentKind = kind;
const epoch = conversation;
const signal = options.signal
? AbortSignal.any([options.signal, attachment.signal])
: attachment.signal;
const result = await current(name, args, {
...options,
signal,
isCurrent: () =>
epoch === conversation &&
!disposed &&
current === inner &&
(typeof options.isCurrent !== 'function' || options.isCurrent()),
});
// Say which engine answered: after a switch the model reads the result
// of the globe the person now sees.
return result &&
typeof result === 'object' &&
!Array.isArray(result) &&
result.engine === undefined
? { ...result, engine: currentKind }
: result;
}
run.switchWaitMs = switchWaitMs;
/** The engine kind of the attached scene, or null. */
run.engineKind = () => kind;
/** Build the runner for `scene`; waiting calls run on it. */
run.attach = (scene) => {
if (disposed) return;
run.detach();
attachment = new AbortController();
inner = build(scene);
kind = scene?.engine?.kind ?? (scene?.viewer ? 'cesium' : null);
attachments++;
wake();
};
/** Stop the attached runner: its calls in flight see their signal abort. */
run.detach = () => {
attachment?.abort();
attachment = null;
const previous = inner;
inner = null;
kind = null;
previous?.dispose?.();
};
/** The page's switch settled: waiting calls look again. */
run.wake = wake;
run.resetConversation = () => {
conversation++;
inner?.resetConversation?.();
};
run.dispose = () => {
if (disposed) return;
run.resetConversation();
disposed = true;
run.detach();
wake();
};
return run;
}
+107
View File
@@ -0,0 +1,107 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createSceneRunner, engineSwitchingRefusal } from './sceneRunner.js';
const engine = (kind) => ({ kind });
function runnerFor(log) {
return (scene) => {
const run = async (name, args, options) => {
log.push({ name, kind: scene.engine.kind, signal: options.signal });
return { ok: true, action: name };
};
run.dispose = () => log.push({ disposed: scene.engine.kind });
run.resetConversation = () => log.push({ reset: scene.engine.kind });
return run;
};
}
test('calls run on the attached engine and say which engine answered', async () => {
const log = [];
const run = createSceneRunner({ build: runnerFor(log) });
run.attach({ engine: engine('cesium') });
assert.deepEqual(await run('set_layer', {}), {
ok: true,
action: 'set_layer',
engine: 'cesium',
});
run.attach({ engine: engine('navara') });
assert.equal((await run('set_layer', {})).engine, 'navara');
assert.deepEqual(log[1], { disposed: 'cesium' });
assert.equal(run.engineKind(), 'navara');
});
test('a call during a switch waits for the next engine and runs there', async () => {
const log = [];
let switching = false;
const run = createSceneRunner({
build: runnerFor(log),
isSwitching: () => switching,
});
run.attach({ engine: engine('cesium') });
switching = true;
const pending = run('set_layer', {});
run.detach();
await new Promise((resolve) => setTimeout(resolve, 5));
assert.equal(log.filter((entry) => entry.name).length, 0);
run.attach({ engine: engine('navara') });
const result = await pending;
assert.equal(result.engine, 'navara');
switching = false;
});
test('a call refused when no engine attaches in time names ENGINE_SWITCHING', async () => {
const run = createSceneRunner({
build: runnerFor([]),
isSwitching: () => true,
switchWaitMs: 10,
});
run.attach({ engine: engine('cesium') });
run.detach();
const result = await run('annotate_map', {});
assert.deepEqual(result, engineSwitchingRefusal('annotate_map'));
assert.equal(result.code, 'ENGINE_SWITCHING');
});
test('a switch that settles without a new engine lets waiting calls run', async () => {
const log = [];
let switching = true;
const run = createSceneRunner({
build: runnerFor(log),
isSwitching: () => switching,
});
run.attach({ engine: engine('cesium') });
const pending = run('set_layer', {});
switching = false;
run.wake();
assert.equal((await pending).engine, 'cesium');
});
test('the attachment signal aborts calls in flight when the engine detaches', async () => {
let seen;
const run = createSceneRunner({
build: () => async (name, args, options) => {
seen = options.signal;
await new Promise((resolve) => setTimeout(resolve, 5));
return { ok: !options.signal.aborted };
},
});
run.attach({ engine: engine('cesium') });
const pending = run('fly_to_location', {});
await new Promise((resolve) => setTimeout(resolve, 0));
run.detach();
assert.equal(seen.aborted, true);
assert.equal((await pending).ok, false);
});
test('a disposed runner refuses and wakes its waiting calls', async () => {
const run = createSceneRunner({
build: runnerFor([]),
isSwitching: () => true,
});
const pending = run('set_layer', {});
run.dispose();
const result = await pending;
assert.equal(result.cancelled, true);
assert.equal((await run('set_layer', {})).cancelled, true);
});
+117
View File
@@ -0,0 +1,117 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createSceneVoice } from './gevRealtime.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
function fakeEngine(kind) {
const canvas = new EventTarget();
canvas.getBoundingClientRect = () => ({ left: 0, top: 0 });
return {
kind,
host: { canvas, size: () => ({ width: 0, height: 0 }) },
camera: { pose: () => ({}) },
};
}
function fixture() {
const previous = globalThis.window;
globalThis.window = {};
const lifetime = new AbortController();
const adapter = { starts: 0, stops: [], events: [] };
let runAction;
const built = [];
const voice = createSceneVoice({
signal: lifetime.signal,
createControl: () => createTestVoiceUi(),
createCard: () => null,
createReticle: () => null,
createRunner: (scene) => {
built.push(scene);
return async (name) => ({ ok: true, action: name });
},
createSession(hooks) {
runAction = hooks.runAction;
return {
async start() {
adapter.starts++;
hooks.emit({ type: 'state', state: 'listening' });
},
stop(options) {
adapter.stops.push(options);
},
sendText() {},
sendMapEvent(event) {
adapter.events.push(event);
return true;
},
};
},
});
return {
voice,
adapter,
built,
lifetime,
run: (...args) => runAction(...args),
restore: () => {
globalThis.window = previous;
},
};
}
test('an engine switch keeps the voice session and its transport', async () => {
const f = fixture();
try {
const cesium = fakeEngine('cesium');
f.voice.attachScene({ engine: cesium, viewer: {} });
await f.voice.commands.start();
assert.equal(f.adapter.starts, 1);
assert.equal(f.voice.commands.pointer.element, cesium.host.canvas);
assert.equal((await f.run('set_layer', {})).engine, 'cesium');
f.voice.detachScene();
assert.deepEqual(f.adapter.stops, [], 'a detach never stops the adapter');
assert.equal(f.voice.commands.isActive(), true);
assert.equal(f.voice.commands.pointer.element, null);
// A call made between the engines waits for the next one.
const pending = f.run('set_layer', {});
const navara = fakeEngine('navara');
f.voice.attachScene({ engine: navara, viewer: null });
assert.equal((await pending).engine, 'navara');
assert.equal(f.voice.commands.pointer.element, navara.host.canvas);
assert.deepEqual(f.adapter.events, [
{ type: 'engine-changed', engine: 'navara' },
]);
assert.equal(f.adapter.starts, 1);
assert.equal(f.built.length, 2);
assert.equal(f.built[1].engine, navara);
f.lifetime.abort();
assert.equal(f.adapter.stops.at(-1)?.removeUi, true);
} finally {
f.restore();
}
});
test('annotation outline events reach the session from the attached engine only', async () => {
const f = fixture();
try {
const listeners = new Set();
const annotations = {
onOutlineEvent(listener) {
listeners.add(listener);
return () => listeners.delete(listener);
},
};
f.voice.attachScene({ engine: fakeEngine('cesium'), annotations });
await f.voice.commands.start();
for (const listener of listeners) listener({ status: 'resolved' });
assert.deepEqual(f.adapter.events, [
{ type: 'map_annotation_outline', status: 'resolved' },
]);
f.voice.detachScene();
assert.equal(listeners.size, 0);
} finally {
f.restore();
}
});
+4 -3
View File
@@ -6,10 +6,11 @@ import { VOICE_PROVIDERS } from './providerCapabilities.js';
import { t } from '../i18n/index.js';
function createDefaultReticle(pointer) {
const element = pointer?.element;
if (!element?.ownerDocument) return null;
// A pointer may follow another element later (an engine switch), so the
// reticle reads it each time; a page without a document has none.
if (!pointer || !globalThis.document) return null;
return new PointerReticle({
rectOf: () => element.getBoundingClientRect?.() || null,
rectOf: () => pointer.element?.getBoundingClientRect?.() || null,
});
}