Keep an active voice session through engine switches

The page creates the voice once; each engine attachment only gives it that scene's action runner, pointer canvas and annotation events. Calls made during a switch wait for the incoming engine and are refused with ENGINE_SWITCHING when none attaches in time; results name the engine that answered, and the session hears engine-changed after a switch. The analyst's follow-up memory and layer enable times outlive one engine's runner. scripts/qa-voice-engine-switch.mjs drives a fake microphone and Realtime transport through the switches.
This commit is contained in:
German Khokhlov committed 2026-11-03 17:20:40 +04:00
1 parent 35ed499ae4
commit 05578c61b9
13 files changed
+998 -139

No files matched your search

+139 -39
View File
@@ -5,6 +5,7 @@ import { createVoiceCommands } from './commands.js';
import { cameraPoseKey, createPointerTracker } from './pointerContext.js';
import { createPointerPicker } from './pointerPick.js';
import { createReferentRegistry } from './referents.js';
import { createSceneRunner } from './sceneRunner.js';
import { engineFromViewer } from '../globe/cesium/engineFromViewer.js';
export * from './realtimeController.js';
@@ -31,49 +32,148 @@ export function withToolCatalog(runner, loadCatalog) {
return runGevTool;
}
/**
* Voice for a page whose globe engine can change while it runs: the
* microphone, transport, controls and conversation belong to the page
* session; the action runner, the pointer's canvas and picker and the
* annotation events belong to one engine attachment (`attachScene`), and a
* switch only replaces those. Returns `{ commands, runner, attachScene,
* detachScene }`; `commands` is what initGevVoiceCommands returns.
*
* `isSwitching()` tells the runner that the page has started a switch, so a
* call made before the outgoing scene detaches waits for the next one too.
* `createRunner` replaces createGevActionRunner (tests).
*/
export function createSceneVoice(options) {
const scene = { engine: null };
const cameraKey = () => cameraPoseKey(scene.engine);
let pick = null;
// Point-and-ask: the tracker and referents are shared by the runner (which
// resolves 'pointer' and referent:n) and the controls (which snapshot the
// pointer at turn start and clear both when the session ends).
const pointer = createPointerTracker({
element: null,
cameraKey,
pick: (x, y) => pick?.(x, y) ?? null,
});
const referents = createReferentRegistry();
// The controller exists only after the runner, so its hooks bind late.
const deixis = { pointer, referents, cameraKey };
// The analyst's follow-up memory and the layer enable times outlive one
// engine's runner.
const analystMemory = { lastResult: null };
const layerEnabledAt = new Map();
let commands;
const runner = createSceneRunner({
isSwitching: options.isSwitching,
switchWaitMs: options.switchWaitMs,
build: (attached) =>
withToolCatalog(
(options.createRunner ?? createGevActionRunner)({
...options,
...attached,
deixis,
analystMemory,
layerEnabledAt,
readVoicePanelOperation: () =>
commands?.captureVoicePanelOperation?.(),
}),
options.toolCatalog,
),
});
const resetConversation = runner.resetConversation;
runner.resetConversation = () => {
analystMemory.lastResult = null;
resetConversation();
};
// Annotation outline events of whichever engine is attached.
const outlineListeners = new Set();
let offOutline = null;
const annotations = {
onOutlineEvent(listener) {
outlineListeners.add(listener);
return () => outlineListeners.delete(listener);
},
};
// The panel reads the cockpit of the attached shell.
let shell = null;
const styleManager = {
getCockpitState: () => shell?.getCockpitState?.() ?? null,
};
commands = createVoiceCommands({
...options,
styleManager,
annotations,
pointer,
referents,
runner,
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();
const lifetime = commands.signal ?? commands.session?.signal;
let attachedOnce = false;
const detachScene = () => {
runner.detach();
offOutline?.();
offOutline = null;
pointer.setElement(null);
pick = null;
scene.engine = null;
shell = null;
};
lifetime?.addEventListener(
'abort',
() => {
detachScene();
pointer.destroy();
},
{ once: true },
);
if (lifetime?.aborted) pointer.destroy();
return {
commands,
runner,
/**
* Attach the voice to one engine: `{ viewer, engine, styleManager,
* dataManager, sceneDirector, annotations }` (viewer and annotations
* null on an engine without them).
*/
attachScene(attached) {
if (lifetime?.aborted) return;
detachScene();
const engine = attached.engine ?? engineFromViewer(attached.viewer);
scene.engine = engine;
shell = attached.styleManager ?? null;
pick = createPointerPicker({
engine,
dataManager: attached.dataManager ?? options.dataManager,
});
pointer.setElement(engine?.host.canvas || null);
offOutline =
attached.annotations?.onOutlineEvent?.((event) => {
for (const listener of [...outlineListeners]) listener(event);
}) ?? null;
runner.attach({ ...attached, engine });
// A session that outlived the switch hears which globe it now drives.
if (attachedOnce && commands.isActive?.()) {
const event = { type: 'engine-changed', engine: engine?.kind ?? null };
if (typeof commands.notifyMapEvent === 'function')
commands.notifyMapEvent(event);
else commands.session?.sendMapEvent?.(event);
}
attachedOnce = true;
},
detachScene,
};
}
/**
* Compose the standalone action runner with the voice controls, on the globe
* `engine` (by default the one registered for `options.viewer`). Without a
* viewer (the Navara engine) the runner refuses the viewer-only tools.
*/
export function initGevVoiceCommands(options) {
const engine = options.engine ?? engineFromViewer(options.viewer);
const canvas = engine?.host.canvas || null;
const cameraKey = () => cameraPoseKey(engine);
// Point-and-ask: the tracker and referents are shared by the runner (which
// resolves 'pointer' and referent:n) and the controls (which snapshot the
// pointer at turn start and clear both when the session ends).
const pointer = canvas
? createPointerTracker({
element: canvas,
cameraKey,
pick: createPointerPicker({
engine,
dataManager: options.dataManager,
}),
})
: null;
const referents = createReferentRegistry();
// The controller exists only after the runner, so its hooks bind late.
const deixis = { pointer, referents, cameraKey };
let commands;
commands = createVoiceCommands({
...options,
pointer,
referents,
runner: withToolCatalog(
createGevActionRunner({
...options,
deixis,
readVoicePanelOperation: () => commands?.captureVoicePanelOperation?.(),
}),
options.toolCatalog,
),
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();
const lifetime = commands.signal ?? commands.session?.signal;
lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true });
if (lifetime?.aborted) pointer?.destroy();
return commands;
const voice = createSceneVoice(options);
voice.attachScene(options);
return voice.commands;
}