Keep an active voice session through engine switches

The page creates the voice once; each engine attachment only gives it that scene's action runner, pointer canvas and annotation events. Calls made during a switch wait for the incoming engine and are refused with ENGINE_SWITCHING when none attaches in time; results name the engine that answered, and the session hears engine-changed after a switch. The analyst's follow-up memory and layer enable times outlive one engine's runner. scripts/qa-voice-engine-switch.mjs drives a fake microphone and Realtime transport through the switches.
This commit is contained in:
German Khokhlov committed 2026-11-03 17:20:40 +04:00
1 parent 35ed499ae4
commit 05578c61b9
13 files changed
+998 -139

No files matched your search

+13 -3
View File
@@ -349,6 +349,8 @@ export function createGevActionRunner({
speechBuilders = null,
deixis = null,
readVoicePanelOperation = null,
analystMemory = null,
layerEnabledAt = null,
}) {
// Voice enable times and analyst follow-up memory belong to this runner.
// Speech builders turn results into say/display envelopes; callers extend
@@ -357,7 +359,10 @@ export function createGevActionRunner({
? { ...SPEECH_BUILDERS, ...speechBuilders }
: SPEECH_BUILDERS;
const engine = givenEngine ?? engineFromViewer(viewer);
const _layerEnabledAt = new Map();
// A host that replaces this runner when the globe engine changes passes
// the analyst's follow-up memory and the layer enable times, so the next
// runner keeps them.
const _layerEnabledAt = layerEnabledAt ?? new Map();
let analystEngine;
const resolveRegionRing = (name, signal) =>
annotationResolver.resolveRegionRingForQuery(name, signal, placeSearch);
@@ -368,6 +373,7 @@ export function createGevActionRunner({
resolveRegionRing,
isWarming: (layerKey) => layerIsWarming(_layerEnabledAt, layerKey),
}),
{ memory: analystMemory },
));
installViewTargetPrewarm(engine);
// The camera verbs (move_camera, fly_route) still run on the viewer.
@@ -471,7 +477,9 @@ export function createGevActionRunner({
};
if (
!viewer &&
((name === 'control_scene' && !sceneDirector) || VIEWER_ONLY_ACTIONS.includes(name) || CESIUM_LOOK_ACTIONS.includes(name))
((name === 'control_scene' && !sceneDirector) ||
VIEWER_ONLY_ACTIONS.includes(name) ||
CESIUM_LOOK_ACTIONS.includes(name))
)
return unavailableOnEngine(name, engine);
@@ -1077,7 +1085,9 @@ export function createGevActionRunner({
contextState?.mode === 'flights' &&
contextState?.active !== false &&
contextState?.changing !== true;
const directEntry = !targetLayer && !aircraftClass &&
const directEntry =
!targetLayer &&
!aircraftClass &&
styleManager.getCockpitState?.()?.entryAllowed === true;
if (!contactsReady && !directEntry) {
const contextResult = await styleManager.setContextMode('flights', {
+139 -39
View File
@@ -5,6 +5,7 @@ import { createVoiceCommands } from './commands.js';
import { cameraPoseKey, createPointerTracker } from './pointerContext.js';
import { createPointerPicker } from './pointerPick.js';
import { createReferentRegistry } from './referents.js';
import { createSceneRunner } from './sceneRunner.js';
import { engineFromViewer } from '../globe/cesium/engineFromViewer.js';
export * from './realtimeController.js';
@@ -31,49 +32,148 @@ export function withToolCatalog(runner, loadCatalog) {
return runGevTool;
}
/**
* Voice for a page whose globe engine can change while it runs: the
* microphone, transport, controls and conversation belong to the page
* session; the action runner, the pointer's canvas and picker and the
* annotation events belong to one engine attachment (`attachScene`), and a
* switch only replaces those. Returns `{ commands, runner, attachScene,
* detachScene }`; `commands` is what initGevVoiceCommands returns.
*
* `isSwitching()` tells the runner that the page has started a switch, so a
* call made before the outgoing scene detaches waits for the next one too.
* `createRunner` replaces createGevActionRunner (tests).
*/
export function createSceneVoice(options) {
const scene = { engine: null };
const cameraKey = () => cameraPoseKey(scene.engine);
let pick = null;
// Point-and-ask: the tracker and referents are shared by the runner (which
// resolves 'pointer' and referent:n) and the controls (which snapshot the
// pointer at turn start and clear both when the session ends).
const pointer = createPointerTracker({
element: null,
cameraKey,
pick: (x, y) => pick?.(x, y) ?? null,
});
const referents = createReferentRegistry();
// The controller exists only after the runner, so its hooks bind late.
const deixis = { pointer, referents, cameraKey };
// The analyst's follow-up memory and the layer enable times outlive one
// engine's runner.
const analystMemory = { lastResult: null };
const layerEnabledAt = new Map();
let commands;
const runner = createSceneRunner({
isSwitching: options.isSwitching,
switchWaitMs: options.switchWaitMs,
build: (attached) =>
withToolCatalog(
(options.createRunner ?? createGevActionRunner)({
...options,
...attached,
deixis,
analystMemory,
layerEnabledAt,
readVoicePanelOperation: () =>
commands?.captureVoicePanelOperation?.(),
}),
options.toolCatalog,
),
});
const resetConversation = runner.resetConversation;
runner.resetConversation = () => {
analystMemory.lastResult = null;
resetConversation();
};
// Annotation outline events of whichever engine is attached.
const outlineListeners = new Set();
let offOutline = null;
const annotations = {
onOutlineEvent(listener) {
outlineListeners.add(listener);
return () => outlineListeners.delete(listener);
},
};
// The panel reads the cockpit of the attached shell.
let shell = null;
const styleManager = {
getCockpitState: () => shell?.getCockpitState?.() ?? null,
};
commands = createVoiceCommands({
...options,
styleManager,
annotations,
pointer,
referents,
runner,
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();
const lifetime = commands.signal ?? commands.session?.signal;
let attachedOnce = false;
const detachScene = () => {
runner.detach();
offOutline?.();
offOutline = null;
pointer.setElement(null);
pick = null;
scene.engine = null;
shell = null;
};
lifetime?.addEventListener(
'abort',
() => {
detachScene();
pointer.destroy();
},
{ once: true },
);
if (lifetime?.aborted) pointer.destroy();
return {
commands,
runner,
/**
* Attach the voice to one engine: `{ viewer, engine, styleManager,
* dataManager, sceneDirector, annotations }` (viewer and annotations
* null on an engine without them).
*/
attachScene(attached) {
if (lifetime?.aborted) return;
detachScene();
const engine = attached.engine ?? engineFromViewer(attached.viewer);
scene.engine = engine;
shell = attached.styleManager ?? null;
pick = createPointerPicker({
engine,
dataManager: attached.dataManager ?? options.dataManager,
});
pointer.setElement(engine?.host.canvas || null);
offOutline =
attached.annotations?.onOutlineEvent?.((event) => {
for (const listener of [...outlineListeners]) listener(event);
}) ?? null;
runner.attach({ ...attached, engine });
// A session that outlived the switch hears which globe it now drives.
if (attachedOnce && commands.isActive?.()) {
const event = { type: 'engine-changed', engine: engine?.kind ?? null };
if (typeof commands.notifyMapEvent === 'function')
commands.notifyMapEvent(event);
else commands.session?.sendMapEvent?.(event);
}
attachedOnce = true;
},
detachScene,
};
}
/**
* Compose the standalone action runner with the voice controls, on the globe
* `engine` (by default the one registered for `options.viewer`). Without a
* viewer (the Navara engine) the runner refuses the viewer-only tools.
*/
export function initGevVoiceCommands(options) {
const engine = options.engine ?? engineFromViewer(options.viewer);
const canvas = engine?.host.canvas || null;
const cameraKey = () => cameraPoseKey(engine);
// Point-and-ask: the tracker and referents are shared by the runner (which
// resolves 'pointer' and referent:n) and the controls (which snapshot the
// pointer at turn start and clear both when the session ends).
const pointer = canvas
? createPointerTracker({
element: canvas,
cameraKey,
pick: createPointerPicker({
engine,
dataManager: options.dataManager,
}),
})
: null;
const referents = createReferentRegistry();
// The controller exists only after the runner, so its hooks bind late.
const deixis = { pointer, referents, cameraKey };
let commands;
commands = createVoiceCommands({
...options,
pointer,
referents,
runner: withToolCatalog(
createGevActionRunner({
...options,
deixis,
readVoicePanelOperation: () => commands?.captureVoicePanelOperation?.(),
}),
options.toolCatalog,
),
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();
const lifetime = commands.signal ?? commands.session?.signal;
lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true });
if (lifetime?.aborted) pointer?.destroy();
return commands;
const voice = createSceneVoice(options);
voice.attachScene(options);
return voice.commands;
}
+20 -3
View File
@@ -276,7 +276,8 @@ export function createPointerTracker({
let held = null;
let active = null;
let destroyed = false;
const listeners = new AbortController();
// The element's own listeners; replaced with the element (setElement).
let listeners = new AbortController();
function size() {
return {
@@ -328,12 +329,14 @@ export function createPointerTracker({
cancelDwell();
}
if (element?.addEventListener) {
function listen() {
if (!element?.addEventListener) return;
const options = { passive: true, signal: listeners.signal };
element.addEventListener('pointermove', onMove, options);
element.addEventListener('pointerdown', onMove, options);
element.addEventListener('pointerleave', onLeave, options);
}
listen();
/** Is the pointer a referent right now? */
function isFresh(recentMs = POINTER_FRESH_MS) {
@@ -372,7 +375,21 @@ export function createPointerTracker({
}
return {
element,
get element() {
return element;
},
/**
* Follow another element (the next globe's canvas after an engine
* switch), or none. The pointer starts outside it.
*/
setElement(next) {
if (destroyed || next === element) return;
listeners.abort();
listeners = new AbortController();
element = next ?? null;
onLeave();
listen();
},
capture,
isFresh,
/** Push-to-talk keydown: snapshot now, keep it for the hold. */
+158
View File
@@ -0,0 +1,158 @@
// The voice action runner for a page whose globe engine can change while a
// voice session runs. The session keeps one runner; each engine attachment
// builds the runner of that scene (createGevActionRunner over its viewer,
// engine, shell and annotations) and the forwarder sends every call to the
// current one. A call made while the engines switch waits for the next
// attachment, up to a bound, and then runs on the engine that attached; when
// none attaches in time it is refused with ENGINE_SWITCHING.
/** How long a call waits for the next engine before it is refused. */
export const ENGINE_SWITCH_WAIT_MS = 15_000;
/** The refusal for a call that found no engine within the wait. */
export function engineSwitchingRefusal(name) {
return {
ok: false,
action: name,
applied: false,
code: 'ENGINE_SWITCHING',
error: 'The globe is switching engines; this action was not run.',
say: 'The globe is switching engines. Ask again in a moment.',
};
}
/**
* @param {object} options
* @param {(scene: object) => Function} options.build The runner for one
* attached scene; its optional `dispose()` runs when the scene detaches.
* @param {() => boolean} [options.isSwitching] True while the page switches
* engines, even before the outgoing scene has detached.
* @param {number} [options.switchWaitMs]
*/
export function createSceneRunner({
build,
isSwitching = () => false,
switchWaitMs = ENGINE_SWITCH_WAIT_MS,
}) {
if (typeof build !== 'function')
throw new TypeError('A scene runner needs a build function');
let inner = null;
let kind = null;
let attachment = null;
// Bumped by every attachment, so a waiting call knows one happened.
let attachments = 0;
let conversation = 0;
let disposed = false;
const waiters = new Set();
const wake = () => {
for (const resolve of [...waiters]) resolve();
waiters.clear();
};
// Resolves true once a scene is ready (one attached since the wait began,
// or the switch settled with a scene), false when the wait is cut short.
async function waitForScene(signal, timeoutMs) {
const since = attachments;
const deadline = Date.now() + Math.max(0, timeoutMs);
for (;;) {
if (disposed || signal?.aborted) return false;
if (inner && (attachments !== since || !isSwitching())) return true;
const remaining = deadline - Date.now();
if (remaining <= 0) return false;
await new Promise((resolve) => {
let timer = null;
const done = () => {
clearTimeout(timer);
signal?.removeEventListener('abort', done);
waiters.delete(done);
resolve();
};
waiters.add(done);
signal?.addEventListener('abort', done, { once: true });
timer = setTimeout(done, remaining);
});
}
}
async function run(name, args, options = {}) {
if (disposed)
return {
ok: false,
action: name,
cancelled: true,
error: 'Voice ended.',
};
if (!inner || isSwitching()) {
const found = await waitForScene(options.signal, run.switchWaitMs);
if (disposed)
return {
ok: false,
action: name,
cancelled: true,
error: 'Voice ended.',
};
if (!found) return engineSwitchingRefusal(name);
}
const current = inner;
const currentKind = kind;
const epoch = conversation;
const signal = options.signal
? AbortSignal.any([options.signal, attachment.signal])
: attachment.signal;
const result = await current(name, args, {
...options,
signal,
isCurrent: () =>
epoch === conversation &&
!disposed &&
current === inner &&
(typeof options.isCurrent !== 'function' || options.isCurrent()),
});
// Say which engine answered: after a switch the model reads the result
// of the globe the person now sees.
return result &&
typeof result === 'object' &&
!Array.isArray(result) &&
result.engine === undefined
? { ...result, engine: currentKind }
: result;
}
run.switchWaitMs = switchWaitMs;
/** The engine kind of the attached scene, or null. */
run.engineKind = () => kind;
/** Build the runner for `scene`; waiting calls run on it. */
run.attach = (scene) => {
if (disposed) return;
run.detach();
attachment = new AbortController();
inner = build(scene);
kind = scene?.engine?.kind ?? (scene?.viewer ? 'cesium' : null);
attachments++;
wake();
};
/** Stop the attached runner: its calls in flight see their signal abort. */
run.detach = () => {
attachment?.abort();
attachment = null;
const previous = inner;
inner = null;
kind = null;
previous?.dispose?.();
};
/** The page's switch settled: waiting calls look again. */
run.wake = wake;
run.resetConversation = () => {
conversation++;
inner?.resetConversation?.();
};
run.dispose = () => {
if (disposed) return;
run.resetConversation();
disposed = true;
run.detach();
wake();
};
return run;
}
+107
View File
@@ -0,0 +1,107 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createSceneRunner, engineSwitchingRefusal } from './sceneRunner.js';
const engine = (kind) => ({ kind });
function runnerFor(log) {
return (scene) => {
const run = async (name, args, options) => {
log.push({ name, kind: scene.engine.kind, signal: options.signal });
return { ok: true, action: name };
};
run.dispose = () => log.push({ disposed: scene.engine.kind });
run.resetConversation = () => log.push({ reset: scene.engine.kind });
return run;
};
}
test('calls run on the attached engine and say which engine answered', async () => {
const log = [];
const run = createSceneRunner({ build: runnerFor(log) });
run.attach({ engine: engine('cesium') });
assert.deepEqual(await run('set_layer', {}), {
ok: true,
action: 'set_layer',
engine: 'cesium',
});
run.attach({ engine: engine('navara') });
assert.equal((await run('set_layer', {})).engine, 'navara');
assert.deepEqual(log[1], { disposed: 'cesium' });
assert.equal(run.engineKind(), 'navara');
});
test('a call during a switch waits for the next engine and runs there', async () => {
const log = [];
let switching = false;
const run = createSceneRunner({
build: runnerFor(log),
isSwitching: () => switching,
});
run.attach({ engine: engine('cesium') });
switching = true;
const pending = run('set_layer', {});
run.detach();
await new Promise((resolve) => setTimeout(resolve, 5));
assert.equal(log.filter((entry) => entry.name).length, 0);
run.attach({ engine: engine('navara') });
const result = await pending;
assert.equal(result.engine, 'navara');
switching = false;
});
test('a call refused when no engine attaches in time names ENGINE_SWITCHING', async () => {
const run = createSceneRunner({
build: runnerFor([]),
isSwitching: () => true,
switchWaitMs: 10,
});
run.attach({ engine: engine('cesium') });
run.detach();
const result = await run('annotate_map', {});
assert.deepEqual(result, engineSwitchingRefusal('annotate_map'));
assert.equal(result.code, 'ENGINE_SWITCHING');
});
test('a switch that settles without a new engine lets waiting calls run', async () => {
const log = [];
let switching = true;
const run = createSceneRunner({
build: runnerFor(log),
isSwitching: () => switching,
});
run.attach({ engine: engine('cesium') });
const pending = run('set_layer', {});
switching = false;
run.wake();
assert.equal((await pending).engine, 'cesium');
});
test('the attachment signal aborts calls in flight when the engine detaches', async () => {
let seen;
const run = createSceneRunner({
build: () => async (name, args, options) => {
seen = options.signal;
await new Promise((resolve) => setTimeout(resolve, 5));
return { ok: !options.signal.aborted };
},
});
run.attach({ engine: engine('cesium') });
const pending = run('fly_to_location', {});
await new Promise((resolve) => setTimeout(resolve, 0));
run.detach();
assert.equal(seen.aborted, true);
assert.equal((await pending).ok, false);
});
test('a disposed runner refuses and wakes its waiting calls', async () => {
const run = createSceneRunner({
build: runnerFor([]),
isSwitching: () => true,
});
const pending = run('set_layer', {});
run.dispose();
const result = await pending;
assert.equal(result.cancelled, true);
assert.equal((await run('set_layer', {})).cancelled, true);
});
+117
View File
@@ -0,0 +1,117 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createSceneVoice } from './gevRealtime.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
function fakeEngine(kind) {
const canvas = new EventTarget();
canvas.getBoundingClientRect = () => ({ left: 0, top: 0 });
return {
kind,
host: { canvas, size: () => ({ width: 0, height: 0 }) },
camera: { pose: () => ({}) },
};
}
function fixture() {
const previous = globalThis.window;
globalThis.window = {};
const lifetime = new AbortController();
const adapter = { starts: 0, stops: [], events: [] };
let runAction;
const built = [];
const voice = createSceneVoice({
signal: lifetime.signal,
createControl: () => createTestVoiceUi(),
createCard: () => null,
createReticle: () => null,
createRunner: (scene) => {
built.push(scene);
return async (name) => ({ ok: true, action: name });
},
createSession(hooks) {
runAction = hooks.runAction;
return {
async start() {
adapter.starts++;
hooks.emit({ type: 'state', state: 'listening' });
},
stop(options) {
adapter.stops.push(options);
},
sendText() {},
sendMapEvent(event) {
adapter.events.push(event);
return true;
},
};
},
});
return {
voice,
adapter,
built,
lifetime,
run: (...args) => runAction(...args),
restore: () => {
globalThis.window = previous;
},
};
}
test('an engine switch keeps the voice session and its transport', async () => {
const f = fixture();
try {
const cesium = fakeEngine('cesium');
f.voice.attachScene({ engine: cesium, viewer: {} });
await f.voice.commands.start();
assert.equal(f.adapter.starts, 1);
assert.equal(f.voice.commands.pointer.element, cesium.host.canvas);
assert.equal((await f.run('set_layer', {})).engine, 'cesium');
f.voice.detachScene();
assert.deepEqual(f.adapter.stops, [], 'a detach never stops the adapter');
assert.equal(f.voice.commands.isActive(), true);
assert.equal(f.voice.commands.pointer.element, null);
// A call made between the engines waits for the next one.
const pending = f.run('set_layer', {});
const navara = fakeEngine('navara');
f.voice.attachScene({ engine: navara, viewer: null });
assert.equal((await pending).engine, 'navara');
assert.equal(f.voice.commands.pointer.element, navara.host.canvas);
assert.deepEqual(f.adapter.events, [
{ type: 'engine-changed', engine: 'navara' },
]);
assert.equal(f.adapter.starts, 1);
assert.equal(f.built.length, 2);
assert.equal(f.built[1].engine, navara);
f.lifetime.abort();
assert.equal(f.adapter.stops.at(-1)?.removeUi, true);
} finally {
f.restore();
}
});
test('annotation outline events reach the session from the attached engine only', async () => {
const f = fixture();
try {
const listeners = new Set();
const annotations = {
onOutlineEvent(listener) {
listeners.add(listener);
return () => listeners.delete(listener);
},
};
f.voice.attachScene({ engine: fakeEngine('cesium'), annotations });
await f.voice.commands.start();
for (const listener of listeners) listener({ status: 'resolved' });
assert.deepEqual(f.adapter.events, [
{ type: 'map_annotation_outline', status: 'resolved' },
]);
f.voice.detachScene();
assert.equal(listeners.size, 0);
} finally {
f.restore();
}
});
+4 -3
View File
@@ -6,10 +6,11 @@ import { VOICE_PROVIDERS } from './providerCapabilities.js';
import { t } from '../i18n/index.js';
function createDefaultReticle(pointer) {
const element = pointer?.element;
if (!element?.ownerDocument) return null;
// A pointer may follow another element later (an engine switch), so the
// reticle reads it each time; a page without a document has none.
if (!pointer || !globalThis.document) return null;
return new PointerReticle({
rectOf: () => element.getBoundingClientRect?.() || null,
rectOf: () => pointer.element?.getBoundingClientRect?.() || null,
});
}