mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-06 03:14:00 +00:00
Keep an active voice session through engine switches
The page creates the voice once; each engine attachment only gives it that scene's action runner, pointer canvas and annotation events. Calls made during a switch wait for the incoming engine and are refused with ENGINE_SWITCHING when none attaches in time; results name the engine that answered, and the session hears engine-changed after a switch. The analyst's follow-up memory and layer enable times outlive one engine's runner. scripts/qa-voice-engine-switch.mjs drives a fake microphone and Realtime transport through the switches.
This commit is contained in:
1 parent
35ed499ae4
commit
05578c61b9
13 files changed
+998
-139
No files matched your search
@@ -0,0 +1,247 @@
|
||||
// An active voice session survives engine switches. With a fake microphone
|
||||
// and a fake Realtime transport (no model is reached: /api/openai, /api/realtime
|
||||
// and api.openai.com are blocked or answered in the page), this gate starts
|
||||
// voice on Cesium, switches Cesium→Navara→Cesium→Navara and checks that:
|
||||
// - the microphone was requested once, one peer connection was made, and
|
||||
// neither it nor its data channel closed; no WebSocket opened;
|
||||
// - every session object kept its identity and each switch attached once;
|
||||
// - a set_layer_visibility call after each switch ran on the engine shown;
|
||||
// - a call made during a switch waited and ran on the incoming engine, and
|
||||
// one whose wait ran out was refused with ENGINE_SWITCHING;
|
||||
// - annotate_map works on Cesium, is refused on Navara, and the mark drawn
|
||||
// before the switches is still on the board back on Cesium;
|
||||
// - the session heard which engine it drives after each switch.
|
||||
//
|
||||
// node scripts/qa-voice-engine-switch.mjs http://127.0.0.1:<port>/
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
idDifference,
|
||||
launch,
|
||||
openSessionPage,
|
||||
previewBase,
|
||||
readSession,
|
||||
readVoiceEvidence,
|
||||
requestEngine,
|
||||
sessionAddress,
|
||||
switchEngine,
|
||||
waitReady,
|
||||
} from './qa/appSessionHarness.mjs';
|
||||
|
||||
const base = previewBase(process.argv[2]);
|
||||
|
||||
// Play one server function call on the fake data channel and wait for the
|
||||
// client's function_call_output.
|
||||
function callTool(page, name, args, { beforeCall = null } = {}) {
|
||||
return page.evaluate(
|
||||
async (name, args, beforeCall) => {
|
||||
const evidence = window.__qaVoice;
|
||||
const callId = `qa-call-${(window.__qaCalls = (window.__qaCalls ?? 0) + 1)}`;
|
||||
const responseId = `qa-response-${callId}`;
|
||||
const deliver = (event) =>
|
||||
evidence.channel.dispatchEvent(
|
||||
new MessageEvent('message', { data: JSON.stringify(event) }),
|
||||
);
|
||||
if (beforeCall === 'switch-navara' || beforeCall === 'switch-cesium') {
|
||||
const select = document.querySelector('#globe-engine');
|
||||
select.value = beforeCall.slice('switch-'.length);
|
||||
select.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
}
|
||||
deliver({ type: 'response.created', response: { id: responseId } });
|
||||
deliver({
|
||||
type: 'response.function_call_arguments.done',
|
||||
response_id: responseId,
|
||||
item_id: `item-${callId}`,
|
||||
call_id: callId,
|
||||
name,
|
||||
arguments: JSON.stringify(args),
|
||||
});
|
||||
const started = performance.now();
|
||||
for (;;) {
|
||||
const sent = evidence.sent.find(
|
||||
(event) =>
|
||||
event?.type === 'conversation.item.create' &&
|
||||
event.item?.type === 'function_call_output' &&
|
||||
event.item.call_id === callId,
|
||||
);
|
||||
if (sent) {
|
||||
deliver({
|
||||
type: 'response.done',
|
||||
response: { id: responseId, status: 'completed', output: [] },
|
||||
});
|
||||
return JSON.parse(sent.item.output);
|
||||
}
|
||||
if (performance.now() - started > 60_000)
|
||||
throw new Error(`No output for ${name}`);
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
},
|
||||
name,
|
||||
args,
|
||||
beforeCall,
|
||||
);
|
||||
}
|
||||
|
||||
const engineChanges = (page) =>
|
||||
page.evaluate(() =>
|
||||
window.__qaVoice.sent
|
||||
.filter(
|
||||
(event) =>
|
||||
event?.type === 'conversation.item.create' &&
|
||||
event.item?.role === 'system',
|
||||
)
|
||||
.map((event) => {
|
||||
try {
|
||||
return JSON.parse(event.item.content[0].text);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((event) => event?.type === 'engine-changed')
|
||||
.map((event) => event.engine),
|
||||
);
|
||||
|
||||
const ANNOTATION = {
|
||||
annotations: [
|
||||
{
|
||||
type: 'pin',
|
||||
latitude: 42.3554,
|
||||
longitude: -71.0605,
|
||||
label: 'QA voice mark',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const browser = await launch();
|
||||
const results = [];
|
||||
try {
|
||||
const { page, errors } = await openSessionPage(
|
||||
browser,
|
||||
sessionAddress(base, { engine: 'cesium' }),
|
||||
);
|
||||
await waitReady(page, 'cesium');
|
||||
await page.evaluate(async () => {
|
||||
const close = document.querySelector('[data-key-setup-close]');
|
||||
if (close?.offsetParent) close.click();
|
||||
await window.__godsEyeView.voiceCommands.start({ pushToTalk: false });
|
||||
});
|
||||
await page.waitForFunction(
|
||||
() =>
|
||||
window.__qaVoice.channel?.readyState === 'open' &&
|
||||
window.__godsEyeView.voiceCommands.state === 'listening',
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
const started = await readVoiceEvidence(page);
|
||||
const boot = await readSession(page);
|
||||
assert.equal(started.getUserMedia, 1, 'voice asks for the microphone once');
|
||||
assert.equal(started.peerConnections, 1);
|
||||
|
||||
const annotated = await callTool(page, 'annotate_map', ANNOTATION);
|
||||
assert.equal(annotated.ok, true, JSON.stringify(annotated));
|
||||
assert.equal(annotated.engine, 'cesium');
|
||||
const marks = await page.evaluate(() => window.__gevAnnotations.count());
|
||||
assert.ok(marks >= 1, 'the mark is on the board');
|
||||
results.push('PASS annotate_map drew a mark on Cesium');
|
||||
|
||||
const sequence = ['navara', 'cesium', 'navara'];
|
||||
for (const kind of sequence) {
|
||||
await switchEngine(page, kind);
|
||||
const layer = await callTool(page, 'set_layer_visibility', {
|
||||
layerId: 'earthquakes',
|
||||
enabled: kind === 'navara',
|
||||
});
|
||||
assert.equal(layer.engine, kind, JSON.stringify(layer));
|
||||
assert.equal(
|
||||
await page.evaluate(() => window.__godsEyeView.engine.kind),
|
||||
kind,
|
||||
);
|
||||
if (kind === 'navara') {
|
||||
const refused = await callTool(page, 'annotate_map', ANNOTATION);
|
||||
assert.equal(refused.ok, false, JSON.stringify(refused));
|
||||
assert.equal(refused.engine, 'navara');
|
||||
} else {
|
||||
const count = await page.evaluate(() => window.__gevAnnotations.count());
|
||||
assert.equal(count, marks, 'the mark survived the round trip');
|
||||
}
|
||||
results.push(`PASS ${kind}: set_layer_visibility ran on ${layer.engine}`);
|
||||
}
|
||||
|
||||
// A call made during a switch waits for the incoming engine.
|
||||
const midSwitch = await callTool(
|
||||
page,
|
||||
'set_layer_visibility',
|
||||
{ layerId: 'earthquakes', enabled: true },
|
||||
{ beforeCall: 'switch-cesium' },
|
||||
);
|
||||
await waitReady(page, 'cesium');
|
||||
assert.equal(midSwitch.ok, true, JSON.stringify(midSwitch));
|
||||
assert.equal(midSwitch.engine, 'cesium', JSON.stringify(midSwitch));
|
||||
results.push('PASS a call made during the switch ran on the incoming engine');
|
||||
assert.equal(
|
||||
await page.evaluate(() => window.__gevAnnotations.count()),
|
||||
marks,
|
||||
'the mark is still on the board after Cesium→Navara→Cesium→Navara→Cesium',
|
||||
);
|
||||
|
||||
// A call whose wait runs out is refused with a clear code.
|
||||
await page.evaluate(() => {
|
||||
window.__godsEyeView.voiceCommands.runner.switchWaitMs = 1;
|
||||
});
|
||||
const refused = await callTool(
|
||||
page,
|
||||
'set_layer_visibility',
|
||||
{ layerId: 'earthquakes', enabled: false },
|
||||
{ beforeCall: 'switch-navara' },
|
||||
);
|
||||
await waitReady(page, 'navara');
|
||||
await page.evaluate(() => {
|
||||
window.__godsEyeView.voiceCommands.runner.switchWaitMs = 15_000;
|
||||
});
|
||||
assert.equal(refused.ok, false, JSON.stringify(refused));
|
||||
assert.equal(refused.code, 'ENGINE_SWITCHING', JSON.stringify(refused));
|
||||
results.push(
|
||||
'PASS a call whose wait ran out was refused with ENGINE_SWITCHING',
|
||||
);
|
||||
|
||||
await switchEngine(page, 'cesium');
|
||||
const after = await readVoiceEvidence(page);
|
||||
const state = await readSession(page);
|
||||
const changes = await engineChanges(page);
|
||||
const switches = sequence.length + 3;
|
||||
const report = {
|
||||
started,
|
||||
after,
|
||||
ids: idDifference(boot.ids, state.ids),
|
||||
counters: state.counters,
|
||||
changes,
|
||||
voiceState: await page.evaluate(
|
||||
() => window.__godsEyeView.voiceCommands.state,
|
||||
),
|
||||
};
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
assert.equal(after.getUserMedia, 1, 'no new microphone request');
|
||||
assert.equal(after.peerConnections, 1, 'no new peer connection');
|
||||
assert.equal(after.peerConnectionsClosed, 0, 'the connection stayed open');
|
||||
assert.equal(after.channelsClosed, 0, 'the data channel stayed open');
|
||||
assert.equal(after.channelState, 'open');
|
||||
assert.equal(after.webSockets, started.webSockets, 'no new WebSocket');
|
||||
assert.equal(after.tokenRequests, 1, 'no new token');
|
||||
assert.equal(report.voiceState, 'listening');
|
||||
assert.deepEqual(report.ids, {}, 'session objects kept their identity');
|
||||
assert.equal(
|
||||
state.counters.attachments,
|
||||
boot.counters.attachments + switches,
|
||||
);
|
||||
assert.equal(changes.length, switches, JSON.stringify(changes));
|
||||
assert.equal(changes.at(-1), 'cesium');
|
||||
assert.deepEqual(errors, []);
|
||||
results.push('PASS the voice session kept its microphone and transport');
|
||||
|
||||
await page.evaluate(() => window.__godsEyeView.voiceCommands.stop());
|
||||
await page.waitForFunction(
|
||||
() => window.__qaVoice.peerConnectionsClosed === 1,
|
||||
);
|
||||
results.push('PASS stopping voice closes the transport');
|
||||
console.log(results.join('\n'));
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
+100
-53
@@ -17,7 +17,7 @@ import {
|
||||
installRenderGovernor,
|
||||
uninstallRenderGovernor,
|
||||
} from '../renderGovernor.js';
|
||||
import { initGevVoiceCommands } from '../voice/gevRealtime.js';
|
||||
import { attachSessionVoice } from './sessionVoice.js';
|
||||
import { initNavaraImageryBoxTool } from '../ui/imageryBoxNavara.js';
|
||||
import { createRecentImageryPanel } from '../ui/recentImagery.js';
|
||||
|
||||
@@ -38,7 +38,9 @@ export function navaraModeNotice(availability) {
|
||||
const names = availability.available().map(({ name }) => name);
|
||||
return Object.freeze({
|
||||
label: t('Experimental Navara engine'),
|
||||
detail: names.length ? `Data layers: ${names.join(', ')}` : 'No registered data layers',
|
||||
detail: names.length
|
||||
? `Data layers: ${names.join(', ')}`
|
||||
: 'No registered data layers',
|
||||
});
|
||||
}
|
||||
|
||||
@@ -112,56 +114,97 @@ export async function registerNavaraLayerRenderers() {
|
||||
bikeshare: async () =>
|
||||
(await import('../layers/bikeshare/render/navara.js'))
|
||||
.createBikeshareNavaraRenderer,
|
||||
...Object.fromEntries(['weather-radar', 'weather-satellite', 'weather-lightning'].map((id) => [
|
||||
id, async () => {
|
||||
const { createWeatherNavaraRenderer } = await import('../layers/weather/render/navara.js');
|
||||
return (engine, options) => createWeatherNavaraRenderer(engine, navaraViews.get(engine), options);
|
||||
},
|
||||
])),
|
||||
...Object.fromEntries(
|
||||
['weather-radar', 'weather-satellite', 'weather-lightning'].map((id) => [
|
||||
id,
|
||||
async () => {
|
||||
const { createWeatherNavaraRenderer } =
|
||||
await import('../layers/weather/render/navara.js');
|
||||
return (engine, options) =>
|
||||
createWeatherNavaraRenderer(
|
||||
engine,
|
||||
navaraViews.get(engine),
|
||||
options,
|
||||
);
|
||||
},
|
||||
]),
|
||||
),
|
||||
'bhote-koshi-2026': async () => {
|
||||
const { createNepalEventDisplay } = await import('../layers/bhoteKoshi/render/navara.js');
|
||||
return (engine) => createNepalEventDisplay(engine, navaraViews.get(engine));
|
||||
const { createNepalEventDisplay } =
|
||||
await import('../layers/bhoteKoshi/render/navara.js');
|
||||
return (engine) =>
|
||||
createNepalEventDisplay(engine, navaraViews.get(engine));
|
||||
},
|
||||
'bhote-koshi-locator': async () => {
|
||||
const { drawStorySource } = await import('../layers/shared/render/navaraStory.js');
|
||||
return (engine, { data }) => drawStorySource(engine, navaraViews.get(engine), data);
|
||||
const { drawStorySource } =
|
||||
await import('../layers/shared/render/navaraStory.js');
|
||||
return (engine, { data }) =>
|
||||
drawStorySource(engine, navaraViews.get(engine), data);
|
||||
},
|
||||
'street-level': async () => {
|
||||
const { createStreetMap } = await import('../layers/streetLevel/render/navara.js');
|
||||
return engine => createStreetMap(engine, navaraViews.get(engine));
|
||||
const { createStreetMap } =
|
||||
await import('../layers/streetLevel/render/navara.js');
|
||||
return (engine) => createStreetMap(engine, navaraViews.get(engine));
|
||||
},
|
||||
traffic: async () => {
|
||||
const { createTrafficNavaraRenderer } = await import('../layers/traffic/render/navara.js');
|
||||
return engine => createTrafficNavaraRenderer(engine, navaraViews.get(engine));
|
||||
const { createTrafficNavaraRenderer } =
|
||||
await import('../layers/traffic/render/navara.js');
|
||||
return (engine) =>
|
||||
createTrafficNavaraRenderer(engine, navaraViews.get(engine));
|
||||
},
|
||||
directions: async () => {
|
||||
const { createDirectionsNavaraRenderer } = await import('../layers/directions/render/navara.js');
|
||||
return (engine) => createDirectionsNavaraRenderer(engine, navaraViews.get(engine));
|
||||
const { createDirectionsNavaraRenderer } =
|
||||
await import('../layers/directions/render/navara.js');
|
||||
return (engine) =>
|
||||
createDirectionsNavaraRenderer(engine, navaraViews.get(engine));
|
||||
},
|
||||
'telegeography-submarine-cables': async () => {
|
||||
const { createNavaraCables } = await import('../layers/submarineCables/render/navara.js');
|
||||
return (engine, { overlayHost }) => createNavaraCables(engine, navaraViews.get(engine), overlayHost);
|
||||
const { createNavaraCables } =
|
||||
await import('../layers/submarineCables/render/navara.js');
|
||||
return (engine, { overlayHost }) =>
|
||||
createNavaraCables(engine, navaraViews.get(engine), overlayHost);
|
||||
},
|
||||
wind: async () => {
|
||||
const { createWindNavaraRenderer } = await import('../layers/wind/render/navara.js');
|
||||
return (engine) => createWindNavaraRenderer(engine, navaraViews.get(engine));
|
||||
const { createWindNavaraRenderer } =
|
||||
await import('../layers/wind/render/navara.js');
|
||||
return (engine) =>
|
||||
createWindNavaraRenderer(engine, navaraViews.get(engine));
|
||||
},
|
||||
'weather-cyclones': async () => {
|
||||
const {createCyclonesNavaraRenderer} = await import('../layers/cyclones/render/navara.js');
|
||||
return (engine,options) => createCyclonesNavaraRenderer(engine,navaraViews.get(engine).view,options);
|
||||
const { createCyclonesNavaraRenderer } =
|
||||
await import('../layers/cyclones/render/navara.js');
|
||||
return (engine, options) =>
|
||||
createCyclonesNavaraRenderer(
|
||||
engine,
|
||||
navaraViews.get(engine).view,
|
||||
options,
|
||||
);
|
||||
},
|
||||
...Object.fromEntries(['local-datacenters','local-dams'].map((id) => [
|
||||
id, async () => {
|
||||
const {createInfrastructureNavaraRenderer} = await import('../layers/infrastructure/render/navara.js');
|
||||
return (engine,{config,services}) => createInfrastructureNavaraRenderer(engine,navaraViews.get(engine),config,services);
|
||||
},
|
||||
])),
|
||||
...Object.fromEntries(
|
||||
['local-datacenters', 'local-dams'].map((id) => [
|
||||
id,
|
||||
async () => {
|
||||
const { createInfrastructureNavaraRenderer } =
|
||||
await import('../layers/infrastructure/render/navara.js');
|
||||
return (engine, { config, services }) =>
|
||||
createInfrastructureNavaraRenderer(
|
||||
engine,
|
||||
navaraViews.get(engine),
|
||||
config,
|
||||
services,
|
||||
);
|
||||
},
|
||||
]),
|
||||
),
|
||||
's2-ghostmaps': async () => {
|
||||
const { createGhostMapsNavaraRenderer } = await import('../layers/ghostMaps/render/navara.js');
|
||||
return (engine) => createGhostMapsNavaraRenderer(engine, navaraViews.get(engine).view);
|
||||
const { createGhostMapsNavaraRenderer } =
|
||||
await import('../layers/ghostMaps/render/navara.js');
|
||||
return (engine) =>
|
||||
createGhostMapsNavaraRenderer(engine, navaraViews.get(engine).view);
|
||||
},
|
||||
'military-awareness': async () =>
|
||||
(await import('../layers/awareness/render/navara.js')).createAwarenessNavaraRenderer,
|
||||
(await import('../layers/awareness/render/navara.js'))
|
||||
.createAwarenessNavaraRenderer,
|
||||
'military-installations': async () => {
|
||||
const { createInstallationsNavaraRenderer } =
|
||||
await import('../layers/installations/render/navara.js');
|
||||
@@ -181,17 +224,22 @@ export async function registerNavaraLayerRenderers() {
|
||||
});
|
||||
},
|
||||
'fire-perimeters': async () => {
|
||||
const { createPerimetersNavaraRenderer } = await import('../layers/perimeters/render/navara.js');
|
||||
return (engine) => createPerimetersNavaraRenderer(engine, navaraViews.get(engine));
|
||||
const { createPerimetersNavaraRenderer } =
|
||||
await import('../layers/perimeters/render/navara.js');
|
||||
return (engine) =>
|
||||
createPerimetersNavaraRenderer(engine, navaraViews.get(engine));
|
||||
},
|
||||
'gnss-integrity': async () => {
|
||||
const { createGnssNavaraRenderer } = await import('../layers/gnss/render/navara.js');
|
||||
return (engine) => createGnssNavaraRenderer(engine, navaraViews.get(engine));
|
||||
const { createGnssNavaraRenderer } =
|
||||
await import('../layers/gnss/render/navara.js');
|
||||
return (engine) =>
|
||||
createGnssNavaraRenderer(engine, navaraViews.get(engine));
|
||||
},
|
||||
'air-quality': async () => {
|
||||
const { createAirQualityNavaraRenderer } =
|
||||
await import('../layers/airQuality/render/navara.js');
|
||||
return (engine) => createAirQualityNavaraRenderer(engine, navaraViews.get(engine));
|
||||
return (engine) =>
|
||||
createAirQualityNavaraRenderer(engine, navaraViews.get(engine));
|
||||
},
|
||||
earthquakes: async () => {
|
||||
const { createEarthquakesNavaraRenderer } =
|
||||
@@ -355,6 +403,7 @@ export function createViewerlessTools({
|
||||
voice = {},
|
||||
startChrome,
|
||||
sceneDirector,
|
||||
layerSession = null,
|
||||
signal,
|
||||
defer,
|
||||
}) {
|
||||
@@ -442,23 +491,21 @@ export function createViewerlessTools({
|
||||
});
|
||||
// Voice on the engine (phase 3, Task 9): the W1 tools drive the engine;
|
||||
// the Director uses the shared session; viewer-only tools declare availability.
|
||||
const voiceCommands = initGevVoiceCommands({
|
||||
...voice,
|
||||
annotationResolver: operations.annotationResolver,
|
||||
searchNavigation: operations.searchAndFlyTo,
|
||||
signal,
|
||||
const voiceCommands = attachSessionVoice({
|
||||
layerSession,
|
||||
voice,
|
||||
operations,
|
||||
placeSearch,
|
||||
viewer: null,
|
||||
engine,
|
||||
styleManager,
|
||||
dataManager,
|
||||
sceneDirector,
|
||||
annotations: null,
|
||||
});
|
||||
defer(() => {
|
||||
voiceCommands.stop({ removeUi: true });
|
||||
if (window.__gevVoiceCommands === voiceCommands)
|
||||
delete window.__gevVoiceCommands;
|
||||
scene: {
|
||||
viewer: null,
|
||||
engine,
|
||||
styleManager,
|
||||
sceneDirector,
|
||||
annotations: null,
|
||||
},
|
||||
signal,
|
||||
defer,
|
||||
});
|
||||
debug.voiceCommands = voiceCommands;
|
||||
return { sceneDirector, annotations: null, voiceCommands };
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import { createSceneVoice } from '../voice/gevRealtime.js';
|
||||
|
||||
/**
|
||||
* Attach the page's voice to this engine's scene. With a layer session the
|
||||
* voice is created once, for the page lifetime, and every engine attachment
|
||||
* only gives it the scene (runner, pointer canvas, annotation events): an
|
||||
* active voice session keeps its microphone and transport through a switch.
|
||||
* Without one (a host that never switches) the voice lives with this
|
||||
* application. Returns the voice commands.
|
||||
*/
|
||||
export function attachSessionVoice({
|
||||
layerSession = null,
|
||||
voice = {},
|
||||
operations,
|
||||
placeSearch,
|
||||
dataManager,
|
||||
scene,
|
||||
signal,
|
||||
defer,
|
||||
}) {
|
||||
const options = {
|
||||
...voice,
|
||||
floorServices: operations.surface?.groundFloor,
|
||||
annotationResolver: operations.annotationResolver,
|
||||
searchNavigation: operations.searchAndFlyTo,
|
||||
placeSearch,
|
||||
dataManager,
|
||||
};
|
||||
let sceneVoice;
|
||||
if (layerSession) {
|
||||
sceneVoice = layerSession.voice ??= createSceneVoice({
|
||||
...options,
|
||||
signal: layerSession.lifetime.signal,
|
||||
isSwitching: () => layerSession.switching?.() === true,
|
||||
});
|
||||
} else {
|
||||
sceneVoice = createSceneVoice({ ...options, signal });
|
||||
defer(() => {
|
||||
sceneVoice.commands.stop({ removeUi: true });
|
||||
if (window.__gevVoiceCommands === sceneVoice.commands)
|
||||
delete window.__gevVoiceCommands;
|
||||
});
|
||||
}
|
||||
sceneVoice.attachScene({ ...scene, dataManager });
|
||||
defer(() => sceneVoice.detachScene());
|
||||
return sceneVoice.commands;
|
||||
}
|
||||
+18
-21
@@ -6,7 +6,7 @@ import { useDisplayStore } from '../ui/stores/display.js';
|
||||
import { applicationPinia } from '../ui/vueRuntime.js';
|
||||
import { initImageryBoxTool } from '../ui/imageryBoxTool.js';
|
||||
import { createRecentImageryPanel } from '../ui/recentImagery.js';
|
||||
import { initGevVoiceCommands } from '../voice/gevRealtime.js';
|
||||
import { attachSessionVoice } from './sessionVoice.js';
|
||||
import { installViews, isEmbeddedInline } from './embed.js';
|
||||
import { installScopeMask, destroyScopeMask } from '../scopeMask.js';
|
||||
import { installLayerReleaseTrim } from '../sceneCommandRelease.js';
|
||||
@@ -42,10 +42,15 @@ export function createApplicationTools({
|
||||
const { viewer, engine, tileset, mapStackController, operations } = scene;
|
||||
const { styleManager, weatherEffects, cockpitCloudEffects } = controls;
|
||||
const { dataManager } = data;
|
||||
const isMapStackAvailable = id => mapStackController?.isStackAvailable(id) === true;
|
||||
const sceneDirector = layerSession?.director ?? new SceneDirector(viewer, styleManager, dataManager, {
|
||||
engine, dataPacks: {...sceneDataPacks,native:scene.packRenderer}, isMapStackAvailable,
|
||||
});
|
||||
const isMapStackAvailable = (id) =>
|
||||
mapStackController?.isStackAvailable(id) === true;
|
||||
const sceneDirector =
|
||||
layerSession?.director ??
|
||||
new SceneDirector(viewer, styleManager, dataManager, {
|
||||
engine,
|
||||
dataPacks: { ...sceneDataPacks, native: scene.packRenderer },
|
||||
isMapStackAvailable,
|
||||
});
|
||||
if (layerSession?.director)
|
||||
sceneDirector.attachScene(
|
||||
viewer,
|
||||
@@ -79,6 +84,7 @@ export function createApplicationTools({
|
||||
voice,
|
||||
sceneDirector,
|
||||
startChrome,
|
||||
layerSession,
|
||||
signal,
|
||||
defer,
|
||||
}),
|
||||
@@ -239,24 +245,15 @@ export function createApplicationTools({
|
||||
defer(() => {
|
||||
if (window.__godsEyeView === debug) delete window.__godsEyeView;
|
||||
});
|
||||
const voiceCommands = initGevVoiceCommands({
|
||||
...voice,
|
||||
floorServices: operations.surface.groundFloor,
|
||||
annotationResolver: operations.annotationResolver,
|
||||
searchNavigation: operations.searchAndFlyTo,
|
||||
signal,
|
||||
const voiceCommands = attachSessionVoice({
|
||||
layerSession,
|
||||
voice,
|
||||
operations,
|
||||
placeSearch,
|
||||
viewer,
|
||||
engine,
|
||||
styleManager,
|
||||
dataManager,
|
||||
sceneDirector,
|
||||
annotations,
|
||||
});
|
||||
defer(() => {
|
||||
voiceCommands.stop({ removeUi: true });
|
||||
if (window.__gevVoiceCommands === voiceCommands)
|
||||
delete window.__gevVoiceCommands;
|
||||
scene: { viewer, engine, styleManager, sceneDirector, annotations },
|
||||
signal,
|
||||
defer,
|
||||
});
|
||||
debug.voiceCommands = voiceCommands;
|
||||
defer(
|
||||
|
||||
+22
-16
@@ -644,8 +644,11 @@ export async function selectAnalystRows(
|
||||
* Create an engine bound to live providers. All spatial/text/number logic is
|
||||
* in the pure helpers above; this closure only sequences and remembers.
|
||||
*/
|
||||
export function createAnalystEngine(providers) {
|
||||
let lastResult = null;
|
||||
export function createAnalystEngine(providers, { memory = null } = {}) {
|
||||
// The follow-up memory. A caller may own it (`memory`, an object whose
|
||||
// `lastResult` this engine reads and writes) so a replacement engine over
|
||||
// another globe keeps the last answer; otherwise it is this engine's.
|
||||
memory ??= { lastResult: null };
|
||||
|
||||
/**
|
||||
* Run one query.
|
||||
@@ -666,7 +669,7 @@ export function createAnalystEngine(providers) {
|
||||
) {
|
||||
const cancelled = () =>
|
||||
refusal('CANCELLED', 'This query was superseded.', { cancelled: true });
|
||||
if (spec.followUp && !lastResult)
|
||||
if (spec.followUp && !memory.lastResult)
|
||||
return refusal(
|
||||
'NO_RESULT_CONTEXT',
|
||||
'There is no previous answer to follow up on. Ask the full question.',
|
||||
@@ -677,7 +680,7 @@ export function createAnalystEngine(providers) {
|
||||
// must be that set: "those hospitals" after an aircraft answer is a new
|
||||
// question, never a filter over the old aircraft.
|
||||
if (followUp && Array.isArray(spec.layers) && spec.layers.length) {
|
||||
const remembered = new Set(lastResult.layerKeys);
|
||||
const remembered = new Set(memory.lastResult.layerKeys);
|
||||
const named = new Set(spec.layers);
|
||||
const same =
|
||||
remembered.size === named.size &&
|
||||
@@ -685,16 +688,16 @@ export function createAnalystEngine(providers) {
|
||||
if (!same)
|
||||
return refusal(
|
||||
'FOLLOW_UP_MISMATCH',
|
||||
`The last answer was about ${lastResult.layerKeys.join(', ')}, not ${spec.layers.join(', ')}. Ask again without followUp.`,
|
||||
`The last answer was about ${memory.lastResult.layerKeys.join(', ')}, not ${spec.layers.join(', ')}. Ask again without followUp.`,
|
||||
{
|
||||
layers: [...spec.layers],
|
||||
remembered: [...lastResult.layerKeys],
|
||||
remembered: [...memory.lastResult.layerKeys],
|
||||
coverage: { layersQueried: [], scope: 'none' },
|
||||
},
|
||||
);
|
||||
}
|
||||
const layers = followUp
|
||||
? lastResult.layerKeys
|
||||
? memory.lastResult.layerKeys
|
||||
: Array.isArray(spec.layers) && spec.layers.length
|
||||
? spec.layers
|
||||
: ['flights'];
|
||||
@@ -729,10 +732,11 @@ export function createAnalystEngine(providers) {
|
||||
let queriedSnapshots;
|
||||
let unanswered = [];
|
||||
if (followUp) {
|
||||
groups = lastResult.matched;
|
||||
layersQueried = lastResult.coverage.layersQueried;
|
||||
queriedSnapshots = lastResult.coverage.feedProvenance?.layers || [];
|
||||
unanswered = [...(lastResult.unanswered || [])];
|
||||
groups = memory.lastResult.matched;
|
||||
layersQueried = memory.lastResult.coverage.layersQueried;
|
||||
queriedSnapshots =
|
||||
memory.lastResult.coverage.feedProvenance?.layers || [];
|
||||
unanswered = [...(memory.lastResult.unanswered || [])];
|
||||
} else {
|
||||
groups = [];
|
||||
layersQueried = [];
|
||||
@@ -816,9 +820,11 @@ export function createAnalystEngine(providers) {
|
||||
|
||||
// 2) Spatial scope
|
||||
const rememberedScope =
|
||||
followUp && !spec.scope ? lastResult.scopePresentation || null : null;
|
||||
followUp && !spec.scope
|
||||
? memory.lastResult.scopePresentation || null
|
||||
: null;
|
||||
const rememberedDistanceOf =
|
||||
followUp && !spec.scope ? lastResult.distanceOf || null : null;
|
||||
followUp && !spec.scope ? memory.lastResult.distanceOf || null : null;
|
||||
let resolvedScope = null;
|
||||
let scopeNote = rememberedScope?.coverage || 'anywhere';
|
||||
let scopeSource = rememberedScope?.source || null;
|
||||
@@ -1018,7 +1024,7 @@ export function createAnalystEngine(providers) {
|
||||
: null;
|
||||
if (!isCurrent()) return cancelled();
|
||||
const committedCoverage = memoryOverride?.coverage || result.coverage;
|
||||
lastResult = {
|
||||
memory.lastResult = {
|
||||
matched: memoryOverride?.matched || selection.matched,
|
||||
distanceOf: memoryOverride?.distanceOf || rememberedDistanceOf,
|
||||
layerKeys: [...(memoryOverride?.layerKeys || layers)],
|
||||
@@ -1042,10 +1048,10 @@ export function createAnalystEngine(providers) {
|
||||
return {
|
||||
query,
|
||||
reset() {
|
||||
lastResult = null;
|
||||
memory.lastResult = null;
|
||||
},
|
||||
hasMemory() {
|
||||
return Boolean(lastResult);
|
||||
return Boolean(memory.lastResult);
|
||||
},
|
||||
};
|
||||
}
|
||||
+6
-1
@@ -44,7 +44,7 @@ const session = createSessionDiagnostics(() => {
|
||||
navigation: style?._navigation,
|
||||
placeSearch: style?.placeSearch,
|
||||
dataManager: components.data?.dataManager,
|
||||
voice: components.tools?.voiceCommands,
|
||||
voice: layerSession.voice?.commands ?? components.tools?.voiceCommands,
|
||||
annotations: components.tools?.annotations,
|
||||
geoImport: window.__gevGeoImport,
|
||||
drawTool: window.__gevDrawTool,
|
||||
@@ -171,8 +171,13 @@ async function settle() {
|
||||
components.controls.styleManager.shareLinkManager?.resumeNavigation?.();
|
||||
}
|
||||
|
||||
// Voice calls made while the engines switch wait for the next engine
|
||||
// (src/voice/sceneRunner.js); a settled switch lets them look again.
|
||||
layerSession.switching = () => engineSwitch?.state().busy === true;
|
||||
|
||||
function present() {
|
||||
const { kind, busy } = engineSwitch.state();
|
||||
if (!busy) layerSession.voice?.runner.wake();
|
||||
display.presentGlobeEngine({ kind, busy, select });
|
||||
for (const button of loading.querySelectorAll('.engine-switch-retry button'))
|
||||
button.disabled = busy;
|
||||
|
||||
+13
-3
@@ -349,6 +349,8 @@ export function createGevActionRunner({
|
||||
speechBuilders = null,
|
||||
deixis = null,
|
||||
readVoicePanelOperation = null,
|
||||
analystMemory = null,
|
||||
layerEnabledAt = null,
|
||||
}) {
|
||||
// Voice enable times and analyst follow-up memory belong to this runner.
|
||||
// Speech builders turn results into say/display envelopes; callers extend
|
||||
@@ -357,7 +359,10 @@ export function createGevActionRunner({
|
||||
? { ...SPEECH_BUILDERS, ...speechBuilders }
|
||||
: SPEECH_BUILDERS;
|
||||
const engine = givenEngine ?? engineFromViewer(viewer);
|
||||
const _layerEnabledAt = new Map();
|
||||
// A host that replaces this runner when the globe engine changes passes
|
||||
// the analyst's follow-up memory and the layer enable times, so the next
|
||||
// runner keeps them.
|
||||
const _layerEnabledAt = layerEnabledAt ?? new Map();
|
||||
let analystEngine;
|
||||
const resolveRegionRing = (name, signal) =>
|
||||
annotationResolver.resolveRegionRingForQuery(name, signal, placeSearch);
|
||||
@@ -368,6 +373,7 @@ export function createGevActionRunner({
|
||||
resolveRegionRing,
|
||||
isWarming: (layerKey) => layerIsWarming(_layerEnabledAt, layerKey),
|
||||
}),
|
||||
{ memory: analystMemory },
|
||||
));
|
||||
installViewTargetPrewarm(engine);
|
||||
// The camera verbs (move_camera, fly_route) still run on the viewer.
|
||||
@@ -471,7 +477,9 @@ export function createGevActionRunner({
|
||||
};
|
||||
if (
|
||||
!viewer &&
|
||||
((name === 'control_scene' && !sceneDirector) || VIEWER_ONLY_ACTIONS.includes(name) || CESIUM_LOOK_ACTIONS.includes(name))
|
||||
((name === 'control_scene' && !sceneDirector) ||
|
||||
VIEWER_ONLY_ACTIONS.includes(name) ||
|
||||
CESIUM_LOOK_ACTIONS.includes(name))
|
||||
)
|
||||
return unavailableOnEngine(name, engine);
|
||||
|
||||
@@ -1077,7 +1085,9 @@ export function createGevActionRunner({
|
||||
contextState?.mode === 'flights' &&
|
||||
contextState?.active !== false &&
|
||||
contextState?.changing !== true;
|
||||
const directEntry = !targetLayer && !aircraftClass &&
|
||||
const directEntry =
|
||||
!targetLayer &&
|
||||
!aircraftClass &&
|
||||
styleManager.getCockpitState?.()?.entryAllowed === true;
|
||||
if (!contactsReady && !directEntry) {
|
||||
const contextResult = await styleManager.setContextMode('flights', {
|
||||
|
||||
+139
-39
@@ -5,6 +5,7 @@ import { createVoiceCommands } from './commands.js';
|
||||
import { cameraPoseKey, createPointerTracker } from './pointerContext.js';
|
||||
import { createPointerPicker } from './pointerPick.js';
|
||||
import { createReferentRegistry } from './referents.js';
|
||||
import { createSceneRunner } from './sceneRunner.js';
|
||||
import { engineFromViewer } from '../globe/cesium/engineFromViewer.js';
|
||||
export * from './realtimeController.js';
|
||||
|
||||
@@ -31,49 +32,148 @@ export function withToolCatalog(runner, loadCatalog) {
|
||||
return runGevTool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Voice for a page whose globe engine can change while it runs: the
|
||||
* microphone, transport, controls and conversation belong to the page
|
||||
* session; the action runner, the pointer's canvas and picker and the
|
||||
* annotation events belong to one engine attachment (`attachScene`), and a
|
||||
* switch only replaces those. Returns `{ commands, runner, attachScene,
|
||||
* detachScene }`; `commands` is what initGevVoiceCommands returns.
|
||||
*
|
||||
* `isSwitching()` tells the runner that the page has started a switch, so a
|
||||
* call made before the outgoing scene detaches waits for the next one too.
|
||||
* `createRunner` replaces createGevActionRunner (tests).
|
||||
*/
|
||||
export function createSceneVoice(options) {
|
||||
const scene = { engine: null };
|
||||
const cameraKey = () => cameraPoseKey(scene.engine);
|
||||
let pick = null;
|
||||
// Point-and-ask: the tracker and referents are shared by the runner (which
|
||||
// resolves 'pointer' and referent:n) and the controls (which snapshot the
|
||||
// pointer at turn start and clear both when the session ends).
|
||||
const pointer = createPointerTracker({
|
||||
element: null,
|
||||
cameraKey,
|
||||
pick: (x, y) => pick?.(x, y) ?? null,
|
||||
});
|
||||
const referents = createReferentRegistry();
|
||||
// The controller exists only after the runner, so its hooks bind late.
|
||||
const deixis = { pointer, referents, cameraKey };
|
||||
// The analyst's follow-up memory and the layer enable times outlive one
|
||||
// engine's runner.
|
||||
const analystMemory = { lastResult: null };
|
||||
const layerEnabledAt = new Map();
|
||||
let commands;
|
||||
const runner = createSceneRunner({
|
||||
isSwitching: options.isSwitching,
|
||||
switchWaitMs: options.switchWaitMs,
|
||||
build: (attached) =>
|
||||
withToolCatalog(
|
||||
(options.createRunner ?? createGevActionRunner)({
|
||||
...options,
|
||||
...attached,
|
||||
deixis,
|
||||
analystMemory,
|
||||
layerEnabledAt,
|
||||
readVoicePanelOperation: () =>
|
||||
commands?.captureVoicePanelOperation?.(),
|
||||
}),
|
||||
options.toolCatalog,
|
||||
),
|
||||
});
|
||||
const resetConversation = runner.resetConversation;
|
||||
runner.resetConversation = () => {
|
||||
analystMemory.lastResult = null;
|
||||
resetConversation();
|
||||
};
|
||||
// Annotation outline events of whichever engine is attached.
|
||||
const outlineListeners = new Set();
|
||||
let offOutline = null;
|
||||
const annotations = {
|
||||
onOutlineEvent(listener) {
|
||||
outlineListeners.add(listener);
|
||||
return () => outlineListeners.delete(listener);
|
||||
},
|
||||
};
|
||||
// The panel reads the cockpit of the attached shell.
|
||||
let shell = null;
|
||||
const styleManager = {
|
||||
getCockpitState: () => shell?.getCockpitState?.() ?? null,
|
||||
};
|
||||
commands = createVoiceCommands({
|
||||
...options,
|
||||
styleManager,
|
||||
annotations,
|
||||
pointer,
|
||||
referents,
|
||||
runner,
|
||||
});
|
||||
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
|
||||
deixis.onPointerUsed = () => commands.announcePointer?.();
|
||||
const lifetime = commands.signal ?? commands.session?.signal;
|
||||
let attachedOnce = false;
|
||||
const detachScene = () => {
|
||||
runner.detach();
|
||||
offOutline?.();
|
||||
offOutline = null;
|
||||
pointer.setElement(null);
|
||||
pick = null;
|
||||
scene.engine = null;
|
||||
shell = null;
|
||||
};
|
||||
lifetime?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
detachScene();
|
||||
pointer.destroy();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
if (lifetime?.aborted) pointer.destroy();
|
||||
return {
|
||||
commands,
|
||||
runner,
|
||||
/**
|
||||
* Attach the voice to one engine: `{ viewer, engine, styleManager,
|
||||
* dataManager, sceneDirector, annotations }` (viewer and annotations
|
||||
* null on an engine without them).
|
||||
*/
|
||||
attachScene(attached) {
|
||||
if (lifetime?.aborted) return;
|
||||
detachScene();
|
||||
const engine = attached.engine ?? engineFromViewer(attached.viewer);
|
||||
scene.engine = engine;
|
||||
shell = attached.styleManager ?? null;
|
||||
pick = createPointerPicker({
|
||||
engine,
|
||||
dataManager: attached.dataManager ?? options.dataManager,
|
||||
});
|
||||
pointer.setElement(engine?.host.canvas || null);
|
||||
offOutline =
|
||||
attached.annotations?.onOutlineEvent?.((event) => {
|
||||
for (const listener of [...outlineListeners]) listener(event);
|
||||
}) ?? null;
|
||||
runner.attach({ ...attached, engine });
|
||||
// A session that outlived the switch hears which globe it now drives.
|
||||
if (attachedOnce && commands.isActive?.()) {
|
||||
const event = { type: 'engine-changed', engine: engine?.kind ?? null };
|
||||
if (typeof commands.notifyMapEvent === 'function')
|
||||
commands.notifyMapEvent(event);
|
||||
else commands.session?.sendMapEvent?.(event);
|
||||
}
|
||||
attachedOnce = true;
|
||||
},
|
||||
detachScene,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose the standalone action runner with the voice controls, on the globe
|
||||
* `engine` (by default the one registered for `options.viewer`). Without a
|
||||
* viewer (the Navara engine) the runner refuses the viewer-only tools.
|
||||
*/
|
||||
export function initGevVoiceCommands(options) {
|
||||
const engine = options.engine ?? engineFromViewer(options.viewer);
|
||||
const canvas = engine?.host.canvas || null;
|
||||
const cameraKey = () => cameraPoseKey(engine);
|
||||
// Point-and-ask: the tracker and referents are shared by the runner (which
|
||||
// resolves 'pointer' and referent:n) and the controls (which snapshot the
|
||||
// pointer at turn start and clear both when the session ends).
|
||||
const pointer = canvas
|
||||
? createPointerTracker({
|
||||
element: canvas,
|
||||
cameraKey,
|
||||
pick: createPointerPicker({
|
||||
engine,
|
||||
dataManager: options.dataManager,
|
||||
}),
|
||||
})
|
||||
: null;
|
||||
const referents = createReferentRegistry();
|
||||
// The controller exists only after the runner, so its hooks bind late.
|
||||
const deixis = { pointer, referents, cameraKey };
|
||||
let commands;
|
||||
commands = createVoiceCommands({
|
||||
...options,
|
||||
pointer,
|
||||
referents,
|
||||
runner: withToolCatalog(
|
||||
createGevActionRunner({
|
||||
...options,
|
||||
deixis,
|
||||
readVoicePanelOperation: () => commands?.captureVoicePanelOperation?.(),
|
||||
}),
|
||||
options.toolCatalog,
|
||||
),
|
||||
});
|
||||
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
|
||||
deixis.onPointerUsed = () => commands.announcePointer?.();
|
||||
const lifetime = commands.signal ?? commands.session?.signal;
|
||||
lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true });
|
||||
if (lifetime?.aborted) pointer?.destroy();
|
||||
return commands;
|
||||
const voice = createSceneVoice(options);
|
||||
voice.attachScene(options);
|
||||
return voice.commands;
|
||||
}
|
||||
@@ -276,7 +276,8 @@ export function createPointerTracker({
|
||||
let held = null;
|
||||
let active = null;
|
||||
let destroyed = false;
|
||||
const listeners = new AbortController();
|
||||
// The element's own listeners; replaced with the element (setElement).
|
||||
let listeners = new AbortController();
|
||||
|
||||
function size() {
|
||||
return {
|
||||
@@ -328,12 +329,14 @@ export function createPointerTracker({
|
||||
cancelDwell();
|
||||
}
|
||||
|
||||
if (element?.addEventListener) {
|
||||
function listen() {
|
||||
if (!element?.addEventListener) return;
|
||||
const options = { passive: true, signal: listeners.signal };
|
||||
element.addEventListener('pointermove', onMove, options);
|
||||
element.addEventListener('pointerdown', onMove, options);
|
||||
element.addEventListener('pointerleave', onLeave, options);
|
||||
}
|
||||
listen();
|
||||
|
||||
/** Is the pointer a referent right now? */
|
||||
function isFresh(recentMs = POINTER_FRESH_MS) {
|
||||
@@ -372,7 +375,21 @@ export function createPointerTracker({
|
||||
}
|
||||
|
||||
return {
|
||||
element,
|
||||
get element() {
|
||||
return element;
|
||||
},
|
||||
/**
|
||||
* Follow another element (the next globe's canvas after an engine
|
||||
* switch), or none. The pointer starts outside it.
|
||||
*/
|
||||
setElement(next) {
|
||||
if (destroyed || next === element) return;
|
||||
listeners.abort();
|
||||
listeners = new AbortController();
|
||||
element = next ?? null;
|
||||
onLeave();
|
||||
listen();
|
||||
},
|
||||
capture,
|
||||
isFresh,
|
||||
/** Push-to-talk keydown: snapshot now, keep it for the hold. */
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
// The voice action runner for a page whose globe engine can change while a
|
||||
// voice session runs. The session keeps one runner; each engine attachment
|
||||
// builds the runner of that scene (createGevActionRunner over its viewer,
|
||||
// engine, shell and annotations) and the forwarder sends every call to the
|
||||
// current one. A call made while the engines switch waits for the next
|
||||
// attachment, up to a bound, and then runs on the engine that attached; when
|
||||
// none attaches in time it is refused with ENGINE_SWITCHING.
|
||||
|
||||
/** How long a call waits for the next engine before it is refused. */
|
||||
export const ENGINE_SWITCH_WAIT_MS = 15_000;
|
||||
|
||||
/** The refusal for a call that found no engine within the wait. */
|
||||
export function engineSwitchingRefusal(name) {
|
||||
return {
|
||||
ok: false,
|
||||
action: name,
|
||||
applied: false,
|
||||
code: 'ENGINE_SWITCHING',
|
||||
error: 'The globe is switching engines; this action was not run.',
|
||||
say: 'The globe is switching engines. Ask again in a moment.',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} options
|
||||
* @param {(scene: object) => Function} options.build The runner for one
|
||||
* attached scene; its optional `dispose()` runs when the scene detaches.
|
||||
* @param {() => boolean} [options.isSwitching] True while the page switches
|
||||
* engines, even before the outgoing scene has detached.
|
||||
* @param {number} [options.switchWaitMs]
|
||||
*/
|
||||
export function createSceneRunner({
|
||||
build,
|
||||
isSwitching = () => false,
|
||||
switchWaitMs = ENGINE_SWITCH_WAIT_MS,
|
||||
}) {
|
||||
if (typeof build !== 'function')
|
||||
throw new TypeError('A scene runner needs a build function');
|
||||
let inner = null;
|
||||
let kind = null;
|
||||
let attachment = null;
|
||||
// Bumped by every attachment, so a waiting call knows one happened.
|
||||
let attachments = 0;
|
||||
let conversation = 0;
|
||||
let disposed = false;
|
||||
const waiters = new Set();
|
||||
|
||||
const wake = () => {
|
||||
for (const resolve of [...waiters]) resolve();
|
||||
waiters.clear();
|
||||
};
|
||||
|
||||
// Resolves true once a scene is ready (one attached since the wait began,
|
||||
// or the switch settled with a scene), false when the wait is cut short.
|
||||
async function waitForScene(signal, timeoutMs) {
|
||||
const since = attachments;
|
||||
const deadline = Date.now() + Math.max(0, timeoutMs);
|
||||
for (;;) {
|
||||
if (disposed || signal?.aborted) return false;
|
||||
if (inner && (attachments !== since || !isSwitching())) return true;
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) return false;
|
||||
await new Promise((resolve) => {
|
||||
let timer = null;
|
||||
const done = () => {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener('abort', done);
|
||||
waiters.delete(done);
|
||||
resolve();
|
||||
};
|
||||
waiters.add(done);
|
||||
signal?.addEventListener('abort', done, { once: true });
|
||||
timer = setTimeout(done, remaining);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function run(name, args, options = {}) {
|
||||
if (disposed)
|
||||
return {
|
||||
ok: false,
|
||||
action: name,
|
||||
cancelled: true,
|
||||
error: 'Voice ended.',
|
||||
};
|
||||
if (!inner || isSwitching()) {
|
||||
const found = await waitForScene(options.signal, run.switchWaitMs);
|
||||
if (disposed)
|
||||
return {
|
||||
ok: false,
|
||||
action: name,
|
||||
cancelled: true,
|
||||
error: 'Voice ended.',
|
||||
};
|
||||
if (!found) return engineSwitchingRefusal(name);
|
||||
}
|
||||
const current = inner;
|
||||
const currentKind = kind;
|
||||
const epoch = conversation;
|
||||
const signal = options.signal
|
||||
? AbortSignal.any([options.signal, attachment.signal])
|
||||
: attachment.signal;
|
||||
const result = await current(name, args, {
|
||||
...options,
|
||||
signal,
|
||||
isCurrent: () =>
|
||||
epoch === conversation &&
|
||||
!disposed &&
|
||||
current === inner &&
|
||||
(typeof options.isCurrent !== 'function' || options.isCurrent()),
|
||||
});
|
||||
// Say which engine answered: after a switch the model reads the result
|
||||
// of the globe the person now sees.
|
||||
return result &&
|
||||
typeof result === 'object' &&
|
||||
!Array.isArray(result) &&
|
||||
result.engine === undefined
|
||||
? { ...result, engine: currentKind }
|
||||
: result;
|
||||
}
|
||||
|
||||
run.switchWaitMs = switchWaitMs;
|
||||
/** The engine kind of the attached scene, or null. */
|
||||
run.engineKind = () => kind;
|
||||
/** Build the runner for `scene`; waiting calls run on it. */
|
||||
run.attach = (scene) => {
|
||||
if (disposed) return;
|
||||
run.detach();
|
||||
attachment = new AbortController();
|
||||
inner = build(scene);
|
||||
kind = scene?.engine?.kind ?? (scene?.viewer ? 'cesium' : null);
|
||||
attachments++;
|
||||
wake();
|
||||
};
|
||||
/** Stop the attached runner: its calls in flight see their signal abort. */
|
||||
run.detach = () => {
|
||||
attachment?.abort();
|
||||
attachment = null;
|
||||
const previous = inner;
|
||||
inner = null;
|
||||
kind = null;
|
||||
previous?.dispose?.();
|
||||
};
|
||||
/** The page's switch settled: waiting calls look again. */
|
||||
run.wake = wake;
|
||||
run.resetConversation = () => {
|
||||
conversation++;
|
||||
inner?.resetConversation?.();
|
||||
};
|
||||
run.dispose = () => {
|
||||
if (disposed) return;
|
||||
run.resetConversation();
|
||||
disposed = true;
|
||||
run.detach();
|
||||
wake();
|
||||
};
|
||||
return run;
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createSceneRunner, engineSwitchingRefusal } from './sceneRunner.js';
|
||||
|
||||
const engine = (kind) => ({ kind });
|
||||
|
||||
function runnerFor(log) {
|
||||
return (scene) => {
|
||||
const run = async (name, args, options) => {
|
||||
log.push({ name, kind: scene.engine.kind, signal: options.signal });
|
||||
return { ok: true, action: name };
|
||||
};
|
||||
run.dispose = () => log.push({ disposed: scene.engine.kind });
|
||||
run.resetConversation = () => log.push({ reset: scene.engine.kind });
|
||||
return run;
|
||||
};
|
||||
}
|
||||
|
||||
test('calls run on the attached engine and say which engine answered', async () => {
|
||||
const log = [];
|
||||
const run = createSceneRunner({ build: runnerFor(log) });
|
||||
run.attach({ engine: engine('cesium') });
|
||||
assert.deepEqual(await run('set_layer', {}), {
|
||||
ok: true,
|
||||
action: 'set_layer',
|
||||
engine: 'cesium',
|
||||
});
|
||||
run.attach({ engine: engine('navara') });
|
||||
assert.equal((await run('set_layer', {})).engine, 'navara');
|
||||
assert.deepEqual(log[1], { disposed: 'cesium' });
|
||||
assert.equal(run.engineKind(), 'navara');
|
||||
});
|
||||
|
||||
test('a call during a switch waits for the next engine and runs there', async () => {
|
||||
const log = [];
|
||||
let switching = false;
|
||||
const run = createSceneRunner({
|
||||
build: runnerFor(log),
|
||||
isSwitching: () => switching,
|
||||
});
|
||||
run.attach({ engine: engine('cesium') });
|
||||
switching = true;
|
||||
const pending = run('set_layer', {});
|
||||
run.detach();
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
assert.equal(log.filter((entry) => entry.name).length, 0);
|
||||
run.attach({ engine: engine('navara') });
|
||||
const result = await pending;
|
||||
assert.equal(result.engine, 'navara');
|
||||
switching = false;
|
||||
});
|
||||
|
||||
test('a call refused when no engine attaches in time names ENGINE_SWITCHING', async () => {
|
||||
const run = createSceneRunner({
|
||||
build: runnerFor([]),
|
||||
isSwitching: () => true,
|
||||
switchWaitMs: 10,
|
||||
});
|
||||
run.attach({ engine: engine('cesium') });
|
||||
run.detach();
|
||||
const result = await run('annotate_map', {});
|
||||
assert.deepEqual(result, engineSwitchingRefusal('annotate_map'));
|
||||
assert.equal(result.code, 'ENGINE_SWITCHING');
|
||||
});
|
||||
|
||||
test('a switch that settles without a new engine lets waiting calls run', async () => {
|
||||
const log = [];
|
||||
let switching = true;
|
||||
const run = createSceneRunner({
|
||||
build: runnerFor(log),
|
||||
isSwitching: () => switching,
|
||||
});
|
||||
run.attach({ engine: engine('cesium') });
|
||||
const pending = run('set_layer', {});
|
||||
switching = false;
|
||||
run.wake();
|
||||
assert.equal((await pending).engine, 'cesium');
|
||||
});
|
||||
|
||||
test('the attachment signal aborts calls in flight when the engine detaches', async () => {
|
||||
let seen;
|
||||
const run = createSceneRunner({
|
||||
build: () => async (name, args, options) => {
|
||||
seen = options.signal;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
return { ok: !options.signal.aborted };
|
||||
},
|
||||
});
|
||||
run.attach({ engine: engine('cesium') });
|
||||
const pending = run('fly_to_location', {});
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
run.detach();
|
||||
assert.equal(seen.aborted, true);
|
||||
assert.equal((await pending).ok, false);
|
||||
});
|
||||
|
||||
test('a disposed runner refuses and wakes its waiting calls', async () => {
|
||||
const run = createSceneRunner({
|
||||
build: runnerFor([]),
|
||||
isSwitching: () => true,
|
||||
});
|
||||
const pending = run('set_layer', {});
|
||||
run.dispose();
|
||||
const result = await pending;
|
||||
assert.equal(result.cancelled, true);
|
||||
assert.equal((await run('set_layer', {})).cancelled, true);
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createSceneVoice } from './gevRealtime.js';
|
||||
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
|
||||
|
||||
function fakeEngine(kind) {
|
||||
const canvas = new EventTarget();
|
||||
canvas.getBoundingClientRect = () => ({ left: 0, top: 0 });
|
||||
return {
|
||||
kind,
|
||||
host: { canvas, size: () => ({ width: 0, height: 0 }) },
|
||||
camera: { pose: () => ({}) },
|
||||
};
|
||||
}
|
||||
|
||||
function fixture() {
|
||||
const previous = globalThis.window;
|
||||
globalThis.window = {};
|
||||
const lifetime = new AbortController();
|
||||
const adapter = { starts: 0, stops: [], events: [] };
|
||||
let runAction;
|
||||
const built = [];
|
||||
const voice = createSceneVoice({
|
||||
signal: lifetime.signal,
|
||||
createControl: () => createTestVoiceUi(),
|
||||
createCard: () => null,
|
||||
createReticle: () => null,
|
||||
createRunner: (scene) => {
|
||||
built.push(scene);
|
||||
return async (name) => ({ ok: true, action: name });
|
||||
},
|
||||
createSession(hooks) {
|
||||
runAction = hooks.runAction;
|
||||
return {
|
||||
async start() {
|
||||
adapter.starts++;
|
||||
hooks.emit({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop(options) {
|
||||
adapter.stops.push(options);
|
||||
},
|
||||
sendText() {},
|
||||
sendMapEvent(event) {
|
||||
adapter.events.push(event);
|
||||
return true;
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
return {
|
||||
voice,
|
||||
adapter,
|
||||
built,
|
||||
lifetime,
|
||||
run: (...args) => runAction(...args),
|
||||
restore: () => {
|
||||
globalThis.window = previous;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('an engine switch keeps the voice session and its transport', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const cesium = fakeEngine('cesium');
|
||||
f.voice.attachScene({ engine: cesium, viewer: {} });
|
||||
await f.voice.commands.start();
|
||||
assert.equal(f.adapter.starts, 1);
|
||||
assert.equal(f.voice.commands.pointer.element, cesium.host.canvas);
|
||||
assert.equal((await f.run('set_layer', {})).engine, 'cesium');
|
||||
|
||||
f.voice.detachScene();
|
||||
assert.deepEqual(f.adapter.stops, [], 'a detach never stops the adapter');
|
||||
assert.equal(f.voice.commands.isActive(), true);
|
||||
assert.equal(f.voice.commands.pointer.element, null);
|
||||
// A call made between the engines waits for the next one.
|
||||
const pending = f.run('set_layer', {});
|
||||
const navara = fakeEngine('navara');
|
||||
f.voice.attachScene({ engine: navara, viewer: null });
|
||||
assert.equal((await pending).engine, 'navara');
|
||||
assert.equal(f.voice.commands.pointer.element, navara.host.canvas);
|
||||
assert.deepEqual(f.adapter.events, [
|
||||
{ type: 'engine-changed', engine: 'navara' },
|
||||
]);
|
||||
assert.equal(f.adapter.starts, 1);
|
||||
assert.equal(f.built.length, 2);
|
||||
assert.equal(f.built[1].engine, navara);
|
||||
|
||||
f.lifetime.abort();
|
||||
assert.equal(f.adapter.stops.at(-1)?.removeUi, true);
|
||||
} finally {
|
||||
f.restore();
|
||||
}
|
||||
});
|
||||
|
||||
test('annotation outline events reach the session from the attached engine only', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const listeners = new Set();
|
||||
const annotations = {
|
||||
onOutlineEvent(listener) {
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
},
|
||||
};
|
||||
f.voice.attachScene({ engine: fakeEngine('cesium'), annotations });
|
||||
await f.voice.commands.start();
|
||||
for (const listener of listeners) listener({ status: 'resolved' });
|
||||
assert.deepEqual(f.adapter.events, [
|
||||
{ type: 'map_annotation_outline', status: 'resolved' },
|
||||
]);
|
||||
f.voice.detachScene();
|
||||
assert.equal(listeners.size, 0);
|
||||
} finally {
|
||||
f.restore();
|
||||
}
|
||||
});
|
||||
@@ -6,10 +6,11 @@ import { VOICE_PROVIDERS } from './providerCapabilities.js';
|
||||
import { t } from '../i18n/index.js';
|
||||
|
||||
function createDefaultReticle(pointer) {
|
||||
const element = pointer?.element;
|
||||
if (!element?.ownerDocument) return null;
|
||||
// A pointer may follow another element later (an engine switch), so the
|
||||
// reticle reads it each time; a page without a document has none.
|
||||
if (!pointer || !globalThis.document) return null;
|
||||
return new PointerReticle({
|
||||
rectOf: () => element.getBoundingClientRect?.() || null,
|
||||
rectOf: () => pointer.element?.getBoundingClientRect?.() || null,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user