Merge main and switch Cesium and Navara in place through Vue

Preserve the camera, map source and layer selection across engine sessions. Restore the previous engine if startup fails and retain local history navigation.
This commit is contained in:
German Khokhlov committed 2026-11-02 13:10:36 +04:00
commit b8f4d097c4
1691 files changed
+225876 -41594

No files matched your search

+15 -10
View File
@@ -178,6 +178,9 @@ const schemas = [
'global-context-panel',
'weather-panel',
'recent-imagery-panel',
'voice-settings',
'voice-preview',
'provider-settings',
],
},
open: {
@@ -187,6 +190,15 @@ const schemas = [
required: ['panelId', 'open'],
},
},
{
name: 'set_voice_inactivity_timeout',
parameters: {
type: 'object',
additionalProperties: false,
properties: { minutes: { type: 'integer', minimum: 0, maximum: 60 } },
required: ['minutes'],
},
},
{
name: 'set_context_mode',
parameters: {
@@ -223,6 +235,7 @@ const schemas = [
aircraftClass: {
type: 'string',
},
voicePanel: { type: 'string', enum: ['preview', 'settings'] },
},
required: ['action'],
},
@@ -235,15 +248,7 @@ const schemas = [
properties: {
style: {
type: 'string',
enum: [
'normal',
'retro',
'surveillance',
'thermal',
'anime',
'noir',
'snow',
],
enum: ['normal', 'retro', 'surveillance', 'thermal', 'anime', 'noir'],
},
},
required: ['style'],
@@ -729,7 +734,7 @@ const schemas = [
properties: {
motion: {
type: 'string',
enum: ['orbit', 'pan', 'tilt', 'rotate', 'stop'],
enum: ['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe', 'stop'],
},
direction: {
type: 'string',
+94 -7
View File
@@ -1,3 +1,4 @@
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
import test from 'node:test';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
@@ -31,13 +32,22 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
.digest('hex');
assert.equal(
digest,
// Re-derived for the voice layer manifest (generated layer enums and
// Re-derived for Provider Settings and explicit Visual Presets/Scenes routing,
// plus the deliberate direct Panel tool/target additions and voice layer manifest (generated layer enums and
// aliases), point-and-ask (pointer sentinels, referent args) and the
// consolidated tool wording (each policy stated once); model-facing tools
// omit top-level anyOf (OpenAI rejects it). Contacts list routing preserves
// requested radius and center authority in descriptions only.
// Street Level adds its toggle enum values and generated alias hint.
'9eb113454c5af5e32e27ffcdc7919d0e4600ba1d204f8f175e071152be41ad0f',
// Street Level and GNSS extend the manifest; tracked and analyst aircraft describe broadcasts.
// Air quality (ECCC AQHI) extends the manifest.
// Globe rotation adds one motion; visual keyhole fit and explicit
// reset guidance/defaults update only move_camera and mode descriptions.
// S2 GhostMaps extends the generated layer enums.
// Panel adds named voice targets and timeout; shared Panel/Cockpit guidance
// is consolidated without changing either feature's arguments.
// Snow leaves the set_visual_style enum.
// The Operator HUD layout leaves set_hud's layout enum.
'f47a7f78f13b1f79978fa524fad2618e32bd813e1a5dfd746bf35d29909fc91b',
);
});
@@ -95,19 +105,43 @@ test('metadata cannot add tools, fields, types or enum values', () => {
test('all legacy action arguments are byte-identical after removing the deliberate additions', () => {
const legacy = structuredClone(GEV_ACTION_SCHEMAS).filter(
(tool) => !['next_satellite_pass', 'set_cyber_sonar'].includes(tool.name),
(tool) =>
![
'next_satellite_pass',
'set_cyber_sonar',
'set_voice_inactivity_timeout',
].includes(tool.name),
);
// Layer enums are generated from the voice layer manifest and pinned by
// layerManifest.test.mjs; the two shipped right-rail panels are additive.
const property = (name) =>
legacy.find((tool) => tool.name === name).parameters.properties;
delete property('control_cockpit').voicePanel;
delete property('set_layer_visibility').layerId.enum;
delete property('show_data_layers_menu').layerId.enum;
delete property('get_entity_context').layerId.enum;
delete property('analyst_query').layers.items.enum;
// Earth-centered rotation is additive; local camera yaw keeps its contract.
const motion = property('move_camera').motion;
assert.deepEqual(motion.enum, [
'orbit',
'pan',
'tilt',
'rotate',
'rotate_globe',
'stop',
]);
motion.enum = motion.enum.filter((value) => value !== 'rotate_globe');
const panels = property('set_panel_open').panelId;
panels.enum = panels.enum.filter(
(id) => !['weather-panel', 'recent-imagery-panel'].includes(id),
(id) =>
![
'weather-panel',
'recent-imagery-panel',
'voice-settings',
'voice-preview',
'provider-settings',
].includes(id),
);
for (const tool of legacy) {
// Point-and-ask adds `referent` arguments and 'pointer' enum values.
@@ -134,9 +168,22 @@ test('all legacy action arguments are byte-identical after removing the delibera
delete center.properties[axis].minimum;
delete center.properties[axis].maximum;
}
// Cyber adds one HUD layout.
// Cyber adds one HUD layout and Operator was retired.
const hud = property('set_hud').layout;
hud.enum = hud.enum.filter((layout) => layout !== 'cyber');
assert.deepEqual(hud.enum, ['tactical', 'minimal', 'cyber']);
hud.enum = ['tactical', 'operator', 'minimal'];
// Snow is retired from set_visual_style; restore it to keep the legacy pin.
const visualStyle = legacy.find((tool) => tool.name === 'set_visual_style')
.parameters.properties.style;
assert.deepEqual(visualStyle.enum, [
'normal',
'retro',
'surveillance',
'thermal',
'anime',
'noir',
]);
visualStyle.enum = [...visualStyle.enum, 'snow'];
// Derived by applying the same removals to the 4b56d0e9 actionSchemas.
assert.equal(
createHash('sha256').update(JSON.stringify(legacy)).digest('hex'),
@@ -144,6 +191,24 @@ test('all legacy action arguments are byte-identical after removing the delibera
);
});
test('globe rotation is an additive camera motion with the existing control arguments', () => {
const schema = GEV_ACTION_SCHEMAS.find(
(tool) => tool.name === 'move_camera',
).parameters;
for (const motion of ['rotate_globe', 'rotate', 'orbit', 'pan', 'tilt']) {
assert.deepEqual(
validateArgs(schema, {
motion,
direction: 'right',
speed: 'slow',
mode: 'continuous',
}),
[],
);
}
assert.ok(validateArgs(schema, { motion: 'spin_camera' }).length > 0);
});
test('track_entity accepts a referent alone but rejects an empty target', () => {
const schema = GEV_ACTION_SCHEMAS.find(
(tool) => tool.name === 'track_entity',
@@ -182,3 +247,25 @@ test('the benchmark still rejects an empty track_entity although the model-facin
[],
);
});
test('globe reset guidance requires a separate explicit command after refusal', () => {
const paragraph = realtimeInstructions()
.split('\n')
.find((line) => line.includes('globe-view-required'));
assert.ok(paragraph);
assert.match(paragraph, /calls only move_camera motion rotate_globe/);
assert.match(paragraph, /explicit user Reset\/Show Globe View/);
assert.match(paragraph, /never from your own plan/);
assert.match(paragraph, /bare yes\/ok/);
assert.match(paragraph, /Do not ask a question/);
});
test('direct voice timeout schema accepts None and exact integers, rejects malformed duration', () => {
const tool = GEV_REALTIME_TOOLS.find(
(tool) => tool.name === 'set_voice_inactivity_timeout',
);
for (const minutes of [0, 1, 5, 60])
assert.equal(validateArgs(tool.parameters, { minutes }).length, 0);
for (const minutes of [-1, 61, 1.5, '5'])
assert.ok(validateArgs(tool.parameters, { minutes }).length > 0);
});
+102
View File
@@ -0,0 +1,102 @@
// The analyst card shows its headline, scope, chips and caveats in the UI
// language; the spoken headline stays English for the model.
import assert from 'node:assert/strict';
import test from 'node:test';
import { setLocale } from '../i18n/index.js';
import { attachVoiceResult as withSpeech } from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
function cardFor(name, result, args = {}) {
const shaped = withSpeech(name, result, args);
return {
shaped,
view: voiceCardView(
[
{ type: 'action-call', callId: 'c1', name, arguments: args },
{ type: 'action-result', callId: 'c1', name, result: shaped },
].reduce(reduceVoiceCard, initialVoiceCardState()),
),
};
}
const analyst = {
ok: true,
count: 3,
complete: true,
scopeLabel: 'anywhere in the loaded data',
partial: true,
feedState: 'degraded',
unanswered: ['earthquakes'],
coverage: { layersQueried: [{ layerKey: 'flights', source: 'OpenSky' }] },
display: { scope: 'anywhere in the loaded data' },
items: [],
};
const radius = {
...analyst,
count: 1234,
complete: false,
partial: false,
feedState: 'nominal',
unanswered: [],
scopeLabel: 'in view',
display: {
scope: 'within 42 km of the view centre',
window: 'last 24 h',
caveat:
'counted 1,000 of 2,345 loaded records; earthquakes loading; flights: distance is ground distance, not overhead',
},
coverage: {
layersQueried: [{ layerKey: 'flights', source: 'OpenSky' }],
note: 'the flights layer loads by viewport',
},
};
test('analyst card in es has no English text', () => {
setLocale('es');
try {
const { view } = cardFor('analyst_query', analyst);
assert.doesNotMatch(view.result.title, /anywhere in the loaded data/);
assert.ok(!view.result.chips.includes('partial'));
assert.ok(!view.result.chips.includes('degraded'));
assert.ok(!view.result.notes.some((n) => n.startsWith('Not answered')));
} finally {
setLocale('en');
}
});
test('analyst scope, window and caveats in ru', () => {
setLocale('ru');
try {
const { shaped, view } = cardFor('analyst_query', radius);
assert.match(shaped.say, /^At least 1,234 aircraft in view/);
assert.doesNotMatch(
JSON.stringify(view.result),
/aircraft|within|view centre|last 24 h|counted|loading|ground distance|loads by viewport/,
);
assert.match(view.result.title, /^1\s234 /);
} finally {
setLocale('en');
}
});
test('en: the card keeps its English text and names layers', () => {
const { shaped, view } = cardFor('analyst_query', analyst);
assert.equal(view.result.title, '3 aircraft anywhere in the loaded data');
assert.deepEqual(view.result.chips, ['degraded', 'partial']);
assert.ok(view.result.notes.includes('Not answered: Earthquakes (24h)'));
// The model still gets the layer key.
assert.match(shaped.say, /earthquakes not answered/);
const capped = cardFor('analyst_query', radius).view.result;
assert.equal(capped.title, 'At least 1,234 aircraft in view');
assert.deepEqual(capped.lines, [
'within 42 km of the view centre',
'last 24 h',
]);
assert.ok(capped.notes.includes('counted 1,000 of 2,345 loaded records'));
assert.ok(capped.notes.includes('the flights layer loads by viewport'));
});
+114
View File
@@ -0,0 +1,114 @@
// The analyst card's empty notes can be a layer's status line: Local ADS-B
// with every decoder feed stale, Transit with a quiet feed ("(95s old)") or
// its getStats().coverage text. The card shows them in the UI language; the
// model gets the English.
import assert from 'node:assert/strict';
import test from 'node:test';
import { setLocale } from '../i18n/index.js';
import { attachVoiceResult } from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
function viewFor(layer, caveat) {
const shaped = attachVoiceResult(
'analyst_query',
{
ok: true,
count: 0,
complete: true,
scopeLabel: 'in view',
feedState: 'nominal',
unanswered: [],
coverage: { layersQueried: [{ ...layer, status: 'empty' }] },
display: { scope: 'in view', caveat },
items: [],
},
{},
);
return voiceCardView(
[
{
type: 'action-call',
callId: 'c1',
name: 'analyst_query',
arguments: {},
},
{
type: 'action-result',
callId: 'c1',
name: 'analyst_query',
result: shaped,
},
].reduce(reduceVoiceCard, initialVoiceCardState()),
);
}
function notesFor(layerKey, caveat) {
return viewFor({ layerKey }, caveat).result.notes;
}
function inLocale(locale, run) {
setLocale(locale);
try {
run();
} finally {
setLocale('en');
}
}
// Caveat = `${layerKey}: ${getEmptyNote(layerKey)}` (data/analystEngine.js;
// gevActions.js getEmptyNote → stats.coverage || stats.statusMessage).
const cases = [
['local-adsb', 'local-adsb: feeds 1090, 978 stale · 0 heard'],
['local-adsb', 'local-adsb: feed 1090 stale · 0 heard'],
['transit', 'transit: TTC 0 (95s old)'],
['transit', 'transit: TTC 0 · GO Transit 0 (95s old)'],
];
// Transit's coverage text when nothing is in view (transit/queries.js).
const transitCoverage = [
'transit: Fly below 3,000 km to a covered region',
'transit: No feed here yet · 12 regions available',
];
for (const locale of ['es', 'ru']) {
for (const [layerKey, caveat] of cases) {
test(`${locale}: analyst note "${caveat}" is translated`, () => {
inLocale(locale, () => {
const notes = notesFor(layerKey, caveat);
assert.ok(notes.length > 0, 'the note is shown');
assert.doesNotMatch(notes.join(' | '), /\b(stale|heard|old)\b/);
});
});
}
test(`${locale}: Transit's empty note is translated on the analyst card`, () => {
inLocale(locale, () => {
for (const caveat of transitCoverage) {
const view = viewFor(
{ layerKey: 'transit', source: 'GTFS-RT' },
caveat,
);
const text = JSON.stringify(view.result);
assert.doesNotMatch(
text,
/Fly below|No feed here yet|regions available/,
text,
);
}
});
});
}
for (const [layerKey, caveat] of cases) {
test(`en: analyst note "${caveat}" keeps its status line`, () => {
setLocale('en');
const line = caveat.slice(caveat.indexOf(': ') + 2);
assert.ok(
notesFor(layerKey, caveat).some((note) => note.endsWith(`: ${line}`)),
);
});
}
+496
View File
@@ -0,0 +1,496 @@
import { after, before, test } from 'node:test';
import assert from 'node:assert/strict';
import { DataLayerManager } from '../data/manager.js';
import { CCTV_FOCUS_RESULT } from '../layers/cctv/index.js';
import { controlCctv, createGevActionRunner } from './gevActions.js';
const originalWindow = globalThis.window;
before(() => {
globalThis.window ??= { clearTimeout, setTimeout, requestIdleCallback: null };
});
after(() => {
if (originalWindow === undefined) delete globalThis.window;
else globalThis.window = originalWindow;
});
function viewerFixture() {
return {
clock: { onTick: { addEventListener: () => () => {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
camera: { moveEnd: { addEventListener() {} } },
};
}
function deferred() {
let resolve;
const promise = new Promise((done) => {
resolve = done;
});
return { promise, resolve };
}
function runnerFixture(t, dataManager, options = {}) {
const runner = createGevActionRunner({
viewer: viewerFixture(),
styleManager: {},
dataManager,
...options,
});
t.after(() => runner.dispose());
return runner;
}
// Only the inert module's lifecycle completion is controlled. The manager's
// intent queue, cancellation, compensation, events, and runner are production.
async function lifecycleFixture(t, targetEnabled) {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const calls = [];
const gates = [];
const pending = [];
let nextGate = null;
let moduleEnabled = false;
const phase = targetEnabled ? 'enable' : 'disable';
const lifecycle = async (name, options = {}) => {
calls.push(name);
if (name === phase && nextGate) {
const gate = nextGate;
nextGate = null;
gate.entered.resolve(options.signal);
await gate.release.promise;
if (gate.outcome instanceof Error) throw gate.outcome;
if (gate.outcome === false) return false;
}
moduleEnabled = name === 'enable';
return true;
};
manager.register({
id: 'cctv',
name: 'CCTV',
source: 'test',
updateInterval: -1,
init: async () => true,
enable: (_viewer, options) => lifecycle('enable', options),
disable: (_viewer, options) => lifecycle('disable', options),
update: async () => true,
getStats: () => ({ count: 0, lastUpdate: 0 }),
});
if (!targetEnabled)
assert.equal(await manager.setEnabled('cctv', true), true);
calls.length = 0;
const events = [];
const unsubscribe = manager.subscribe((event) => events.push(event));
const runner = runnerFixture(t, manager, { viewer });
t.after(async () => {
for (const gate of gates) gate.release.resolve();
await Promise.allSettled(pending);
unsubscribe();
await manager.destroyAll();
});
return {
manager,
events,
calls,
get moduleEnabled() {
return moduleEnabled;
},
gate(outcome = true) {
const gate = { entered: deferred(), release: deferred(), outcome };
gates.push(gate);
nextGate = gate;
return gate;
},
run(action, options = {}) {
const work = runner('control_cctv', { action }, options);
pending.push(work);
return work;
},
};
}
function assertLifecycle(result, manager, enabled, uncertain = false) {
assert.equal(result.action, 'control_cctv');
assert.equal(result.layerId, 'cctv');
assert.equal(result.enabled, enabled);
assert.equal(result.lifecycleState, enabled ? 'enabled' : 'disabled');
assert.equal(result.lifecycleUncertain, uncertain);
assert.deepEqual(manager.getLayerLifecycleState('cctv'), {
enabled,
lifecycleState: result.lifecycleState,
uncertain,
});
}
const terminalTypes = new Set([
'visibility',
'visibility-cancelled',
'visibility-failed',
'visibility-blocked',
]);
for (const action of ['enable', 'disable']) {
const targetEnabled = action === 'enable';
for (const outcome of [
'success',
'false',
'throw',
'caller-abort',
'resource-abort',
'late-abort',
]) {
test(
`CCTV ${action}: ${outcome} preserves the manager outcome`,
{ timeout: 2000 },
async (t) => {
const h = await lifecycleFixture(t, targetEnabled);
const controller = new AbortController();
const failure =
outcome === 'resource-abort'
? Object.assign(new Error('fixture resource ended'), {
name: 'AbortError',
})
: outcome === 'throw'
? new Error('fixture lifecycle failed')
: null;
const gate = h.gate(failure || (outcome === 'false' ? false : true));
let lateAbortObserved = false;
if (outcome === 'late-abort') {
t.after(
h.manager.subscribe((event) => {
if (event.type === 'visibility' && event.origin === 'voice') {
lateAbortObserved = true;
controller.abort();
}
}),
);
}
let settled = false;
// Uppercase exercises the existing case-insensitive action contract.
const work = h.run(
outcome === 'success' ? action.toUpperCase() : action,
{
signal: controller.signal,
isCurrent: () => !controller.signal.aborted,
},
);
void work.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
const moduleSignal = await gate.entered.promise;
assert.equal(
settled,
false,
'the tool must wait for lifecycle completion',
);
if (outcome === 'caller-abort') {
controller.abort();
assert.ok(
moduleSignal,
'the manager must receive a lifecycle signal',
);
assert.equal(
moduleSignal.aborted,
true,
'caller abort must reach the manager lifecycle',
);
}
gate.release.resolve();
const result = await work;
const committed = outcome === 'success' || outcome === 'late-abort';
const cancelled =
outcome === 'caller-abort' || outcome === 'resource-abort';
assert.equal(result.ok, committed);
assert.equal(result.cancelled, cancelled ? true : undefined);
assertLifecycle(
result,
h.manager,
committed ? targetEnabled : !targetEnabled,
!targetEnabled && (outcome === 'false' || outcome === 'throw'),
);
assert.equal(
h.moduleEnabled,
committed ? targetEnabled : !targetEnabled,
'the module must match the committed or compensated manager state',
);
if (cancelled) {
assert.equal(result.cancellationReason, outcome);
assert.equal(result.phase, action);
assert.equal(result.successorIntentEpoch, null);
assert.equal(result.successorEnabled, null);
assert.equal(result.successorOrigin, null);
} else if (!committed) {
assert.equal(result.error, `Could not ${action} the requested layer`);
}
const terminal = h.events.filter(
(event) => event.origin === 'voice' && terminalTypes.has(event.type),
);
assert.equal(
terminal.length,
1,
'each voice intent has one terminal manager event',
);
assert.equal(
terminal[0].type,
committed
? 'visibility'
: cancelled
? 'visibility-cancelled'
: 'visibility-failed',
);
if (outcome === 'resource-abort')
assert.equal(controller.signal.aborted, false);
if (outcome === 'late-abort') assert.equal(lateAbortObserved, true);
},
);
}
test(
`CCTV ${action}: already-aborted caller cannot enter lifecycle work`,
{ timeout: 2000 },
async (t) => {
const h = await lifecycleFixture(t, targetEnabled);
const controller = new AbortController();
controller.abort();
const result = await h.run(action, { signal: controller.signal });
assert.equal(result.ok, false);
assert.equal(result.cancelled, true);
assert.equal(result.cancellationReason, 'caller-abort');
assert.equal(result.phase, 'queued');
assertLifecycle(result, h.manager, !targetEnabled);
assert.equal(h.moduleEnabled, !targetEnabled);
assert.deepEqual(h.calls, []);
assert.equal(
h.events.filter((event) => terminalTypes.has(event.type)).length,
1,
);
},
);
}
test(
'CCTV toggles retain missing-module errors without calling visibility',
{ timeout: 2000 },
async (t) => {
for (const layers of [new Map(), new Map([['cctv', {}]])]) {
const runner = runnerFixture(t, {
layers,
getAll: () => [],
setEnabled: () =>
assert.fail('missing CCTV must not dispatch visibility'),
});
for (const action of ['enable', 'disable']) {
assert.deepEqual(await runner('control_cctv', { action }), {
ok: false,
action: 'control_cctv',
error: 'CCTV layer unavailable',
});
}
}
},
);
test('exported controlCctv refuses toggles without calling the manager', async () => {
const manager = {
layers: new Map([['cctv', { module: {} }]]),
setEnabled: () => assert.fail('direct helper must not change visibility'),
isEnabled: () => assert.fail('direct toggle must not read enabled state'),
};
for (const action of ['ENABLE', 'disable']) {
const result = await controlCctv(manager, { action });
assert.equal(result.ok, false);
assert.equal(result.action, 'control_cctv');
assert.match(result.error, /set_layer_visibility/);
}
});
test(
'CCTV fallback managers report rejection, transitional state, and uncertainty',
{ timeout: 2000 },
async (t) => {
for (const enabled of [true, false]) {
const action = enabled ? 'enable' : 'disable';
for (const mode of [
'rejection',
'mismatch',
'transitional',
'uncertain',
]) {
const actualEnabled =
mode === 'rejection' || mode === 'mismatch' ? !enabled : enabled;
const lifecycleState =
mode === 'transitional'
? enabled
? 'enabling'
: 'disabling'
: actualEnabled
? 'enabled'
: 'disabled';
const manager = {
layers: new Map([['cctv', { module: {} }]]),
getAll: () => [{ id: 'cctv', name: 'CCTV' }],
isEnabled: () => actualEnabled,
getLayerLifecycleState: () => ({
enabled: actualEnabled,
lifecycleState,
uncertain: mode === 'uncertain',
}),
async setEnabled() {
if (mode === 'rejection') throw new Error('adapter rejected');
return true;
},
};
const runner = runnerFixture(t, manager);
const result = await runner('control_cctv', { action });
assert.equal(result.ok, false, `${action}: ${mode}`);
assert.equal(result.action, 'control_cctv');
assert.equal(result.enabled, actualEnabled);
assert.equal(result.lifecycleState, lifecycleState);
assert.equal(result.lifecycleUncertain, mode === 'uncertain');
assert.equal(
result.error,
mode === 'rejection'
? 'adapter rejected'
: `Could not ${action} the requested layer`,
);
}
}
},
);
test(
'CCTV visibility composes only its outer speech envelope',
{ timeout: 2000 },
async (t) => {
let enabled = false;
const calls = [];
const runner = runnerFixture(
t,
{
layers: new Map([['cctv', { module: {} }]]),
getAll: () => [{ id: 'cctv', name: 'CCTV' }],
isEnabled: () => enabled,
async setEnabled(_id, value) {
enabled = value;
return true;
},
},
{
speechBuilders: {
control_cctv(result, args) {
calls.push(['cctv', result.action, args.action]);
return {
say: 'CCTV response',
display: { title: 'CCTV', lines: [] },
};
},
set_layer_visibility() {
calls.push(['generic']);
return { say: 'Layer response' };
},
},
},
);
const result = await runner('control_cctv', { action: 'enable' });
assert.equal(result.ok, true);
assert.equal(result.action, 'control_cctv');
assert.equal(result.say, 'CCTV response');
assert.deepEqual(result.display, { title: 'CCTV', lines: [] });
assert.deepEqual(calls, [['cctv', 'control_cctv', 'enable']]);
},
);
test(
'CCTV non-toggle actions retain camera navigation and durable coverage',
{ timeout: 2000 },
async (t) => {
const calls = [];
const cameras = [
{ id: 'a', name: 'Camera A' },
{ id: 'b', name: 'Camera B' },
];
let activeCameraId = 'a';
let coverageMode = 'viewshed';
const module = {
getUIState: () => ({
cameras,
activeCameraId,
activeCamera: cameras.find((camera) => camera.id === activeCameraId),
coverageMode,
showCoverage: coverageMode !== 'off',
}),
selectCamera(id) {
calls.push(['select', id]);
activeCameraId = id;
return true;
},
cycleCamera(step) {
calls.push(['cycle', step]);
activeCameraId = step > 0 ? 'b' : 'a';
return activeCameraId;
},
focusNearest(options) {
calls.push(['nearest', options]);
activeCameraId = 'a';
return 'a';
},
focusCamera(id, duration) {
calls.push(['focus', id, duration]);
return CCTV_FOCUS_RESULT.TRACKING_HOLDS_VIEW;
},
};
let superseded = 0;
const runner = runnerFixture(
t,
{
layers: new Map([['cctv', { module }]]),
isEnabled: () => true,
setEnabled: () =>
assert.fail('non-toggle action must not dispatch visibility'),
setLayerParams(id, params, options) {
calls.push(['params', id, params, options]);
coverageMode = params.coverageMode;
},
},
{
styleManager: {
supersedeDeferredNavigation() {
superseded += 1;
},
},
},
);
for (const args of [
{ action: 'select', cameraQuery: 'Camera B' },
{ action: 'next' },
{ action: 'prev' },
{ action: 'nearest' },
{ action: 'focus' },
]) {
const result = await runner('control_cctv', args);
assert.equal(result.ok, false);
assert.equal(result.action, 'control_cctv');
assert.match(result.error, /tracking holds the view/);
}
const coverage = await runner('control_cctv', { action: 'coverage' });
assert.equal(coverage.ok, true);
assert.equal(coverage.coverageMode, 'off');
assert.equal(superseded, 5, 'coverage must not acquire camera navigation');
assert.deepEqual(calls, [
['select', 'b'],
['focus', 'b', 1.8],
['cycle', 1],
['focus', 'b', 1.8],
['cycle', -1],
['focus', 'a', 1.8],
['nearest', { focus: false }],
['focus', 'a', 1.8],
['focus', 'a', undefined],
['params', 'cctv', { coverageMode: 'off' }, { origin: 'voice' }],
]);
},
);
+3 -3
View File
@@ -1,5 +1,5 @@
export const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']);
export const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key';
const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']);
const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key';
export const CLOUD_VOICE_AUTH_STORAGE_KEY = 'godsEyeView.voice.cloudAuth';
function storageRef(storage) {
@@ -11,7 +11,7 @@ function storageRef(storage) {
}
}
export function normalizeCloudVoiceAuthMode(mode) {
function normalizeCloudVoiceAuthMode(mode) {
const normalized = String(mode || '')
.trim()
.toLowerCase();
+46 -5
View File
@@ -1,10 +1,51 @@
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
import { createRealtimeSession } from './realtimeSession.js';
import { createGeminiSession } from './geminiSession.js';
import { createProviderCommands } from './providerCommands.js';
import { createVoicePanelHost } from './panel/host.js';
/** Default composition; callers may supply another session adapter factory. */
export function createVoiceCommands(options) {
return bindVoiceCommands({
createSession: createRealtimeSession,
...options,
});
export function createVoiceCommands(options = {}) {
const panelHost =
options.panel === true
? createVoicePanelHost({
storage: options.panelStorage,
clock: options.panelClock,
readCockpitState: options.styleManager?.getCockpitState
? () => options.styleManager.getCockpitState()
: undefined,
})
: null;
const bind = (settings) =>
bindVoiceCommands({
...settings,
...(panelHost
? {
panelHost,
nativePanelProvider: settings.createController
? null
: settings.createSession === createRealtimeSession
? 'openai'
: settings.createSession === createGeminiSession
? 'gemini'
: null,
}
: {}),
});
let commands;
// Generic hosts retain the existing default; standalone opts in explicitly.
if (!options.createSession && options.supportedProviders !== undefined)
commands = createProviderCommands(options, {
bind,
factories: { openai: createRealtimeSession, gemini: createGeminiSession },
});
else commands = bind({ createSession: createRealtimeSession, ...options });
if (panelHost) {
const lifetime = commands.signal ?? commands.session?.signal;
lifetime?.addEventListener('abort', () => panelHost.destroy(), {
once: true,
});
if (lifetime?.aborted) panelHost.destroy();
}
return commands;
}
+321 -91
View File
@@ -1,95 +1,325 @@
/** Build the voice control independently of its connection backend. */
export function createVoiceControl({ reset = false } = {}) {
let root = document.getElementById('gev-voice-control');
if (root && reset) {
root.remove();
root = null;
}
if (!root) {
root = document.createElement('div');
root.id = 'gev-voice-control';
root.dataset.status = 'idle';
root.dataset.speaker = 'idle';
root.innerHTML = `
<div class="gev-voice-heading">
<div class="gev-voice-kicker">AI AGENT</div>
<div id="gev-voice-status">OFF</div>
<div class="gev-voice-cost">
<button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="Voice model tier — applies next session">STD</button>
<span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="Estimated session cost">~$0.00</span>
</div>
</div>
<button id="gev-voice-button" type="button" aria-label="Voice control — activate to toggle voice; hold Space to speak" aria-describedby="gev-voice-help">
<span class="gev-mic-orbit"><img src="/mic.svg" alt="" /></span>
<span class="gev-mic-label">ON/OFF</span>
</button>
<div class="gev-voice-visualizer" aria-hidden="true">
${Array.from({ length: 15 }, (_, index) => `<span style="--bar:${index}"></span>`).join('')}
</div>
<div class="gev-voice-readout">
<div id="gev-voice-detail">VOICE STANDBY</div>
</div>
<div id="gev-voice-help" class="gev-voice-help-tray" role="tooltip">
<span class="gev-voice-help-kicker">VOICE CONTROL</span>
<span class="gev-voice-help-detail">Hold Space to speak · tap Space to activate focused controls</span>
</div>
<section id="gev-voice-card" class="gev-voice-card" hidden aria-label="Voice assistant">
<div class="gev-voice-card-header">
<span class="gev-voice-card-kicker">VOICE</span>
<span id="gev-voice-card-phase" class="gev-voice-card-phase"></span>
<button id="gev-voice-card-dismiss" class="gev-voice-card-dismiss" type="button" aria-label="Dismiss voice card">×</button>
</div>
<p id="gev-voice-card-user" class="gev-voice-card-caption" data-role="user" hidden><span class="gev-voice-card-who">YOU</span><span id="gev-voice-card-user-text" class="gev-voice-card-text"></span></p>
<p id="gev-voice-card-pointer" class="gev-voice-card-pointer" hidden><span id="gev-voice-card-pointer-kind" class="gev-voice-card-who">THIS</span><span class="gev-voice-card-sr">: </span><span id="gev-voice-card-pointer-text" class="gev-voice-card-pointer-label"></span></p>
<p id="gev-voice-card-assistant" class="gev-voice-card-caption" data-role="assistant" hidden><span class="gev-voice-card-who">GEV</span><span id="gev-voice-card-assistant-text" class="gev-voice-card-text"></span></p>
<ol id="gev-voice-card-plan" class="gev-voice-card-plan" aria-label="Plan" hidden></ol>
<div id="gev-voice-card-result" class="gev-voice-card-result" hidden>
<div id="gev-voice-card-result-title" class="gev-voice-card-result-title"></div>
<div id="gev-voice-card-chips" class="gev-voice-card-chips" hidden></div>
<ul id="gev-voice-card-lines" class="gev-voice-card-lines"></ul>
<ol id="gev-voice-card-referents" class="gev-voice-card-referents" aria-label="Referenced items" hidden></ol>
<details id="gev-voice-card-notes" class="gev-voice-card-notes" hidden>
<summary>NOTES &amp; SOURCES</summary>
<ul id="gev-voice-card-note-list" class="gev-voice-card-lines"></ul>
</details>
</div>
<div id="gev-voice-card-live" class="gev-voice-card-sr" aria-live="polite" aria-atomic="true"></div>
</section>
<div class="gev-voice-error-tray" role="alert" aria-live="assertive">
<div class="gev-voice-error-header">
<span>VOICE SYSTEM ERROR</span>
<button class="gev-voice-error-dismiss" type="button">DISMISS</button>
</div>
<div id="gev-voice-error-detail"></div>
<div class="gev-voice-error-hint">Check microphone permission and network access, then try again.</div>
</div>
`;
const commandDock = document.getElementById('command-dock');
if (commandDock) {
const locationBar = document.getElementById('location-bar');
const controlPanel = document.getElementById('control-panel');
commandDock.appendChild(root);
if (locationBar) commandDock.insertBefore(locationBar, root);
if (controlPanel) commandDock.appendChild(controlPanel);
} else {
document.body.appendChild(root);
}
root
.querySelector('.gev-voice-error-dismiss')
?.addEventListener('click', () => {
root.classList.add('error-dismissed');
});
import { nextTick } from 'vue';
import { onLocaleChange, t } from '../i18n/index.js';
import { useVoiceStore } from '../ui/stores/voice.js';
import { applicationPinia } from '../ui/vueRuntime.js';
// The assistant's name on its card. i18n-ignore: product name
export const ASSISTANT_NAME = 'GEV';
/** Bars in the dock waveform. */
export const VOICE_VISUALIZER_BARS = 15;
/**
* Short state label for the narrow voice heading. The `voice` context lets
* languages use shorter words; unknown states show upper-cased.
* @param {string} status machine state (dataset.status)
* @returns {string}
*/
export function voiceStatusLabel(status) {
switch (status) {
case 'idle':
return t('OFF', { context: 'voice' });
case 'connecting':
return t('CONNECTING', { context: 'voice' });
case 'listening':
return t('LISTENING', { context: 'voice' });
case 'executing':
return t('EXECUTING', { context: 'voice' });
case 'error':
return t('ERROR', { context: 'voice' });
default:
return String(status || '').toUpperCase();
}
}
/** The card before its first render: hidden, with its default labels. */
function initialCard() {
return {
root,
button: root.querySelector('#gev-voice-button'),
buttonLabel: root.querySelector('.gev-mic-label'),
status: root.querySelector('#gev-voice-status'),
detail: root.querySelector('#gev-voice-detail'),
helpDetail: root.querySelector('.gev-voice-help-detail'),
errorDetail: root.querySelector('#gev-voice-error-detail'),
tierButton: root.querySelector('#gev-voice-tier'),
costValue: root.querySelector('#gev-voice-cost-value'),
card: root.querySelector('#gev-voice-card'),
visible: false,
busy: null,
phase: '',
user: { visible: false, text: '', label: t('YOU') },
pointer: { visible: false, kind: t('THIS'), text: '' },
assistant: { visible: false, text: '' },
steps: [],
result: {
visible: false,
title: '',
chips: [],
lines: [],
referents: [],
notes: [],
},
announce: '',
};
}
/** The control as it first renders, before any owner writes to it. */
function initialState(panel) {
return {
panel,
root: {
status: 'idle',
speaker: 'idle',
provider: null,
providerChoice: null,
pushToTalk: null,
microphone: null,
rowGrace: false,
errorDismissed: false,
},
// The provider field and its select.
provider: {
hidden: true,
options: [],
value: '',
title: () => t('Changing provider reconnects voice if it is on.'),
},
tier: {
hidden: false,
text: () => t('STD'),
pressed: false,
title: () => t('Voice model tier — applies next session'),
},
cost: {
hidden: false,
text: '~$0.00',
level: 'ok',
title: () => t('Estimated session cost'),
},
costWarning: { hidden: true, text: '', level: null, title: null },
button: {
label: () =>
t('Voice control — activate to toggle voice; hold Space to speak'),
pressed: null,
},
mic: { text: () => t('ON/OFF') },
status: { text: () => voiceStatusLabel('idle') },
detail: { text: () => t('VOICE STANDBY'), title: null },
help: {
text: () =>
t('Hold Space to speak · tap Space to activate focused controls'),
},
error: { text: '' },
voicePanel: {
hidden: true,
view: null,
previewHidden: false,
settingsHidden: true,
settingsExpanded: false,
selected: 'preview',
},
previewEmpty: {
hidden: false,
text: () => t('Voice replies appear here.'),
},
preferences: {
previewChecked: true,
previewNote: '',
timeoutChoice: '5',
timeoutMinutes: '5',
timeoutInputHidden: false,
timeoutNote: () => t('These preferences stay in this browser.'),
timeoutStatus: '',
// Bumped by every owner write of the fields, so the view restores a
// refused edit but leaves a draft alone across other renders.
revision: 0,
},
card: initialCard(),
};
}
// The rendered elements the presenter focuses or measures, by name.
const ELEMENT_IDS = Object.freeze({
root: 'gev-voice-control',
button: 'gev-voice-button',
provider: 'gev-voice-provider',
voicePanel: 'gev-voice-panel',
previewPanel: 'gev-voice-preview-panel',
settingsPanel: 'gev-voice-settings-panel',
previewTab: 'gev-voice-preview-tab',
settingsTab: 'gev-voice-settings-tab',
settingsButton: 'gev-voice-settings-button',
card: 'gev-voice-card',
});
let controlCount = 0;
const sameBars = (a, b) =>
a === b ||
(Array.isArray(a) &&
Array.isArray(b) &&
a.length === b.length &&
a.every(
(bar, index) =>
bar.level === b[index].level && bar.opacity === b[index].opacity,
));
/**
* The voice control, independent of its connection backend. Session owners
* write its state by part (`update('detail', {text})`) and listen for what
* the person does (`on('button', fn)`); the command dock's VoiceControl view
* renders it from the voice store. A newer control replaces this one there.
* @param {object} [options]
* @param {boolean} [options.panel] Show the Voice Settings panel and preview.
*/
export function createVoiceControl({
panel = false,
store = useVoiceStore(applicationPinia()),
} = {}) {
const id = ++controlCount;
const listeners = new Map();
// The fields given as functions, by part: they are called again when the
// locale changes, so the control's words follow the language.
const wording = new Map();
const word = (part, patch) => {
let fields = wording.get(part);
let worded = patch;
for (const [field, value] of Object.entries(patch)) {
if (typeof value !== 'function') {
fields?.delete(field);
continue;
}
if (!fields) wording.set(part, (fields = new Map()));
fields.set(field, value);
if (worded === patch) worded = { ...patch };
worded[field] = value();
}
return worded;
};
let state = initialState(Boolean(panel));
for (const [part, value] of Object.entries(state))
if (part !== 'card' && value && typeof value === 'object')
state[part] = word(part, value);
// The default card's labels follow the locale until a card replaces it.
let defaultCard = true;
// Each bar's `--audio-level` and `--audio-opacity`, or null at rest.
let bars = null;
let removed = false;
const owns = () => !removed && store.controls === ui;
const publish = () => {
if (owns()) store.publish({ ...state, id });
};
const element = (name) =>
owns()
? (globalThis.document?.getElementById(ELEMENT_IDS[name]) ?? null)
: null;
const shown = (node) =>
Boolean(
node?.checkVisibility?.({
opacityProperty: true,
visibilityProperty: true,
}) ?? node,
);
const ui = {
get panel() {
return state.panel;
},
/** The current state; read-only, replaced on every update. */
get state() {
return state;
},
/**
* Merge `patch` into one part of the control. A field given as a
* function is shown as what it returns, and called again when the
* locale changes; give translated words that way.
*/
update(part, patch) {
if (removed) return;
state = { ...state, [part]: { ...state[part], ...word(part, patch) } };
publish();
},
/** Replace one part of the control. */
replace(part, value) {
if (removed) return;
wording.delete(part);
if (part === 'card') defaultCard = false;
state = { ...state, [part]: value };
publish();
},
/**
* Listen for an interaction the view reports: `button`, `tier`,
* `provider-change`, `settings-button`, `settings-close`, `tab`,
* `tab-keydown`, `panel-keydown`, `pointerenter`, `pointerleave`,
* `preview-change`, `timeout-select`, `timeout-input`, `card-dismiss`;
* and `locale` after a locale change, for an owner's own words.
* @returns {() => void} Removes the listener.
*/
on(name, listener) {
if (!listeners.has(name)) listeners.set(name, new Set());
listeners.get(name).add(listener);
return () => listeners.get(name)?.delete(listener);
},
/** The view reports an interaction; listeners run in the order added. */
emit(name, ...args) {
if (removed) return;
for (const listener of [...(listeners.get(name) ?? [])])
listener(...args);
},
/** The waveform's bars, or null at rest. */
get bars() {
return bars;
},
/**
* Show the waveform's bars (null rests it). Published apart from the
* rest of the control, and only when they change.
* @param {?Array<{level: string, opacity: string}>} next
*/
setBars(next) {
if (removed || sameBars(bars, next)) return;
bars = next;
if (owns()) store.publishBars(bars);
},
/** The error tray's DISMISS. */
dismissError() {
ui.update('root', { errorDismissed: true });
},
/** The rendered element by name, or null before it renders. */
element,
/** Whether the named element is rendered and visible on the page. */
visible: (name) => shown(element(name)),
/**
* Once the view has rendered the latest state, focus the first of the
* named elements that is visible, or with `anyVisibility`, that is
* rendered (focus may be what reveals it).
* @param {string[]} names
* @param {{anyVisibility?: boolean}} [options]
*/
focus(names, { anyVisibility = false } = {}) {
return nextTick(() => {
const target = names
.map(element)
.find((node) => node && (anyVisibility || shown(node)));
target?.focus?.();
return target ?? null;
});
},
/** Remove the control from the dock, unless a newer one replaced it. */
remove() {
if (removed) return;
if (store.controls === ui) {
store.connect(null);
store.publish(null);
store.publishBars(null);
}
removed = true;
offLocale();
listeners.clear();
},
};
const offLocale = onLocaleChange(() => {
if (removed) return;
for (const [part, fields] of wording) {
const patch = {};
for (const [field, value] of fields) patch[field] = value();
state = { ...state, [part]: { ...state[part], ...patch } };
}
if (defaultCard) state = { ...state, card: initialCard() };
publish();
ui.emit('locale');
});
store.connect(ui);
store.publishBars(null);
publish();
return ui;
}
+1 -1
View File
@@ -18,7 +18,7 @@
import { finiteOrNull } from './pointerContext.js';
export const POINTER_SENTINEL = 'pointer';
const POINTER_SENTINEL = 'pointer';
const TRACKABLE_LAYERS = new Set([
'flights',
+142
View File
@@ -0,0 +1,142 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
deferred,
tick,
} from '../testSupport/geminiNativeHarness.mjs';
function unreadResponse(bytes = null) {
const state = { reads: 0, cancelled: 0 };
const response = new Response(
new ReadableStream(
{
pull(controller) {
state.reads++;
if (bytes) controller.enqueue(bytes);
},
cancel() {
state.cancelled++;
},
},
{ highWaterMark: 0 },
),
);
return { response, state };
}
test('already aborted token request does not acquire or read a response', async (t) => {
let fetches = 0;
const h = harness({
fetchImpl: () => {
fetches++;
throw Error('must not fetch');
},
});
t.after(() => h.c.stop());
const abort = new AbortController();
abort.abort();
await assert.rejects(h.c.requestToken(abort.signal), { name: 'AbortError' });
assert.equal(fetches, 0);
});
for (const phase of [
'before response',
'between resolution and continuation',
]) {
test(`aborted token acquisition closes the unread body ${phase}`, async (t) => {
const gate = deferred();
const h = harness({ fetchImpl: () => gate.promise });
t.after(() => h.c.stop());
const abort = new AbortController();
const { response, state } = unreadResponse();
const result = assert.rejects(h.c.requestToken(abort.signal), {
name: 'AbortError',
});
if (phase === 'before response') abort.abort();
gate.resolve(response);
if (phase !== 'before response') queueMicrotask(() => abort.abort());
await result;
await tick();
assert.deepEqual(state, { reads: 0, cancelled: 1 });
});
}
test('token body cap cancels a native response stream', async (t) => {
const { response, state } = unreadResponse(new Uint8Array(256 * 1024 + 1));
const h = harness({ fetchImpl: async () => response });
t.after(() => h.c.stop());
await assert.rejects(
h.c.requestToken(new AbortController().signal),
/too large|exceed|limit/i,
);
assert.equal(state.cancelled, 1);
assert.equal(state.reads, 1);
});
test('token body read ends promptly on caller abort and releases the reader', async (t) => {
const { response, state } = unreadResponse();
const h = harness({ fetchImpl: async () => response });
t.after(() => h.c.stop());
const abort = new AbortController();
const result = assert.rejects(h.c.requestToken(abort.signal), {
name: 'AbortError',
});
await tick();
assert.equal(state.reads, 1);
abort.abort();
await result;
assert.equal(state.cancelled, 1);
assert.equal(response.body.locked, false);
});
test('permission grant resolved just before Stop releases its tracks and preserves a fresh start', async (t) => {
const gate = deferred();
const h = harness({ getUserMedia: () => gate.promise });
t.after(() => h.c.stop());
const stream = h.makeStream();
const start = h.c.start();
gate.resolve(stream);
queueMicrotask(() => h.c.stop());
await start;
await tick();
assert.equal(h.tracks[0].readyState, 'ended');
assert.equal(h.requests.length, 0);
assert.equal(h.sockets.length, 0);
assert.equal(h.captures[0].closed, true);
h.c.getUserMedia = async () => h.makeStream();
await h.c.start();
assert.equal(h.c.status, 'listening');
assert.equal(h.tracks[1].readyState, 'live');
assert.equal(h.captures[1].closed, false);
});
test('late audio initialization cannot connect a retired microphone or close successor audio', async (t) => {
const gate = deferred();
const h = harness();
t.after(() => h.c.stop());
const createAudio = h.c.createAudio;
let first = true;
h.c.createAudio = (options) => {
const audio = createAudio(options);
if (first) {
first = false;
audio.initialize = () => gate.promise;
audio.connectMicrophone = () =>
assert.fail('retired initialization must not connect');
}
return audio;
};
const oldStart = h.c.start();
await tick();
h.c.stop();
await h.c.start();
gate.resolve();
await oldStart;
await tick();
assert.equal(h.c.status, 'listening');
assert.equal(h.captures[0].closed, true);
assert.equal(h.captures[1].closed, false);
assert.equal(h.tracks[0].readyState, 'ended');
assert.equal(h.tracks[1].readyState, 'live');
});
+380
View File
@@ -0,0 +1,380 @@
import { msg } from '../i18n/marker.js';
const MAX_AUDIO_BYTES = 2 * 1024 * 1024;
const MAX_QUEUED_SECONDS = 60;
const MAX_QUEUED_SOURCES = 256;
const PTT_WINDOW_SAMPLES = 160; // 10 ms at the worklet's fixed 16 kHz.
const PTT_ENERGY_FLOOR = 0.002;
const PTT_PRE_ROLL_BYTES = 16000 * 2 * 0.4;
/** Admit manual input only after sustained energy; not an utterance detector. */
function pttCapture(captureId) {
return {
captureId,
admitted: false,
buffers: [],
bytes: 0,
samples: 0,
energy: 0,
aboveFloor: 0,
voicedWindows: 0,
};
}
function hasPttEnergy(state, pcm) {
for (let index = 0; index + 1 < pcm.byteLength; index += 2) {
const sample = pcm.getInt16(index, true) / 32768;
state.energy += sample * sample;
if (Math.abs(sample) >= PTT_ENERGY_FLOOR) state.aboveFloor++;
if (++state.samples !== PTT_WINDOW_SAMPLES) continue;
// Requiring occupancy as well as RMS rejects isolated high-peak key clicks.
const voiced =
state.energy / state.samples >= PTT_ENERGY_FLOOR ** 2 &&
state.aboveFloor >= PTT_WINDOW_SAMPLES / 8;
state.voicedWindows = voiced ? state.voicedWindows + 1 : 0;
state.samples = state.energy = state.aboveFloor = 0;
if (state.voicedWindows >= 2) return true;
}
return false;
}
export function pcmBytesToBase64(buffer) {
const bytes = new Uint8Array(buffer);
let binary = '';
for (let offset = 0; offset < bytes.length; offset += 8192) {
binary += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
}
return btoa(binary);
}
export function decodeGeminiPcm(data, mimeType = 'audio/pcm;rate=24000') {
if (
typeof data !== 'string' ||
data.length > 4 * Math.ceil(MAX_AUDIO_BYTES / 3)
)
throw new Error(msg('Gemini returned an oversized audio chunk'));
const match = /^audio\/pcm(?:;rate=(\d+))?$/.exec(mimeType);
if (!match)
throw new Error(msg('Gemini returned an unsupported audio format'));
const sampleRate = Number(match[1] || 24000);
if (!Number.isInteger(sampleRate) || sampleRate < 8000 || sampleRate > 96000)
throw new Error(msg('Gemini returned an invalid audio sample rate'));
const binary = atob(data);
if (binary.length > MAX_AUDIO_BYTES)
throw new Error(msg('Gemini returned an oversized audio chunk'));
if (!binary.length || binary.length % 2)
throw new Error(msg('Gemini returned incomplete PCM audio'));
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index++)
bytes[index] = binary.charCodeAt(index);
const view = new DataView(bytes.buffer);
const samples = new Float32Array(bytes.length / 2);
for (let index = 0; index < samples.length; index++)
samples[index] = view.getInt16(index * 2, true) / 32768;
return { samples, sampleRate };
}
/** Own capture and playback independently from the Live protocol. */
export function createGeminiAudio({
pushToTalk = false,
onAudioStart,
onAudio,
onAudioEnd,
onAudioDiscard,
onDrain,
onError,
AudioContextClass = globalThis.AudioContext || globalThis.webkitAudioContext,
AudioWorkletNodeClass = globalThis.AudioWorkletNode,
} = {}) {
if (!AudioContextClass || !AudioWorkletNodeClass)
throw new Error(
msg('This browser does not support Gemini microphone audio'),
);
let captureContext;
let playbackContext;
let output;
let meter;
let resumed;
const closeContext = (context) => {
try {
return Promise.resolve(context?.close());
} catch (error) {
return Promise.reject(error);
}
};
try {
// Capture at the device rate; the worklet explicitly resamples to 16 kHz.
captureContext = new AudioContextClass();
playbackContext = new AudioContextClass({ sampleRate: 24000 });
// Resume both synchronously within the mic gesture, before any awaits.
const resume = (context) => {
try {
return Promise.resolve(context.resume());
} catch (error) {
return Promise.reject(error);
}
};
resumed = Promise.all([resume(captureContext), resume(playbackContext)]);
// Permission may remain pending before initialize observes this failure.
void resumed.catch(() => {});
output = playbackContext.createGain();
meter = playbackContext.createMediaStreamDestination();
output.connect(playbackContext.destination);
output.connect(meter);
} catch (error) {
output?.disconnect();
meter?.disconnect();
meter?.stream.getTracks?.().forEach((track) => track.stop());
void Promise.allSettled([
closeContext(captureContext),
closeContext(playbackContext),
]);
throw error;
}
let closed = false;
let closeResult;
let capture = null;
let microphone = null;
let mute = null;
let nextTime = 0;
let inputChunks = 0;
let inputPeak = 0;
let nonSilentInputChunks = 0;
let outputChunks = 0;
let outputSamples = 0;
let completedOutputSamples = 0;
let discardedOutputSamples = 0;
let pendingPtt = null;
let latestPttCaptureId = 0;
let suppressedPttCaptures = 0;
let admittedPttCaptures = 0;
const sources = new Map();
const emitAudio = (buffer, metadata) =>
onAudio?.(
{ data: pcmBytesToBase64(buffer), mimeType: 'audio/pcm;rate=16000' },
metadata,
);
function clearPlayback() {
const oldSources = [...sources];
sources.clear();
nextTime = playbackContext.currentTime;
for (const [source, sampleCount] of oldSources) {
discardedOutputSamples += sampleCount;
source.onended = null;
try {
source.stop();
} catch {
/* Already ended. */
}
source.disconnect();
}
}
return {
outputStream: meter.stream,
get pending() {
return sources.size > 0;
},
async initialize(signal) {
try {
await Promise.all([
resumed,
captureContext.audioWorklet.addModule(
new URL('./geminiCaptureWorklet.js', import.meta.url),
),
]);
signal?.throwIfAborted();
if (closed) throw new DOMException('Voice stopped', 'AbortError');
} catch (error) {
await this.close();
throw error;
}
},
connectMicrophone(stream, enabled, metadata) {
if (closed) throw new DOMException('Voice stopped', 'AbortError');
capture = new AudioWorkletNodeClass(
captureContext,
'gev-gemini-capture',
{
numberOfInputs: 1,
numberOfOutputs: 1,
outputChannelCount: [1],
},
);
capture.port.onmessage = ({ data }) => {
if (closed) return;
// Capture ownership is transport metadata, never part of Live audio.
if (!Number.isSafeInteger(data?.captureId) || data.captureId <= 0)
return;
const metadata = { captureId: data.captureId };
try {
if (data?.type === 'audio' && data.buffer instanceof ArrayBuffer) {
inputChunks++;
const pcm = new DataView(data.buffer);
let peak = 0;
for (let index = 0; index + 1 < pcm.byteLength; index += 2)
peak = Math.max(
peak,
Math.abs(pcm.getInt16(index, true)) / 32768,
);
inputPeak = Math.max(inputPeak, peak);
if (peak >= 0.01) nonSilentInputChunks++;
if (!pushToTalk) emitAudio(data.buffer, metadata);
else {
const state = pendingPtt;
if (state?.captureId !== data.captureId) return;
if (state.admitted) emitAudio(data.buffer, metadata);
else {
state.buffers.push(data.buffer);
state.bytes += data.buffer.byteLength;
if (hasPttEnergy(state, pcm)) {
state.admitted = true;
admittedPttCaptures++;
// Context and activityStart are gated together with PCM.
onAudioStart?.(metadata);
for (const buffer of state.buffers) {
if (closed || pendingPtt !== state) break;
emitAudio(buffer, metadata);
}
state.buffers = [];
state.bytes = 0;
} else {
while (state.bytes > PTT_PRE_ROLL_BYTES) {
state.bytes -= state.buffers.shift().byteLength;
}
}
}
}
} else if (data?.type === 'start') {
if (!pushToTalk) onAudioStart?.(metadata);
else if (data.captureId > latestPttCaptureId) {
latestPttCaptureId = data.captureId;
pendingPtt = pttCapture(data.captureId);
}
} else if (data?.type === 'end') {
if (!pushToTalk) onAudioEnd?.(metadata);
else if (pendingPtt?.captureId === data.captureId) {
const admitted = pendingPtt.admitted;
pendingPtt = null;
if (admitted) onAudioEnd?.(metadata);
else {
suppressedPttCaptures++;
onAudioDiscard?.(metadata);
}
}
}
} catch (error) {
onError?.(error);
}
};
capture.onprocessorerror = () => {
if (!closed)
onError?.(new Error(msg('Microphone audio processing failed')));
};
microphone = captureContext.createMediaStreamSource(stream);
mute = captureContext.createGain();
mute.gain.value = 0;
capture.port.postMessage({
enabled: Boolean(enabled),
captureId: metadata?.captureId,
});
microphone.connect(capture);
capture.connect(mute);
mute.connect(captureContext.destination);
},
setMicrophoneEnabled(enabled, metadata) {
if (closed) return;
capture?.port.postMessage({
enabled: Boolean(enabled),
captureId: metadata?.captureId,
});
},
play(data, mimeType) {
if (closed) return;
// Tiny chunks can exhaust source nodes long before the duration limit.
if (sources.size >= MAX_QUEUED_SOURCES)
throw new Error(
msg('Gemini audio playback fell behind. Reconnect to continue.'),
);
const { samples, sampleRate } = decodeGeminiPcm(data, mimeType);
const start = Math.max(playbackContext.currentTime + 0.02, nextTime);
if (
start + samples.length / sampleRate - playbackContext.currentTime >
MAX_QUEUED_SECONDS
)
throw new Error(
msg('Gemini audio playback fell behind. Reconnect to continue.'),
);
const buffer = playbackContext.createBuffer(
1,
samples.length,
sampleRate,
);
buffer.copyToChannel(samples, 0);
const source = playbackContext.createBufferSource();
source.buffer = buffer;
source.connect(output);
sources.set(source, samples.length);
source.onended = () => {
if (!sources.delete(source)) return;
completedOutputSamples += samples.length;
source.disconnect();
if (!closed && !sources.size) onDrain?.();
};
nextTime = start + buffer.duration;
outputChunks++;
outputSamples += samples.length;
source.start(start);
},
clearPlayback,
diagnostics() {
return {
inputChunks,
inputPeak,
nonSilentInputChunks,
outputChunks,
outputSamples,
completedOutputSamples,
discardedOutputSamples,
queuedSeconds: Math.max(0, nextTime - playbackContext.currentTime),
contextState:
captureContext.state === playbackContext.state
? captureContext.state
: 'suspended',
captureContextState: captureContext.state,
playbackContextState: playbackContext.state,
captureSampleRate: captureContext.sampleRate,
playbackSampleRate: playbackContext.sampleRate,
inputSampleRate: 16000,
suppressedPttCaptures,
admittedPttCaptures,
pttBufferedBytes: pendingPtt?.bytes || 0,
};
},
close() {
if (closed) return closeResult;
closed = true;
pendingPtt = null;
clearPlayback();
if (capture) {
capture.port.onmessage = null;
capture.onprocessorerror = null;
capture.port.close();
capture.disconnect();
}
microphone?.disconnect();
mute?.disconnect();
output.disconnect();
meter.disconnect();
meter.stream.getTracks?.().forEach((track) => track.stop());
closeResult = Promise.allSettled([
closeContext(captureContext),
closeContext(playbackContext),
]).then((results) =>
results.every((result) => result.status === 'fulfilled'),
);
return closeResult;
},
};
}
File diff suppressed because it is too large. Load diff
+791
View File
@@ -0,0 +1,791 @@
// Native automatic activity, reply ownership and expiry lifecycle regressions.
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
const vad = (ws, type) => ws.message({ voiceActivity: { type } });
const onset = (ws) => {
vad(ws, 'ACTIVITY_START');
vad(ws, 'ACTIVITY_END');
};
const split = (ws) => {
onset(ws);
onset(ws);
};
const audio = {
serverContent: {
modelTurn: {
parts: [
{ inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } },
],
},
},
};
const done = (ws) => ws.message({ serverContent: { turnComplete: true } });
const tools = (ws, id) =>
ws.message({
toolCall: {
functionCalls: [
{
id,
name: 'adjust_camera_zoom',
args: { direction: 'out', amount: 'little' },
},
],
},
});
const responses = (ws) =>
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
function close(h) {
h.c.stop();
assert.equal(h.timers.size, 0, 'owned timers released');
}
test('N1 VAD-only split admits the first real answer and completes after playback', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
ws.message(audio);
ws.message({ serverContent: { outputTranscription: { text: 'Answer' } } });
done(ws);
assert.equal(h.captures[0].played.length, 1);
h.captures[0].drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
assert.equal(h.c.retiredReplyPending, false);
} finally {
close(h);
}
});
test('N2 VAD-only split does not arm a nonexistent reply timeout', async () => {
const h = harness();
try {
await h.c.start();
split(h.sockets[0]);
h.advance(5001);
assert.equal(h.c.status, 'listening');
assert.equal(h.c.retiredReplyPending, false);
assert.equal(h.c.retiredBoundaryWait, null);
} finally {
close(h);
}
});
test('N3 server interruption before output still fences until ordered turnComplete', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
ws.message({ serverContent: { interrupted: true } });
assert.equal(h.c.retiredReplyPending, true);
ws.message(audio);
tools(ws, 'retired-before-boundary');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 0);
assert.equal(h.calls.length, 0);
assert.equal(responses(ws).length, 0);
ws.message({
serverContent: { ...audio.serverContent, turnComplete: true },
});
assert.equal(
h.captures[0].played.length,
0,
'coalesced retired content is discarded',
);
ws.message(audio);
assert.equal(h.captures[0].played.length, 1);
done(ws);
h.captures[0].drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
} finally {
close(h);
}
});
test('N4 generationComplete is server evidence and next onset retires it', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message({ serverContent: { generationComplete: true } });
onset(ws);
assert.equal(h.c.retiredReplyPending, true);
assert.ok(h.c.retiredBoundaryWait);
ws.message(audio);
tools(ws, 'old-generation');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 0);
assert.equal(h.calls.length, 0);
done(ws);
assert.equal(h.c.retiredReplyPending, false);
} finally {
close(h);
}
});
test('N5 goAway during VAD-only activity waits for actual reply and playback', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
assert.ok(h.c.goAway);
assert.equal(ws.closed, undefined);
h.advance(5001);
assert.equal(h.c.isActive(), true);
ws.message(audio);
done(ws);
assert.equal(
h.c.isActive(),
true,
'pending actual PCM drains before expiry',
);
h.captures[0].drain();
assert.equal(h.c.status, 'error');
assert.equal(ws.closed, true);
} finally {
close(h);
}
});
test('N6 first server tool after VAD-only split executes exactly once', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
tools(ws, 'new-generation');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(
responses(ws).filter((x) => x.id === 'new-generation').length,
1,
);
assert.equal(responses(ws).at(-1).response.ok, true);
tools(ws, 'new-generation');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(
responses(ws).filter((x) => x.id === 'new-generation').length,
1,
);
} finally {
close(h);
}
});
test('C1 actual old audio followed by onset fences late audio and tool until boundary', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
onset(ws);
assert.equal(h.c.retiredReplyPending, true);
ws.message(audio);
tools(ws, 'late-old');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 1);
assert.equal(h.calls.length, 0);
ws.message({ serverContent: { interrupted: true } });
ws.message(audio);
assert.equal(h.captures[0].played.length, 1);
done(ws);
ws.message(audio);
assert.equal(h.captures[0].played.length, 2);
} finally {
close(h);
}
});
test('C2 first server tool in flight is aborted and never answered after new onset', async () => {
const gate = deferred(),
entered = [];
const h = harness({
runAction: async (_n, _a, o) => {
entered.push(o);
await gate.promise;
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
tools(ws, 'old-running');
await tick();
assert.equal(entered.length, 1);
onset(ws);
assert.equal(entered[0].signal.aborted, true);
gate.resolve();
await h.c.toolQueue;
assert.equal(responses(ws).filter((x) => x.id === 'old-running').length, 0);
assert.equal(h.c.retiredReplyPending, true);
} finally {
gate.resolve();
close(h);
}
});
test('C5 typed request before first server output still retires on next onset', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
assert.equal(h.c.sendTextCommand('Zoom in'), true);
onset(ws);
assert.equal(h.c.retiredReplyPending, true);
ws.message(audio);
tools(ws, 'late-typed');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 0);
assert.equal(h.calls.length, 0);
} finally {
close(h);
}
});
test('N7 old retired boundary preserves the newer automatic owner for goAway', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
const oldOwner = h.c.automaticActivityOwner;
ws.message(audio);
onset(ws);
const successor = h.c.automaticActivityOwner;
assert.notEqual(successor, oldOwner);
assert.equal(h.c.retiredAutomaticActivityOwner, oldOwner);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.automaticActivityOwner, successor);
assert.equal(h.c.automaticTurnOpen, true);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message(audio);
done(ws);
h.captures[0].drain();
assert.equal(h.c.status, 'error');
} finally {
close(h);
}
});
test('N8 old automatic owner with no successor is cleared by its retired boundary', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
const owner = h.c.automaticActivityOwner;
ws.message(audio);
ws.message({ serverContent: { interrupted: true } });
assert.equal(h.c.retiredAutomaticActivityOwner, owner);
done(ws);
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), false, 'no invented successor wait');
} finally {
close(h);
}
});
test('N9 coalesced old completion and new VAD preserves the new owner', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
vad(ws, 'ACTIVITY_END');
ws.message({
voiceActivity: { type: 'ACTIVITY_START' },
serverContent: { interrupted: true, turnComplete: true },
});
assert.equal(h.c.automaticTurnOpen, true);
assert.ok(h.c.automaticActivityOwner);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
} finally {
close(h);
}
});
test('N10 stop, restart and destroy release automatic owner identities and timers', async () => {
const h = harness();
try {
await h.c.start();
onset(h.sockets[0]);
const first = h.c.automaticActivityOwner;
h.c.stop();
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
assert.equal(h.c.automaticActivitySequence, 0);
await h.c.start();
onset(h.sockets[1]);
assert.notEqual(h.c.automaticActivityOwner, first);
assert.notEqual(h.c.automaticActivityOwner.epoch, first.epoch);
h.c.stop({ removeUi: true });
assert.equal(h.c.disposed, true);
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
assert.equal(h.timers.size, 0);
} finally {
close(h);
}
});
test('N11 deferred typed ownership counts discarded open-mic PCM without queuing or replay', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
assert.equal(h.c.sendTextCommand('Zoom in'), true);
assert.equal(h.c.sendTextCommand('Zoom out'), true);
assert.equal(h.c.deferredInput.kind, 'user-text');
const before = ws.sent.filter((x) => x.realtimeInput?.audio).length;
const metadata = { captureId: h.c.captureOwner.captureId };
h.captures[0].callbacks.onAudio(
{ data: 'AAAA', mimeType: 'audio/pcm;rate=16000' },
metadata,
);
assert.equal(ws.sent.filter((x) => x.realtimeInput?.audio).length, before);
assert.equal(
h.c.deferredInput.messages.some((x) => x.realtimeInput?.audio),
false,
);
assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1);
assert.deepEqual(h.c.getDiagnostics().deferredAudioDrops, {
nonPttOwner: 1,
closedActivity: 0,
});
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
done(ws);
assert.equal(
ws.sent.filter((x) => x.realtimeInput?.audio).length,
before,
'no wrong-owner PCM replay after boundaries',
);
} finally {
close(h);
}
});
test('N12 goAway before old completion still waits for the newer automatic owner', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
onset(ws);
const successor = h.c.automaticActivityOwner;
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.automaticActivityOwner, successor);
assert.equal(
h.c.isActive(),
true,
'old boundary during goAway cannot end successor wait',
);
ws.message(audio);
done(ws);
h.captures[0].drain();
assert.equal(h.c.status, 'error');
} finally {
close(h);
}
});
for (const interruptedBeforeExpiry of [false, true])
test(`N13 no successor clears retired ownership with goAway ${interruptedBeforeExpiry ? 'after interruption' : 'before interruption'}`, async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
if (interruptedBeforeExpiry)
ws.message({ serverContent: { interrupted: true } });
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(
h.c.isActive(),
false,
'no successor finishes at its actual boundary',
);
} finally {
close(h);
}
});
test('N14 native interruption during goAway aborts pending old tool and suppresses late response', async () => {
const gate = deferred(),
entered = [];
const h = harness({
runAction: async (_n, _a, o) => {
entered.push(o);
await gate.promise;
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
tools(ws, 'expiring-old-tool');
await tick();
assert.equal(entered.length, 1);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(entered[0].signal.aborted, true);
gate.resolve();
await h.c.toolQueue;
assert.equal(
responses(ws).filter((x) => x.id === 'expiring-old-tool').length,
0,
);
assert.equal(h.c.isActive(), false);
} finally {
gate.resolve();
close(h);
}
});
test('N15 goAway still refuses local PTT and typed replacement without reopening microphone', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
ws.message({ goAway: { timeLeft: '30s' } });
const turn = h.c.turn,
sent = ws.sent.length;
assert.equal(h.c.bargeIn(), false);
assert.equal(h.c.sendTextCommand('Zoom out'), false);
h.c.interrupt('push-to-talk');
assert.equal(h.c.turn, turn);
assert.equal(ws.sent.length, sent);
assert.equal(
h.tracks.every((t) => !t.enabled),
true,
);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.isActive(), false);
assert.equal(
h.tracks.every((t) => t.readyState === 'ended'),
true,
);
} finally {
close(h);
}
});
test('T-F1a PTT successor PCM survives retired boundary and a second input fences its late output', async () => {
const h = harness();
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
a = h.captures[0];
ws.message(audio);
h.c.input.pushToTalkKeyHeld = true;
assert.equal(h.c.input.bargeIn(), true);
h.c.input.setMicrophoneEnabled(true);
a.callbacks.onAudio(
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
a.metadata,
);
h.c.input.setMicrophoneEnabled(false);
assert.equal(h.c.replyPending, true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(
h.c.replyPending,
true,
'old boundary cannot clear submitted successor',
);
assert.equal(h.c.sendTextCommand('Zoom out second request'), true);
assert.ok(h.c.currentTypedInput());
const played = a.played.length;
ws.message(audio);
tools(ws, 'late-ptt-predecessor');
await h.c.toolQueue;
assert.equal(a.played.length, played);
assert.equal(
h.calls.length,
0,
'no old tool under newer local typed input',
);
done(ws);
tools(ws, 'current-typed');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
assert.equal(h.c.currentTypedInput().text, 'Zoom out second request');
} finally {
close(h);
}
});
for (const expiryBeforeBoundary of [false, true])
test(`T-F1b VAD then submitted text survives goAway ${expiryBeforeBoundary ? 'before' : 'after'} retired boundary`, async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
onset(ws);
assert.equal(h.c.sendTextCommand('Zoom out successor'), true);
const typed = h.c.currentTypedInput();
assert.ok(typed);
assert.equal(h.c.replyPending, true);
if (expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } });
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.replyPending, true);
if (!expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true, 'submitted text must finish');
tools(ws, 'valid-successor');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
assert.equal(h.c.currentTypedInput(), typed);
ws.message(audio);
done(ws);
assert.equal(h.c.isActive(), true);
h.captures[0].drain();
assert.equal(h.c.isActive(), false);
} finally {
close(h);
}
});
for (const repeat of [false, true])
test(`T-F2 forwarded A superseded by deferred B cannot run after goAway${repeat ? ' shortened twice' : ''}`, async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
assert.equal(h.c.sendTextCommand('Zoom out A'), true);
assert.equal(h.c.replyPending, true);
assert.equal(h.c.sendTextCommand('Zoom out B'), true);
assert.equal(h.c.deferredInput.kind, 'user-text');
assert.ok(h.c.currentTypedInput());
onset(ws);
assert.equal(h.c.automaticTurnOpen, true);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.deferredInput, null);
if (repeat) {
ws.message({ goAway: { timeLeft: '20s' } });
ws.message({ goAway: { timeLeft: '10s' } });
}
assert.equal(h.c.goAway.supersededSuccessor, true);
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
tools(ws, 'superseded-A');
await h.c.toolQueue;
assert.equal(h.calls.length, 0, 'A cannot run with dropped B authority');
assert.equal(
h.c.isActive(),
false,
'ordered boundary closes superseded forwarded predecessor',
);
} finally {
close(h);
}
});
test('T-F3 VAD-only no-generation expiry waits to bounded native deadline and releases resources', async () => {
const h = harness();
try {
await h.c.start();
onset(h.sockets[0]);
h.sockets[0].message({ goAway: { timeLeft: '30s' } });
h.advance(29999);
assert.equal(h.c.isActive(), true);
h.advance(1);
assert.equal(h.c.isActive(), false);
assert.equal(h.timers.size, 0);
assert.equal(
h.tracks.every((t) => t.readyState === 'ended'),
true,
);
} finally {
close(h);
}
});
test('T-C1 coalesced new START and natural old completion preserve newer automatic activity', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
ws.message({
voiceActivity: { type: 'ACTIVITY_START' },
serverContent: { turnComplete: true },
});
assert.equal(h.c.retiredReplyPending, false);
assert.equal(h.c.automaticTurnOpen, true);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
} finally {
close(h);
}
});
test('T-C2 generationComplete alone followed by natural completion emits completion with no retirement timer', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message({ serverContent: { generationComplete: true } });
assert.equal(h.c.replyPending, true);
done(ws);
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
assert.equal(h.c.replyPending, false);
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.retiredBoundaryWait, null);
assert.equal(h.timers.size, 0);
} finally {
close(h);
}
});
test('T-C3 silent PTT terminal completion emits spoken false without fabricated audio', async () => {
const h = harness();
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
a = h.captures[0];
h.c.input.setMicrophoneEnabled(true);
a.callbacks.onAudio(
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
a.metadata,
);
h.c.input.setMicrophoneEnabled(false);
assert.equal(h.c.userTurnPending, true);
ws.message({ serverContent: { generationComplete: true } });
done(ws);
const event = h.events.findLast((e) => e.type === 'completion');
assert.equal(event.status, 'completed');
assert.equal(event.spoken, false);
assert.equal(a.played.length, 0);
assert.equal(h.c.userTurnPending, false);
} finally {
close(h);
}
});
test('T-C4 silent terminal boundary waits for pending tool and native follow-up completion', async () => {
const gate = deferred(),
entered = [];
const h = harness({
runAction: async (_n, _a, o) => {
entered.push(o);
await gate.promise;
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
tools(ws, 'pending-terminal-tool');
await tick();
assert.equal(entered.length, 1);
done(ws);
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
assert.equal(h.c.userTurnPending, false);
gate.resolve();
await h.c.toolQueue;
h.c.finishPlayback();
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
assert.equal(h.c.replyPending, true, 'tool result awaits native follow-up');
done(ws);
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
} finally {
gate.resolve();
close(h);
}
});
test('T-C5 terminal boundary with queued PCM waits for drain and preserves real spoken state', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
ws.message({ serverContent: { generationComplete: true } });
done(ws);
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
h.captures[0].drain();
const event = h.events.findLast((e) => e.type === 'completion');
assert.equal(event.status, 'completed');
assert.equal(event.spoken, true);
} finally {
close(h);
}
});
test('T-C6 old retired terminal boundary cannot clear successor userTurnPending', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
assert.equal(h.c.sendTextCommand('Zoom out successor'), true);
assert.equal(h.c.userTurnPending, true);
done(ws);
assert.equal(h.c.userTurnPending, true);
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
} finally {
close(h);
}
});
test('T-C7 reentrant silent completion observer keeps new input pending', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
const emit = h.c.emit.bind(h.c);
h.c.emit = (event) => {
emit(event);
if (event.type === 'completion')
h.c.sendTextCommand('Zoom out reentrant');
};
onset(ws);
ws.message({ serverContent: { generationComplete: true } });
done(ws);
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
assert.equal(h.c.replyPending, true);
assert.equal(h.c.userTurnPending, true);
assert.ok(h.c.currentTypedInput());
assert.equal(h.c.modelComplete, false);
} finally {
close(h);
}
});
+125
View File
@@ -0,0 +1,125 @@
const OUTPUT_RATE = 16000;
const FILTER_TAPS = 64;
const FILTER_PHASES = 128;
// A causal, windowed-sinc low-pass filter avoids folding device frequencies
// above 8 kHz into speech. Precompute phases; process allocates no filter data.
function resamplingFilters(inputRate) {
const cutoff = 0.45 * Math.min(1, OUTPUT_RATE / inputRate);
const filters = new Float32Array(FILTER_TAPS * FILTER_PHASES);
for (let phase = 0; phase < FILTER_PHASES; phase++) {
let total = 0;
for (let tap = 0; tap < FILTER_TAPS; tap++) {
const distance = tap - (FILTER_TAPS - 1) / 2 - phase / FILTER_PHASES;
const sinc =
distance === 0
? 2 * cutoff
: Math.sin(2 * Math.PI * cutoff * distance) / (Math.PI * distance);
const window =
0.5 - 0.5 * Math.cos((2 * Math.PI * tap) / (FILTER_TAPS - 1));
const weight = sinc * window;
filters[phase * FILTER_TAPS + tap] = weight;
total += weight;
}
for (let tap = 0; tap < FILTER_TAPS; tap++)
filters[phase * FILTER_TAPS + tap] /= total;
}
return filters;
}
/** PCM microphone capture. Only explicitly enabled frames leave this processor. */
class GeminiCaptureProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.enabled = false;
this.captureId = null;
this.samples = new Int16Array(1024);
this.length = 0;
this.inputRate = sampleRate;
this.filters =
this.inputRate === OUTPUT_RATE ? null : resamplingFilters(this.inputRate);
this.history = new Float32Array(FILTER_TAPS);
this.resetResampler();
this.port.onmessage = ({ data }) => {
if (data?.enabled === true) {
if (
this.enabled ||
!Number.isSafeInteger(data.captureId) ||
data.captureId <= 0
)
return;
this.captureId = data.captureId;
this.resetResampler();
this.enabled = true;
this.port.postMessage({ type: 'start', captureId: this.captureId });
} else if (
data?.enabled === false &&
this.enabled &&
data.captureId === this.captureId
) {
this.flush();
this.port.postMessage({ type: 'end', captureId: this.captureId });
this.enabled = false;
this.captureId = null;
this.length = 0;
// Do not drain filter tails into a later physical microphone capture.
this.resetResampler();
}
};
}
resetResampler() {
this.phase = 0;
this.historyIndex = 0;
this.history.fill(0);
}
appendSample(sample) {
const value = Math.max(-1, Math.min(1, sample));
this.samples[this.length++] = Math.round(
value * (value < 0 ? 32768 : 32767),
);
if (this.length === this.samples.length) this.flush();
}
flush() {
if (!this.length) return;
const buffer = new ArrayBuffer(this.length * 2);
const view = new DataView(buffer);
for (let i = 0; i < this.length; i++)
view.setInt16(i * 2, this.samples[i], true);
this.port.postMessage(
{ type: 'audio', buffer, captureId: this.captureId },
[buffer],
);
this.length = 0;
}
process(inputs) {
if (!this.enabled) return true;
const channel = inputs[0]?.[0];
if (!channel) return true;
for (const sample of channel) {
if (!this.filters) {
this.appendSample(sample);
continue;
}
this.history[this.historyIndex] = sample;
this.historyIndex = (this.historyIndex + 1) % FILTER_TAPS;
this.phase += OUTPUT_RATE;
while (this.phase >= this.inputRate) {
this.phase -= this.inputRate;
const phase = Math.min(
FILTER_PHASES - 1,
Math.floor((this.phase * FILTER_PHASES) / OUTPUT_RATE),
);
let value = 0;
for (let tap = 0; tap < FILTER_TAPS; tap++) {
const index =
(this.historyIndex - 1 - tap + FILTER_TAPS) % FILTER_TAPS;
value +=
this.history[index] * this.filters[phase * FILTER_TAPS + tap];
}
this.appendSample(value);
}
}
return true;
}
}
registerProcessor('gev-gemini-capture', GeminiCaptureProcessor);
+285
View File
@@ -0,0 +1,285 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
nativeIntegrationFixture,
assertSuccessfulZoom,
} from '../testSupport/geminiIntegrationFixture.mjs';
const options = { concurrency: false, timeout: 5000 };
const OLD_PCM = 'BAAFAA==';
const TEXT = 'Zoom in for the owned typed successor';
function exposeCapture(h) {
const createAudio = h.controller.createAudio;
h.controller.createAudio = (callbacks) => {
const audio = createAudio(callbacks);
audio.captureCallbacks = callbacks;
return audio;
};
}
async function startManual(h) {
await h.session.start({ pushToTalk: true });
assert.equal(h.session.state, 'listening');
assert.equal(h.controller.input.pushToTalkMode, true);
assert.equal(Boolean(h.controller.input.pushToTalkKeyHeld), false);
return h.sockets.at(-1);
}
// Explicit fake worklet callbacks retain the native physical-capture path;
// this helper does not populate controller queues or ownership fields itself.
function speakAndRelease(h, data) {
const c = h.controller;
const audio = h.audios.at(-1);
c.input.pushToTalkKeyHeld = true;
c.input.setMicrophoneEnabled(true);
const metadata = audio.metadata;
assert.ok(Number.isSafeInteger(metadata?.captureId));
audio.captureCallbacks.onAudioStart(metadata);
audio.captureCallbacks.onAudio(
{ data, mimeType: 'audio/pcm;rate=16000' },
metadata,
);
c.input.pushToTalkKeyHeld = false;
c.input.setMicrophoneEnabled(false);
audio.captureCallbacks.onAudioEnd(metadata);
return metadata;
}
async function prepareDeferredPtt(h) {
exposeCapture(h);
const socket = await startManual(h);
speakAndRelease(h, 'AAABAA=='); // A is submitted and awaits its reply.
speakAndRelease(h, 'AgADAA=='); // B is submitted, retiring A.
const wireBeforeC = socket.sent.length;
const metadata = speakAndRelease(h, OLD_PCM); // C waits behind A's boundary.
const owner = h.controller.deferredInput;
assert.equal(owner?.kind, 'push-to-talk');
assert.equal(owner.captureId, metadata.captureId);
assert.equal(
socket.sent.length,
wireBeforeC,
'C must still be wholly deferred',
);
assert.deepEqual(
owner.messages.map((message) =>
message.realtimeInput?.activityStart
? 'start'
: message.realtimeInput?.audio
? 'audio'
: message.realtimeInput?.activityEnd
? 'end'
: 'other',
),
['start', 'audio', 'end'],
);
return { socket, owner, metadata };
}
async function assertHealthySuccessor(h, socket, callId) {
const beforeHeight = h.state.height;
await h.tool(callId, socket);
assertSuccessfulZoom(h, socket, callId, beforeHeight);
assert.equal(h.controller.pendingTools, 0);
h.complete(socket); // Native post-tool response boundary, with no output PCM.
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.session.state, 'listening');
}
for (const mode of ['typed replacement', 'Stop', 'Stop+restart']) {
test(
`F04-5 deferred PTT flush loses its owner on ${mode}`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const { socket: oldSocket, owner } = await prepareDeferredPtt(h);
const oldAudio = h.audios.at(-1);
let observed = 0;
let sentAtObserverReturn;
let restart;
let stateAtObserverReturn;
const unsubscribe = h.session.subscribe((event) => {
if (
event.type !== 'transcript' ||
event.role !== 'user' ||
event.text !== '' ||
observed
)
return;
observed++;
if (mode === 'typed replacement') {
assert.equal(h.session.sendText(TEXT), true);
} else {
h.session.stop();
if (mode === 'Stop+restart')
restart = h.session.start({ pushToTalk: true });
}
sentAtObserverReturn = oldSocket.sent.length;
stateAtObserverReturn = {
epoch: h.controller.epoch,
turn: h.controller.turn,
replyPending: h.controller.replyPending,
forwardedActivityOpen: h.controller.forwardedActivityOpen,
};
});
t.after(unsubscribe);
h.complete(oldSocket); // A's boundary starts flushing C. C's start emits.
assert.equal(
observed,
1,
'must enter the actual flush/onset observer boundary',
);
assert.ok(
stateAtObserverReturn,
'observer must complete its replacement action',
);
assert.deepEqual(
oldSocket.sent.slice(sentAtObserverReturn),
[],
'retired C must send neither queued PCM nor an extra end after the observer returns',
);
assert.deepEqual(
{
epoch: h.controller.epoch,
turn: h.controller.turn,
replyPending: h.controller.replyPending,
forwardedActivityOpen: h.controller.forwardedActivityOpen,
},
stateAtObserverReturn,
'old flush must not write successor state',
);
assert.equal(h.controller.deferredInput, null);
assert.equal(
h.controller.typedIntent?.text ?? null,
mode === 'typed replacement' ? TEXT : null,
);
let successorSocket;
if (mode === 'typed replacement') {
assert.equal(h.controller.epoch, owner.epoch);
assert.notEqual(h.controller.turn, owner.turn);
assert.equal(
oldSocket.sent.some(
(message) => message.realtimeInput?.audio?.data === OLD_PCM,
),
false,
);
const typedIndex = oldSocket.sent.findIndex((message) =>
message.clientContent?.turns?.some((turn) =>
turn.parts?.some((part) => part.text === TEXT),
),
);
assert.ok(
typedIndex >= 0,
'owned replacement must actually reach transport',
);
// The replacement legitimately closes the already-forwarded C start once.
assert.ok(oldSocket.sent[typedIndex - 1]?.realtimeInput?.activityEnd);
h.complete(oldSocket); // B's remaining retired boundary, deliberately separate.
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(h.controller.replyPending, true);
successorSocket = oldSocket;
} else {
assert.equal(oldAudio.closed, true);
assert.equal(oldSocket.closed, true);
if (!restart) restart = h.session.start({ pushToTalk: true });
await restart;
successorSocket = h.sockets.at(-1);
assert.notEqual(successorSocket, oldSocket);
assert.equal(
successorSocket.sent.some((message) => message.realtimeInput?.audio),
false,
);
assert.equal(
successorSocket.sent.some(
(message) => message.realtimeInput?.activityEnd,
),
false,
);
assert.equal(h.session.sendText(TEXT), true);
}
await assertHealthySuccessor(
h,
successorSocket,
`healthy-after-deferred-${mode}`,
);
},
);
}
test(
'F04-5 deferred typed onSent cannot resurrect replyPending after Stop+restart',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const oldSocket = await h.start();
assert.equal(h.session.sendText('A'), true);
assert.equal(h.session.sendText('B'), true);
assert.equal(h.session.sendText('C deferred text'), true);
assert.equal(h.controller.deferredInput?.kind, 'user-text');
let restart;
let observed = 0;
const unsubscribe = h.session.subscribe((event) => {
if (
event.type !== 'transcript' ||
event.role !== 'user' ||
event.text !== 'C deferred text'
)
return;
observed++;
h.session.stop();
restart = h.session.start();
});
t.after(unsubscribe);
h.complete(oldSocket);
assert.equal(observed, 1);
assert.equal(
h.controller.replyPending,
false,
'old onSent wrapper must not resurrect a stopped reply',
);
await restart;
assert.equal(h.controller.replyPending, false);
const socket = h.sockets.at(-1);
assert.equal(
socket.sent.length,
1,
'replacement connection should contain setup only',
);
assert.equal(h.session.sendText(TEXT), true);
await assertHealthySuccessor(h, socket, 'healthy-after-deferred-text');
},
);
test(
'F04-5 ordinary deferred PTT flush preserves start, PCM and end exactly once',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const { socket } = await prepareDeferredPtt(h);
const before = socket.sent.length;
h.complete(socket);
assert.deepEqual(
socket.sent
.slice(before)
.map((message) =>
message.realtimeInput?.activityStart
? 'start'
: message.realtimeInput?.audio?.data === OLD_PCM
? 'audio'
: message.realtimeInput?.activityEnd
? 'end'
: 'other',
),
['start', 'audio', 'end'],
);
assert.equal(h.controller.deferredInput, null);
assert.equal(h.controller.replyPending, true);
assert.equal(h.controller.retiredReplyPending, true);
h.complete(socket); // B's boundary; C remains the owned submitted input.
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(h.controller.replyPending, true);
await assertHealthySuccessor(h, socket, 'healthy-ordinary-deferred');
},
);
+287
View File
@@ -0,0 +1,287 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
nativeIntegrationFixture,
assertSuccessfulZoom,
turnOfEventLoop,
} from '../testSupport/geminiIntegrationFixture.mjs';
const options = { concurrency: false, timeout: 5000 };
const resultFor = (h, id) =>
h.events.filter(
(event) => event.type === 'action-result' && event.callId === id,
);
const callFor = (h, id) =>
h.events.filter(
(event) => event.type === 'action-call' && event.callId === id,
);
test(
'native tool reaches actual common session and camera runner without a caption',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
h.activity(ws); // Native activity is input framing, not transcript admission.
await h.tool('ordinary', ws);
assertSuccessfulZoom(h, ws, 'ordinary', 1000);
assert.equal(h.runnerCalls.length, 1);
assert.equal(h.runnerCalls[0].options.callId, 'ordinary');
assert.equal(h.runnerCalls[0].options.signal.aborted, false);
assert.equal(h.runnerCalls[0].options.isCurrent(), true);
h.complete(ws);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
},
);
for (const cancellation of ['stop', 'native interruption']) {
test(
`${cancellation} discards a stale completed action result; successor changes camera once`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const oldSocket = await h.start();
h.activity(oldSocket);
const gate = h.holdResult('old');
const oldDispatch = h.tool('old', oldSocket);
const committedResult = await gate.ready;
assert.equal(committedResult.ok, true);
assert.equal(
h.state.height,
750,
'the legitimate first action committed before cancellation',
);
assert.equal(callFor(h, 'old').length, 1);
assert.deepEqual(resultFor(h, 'old'), []);
assert.deepEqual(h.outputs(oldSocket), []);
if (cancellation === 'stop') h.session.stop();
else
oldSocket.message({
serverContent: { interrupted: true, turnComplete: true },
});
assert.equal(h.runnerCalls[0].options.signal.aborted, true);
assert.equal(h.runnerCalls[0].options.isCurrent(), false);
await oldDispatch; // Cancellation must settle before the held result returns.
gate.release();
await turnOfEventLoop();
assert.equal(
h.state.height,
750,
'settling a cancelled result adds no camera mutation',
);
assert.equal(h.state.zooms.length, 1);
assert.deepEqual(resultFor(h, 'old'), []);
assert.deepEqual(h.outputs(oldSocket), []);
if (cancellation === 'native interruption')
assert.ok(h.events.some((event) => event.type === 'interruption'));
const successorSocket =
cancellation === 'stop' ? await h.start() : oldSocket;
h.activity(successorSocket);
await h.tool('successor', successorSocket);
assertSuccessfulZoom(h, successorSocket, 'successor', 750);
assert.equal(h.state.zooms.length, 2);
assert.equal(h.runnerCalls.length, 2);
assert.equal(
h.outputs(successorSocket).some((output) => output.id === 'old'),
false,
);
h.complete(successorSocket);
},
);
}
for (const phase of ['common connecting', 'adapter connecting']) {
test(
`Stop from ${phase} observer prevents acquisition and permits fresh Start`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
let stopped = false;
const remove = h.session.subscribe((event) => {
const matchingDetail =
phase === 'common connecting'
? event.detail === undefined
: typeof event.detail === 'string' && event.detail.length > 0;
if (
!stopped &&
event.type === 'state' &&
event.state === 'connecting' &&
matchingDetail
) {
stopped = true;
h.session.stop();
}
});
await h.session.start();
remove();
assert.equal(
stopped,
true,
'the chosen production observer boundary was exercised',
);
assert.deepEqual(h.acquisitions, {
microphone: 0,
audio: 0,
token: 0,
socket: 0,
});
assert.equal(h.session.state, 'idle');
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
const ws = await h.start();
h.activity(ws);
await h.tool('fresh-after-start-cancel', ws);
assertSuccessfulZoom(h, ws, 'fresh-after-start-cancel', 1000);
h.complete(ws);
},
);
}
test(
'Stop from common action-call observer prevents runner and leaves no unhandled rejection',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
const remove = h.session.subscribe((event) => {
if (event.type === 'action-call' && event.callId === 'cancel-at-call')
h.session.stop();
});
h.activity(ws);
await h.tool('cancel-at-call', ws);
// Do not install a swallowing process rejection listener. node:test records
// an unhandled rejection as a failure; these checkpoints let it surface here.
await turnOfEventLoop();
await turnOfEventLoop();
remove();
assert.equal(callFor(h, 'cancel-at-call').length, 1);
assert.deepEqual(resultFor(h, 'cancel-at-call'), []);
assert.equal(h.runnerCalls.length, 0);
assert.equal(h.state.height, 1000);
assert.deepEqual(h.outputs(ws), []);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
const successor = await h.start();
h.activity(successor);
await h.tool('after-call-cancel', successor);
assertSuccessfulZoom(h, successor, 'after-call-cancel', 1000);
h.complete(successor);
},
);
test(
'Stop from executing observer cannot recreate narration or dispatch the runner',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
let stopped = false;
const remove = h.session.subscribe((event) => {
if (!stopped && event.type === 'state' && event.state === 'executing') {
stopped = true;
h.session.stop();
}
});
h.activity(ws);
await h.tool('cancel-at-executing', ws);
remove();
assert.equal(stopped, true);
assert.equal(h.runnerCalls.length, 0);
assert.deepEqual(callFor(h, 'cancel-at-executing'), []);
assert.deepEqual(resultFor(h, 'cancel-at-executing'), []);
assert.equal(h.state.height, 1000);
assert.deepEqual(h.outputs(ws), []);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
assert.equal(ws.closed, true);
assert.ok(h.audios.every((audio) => audio.closed));
assert.ok(h.tracks.every((track) => track.readyState === 'ended'));
const before = {
events: h.events.length,
wire: ws.sent.length,
earcon: h.state.earconStarts,
};
h.clock.advance(60_000);
assert.deepEqual(
{
events: h.events.length,
wire: ws.sent.length,
earcon: h.state.earconStarts,
},
before,
);
const successor = await h.start();
h.activity(successor);
await h.tool('after-executing-cancel', successor);
assertSuccessfulZoom(h, successor, 'after-executing-cancel', 1000);
h.complete(successor);
},
);
test(
'native idle observer restart remains connecting and public Stop cancels its pending permission',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
await h.start();
let grant;
const permission = new Promise((resolve) => {
grant = resolve;
});
h.controller.getUserMedia = () => permission;
const states = [];
let restarted = false;
let startup;
const remove = h.session.subscribe((event) => {
if (event.type === 'state') states.push(event.state);
if (!restarted && event.type === 'state' && event.state === 'idle') {
restarted = true;
startup = h.session.start();
}
});
h.session.stop();
remove();
assert.equal(restarted, true);
assert.equal(states.filter((state) => state === 'idle').length, 1);
assert.equal(h.session.state, 'connecting');
assert.equal(h.controller.status, 'connecting');
assert.equal(
h.session.isActive(),
true,
'the common button/facade must offer Stop',
);
h.session.stop();
assert.equal(h.session.state, 'idle');
assert.equal(h.controller.status, 'idle');
const lateTrack = {
readyState: 'live',
stopCalls: 0,
stop() {
this.stopCalls++;
this.readyState = 'ended';
},
};
h.tracks.push(lateTrack);
grant({ getTracks: () => [lateTrack], getAudioTracks: () => [lateTrack] });
await startup;
await turnOfEventLoop();
assert.equal(lateTrack.stopCalls, 1);
assert.equal(
h.requests.length,
1,
'cancelled restart must not mint another token',
);
assert.equal(
h.sockets.length,
1,
'cancelled restart must not open another socket',
);
assert.equal(h.session.state, 'idle');
},
);
@@ -0,0 +1,85 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { harness } from '../testSupport/geminiNativeHarness.mjs';
const caption = (socket, text) =>
socket.message({ serverContent: { inputTranscription: { text } } });
const userEvents = (events) =>
events.filter(
(event) => event.type === 'transcript' && event.role === 'user',
);
test('native caption events preserve received advisory segments without granting tool authority', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
const start = h.events.length;
for (const text of ['little. ', 'Zoom in a ']) caption(ws, text);
const received = userEvents(h.events.slice(start));
assert.deepEqual(
received.map((event) => event.text),
['little. ', 'Zoom in a '],
);
assert.equal(
received.every(
(event) =>
event.delta === true &&
event.final === false &&
event.attribution === 'unverified' &&
event.responseId === null,
),
true,
);
assert.equal(h.c.currentTypedInput(), null);
ws.message({
toolCall: {
functionCalls: [
{
id: 'ordinary-native',
name: 'get_current_view_state',
args: {},
},
],
},
});
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
ws.message({ serverContent: { turnComplete: true } });
assert.equal(
userEvents(h.events.slice(start)).some((event) => event.final),
false,
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
test('native old socket captions cannot enter a restarted controller event stream', async () => {
const h = harness();
try {
await h.c.start();
const old = h.sockets[0];
caption(old, 'old partial');
h.c.stop();
await h.c.start();
const start = h.events.length;
const current = h.sockets[1];
caption(current, 'Zoom ');
caption(old, 'stale old socket');
caption(current, 'in a little.');
current.message({ serverContent: { turnComplete: true } });
assert.deepEqual(
userEvents(h.events.slice(start)).map((event) => event.text),
['Zoom ', 'in a little.'],
);
assert.equal(h.tracks[0].readyState, 'ended');
assert.equal(h.captures[0].closed, true);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
+667
View File
@@ -0,0 +1,667 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
import { withToolCatalog } from './gevRealtime.js';
const audioContent = (data) => ({
modelTurn: {
parts: [{ inlineData: { data, mimeType: 'audio/pcm;rate=24000' } }],
},
});
const request = (ws, id, name = 'get_current_view_state', args = {}) =>
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
const responses = (ws) =>
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
const onset = (ws) => {
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
};
const caption = (ws, text) =>
ws.message({ serverContent: { inputTranscription: { text } } });
for (const [id, name, args] of [
['H01', 'get_entity_context', {}],
['H02', 'control_radio', { action: 'status' }],
]) {
test(`${id} native: cancelling a current ${name} query preserves unrelated Radio preflight`, async () => {
const ready = deferred(),
query = deferred(),
executions = [];
const h = harness({
runAction: (name, args, options) => {
executions.push({ name, args, options });
return args.action === 'play'
? Promise.resolve({
ok: true,
action: 'control_radio',
radioAction: 'play',
radioPlaybackRequested: true,
})
: query.promise;
},
});
h.c.radio.radioLayer.playForVoice = () => ready.promise;
try {
await h.c.start();
h.c.sendTextCommand('Play radio');
const ws = h.sockets[0];
request(ws, 'owner', 'control_radio', { action: 'play' });
await h.c.toolQueue;
ws.message({
serverContent: { ...audioContent('confirmation'), turnComplete: true },
});
h.captures[0].drain();
assert.equal(h.c.radio.radioHandoffInFlight, true);
request(ws, 'query', name, args);
await tick();
assert.equal(
executions.length,
2,
'current native query may start after provider completion',
);
const epoch = h.c.radio.radioHandoffEpoch;
const pending = h.c.radio.pendingRadioPlaybackResult;
const attempt = h.c.radio.radioHandoffAttemptId;
const ownerCurrent = executions[0].options.isCurrent();
ws.message({ toolCallCancellation: { ids: ['unknown', 'query'] } });
await h.c.toolQueue;
assert.equal(executions[1].options.signal.aborted, true);
assert.equal(executions[1].options.isCurrent(), false);
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
assert.equal(executions[0].options.signal.aborted, false);
assert.equal(executions[0].options.isCurrent(), ownerCurrent);
assert.equal(h.c.radio.pendingRadioPlaybackResult, pending);
assert.equal(h.c.radio.radioHandoffAttemptId, attempt);
assert.equal(h.c.radio.radioHandoffInFlight, true);
assert.equal(h.state.stopped, 0);
assert.equal(h.c.pendingTools, 0);
assert.deepEqual(
responses(ws).map((r) => r.id),
['owner'],
);
assert.equal(responses(ws)[0].response.ok, true);
query.resolve({ ok: true });
ready.resolve(true);
await tick();
assert.equal(h.c.status, 'idle');
assert.equal(h.state.ducked, false);
assert.deepEqual(
responses(ws).map((r) => r.id),
['owner'],
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const [id, prior] of [
['H03', 'playing'],
['H04', 'before-first-output'],
]) {
test(`${id} native: PTT fences ${prior} reply and recovers the new call after its boundary`, async () => {
const h = harness();
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
audio = h.captures[0];
if (prior === 'playing')
ws.message({ serverContent: audioContent('old') });
else {
h.c.input.setMicrophoneEnabled(true);
audio.callbacks.onAudio(
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
audio.metadata,
);
h.c.input.setMicrophoneEnabled(false);
}
h.c.input.pushToTalkKeyHeld = true;
assert.equal(h.c.input.bargeIn(), true);
h.c.input.setMicrophoneEnabled(true);
h.c.input.setMicrophoneEnabled(false);
const turn = h.c.turn,
played = audio.played.length,
firstEvent = h.events.length;
ws.message({
serverContent: {
...audioContent('late-old'),
outputTranscription: { text: 'retired output' },
inputTranscription: { text: 'new request' },
},
});
request(ws, 'retired-call', 'fly_to_location', { query: 'old target' });
await h.c.toolQueue;
assert.equal(audio.played.length, played);
assert.equal(audio.pending, false);
assert.equal(h.calls.length, 0);
assert.equal(
h.events.slice(firstEvent).some((e) => e.role === 'assistant'),
false,
);
assert.equal(
h.events
.slice(firstEvent)
.some((e) => e.role === 'user' && e.text === 'new request'),
true,
);
ws.message({ serverContent: { interrupted: true } });
assert.equal(h.c.turn, turn);
ws.message({ serverContent: audioContent('after-interrupted') });
assert.equal(
audio.played.length,
played,
'interrupted alone cannot release the fence',
);
ws.message({
serverContent: {
...audioContent('boundary-old'),
outputTranscription: { text: 'still retired' },
turnComplete: true,
},
});
assert.equal(
audio.played.length,
played,
'coalesced old boundary output is discarded',
);
assert.equal(
h.events.slice(firstEvent).some((e) => e.type === 'completion'),
false,
);
caption(ws, 'unrelated advisory words');
ws.message({
toolCall: {
functionCalls: [
{
id: 'retired-call',
name: 'fly_to_location',
args: { query: 'replayed old target' },
},
{
id: 'new-call',
name: 'fly_to_location',
args: { query: 'new target' },
},
],
},
});
await h.c.toolQueue;
assert.deepEqual(
h.calls.map((c) => c.args.query),
['new target'],
);
assert.equal(h.calls[0].options.userTurn, undefined);
ws.message({
serverContent: {
...audioContent('new'),
outputTranscription: { text: 'new reply' },
turnComplete: true,
},
});
assert.equal(audio.played.at(-1).data, 'new');
audio.drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion').status,
'completed',
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
test('H05 native: local typed presentation survives audio drain and new input revokes old call ownership', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0],
audio = h.captures[0];
h.c.sendTextCommand('Show the current view');
const response = h.c.currentTypedInput();
assert.equal(response.text, 'Show the current view');
assert.equal(
h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
.text,
response.text,
);
request(ws, 'first');
await h.c.toolQueue;
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
ws.message({
serverContent: {
...audioContent('confirmation'),
outputTranscription: { text: 'Here is the current view' },
turnComplete: true,
},
});
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
assert.equal(
h.events.findLast((e) => e.role === 'assistant').responseId,
response.responseId,
);
audio.drain();
const completion = h.events.findLast((e) => e.type === 'completion');
assert.equal(completion.responseId, response.responseId);
assert.equal(completion.spoken, true);
assert.equal(completion.audioDrained, true);
assert.equal(h.c.currentTypedInput(), response);
assert.equal(response.text, 'Show the current view');
request(ws, 'same-turn');
await h.c.toolQueue;
assert.equal(h.calls[1].options.userTurn, undefined);
assert.equal(h.c.currentTypedInput(), response);
h.c.sendTextCommand('New request');
assert.notEqual(h.c.currentTypedInput(), response);
assert.equal(h.c.currentTypedInput().text, 'New request');
assert.equal(h.calls[0].options.isCurrent(), false);
assert.equal(h.calls[1].options.isCurrent(), false);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
test('H08 native: replacing pending runner work cannot underflow tool counts or block successor', async () => {
const gate = deferred(),
executions = [];
const h = harness({
runAction: (name, args, options) => {
executions.push({ name, args, options });
return options.callId === 'old'
? gate.promise
: Promise.resolve({ ok: true });
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
request(ws, 'old');
await tick();
const oldWork = h.c.toolQueue;
assert.equal(h.c.pendingTools, 1);
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
assert.equal(h.c.pendingTools, 0);
await oldWork;
assert.equal(h.c.pendingTools, 0);
assert.equal(executions[0].options.signal.aborted, true);
assert.equal(executions[0].options.isCurrent(), false);
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
request(ws, 'fresh');
await h.c.toolQueue;
assert.equal(executions.at(-1).options.callId, 'fresh');
assert.equal(h.c.pendingTools, 0);
gate.resolve({ ok: true });
await tick();
assert.equal(h.c.pendingTools, 0);
assert.deepEqual(
responses(ws).map((r) => r.id),
['fresh'],
);
ws.message({
serverContent: { ...audioContent('fresh reply'), turnComplete: true },
});
h.captures[0].drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion').status,
'completed',
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
test('H10 native: held speech dispatches without caption authority and new text revokes its call', async () => {
const gate = deferred(),
executions = [];
const h = harness({
runAction: (name, args, options) => {
executions.push({ name, args, options });
return gate.promise;
},
});
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
audio = h.captures[0];
h.c.input.bargeIn();
h.c.input.setMicrophoneEnabled(true);
audio.callbacks.onAudio(
{ data: 'AA==', mimeType: 'audio/pcm;rate=16000' },
audio.metadata,
);
h.c.input.setMicrophoneEnabled(false);
caption(ws, 'Show the current view');
assert.equal(h.c.currentTypedInput(), null);
assert.equal(
h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
.text,
'Show the current view',
);
request(ws, 'held');
await tick();
const oldWork = h.c.toolQueue;
assert.equal(executions.length, 1);
assert.equal(executions[0].options.userTurn, undefined);
assert.equal(executions[0].options.isCurrent(), true);
h.c.sendTextCommand('new intent');
await oldWork;
assert.equal(executions[0].options.signal.aborted, true);
assert.equal(executions[0].options.isCurrent(), false);
gate.resolve({ ok: true });
await tick();
assert.equal(
responses(ws).some((r) => r.id === 'held'),
false,
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
for (const [id, restart] of [
['H11', false],
['H12', true],
]) {
test(`${id} native: pending tool cannot revive voice after Stop${restart ? '/restart' : ''}`, async () => {
const gate = deferred(),
mutations = [],
executions = [];
const h = harness({
runAction: async (name, args, options) => {
executions.push(options);
await gate.promise;
if (options.isCurrent()) mutations.push(name);
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
request(ws, 'old');
await tick();
const pending = h.c.toolQueue;
assert.equal(h.c.pendingTools, 1);
h.c.stop();
const boundary = h.events.length;
const successor = restart ? h.c.start() : null;
await pending;
assert.equal(
h.events
.slice(boundary)
.some((e) => e.type === 'state' && e.state === 'listening'),
false,
);
assert.equal(executions[0].signal.aborted, true);
assert.equal(executions[0].isCurrent(), false);
assert.equal(h.c.pendingTools, 0);
if (successor) await successor;
const laterBoundary = h.events.length;
gate.resolve();
caption(ws, 'late old speech');
await tick();
assert.deepEqual(mutations, []);
assert.equal(
h.events.slice(laterBoundary).some((e) => e.type === 'state'),
false,
);
assert.equal(responses(ws).length, 0);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const [id, boundary] of [
['H14', 'interrupted'],
['H15', 'natural'],
]) {
test(`${id} native: shared runner zoom executes promptly after ${boundary} retired boundary`, async () => {
const mutations = [],
observed = [];
const guarded = withToolCatalog(
async (name, args) => {
mutations.push({ name, args });
return { ok: true };
},
async () => null,
);
const h = harness({
runAction: (name, args, options) => {
observed.push(options);
return guarded(name, args, options);
},
});
try {
await h.c.start();
const ws = h.sockets[0];
ws.message({ serverContent: audioContent('old reply') });
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({
serverContent: {
...(boundary === 'interrupted' ? { interrupted: true } : {}),
turnComplete: true,
},
});
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
caption(ws, 'zoom in');
request(ws, 'zoom', 'adjust_camera_zoom', { direction: 'in' });
await tick();
assert.equal(
mutations.length,
1,
'native action executes without a transcript wait',
);
assert.equal(observed[0].userTurn, undefined);
assert.equal(h.c.pendingTools, 0);
assert.equal(
responses(ws).some((r) => r.response.voiceIntentUnverified),
false,
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const [id, exit] of [
['H18', 'stop'],
['H19', 'replacement'],
]) {
test(`${id} native: ${exit} prevents queued camera dispatch and late captions cannot revive it`, async () => {
const mutations = [],
observed = [];
const guarded = withToolCatalog(
async (name) => {
mutations.push(name);
return { ok: true };
},
async () => null,
);
const h = harness({
runAction: (name, args, options) => {
observed.push(name);
return guarded(name, args, options);
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
request(ws, 'queued-old', 'adjust_camera_zoom', { direction: 'in' });
const oldWork = h.c.toolQueue;
assert.equal(h.c.pendingTools, 1);
if (exit === 'stop') h.c.stop();
else h.c.sendTextCommand('Show the whole globe');
caption(ws, 'zoom in');
await oldWork;
assert.deepEqual(mutations, []);
assert.deepEqual(observed, []);
assert.equal(h.c.pendingTools, 0);
if (exit === 'replacement') {
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
request(ws, 'fresh-camera', 'zoom_to_globe');
await h.c.toolQueue;
assert.deepEqual(mutations, ['zoom_to_globe']);
assert.deepEqual(observed, ['zoom_to_globe']);
assert.equal(responses(ws).at(-1).response.ok, true);
}
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const settlement of ['queued', 'running']) {
test(`native: cancelled ${settlement} tool settles a completed provider turn`, async () => {
const result = deferred();
let executions = 0;
const h = harness({
runAction: () => {
executions++;
return result.promise;
},
});
try {
await h.c.start();
h.c.sendTextCommand('Read current view');
const ws = h.sockets[0];
request(ws, 'cancelled-query');
if (settlement === 'running') await tick();
ws.message({ toolCallCancellation: { ids: ['cancelled-query'] } });
ws.message({ serverContent: { turnComplete: true } });
await h.c.toolQueue;
assert.equal(h.c.pendingTools, 0);
assert.equal(executions, settlement === 'queued' ? 0 : 1);
assert.deepEqual(responses(ws), []);
assert.equal(
h.events.filter((event) => event.type === 'completion').length,
1,
);
assert.equal(h.c.pendingCompletionStatus, null);
result.resolve({ ok: true });
await tick();
h.captures[0].drain();
assert.equal(
h.events.filter((event) => event.type === 'completion').length,
1,
);
assert.deepEqual(responses(ws), []);
} finally {
result.resolve({ ok: true });
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const order of ['before', 'after']) {
test(`native: cancelling queued output ${order} turnComplete emits one completion`, async () => {
const h = harness();
try {
await h.c.start();
h.c.sendTextCommand('Read current view');
const ws = h.sockets[0];
ws.bufferedAmount = 1e9;
request(ws, 'queued-output');
await h.c.toolQueue;
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.controlQueue.length, 1);
assert.equal(responses(ws).length, 0);
if (order === 'after')
ws.message({ serverContent: { turnComplete: true } });
ws.message({ toolCallCancellation: { ids: ['queued-output'] } });
if (order === 'before')
ws.message({ serverContent: { turnComplete: true } });
assert.equal(h.c.controlQueue.length, 0);
assert.equal(h.c.controlTimer, null);
assert.equal(h.c.pendingCompletionStatus, null);
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
ws.bufferedAmount = 0;
h.advance(1000);
h.captures[0].drain();
assert.deepEqual(responses(ws), []);
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const successor of ['input', 'stop']) {
test(`native: queued-output cancellation completion fences reentrant ${successor}`, async () => {
let h;
let successorTurn;
h = harness({
emit: (event) => {
h.events.push(event);
if (event.type !== 'completion') return;
h.sockets[0].bufferedAmount = 0;
if (successor === 'stop') h.c.stop();
else {
h.c.sendTextCommand('Show the whole globe');
successorTurn = h.c.turn;
}
},
});
try {
await h.c.start();
h.c.sendTextCommand('Read current view');
const ws = h.sockets[0];
ws.bufferedAmount = 1e9;
request(ws, 'queued-output');
await h.c.toolQueue;
ws.message({ serverContent: { turnComplete: true } });
const callCount = h.calls.length;
ws.message({
toolCallCancellation: { ids: ['queued-output'] },
serverContent: {
outputTranscription: { text: 'Retired packet caption' },
turnComplete: true,
},
toolCall: {
functionCalls: [
{ id: 'retired-tool', name: 'get_current_view_state', args: {} },
],
},
});
await h.c.toolQueue;
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
assert.equal(h.calls.length, callCount);
assert.deepEqual(responses(ws), []);
assert.ok(
!h.events.some((e) =>
JSON.stringify(e).includes('Retired packet caption'),
),
);
if (successor === 'input') {
assert.equal(h.c.turn, successorTurn);
assert.equal(h.c.modelComplete, false);
assert.equal(h.c.replyPending, true);
} else assert.equal(h.c.isActive(), false);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
+734
View File
@@ -0,0 +1,734 @@
// Native ownership replacements for historical Radio regression cases.
// Real Gemini controller, action runner, DataLayerManager and RealtimeRadio consumers.
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
import { createGevActionRunner } from './gevActions.js';
import { createVoiceSession } from './session.js';
import { DataLayerManager } from '../data/manager.js';
const viewerFixture = () => ({
clock: { onTick: { addEventListener: () => () => {} } },
camera: { moveEnd: { addEventListener() {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
});
const request = (ws, id, name, args = {}) =>
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
const responses = (ws) =>
ws.sent.flatMap((frame) => frame.toolResponse?.functionResponses || []);
const responseFor = (ws, id) =>
responses(ws).findLast((response) => response.id === id)?.response;
const speechTurn = (ws) => {
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
};
function drainRadioReply(h) {
h.sockets.at(-1).message({
serverContent: {
modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] },
turnComplete: true,
},
});
h.captures.at(-1).drain();
}
function radioSiblingFixture() {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const lookupEntered = deferred();
const lookupReleased = deferred();
const state = {
stationCount: 2,
selected: { id: 'existing-station', name: 'Existing Station' },
audioState: 'stopped',
};
const trace = { stops: 0, pauses: 0, selections: 0, options: new Map() };
const radio = {
async init() {},
async enable() {},
async disable() {},
async update() {},
getStats: () => ({ count: 2 }),
getUIState: () => ({ ...state }),
setVoiceDucked() {},
pause({ origin } = {}) {
if (origin === 'voice') trace.pauses++;
state.audioState = 'paused';
return true;
},
stopPlayback() {
trace.stops++;
state.audioState = 'stopped';
return true;
},
selectRequestedStation() {
trace.selections++;
state.selected = { id: 'late-station', name: 'Late Station' };
return state.selected;
},
subscribePlaybackControls: () => () => {},
};
manager.register(
Object.assign(radio, {
id: 'radio',
name: 'Radio',
updateInterval: -1,
}),
);
const runner = createGevActionRunner({
viewer,
dataManager: manager,
placeSearch: {
async geocode(_query, { signal }) {
trace.lookupSignal = signal;
lookupEntered.resolve();
// Deliberately ignore abort: production currentness must still prevent
// this late service completion from committing a station selection.
await lookupReleased.promise;
return { place: { lat: 10, lng: 20, label: 'Deferred Place' } };
},
},
});
const h = harness({
dataManager: manager,
radioLayer: radio,
runAction(name, args, options) {
trace.options.set(options.callId, options);
return runner(name, args, options);
},
});
return {
h,
manager,
runner,
radio,
trace,
state,
lookupEntered,
lookupReleased,
};
}
for (const action of ['select', 'play']) {
for (const stronger of ['stop', 'pause', 'disable', 'off']) {
test(`native same-turn Radio ${stronger} preempts deferred ${action} without cancelling a query sibling`, async () => {
const f = radioSiblingFixture();
const { h, trace, state } = f;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
const turn = h.c.turn;
request(ws, 'slow-selection', 'control_radio', {
action,
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'query-sibling', 'control_radio', { action: 'status' });
request(
ws,
'stronger-radio',
stronger === 'off' ? 'set_layer_visibility' : 'control_radio',
stronger === 'off'
? { layerId: 'radio', enabled: false }
: { action: stronger },
);
await tick();
assert.equal(
responseFor(ws, 'stronger-radio')?.ok,
true,
'stronger control must finish before lookup release',
);
assert.equal(trace.lookupSignal.aborted, true);
await h.c.toolQueue;
assert.equal(
h.c.turn,
turn,
'sibling control cannot invent a new turn',
);
assert.equal(trace.selections, 0);
assert.equal(state.selected.id, 'existing-station');
assert.equal(responseFor(ws, 'slow-selection').ok, false);
assert.match(responseFor(ws, 'slow-selection').error, /cancelled/);
assert.equal(
responseFor(ws, 'slow-selection').enabled,
!['disable', 'off'].includes(stronger),
);
assert.equal(responseFor(ws, 'query-sibling').ok, true);
assert.equal(trace.options.get('query-sibling').signal.aborted, false);
assert.equal(
responseFor(ws, 'stronger-radio').enabled,
!['disable', 'off'].includes(stronger),
);
assert.equal(trace.stops, stronger === 'stop' ? 1 : 0);
assert.equal(trace.pauses, stronger === 'pause' ? 1 : 0);
f.lookupReleased.resolve();
await tick();
assert.equal(trace.selections, 0, 'late lookup cannot commit');
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
for (const id of ['slow-selection', 'query-sibling', 'stronger-radio'])
assert.equal(responses(ws).filter((r) => r.id === id).length, 1);
} finally {
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
}
for (const guard of ['malformed', 'replay', 'invalid-arguments']) {
test(`native stronger Radio ${guard} cannot revoke a same-turn lookup`, async () => {
const f = radioSiblingFixture();
const { h, trace } = f;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
if (guard === 'replay') {
request(ws, 'stronger-radio', 'control_radio', { action: 'stop' });
await h.c.toolQueue;
}
speechTurn(ws);
if (guard === 'replay')
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
const stops = trace.stops;
const handoffEpoch = h.c.radio.radioHandoffEpoch;
request(
ws,
guard === 'malformed' ? '\n' : 'stronger-radio',
'control_radio',
{
action: 'stop',
...(guard === 'invalid-arguments'
? { latitude: 91, longitude: 0 }
: {}),
},
);
await tick();
assert.equal(trace.lookupSignal.aborted, false);
assert.equal(trace.stops, stops);
assert.equal(h.c.radio.radioHandoffEpoch, handoffEpoch);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
if (guard === 'invalid-arguments') {
assert.equal(responseFor(ws, 'stronger-radio').ok, false);
assert.match(responseFor(ws, 'stronger-radio').error, /coordinates/);
}
f.lookupReleased.resolve();
await h.c.toolQueue;
assert.equal(
trace.selections,
1,
'older eligible work retains authority',
);
assert.equal(responseFor(ws, 'slow-selection').ok, true);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
} finally {
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
test('native failed Stop releases its reservation and lets the older selection finish', async () => {
const f = radioSiblingFixture();
const { h, trace } = f;
const stopReleased = deferred();
f.radio.stopPlayback = () => stopReleased.promise;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'failed-stop', 'control_radio', { action: 'stop' });
await tick();
assert.equal(h.c.radio.radioToolHandoffReservations.size, 1);
assert.equal(
trace.lookupSignal.aborted,
false,
'provisional Stop cannot revoke work',
);
assert.equal(h.c.pendingTools, 2);
stopReleased.resolve(false);
await tick();
assert.equal(responseFor(ws, 'failed-stop').ok, false);
assert.match(responseFor(ws, 'failed-stop').error, /could not be stopped/);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
assert.equal(trace.lookupSignal.aborted, false);
f.lookupReleased.resolve();
await h.c.toolQueue;
assert.equal(responseFor(ws, 'slow-selection').ok, true);
assert.equal(trace.selections, 1);
assert.equal(f.state.selected.id, 'late-station');
assert.equal(h.c.pendingTools, 0);
} finally {
stopReleased.resolve(false);
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
test('native pending Stop preserves selection completed before its own successful settlement', async () => {
const f = radioSiblingFixture();
const { h } = f;
const stopReleased = deferred();
f.radio.stopPlayback = () => stopReleased.promise;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'pending-stop', 'control_radio', { action: 'stop' });
await tick();
f.lookupReleased.resolve();
await tick();
assert.equal(responseFor(ws, 'slow-selection').ok, true);
assert.equal(f.state.selected.id, 'late-station');
assert.equal(h.c.radio.radioToolHandoffReservations.size, 1);
assert.ok(h.c.radio.pendingRadioPlaybackResult);
stopReleased.resolve(true);
await h.c.toolQueue;
assert.equal(responseFor(ws, 'pending-stop').ok, true);
assert.equal(responseFor(ws, 'pending-stop').stationId, 'late-station');
assert.equal(f.state.selected.id, 'late-station');
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
assert.equal(h.c.pendingTools, 0);
} finally {
stopReleased.resolve(false);
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
for (const restart of [false, true]) {
test(`native Stop${restart ? '/restart' : ''} retires concurrent stronger Radio work without late output or count underflow`, async () => {
const f = radioSiblingFixture();
const { h } = f;
const stopReleased = deferred();
f.radio.stopPlayback = () => stopReleased.promise;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'pending-stop', 'control_radio', { action: 'stop' });
await tick();
const oldDrain = h.c.toolQueue;
const stopOptions = f.trace.options.get('pending-stop');
assert.equal(h.c.pendingTools, 2);
h.c.stop();
assert.equal(stopOptions.signal.aborted, true);
assert.equal(f.trace.lookupSignal.aborted, true);
if (restart) {
await h.c.start();
request(h.sockets[1], 'new-query', 'control_radio', {
action: 'status',
});
await h.c.toolQueue;
assert.equal(responseFor(h.sockets[1], 'new-query').ok, true);
}
stopReleased.resolve(true);
f.lookupReleased.resolve();
await oldDrain;
await tick();
assert.equal(responseFor(ws, 'pending-stop'), undefined);
assert.equal(responseFor(ws, 'slow-selection'), undefined);
assert.equal(f.trace.selections, 0);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
assert.equal(h.c.isActive(), restart);
} finally {
stopReleased.resolve(false);
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
for (const phase of ['prepared', 'preflight']) {
test(`native same-turn Stop revokes only voice-owned ${phase} playback and preserves completed station`, async () => {
const f = radioSiblingFixture();
const { h } = f;
const ready = deferred();
const stopped = [];
let preflight;
f.radio.playForVoice = () => ready.promise;
f.radio.stopPlayback = (options) => {
stopped.push(options);
return true;
};
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'prepared-play', 'control_radio', { action: 'play' });
await h.c.toolQueue;
assert.equal(
responseFor(ws, 'prepared-play').radioPlaybackRequested,
true,
);
if (phase === 'preflight')
preflight = h.c.radio.startPendingRadioHandoff();
const attemptId = h.c.radio.radioHandoffAttemptId;
request(ws, 'stronger-stop', 'control_radio', { action: 'stop' });
await h.c.toolQueue;
assert.equal(responseFor(ws, 'stronger-stop').ok, true);
assert.equal(f.state.selected.id, 'existing-station');
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
assert.equal(h.c.radio.radioHandoffInFlight, false);
const cleanups = stopped.filter((o) => o.origin === 'voice-cleanup');
assert.deepEqual(
cleanups,
phase === 'preflight' ? [{ origin: 'voice-cleanup', attemptId }] : [],
);
ready.resolve(true);
await preflight;
await tick();
assert.equal(
h.c.isActive(),
true,
'late old readiness cannot stop voice',
);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
assert.equal(h.c.pendingTools, 0);
} finally {
ready.resolve(false);
h.c.stop();
await preflight;
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
for (const phase of ['prepared', 'preflight', 'settling']) {
test(`native Radio cancellation: ${phase} preserves selection truth and prevents stale speaker transfer`, async () => {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const prepared = deferred();
const settleResult = deferred();
const playbackReady = deferred();
let playCalls = 0;
let stopCalls = 0;
let selections = 0;
let activeOptions;
const state = {
stationCount: 2,
selected: null,
audioState: 'stopped',
voiceDucked: false,
};
const radio = {
async init() {},
async enable() {},
async disable() {},
async update() {},
getStats: () => ({ count: 2 }),
getUIState: () => ({ ...state }),
setVoiceDucked(value) {
state.voiceDucked = value;
},
pause() {
state.audioState = 'paused';
},
subscribePlaybackControls() {
return () => {};
},
selectRequestedStation(_criteria, options) {
assert.equal(options.autoplay, false);
selections++;
state.selected = { id: 'station-1', name: 'Station One' };
return state.selected;
},
playForVoice() {
playCalls++;
return playbackReady.promise;
},
stopPlayback() {
stopCalls++;
state.audioState = 'stopped';
},
};
manager.register({
id: 'radio',
name: 'Radio',
updateInterval: -1,
...radio,
});
const runner = createGevActionRunner({ viewer, dataManager: manager });
let h;
let preflight;
const session = createVoiceSession({
runner: async (name, args, options) => {
activeOptions = options;
const result = await runner(name, args, options);
prepared.resolve(result);
if (phase === 'settling') await settleResult.promise;
return result;
},
createAdapter: (shared) => {
h = harness({ ...shared, dataManager: manager, radioLayer: radio });
return {
start: (settings) => h.c.start(settings),
stop: (settings) => h.c.stop(settings),
sendText: (text) => h.c.sendTextCommand(text),
sendMapEvent: (event) => h.c.notifyMapEvent(event),
};
},
});
try {
await session.start();
const ws = h.sockets[0];
assert.equal(session.sendText('Select Station One on Radio'), true);
request(ws, 'prepared-radio', 'control_radio', {
action: 'select',
stationQuery: 'Station One',
});
const actual = await prepared.promise;
assert.equal(actual.ok, true);
assert.equal(actual.radioPlaybackRequested, true);
assert.equal(actual.stationId, 'station-1');
assert.equal(manager.isEnabled('radio'), true);
if (phase !== 'settling') {
await h.c.toolQueue;
assert.ok(h.c.radio.pendingRadioPlaybackResult);
}
if (phase === 'preflight') {
preflight = h.c.radio.startPendingRadioHandoff();
assert.equal(h.c.radio.radioHandoffInFlight, true);
assert.equal(playCalls, 1);
}
// Native call ownership revokes only this deferred playback. No caption
// parsing or caption-driven provider policy participates in cancellation.
ws.message({ toolCallCancellation: { ids: ['prepared-radio'] } });
if (phase === 'settling')
assert.equal(activeOptions.signal.aborted, true);
settleResult.resolve();
await h.c.toolQueue;
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
assert.equal(h.c.radio.radioHandoffInFlight, false);
assert.equal(manager.isEnabled('radio'), true);
assert.equal(state.selected.id, 'station-1');
assert.equal(selections, 1);
const modelResult = responseFor(ws, 'prepared-radio');
if (phase === 'settling')
assert.equal(
modelResult,
undefined,
'cancelled active call never sends a late result',
);
else {
assert.equal(modelResult.ok, true);
assert.equal(modelResult.stationId, 'station-1');
}
const stopsAfterCancellation = stopCalls;
assert.equal(stopsAfterCancellation, phase === 'preflight' ? 1 : 0);
drainRadioReply(h);
playbackReady.resolve(true);
await preflight;
await tick();
assert.equal(
playCalls,
phase === 'preflight' ? 1 : 0,
'drain cannot revive cancelled playback',
);
assert.equal(h.c.isActive(), true, 'late readiness cannot stop voice');
assert.equal(h.c.radio.radioHandoffInFlight, false);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
assert.equal(session.sendText('Play radio'), true);
request(ws, 'fresh-radio', 'control_radio', { action: 'play' });
await h.c.toolQueue;
assert.ok(h.c.radio.pendingRadioPlaybackResult);
drainRadioReply(h);
await tick();
assert.equal(playCalls, phase === 'preflight' ? 2 : 1);
assert.equal(
h.c.isActive(),
false,
'fresh typed Radio handoff alone ends voice',
);
} finally {
settleResult.resolve();
playbackReady.resolve(false);
session.destroy();
await preflight;
runner.dispose();
await manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
// A stronger Radio control skips the ordered queue. An older Radio call still
// queued behind a slow sibling must not start after that control succeeded,
// while a failed control or a later call in the same turn keeps its order.
for (const [label, ids, stopOk, expectPending] of [
[
'older queued select is revoked by a later successful stop',
['slow', 'sel', 'stop'],
true,
false,
],
[
'a failed stop leaves the older queued select current',
['slow', 'sel', 'stop'],
false,
true,
],
[
'a select after the stop in the same turn still runs',
['slow', 'stop', 'sel'],
true,
true,
],
]) {
test(`stronger Radio ordering: ${label}`, async () => {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const state = {
stationCount: 2,
selected: { id: 'existing', name: 'Existing' },
audioState: 'stopped',
};
const radio = {
id: 'radio',
name: 'Radio',
updateInterval: -1,
async init() {},
async enable() {},
async disable() {},
async update() {},
getStats: () => ({ count: 2 }),
getUIState: () => ({ ...state }),
setVoiceDucked() {},
pause() {
state.audioState = 'paused';
return true;
},
stopPlayback() {
state.audioState = 'stopped';
return stopOk;
},
selectRequestedStation() {
state.selected = { id: 'late', name: 'Late' };
return state.selected;
},
async playForVoice() {
return true;
},
subscribePlaybackControls: () => () => {},
};
manager.register(radio);
const runner = createGevActionRunner({
viewer,
dataManager: manager,
placeSearch: {
async geocode() {
return { place: { lat: 10, lng: 20, label: 'Place' } };
},
},
});
const slow = deferred();
const h = harness({
dataManager: manager,
radioLayer: radio,
async runAction(name, args, options) {
if (name === 'slow_sibling') {
await slow.promise;
return { ok: true, action: 'slow_sibling' };
}
if (name === 'control_radio' && args.action === 'stop' && !stopOk)
return {
ok: false,
action: 'control_radio',
error: 'Radio stop failed',
};
return runner(name, args, options);
},
});
const calls = {
slow: { id: 'slow', name: 'slow_sibling', args: {} },
sel: {
id: 'sel',
name: 'control_radio',
args: { action: 'select', locationQuery: 'Place' },
},
stop: { id: 'stop', name: 'control_radio', args: { action: 'stop' } },
};
try {
await manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
ws.message({ toolCall: { functionCalls: ids.map((id) => calls[id]) } });
await tick();
assert.equal(
responseFor(ws, 'stop')?.ok,
stopOk,
'stop settles without waiting',
);
slow.resolve();
await h.c.toolQueue;
await tick();
assert.equal(
Boolean(h.c.radio.pendingRadioPlaybackResult),
expectPending,
);
assert.equal(responseFor(ws, 'sel')?.ok, expectPending);
} finally {
slow.resolve();
h.c.stop();
runner.dispose();
await manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
+282
View File
@@ -0,0 +1,282 @@
import { createCesiumEngine } from '../globe/cesium/createCesiumEngine.js';
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
import { createGevActionRunner } from './gevActions.js';
const tool = (ws, id) =>
ws.message({
toolCall: {
functionCalls: [
{
id,
name: 'adjust_camera_zoom',
args: { direction: 'in', amount: 'little' },
},
],
},
});
const caption = (ws, text) =>
ws.message({ serverContent: { inputTranscription: { text } } });
const onset = (ws) => {
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
};
const complete = (ws) => ws.message({ serverContent: { turnComplete: true } });
const oldAudio = {
serverContent: {
modelTurn: {
parts: [
{
inlineData: {
data: 'AAAA',
mimeType: 'audio/pcm;rate=24000',
},
},
],
},
},
};
const outputs = (ws) =>
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
// Socket/media are deterministic. Mutations use the production action runner.
function fixture({ holdFirst = false } = {}) {
const state = { height: 1000, renders: 0, providerWrites: [], panels: [] };
const position = { x: 6379137, y: 0, z: 0 };
const viewer = {
clock: { onTick: { addEventListener: () => () => {} } },
camera: {
positionWC: position,
positionCartographic: { height: 1000 },
moveEnd: { addEventListener() {} },
cancelFlight() {},
zoomIn(n) {
position.x -= n;
state.height -= n;
this.positionCartographic.height = state.height;
},
zoomOut(n) {
position.x += n;
state.height += n;
this.positionCartographic.height = state.height;
},
},
scene: {
canvas: {
clientWidth: 800,
clientHeight: 600,
addEventListener() {},
removeEventListener() {},
},
requestRender() {
state.renders++;
},
},
};
const runner = createGevActionRunner({
engine: createCesiumEngine(viewer),
viewer,
dataManager: { layers: new Map() },
styleManager: {
setVoiceProvider: (p) => state.providerWrites.push(p),
setPanelCollapsed: (...args) => state.panels.push(args),
},
});
const entered = [];
const gate = deferred();
const h = harness({
runAction: async (name, args, options) => {
entered.push({ name, args, options });
if (holdFirst && entered.length === 1) await gate.promise;
if (!options.isCurrent())
return { ok: false, action: name, changed: false };
return runner(name, args, options);
},
});
return {
...h,
state,
entered,
gate,
close() {
gate.resolve();
h.c.stop();
runner.dispose();
assert.equal(h.timers.size, 0);
assert.deepEqual(state.providerWrites, []);
assert.deepEqual(state.panels, []);
},
};
}
for (const [id, order] of [
['H20', 'before-boundary'],
['H21', 'after-boundary'],
]) {
test(`native replacement ${id}: ${order} tool respects retired output and next onset recovers`, async () => {
const h = fixture();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(oldAudio);
onset(ws);
caption(ws, 'zoom in');
if (order === 'before-boundary') {
tool(ws, 'ambiguous-before');
await tick();
assert.equal(h.state.height, 1000);
assert.equal(h.entered.length, 0);
assert.equal(outputs(ws).length, 0);
}
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
if (order === 'after-boundary') {
tool(ws, 'native-after');
await h.c.toolQueue;
assert.equal(h.state.height, 750);
assert.equal(h.entered.length, 1);
assert.equal(outputs(ws).at(-1).response.ok, true);
complete(ws);
}
onset(ws);
caption(ws, 'zoom in');
tool(ws, 'fresh-clean-onset');
await h.c.toolQueue;
assert.equal(h.state.height, order === 'before-boundary' ? 750 : 562.5);
assert.equal(h.entered.length, order === 'before-boundary' ? 1 : 2);
assert.equal(h.entered.at(-1).options.userTurn, undefined);
assert.equal(h.c.pendingTools, 0);
assert.equal(
outputs(ws).some((o) => o.id === 'ambiguous-before'),
false,
);
assert.equal(
outputs(ws).find((o) => o.id === 'fresh-clean-onset').response.ok,
true,
);
} finally {
h.close();
}
});
}
test('native replacement H30: ordinary tools run promptly and late caption fragments cannot strand siblings', async () => {
const h = fixture();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
caption(ws, 'switch to');
tool(ws, 'ordinary-before-caption-resolution');
await tick();
await h.c.toolQueue;
assert.equal(h.state.height, 750);
assert.equal(h.c.pendingTools, 0);
h.advance(1499);
caption(ws, 'Austin');
h.advance(5000);
tool(ws, 'released-sibling');
await h.c.toolQueue;
assert.equal(h.state.height, 562.5);
assert.equal(h.entered.length, 2);
assert.equal(
h.entered.every((c) => c.options.userTurn === undefined),
true,
);
assert.equal(outputs(ws).length, 2);
assert.equal(
outputs(ws).every((o) => o.response.ok),
true,
);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.status, 'listening');
} finally {
h.close();
}
});
for (const [id, exit] of [
['H32', 'stop'],
['H33', 'restart'],
['H34', 'new-input'],
]) {
test(`native replacement ${id}: ${exit} retires owned in-flight work and late callbacks`, async () => {
const h = fixture({ holdFirst: true });
try {
await h.c.start();
let ws = h.sockets[0];
onset(ws);
caption(ws, 'switch to');
tool(ws, 'old-owned-action');
await tick();
assert.equal(h.entered.length, 1);
assert.equal(h.c.pendingTools, 1);
assert.equal(h.state.height, 1000);
const oldWork = h.c.toolQueue;
const oldMessage = ws.onmessage;
const oldOwner = h.entered[0].options;
if (exit === 'new-input') h.c.sendTextCommand('zoom out');
else h.c.stop();
assert.equal(oldOwner.signal.aborted, true);
assert.equal(oldOwner.isCurrent(), false);
if (exit === 'restart') {
await h.c.start();
ws = h.sockets.at(-1);
}
if (exit !== 'new-input')
oldMessage({
data: JSON.stringify({
toolCall: {
functionCalls: [
{
id: 'late-old-call',
name: 'adjust_camera_zoom',
args: { direction: 'in' },
},
],
},
}),
});
caption(ws, 'Gemini');
h.advance(1500);
h.gate.resolve();
await oldWork;
await tick();
assert.equal(h.state.height, 1000);
assert.equal(h.c.pendingTools, 0);
assert.equal(
outputs(ws).some(
(o) => o.id === 'old-owned-action' || o.id === 'late-old-call',
),
false,
);
if (exit === 'stop') {
assert.equal(h.c.status, 'idle');
assert.equal(h.c.isActive(), false);
} else {
if (exit === 'new-input') {
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
} else h.c.sendTextCommand('zoom out');
tool(ws, 'fresh-after-retirement');
await h.c.toolQueue;
assert.equal(h.state.height, 750);
assert.equal(h.entered.length, 2);
assert.equal(
outputs(ws).find((o) => o.id === 'fresh-after-retirement').response
.ok,
true,
);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.isActive(), true);
}
} finally {
h.close();
}
});
}
+529
View File
@@ -0,0 +1,529 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
nativeIntegrationFixture,
assertSuccessfulZoom,
turnOfEventLoop,
} from '../testSupport/geminiIntegrationFixture.mjs';
const options = { concurrency: false, timeout: 5000 };
const OLD_TEXT = 'Old packet caption.';
const OLD_PCM = 'AAABAA==';
const POST_TOOL_PCM = 'AgADAA==';
const OLD_CALL = 'old-coalesced-call';
function coalescedPacket(callId = OLD_CALL) {
return {
serverContent: {
outputTranscription: { text: OLD_TEXT },
modelTurn: {
parts: [
{ inlineData: { data: OLD_PCM, mimeType: 'audio/pcm;rate=24000' } },
],
},
turnComplete: true,
},
toolCall: {
functionCalls: [
{
id: callId,
name: 'adjust_camera_zoom',
args: { direction: 'in', amount: 'little' },
},
],
},
};
}
// Local test observations wrap only injected device fakes and the actual
// scheduler methods, delegating unchanged. No frozen fixture file is modified.
function observePacketEffects(h) {
const played = [];
const narration = { audible: 0, toolStarted: 0, toolFinished: 0 };
const createAudio = h.controller.createAudio;
h.controller.createAudio = (callbacks) => {
const audio = createAudio(callbacks);
const play = audio.play.bind(audio);
audio.play = (data, mimeType) => {
played.push({ audio, data, mimeType });
return play(data, mimeType);
};
return audio;
};
for (const method of Object.keys(narration)) {
const invoke = h.controller.narration[method].bind(h.controller.narration);
h.controller.narration[method] = (...args) => {
narration[method]++;
return invoke(...args);
};
}
return { played, narration };
}
function packetState(h) {
const c = h.controller;
return {
epoch: c.epoch,
inputGeneration: c.inputGeneration,
turn: c.turn,
completedTurns: c.completedTurns,
pendingTools: c.pendingTools,
modelComplete: c.modelComplete,
pendingCompletionStatus: c.pendingCompletionStatus,
postToolOutput: c.postToolOutput,
modelSpoke: c.modelSpoke,
assistant: c.transcripts.assistant,
replyPending: c.replyPending,
retiredReplyPending: c.retiredReplyPending,
typedText: c.typedIntent?.text ?? null,
typedResponseId: c.typedIntent?.responseId ?? null,
narrationTools: c.narration.state().tools,
narrationTimers: c.narration.state().pendingTimers,
ownedTimers: h.clock.timers.size,
};
}
const completions = (h) =>
h.events.filter((event) => event.type === 'completion');
const toolEvents = (h, callId) =>
h.events.filter(
(event) =>
['action-call', 'action-result', 'progress'].includes(event.type) &&
event.callId === callId,
);
function assertOldRemainderAbsent(h, observed) {
assert.equal(h.state.height, 1000);
assert.deepEqual(h.state.zooms, []);
assert.deepEqual(h.runnerCalls, []);
assert.deepEqual(toolEvents(h, OLD_CALL), []);
assert.deepEqual(observed.played, []);
assert.deepEqual(observed.narration, {
audible: 0,
toolStarted: 0,
toolFinished: 0,
});
assert.deepEqual(completions(h), []);
assert.equal(h.controller.completedTurns, 0);
assert.equal(h.controller.pendingTools, 0);
assert.equal(
h.events.filter(
(event) =>
event.type === 'transcript' &&
event.role === 'assistant' &&
event.final &&
event.text === OLD_TEXT,
).length,
0,
);
for (const socket of h.sockets)
assert.equal(
h.outputs(socket).some((output) => output.id === OLD_CALL),
false,
);
}
for (const mode of ['Stop', 'Stop then Start', 'typed successor']) {
test(
`assistant transcript observer ${mode} fences the rest of a coalesced native packet`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const observed = observePacketEffects(h);
const oldSocket = await h.start();
const oldAudio = h.audios[0];
h.activity(oldSocket);
const before = packetState(h);
let reached = false;
let restart;
let typedAccepted;
let afterObserver;
const remove = h.session.subscribe((event) => {
if (
reached ||
event.type !== 'transcript' ||
event.role !== 'assistant' ||
event.final ||
event.text !== OLD_TEXT
)
return;
reached = true;
if (mode === 'typed successor') {
typedAccepted = h.session.sendText('Zoom in for the typed successor');
} else {
h.session.stop();
if (mode === 'Stop then Start') restart = h.session.start();
}
afterObserver = packetState(h);
});
oldSocket.message(coalescedPacket());
remove();
assert.equal(
reached,
true,
'the actual common transcript observer ran synchronously',
);
// A retired reply may consume its own received terminal, but cannot
// complete or overwrite the successor. Stop/restart permits no old writes.
const afterPacketExpected =
mode === 'typed successor'
? {
...afterObserver,
retiredReplyPending: false,
ownedTimers: afterObserver.ownedTimers - 1,
}
: afterObserver;
assert.deepEqual(packetState(h), afterPacketExpected);
await h.controller.toolQueue;
await turnOfEventLoop();
if (restart) await restart;
assertOldRemainderAbsent(h, observed);
let successorSocket;
if (mode === 'typed successor') {
assert.equal(typedAccepted, true);
assert.equal(
h.controller.epoch,
before.epoch,
'typed replacement shares the socket epoch',
);
assert.ok(h.controller.inputGeneration > before.inputGeneration);
assert.ok(h.controller.turn > before.turn);
assert.deepEqual(packetState(h), afterPacketExpected);
assert.equal(
h.controller.typedIntent.text,
'Zoom in for the typed successor',
);
assert.equal(h.controller.replyPending, true);
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(oldAudio.closed, false);
const typedMessages = oldSocket.sent.filter((message) =>
message.clientContent?.turns?.some((turn) =>
turn.parts?.some(
(part) => part.text === 'Zoom in for the typed successor',
),
),
);
assert.equal(typedMessages.length, 1);
// The received packet already carried the old terminal. No extra
// provider boundary may be invented before testing the actual successor.
assert.equal(h.controller.retiredBoundaryWait, null);
assert.equal(h.controller.cancelledCalls.has(OLD_CALL), true);
assert.equal(h.controller.replyPending, true);
assert.equal(h.controller.completedTurns, 0);
assert.deepEqual(completions(h), []);
successorSocket = oldSocket;
} else {
assert.equal(oldSocket.closed, true);
assert.equal(oldAudio.closed, true);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
if (mode === 'Stop') {
assert.equal(h.session.state, 'idle');
successorSocket = await h.start();
} else {
assert.equal(h.session.state, 'listening');
assert.equal(h.sockets.length, 2);
assert.equal(h.audios.length, 2);
assert.notEqual(h.audios[1], oldAudio);
assert.equal(h.audios[1].closed, false);
assert.equal(h.audios[1].pending, false);
successorSocket = h.sockets[1];
}
h.activity(successorSocket);
}
await h.tool('owned-after-packet-cancel', successorSocket);
assertSuccessfulZoom(
h,
successorSocket,
'owned-after-packet-cancel',
1000,
);
assert.equal(h.state.zooms.length, 1);
assert.equal(h.runnerCalls.length, 1);
assert.equal(
h.outputs(successorSocket).some((output) => output.id === OLD_CALL),
false,
);
h.complete(successorSocket);
assert.equal(completions(h).length, 1);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
},
);
}
test(
'ordinary coalesced packet plays and executes once, then completes only after its post-tool reply',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const observed = observePacketEffects(h);
const ws = await h.start();
h.activity(ws);
ws.message(coalescedPacket('ordinary-coalesced'));
assert.equal(
h.controller.completedTurns,
1,
'first generation terminal is counted once',
);
assert.equal(h.controller.pendingTools, 1);
assert.deepEqual(
observed.played.map(({ data }) => data),
[OLD_PCM],
);
assert.equal(observed.played[0].audio, h.audios[0]);
assert.deepEqual(completions(h), [], 'tool and playback are still pending');
await h.controller.toolQueue;
assertSuccessfulZoom(h, ws, 'ordinary-coalesced', 1000);
assert.equal(h.controller.pendingTools, 0);
assert.equal(observed.narration.toolStarted, 1);
assert.equal(observed.narration.toolFinished, 1);
assert.equal(h.controller.narration.state().tools, 0);
h.audios[0].drain();
assert.equal(
h.controller.modelComplete,
false,
'function response waits for the post-tool generation',
);
assert.equal(h.controller.completedTurns, 1);
assert.deepEqual(
completions(h),
[],
'draining first-generation PCM cannot finish the tool reply',
);
// The accepted contract has two generations: the function response prompts
// a fresh model reply. This boundary is intentional, not a stale-packet fix.
ws.message({
serverContent: {
outputTranscription: { text: 'The requested zoom is complete.' },
modelTurn: {
parts: [
{
inlineData: {
data: POST_TOOL_PCM,
mimeType: 'audio/pcm;rate=24000',
},
},
],
},
turnComplete: true,
},
});
assert.equal(h.controller.completedTurns, 2);
assert.deepEqual(
observed.played.map(({ data }) => data),
[OLD_PCM, POST_TOOL_PCM],
);
assert.deepEqual(
completions(h),
[],
'post-tool playback must drain before completion',
);
h.audios[0].drain();
assert.equal(completions(h).length, 1);
assert.equal(completions(h)[0].status, 'completed');
assert.equal(completions(h)[0].spoken, true);
assert.equal(completions(h)[0].audioDrained, true);
assert.equal(h.controller.pendingCompletionStatus, null);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
assert.equal(observed.narration.audible, 2);
assert.equal(h.runnerCalls.length, 1);
assert.equal(h.state.zooms.length, 1);
assert.equal(h.outputs(ws).length, 1);
h.audios[0].drain();
assert.equal(
completions(h).length,
1,
'a duplicate device drain cannot complete twice',
);
},
);
// Real native message-type fields are separate packets here; no extra old
// completion is inserted before the accepted successor.
const BOUNDARY_OLD = 'Prior reply caption.';
const BOUNDARY_NEXT = 'Zoom in for the typed successor';
const boundaryZoomCall = (id) => ({
toolCall: {
functionCalls: [
{
id,
name: 'adjust_camera_zoom',
args: { direction: 'in', amount: 'little' },
},
],
},
});
function boundarySnapshot(h) {
const c = h.controller;
return {
retiredReplyPending: c.retiredReplyPending,
retiredBoundaryWait: Boolean(c.retiredBoundaryWait),
replyPending: c.replyPending,
typedText: c.typedIntent?.text ?? null,
completions: h.events.filter((event) => event.type === 'completion').length,
};
}
for (const mode of [
'partial assistant transcript',
'input transcript',
'interruption event',
'listening event',
'final assistant transcript control',
]) {
test(
`received native terminal preserves typed successor after ${mode}`,
{ concurrency: false, timeout: 5000 },
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
h.activity(ws);
let oldDispatch, gate;
const interruptionMode =
mode === 'interruption event' || mode === 'listening event';
if (interruptionMode) {
h.session.sendText('Prior zoom request');
gate = h.holdResult('already-committed-old-tool');
oldDispatch = h.tool('already-committed-old-tool', ws);
await gate.ready;
} else if (mode === 'input transcript') {
ws.message({
serverContent: {
outputTranscription: { text: 'Previously emitted output' },
},
});
}
let entered = 0,
accepted;
const remove = h.session.subscribe((event) => {
if (entered) return;
const matches =
mode === 'interruption event'
? event.type === 'interruption' && event.reason === 'user-speech'
: mode === 'listening event'
? event.type === 'state' && event.state === 'listening'
: mode === 'input transcript'
? event.type === 'transcript' &&
event.role === 'user' &&
event.text === 'Old input caption'
: event.type === 'transcript' &&
event.role === 'assistant' &&
event.text === BOUNDARY_OLD &&
event.final ===
(mode === 'final assistant transcript control');
if (!matches) return;
entered++;
accepted = h.session.sendText(BOUNDARY_NEXT);
});
if (interruptionMode)
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
else
ws.message({
serverContent: {
...(mode === 'input transcript'
? { inputTranscription: { text: 'Old input caption' } }
: {}),
outputTranscription: { text: BOUNDARY_OLD },
turnComplete: true,
},
});
remove();
if (oldDispatch) {
await oldDispatch;
gate.release();
}
await turnOfEventLoop();
const afterPacket = boundarySnapshot(h);
const beforeSuccessorHeight = h.state.height;
// No extra synthetic old boundary. This is the next reply to the accepted text.
ws.message({
serverContent: {
outputTranscription: { text: 'Handling your new zoom.' },
},
});
ws.message(boundaryZoomCall('typed-successor-call'));
await h.controller.toolQueue;
await turnOfEventLoop();
ws.message({ serverContent: { turnComplete: true } });
const afterSuccessor = boundarySnapshot(h);
assert.equal(entered, 1);
assert.equal(accepted, true);
assert.equal(
afterPacket.retiredReplyPending,
false,
'received old terminal must settle its retirement',
);
assert.equal(afterPacket.retiredBoundaryWait, false);
assert.equal(afterPacket.typedText, BOUNDARY_NEXT);
assert.equal(
h.runnerCalls.filter((x) => x.options.callId === 'typed-successor-call')
.length,
1,
'accepted typed successor must reach the actual runner',
);
assert.equal(h.state.height, beforeSuccessorHeight * 0.75);
assert.equal(afterSuccessor.completions, 1);
assert.equal(afterSuccessor.replyPending, false);
assert.equal(
h.controller.cancelledCalls.has('typed-successor-call'),
false,
);
},
);
}
for (const terminalAlreadyApplied of [false, true]) {
test(
`typed reentry retains a genuinely pending boundary when terminal already applied is ${terminalAlreadyApplied}`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
h.activity(ws);
let entered = false;
const remove = h.session.subscribe((event) => {
if (
entered ||
event.type !== 'transcript' ||
event.role !== 'assistant' ||
event.text !== 'Boundary control' ||
event.final !== terminalAlreadyApplied
)
return;
entered = true;
assert.equal(h.session.sendText('First typed replacement'), true);
if (terminalAlreadyApplied)
assert.equal(h.session.sendText('Second typed replacement'), true);
});
ws.message({
serverContent: {
outputTranscription: { text: 'Boundary control' },
...(terminalAlreadyApplied ? { turnComplete: true } : {}),
},
});
remove();
assert.equal(entered, true);
// With no old terminal received, or when that terminal was consumed
// before the observer, a real predecessor boundary is still outstanding.
assert.equal(h.controller.retiredReplyPending, true);
assert.ok(h.controller.retiredBoundaryWait);
assert.equal(completions(h).length, 0);
assert.equal(h.runnerCalls.length, 0);
ws.message({ serverContent: { turnComplete: true } });
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(h.controller.retiredBoundaryWait, null);
await h.tool('after-real-pending-boundary', ws);
assertSuccessfulZoom(h, ws, 'after-real-pending-boundary', 1000);
h.complete(ws);
assert.equal(completions(h).length, 1);
},
);
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+335 -191
View File
@@ -1,6 +1,6 @@
import { interruptGlobeRotation } from '../cameraVerbs.js';
import {
layerSnapshot,
layerSnapshots,
feedProvenanceEnvelope,
} from '../data/layerSnapshot.js';
import { readLayerLifecycleSummary } from './layerSummary.js';
@@ -8,10 +8,19 @@ import { SPEECH_BUILDERS, attachVoiceResult } from './speech.js';
import { resolveDeicticArgs } from './deixis.js';
export { readLayerLifecycleSummary } from './layerSummary.js';
import { defaultGeospatial } from '../search/defaults.js';
import { msg } from '../i18n/index.js';
import { useProviderSettingsStore } from '../ui/stores/providerSettings.js';
import { applicationPinia } from '../ui/vueRuntime.js';
import * as Cesium from 'cesium';
import * as geo from '../geo/index.js';
import { CITY_POIS } from '../cityPois.js';
import { greatCircleKm } from '../geoDistance.js';
import {
hudViewportSamples,
placeLookupView,
placeLookupViewChanged,
} from '../hudPlaceLookup.js';
import {
CITY_POIS,
findPoiByName,
flyToGlobeView,
flyToLandmark,
@@ -26,7 +35,6 @@ import {
isContextRecordActive,
} from '../data/contextStore.js';
import { CCTV_FOCUS_RESULT } from '../layers/cctv/index.js';
import { contextModeWord } from '../contextModePolicy.js';
import {
createAnalystEngine,
normalizeAnalystSpec,
@@ -48,6 +56,12 @@ import { unavailablePlaceSearch } from '../search/placeSearch.js';
import * as defaultAnnotationResolver from '../annotations/annotationResolver.js';
import { normalizeRadioCountryInput } from '../data/radioCountry.js';
import { TR3B_CLASS } from '../data/tr3bRegistry.js';
import {
getWorldPacket,
withContextModeVocabulary,
activeContactsWindow,
TRACKABLE_FAMILIES,
} from '../agent/worldPacket.js';
const ALLOWED_STYLES = new Set([
'normal',
@@ -56,7 +70,6 @@ const ALLOWED_STYLES = new Set([
'thermal',
'anime',
'noir',
'snow',
]);
const PANEL_ALIASES = new Map([
['data', 'data-panel'],
@@ -69,8 +82,17 @@ const PANEL_ALIASES = new Map([
['styles', 'control-panel'],
['filters', 'control-panel'],
['visual styles', 'control-panel'],
['visual preset', 'control-panel'],
['visual presets', 'control-panel'],
['provider settings', 'provider-settings'],
['cctv', 'cctv-panel'],
['cameras', 'cctv-panel'],
['voice settings', 'voice-settings'],
['voice-settings', 'voice-settings'],
['voice preview', 'voice-preview'],
['voice-preview', 'voice-preview'],
['voice control', 'voice-preview'],
['voice controls', 'voice-preview'],
['radio', 'radio-panel'],
['internet radio', 'radio-panel'],
['radio stations', 'radio-panel'],
@@ -79,6 +101,7 @@ const PANEL_ALIASES = new Map([
['global context', 'global-context-panel'],
['right context', 'global-context-panel'],
['context right panel', 'global-context-panel'],
['authoring', 'scene-panel'],
['scenes', 'scene-panel'],
['scene', 'scene-panel'],
['post processing', 'pp-toggles'],
@@ -105,6 +128,7 @@ const PANEL_IDS = new Set([
'pp-toggles',
'weather-panel',
'recent-imagery-panel',
'provider-settings',
]);
const CONTEXT_MODE_ALIASES = new Map([
['off', 'off'],
@@ -119,64 +143,6 @@ const CONTEXT_MODE_ALIASES = new Map([
['space-missions', 'space-missions'],
['missions', 'space-missions'],
]);
/**
* Every model-readable field that carries a context-mode id, and what an
* absent value means for each.
*
* `mode` always names a mode, so nothing is 'off'. `entering` and `priorMode`
* are absent when there is no such mode at all — calling those 'off' would
* assert a state that does not exist.
*/
const CONTEXT_MODE_RESULT_FIELDS = Object.freeze([
{ field: 'mode', emptyAs: 'off' },
{ field: 'entering', emptyAs: null },
{ field: 'priorMode', emptyAs: null },
]);
/** Nested results that are themselves context-mode payloads the model reads. */
const NESTED_CONTEXT_RESULT_FIELDS = Object.freeze([
'context',
'contextRollback',
]);
/**
* Report a context-mode payload in the tools' own vocabulary.
*
* `set_context_mode` accepts 'contacts' while the mode's internal id is
* 'flights'. Reporting the internal id back made the model read
* `mode:'flights'` as "Contacts is off" and refuse to answer from the Contacts
* window counts sitting in the very same payload (owner field session
* 2026-08-21). Secondary fields and nested transition/rollback results are
* translated too — one leaked internal id is enough to recreate the confusion,
* and a rollback result is exactly what the model reads when something went
* wrong. Each internal id is kept alongside as `<field>Internal` for anything
* reasoning about layers.
* @param {object|null|undefined} state Any payload carrying context-mode fields.
* @returns {object|null|undefined} The same payload, modes translated.
*/
function withContextModeVocabulary(state) {
if (!state || typeof state !== 'object') return state;
let out = state;
const mutable = () => {
if (out === state) out = { ...state };
return out;
};
for (const { field, emptyAs } of CONTEXT_MODE_RESULT_FIELDS) {
if (!(field in state)) continue;
const internal = state[field] ?? null;
const target = mutable();
target[field] = contextModeWord(internal, { emptyAs });
target[`${field}Internal`] = internal;
}
for (const field of NESTED_CONTEXT_RESULT_FIELDS) {
const nested = state[field];
if (!nested || typeof nested !== 'object') continue;
const translated = withContextModeVocabulary(nested);
if (translated !== nested) mutable()[field] = translated;
}
return out;
}
const COCKPIT_ACTION_ALIASES = new Map([
['next', 'next'],
['previous', 'previous'],
@@ -246,14 +212,6 @@ const STACK_ALIASES = new Map([
['road map', 'osm'],
]);
/** Search order for track_entity across entity layer families. */
const TRACKABLE_FAMILIES = [
{ layerId: 'flights', kind: 'aircraft' },
{ layerId: 'military', kind: 'aircraft' },
{ layerId: 'ais-live-vessels', kind: 'vessel' },
{ layerId: 'satellites', kind: 'satellite' },
];
const FRAME_TARGETS = new Map([
['flights', 'flights'],
['planes', 'flights'],
@@ -388,6 +346,7 @@ export function createGevActionRunner({
searchNavigation = searchAndFlyTo,
speechBuilders = null,
deixis = null,
readVoicePanelOperation = null,
}) {
// Voice enable times and analyst follow-up memory belong to this runner.
// Speech builders turn results into say/display envelopes; callers extend
@@ -410,9 +369,87 @@ export function createGevActionRunner({
));
installViewTargetPrewarm(engine);
// The camera verbs (move_camera, fly_route) still run on the viewer.
if (viewer) initCameraVerbs(viewer, () => getViewTargetCartesian(engine));
if (viewer)
initCameraVerbs(viewer, () => getViewTargetCartesian(engine), {
isCockpitActive: () => Boolean(styleManager?.getCockpitState?.()?.active),
});
// Nested calls (select_nearest_aircraft composing layer + flight) use the
// raw runner; only the outer call gets a voice envelope.
const voiceUnavailable = (action) => ({
ok: false,
action,
applied: false,
unavailable: true,
error: 'Voice panel controls are unavailable in this host.',
say: 'Voice panel controls are unavailable in this host.',
});
async function showVoicePanel(
view,
open,
options,
operation = readVoicePanelOperation?.(),
) {
const current = () =>
!options.signal?.aborted &&
(!options.isCurrent || options.isCurrent()) &&
operation?.isCurrent();
if (!operation) return voiceUnavailable('set_panel_open');
if (!current())
return {
ok: false,
action: 'set_panel_open',
applied: false,
cancelled: true,
};
let exit = null;
if (open) {
if (typeof styleManager?.getCockpitState !== 'function')
return voiceUnavailable('set_panel_open');
if (styleManager.getCockpitState()?.active) {
if (typeof styleManager.controlCockpit !== 'function')
return voiceUnavailable('set_panel_open');
exit = await styleManager.controlCockpit('exit');
if (styleManager.getCockpitState()?.active)
return {
ok: false,
action: 'set_panel_open',
applied: false,
exit,
error: exit?.error || 'Cockpit did not exit.',
};
}
}
if (!current())
return {
ok: false,
action: 'set_panel_open',
applied: false,
cancelled: true,
exit,
cockpitExited: Boolean(exit && !styleManager.getCockpitState()?.active),
say:
exit && !styleManager.getCockpitState()?.active
? 'Cockpit exited; the voice panel was left as you last set it.'
: 'The voice panel was left as you last set it.',
};
const result = operation.show(view, open);
return {
...result,
action: 'set_panel_open',
panelId: view === 'settings' ? 'voice-settings' : 'voice-preview',
exit,
state: styleManager?.getCockpitState?.() ?? null,
say: !result.ok
? 'The voice panel was not changed.'
: result.settingsOpen
? 'Voice Settings is open.'
: result.panelOpen
? result.previewEnabled
? 'Voice Preview is open.'
: 'Voice control is open; Preview remains off.'
: 'The voice panel is closed.',
};
}
const runGevAction = async function runGevAction(
name,
rawArgs = {},
@@ -451,6 +488,7 @@ export function createGevActionRunner({
// Zoom during an active orbit adjusts the orbit RADIUS (spiral in/out) —
// a straight camera move would be snapped back by the per-frame lookAt.
if (name === 'adjust_camera_zoom') {
interruptGlobeRotation('voice-zoom-intent');
const zoomOut = String(args.direction || '').toLowerCase() === 'out';
const amt = String(args.amount || 'medium').toLowerCase();
const factor = { little: 1.25, medium: 1.6, lot: 2.4 }[amt] || 1.6;
@@ -799,6 +837,25 @@ export function createGevActionRunner({
return { ok: true, action: 'set_visual_style', style };
}
if (name === 'set_voice_inactivity_timeout') {
const operation = readVoicePanelOperation?.();
if (!operation) return voiceUnavailable(name);
if (!current() || !operation.isCurrent())
return { ok: false, action: name, applied: false, cancelled: true };
const result = operation.setMinutes(args.minutes);
const duration =
result.minutes === 0 ? 'disabled' : `${result.minutes} minutes`;
return {
...result,
action: name,
say: !result.applied
? 'Voice inactivity was not changed.'
: result.persisted
? `Voice inactivity is ${duration}, saved in this browser.`
: `Voice inactivity is ${duration} for now, but could not be saved in this browser.`,
};
}
if (name === 'set_panel_open') {
const panelId = normalizePanelId(args.panelId || args.panel);
if (!panelId)
@@ -806,7 +863,76 @@ export function createGevActionRunner({
`Unknown panel: ${args.panelId || args.panel || 'missing'}`,
);
const open = args.open !== false;
setPanelOpen(styleManager, panelId, open);
if (panelId === 'voice-settings' || panelId === 'voice-preview')
return showVoicePanel(
panelId === 'voice-settings' ? 'settings' : 'preview',
open,
runOptions,
);
if (panelId === 'provider-settings') {
if (!current())
return {
ok: false,
action: name,
panelId,
applied: false,
cancelled: true,
};
const doc = globalThis.document;
// The setup owner (src/keySetup.js) publishes the dialog and takes
// its open and close requests.
const settings = useProviderSettingsStore(applicationPinia());
const concealed =
open &&
[
'ui-clean-view',
'recording-mode',
'cockpit-mode',
'scene-playback-mode',
].some((mode) => doc?.body?.classList?.contains(mode));
if (concealed || settings.view?.ready !== true || !settings.controls) {
const error = concealed
? 'Provider Settings is hidden in the current view. Leave that view before opening it.'
: 'Provider Settings is unavailable in this runtime.';
return {
ok: false,
action: name,
panelId,
applied: false,
unavailable: true,
error,
say: error,
};
}
if (open) settings.controls.open();
else settings.controls.close();
// Opening unhides at once; closing drops visible before its fade ends.
const dialog = settings.view.dialog;
const applied = open ? !dialog.hidden : !dialog.visible;
const say = applied
? `Provider Settings is ${open ? 'open' : 'closed'}.`
: 'Provider Settings did not change.';
return {
ok: applied,
action: name,
panelId,
applied,
open: applied ? open : !dialog.hidden,
say,
...(!applied ? { error: say } : {}),
};
}
if (!setPanelOpen(styleManager, panelId, open)) {
const say = 'There is no panel to open here.';
return {
ok: false,
action: 'set_panel_open',
panelId,
open,
say,
error: say,
};
}
return { ok: true, action: 'set_panel_open', panelId, open };
}
@@ -894,6 +1020,25 @@ export function createGevActionRunner({
error: `Unknown cockpit action: ${args.action || args.command || 'missing'}`,
};
}
if (args.voicePanel !== undefined) {
if (
action !== 'exit' ||
!['preview', 'settings'].includes(args.voicePanel)
)
return {
ok: false,
action: 'control_cockpit',
applied: false,
error: 'Voice panel reveal requires exit and a valid view.',
};
const result = await showVoicePanel(args.voicePanel, true, runOptions);
return {
...result,
action: 'control_cockpit',
cockpitAction: 'exit',
voicePanel: args.voicePanel,
};
}
const inferred = normalizeCockpitNavigationHints(rawAction);
const targetLayer = normalizeCockpitTargetLayer(
args.targetLayer || inferred.targetLayer || args.layer || args.layerId,
@@ -1184,12 +1329,13 @@ export function createGevActionRunner({
}
if (name === 'get_current_view_state') {
return getCurrentViewState(
return getWorldPacket({
viewer,
engine,
styleManager,
dataManager,
sceneDirector,
);
});
}
if (name === 'set_hud') {
@@ -1279,6 +1425,18 @@ export function createGevActionRunner({
}
if (name === 'control_cctv') {
const action = String(args.action || '').toLowerCase();
if (
(action === 'enable' || action === 'disable') &&
dataManager.layers.get('cctv')?.module
) {
const result = await runGevAction(
'set_layer_visibility',
{ layerId: 'cctv', enabled: action === 'enable' },
runOptions,
);
return { ...result, action: 'control_cctv' };
}
return controlCctv(dataManager, args, styleManager);
}
@@ -1569,7 +1727,7 @@ function controlScene(sceneDirector, args = {}) {
};
}
if (sceneAction === 'stop') {
sceneDirector.stopScene('Stopped by voice');
sceneDirector.stopScene(msg('Stopped by voice'));
return { ok: true, action: 'control_scene', running: false };
}
if (sceneAction === 'next') {
@@ -1619,13 +1777,10 @@ export async function controlCctv(dataManager, args = {}, styleManager = null) {
}
if (action === 'enable' || action === 'disable') {
await dataManager.setEnabled('cctv', action === 'enable', {
origin: 'voice',
});
return {
ok: true,
ok: false,
action: 'control_cctv',
enabled: dataManager.isEnabled('cctv'),
error: 'Enable or disable CCTV through set_layer_visibility',
};
}
if (!dataManager.isEnabled('cctv')) {
@@ -1641,9 +1796,8 @@ export async function controlCctv(dataManager, args = {}, styleManager = null) {
return {
activeCameraId: ui.activeCameraId || null,
activeCamera: ui.activeCamera?.name || ui.activeCamera?.id || null,
cameraCount: Array.isArray(ui.cameras)
? ui.cameras.length
: ui.count || 0,
// `count`, not `cameras.length`: reading `cameras` builds every camera.
cameraCount: Number.isFinite(ui.count) ? ui.count : 0,
showCoverage: !!ui.showCoverage,
coverageMode: ui.coverageMode || (ui.showCoverage ? 'on' : 'off'),
showProjection: !!ui.showProjection,
@@ -1770,14 +1924,18 @@ export async function controlCctv(dataManager, args = {}, styleManager = null) {
}
const RADIO_COUNTRY_CENTERS = new Map([
// i18n-ignore: place name in tool data
['us', { lat: 39.8, lon: -98.6, country: 'US', label: 'United States' }],
// i18n-ignore: place name in tool data
['usa', { lat: 39.8, lon: -98.6, country: 'US', label: 'United States' }],
[
'united states',
// i18n-ignore: place name in tool data
{ lat: 39.8, lon: -98.6, country: 'US', label: 'United States' },
],
[
'united states of america',
// i18n-ignore: place name in tool data
{ lat: 39.8, lon: -98.6, country: 'US', label: 'United States' },
],
]);
@@ -2378,6 +2536,7 @@ async function trackEntity(
action: 'track_entity',
kind: 'fire',
layerId: 'local-firms',
// i18n-ignore: entity name in a tool result for the voice model
label: strongest.label || 'Strongest fire',
latitude: strongest.latitude,
longitude: strongest.longitude,
@@ -2680,24 +2839,6 @@ function runManagedVoiceNavigation(
}
/** Gathers tracked/selected entities across layer families for read-back. */
function collectTrackedEntities(dataManager) {
const tracked = [];
for (const family of TRACKABLE_FAMILIES) {
const module = dataManager.layers.get(family.layerId)?.module;
if (!module) continue;
try {
const info =
family.kind === 'vessel'
? module.getSelectedInfo?.()
: module.getTrackedInfo?.();
if (info)
tracked.push({ kind: family.kind, layerId: family.layerId, ...info });
} catch {
// layer not ready
}
}
return tracked;
}
/**
* @param {object} target The globe engine, or a Cesium viewer with a
@@ -2706,7 +2847,7 @@ function collectTrackedEntities(dataManager) {
export async function getBasemapLabelContext(
target,
service = defaultGeospatial,
{ cachedOnly = false } = {},
{ cachedOnly = false, previous = null } = {},
) {
const engine = engineFromViewer(target);
const { reverseGeocodeCache, nearbyPlacesCache } = cachesFor(service);
@@ -2716,6 +2857,8 @@ export async function getBasemapLabelContext(
placeLabels: [],
streetLabels: [],
nearbyPlaceLabels: [],
view: null,
lookup: null,
};
}
const samples = sampleViewportCartographics(engine);
@@ -2723,6 +2866,73 @@ export async function getBasemapLabelContext(
const latitude = Number(geo.math.toDegrees(viewTarget.latitude).toFixed(6));
const longitude = Number(geo.math.toDegrees(viewTarget.longitude).toFixed(6));
const view = placeLookupView({
latitude,
longitude,
rangeM: viewTargetRangeM(engine, viewTarget, cameraHeightM),
fovRad: engine.camera.pose().fovy,
aspectRatio: engine.camera.pose().aspectRatio,
tier: basemapLookupTier(cameraHeightM),
});
if (
!cachedOnly &&
previous?.lookup &&
!placeLookupViewChanged(previous.view, view)
) {
// The same points again: what the last lookup got is a cache hit, and
// only what it did not (a timeout, a failure, an empty answer) is asked.
const labels = await basemapLabelsAt(previous.lookup, service, {
cachedOnly: false,
});
return { ...labels, view: previous.view, lookup: previous.lookup };
}
const lookup = {
latitude,
longitude,
cameraHeightM,
// Cache reads are free, so a cached-only host keeps every sample.
samples: cachedOnly
? samples
: hudViewportSamples(samples, shouldReverseGeocode(cameraHeightM)),
};
return {
...(await basemapLabelsAt(lookup, service, { cachedOnly })),
view,
lookup,
};
}
/** Which of the height-gated lookups a camera height calls for. */
function basemapLookupTier(cameraHeightM) {
return [
shouldReverseGeocode(cameraHeightM),
shouldReverseGeocodeViewport(cameraHeightM) && cameraHeightM > 10000,
shouldFetchNearbyPlaces(cameraHeightM) &&
nearbyPlacesRadiusM(cameraHeightM),
].join(':');
}
/** Camera distance to the view target, or the camera height without one. */
function viewTargetRangeM(engine, target, cameraHeightM) {
const position = engine.camera.pose().position;
if (!position) return cameraHeightM;
const range = geo.cartesian3.distance(
position,
geo.cartesian3.fromRadians(
target.longitude,
target.latitude,
target.height,
),
);
return Number.isFinite(range) ? range : cameraHeightM;
}
async function basemapLabelsAt(
{ latitude, longitude, cameraHeightM, samples },
service,
{ cachedOnly },
) {
const { reverseGeocodeCache, nearbyPlacesCache } = cachesFor(service);
const cachedViewportPlaces = viewportPlacesFromCache(
samples,
cameraHeightM,
@@ -2760,7 +2970,6 @@ export async function getBasemapLabelContext(
resolveWithin(placePromise, BASEMAP_CONTEXT_WAIT_MS, null),
resolveWithin(nearbyPromise, BASEMAP_CONTEXT_WAIT_MS, []),
]);
return {
placeLabels: uniqueStrings([
place?.formattedAddress,
@@ -3082,13 +3291,14 @@ function normalizeAircraftClassFilter(value) {
return raw.replace(/[\s-]+/g, '') === TR3B_CLASS ? TR3B_CLASS : raw;
}
/**
* @returns {boolean} Whether a shell took the change: its panel chrome owns
* panel state, so without one there is no panel to open.
*/
function setPanelOpen(styleManager, panelId, open) {
if (styleManager && typeof styleManager.setPanelCollapsed === 'function') {
styleManager.setPanelCollapsed(panelId, !open, { explicit: true });
} else {
const panel = document.getElementById(panelId);
if (panel) panel.classList.toggle('collapsed', !open);
}
if (typeof styleManager?.setPanelCollapsed !== 'function') return false;
styleManager.setPanelCollapsed(panelId, !open, { explicit: true });
return true;
}
function normalizeContextMode(value) {
@@ -3135,7 +3345,11 @@ function focusDataLayerRow(layerId) {
void row.offsetWidth;
row.classList.add('gev-voice-focus');
window.setTimeout(() => row.classList.remove('gev-voice-focus'), 3000);
const name = row.querySelector('.data-name')?.textContent?.trim() || layerId;
// The English name: tool results stay English whatever the UI language.
const name =
row.dataset.layerName ||
row.querySelector('.data-name')?.textContent?.trim() ||
layerId;
return { id: layerId, name };
}
@@ -3331,66 +3545,6 @@ function normalizeLocationId(value) {
return null;
}
function getCurrentViewState(
engine,
styleManager,
dataManager,
sceneDirector = null,
) {
const cartographic = cameraCartographic(engine);
return {
ok: true,
action: 'get_current_view_state',
camera: {
latitude: geo.math.toDegrees(cartographic.latitude),
longitude: geo.math.toDegrees(cartographic.longitude),
heightM: cartographic.height,
},
style: styleManager.activeStyle || 'normal',
context:
typeof styleManager.getContextModeState === 'function'
? {
...withContextModeVocabulary(styleManager.getContextModeState()),
// The numbers on the operator's Contacts panel, so a window/count
// question can be answered from what they are looking at.
...(activeContactsWindow(dataManager)
? { contactsWindow: activeContactsWindow(dataManager) }
: {}),
}
: null,
cockpit:
typeof styleManager.getCockpitState === 'function'
? styleManager.getCockpitState()
: null,
controls:
typeof styleManager.getControlState === 'function'
? styleManager.getControlState()
: null,
scenePlayback: sceneDirector?.getPlaybackStatus?.() || null,
tracked: collectTrackedEntities(dataManager),
// Enabled layers only: the full registry cost ~1k tokens per call and
// the model only reasons about what is on.
disabledLayerCount: dataManager.getAll().filter((layer) => !layer.enabled)
.length,
layers: dataManager
.getAll()
.filter((layer) => layer.enabled)
.map((layer) => ({
id: layer.id,
name: layer.name,
enabled: layer.enabled,
count: layer.stats?.count || 0,
error: layer.stats?.error || null,
feedState: layerSnapshot(layer).feedState,
source: layerSnapshot(layer).source,
lastUpdate: layerSnapshot(layer).lastUpdate,
})),
feedProvenance: feedProvenanceEnvelope(
layerSnapshots(dataManager.getAll()).filter((layer) => layer.enabled),
),
};
}
async function getEntityContext(
engine,
dataManager,
@@ -3868,8 +4022,10 @@ function centerMatchesSubject(center, subjectPosition) {
const subjectLat = geo.math.toDegrees(carto.latitude);
const subjectLon = geo.math.toDegrees(carto.longitude);
return (
haversineKm(subjectLat, subjectLon, center.lat, center.lon) <=
SUBJECT_CENTER_TOLERANCE_KM
greatCircleKm(
{ lat: subjectLat, lon: subjectLon },
{ lat: center.lat, lon: center.lon },
) <= SUBJECT_CENTER_TOLERANCE_KM
);
}
@@ -4147,7 +4303,7 @@ function nearbyKnownLandmarks(latitude, longitude, cameraHeightM) {
for (const [cityId, city] of Object.entries(CITY_POIS)) {
for (let poiIndex = 0; poiIndex < city.pois.length; poiIndex++) {
const poi = city.pois[poiIndex];
const distanceKm = haversineKm(latitude, longitude, poi.lat, poi.lon);
const distanceKm = greatCircleKm({ lat: latitude, lon: longitude }, poi);
if (distanceKm > maxDistanceKm) continue;
matches.push({
name: poi.name,
@@ -4672,23 +4828,6 @@ function clampNumber(value, min, max, fallback) {
* ("which of those is closest?") scoped to the session.
*/
/**
* The Contacts panel's own counts, or null when Contacts has no subject.
* Read through the awareness snapshot the panel renders, so the two cannot
* diverge no matter which surface asks.
* @returns {object|null} Panel-equivalent window counts.
*/
function activeContactsWindow(dataManager) {
try {
const layer = dataManager?.layers?.get('military-awareness')?.module;
return (
layer?.contactsWindowFromSnapshot?.(layer.getContextSnapshot?.()) ?? null
);
} catch {
return null;
}
}
/**
* Most records one layer hands a single query. Above every loaded feed seen
* so far (flights ~8k, vessels ~12k, a 24 h global FIRMS day ~160k), so
@@ -4845,7 +4984,12 @@ function compactAnalystItem(r) {
if (!present(r[key]) || key in compact) continue;
if (spec.type === 'time') {
const ms = Number(r[key]);
if (Number.isFinite(ms)) compact.time = new Date(ms).toISOString();
// A finite value can still be past Date's range, where toISOString throws.
const time = new Date(ms);
if (Number.isFinite(time.getTime())) {
if (key === 'lastContactAtMs') compact.lastContactAtMs = ms;
else compact.time = time.toISOString();
}
} else compact[key] = r[key];
}
if (present(r.ageHours)) compact.ageHours = r.ageHours;
+73 -4
View File
@@ -1089,7 +1089,9 @@ test('generic layer visibility exposes lifecycle truth for every manager phase a
});
// The voice envelope is additive and states the failure plainly.
assert.equal(say, "Couldn't turn on radio.");
assert.equal(display.title, 'radio');
// The card names the layer, not its id; the result keeps the id.
assert.equal(display.title, 'Radio');
assert.deepEqual(display.lines, ['Radio layer unavailable']);
});
test('generic voice visibility maps Space Missions to the explicit mission layer', async () => {
@@ -3187,7 +3189,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl
const requests = [];
installKeyedFetch(t, async (url) => {
requests.push(String(url));
if (String(url).startsWith('https://maps.googleapis.com/')) {
if (String(url).startsWith('/api/google/geocode')) {
return { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) };
}
assert.match(String(url), /^https:\/\/photon\.komoot\.io\/api\/\?/);
@@ -3212,7 +3214,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl
assert.ok(Math.abs(calls[0].criteria.anchor.lat - 20.9101) < 1e-9);
assert.ok(Math.abs(calls[0].criteria.anchor.lon - 107.1839) < 1e-9);
// Google is asked first and exactly once; Photon answers unbiased, in one call.
assert.equal(requests.filter((url) => url.includes('maps.googleapis.com')).length, 1);
assert.equal(requests.filter((url) => url.includes('/api/google/geocode')).length, 1);
assert.equal(requests.filter((url) => url.includes('photon.komoot.io')).length, 1);
assert.match(requests.at(-1), /[?&]q=H%E1%BA%A1\+Long\+Bay/);
assert.doesNotMatch(requests.at(-1), /[?&](lat|lon|bbox)=/, 'a named radio location is not viewport-biased');
@@ -3225,7 +3227,7 @@ test('voice Radio: the keyless path applies no country filter the keyed path wou
// "No Radio station matched" for exactly the places it just resolved, while a
// keyed install placed a station. The label may carry it; the filter may not.
const { calls, dataManager } = radioSelectionHarness();
installKeyedFetch(t, async (url) => (String(url).startsWith('https://maps.googleapis.com/')
installKeyedFetch(t, async (url) => (String(url).startsWith('/api/google/geocode')
? { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) }
: {
ok: true,
@@ -3290,6 +3292,30 @@ test('ALPR common names toggle only the registered camera layer through the norm
}
});
test('GNSS integrity and interference common names toggle only the GNSS layer through the normal voice action', async () => {
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
const viewer = { clock: { onTick: { addEventListener: () => () => {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
camera: { moveEnd: { addEventListener() {} } } };
const calls = [];
let enabled = false;
const dataManager = {
layers: new Map([['gnss-integrity', { module: {} }]]),
getAll: () => [{ id: 'gnss-integrity', name: 'GNSS Integrity' }],
isEnabled: () => enabled,
setEnabled: async (id, value) => { calls.push([id, value]); enabled = value; return true; },
};
const runner = createGevActionRunner({ viewer, styleManager: {}, dataManager });
for (const alias of ['gnss-integrity', 'gnss integrity', 'navigation integrity', 'low navigation accuracy', 'gnss interference', 'gps interference', 'gps jamming', 'gnss']) {
for (const value of [true, false]) {
const result = await runner('set_layer_visibility', { layerId: alias, enabled: value });
assert.equal(result.ok, true);
assert.equal(result.layerId, 'gnss-integrity');
assert.deepEqual(calls.at(-1), ['gnss-integrity', value]);
}
}
});
test('Local ADS-B common names toggle only the receiver layer through the normal voice action', async () => {
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
const viewer = { clock: { onTick: { addEventListener: () => () => {} } },
@@ -3486,6 +3512,26 @@ test('analyst_query refusals carry their code and the allowed values', async ()
assert.match(ok.items[0].time, /^\d{4}-\d\d-\d\dT/);
});
test('analyst_query keeps its answer when one record has an out-of-range time', async () => {
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
const now = Date.now();
const earthquakes = {
getStats: () => ({ count: 2, lastUpdate: now }),
getAnalystRecords: () => [
{ id: 'q', lat: 0, lon: 0, magnitude: 5, timeMs: now },
{ id: 'r', lat: 1, lon: 1, magnitude: 4, timeMs: 8.64e15 + 1 },
],
};
const { runner } = manifestRunner({ earthquakes }, { enabled: ['earthquakes'] });
const result = await runner('analyst_query', { layers: ['earthquakes'], scope: { kind: 'anywhere' } });
assert.equal(result.ok, true);
const q = result.items.find((item) => item.id === 'q');
assert.match(q.time, /^\d{4}-\d\d-\d\dT/);
const r = result.items.find((item) => item.id === 'r');
assert.ok(r, 'the record is still listed');
assert.equal(r.time, undefined, 'an unrepresentable time is omitted');
});
test('spoken layer names from the manifest reach the newer layers', async () => {
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
const stub = () => ({ getStats: () => ({ count: 0 }) });
@@ -3912,3 +3958,26 @@ test('analyst-only lookup distinguishes duplicate names and uses canonical aircr
assert.equal(plane.ok, true);
assert.equal(plane.latitude, 36, 'canonical ID resolves the current position, not old coordinates');
});
test('set_panel_open accepts the Authoring name for the scene panel', async () => {
const calls = [];
const runner = createGevActionRunner({
viewer: {
clock: { onTick: { addEventListener: () => () => {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
camera: {
moveEnd: { addEventListener() {} },
positionWC: Cesium.Cartesian3.fromDegrees(-97.7, 30.2, 1000),
},
},
styleManager: {
setPanelCollapsed: (...args) => calls.push(args),
},
dataManager: { layers: new Map(), isEnabled: () => false, getAll: () => [] },
});
for (const panel of ['authoring', 'Authoring', 'scenes']) {
const result = await runner('set_panel_open', { panelId: panel, open: true });
assert.equal(result.panelId, 'scene-panel');
}
assert.deepEqual(calls.at(-1), ['scene-panel', false, { explicit: true }]);
});
+10 -7
View File
@@ -56,21 +56,24 @@ export function initGevVoiceCommands(options) {
const referents = createReferentRegistry();
// The controller exists only after the runner, so its hooks bind late.
const deixis = { pointer, referents, cameraKey };
const commands = createVoiceCommands({
let commands;
commands = createVoiceCommands({
...options,
pointer,
referents,
runner: withToolCatalog(
createGevActionRunner({ ...options, engine, deixis }),
createGevActionRunner({
...options,
deixis,
readVoicePanelOperation: () => commands?.captureVoicePanelOperation?.(),
}),
options.toolCatalog,
),
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();
commands.session?.signal?.addEventListener(
'abort',
() => pointer?.destroy(),
{ once: true },
);
const lifetime = commands.signal ?? commands.session?.signal;
lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true });
if (lifetime?.aborted) pointer?.destroy();
return commands;
}
+47 -212
View File
@@ -34,6 +34,7 @@ import {
withToolCatalog,
} from './gevRealtime.js';
import { createVoiceCostTracker } from './voiceCost.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
test('push-to-talk recognizes Space by code or key', () => {
assert.equal(PUSH_TO_TALK_HOLD_DELAY_MS, 500);
@@ -114,10 +115,7 @@ function createPushToTalkFixture(t) {
const makeController = () => {
const microphone = { enabled: false, stopped: false, stop() { this.stopped = true; } };
const controller = new GevRealtimeController({
ui: {
root: { dataset: {}, querySelectorAll: () => [], remove() {} },
status: {}, detail: {},
},
ui: createTestVoiceUi(),
runner: async () => ({}),
radioLayer: {
pause: () => radio.push('pause'),
@@ -318,8 +316,8 @@ test('push-to-talk starts once at 500ms and repeat does not reset the threshold'
assert.equal(f.controller.spaceKeyHeld, false);
assert.equal(f.controller.pushToTalkKeyHeld, false);
assert.equal(f.microphone.enabled, false);
assert.equal(f.controller.ui.root.dataset.pushToTalk, undefined);
assert.equal(f.controller.ui.detail.textContent, 'Hold Space to talk');
assert.equal(f.controller.ui.state.root.pushToTalk, null);
assert.equal(f.controller.ui.state.detail.text, 'Hold Space to talk');
});
test('push-to-talk treats a Space release before 500ms as a background tap', (t) => {
@@ -392,7 +390,7 @@ test('push-to-talk cancels pending and active holds on blur or hidden document',
assert.equal(f.controller.spaceKeyHeld, false);
assert.equal(f.controller.pushToTalkKeyHeld, false);
assert.equal(f.microphone.enabled, false);
assert.equal(f.controller.ui.root.dataset.pushToTalk, undefined);
assert.equal(f.controller.ui.state.root.pushToTalk, null);
assert.equal(f.key('keyup', background).defaultPrevented, false);
f.document.visibilityState = 'visible';
}
@@ -579,7 +577,7 @@ test('failed or superseded Radio preflight keeps voice open and cancels audio',
test('replacement input invalidates an in-flight Radio preflight immediately', () => {
const order = [];
const controller = new GevRealtimeController({
ui: {},
ui: createTestVoiceUi(),
runner: async () => ({}),
radioLayer: { stopPlayback: () => order.push('radio-stop') },
});
@@ -596,12 +594,7 @@ test('replacement input invalidates an in-flight Radio preflight immediately', (
test('direct Radio pause or stop cancels a pending voice handoff', async () => {
const order = [];
let playbackControl = null;
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async () => ({ ok: true }),
@@ -637,12 +630,7 @@ test('direct Radio pause or stop cancels a pending voice handoff', async () => {
test('confirmed manual Radio playback closes active voice without stopping Radio', () => {
const order = [];
let playbackControl = null;
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async () => ({ ok: true }),
@@ -675,12 +663,7 @@ test('manual playback takeover survives stale voice preflight cleanup', async ()
let resolvePreflight;
let activeAttemptId = null;
const stopRequests = [];
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const radioLayer = {
subscribePlaybackControls(listener) {
playbackControl = listener;
@@ -732,7 +715,7 @@ test('manual playback takeover survives stale voice preflight cleanup', async ()
test('internal Radio cleanup controls do not masquerade as newer user input', () => {
let playbackControl = null;
const controller = new GevRealtimeController({
ui: {},
ui: createTestVoiceUi(),
runner: async () => ({}),
radioLayer: {
subscribePlaybackControls(listener) {
@@ -757,7 +740,7 @@ test('manual Radio playback does not close voice when voice is already idle', ()
let playbackControl = null;
let voiceStops = 0;
const controller = new GevRealtimeController({
ui: {},
ui: createTestVoiceUi(),
runner: async () => ({}),
radioLayer: {
subscribePlaybackControls(listener) {
@@ -777,12 +760,7 @@ test('manual Radio playback does not close voice when voice is already idle', ()
test('active-response Realtime errors invalidate pending and in-flight Radio handoffs', async () => {
const order = [];
let playbackControl = null;
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async () => ({ ok: true }),
@@ -830,16 +808,7 @@ test('voice ownership ducks tuner static before pausing broadcaster audio', () =
test('Radio handoff waits for response.done so later multi-intent tools execute before voice closes', async () => {
const order = [];
const confirmationInstructions = [];
const ui = {
root: {
dataset: {},
classList: { remove() {} },
querySelectorAll: () => [],
},
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const runner = async (name) => {
order.push(`tool:${name}`);
if (name === 'control_radio') {
@@ -924,12 +893,7 @@ test('Radio handoff waits for response.done so later multi-intent tools execute
test('Radio playback failure leaves voice connected and speaks a correction', async () => {
const sent = [];
const order = [];
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async () => ({
@@ -982,16 +946,7 @@ test('Radio playback failure leaves voice connected and speaks a correction', as
test('detected speech cancels a prepared Radio handoff before a cancelled response completes', async () => {
const order = [];
const ui = {
root: {
dataset: {},
classList: { remove() {} },
querySelectorAll: () => [],
},
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async () => ({
@@ -1050,12 +1005,7 @@ test('a Radio tool result that resolves after speech interruption cannot re-arm
let resolveRunner;
const runnerResult = new Promise((resolve) => { resolveRunner = resolve; });
let receivedSignal = null;
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async (_name, _args, options) => {
@@ -1110,12 +1060,7 @@ test('sibling tool calls in one response do not abort an in-flight Radio action'
let resolveRadio;
let radioSignal = null;
const radioResult = new Promise((resolve) => { resolveRadio = resolve; });
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async (name, _args, options) => {
@@ -1156,12 +1101,7 @@ test('Radio stop does not abort an unrelated sibling tool from the same response
let releaseStyle;
let styleSignal = null;
const styleResult = new Promise((resolve) => { releaseStyle = resolve; });
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async (name, _args, options) => {
@@ -1235,12 +1175,7 @@ test('real Radio Select cannot enable or play after a same-response Disable comp
return true;
},
};
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
globalThis.window = { __GOOGLE_MAPS_API_KEY__: 'test-key' };
globalThis.fetch = async () => {
announceLookupStarted();
@@ -1366,12 +1301,7 @@ test('generic same-response Radio visibility disable supersedes delayed Select',
camera: { moveEnd: { addEventListener() {} } },
};
const genericRunner = createGevActionRunner({ viewer, styleManager: {}, dataManager });
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
globalThis.window = { __GOOGLE_MAPS_API_KEY__: 'test-key' };
globalThis.fetch = async () => {
announceLookupStarted();
@@ -1501,12 +1431,7 @@ test('same-response Pause aborts a real manager enable already in flight', async
dataManager.subscribe((change) => {
if (change.type === 'visibility') trace.push(`visibility:${change.enabled}`);
});
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
dataManager,
@@ -1632,12 +1557,7 @@ test('Pause and Stop preserve independent dedicated and generic Radio ON across
styleManager: {},
dataManager,
});
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
dataManager,
@@ -1766,12 +1686,7 @@ test('successful same- or newer-response Stop aborts Select across real manager
if (change.origin === 'voice') persistenceWrites.push(change.enabled);
}
});
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
radioLayer,
@@ -1837,12 +1752,7 @@ test('same-response pause and disable suppress play/select handoffs without canc
const playbackGate = new Promise((resolve) => { releasePlayback = resolve; });
const sent = [];
const radioState = { enabled: true, audioState: 'playing' };
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
radioLayer: { getUIState: () => ({ ...radioState }) },
@@ -1918,12 +1828,7 @@ test('Realtime Radio route exceptions preserve the authoritative lifecycle summa
for (const route of ['dedicated', 'generic']) {
const sent = [];
const lifecycle = { enabled: true, lifecycleState: 'disabling', uncertain: true };
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
dataManager: {
@@ -1970,12 +1875,7 @@ test('different-response Radio stop remains a cancellation authority for an olde
let releaseSelect;
const selectGate = new Promise((resolve) => { releaseSelect = resolve; });
const sent = [];
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async (_name, args) => {
@@ -2048,12 +1948,7 @@ test('failed same-response stop does not suppress a successful playback request'
}]]),
isEnabled: () => true,
};
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async (_name, args) => {
@@ -2156,12 +2051,7 @@ test('failed same-response Stop preserves Select auto-enable held inside real ma
};
dataManager = new DataLayerManager({});
dataManager.register(radioLayer);
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
dataManager,
@@ -2244,12 +2134,7 @@ test('failed same-response Pause and Disable do not suppress valid older playbac
return () => visibilityListeners.delete(listener);
},
};
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
dataManager,
@@ -2307,12 +2192,7 @@ test('failed generic same-response Radio OFF does not suppress valid older playb
let releaseSelect;
const selectGate = new Promise((resolve) => { releaseSelect = resolve; });
const sent = [];
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async (name, args) => {
@@ -2401,12 +2281,7 @@ test('direct user Radio OFF aborts an in-flight voice enable before settled publ
dataManager.subscribe((change) => {
if (change.type === 'visibility') visibility.push(`${change.origin}:${change.enabled}`);
});
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
dataManager,
@@ -2492,12 +2367,7 @@ test('direct user Radio OFF freezes a prepared handoff until disable success or
dataManager = new DataLayerManager({});
dataManager.register(radioLayer);
await dataManager.setEnabled('radio', true);
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
dataManager,
@@ -2571,12 +2441,7 @@ test('dedicated and generic Radio OFF reservations freeze an in-flight playback
},
getUIState: () => ({ enabled: true, audioState: 'stopped' }),
};
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
radioLayer,
@@ -2673,12 +2538,7 @@ test('delayed Stop reports its result before committing or resuming a prepared h
},
getUIState: () => ({ enabled: true, audioState: 'stopped' }),
};
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
radioLayer,
@@ -2756,12 +2616,7 @@ test('stale handoff cleanup cannot erase a resumed successor across repeated fai
return true;
},
};
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
radioLayer,
@@ -2808,12 +2663,7 @@ test('stale handoff cleanup cannot erase a resumed successor across repeated fai
});
test('a later same-response stop clears an already prepared playback result', async () => {
const ui = {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: async (_name, args) => args.action === 'select'
@@ -3107,22 +2957,7 @@ const usdUsage = (usd) => ({
/** A controller wired to inert UI stubs, with the cost surface present. */
function costControllerHarness({ runner } = {}) {
const toolCalls = [];
const ui = {
root: { dataset: {}, classList: { remove() {}, add() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
button: { dataset: {} },
buttonLabel: { textContent: '' },
tierButton: {
textContent: '',
title: '',
attrs: {},
setAttribute(key, value) { this.attrs[key] = value; },
getAttribute(key) { return this.attrs[key] ?? null; },
},
costValue: { textContent: '', title: '', dataset: {} },
};
const ui = createTestVoiceUi();
const controller = new GevRealtimeController({
ui,
runner: runner || (async (name) => { toolCalls.push(name); return { ok: true }; }),
@@ -3190,9 +3025,9 @@ test('F1: the toggle still records the next-session preference while live', () =
controller.status = 'listening';
controller.setVoiceTier('mini');
assert.equal(controller.voiceTier, 'mini');
assert.equal(ui.tierButton.textContent, 'MINI');
assert.equal(ui.state.tier.text, 'MINI');
// ...and says so, rather than implying the live session switched.
assert.match(ui.tierButton.title, /this session stays on/i);
assert.match(ui.state.tier.title, /this session stays on/i);
});
test('OAuth cloud sessions report unknown cost and usage without tripping the API spend guard', () => {
@@ -3202,8 +3037,8 @@ test('OAuth cloud sessions report unknown cost and usage without tripping the AP
controller._cost.sessionCloudVoiceAuth = 'oauth';
controller.syncCostUi();
const before = controller.costTracker.state();
assert.equal(ui.costValue.hidden, false);
assert.equal(ui.costValue.textContent, 'COST UNKNOWN');
assert.equal(ui.state.cost.hidden, false);
assert.equal(ui.state.cost.text, 'COST UNKNOWN');
assert.deepEqual(controller.recordUsage(usdUsage(100)), {
responses: 1,
input: 0,
@@ -3214,11 +3049,11 @@ test('OAuth cloud sessions report unknown cost and usage without tripping the AP
const after = controller.costTracker.state();
assert.equal(after.totalUsd, before.totalUsd);
assert.equal(controller.costCapStopped, false);
assert.match(ui.costValue.title, /1 response\(s\).*1562500 output tokens/);
assert.match(ui.state.cost.title, /1 response\(s\).*1562500 output tokens/);
controller.dc = {};
controller.responseActive = true;
controller.stop();
assert.match(ui.costValue.title, /Usage is incomplete/);
assert.match(ui.state.cost.title, /Usage is incomplete/);
});
test('OAuth cloud sessions count caption transcriptions without pricing them into the API cap', () => {
@@ -3239,8 +3074,8 @@ test('OAuth cloud sessions count caption transcriptions without pricing them int
assert.equal(after.totalUsd, before.totalUsd);
assert.equal(after.capReached, false);
assert.equal(controller.costCapStopped, false);
assert.equal(ui.costValue.textContent, 'COST UNKNOWN');
assert.match(ui.costValue.title, /1 caption transcription\(s\)/);
assert.equal(ui.state.cost.text, 'COST UNKNOWN');
assert.match(ui.state.cost.title, /1 caption transcription\(s\)/);
});
test('F1: when idle, toggling does re-price the preview meter', () => {
@@ -3444,7 +3279,7 @@ test('F4: two clicks during a live session return to the original preference', (
controller.toggleVoiceTier();
assert.equal(controller.voiceTier, 'standard', 'second click switches back');
assert.equal(ui.tierButton.textContent, 'STD');
assert.equal(ui.state.tier.text, 'STD');
});
test('F4: the toggle alternates across many clicks mid-session', () => {
@@ -3481,7 +3316,7 @@ test('F3: an unrecognised session model bills at the most expensive known rates'
function textCommandController() {
const sent = [];
const controller = new GevRealtimeController({
ui: {},
ui: createTestVoiceUi(),
runner: async () => ({}),
});
controller.dc = { readyState: 'open' };
File diff suppressed because it is too large. Load diff
+365
View File
@@ -0,0 +1,365 @@
import { createCesiumEngine } from '../globe/cesium/createCesiumEngine.js';
// The HUD's passive place lookup spends paid Google requests on its own, so
// it must look up the view center and one more point, and only again once the
// view shows a different area.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import * as Cesium from 'cesium';
import { getBasemapLabelContext } from './gevActions.js';
import {
hudViewportSamples,
placeLookupView,
placeLookupViewChanged,
} from '../hudPlaceLookup.js';
globalThis.window = globalThis.window || { setTimeout, clearTimeout };
/**
* A looking-down camera `heightM` above the ellipsoid. Ellipsoid picks map
* screen pixels linearly to degrees; the view target sits on terrain
* `groundM` high under the camera.
*/
function createViewer({ latitude, longitude, heightM, groundM = 0 }) {
const state = { latitude, longitude, heightM, groundM };
const canvas = { clientWidth: 1200, clientHeight: 800 };
const degreesPerPixel = () => state.heightM / 111_000 / canvas.clientHeight;
const camera = {
heading: 0,
pitch: -Math.PI / 2,
roll: 0,
frustum: { fovy: Math.PI / 3, aspectRatio: 1.5 },
pickEllipsoid(point) {
return Cesium.Cartesian3.fromDegrees(
state.longitude +
(point.x - canvas.clientWidth / 2) * degreesPerPixel(),
state.latitude -
(point.y - canvas.clientHeight / 2) * degreesPerPixel(),
);
},
};
Object.defineProperties(camera, {
positionCartographic: {
get: () =>
Cesium.Cartographic.fromDegrees(
state.longitude,
state.latitude,
state.heightM,
),
},
positionWC: {
get: () =>
Cesium.Cartesian3.fromDegrees(
state.longitude,
state.latitude,
state.heightM,
),
},
});
const scene = {
canvas,
pickPositionSupported: true,
pickPosition: () =>
Cesium.Cartesian3.fromDegrees(
state.longitude,
state.latitude,
state.groundM,
),
};
return {
viewer: createCesiumEngine({ camera, scene }),
state,
degreesPerPixel,
};
}
/** Answers every lookup with a label naming the point it was asked about. */
function createService() {
const calls = { reverse: 0, nearby: 0 };
return {
calls,
async reverseGeocode(latitude, longitude) {
calls.reverse += 1;
const point = `${latitude.toFixed(4)},${longitude.toFixed(4)}`;
return { formattedAddress: `Place ${point}`, labels: [`Area ${point}`] };
},
async nearby() {
calls.nearby += 1;
return [{ name: `Nearby ${calls.nearby}` }];
},
};
}
test('a view counts as new after a quarter-width move or a doubled distance', () => {
const view = placeLookupView({ latitude: 30, longitude: -97, rangeM: 1000 });
// A 60° field of view at 1 km shows about 1155 m across.
assert.ok(Math.abs(view.spanM - 1154.7) < 1);
const near = placeLookupView({
latitude: 30.002,
longitude: -97,
rangeM: 1100,
});
assert.equal(
placeLookupViewChanged(view, near),
false,
'~220 m is under a quarter width',
);
const far = placeLookupView({
latitude: 30.003,
longitude: -97,
rangeM: 1000,
});
assert.equal(
placeLookupViewChanged(view, far),
true,
'~330 m is over a quarter width',
);
const closer = placeLookupView({ latitude: 30, longitude: -97, rangeM: 500 });
assert.equal(placeLookupViewChanged(view, closer), true);
const farther = placeLookupView({
latitude: 30,
longitude: -97,
rangeM: 2000,
});
assert.equal(placeLookupViewChanged(view, farther), true);
assert.equal(placeLookupViewChanged(null, view), true);
// The antimeridian is a short hop, not a world away.
const west = placeLookupView({
latitude: 0,
longitude: 179.9999,
rangeM: 1000,
});
const east = placeLookupView({
latitude: 0,
longitude: -179.9999,
rangeM: 1000,
});
assert.equal(placeLookupViewChanged(west, east), false);
});
test('a portrait screen measures its narrower width', () => {
const landscape = placeLookupView({
latitude: 0,
longitude: 0,
rangeM: 1000,
aspectRatio: 1.5,
});
const portrait = placeLookupView({
latitude: 0,
longitude: 0,
rangeM: 1000,
aspectRatio: 0.5,
});
assert.ok(Math.abs(landscape.spanM - 1154.7) < 1);
assert.ok(Math.abs(portrait.spanM - 577.4) < 1);
});
test('a view in another height tier is new however close it is', () => {
const a = placeLookupView({
latitude: 0,
longitude: 0,
rangeM: 1000,
tier: 'x',
});
const b = placeLookupView({
latitude: 0,
longitude: 0,
rangeM: 1000,
tier: 'y',
});
assert.equal(placeLookupViewChanged(a, b), true);
});
test('the HUD geocodes one viewport point besides the center', () => {
const samples = [{ i: 0 }, { i: 1 }, { i: 2 }];
assert.deepEqual(hudViewportSamples(samples, true), [{ i: 1 }]);
assert.deepEqual(hudViewportSamples(samples, false), [{ i: 0 }, { i: 1 }]);
assert.deepEqual(hudViewportSamples([], true), []);
});
test('a regional view spends two reverse geocodes and a nearby search', async () => {
const { viewer } = createViewer({
latitude: 30.26,
longitude: -97.74,
heightM: 20_000,
});
const service = createService();
const labels = await getBasemapLabelContext(viewer, service);
// Before: the center plus three viewport samples (the first of them the
// center again, which shares its cache key in this stub).
assert.deepEqual(service.calls, { reverse: 2, nearby: 1 });
assert.ok(labels.placeLabels.length > 0);
});
test('a small pan reuses the last lookup; a real move looks up again', async () => {
const { viewer, state } = createViewer({
latitude: 30.26,
longitude: -97.74,
heightM: 600,
});
const service = createService();
const first = await getBasemapLabelContext(viewer, service);
assert.deepEqual(service.calls, { reverse: 1, nearby: 1 });
state.latitude += 0.0005; // ~55 m at 600 m height
const panned = await getBasemapLabelContext(viewer, service, {
previous: first,
});
assert.deepEqual(
panned.placeLabels,
first.placeLabels,
'the same area keeps its labels',
);
assert.deepEqual(service.calls, { reverse: 1, nearby: 1 });
state.latitude += 0.002; // ~280 m from the first lookup
const moved = await getBasemapLabelContext(viewer, service, {
previous: panned,
});
assert.notDeepEqual(moved.placeLabels, first.placeLabels);
assert.deepEqual(service.calls, { reverse: 2, nearby: 2 });
});
test('over high terrain the visible width is measured from the ground', async () => {
// 100 m above a 3000 m plateau shows ~115 m across, not the ~3.6 km a
// 3100 m ellipsoid height would suggest.
const { viewer, state } = createViewer({
latitude: 30.26,
longitude: -97.74,
heightM: 3100,
groundM: 3000,
});
const service = createService();
const first = await getBasemapLabelContext(viewer, service);
state.latitude += 0.0045; // ~500 m
await getBasemapLabelContext(viewer, service, { previous: first });
assert.equal(service.calls.reverse, 2);
});
test('crossing into the Nearby Places height looks up again', async () => {
const { viewer, state } = createViewer({
latitude: 30.26,
longitude: -97.74,
heightM: 30_000,
});
const service = createService();
const high = await getBasemapLabelContext(viewer, service);
assert.equal(service.calls.nearby, 0);
state.heightM = 20_000; // a 1.5× zoom, under the distance rule
const low = await getBasemapLabelContext(viewer, service, { previous: high });
assert.equal(service.calls.nearby, 1);
assert.ok(low.nearbyPlaceLabels.length > 0);
});
test('a lookup that timed out is completed from the cache, not frozen', async () => {
const { viewer, state } = createViewer({
latitude: 30.26,
longitude: -97.74,
heightM: 600,
});
const service = createService();
let answer;
service.reverseGeocode = () => {
service.calls.reverse += 1;
return new Promise((resolve) => {
answer = resolve;
});
};
// Expire the 1.5 s wait at once.
const { setTimeout: wait } = globalThis.window;
globalThis.window.setTimeout = (fn) => wait(fn, 0);
let first;
try {
first = await getBasemapLabelContext(viewer, service);
} finally {
globalThis.window.setTimeout = wait;
}
assert.deepEqual(first.placeLabels, []);
assert.ok(
first.nearbyPlaceLabels.length > 0,
'the nearby answer arrived in time',
);
answer({ formattedAddress: 'Late place', labels: [] });
await new Promise((resolve) => setImmediate(resolve));
state.latitude += 0.0005;
const next = await getBasemapLabelContext(viewer, service, {
previous: first,
});
assert.deepEqual(next.placeLabels, ['Late place']);
assert.deepEqual(
service.calls,
{ reverse: 1, nearby: 1 },
'the late answer is a cache hit',
);
});
test('an empty answer is asked again rather than held', async () => {
const { viewer, state } = createViewer({
latitude: 30.26,
longitude: -97.74,
heightM: 600,
});
const service = createService();
service.reverseGeocode = async () => {
service.calls.reverse += 1;
return null;
};
const empty = await getBasemapLabelContext(viewer, service);
state.latitude += 0.0005;
await getBasemapLabelContext(viewer, service, { previous: empty });
assert.deepEqual(
service.calls,
{ reverse: 2, nearby: 1 },
'the cached nearby answer is not asked again',
);
});
test('a failed lookup is asked again once the provider recovers', async () => {
const { viewer, state } = createViewer({
latitude: 30.26,
longitude: -97.74,
heightM: 600,
});
const service = createService();
const working = service.reverseGeocode;
service.reverseGeocode = async () => {
service.calls.reverse += 1;
throw new Error('503');
};
const failed = await getBasemapLabelContext(viewer, service);
assert.deepEqual(failed.placeLabels, []);
service.reverseGeocode = working;
state.latitude += 0.0005;
const recovered = await getBasemapLabelContext(viewer, service, {
previous: failed,
});
assert.ok(recovered.placeLabels.length > 0);
assert.deepEqual(
service.calls,
{ reverse: 2, nearby: 1 },
'only the failed geocode is asked again',
);
});
test('cached-only lookups read every cached viewport sample and never request', async () => {
const { viewer, state, degreesPerPixel } = createViewer({
latitude: 30.26,
longitude: -97.74,
heightM: 20_000,
});
const service = createService();
// Geocode view A's upper-left sample (screen x = 300).
const a = await getBasemapLabelContext(viewer, service);
const cachedLabel = a.placeLabels.find(
(label) => label.startsWith('Area') && label !== 'Area 30.2600,-97.7400',
);
assert.ok(cachedLabel, 'the upper-left sample was geocoded');
// View B puts that same point at its upper-right sample (screen x = 900),
// which only a read of every sample finds.
state.longitude -= 600 * degreesPerPixel();
const before = { ...service.calls };
const b = await getBasemapLabelContext(viewer, service, { cachedOnly: true });
assert.ok(b.placeLabels.includes(cachedLabel));
assert.deepEqual(service.calls, before);
});
+4 -9
View File
@@ -5,6 +5,7 @@
// Run with: npm test
import test from 'node:test';
import assert from 'node:assert/strict';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import {
initialVoiceCardState,
reduceVoiceCard,
@@ -124,7 +125,8 @@ test('a partial answer names the layers it could not read', async () => {
assert.match(result.say, /Partial; earthquakes not answered/);
assert.deepEqual(result.unanswered, ['earthquakes']);
assert.ok(view.chips.includes('partial'));
assert.ok(view.notes.includes('Not answered: earthquakes'));
// The card names the layer; the result keeps its key for the model.
assert.ok(view.notes.includes('Not answered: Earthquakes (24h)'));
});
test('"the second one" is the second row the card lists', async () => {
@@ -205,14 +207,7 @@ test('the voice controls reset the runner on stop and dispose it on removal', as
globalThis.window = {};
try {
const h = harness();
const button = new EventTarget();
button.setAttribute = () => {};
const ui = {
button,
root: { dataset: {}, remove() {} },
status: {},
detail: {},
};
const ui = createTestVoiceUi();
const lifetime = new AbortController();
let emitState;
const controls = createVoiceCommands({
+55 -1
View File
@@ -34,9 +34,43 @@ const AIRCRAFT_FIELDS = {
aircraftClass: ['t'],
military: ['f'],
onGround: ['f'],
squawk: ['t'],
emergency: ['f'],
emergencyKind: ['t'],
emergencySource: ['t'],
emergencyBroadcast: ['t'],
lastContactAtMs: ['ms'],
};
const layers = [
{
id: 'gnss-integrity',
aliases: [
'gnss integrity',
'navigation integrity',
'low navigation accuracy',
'gnss interference',
'gps interference',
'gps jamming',
'gnss',
],
context: false,
noQuery: 'navigation-integrity cells; no countable analyst records',
},
{
id: 's2-ghostmaps',
aliases: [
's2 ghostmaps',
'ghost maps',
'ghostmaps',
's2 underground',
's2 intel',
'intel picture',
'common intelligence picture',
],
context: false,
noQuery: 'published third-party intel placemarks; not analyst records',
},
{
id: 'flights',
track: true,
@@ -230,6 +264,24 @@ const layers = [
context: false,
noQuery: 'map overlay; no countable records',
},
{
id: 'air-quality',
aliases: ['air quality', 'aqhi', 'smog', 'air pollution'],
context: false,
query: {
fields: {
value: ['n', 'AQHI'],
band: ['t'],
scale: ['t'],
provider: ['t'],
name: ['t'],
zone: ['t'],
risk: ['t'],
observedMs: ['ms'],
},
timeField: 'observedMs',
},
},
{
id: 'traffic',
aliases: ['street traffic', 'road traffic', 'congestion'],
@@ -473,6 +525,8 @@ const SELF_EVIDENT_IDS = new Set([
'radio',
'wind',
'directions',
// The brand is already the layer id; all spoken aliases remain accepted.
's2-ghostmaps',
]);
/**
@@ -487,7 +541,7 @@ export function voiceLayerAliasHint() {
entry.toggle && entry.aliases.length && !SELF_EVIDENT_IDS.has(entry.id),
)
.map((entry) => `${entry.aliases.slice(0, 2).join('/')}→${entry.id}`)
.join('; ');
.join(';');
}
/** Field names that already carry their unit (altitudeM, speedKts, …). */
+1 -1
View File
@@ -4,7 +4,7 @@ import { progressLine } from './speech.js';
* Timing policy for filling silence while tools run. Tuned against the turn
* spans' `max_silence_ms`.
*/
export const NARRATION_TIMING = Object.freeze({
const NARRATION_TIMING = Object.freeze({
/** Start the earcon when nothing is audible this long after end of speech. */
earconAfterMs: 700,
/** First spoken progress line when a tool is still running after this. */
+178
View File
@@ -0,0 +1,178 @@
/**
* Private action accounting prototype. Keys describe invocations, never input
* provenance, successful effects, provider admission or physical settlement.
* The supplied runAction continues to own execution and currentness.
*/
export function createPanelActionObserver({
runAction,
onObservation,
signal,
}) {
if (typeof runAction !== 'function')
throw new TypeError('runAction is required');
const pending = new Map();
let disposed = false;
const publish = (type, record, details = {}) => {
try {
onObservation?.(
Object.freeze({
type,
key: record.key,
name: record.name,
pendingCount: pending.size,
...details,
}),
);
} catch {
// A presentation observer cannot change execution or its result.
}
};
const release = (record) => {
if (!pending.delete(record.key)) return false;
try {
record.signal?.removeEventListener('abort', record.abort);
} catch {
/* Observer cleanup only. */
}
return true;
};
const retireOne = (record, reason) => {
if (release(record)) publish('ownership-retired', record, { reason });
};
const retire = (reason = 'owner-retired') => {
for (const record of [...pending.values()])
retireOne(record, typeof reason === 'string' ? reason : 'owner-retired');
};
const dispose = () => {
if (disposed) return;
disposed = true;
try {
signal?.removeEventListener('abort', dispose);
} catch {
/* Observer cleanup only. */
}
retire('mount-retired');
};
signal?.addEventListener('abort', dispose, { once: true });
if (signal?.aborted) dispose();
function observedAction(...args) {
if (disposed) return Reflect.apply(runAction, this, args);
const record = {
key: Object.freeze({}),
name: typeof args[0] === 'string' ? args[0] : undefined,
};
const originalOptions = args[2];
let forwarded = args;
// Main supplies ordinary data options. Leave exotic/accessor options intact
// rather than evaluating getters or pretending all custom factories fit.
try {
const prototype =
originalOptions == null ? null : Object.getPrototypeOf(originalOptions);
const descriptors =
originalOptions == null
? {}
: Object.getOwnPropertyDescriptors(originalOptions);
const callerSignal = descriptors.signal?.value;
if (callerSignal instanceof AbortSignal) record.signal = callerSignal;
const dataOnly = Object.values(descriptors).every((d) => 'value' in d);
const originalProgress = descriptors.progress?.value;
if (
originalOptions !== undefined &&
originalOptions !== null &&
(prototype === Object.prototype || prototype === null) &&
dataOnly &&
(originalProgress == null || typeof originalProgress === 'function')
) {
const progressDescriptor = descriptors.progress;
delete descriptors.progress;
const options = Object.create(prototype, descriptors);
Object.defineProperty(options, 'progress', {
configurable: true,
enumerable: progressDescriptor?.enumerable ?? true,
writable: true,
value: function (...updates) {
const receiver = this === options ? originalOptions : this;
const returned =
originalProgress == null
? undefined
: Reflect.apply(originalProgress, receiver, updates);
if (pending.has(record.key)) {
// No raw mutable result/options/update object escapes to observers.
try {
const step = Object.getOwnPropertyDescriptor(
updates[0] ?? {},
'step',
)?.value;
const label = Object.getOwnPropertyDescriptor(
updates[0] ?? {},
'label',
)?.value;
if (typeof step === 'string')
publish('progress', record, {
step,
...(typeof label === 'string' ? { label } : {}),
});
} catch {
/* Private metadata inspection cannot change progress. */
}
}
return returned;
},
});
forwarded = [...args];
forwarded[2] = options;
}
} catch {
// Unsupported option introspection is not an execution failure.
}
if (!record.signal?.aborted) {
pending.set(record.key, record);
record.abort = () => retireOne(record, 'caller-aborted');
try {
record.signal?.addEventListener('abort', record.abort, { once: true });
} catch {
retireOne(record, 'unobserved-signal');
}
if (pending.has(record.key)) publish('attempt-started', record);
}
const settle = (outcome) => {
if (release(record)) publish('attempt-settled', record, { outcome });
};
let result;
try {
result = Reflect.apply(runAction, this, forwarded);
} catch (error) {
settle('synchronous-throw');
throw error;
}
// The actual common session returns a same-realm native Promise. Do not
// assimilate arbitrary thenables, read their .then getter or invent their
// settlement; custom return values still reach the caller unchanged.
try {
if (result instanceof Promise) {
Promise.prototype.then.call(
result,
() => settle('fulfilled'),
() => settle('rejected'),
);
} else if (
result === null ||
!['object', 'function'].includes(typeof result)
) {
settle('synchronous-return');
} else retireOne(record, 'unobserved-return-kind');
} catch {
retireOne(record, 'unobserved-return-kind');
}
return result;
}
return {
runAction: observedAction,
retire,
dispose,
get pendingCount() {
return pending.size;
},
};
}
+498
View File
@@ -0,0 +1,498 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { getEventListeners } from 'node:events';
import { createPanelActionObserver } from './actionObserver.js';
import { createVoiceSession } from '../session.js';
function deferred() {
let resolve, reject;
const promise = new Promise((yes, no) => {
resolve = yes;
reject = no;
});
return { promise, resolve, reject };
}
test('sync value/throw, receiver, arguments and opaque unsupported result stay exact', () => {
const receiver = {},
args = {},
options = { extra: 7 };
const error = Error('original');
let calls = 0;
const events = [];
const observer = createPanelActionObserver({
onObservation: (event) => events.push(event),
runAction(name, received, opts, tail) {
assert.equal(this, receiver);
assert.equal(received, args);
assert.equal(opts.extra, 7);
assert.equal(tail, 'tail');
calls++;
if (name === 'throw') throw error;
return 42;
},
});
assert.equal(
observer.runAction.call(receiver, 'value', args, options, 'tail'),
42,
);
assert.throws(
() => observer.runAction.call(receiver, 'throw', args, options, 'tail'),
(e) => e === error,
);
assert.equal(calls, 2);
assert.equal(observer.pendingCount, 0);
assert.deepEqual(
events.filter((e) => e.type === 'attempt-settled').map((e) => e.outcome),
['synchronous-return', 'synchronous-throw'],
);
const opaque = Object.defineProperty({}, 'then', {
get() {
throw Error('must not read then');
},
});
const another = createPanelActionObserver({ runAction: () => opaque });
assert.equal(another.runAction('opaque'), opaque);
assert.equal(another.pendingCount, 0);
});
test('native Promise identity, fulfillment object and rejection object are preserved', async () => {
for (const reject of [false, true]) {
const d = deferred(),
value = { exact: true },
events = [];
const observer = createPanelActionObserver({
runAction: () => d.promise,
onObservation: (e) => events.push(e),
});
const returned = observer.runAction('promise');
assert.equal(returned, d.promise);
assert.equal(observer.pendingCount, 1);
if (reject) {
d.reject(value);
await assert.rejects(returned, (e) => e === value);
} else {
d.resolve(value);
assert.equal(await returned, value);
}
assert.equal(observer.pendingCount, 0);
assert.equal(events.at(-1).outcome, reject ? 'rejected' : 'fulfilled');
assert.equal('result' in events.at(-1), false);
assert.equal('error' in events.at(-1), false);
}
});
test('null/reused callIds have independent keys and settlement cannot retire a sibling', async () => {
const a = deferred(),
b = deferred(),
events = [];
const observer = createPanelActionObserver({
runAction: (name) => (name === 'a' ? a.promise : b.promise),
onObservation: (e) => events.push(e),
});
const pa = observer.runAction('a', {}, { callId: null });
const pb = observer.runAction('b', {}, { callId: null });
const starts = events.filter((e) => e.type === 'attempt-started');
assert.notEqual(starts[0].key, starts[1].key);
assert.equal(observer.pendingCount, 2);
b.resolve('b');
await pb;
assert.equal(observer.pendingCount, 1);
assert.equal(events.at(-1).key, starts[1].key);
a.resolve('a');
await pa;
assert.equal(observer.pendingCount, 0);
});
test('progress preserves public-before-original-before-private order, receiver and option references', async () => {
const order = [],
signal = new AbortController().signal,
symbol = Symbol('unknown');
let currentCalls = 0;
const isCurrent = () => {
currentCalls++;
return true;
};
const options = {
signal,
isCurrent,
progress(update) {
assert.equal(this, options);
order.push('original');
return update;
},
};
Object.defineProperty(options, 'hidden', { value: 8 });
options[symbol] = 9;
const custom = {};
const observer = createPanelActionObserver({
onObservation(e) {
if (e.type === 'progress') order.push('private');
},
runAction(name, args, received) {
assert.equal(received.signal, signal);
assert.equal(received.isCurrent, isCurrent);
assert.equal(received.hidden, 8);
assert.equal(received[symbol], 9);
assert.equal(
Object.getOwnPropertyDescriptor(received, 'hidden').enumerable,
false,
);
order.push('public');
assert.equal(received.progress({ step: 'running' }).step, 'running');
return Promise.resolve();
},
});
await observer.runAction('action', {}, options);
assert.deepEqual(order, ['public', 'original', 'private']);
assert.equal(currentCalls, 0);
const explicit = createPanelActionObserver({
runAction(name, args, received) {
received.progress.call(custom, { step: 'running' });
return 0;
},
});
let originalReceiver;
explicit.runAction(
'explicit',
{},
{
progress() {
originalReceiver = this;
},
},
);
assert.equal(originalReceiver, custom);
});
test('observer faults and private metadata faults cannot replace original callback errors', async () => {
const originalError = Error('callback');
const observer = createPanelActionObserver({
onObservation() {
throw Error('observer');
},
runAction(name, args, options) {
options.progress({ step: 'work' });
return Promise.resolve('ok');
},
});
await assert.rejects(
Promise.resolve().then(() =>
observer.runAction(
'error',
{},
{
progress() {
throw originalError;
},
},
),
),
(e) => e === originalError,
);
const hostile = new Proxy(
{},
{
getOwnPropertyDescriptor() {
throw Error('private read');
},
},
);
const safe = createPanelActionObserver({
runAction(name, args, options) {
options.progress(hostile);
return Promise.resolve('unchanged');
},
onObservation() {
throw Error('observer');
},
});
assert.equal(await safe.runAction('safe', {}, {}), 'unchanged');
});
test('accessor/custom options are forwarded untouched without extra getter/currentness probes', async () => {
let reads = 0,
predicates = 0;
const options = Object.defineProperty(
{
isCurrent() {
predicates++;
return true;
},
},
'progress',
{
get() {
reads++;
return () => {};
},
},
);
const observer = createPanelActionObserver({
runAction(name, args, received) {
assert.equal(received, options);
received.progress({});
received.isCurrent();
return Promise.resolve();
},
});
await observer.runAction('accessor', {}, options);
assert.equal(reads, 1);
assert.equal(predicates, 1);
assert.equal(observer.pendingCount, 0);
});
test('real caller abort retires ownership only; late A cannot release or report over B', async () => {
const a = deferred(),
b = deferred(),
caller = new AbortController(),
events = [],
callbacks = [];
let oldOptions;
const observer = createPanelActionObserver({
onObservation: (e) => events.push(e),
runAction(name, args, options) {
if (name === 'a') oldOptions = options;
return name === 'a' ? a.promise : b.promise;
},
});
const pa = observer.runAction(
'a',
{},
{ signal: caller.signal, progress: () => callbacks.push('original') },
);
caller.abort();
assert.equal(observer.pendingCount, 0);
assert.equal(events.at(-1).type, 'ownership-retired');
assert.equal(
events.some((e) => e.type === 'attempt-settled'),
false,
);
const pb = observer.runAction('b');
const count = events.length;
oldOptions.progress({ step: 'late' });
a.resolve('a');
await pa;
assert.deepEqual(callbacks, ['original']);
assert.equal(events.length, count);
assert.equal(observer.pendingCount, 1);
b.resolve('b');
await pb;
assert.equal(observer.pendingCount, 0);
assert.equal(getEventListeners(caller.signal, 'abort').length, 0);
});
test('already-aborted caller and disposed mount still invoke original exactly once', async () => {
const caller = new AbortController(),
mount = new AbortController();
caller.abort();
let calls = 0;
const events = [];
const observer = createPanelActionObserver({
signal: mount.signal,
onObservation: (e) => events.push(e),
runAction: () => {
calls++;
return Promise.resolve('original');
},
});
assert.equal(
await observer.runAction('aborted', {}, { signal: caller.signal }),
'original',
);
assert.equal(observer.pendingCount, 0);
mount.abort();
assert.equal(await observer.runAction('disposed'), 'original');
assert.equal(calls, 2);
assert.equal(events.length, 0);
assert.equal(getEventListeners(mount.signal, 'abort').length, 0);
});
test('reentrant start retirement retains only fresh ownership and never skips execution', async () => {
const a = deferred(),
b = deferred(),
called = [];
let observer, pb;
observer = createPanelActionObserver({
runAction(name) {
called.push(name);
return name === 'a' ? a.promise : b.promise;
},
onObservation(e) {
if (e.type === 'attempt-started' && e.name === 'a') {
observer.retire('actual-interruption');
pb = observer.runAction('b');
}
},
});
const pa = observer.runAction('a');
assert.deepEqual(called, ['b', 'a']);
assert.equal(observer.pendingCount, 1);
a.resolve();
await pa;
assert.equal(observer.pendingCount, 1);
b.resolve();
await pb;
assert.equal(observer.pendingCount, 0);
});
test('actual pinned-main common session: private invocation stream is separate from public events', async (t) => {
const publicEvents = [],
privateEvents = [],
order = [];
let hooks, observer;
const result = { ok: true, unchanged: true };
const session = createVoiceSession({
runner: async (name, args, options) => {
assert.equal(options.isCurrent(), true);
options.progress({ step: 'actual', label: 'Actual' });
return result;
},
createAdapter(h) {
hooks = h;
observer = createPanelActionObserver({
runAction: h.runAction,
signal: h.signal,
onObservation(e) {
privateEvents.push(e);
if (e.type === 'progress') order.push('private');
},
});
return {
start() {
h.emit({ type: 'state', state: 'listening' });
},
stop() {},
sendText() {},
sendMapEvent() {},
};
},
});
t.after(() => session.destroy());
session.subscribe((e) => {
publicEvents.push(e);
if (e.type === 'progress') order.push('public');
});
await session.start();
assert.equal(
await observer.runAction(
'actual',
{},
{
progress() {
order.push('caller');
},
},
),
result,
);
assert.deepEqual(order, ['public', 'caller', 'private']);
const call = publicEvents.find((e) => e.type === 'action-call');
assert.deepEqual(call, {
type: 'action-call',
name: 'actual',
callId: null,
arguments: {},
});
assert.equal(
publicEvents.some((e) => e.type === 'action-settled'),
false,
);
assert.equal(privateEvents.at(-1).type, 'attempt-settled');
assert.equal(observer.pendingCount, 0);
assert.equal(hooks.signal.aborted, false);
});
test('actual common interruption aborts resources; injected real-lifecycle observation retires uncooperative A', async (t) => {
const a = deferred(),
b = deferred(),
events = [];
let hooks, observer, aSignal;
const session = createVoiceSession({
runner: (name, args, options) => {
if (name === 'a') aSignal = options.signal;
return name === 'a' ? a.promise : b.promise;
},
createAdapter(h) {
hooks = h;
observer = createPanelActionObserver({
runAction: h.runAction,
signal: h.signal,
onObservation: (e) => events.push(e),
});
return {
start() {
h.emit({ type: 'state', state: 'listening' });
},
stop() {},
sendText() {},
sendMapEvent() {},
};
},
});
t.after(() => session.destroy());
await session.start();
const pa = observer.runAction('a');
const rejection = assert.rejects(pa, { name: 'AbortError' });
hooks.emit({ type: 'interruption', reason: 'actual-common-event' });
assert.equal(aSignal.aborted, true);
// This bridge is explicitly supplied by the test, not automatically inferred.
observer.retire('observed-common-interruption');
assert.equal(observer.pendingCount, 0);
const pb = observer.runAction('b');
const count = events.length;
a.resolve({ ok: true });
await rejection;
assert.equal(events.length, count);
assert.equal(observer.pendingCount, 1);
b.resolve({ ok: true });
await pb;
assert.equal(observer.pendingCount, 0);
});
test('null-prototype option descriptors survive instrumentation; mount abort does not fabricate settlement', async () => {
const mount = new AbortController(),
d = deferred(),
events = [],
originalProgress = [];
const options = Object.assign(Object.create(null), {
progress: () => originalProgress.push('original'),
});
let received;
const observer = createPanelActionObserver({
signal: mount.signal,
onObservation: (e) => events.push(e),
runAction(name, args, opts) {
received = opts;
return d.promise;
},
});
const promise = observer.runAction('pending', {}, options);
assert.equal(Object.getPrototypeOf(received), null);
mount.abort();
assert.equal(observer.pendingCount, 0);
assert.equal(events.at(-1).type, 'ownership-retired');
const count = events.length;
received.progress({ step: 'late' });
d.resolve('done');
assert.equal(await promise, 'done');
assert.deepEqual(originalProgress, ['original']);
assert.equal(events.length, count);
assert.equal(getEventListeners(mount.signal, 'abort').length, 0);
});
test('omitted and explicit undefined options keep original arity and values', () => {
const calls = [];
const observer = createPanelActionObserver({
runAction: function (...args) {
calls.push(args);
return 1;
},
});
assert.equal(observer.runAction('omitted', {}), 1);
assert.equal(observer.runAction('explicit', {}, undefined), 1);
assert.equal(calls[0].length, 2);
assert.equal(calls[1].length, 3);
assert.equal(calls[0][2], undefined);
assert.equal(calls[1][2], undefined);
});
+314
View File
@@ -0,0 +1,314 @@
import { t } from '../../i18n/index.js';
import { createManualVoicePanel } from './manualPanel.js';
import {
createVoicePreferences,
bindPreferenceControls,
} from './preferences.js';
import { createPanelActionObserver } from './actionObserver.js';
import { observeNativeActivity } from './nativeActivity.js';
import { createVoiceInactivity } from './inactivity.js';
/** One UI/preference owner across provider mounts; no semantic voice routes. */
export function createVoicePanelHost({
storage,
clock = {},
readCockpitState = () => ({
active:
globalThis.document?.body?.classList?.contains('cockpit-mode') === true,
}),
} = {}) {
const preferences = createVoicePreferences({ storage });
let current = null;
let disposed = false;
let saved = { view: 'preview', panelOpen: false, automatic: true };
function observeActions(hooks) {
const observer = createPanelActionObserver({
runAction: hooks.runAction,
signal: hooks.signal,
onObservation: () => {
if (current?.observer === observer) current.workChanged();
},
});
return observer;
}
function mount({ ui, card, session, observer, nativeProvider }) {
if (disposed || !ui.panel || !card?.setEnabled || !card?.setPresented)
return null;
current?.destroy();
let dead = false;
let revision = 0;
let operationEpoch = 0;
let operationWrite = false;
let panel;
let idle;
let refreshQueued = false;
let automatic = saved.automatic;
let inputBusy = false;
let finalTranscript = null;
const presentationLifetime = new AbortController();
const rearm = () => {
if (!owns() || !session.isActive()) return;
automatic = true;
saved.automatic = true;
card.rearm?.();
// Unknown IDs cannot distinguish a fresh identical answer after input.
if (!finalTranscript?.id) finalTranscript = null;
};
const owns = () =>
!dead && !disposed && current === binding && !session.disposed;
const remember = ({ automatic: auto = false } = {}) => {
if (!auto) {
revision++;
automatic = false;
}
if (panel) {
const value = panel.snapshot();
saved = { view: value.view, panelOpen: value.panelOpen, automatic };
}
};
const renderActivity = () => {
ui.update('preferences', {
timeoutStatus: () =>
nativeProvider === 'openai' || nativeProvider === 'gemini'
? ''
: t('Automatic stop is unavailable for this voice connection.'),
});
};
const changed = () => {
if (!owns()) return;
const busy = activity.snapshot().inputBusy;
if (busy && !inputBusy) rearm();
inputBusy = busy;
idle?.update({ activity: true });
renderActivity();
};
const activity = observeNativeActivity({
adapter: session.adapter,
nativeProvider,
signal: session.signal,
onChange: changed,
onInput: rearm,
});
const closeAfterStop = (ticket) => {
if (owns() && ticket === revision && !session.isActive()) panel.close();
};
const binding = {
observer,
get panel() {
return panel;
},
stopTicket: () => revision,
closeAfterStop,
workChanged: changed,
captureOperation() {
const stamp = revision;
const epoch = operationEpoch;
const isCurrent = () =>
owns() && stamp === revision && epoch === operationEpoch;
const cancelled = () => ({
ok: false,
applied: false,
cancelled: true,
});
return {
isCurrent,
setMinutes(value) {
if (!isCurrent()) return cancelled();
if (typeof value !== 'number' || !Number.isInteger(value))
return {
ok: false,
applied: false,
changed: false,
persisted: null,
reason: 'invalid-value',
minutes: preferences.snapshot().minutes,
};
let result;
operationWrite = true;
try {
result = preferences.setMinutes(value);
} finally {
operationWrite = false;
}
return {
ok: result.ok,
applied: result.applied,
changed: result.changed,
persisted: result.persisted,
minutes: result.minutes,
...(result.reason ? { reason: result.reason } : {}),
};
},
show(view, open = true) {
if (!isCurrent()) return cancelled();
const result = open ? panel.select(view) : panel.close(view);
return { ...result, applied: result.ok };
},
};
},
handle(event) {
if (!owns()) return;
if (event.type !== 'state' && !session.isActive()) return;
if (
event.type === 'state' &&
['connecting', 'idle', 'error'].includes(event.state)
) {
finalTranscript = null;
if (event.state !== 'connecting') saved.automatic = automatic = false;
}
if (
event.type === 'interruption' &&
!event.preserveAdvisoryCaption &&
['user-speech', 'user-text', 'push-to-talk'].includes(event.reason)
)
rearm();
if (
event.type === 'transcript' &&
event.role === 'assistant' &&
event.text
) {
const id = event.responseId || event.itemId || null;
if (
finalTranscript &&
(id ? id === finalTranscript.id : !finalTranscript.id) &&
((!event.final && id) ||
(event.final && event.text === finalTranscript.text))
)
return;
if (event.final) finalTranscript = { id, text: event.text };
if (
automatic &&
preferences.snapshot().previewEnabled &&
!readCockpitState()?.active
)
panel.select('preview', { automatic: true });
}
// A retired native boundary is not fresh input and must not revive
// dismissal or erase the advisory caption it explicitly preserves.
if (!(event.type === 'interruption' && event.preserveAdvisoryCaption))
card.handle(event);
if (
event.type === 'interruption' ||
(event.type === 'state' &&
['connecting', 'idle', 'error'].includes(event.state))
) {
operationEpoch++;
observer.retire('session-boundary');
}
activity.handle(event);
idle?.update();
// Native controllers finish processing after forwarding some events.
// One guarded microtask refresh reads the final current local state.
if (!refreshQueued) {
refreshQueued = true;
queueMicrotask(() => {
refreshQueued = false;
if (!owns()) return;
activity.handle({ type: 'refresh' });
idle?.update();
});
}
},
snapshot: () => ({
...panel.snapshot(),
preferences: preferences.snapshot(),
activity: activity.snapshot(),
inactivity: idle.snapshot(),
}),
destroy() {
if (dead) return;
dead = true;
presentationLifetime.abort();
idle?.destroy();
activity.destroy();
fields.dispose();
observer.dispose();
panel.destroy();
session.signal.removeEventListener('abort', binding.destroy);
if (current === binding) current = null;
},
};
current = binding;
panel = createManualVoicePanel({
ui,
card,
getPreviewEnabled: () => preferences.snapshot().previewEnabled,
getCockpitActive: () => readCockpitState()?.active === true,
// Getters, so the panel words them in the current locale.
copy: {
get previewEmpty() {
return t('Voice replies appear here.');
},
get previewDisabled() {
return t('Preview is off.');
},
},
onPresentationIntent: remember,
});
const fields = bindPreferenceControls({
ui,
preferences,
copy: {
previewNote: '',
get timeoutNote() {
return t('These preferences stay in this browser.');
},
get notSaved() {
return t('Your choice applies now, but could not be saved.');
},
},
onChange: (state) => {
if (!owns()) return;
if (!operationWrite) revision++;
card.setEnabled(state.previewEnabled);
panel.sync();
idle?.setMinutes(state.minutes);
},
});
globalThis.window?.addEventListener(
'gev:cockpit-mode-changed',
() => {
if (owns()) panel.sync();
},
{ signal: presentationLifetime.signal },
);
const offDismiss = ui.on('card-dismiss', () => {
remember();
if (panel.snapshot().panelOpen)
void ui.focus(['previewTab'], { anyVisibility: true });
});
presentationLifetime.signal.addEventListener('abort', offDismiss, {
once: true,
});
card.setEnabled(preferences.snapshot().previewEnabled);
panel.sync();
idle = createVoiceInactivity({
readActivity: activity.snapshot,
readPending: () => observer.pendingCount,
minutes: preferences.snapshot().minutes,
signal: session.signal,
...clock,
onExpire: () => {
if (!owns()) return;
const ticket = revision;
session.stop();
closeAfterStop(ticket);
},
});
if (saved.panelOpen) panel.select(saved.view, { automatic: true });
renderActivity();
session.signal.addEventListener('abort', binding.destroy, { once: true });
if (session.signal.aborted) binding.destroy();
return binding;
}
return {
observeActions,
mount,
snapshot: () => preferences.snapshot(),
destroy() {
if (disposed) return;
current?.destroy();
disposed = true;
},
};
}
+828
View File
@@ -0,0 +1,828 @@
import '../../testSupport/registerDom.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
import { createPinia } from 'pinia';
import { createVoiceControl } from '../control.js';
import { useVoiceStore } from '../../ui/stores/voice.js';
import { createGevActionRunner } from '../gevActions.js';
import { createVoiceCommands as bind } from '../sessionCommands.js';
import { createVoiceCommands as compose } from '../commands.js';
import { createProviderCommands } from '../providerCommands.js';
import { createVoicePanelHost } from './host.js';
import { createTestVoiceUi } from '../../testSupport/voiceUi.mjs';
import { mountVoiceView } from '../../testSupport/voiceView.mjs';
const $ = (selector) => globalThis.document.querySelector(selector);
const exists = (selector) => $(selector) !== null;
const active = () => globalThis.document.activeElement?.id ?? null;
const key = (node, name) =>
node.dispatchEvent(
new KeyboardEvent('keydown', {
key: name,
bubbles: true,
cancelable: true,
}),
);
function controlledClock() {
let now = 0,
next = 0;
const tasks = new Map();
return {
now: () => now,
setTimer(fn, ms) {
const id = ++next;
tasks.set(id, { fn, at: now + ms });
return id;
},
clearTimer: (id) => tasks.delete(id),
advance(ms) {
now += ms;
for (const [id, task] of [...tasks])
if (task.at <= now && tasks.delete(id)) task.fn();
},
};
}
// The custom inactivity minutes, then the select's Custom choice.
function chooseCustomTimeout(ui, minutes) {
ui.update('preferences', { timeoutMinutes: minutes });
ui.emit('timeout-select', 'custom');
}
function factory(instances, { stopError = null, native = false } = {}) {
return (hooks) => {
const input = {
pushToTalkKeyHeld: false,
setMicrophoneEnabled() {},
releasePushToTalkKey() {
this.pushToTalkKeyHeld = false;
},
resetSession() {},
};
const controller = native
? {
input,
_input: input,
_turns: { observeSpeech() {} },
status: 'idle',
dc: null,
sendTextCommand(text) {
if (!text) return false;
hooks.emit({
type: 'pointer',
pointer: { kind: 'here', label: 'Typed target' },
});
},
createAudio(options) {
this.audioCallbacks = options;
return {
pending: false,
play() {},
clearPlayback() {},
close() {},
};
},
handleMessage() {},
userActivityOpen: false,
pendingTools: 0,
isActive: () => true,
}
: {};
const adapter = {
controller,
capabilities: { pushToTalk: true },
starts: 0,
stops: 0,
bindControls() {},
start() {
this.starts++;
if (native) {
hooks.emit({ type: 'state', state: 'connecting' });
controller.audio = controller.createAudio({ pushToTalk: false });
hooks.emit({ type: 'state', state: 'connecting' });
// Native open-mic startup marks presentation pending before speech.
controller.userTurnPending = true;
}
hooks.emit({ type: 'state', state: 'listening' });
},
stop() {
this.stops++;
if (stopError) throw stopError;
},
sendText: () => false,
sendMapEvent() {},
getStartOptions: () => ({ pushToTalk: false }),
hooks,
input,
};
instances.push(adapter);
return adapter;
};
}
// With `view`, the real VoiceControl renders the control from its store.
function mounted(t, options = {}) {
const view = options.view ? mountVoiceView(t) : null;
const pinia = view?.pinia ?? createPinia();
const clock = controlledClock();
const host = createVoicePanelHost({
storage: options.storage ?? null,
clock,
readCockpitState: options.readCockpitState,
});
const instances = [];
const commands = bind({
runner: options.runner ?? (async () => ({ ok: true })),
dataManager: { layers: new Map() },
createSession: factory(instances, options),
createControl: (control) => createTestVoiceUi({ ...control, pinia }),
panelHost: host,
nativePanelProvider: options.native
? (options.nativeProvider ?? 'gemini')
: null,
});
t.after(() => {
try {
commands.session.destroy();
} catch {}
host.destroy();
});
return {
clock,
host,
instances,
commands,
ui: commands.ui,
settle: view?.settle,
};
}
test('actual control default remains opt-out; panel markup has unique controlled panes and an outside alert', async (t) => {
const view = mountVoiceView(t);
const store = useVoiceStore(view.pinia);
const baseline = createVoiceControl({ store });
t.after(() => baseline.remove());
assert.equal(baseline.panel, false);
await view.settle();
assert.equal(exists('#gev-voice-control'), true);
assert.equal(exists('#gev-voice-panel'), false);
const ui = createVoiceControl({ panel: true, store });
t.after(() => ui.remove());
await view.settle();
const panel = $('#gev-voice-panel');
assert.equal(panel !== null, true);
assert.equal(
$('#gev-voice-settings-button').getAttribute('aria-controls'),
panel.id,
);
for (const [tab, pane] of [
['#gev-voice-preview-tab', 'gev-voice-preview-panel'],
['#gev-voice-settings-tab', 'gev-voice-settings-panel'],
]) {
assert.equal($(tab).getAttribute('aria-controls'), pane);
assert.equal(exists(`#${pane}`), true);
}
assert.equal(
$('#gev-voice-preview-panel').contains($('#gev-voice-card')),
true,
);
const tray = $('#gev-voice-control .gev-voice-error-tray');
assert.equal(tray !== null, true);
assert.equal(panel.contains(tray), false);
assert.equal(
$('#gev-voice-control').querySelectorAll('#gev-voice-card').length,
1,
);
});
test('actual binder/card has launcher toggle, idempotent tabs, focus and disabled Preview', async (t) => {
const { ui, commands, settle } = mounted(t, { view: true });
await settle();
$('#gev-voice-settings-button').click();
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
$('#gev-voice-settings-button').click();
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
await settle();
$('#gev-voice-settings-tab').click();
key($('#gev-voice-settings-tab'), 'End');
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
$('#gev-voice-settings-tab').click();
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
commands.voicePanel.select('preview');
commands.voiceCard.handle({
type: 'transcript',
role: 'assistant',
final: true,
text: 'A visible reply',
});
await settle();
const dismiss = $('#gev-voice-card-dismiss');
dismiss.focus();
dismiss.click();
await settle();
assert.equal(ui.state.card.visible, false);
assert.equal($('#gev-voice-card').hidden, true);
assert.equal(commands.voicePanel.snapshot().panelOpen, true);
assert.equal(active(), 'gev-voice-preview-tab');
key(globalThis.document.activeElement, 'Escape');
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
commands.voicePanel.select('settings');
ui.emit('preview-change', false);
commands.voicePanel.select('preview');
commands.voiceCard.handle({
type: 'transcript',
role: 'assistant',
final: true,
text: 'A hidden reply',
});
assert.equal(ui.state.card.visible, false);
assert.equal(ui.state.previewEmpty.text, 'Preview is off.');
await settle();
$('#gev-voice-preview-tab').focus();
key(globalThis.document.activeElement, 'Escape');
await settle();
assert.equal(active(), 'gev-voice-settings-button');
commands.voiceCard.handle({
type: 'transcript',
role: 'assistant',
final: true,
text: 'Later reply',
});
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
});
test('actual facade remount keeps failed-save live preferences/view and manual close, with no restore writes or focus theft', async (t) => {
const view = mountVoiceView(t);
const writes = [];
const storage = {
getItem: () => null,
setItem(key, value) {
writes.push([key, value]);
throw Error('blocked');
},
};
const host = createVoicePanelHost({ storage });
const instances = [];
const commands = createProviderCommands(
{
runner: async () => ({}),
dataManager: { layers: new Map() },
supportedProviders: ['openai', 'gemini'],
},
{
bind: (options) =>
bind({
...options,
panelHost: host,
createControl: (control) =>
createTestVoiceUi({ ...control, pinia: view.pinia }),
}),
factories: { openai: factory(instances), gemini: factory(instances) },
storage,
},
);
t.after(() => {
commands.destroy();
host.destroy();
});
const old = commands.ui;
commands.voicePanel.select('settings');
old.emit('preview-change', false);
chooseCustomTimeout(old, '7');
const preferenceWrites = () =>
writes.filter(([key]) => key.startsWith('godsEyeView.voice.')).length;
assert.equal(preferenceWrites(), 2);
await view.settle();
const oldRoot = $('#gev-voice-control');
old.emit('provider-change', 'gemini');
assert.equal(commands.ui !== old, true);
await view.settle();
assert.equal(oldRoot.isConnected, false);
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
assert.deepEqual(commands.getVoicePreferences(), {
previewEnabled: false,
minutes: 7,
previewPersisted: false,
timeoutPersisted: false,
});
assert.equal(commands.ui.state.preferences.previewChecked, false);
assert.equal($('#gev-voice-preview-enabled').checked, false);
assert.match(commands.ui.state.preferences.timeoutNote, /could not be saved/);
assert.equal(preferenceWrites(), 2);
assert.equal(active(), 'gev-voice-provider');
commands.ui.emit('preview-change', true);
commands.voicePanel.close();
await commands.start();
const retired = instances[1];
commands.ui.emit('provider-change', 'openai');
await commands.start();
assert.equal(commands.session.isActive(), true);
retired.hooks.emit({
type: 'transcript',
role: 'assistant',
text: 'Retired output',
final: true,
});
const emit = instances[2].hooks.emit;
emit({
type: 'transcript',
role: 'assistant',
text: 'Current output after remount',
final: true,
});
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
emit({ type: 'interruption', reason: 'user-speech' });
emit({
type: 'transcript',
role: 'assistant',
text: 'Reply after genuine input',
final: true,
});
assert.equal(commands.voicePanel.snapshot().previewVisible, true);
});
test('explicit successful mic Stop closes; failed Stop idle and startup error do not', async (t) => {
const good = mounted(t);
await good.commands.session.start();
good.commands.voicePanel.select('settings');
good.ui.emit('button');
assert.equal(good.commands.voicePanel.snapshot().panelOpen, false);
const error = Error('stop failed');
const bad = mounted(t, { stopError: error });
await bad.commands.session.start();
bad.commands.voicePanel.select('settings');
assert.throws(
() => bad.ui.emit('button'),
(value) => value === error,
);
assert.equal(bad.commands.session.state, 'idle');
assert.equal(bad.commands.voicePanel.snapshot().settingsOpen, true);
bad.instances[0].hooks.emit({
type: 'state',
state: 'error',
detail: 'startup failed',
});
assert.equal(bad.commands.voicePanel.snapshot().settingsOpen, true);
});
test('reentrant manual choice/new start during Stop cannot close the successor choice', async (t) => {
const f = mounted(t);
await f.commands.session.start();
f.commands.voicePanel.select('settings');
const unsubscribe = f.commands.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'idle')
f.commands.voicePanel.select('preview');
});
f.ui.emit('button');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
unsubscribe();
});
test('integrated inactivity rechecks held PTT at expiry, then normal stop closes on later quiet expiry', async (t) => {
const f = mounted(t, { native: true });
await f.commands.session.start();
f.commands.voicePanel.select('settings');
chooseCustomTimeout(f.ui, '1');
f.instances[0].input.pushToTalkKeyHeld = true;
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 0);
f.instances[0].input.releasePushToTalkKey();
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 1);
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
});
test('None cancels integrated app timer and unknown custom hosts never guess expiry', async (t) => {
const native = mounted(t, { native: true });
await native.commands.session.start();
native.ui.emit('timeout-select', '0');
native.clock.advance(600000);
assert.equal(native.instances[0].stops, 0);
assert.equal(native.commands.getVoicePreferences().minutes, 0);
const unknown = mounted(t);
await unknown.commands.session.start();
unknown.clock.advance(600000);
assert.equal(unknown.instances[0].stops, 0);
assert.match(unknown.ui.state.preferences.timeoutStatus, /unavailable/);
});
test('transient native OpenAI response activity does not announce connection unavailability', async (t) => {
const f = mounted(t, { native: true, nativeProvider: 'openai' });
await f.commands.session.start();
chooseCustomTimeout(f.ui, '1');
f.instances[0].controller._turns.observeSpeech({ type: 'response.created' });
assert.equal(f.ui.state.preferences.timeoutStatus, '');
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 0, 'unknown output still prevents expiry');
f.instances[0].controller._turns.observeSpeech({
type: 'output_audio_buffer.cleared',
});
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 1, 'known quiet output permits expiry');
});
test('native factory with custom controller remains uninstrumented despite panel opt-in', (t) => {
let inspected = 0;
const commands = compose({
panel: true,
panelStorage: null,
runner: async () => ({}),
dataManager: { layers: new Map() },
createController: (options) => ({
ui: options.ui,
get _turns() {
inspected++;
throw Error('custom internal getter');
},
start() {},
stop() {},
sendTextCommand() {},
notifyMapEvent() {},
syncCostUi() {},
bindPushToTalkShortcut() {},
toggleVoiceTier() {},
}),
});
t.after(() => commands.session.destroy());
assert.equal(inspected, 0);
assert.equal(commands.getVoicePanelState().activity.known, false);
assert.equal(inspected, 0);
});
test('native connecting phases retain output observation; idle starts before first speech and waits for drain', async (t) => {
const f = mounted(t, { native: true });
await f.commands.session.start();
const adapter = f.instances[0];
const controller = adapter.controller;
assert.equal(
f.commands.getVoicePanelState().inactivity.eligible,
true,
'open microphone presentation-pending is not admitted speech',
);
controller.audio.pending = true;
controller.handleMessage({});
f.clock.advance(300000);
assert.equal(adapter.stops, 0);
assert.equal(f.commands.getVoicePanelState().activity.outputBusy, true);
controller.audio.pending = false;
controller.audioCallbacks.onDrain();
f.clock.advance(300000);
assert.equal(adapter.stops, 1);
});
async function directActions(
t,
{ storage = null, delayExit = false, exposePanel = true } = {},
) {
let f, releaseExit;
const state = { active: true, exits: 0, visual: [] };
const styleManager = {
getCockpitState: () => ({ active: state.active }),
controlCockpit(action) {
assert.equal(action, 'exit');
state.exits++;
state.active = false;
const result = { ok: true, state: { active: false } };
return delayExit
? new Promise((resolve) => {
releaseExit = () => resolve(result);
})
: result;
},
setPanelCollapsed(...args) {
state.visual.push(args);
},
};
const runner = createGevActionRunner({
viewer: {
clock: { onTick: { addEventListener: () => () => {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
camera: { moveEnd: { addEventListener() {} } },
},
styleManager,
dataManager: { layers: new Map(), getAll: () => [] },
readVoicePanelOperation: () =>
exposePanel ? f?.commands.captureVoicePanelOperation() : undefined,
});
f = mounted(t, { native: true, runner, storage });
await f.commands.session.start();
return {
...f,
runner,
state,
run: f.instances[0].hooks.runAction,
releaseExit: () => releaseExit(),
};
}
test('actual runner applies exact timeout directly and reports live-unsaved truth, invalid/cancelled attempts stay inert', async (t) => {
const f = await directActions(t);
const result = await f.run('set_voice_inactivity_timeout', { minutes: 7 });
assert.equal(result.applied, true);
assert.equal(result.persisted, false);
assert.equal(result.minutes, 7);
assert.match(result.say, /could not be saved/);
assert.equal(f.commands.getVoicePreferences().minutes, 7);
const invalid = await f.run('set_voice_inactivity_timeout', { minutes: 7.5 });
assert.equal(invalid.applied, false);
assert.equal(f.commands.getVoicePreferences().minutes, 7);
const abort = new AbortController();
abort.abort();
const cancelled = await f.runner(
'set_voice_inactivity_timeout',
{ minutes: 2 },
{ signal: abort.signal },
);
assert.equal(cancelled.cancelled, true);
assert.equal(f.commands.getVoicePreferences().minutes, 7);
const operation = f.commands.captureVoicePanelOperation();
f.commands.session.stop();
assert.equal(operation.setMinutes(2).cancelled, true);
});
test('named voice settings exits Cockpit directly; Preview stays disabled and visual presets stay separate from Scenes', async (t) => {
const f = await directActions(t);
const result = await f.run('set_panel_open', {
panelId: 'voice-settings',
open: true,
});
assert.equal(f.state.exits, 1);
assert.equal(result.settingsOpen, true);
f.ui.emit('preview-change', false);
const preview = await f.run('set_panel_open', {
panelId: 'voice control',
open: true,
});
assert.equal(preview.view, 'preview');
assert.equal(preview.previewEnabled, false);
assert.equal(f.commands.getVoicePreferences().previewEnabled, false);
for (const panelId of [
'visual styles',
'visual preset',
'visual presets',
'scenes',
])
await f.run('set_panel_open', { panelId, open: true });
assert.equal(f.state.exits, 1);
assert.deepEqual(
f.state.visual.map(([id]) => id),
['control-panel', 'control-panel', 'control-panel', 'scene-panel'],
);
await assert.rejects(
() => f.run('set_panel_open', { panelId: 'settings', open: true }),
/Unknown panel: settings/,
);
assert.equal(
f.state.visual.length,
4,
'an unnamed settings target never guesses a panel',
);
});
test('canonical exit reveals requested voice UI; newer manual choice or Stop defeats pending reveal', async (t) => {
const f = await directActions(t, { delayExit: true });
const pending = f.run('control_cockpit', {
action: 'exit',
voicePanel: 'settings',
});
f.commands.voicePanel.select('preview');
f.releaseExit();
const result = await pending;
assert.equal(result.cancelled, true);
assert.equal(f.state.active, false);
assert.equal(result.exit.ok, true);
assert.equal(result.cockpitExited, true);
assert.match(result.say, /Cockpit exited/);
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
f.state.active = true;
const alias = f.run('control_cockpit', {
action: 'leave cockpit',
voicePanel: 'settings',
});
f.releaseExit();
assert.equal((await alias).settingsOpen, true);
f.commands.voicePanel.select('preview');
f.state.active = true;
const stopped = f.run('control_cockpit', {
action: 'quit cockpit',
voicePanel: 'settings',
});
f.commands.session.stop();
f.releaseExit();
await assert.rejects(stopped, { name: 'AbortError' });
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
});
test('unavailable host refuses direct voice operations before Cockpit or preference effects', async (t) => {
const f = await directActions(t, { exposePanel: false });
const timeout = await f.run('set_voice_inactivity_timeout', { minutes: 1 });
assert.equal(timeout.unavailable, true);
assert.equal(timeout.applied, false);
const panel = await f.run('set_panel_open', {
panelId: 'voice-settings',
open: true,
});
assert.equal(panel.unavailable, true);
assert.equal(f.state.exits, 0);
assert.equal(f.commands.getVoicePreferences().minutes, 5);
});
test('voice timeout change preserves a pending voice reveal', async (t) => {
const f = await directActions(t, { delayExit: true });
const pending = f.run('control_cockpit', {
action: 'exit',
voicePanel: 'settings',
});
const timeout = await f.run('set_voice_inactivity_timeout', { minutes: 0 });
assert.equal(timeout.minutes, 0);
assert.equal(timeout.applied, true);
f.releaseExit();
const result = await pending;
assert.equal(result.settingsOpen, true);
assert.equal(f.commands.getVoicePreferences().minutes, 0);
});
test('active provider output reveals Preview after done; manual choice wins until genuine input, replays do not reset idle', async (t) => {
const f = mounted(t, { native: true, nativeProvider: 'openai', view: true });
await f.commands.session.start();
const emit = f.instances[0].hooks.emit;
const reply = (text, responseId = 'r1', extra = {}) =>
emit({
type: 'transcript',
role: 'assistant',
text,
responseId,
final: true,
...extra,
});
emit({ type: 'completion', responseId: 'r1' });
reply('Late answer');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
const timer = f.commands.voiceCard.hideTimer;
reply('Late answer');
reply('Late', 'r1', { final: false });
assert.equal(f.commands.voiceCard.hideTimer, timer);
assert.equal(f.commands.voiceCard.state.assistant, 'Late answer');
f.commands.voicePanel.close();
emit({
type: 'transcript',
role: 'user',
text: 'advisory caption',
final: false,
});
f.commands.voiceCard.handle({ type: 'dismiss' });
const advisory = f.commands.voiceCard.state.user;
emit({
type: 'interruption',
reason: 'user-speech',
preserveAdvisoryCaption: true,
});
assert.equal(f.commands.voiceCard.state.dismissed, true);
assert.equal(f.commands.voiceCard.state.user, advisory);
reply('Older reply', 'r2');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
emit({ type: 'interruption', reason: 'user-speech' });
reply('A reply with no newest-request attribution', 'older-id');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
f.commands.voicePanel.select('settings');
reply('Another reply', 'r3');
assert.equal(f.commands.voicePanel.snapshot().settingsOpen, true);
f.commands.voicePanel.close();
f.commands.voiceCard.handle({ type: 'dismiss' });
f.commands.sendTextCommand('Show the typed target');
reply('Typed reply', 'typed-id');
assert.equal(f.commands.voicePanel.snapshot().previewVisible, true);
assert.equal(f.commands.voiceCard.state.dismissed, false);
assert.equal(f.commands.voiceCard.state.pointer.label, 'Typed target');
f.commands.session.stop();
reply('Stopped stale output', 'r4');
assert.equal(f.commands.voiceCard.state.assistant, null);
f.commands.session.destroy();
reply('Disposed stale output', 'r5');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
});
test('Preview off and actual Cockpit state suppress automatic presentation without enabling preference', async (t) => {
let cockpit = false;
const f = mounted(t, {
native: true,
view: true,
readCockpitState: () => ({ active: cockpit }),
});
await f.commands.session.start();
const emit = f.instances[0].hooks.emit;
const reply = (text) =>
emit({ type: 'transcript', role: 'assistant', text, final: true });
reply('Visible');
await f.settle();
$('#gev-voice-preview-tab').focus();
cockpit = true;
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
assert.equal(f.ui.state.voicePanel.hidden, true);
await f.settle();
assert.equal(active(), 'gev-voice-settings-button');
f.commands.voicePanel.select('settings', { focus: true });
await f.settle();
assert.equal(active(), 'gev-voice-settings-tab');
f.commands.voicePanel.select('preview', { focus: true });
await f.settle();
assert.equal(active(), 'gev-voice-settings-button');
const outside = document.createElement('button');
outside.id = 'unrelated';
document.body.appendChild(outside);
outside.focus();
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
await f.settle();
assert.equal(active(), 'unrelated');
assert.equal(f.commands.voicePanel.snapshot().previewVisible, false);
reply('Suppressed in Cockpit');
assert.equal(f.ui.state.voicePanel.hidden, true);
f.commands.voicePanel.close();
cockpit = false;
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
assert.equal(f.ui.state.voicePanel.hidden, true);
f.ui.emit('preview-change', false);
emit({ type: 'interruption', reason: 'user-speech' });
reply('Preference stays off');
assert.equal(f.ui.state.voicePanel.hidden, true);
assert.equal(f.commands.getVoicePreferences().previewEnabled, false);
});
test('Provider Settings uses its ready dialog handlers and refuses unavailable or cancelled requests', async (t) => {
const f = await directActions(t);
t.mock.timers.enable({ apis: ['setTimeout'] });
const request = (open, options) =>
f.run('set_panel_open', { panelId: 'provider-settings', open }, options);
assert.equal((await request(true)).unavailable, true);
const doc = globalThis.document;
// The setup owner needs only its dialog element; the view is not mounted,
// so the published dialog state stands in for it.
const panel = doc.createElement('aside');
panel.id = 'key-setup';
doc.body.append(panel);
const dialog = () => settings.view?.dialog;
let frames = 0;
const previousFrame = globalThis.requestAnimationFrame;
globalThis.requestAnimationFrame = (callback) => {
frames++;
callback();
};
t.after(() => {
if (previousFrame === undefined) delete globalThis.requestAnimationFrame;
else globalThis.requestAnimationFrame = previousFrame;
});
const { initKeySetup } = await import('../../keySetup.js');
const { useProviderSettingsStore } =
await import('../../ui/stores/providerSettings.js');
const { applicationPinia } = await import('../../ui/vueRuntime.js');
const settings = useProviderSettingsStore(applicationPinia());
let finishStatus;
const pending = initKeySetup({
documentRef: doc,
fetchImpl: () =>
new Promise((resolve) => {
finishStatus = resolve;
}),
});
assert.equal(
(await request(true)).unavailable,
true,
'initializing is not ready',
);
finishStatus({
ok: true,
json: async () => ({ keys: [], setCount: 0, total: 1 }),
});
const setup = await pending;
t.after(() => setup.destroy());
doc.body.classList.add('cockpit-mode');
assert.equal((await request(true)).unavailable, true);
assert.equal(
dialog().hidden,
true,
'a concealed view cannot schedule a later surprise reveal',
);
doc.body.classList.remove('cockpit-mode');
const abort = new AbortController();
abort.abort();
await assert.rejects(() => request(true, { signal: abort.signal }), {
name: 'AbortError',
});
assert.equal(dialog().hidden, true);
assert.equal((await request(true)).ok, true);
assert.equal(dialog().hidden, false);
assert.equal(dialog().visible, true);
assert.equal((await request(true)).ok, true);
assert.equal(frames, 1, 'repeated open reuses the dialog');
assert.equal((await request(false)).ok, true);
assert.equal(dialog().visible, false);
t.mock.timers.tick(400);
assert.equal(dialog().hidden, true);
assert.equal((await request(false)).ok, true);
assert.equal(
f.state.visual.length,
0,
'provider setup never opens a visual or scene panel',
);
});
+77
View File
@@ -0,0 +1,77 @@
/** Local inactivity only. Unknown activity never authorizes expiry. */
export function createVoiceInactivity({
readActivity,
readPending,
minutes = 5,
onExpire,
signal,
setTimer = setTimeout,
clearTimer = clearTimeout,
now = Date.now,
}) {
if (!Number.isInteger(minutes) || minutes < 0 || minutes > 60) minutes = 5;
let disposed = false,
timer = null,
deadline = null,
eligible = false,
epoch = 0;
function cancel() {
epoch++;
if (timer !== null) clearTimer(timer);
timer = deadline = null;
}
function quiet() {
const a = readActivity();
return (
a.known &&
a.connected &&
!a.held &&
!a.inputBusy &&
!a.outputBusy &&
!a.generationBusy &&
!readPending()
);
}
function arm() {
cancel();
if (!minutes || disposed) return;
const id = epoch;
deadline = now() + minutes * 60000;
timer = setTimer(() => {
if (disposed || id !== epoch) return;
timer = deadline = null;
eligible = quiet();
if (eligible) onExpire?.();
}, minutes * 60000);
}
function update({ activity = false } = {}) {
if (disposed) return;
const next = quiet();
if (!next) cancel();
else if (!eligible || activity) arm();
eligible = next;
}
function setMinutes(value) {
if (!Number.isInteger(value) || value < 0 || value > 60) return false;
if (disposed) return false;
minutes = value;
eligible = false;
update();
return true;
}
function destroy() {
if (disposed) return;
disposed = true;
cancel();
signal?.removeEventListener('abort', destroy);
}
signal?.addEventListener('abort', destroy, { once: true });
if (signal?.aborted) destroy();
update();
return {
update,
setMinutes,
snapshot: () => ({ minutes, deadline, eligible, disposed }),
destroy,
};
}
+108
View File
@@ -0,0 +1,108 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceInactivity } from './inactivity.js';
function fixture() {
let activity = {
known: true,
connected: true,
held: false,
inputBusy: false,
outputBusy: false,
generationBusy: false,
};
let pending = 0,
expired = 0,
time = 100,
serial = 0;
const timers = new Map();
const idle = createVoiceInactivity({
readActivity: () => activity,
readPending: () => pending,
onExpire: () => expired++,
now: () => time,
setTimer: (fn, delay) => {
timers.set(++serial, { fn, delay });
return serial;
},
clearTimer: (id) => timers.delete(id),
});
return {
idle,
activity,
timers,
setPending: (n) => (pending = n),
expired: () => expired,
fire: () => {
const entry = [...timers][0];
timers.delete(entry[0]);
entry[1].fn();
},
tick: (n) => (time += n),
};
}
test('one clock; quiet arms default five minutes; busy blocks and quiescence rearms', () => {
const f = fixture();
assert.equal([...f.timers.values()][0].delay, 300000);
assert.equal(f.idle.snapshot().deadline, 300100);
f.activity.outputBusy = true;
f.idle.update();
assert.equal(f.timers.size, 0);
f.activity.outputBusy = false;
f.tick(200);
f.idle.update();
assert.equal(f.idle.snapshot().deadline, 300300);
f.fire();
assert.equal(f.expired(), 1);
});
test('actual activity resets but silent unchanged snapshots do not', () => {
const f = fixture();
const deadline = f.idle.snapshot().deadline;
f.tick(1000);
f.idle.update();
assert.equal(f.idle.snapshot().deadline, deadline);
f.idle.update({ activity: true });
assert.equal(f.idle.snapshot().deadline, deadline + 1000);
});
test('None cancels; invalid values cannot replace setting; explicit retry rearms', () => {
const f = fixture();
assert.equal(f.idle.setMinutes(0), true);
assert.equal(f.timers.size, 0);
for (const bad of [1.5, -1, 61, '5', NaN])
assert.equal(f.idle.setMinutes(bad), false);
assert.equal(f.idle.setMinutes(1), true);
assert.equal([...f.timers.values()][0].delay, 60000);
});
test('expiry rereads held, unknown and pending counts despite missing update', () => {
for (const kind of ['held', 'unknown', 'pending']) {
const f = fixture();
if (kind === 'held') f.activity.held = true;
if (kind === 'unknown') f.activity.known = false;
if (kind === 'pending') f.setPending(2);
f.fire();
assert.equal(f.expired(), 0);
}
});
test('pending siblings and held-before-connect forbid timer; stale callback cannot expire', () => {
const f = fixture();
const stale = [...f.timers.values()][0].fn;
f.setPending(2);
f.idle.update();
stale();
assert.equal(f.expired(), 0);
f.setPending(1);
f.idle.update();
assert.equal(f.timers.size, 0);
f.setPending(0);
f.activity.connected = false;
f.activity.held = true;
f.idle.update();
assert.equal(f.timers.size, 0);
f.activity.connected = true;
f.idle.update();
assert.equal(f.timers.size, 0);
f.activity.held = false;
f.idle.update();
assert.equal(f.timers.size, 1);
f.idle.destroy();
assert.equal(f.timers.size, 0);
});
+158
View File
@@ -0,0 +1,158 @@
const ROW_GRACE_MS = 700;
/** Manual presentation only. Session and action events belong to the host. */
export function createManualVoicePanel({
ui,
card,
getPreviewEnabled,
getCockpitActive = () => false,
copy,
onPresentationIntent,
}) {
if (!ui?.panel || typeof getPreviewEnabled !== 'function')
throw new TypeError(
'A voice panel and Preview preference reader are required.',
);
if (
typeof copy?.previewEmpty !== 'string' ||
typeof copy?.previewDisabled !== 'string'
)
throw new TypeError('Localized Preview messages are required.');
const offs = [];
const tabs = ['previewTab', 'settingsTab'];
const panes = { preview: 'previewPanel', settings: 'settingsPanel' };
let disposed = false;
let open = false;
let view = 'preview';
let graceTimer;
const enabled = () => getPreviewEnabled() === true;
const suppressed = () => view === 'preview' && getCockpitActive() === true;
const holdsFocus = (name) =>
Boolean(ui.element(name)?.contains?.(globalThis.document?.activeElement));
const snapshot = () => ({
view,
panelOpen: !disposed && open,
settingsOpen: !disposed && open && view === 'settings',
previewEnabled: enabled(),
// The card as presented, on a control the page shows.
previewVisible:
!disposed &&
open &&
view === 'preview' &&
!suppressed() &&
enabled() &&
ui.state.card.visible &&
// The error tray takes the card's place.
ui.state.root.status !== 'error' &&
ui.visible('root'),
});
const sync = () => {
if (disposed) return;
const hidePanel = !open || suppressed();
const restoreFocus = hidePanel && holdsFocus('voicePanel');
ui.update('voicePanel', {
hidden: hidePanel,
view,
previewHidden: !open || view !== 'preview' || suppressed(),
settingsHidden: !open || view !== 'settings',
settingsExpanded: open && view === 'settings',
selected: view,
});
card?.setPresented?.(open && view === 'preview' && !suppressed());
const previewOn = enabled();
ui.update('previewEmpty', {
hidden: ui.state.card.visible,
text: () => (previewOn ? copy.previewEmpty : copy.previewDisabled),
});
if (restoreFocus) void ui.focus(['settingsButton', 'button']);
};
const refuse = (reason) => ({
ok: false,
changed: false,
reason,
...snapshot(),
});
const select = (next, { focus = false, automatic = false } = {}) => {
if (disposed) return refuse('disposed');
if (!['preview', 'settings'].includes(next)) return refuse('unknown-view');
const changed = !open || view !== next;
const transferFocus = changed && holdsFocus(panes[view]);
open = true;
view = next;
onPresentationIntent?.({ automatic });
sync();
// The tab takes focus once it shows, inside a shown panel.
if (focus || transferFocus)
void ui.focus([tabs[next === 'preview' ? 0 : 1]]);
return { ok: true, changed, ...snapshot() };
};
const close = (onlyView) => {
if (disposed) return refuse('disposed');
if (onlyView && !['preview', 'settings'].includes(onlyView))
return refuse('unknown-view');
if (onlyView && onlyView !== view)
return { ok: true, changed: false, ...snapshot() };
const changed = open;
open = false;
onPresentationIntent?.();
sync();
return { ok: true, changed, ...snapshot() };
};
const toggleSettings = () =>
open && view === 'settings'
? close('settings')
: select('settings', { focus: true });
// Keep the raised row reachable across pinned trays without intercepting
// their controls. The same lifetime owns the listeners and pending timer.
const endGrace = () => {
clearTimeout(graceTimer);
if (ui.state.root.rowGrace) ui.update('root', { rowGrace: false });
};
offs.push(
ui.on('settings-button', toggleSettings),
ui.on('tab', (index) =>
select(index === 0 ? 'preview' : 'settings', { focus: true }),
),
ui.on('tab-keydown', (index, event) => {
const next =
event.key === 'Home'
? 0
: event.key === 'End'
? 1
: ['ArrowLeft', 'ArrowRight'].includes(event.key)
? 1 - index
: null;
if (next === null) return;
event.preventDefault();
select(next === 0 ? 'preview' : 'settings', { focus: true });
}),
ui.on('settings-close', () => close()),
ui.on('panel-keydown', (event) => {
if (event.key !== 'Escape' || !open) return;
event.preventDefault();
close();
}),
ui.on('pointerleave', (event) => {
if (event.pointerType === 'touch') return;
endGrace();
ui.update('root', { rowGrace: true });
graceTimer = setTimeout(endGrace, ROW_GRACE_MS);
}),
ui.on('pointerenter', endGrace),
);
sync();
return {
select,
close,
sync,
snapshot,
destroy() {
if (disposed) return;
open = false;
sync();
disposed = true;
for (const off of offs) off();
endGrace();
},
};
}
+209
View File
@@ -0,0 +1,209 @@
import '../../testSupport/registerDom.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
import { createManualVoicePanel } from './manualPanel.js';
import { createTestVoiceUi } from '../../testSupport/voiceUi.mjs';
import { mountVoiceView } from '../../testSupport/voiceView.mjs';
function fixture(t) {
const { pinia, settle } = mountVoiceView(t);
const ui = createTestVoiceUi({ panel: true, pinia });
t.after(() => ui.remove());
let enabled = true,
content = false,
intents = 0;
// The card shows when presented, with Preview on and something to show.
const card = {
setPresented(value) {
this.presented = value;
ui.replace('card', {
...ui.state.card,
visible: Boolean(value && enabled && content),
});
},
};
const copy = {
previewEmpty: 'Habla para ver resultados.',
previewDisabled: 'Vista previa desactivada.',
};
const panel = createManualVoicePanel({
ui,
card,
getPreviewEnabled: () => enabled,
copy,
onPresentationIntent: () => intents++,
});
t.after(() => panel.destroy());
const $ = (id) => document.getElementById(id);
return {
ui,
panel,
card,
copy,
settle,
$,
active: () => document.activeElement?.id ?? null,
intents: () => intents,
enabled(value) {
enabled = value;
panel.sync();
},
content(value) {
content = value;
panel.sync();
},
};
}
const click = (element) => element.click();
function key(element, value) {
const event = new KeyboardEvent('keydown', {
key: value,
bubbles: true,
cancelable: true,
});
element.dispatchEvent(event);
return event;
}
test('Settings launcher toggles while tab and programmatic selection remain idempotent', async (t) => {
const f = fixture(t);
await f.settle();
assert.equal(f.panel.snapshot().panelOpen, false);
click(f.$('gev-voice-settings-button'));
await f.settle();
assert.equal(f.panel.snapshot().settingsOpen, true);
assert.equal(f.active(), 'gev-voice-settings-tab');
assert.equal(
f.$('gev-voice-settings-button').getAttribute('aria-expanded'),
'true',
);
click(f.$('gev-voice-settings-button'));
await f.settle();
assert.equal(f.panel.snapshot().panelOpen, false);
click(f.$('gev-voice-settings-tab'));
assert.equal(f.panel.snapshot().settingsOpen, true);
assert.equal(f.panel.select('settings').changed, false);
assert.equal(f.panel.snapshot().settingsOpen, true);
click(f.$('gev-voice-settings-tab'));
assert.equal(f.panel.snapshot().settingsOpen, true);
assert.equal(f.panel.select('unknown').reason, 'unknown-view');
});
test('tab keyboard navigation and Escape preserve focus and accessibility state', async (t) => {
const f = fixture(t);
await f.settle();
click(f.$('gev-voice-settings-button'));
await f.settle();
assert.equal(
key(f.$('gev-voice-settings-tab'), 'ArrowLeft').defaultPrevented,
true,
);
await f.settle();
assert.equal(f.panel.snapshot().view, 'preview');
assert.equal(f.active(), 'gev-voice-preview-tab');
assert.equal(f.$('gev-voice-preview-tab').tabIndex, 0);
assert.equal(f.$('gev-voice-settings-tab').tabIndex, -1);
assert.equal(
f.$('gev-voice-preview-tab').getAttribute('aria-selected'),
'true',
);
key(f.$('gev-voice-preview-tab'), 'End');
assert.equal(f.panel.snapshot().view, 'settings');
key(f.$('gev-voice-settings-tab'), 'Home');
await f.settle();
assert.equal(f.panel.snapshot().view, 'preview');
assert.equal(key(f.$('gev-voice-panel'), 'Escape').defaultPrevented, true);
await f.settle();
assert.equal(f.panel.snapshot().panelOpen, false);
assert.equal(f.active(), 'gev-voice-settings-button');
assert.equal(key(f.$('gev-voice-panel'), 'Escape').defaultPrevented, false);
});
test('manual switching transfers hidden-panel focus without stealing unrelated focus', async (t) => {
const f = fixture(t);
f.panel.select('settings');
await f.settle();
// Focus inside the Settings pane moves to the Preview tab with the switch.
f.$('gev-voice-timeout-select').focus();
f.panel.select('preview');
await f.settle();
assert.equal(f.active(), 'gev-voice-preview-tab');
const unrelated = document.body.appendChild(document.createElement('button'));
unrelated.id = 'unrelated';
unrelated.focus();
f.panel.select('settings');
await f.settle();
assert.equal(f.active(), 'unrelated');
assert.equal(f.panel.close('preview').changed, false);
assert.equal(f.panel.snapshot().settingsOpen, true);
});
test('manual Preview selection never enables a disabled preference and uses injected copy', async (t) => {
const f = fixture(t);
await f.settle();
f.enabled(false);
f.content(true);
f.panel.select('preview');
await f.settle();
const empty = f.$('gev-voice-preview-empty');
assert.equal(f.panel.snapshot().previewEnabled, false);
assert.equal(f.panel.snapshot().previewVisible, false);
assert.equal(empty.textContent, f.copy.previewDisabled);
assert.equal(empty.hidden, false);
f.enabled(true);
await f.settle();
assert.equal(f.panel.snapshot().previewVisible, true);
assert.equal(empty.hidden, true);
assert.equal(f.$('gev-voice-card').hidden, false);
f.content(false);
await f.settle();
assert.equal(empty.textContent, f.copy.previewEmpty);
});
test('disposal hides owned presentation and removes listeners without destroying the card', async (t) => {
const f = fixture(t);
f.panel.select('settings');
await f.settle();
const count = f.intents();
f.panel.destroy();
f.panel.destroy();
click(f.$('gev-voice-settings-button'));
click(f.$('gev-voice-settings-tab'));
await f.settle();
assert.equal(f.intents(), count);
assert.equal(f.$('gev-voice-panel').hidden, true);
assert.equal(f.card.presented, false);
assert.equal(f.panel.select('preview').reason, 'disposed');
assert.equal(f.panel.close().reason, 'disposed');
});
test('pointer grace expires, resets on reentry, ignores touch and clears on disposal', async (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
const f = fixture(t);
await f.settle();
const root = f.$('gev-voice-control');
const pointer = (type, pointerType = 'mouse') => {
const event = new Event(type);
Object.defineProperty(event, 'pointerType', { value: pointerType });
root.dispatchEvent(event);
};
pointer('pointerleave');
t.mock.timers.tick(699);
await f.settle();
assert.equal(root.dataset.rowGrace, 'true');
pointer('pointerenter');
assert.equal(f.ui.state.root.rowGrace ? 'true' : undefined, undefined);
pointer('pointerleave');
t.mock.timers.tick(699);
assert.equal(f.ui.state.root.rowGrace ? 'true' : undefined, 'true');
t.mock.timers.tick(1);
assert.equal(f.ui.state.root.rowGrace ? 'true' : undefined, undefined);
pointer('pointerleave', 'touch');
assert.equal(f.ui.state.root.rowGrace ? 'true' : undefined, undefined);
pointer('pointerleave');
f.panel.destroy();
assert.equal(f.ui.state.root.rowGrace ? 'true' : undefined, undefined);
pointer('pointerleave');
t.mock.timers.tick(700);
assert.equal(f.ui.state.root.rowGrace ? 'true' : undefined, undefined);
});
+292
View File
@@ -0,0 +1,292 @@
/** Inspected native capabilities only; scheduled/reported output is not acoustics. */
export function observeNativeActivity({
adapter,
nativeProvider,
signal,
onChange,
onInput,
}) {
const native = nativeProvider === 'openai' || nativeProvider === 'gemini';
// Do not even inspect custom controller properties.
const controller = native ? adapter.controller : null;
const input = native
? nativeProvider === 'gemini'
? controller.input
: controller._input
: null;
let dead = false,
connected = false,
inputBusy = false,
recoverableError = false;
function cleanNativeOutput() {
if (nativeProvider === 'gemini') return true;
if (nativeProvider !== 'openai') return false;
const turns = controller._turns;
return (
['idle', 'connecting', 'error'].includes(controller.status) &&
controller.dc == null &&
!turns.assistantAudioPlaying &&
!turns.responseActive &&
!turns.responseCreatePending &&
!turns.userTurnPending &&
!turns.activeResponseId &&
!turns.progressResponse
);
}
let outputKnown = cleanNativeOutput();
let audio = null,
version = 0;
const restores = [];
const audioRestores = [];
const pendingOutput = new Map();
let untrackedOutput = !outputKnown;
function retireAudio() {
audio = null;
for (const restore of audioRestores.splice(0).reverse()) restore();
}
function refreshOutputKnowledge() {
outputKnown = !untrackedOutput && pendingOutput.size === 0;
}
function hasNoAudioOutput(response) {
return (
['completed', 'cancelled', 'failed', 'incomplete'].includes(
response?.status,
) &&
Array.isArray(response.output) &&
response.output.every(
(item) =>
item?.type === 'function_call' ||
(item?.type === 'message' &&
Array.isArray(item.content) &&
item.content.every((part) =>
['text', 'output_text', 'refusal'].includes(part?.type),
)),
)
);
}
function snapshot() {
const turns = nativeProvider === 'openai' ? controller?._turns : controller;
return {
known: native && outputKnown && !dead,
connected: connected && !dead,
held: Boolean(input?.pushToTalkKeyHeld),
inputBusy: Boolean(
inputBusy ||
(nativeProvider === 'gemini' && controller?.userActivityOpen),
),
outputBusy: Boolean(
nativeProvider === 'gemini'
? audio && audio === controller.audio && audio.pending
: turns?.assistantAudioPlaying,
),
generationBusy: Boolean(
turns?.responseActive ||
turns?.responseCreatePending ||
(nativeProvider === 'openai' && turns?.userTurnPending) ||
(nativeProvider === 'gemini' &&
(turns?.replyPending ||
turns?.retiredReplyPending ||
turns?.deferredInput ||
turns?.automaticTurnOpen ||
turns?.pendingTools ||
turns?.controlQueue?.length)),
),
};
}
function notify() {
if (!dead) {
try {
onChange?.(snapshot());
} catch {}
}
}
function wrap(object, key, after, ownedRestores = restores) {
const original = object?.[key];
if (typeof original !== 'function') return;
function wrapped(...args) {
const stamp = version;
const result = Reflect.apply(original, this, args);
if (!dead && stamp === version) {
try {
after?.(args, result);
} catch {}
}
return result;
}
object[key] = wrapped;
ownedRestores.push(() => {
if (object[key] === wrapped) object[key] = original;
});
}
function handle(event) {
if (dead || !native) return;
if (event?.type === 'state') {
const state = event.state;
const wasConnected = connected;
const liveError =
state === 'error' &&
nativeProvider === 'openai' &&
controller.dc?.readyState === 'open';
const restartingAfterError = state === 'connecting' && recoverableError;
recoverableError = liveError;
connected = !['idle', 'connecting', 'error'].includes(state);
// Connecting has multiple native phases around audio allocation. Retire
// a previous connected generation once, not each phase of this start.
// A failed reply can briefly report error on a live OpenAI transport.
// Preserve its output evidence through recovery to listening; a later
// reconnect still retires the failed connection's generation.
if (
state === 'idle' ||
(state === 'error' && !liveError) ||
(state === 'connecting' && (wasConnected || restartingAfterError))
) {
version++;
retireAudio();
inputBusy = false;
outputKnown = cleanNativeOutput();
untrackedOutput = !outputKnown;
pendingOutput.clear();
}
}
notify();
}
if (native) {
wrap(controller, 'sendTextCommand', ([text], result) => {
if (String(text || '').trim() && result !== false) onInput?.();
});
for (const key of [
'setMicrophoneEnabled',
'releasePushToTalkKey',
'resetSession',
])
wrap(input, key, notify);
if (nativeProvider === 'openai') {
wrap(controller._turns, 'observeSpeech', ([event]) => {
if (event?.type === 'input_audio_buffer.speech_started')
inputBusy = true;
if (event?.type === 'input_audio_buffer.speech_stopped')
inputBusy = false;
const type = event?.type;
const id = event?.response_id || event?.response?.id;
// Keep each unresolved response separate: a later silent reply cannot
// establish that an earlier audio reply has drained.
if (type === 'response.created') {
if (id && pendingOutput.size < 64)
pendingOutput.set(id, { audioSeen: false });
else untrackedOutput = true;
}
const pending = pendingOutput.get(id);
if (
pending &&
(type?.startsWith('response.output_audio') ||
type?.startsWith('response.audio') ||
['audio', 'output_audio'].includes(event?.part?.type))
)
pending.audioSeen = true;
if (
type === 'response.done' &&
pending &&
!pending.audioSeen &&
hasNoAudioOutput(event.response)
)
pendingOutput.delete(id);
if (
type === 'output_audio_buffer.started' ||
type === 'output_audio_buffer.cleared'
) {
// Native playback events establish the transport's output lifecycle.
// Generation completion alone never establishes audio drainage.
untrackedOutput = false;
if (id) pendingOutput.delete(id);
else pendingOutput.clear();
}
refreshOutputKnowledge();
notify();
});
} else {
wrap(controller, 'handleMessage', notify);
const original = controller.createAudio;
function createAudio(...args) {
const stamp = version;
const options = args[0];
let current;
const decorated = { ...options };
for (const key of [
'onAudioStart',
'onAudioEnd',
'onAudioDiscard',
'onDrain',
]) {
const callback = options?.[key];
decorated[key] = function (...values) {
const result =
typeof callback === 'function'
? Reflect.apply(
callback,
this === decorated ? options : this,
values,
)
: undefined;
if (
!dead &&
stamp === version &&
audio === current &&
controller.audio === current
) {
if (
key === 'onAudioStart' &&
options.pushToTalk &&
controller.captureOwner?.admitted
)
inputBusy = true;
if (key === 'onAudioEnd' || key === 'onAudioDiscard')
inputBusy = false;
notify();
}
return result;
};
}
const forwarded = args.slice();
forwarded[0] = decorated;
current = Reflect.apply(original, this, forwarded);
if (!dead && stamp === version) {
retireAudio();
audio = current;
inputBusy = false;
for (const key of ['play', 'clearPlayback', 'close'])
wrap(
current,
key,
() => {
if (audio !== current) return;
if (key === 'close') {
retireAudio();
inputBusy = false;
}
if (controller.audio === current) notify();
},
audioRestores,
);
}
return current;
}
controller.createAudio = createAudio;
restores.push(() => {
if (controller.createAudio === createAudio)
controller.createAudio = original;
});
}
}
function destroy() {
if (dead) return;
dead = true;
version++;
retireAudio();
pendingOutput.clear();
for (const restore of restores.splice(0).reverse()) restore();
signal?.removeEventListener('abort', destroy);
}
signal?.addEventListener('abort', destroy, { once: true });
if (signal?.aborted) destroy();
return { snapshot, handle, destroy };
}
+763
View File
@@ -0,0 +1,763 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceInactivity } from './inactivity.js';
import { RealtimeTurns } from '../realtimeTurns.js';
import { observeNativeActivity } from './nativeActivity.js';
import { createTestVoiceUi } from '../../testSupport/voiceUi.mjs';
// Deliberately small native-shaped fixtures exercise wrapper contracts, not devices.
function gemini(onChange = () => {}) {
const callbacks = [];
const input = {
pushToTalkKeyHeld: false,
setMicrophoneEnabled() {},
releasePushToTalkKey() {
this.pushToTalkKeyHeld = false;
},
resetSession() {
this.pushToTalkKeyHeld = false;
},
};
const c = {
input,
captureOwner: { admitted: false },
createAudio(opts) {
callbacks.push(opts);
return {
pending: 0,
play() {
this.pending++;
return 42;
},
clearPlayback() {
this.pending = 0;
},
close() {
this.pending = 0;
},
};
},
handleMessage() {},
};
const observer = observeNativeActivity({
adapter: { controller: c },
nativeProvider: 'gemini',
onChange,
});
const start = () => {
observer.handle({ type: 'state', state: 'connecting' });
c.audio = c.createAudio({ pushToTalk: true });
observer.handle({ type: 'state', state: 'listening' });
return c.audio;
};
return { c, input, observer, callbacks, start };
}
test('custom host never inspected, including hostile controller getter', () => {
const adapter = {
get controller() {
throw Error('must not inspect');
},
};
const o = observeNativeActivity({ adapter, nativeProvider: null });
assert.equal(o.snapshot().known, false);
o.handle({ type: 'state', state: 'listening' });
o.destroy();
});
test('PTT hold is direct before connect; silent automatic stream is not admitted speech', () => {
const f = gemini();
f.input.pushToTalkKeyHeld = true;
assert.equal(f.observer.snapshot().held, true);
assert.equal(f.observer.snapshot().connected, false);
f.start();
f.callbacks[0].onAudioStart({});
assert.equal(f.observer.snapshot().inputBusy, false);
f.c.captureOwner.admitted = true;
f.callbacks[0].onAudioStart({});
assert.equal(f.observer.snapshot().inputBusy, true);
f.callbacks[0].onAudioDiscard();
f.input.releasePushToTalkKey();
assert.equal(f.observer.snapshot().inputBusy, false);
assert.equal(f.observer.snapshot().held, false);
});
test('queue-to-drain, clear and close use current instance, without an acoustic claim', () => {
const f = gemini();
const a = f.start();
assert.equal(a.play(), 42);
assert.equal(f.observer.snapshot().outputBusy, true);
a.clearPlayback();
assert.equal(f.observer.snapshot().outputBusy, false);
a.play();
a.close();
assert.equal(f.observer.snapshot().outputBusy, false);
a.play();
a.pending = 0;
f.callbacks[0].onDrain();
assert.equal(f.observer.snapshot().outputBusy, false);
});
test('same-mount restart old callbacks cannot clear new admitted capture/output', () => {
const f = gemini();
f.start();
const old = f.callbacks[0];
const b = f.start();
f.c.captureOwner.admitted = true;
f.callbacks[1].onAudioStart();
b.play();
old.onAudioDiscard();
old.onDrain();
assert.equal(f.observer.snapshot().inputBusy, true);
assert.equal(f.observer.snapshot().outputBusy, true);
});
test('native callbacks original first with receiver, arguments, return and thrown identity; observation failure isolated', () => {
const f = gemini(() => {
throw Error('observer');
});
const receiver = {};
const argument = {};
let seen;
f.c.audio = f.c.createAudio({
pushToTalk: true,
onAudioEnd: function (...args) {
seen = [this, args];
return argument;
},
});
const cb = f.callbacks[0];
assert.equal(cb.onAudioEnd.call(receiver, argument), argument);
assert.equal(seen[0], receiver);
assert.equal(seen[1][0], argument);
const error = Error('same');
f.c.audio = f.c.createAudio({
onDrain() {
throw error;
},
});
assert.throws(
() => f.callbacks[1].onDrain(),
(e) => e === error,
);
});
test('reentrant Stop in original callback suppresses stale post-callback observation; teardown preserves newer replacement', () => {
let count = 0;
const f = gemini(() => count++);
f.c.audio = f.c.createAudio({
onDrain() {
f.observer.handle({ type: 'state', state: 'idle' });
},
});
const before = count;
f.callbacks[0].onDrain();
assert.equal(count, before + 1);
assert.equal(f.observer.snapshot().connected, false);
const newer = () => {};
f.input.releasePushToTalkKey = newer;
f.observer.destroy();
assert.equal(f.input.releasePushToTalkKey, newer);
});
test('OpenAI response.done does not drain output; absent lifecycle remains unknown', () => {
const turns = {
assistantAudioPlaying: false,
responseActive: false,
observeSpeech(e) {
if (e.type === 'output_audio_buffer.started')
this.assistantAudioPlaying = true;
if (
e.type === 'output_audio_buffer.stopped' ||
e.type === 'output_audio_buffer.cleared'
)
this.assistantAudioPlaying = false;
},
};
const c = { _turns: turns, _input: { pushToTalkKeyHeld: false } };
const original = turns.observeSpeech;
const o = observeNativeActivity({
adapter: { controller: c },
nativeProvider: 'openai',
});
o.handle({ type: 'state', state: 'listening' });
assert.equal(o.snapshot().known, false);
turns.observeSpeech({ type: 'output_audio_buffer.started' });
assert.equal(o.snapshot().known, true);
turns.responseActive = false;
o.handle({ type: 'completion' });
assert.equal(o.snapshot().outputBusy, true);
turns.observeSpeech({ type: 'output_audio_buffer.cleared' });
assert.equal(o.snapshot().outputBusy, false);
o.destroy();
assert.equal(turns.observeSpeech, original);
});
test('method receiver/args/result and throw are exact; abort restores wrappers', () => {
const abort = new AbortController();
const result = Promise.resolve();
let seen;
const error = Error('original');
const input = {
setMicrophoneEnabled(...args) {
seen = [this, args];
if (args[0] === 'throw') throw error;
return result;
},
};
const original = input.setMicrophoneEnabled;
const c = {
input,
createAudio() {
return {};
},
handleMessage() {},
};
const o = observeNativeActivity({
adapter: { controller: c },
nativeProvider: 'gemini',
signal: abort.signal,
onChange() {
throw Error('observe');
},
});
assert.equal(input.setMicrophoneEnabled(1, 2), result);
assert.equal(seen[0], input);
assert.deepEqual(seen[1], [1, 2]);
assert.throws(
() => input.setMicrophoneEnabled('throw'),
(e) => e === error,
);
abort.abort();
assert.equal(input.setMicrophoneEnabled, original);
assert.equal(o.snapshot().known, false);
});
test('options-method callback receiver maps to original options, explicit receiver is preserved', () => {
const f = gemini();
let receiver;
const options = {
onDrain() {
receiver = this;
return 17;
},
};
f.c.audio = f.c.createAudio(options);
assert.equal(f.callbacks[0].onDrain(), 17);
assert.equal(receiver, options);
const explicit = {};
f.callbacks[0].onDrain.call(explicit);
assert.equal(receiver, explicit);
});
test('actual native RealtimeTurns output lifecycle remains busy beyond generation', () => {
const turns = new RealtimeTurns({});
const observer = observeNativeActivity({
adapter: { controller: { _turns: turns, _input: {} } },
nativeProvider: 'openai',
});
observer.handle({ type: 'state', state: 'listening' });
turns.observeSpeech({ type: 'output_audio_buffer.started' });
turns.responseActive = false;
observer.handle({ type: 'completion' });
assert.equal(observer.snapshot().outputBusy, true);
turns.observeSpeech({ type: 'output_audio_buffer.stopped' });
assert.equal(observer.snapshot().outputBusy, false);
turns.observeSpeech({ type: 'response.created' });
turns.observeSpeech({ type: 'output_audio_buffer.stopped' });
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('verified clean native mount can arm initial idle, eventless response invalidates it', () => {
const turns = new RealtimeTurns({});
const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'openai',
});
let armed = 0,
cleared = 0;
const idle = createVoiceInactivity({
readActivity: observer.snapshot,
readPending: () => 0,
setTimer() {
armed++;
return 1;
},
clearTimer() {
cleared++;
},
});
assert.equal(observer.snapshot().known, true);
assert.equal(armed, 0);
controller.status = 'listening';
controller.dc = { readyState: 'open' };
observer.handle({ type: 'state', state: 'listening' });
idle.update();
assert.equal(armed, 1);
turns.observeSpeech({ type: 'response.created' });
idle.update();
assert.equal(observer.snapshot().known, false);
assert.equal(cleared, 1);
turns.responseActive = false;
observer.handle({ type: 'completion' });
idle.update();
assert.equal(armed, 1);
assert.equal(observer.snapshot().known, false);
controller.status = 'idle';
controller.dc = null;
turns.reset();
observer.handle({ type: 'state', state: 'idle' });
assert.equal(observer.snapshot().known, true);
idle.destroy();
observer.destroy();
});
test('attaching to already active native transport never assumes empty playback', () => {
const turns = new RealtimeTurns({});
for (const controller of [
{ status: 'listening', dc: {}, _turns: turns, _input: {} },
{ status: 'idle', dc: {}, _turns: turns, _input: {} },
]) {
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'openai',
});
assert.equal(observer.snapshot().known, false);
observer.destroy();
}
turns.responseActive = true;
const observer = observeNativeActivity({
adapter: {
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
},
nativeProvider: 'openai',
});
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('Gemini queue-only work blocks idle, queue drain rearms, expiry rereads delayed queue', () => {
const f = gemini();
f.start();
f.c.controlQueue = [];
let callback,
arms = 0,
clears = 0,
expired = 0;
const idle = createVoiceInactivity({
readActivity: f.observer.snapshot,
readPending: () => 0,
onExpire() {
expired++;
},
setTimer(fn) {
callback = fn;
return ++arms;
},
clearTimer() {
clears++;
},
});
assert.equal(arms, 1);
f.c.controlQueue.push({});
assert.equal(f.observer.snapshot().generationBusy, true);
idle.update();
assert.equal(clears, 1);
f.c.controlQueue.shift();
assert.equal(f.observer.snapshot().generationBusy, false);
idle.update();
assert.equal(arms, 2);
// A queue may change after the last event-driven snapshot and before the deadline.
f.c.controlQueue.push({});
callback();
assert.equal(expired, 0);
f.c.controlQueue = [];
idle.update();
assert.equal(arms, 3);
callback();
assert.equal(expired, 1);
idle.destroy();
f.observer.destroy();
});
test('Gemini presentation-only pending does not block idle; retired/deferred work still does', () => {
const f = gemini();
f.start();
f.c.userTurnPending = true;
assert.equal(f.observer.snapshot().generationBusy, false);
f.c.retiredReplyPending = true;
assert.equal(f.observer.snapshot().generationBusy, true);
f.c.retiredReplyPending = false;
f.c.deferredInput = {};
assert.equal(f.observer.snapshot().generationBusy, true);
f.c.deferredInput = null;
assert.equal(f.observer.snapshot().generationBusy, false);
f.observer.destroy();
});
test('actual Gemini startup phases keep playback observation and permit quiet open-mic expiry', async () => {
const { harness } = await import('../../testSupport/geminiNativeHarness.mjs');
let observer,
idle,
deadline,
expiries = 0;
const h = harness({ emit: (event) => observer?.handle(event) });
observer = observeNativeActivity({
adapter: { controller: h.c },
nativeProvider: 'gemini',
onChange: () => idle?.update(),
});
idle = createVoiceInactivity({
readActivity: observer.snapshot,
readPending: () => 0,
setTimer(fn) {
deadline = fn;
return fn;
},
clearTimer() {},
onExpire() {
expiries++;
},
});
try {
await h.c.start();
assert.equal(h.c.userTurnPending, true);
assert.equal(idle.snapshot().eligible, true);
const beforeOutput = deadline;
h.sockets[0].message({
serverContent: {
modelTurn: {
parts: [
{
inlineData: { data: 'output', mimeType: 'audio/pcm;rate=24000' },
},
],
},
turnComplete: true,
},
});
assert.equal(observer.snapshot().outputBusy, true);
assert.equal(idle.snapshot().eligible, false);
beforeOutput();
assert.equal(expiries, 0);
h.captures[0].drain();
assert.equal(observer.snapshot().outputBusy, false);
assert.equal(idle.snapshot().eligible, true);
deadline();
assert.equal(expiries, 1);
} finally {
idle.destroy();
observer.destroy();
h.c.stop({ removeUi: true });
}
});
test('native typed input observation preserves receiver/result, refuses failed submissions and retires old wrapper', () => {
let inputs = 0;
const controller = {
input: {},
handleMessage() {},
createAudio() {},
sendTextCommand(text) {
assert.equal(this, controller);
if (text === 'throw') throw new Error('rejected');
return text === 'reject' ? false : undefined;
},
};
const original = controller.sendTextCommand;
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'gemini',
onInput: () => inputs++,
});
const old = controller.sendTextCommand;
assert.equal(controller.sendTextCommand('typed request'), undefined);
controller.sendTextCommand('reject');
controller.sendTextCommand('');
assert.throws(() => controller.sendTextCommand('throw'), /rejected/);
assert.equal(inputs, 1);
observer.destroy();
assert.equal(controller.sendTextCommand, original);
old.call(controller, 'late');
assert.equal(inputs, 1);
});
test('explicit silent OpenAI completion rearms expiry, including cancelled and failed responses', () => {
for (const status of ['completed', 'cancelled', 'failed', 'incomplete']) {
const turns = new RealtimeTurns({});
const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
let idle;
let timer;
let expired = 0;
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'openai',
onChange: () => idle?.update(),
});
idle = createVoiceInactivity({
readActivity: observer.snapshot,
readPending: () => false,
setTimer(callback) {
timer = callback;
return 1;
},
clearTimer() {
timer = null;
},
onExpire: () => expired++,
});
controller.status = 'listening';
controller.dc = {};
observer.handle({ type: 'state', state: 'listening' });
assert.equal(typeof timer, 'function');
turns.responseActive = true;
turns.observeSpeech({
type: 'response.created',
response: { id: 'silent' },
});
assert.equal(timer, null);
turns.observeSpeech({
type: 'response.done',
response: {
id: 'silent',
status,
output:
status === 'completed'
? [{ type: 'function_call', name: 'set_panel_open' }]
: [],
},
});
// The provider updates generation state after forwarding the raw event.
assert.equal(timer, null);
turns.responseActive = false;
observer.handle({ type: 'completion' });
assert.equal(observer.snapshot().known, true);
assert.equal(typeof timer, 'function');
timer();
assert.equal(expired, 1);
idle.destroy();
observer.destroy();
}
});
test('silent OpenAI replies cannot clear unresolved older audio or incomplete output metadata', () => {
const turns = new RealtimeTurns({});
const observer = observeNativeActivity({
adapter: {
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
},
nativeProvider: 'openai',
});
const created = (id) =>
turns.observeSpeech({ type: 'response.created', response: { id } });
const done = (id, output) =>
turns.observeSpeech({
type: 'response.done',
response: { id, status: 'completed', output },
});
created('audio');
turns.observeSpeech({
type: 'response.output_audio.delta',
response_id: 'audio',
});
created('silent');
done('silent', []);
assert.equal(observer.snapshot().known, false);
done('audio', []); // Even contradictory completion metadata cannot erase audio evidence.
assert.equal(observer.snapshot().known, false);
turns.observeSpeech({
type: 'output_audio_buffer.started',
response_id: 'audio',
});
assert.equal(observer.snapshot().outputBusy, true);
done('silent', []);
assert.equal(observer.snapshot().outputBusy, true);
turns.observeSpeech({
type: 'output_audio_buffer.stopped',
response_id: 'audio',
});
assert.equal(observer.snapshot().outputBusy, false);
created('unknown');
done('unknown', undefined);
assert.equal(observer.snapshot().known, false);
done('unobserved', []);
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('overlapping silent replies resolve independently and audio output items remain unknown until playback', () => {
const turns = new RealtimeTurns({});
const observer = observeNativeActivity({
adapter: {
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
},
nativeProvider: 'openai',
});
for (const id of ['a', 'b'])
turns.observeSpeech({ type: 'response.created', response: { id } });
for (const id of ['a', 'b']) {
turns.observeSpeech({
type: 'response.done',
response: {
id,
status: 'completed',
output: [
{ type: 'message', content: [{ type: 'text', text: 'Done' }] },
],
},
});
assert.equal(observer.snapshot().known, id === 'b');
}
turns.observeSpeech({ type: 'response.created', response: { id: 'audio' } });
turns.observeSpeech({
type: 'response.done',
response: {
id: 'audio',
status: 'completed',
output: [{ type: 'message', content: [{ type: 'audio' }] }],
},
});
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('silent completion cannot establish output knowledge when observer attached mid-session', () => {
const turns = new RealtimeTurns({});
const observer = observeNativeActivity({
adapter: {
controller: { status: 'listening', dc: {}, _turns: turns, _input: {} },
},
nativeProvider: 'openai',
});
turns.observeSpeech({ type: 'response.created', response: { id: 'new' } });
turns.observeSpeech({
type: 'response.done',
response: { id: 'new', status: 'completed', output: [] },
});
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('Gemini restores each retired audio immediately across repeated same-mount restarts', () => {
const originals = new Map();
const controller = {
input: {},
createAudio() {
const audio = { pending: 0, play() {}, clearPlayback() {}, close() {} };
originals.set(audio, { ...audio });
return audio;
},
handleMessage() {},
};
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'gemini',
});
const assertRestored = (audio) => {
for (const key of ['play', 'clearPlayback', 'close'])
assert.equal(audio[key], originals.get(audio)[key]);
};
for (let i = 0; i < 5; i++) {
observer.handle({ type: 'state', state: 'connecting' });
controller.audio = controller.createAudio({});
const audio = controller.audio;
observer.handle({ type: 'state', state: 'listening' });
assert.notEqual(audio.close, originals.get(audio).close);
audio.close();
assertRestored(audio);
controller.audio = null;
observer.handle({ type: 'state', state: 'idle' });
}
controller.audio = controller.createAudio({});
const replaced = controller.audio;
controller.audio = controller.createAudio({});
assertRestored(replaced);
const current = controller.audio;
const replacement = () => {};
current.play = replacement;
observer.handle({ type: 'state', state: 'error' });
assert.equal(current.play, replacement);
assert.equal(current.close, originals.get(current).close);
observer.destroy();
});
test('actual OpenAI failed-response error/listening recovery preserves only proven no-audio knowledge', async (t) => {
const { GevRealtimeController } = await import('../realtimeController.js');
t.mock.method(console, 'error', () => {});
for (const [output, known] of [
[[], true],
[[{ type: 'message', content: [{ type: 'audio' }] }], false],
[undefined, false],
]) {
let observer, idle, timer;
const states = [];
let expired = 0;
const controller = new GevRealtimeController({
runner: async () => ({ ok: true }),
backend: {},
debugSink: null,
ui: createTestVoiceUi(),
onSessionEvent(event) {
if (event.type === 'state') states.push(event.state);
observer?.handle(event);
},
});
observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'openai',
onChange: () => idle?.update(),
});
idle = createVoiceInactivity({
readActivity: observer.snapshot,
readPending: () => false,
setTimer(callback) {
timer = callback;
return 1;
},
clearTimer() {
timer = null;
},
onExpire() {
expired++;
},
});
try {
controller.dc = {
readyState: 'open',
send() {},
close() {
this.readyState = 'closed';
},
};
controller.setStatus('listening', 'Ready');
const event = (payload) =>
controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
await event({
type: 'response.created',
response: { id: 'failed-reply' },
});
await event({
type: 'response.done',
response: {
id: 'failed-reply',
status: 'failed',
output,
status_details: { error: { message: 'Synthetic failed reply' } },
},
});
assert.deepEqual(states.slice(-2), ['error', 'listening']);
assert.equal(controller.dc.readyState, 'open');
assert.equal(observer.snapshot().known, known);
assert.equal(idle.snapshot().eligible, known);
if (known) {
timer();
assert.equal(expired, 1);
} else assert.equal(timer, null);
// Fatal teardown can preserve the error status; the next connection must
// still retire the old generation rather than inherit unresolved output.
controller.setStatus('error', 'Transport failed');
controller.stop({ preserveStatus: true });
controller.setStatus('connecting', 'Reconnecting');
assert.equal(observer.snapshot().connected, false);
assert.equal(observer.snapshot().known, true);
controller.stop();
assert.equal(observer.snapshot().connected, false);
assert.equal(observer.snapshot().known, true);
} finally {
idle.destroy();
observer.destroy();
controller.stop();
}
}
});
+189
View File
@@ -0,0 +1,189 @@
export const VOICE_PREVIEW_STORAGE_KEY = 'godsEyeView.voice.previewEnabled';
export const VOICE_TIMEOUT_STORAGE_KEY = 'godsEyeView.voice.inactivityMinutes';
const PRESETS = ['3', '5', '10', '15'];
function parseMinutes(value) {
if (typeof value !== 'number' && typeof value !== 'string') return undefined;
if (typeof value === 'string' && !/^\d+$/.test(value)) return undefined;
const number = Number(value);
return Number.isInteger(number) && number >= 0 && number <= 60
? number
: undefined;
}
/** Outer, browser-local live state. Construction never writes storage. */
export function createVoicePreferences({ storage } = {}) {
const listeners = new Set();
const target = () =>
storage === undefined ? globalThis.localStorage : storage;
const read = (key) => {
try {
return target()?.getItem?.(key);
} catch {
return undefined;
}
};
const rawMinutes = read(VOICE_TIMEOUT_STORAGE_KEY);
let state = {
previewEnabled: read(VOICE_PREVIEW_STORAGE_KEY) !== 'false',
minutes: rawMinutes === 'none' ? 0 : (parseMinutes(rawMinutes) ?? 5),
previewPersisted: null,
timeoutPersisted: null,
};
const snapshot = () => ({ ...state });
function apply(field, status, key, value, serialized) {
const changed = state[field] !== value;
let persisted = false;
state = { ...state, [field]: value };
try {
const destination = target();
if (
typeof destination?.setItem === 'function' &&
typeof destination?.getItem === 'function'
) {
destination.setItem(key, serialized);
persisted = destination.getItem(key) === serialized;
}
} catch {
/* The applied live value remains authoritative across mounts. */
}
state = { ...state, [status]: persisted };
const result = {
ok: true,
applied: true,
changed,
persisted,
...snapshot(),
};
for (const listener of [...listeners]) listener(snapshot());
return result;
}
const invalid = () => ({
ok: false,
applied: false,
changed: false,
persisted: null,
reason: 'invalid-value',
...snapshot(),
});
return {
snapshot,
setPreviewEnabled(value) {
return typeof value === 'boolean'
? apply(
'previewEnabled',
'previewPersisted',
VOICE_PREVIEW_STORAGE_KEY,
value,
String(value),
)
: invalid();
},
setMinutes(value) {
const parsed = parseMinutes(value);
return parsed === undefined
? invalid()
: apply(
'minutes',
'timeoutPersisted',
VOICE_TIMEOUT_STORAGE_KEY,
parsed,
parsed === 0 ? 'none' : String(parsed),
);
},
subscribe(listener) {
if (typeof listener !== 'function')
throw new TypeError('Preference listener required');
listeners.add(listener);
return () => listeners.delete(listener);
},
};
}
/** Mount-local controls, backed by an independently owned preference model. */
export function bindPreferenceControls({ ui, preferences, copy, onChange }) {
let disposed = false;
const initial = preferences.snapshot();
const displayed = parseMinutes(ui.state.preferences.timeoutMinutes);
let custom =
!PRESETS.includes(String(initial.minutes)) && initial.minutes > 0
? initial.minutes
: displayed > 0
? displayed
: 5;
// Every render republishes the fields, so a refused change shows the
// applied value again.
function render(state) {
if (disposed) return;
const value = String(state.minutes);
const customSelected = state.minutes > 0 && !PRESETS.includes(value);
if (customSelected) custom = state.minutes;
ui.update('preferences', {
previewChecked: state.previewEnabled,
// Functions, so a locale change words the notes again.
previewNote: () =>
`${copy.previewNote}${state.previewPersisted === false ? ` ${copy.notSaved}` : ''}`,
timeoutNote: () =>
`${copy.timeoutNote}${state.timeoutPersisted === false ? ` ${copy.notSaved}` : ''}`,
timeoutChoice:
state.minutes === 0 ? '0' : customSelected ? 'custom' : value,
timeoutMinutes: String(custom),
timeoutInputHidden: !customSelected,
revision: ui.state.preferences.revision + 1,
});
}
const unsubscribe = preferences.subscribe((state) => {
render(state);
onChange?.(state);
});
// A custom value may equal a preset: keep the person's explicit editor
// choice.
const showCustom = () =>
ui.update('preferences', {
timeoutChoice: 'custom',
timeoutMinutes: String(custom),
timeoutInputHidden: false,
revision: ui.state.preferences.revision + 1,
});
const offs = [
ui.on('preview-change', (checked) => {
if (!disposed && !preferences.setPreviewEnabled(checked).ok)
render(preferences.snapshot());
}),
ui.on('timeout-select', (value) => {
if (disposed) return;
if (value === 'custom') {
const parsed = parseMinutes(ui.state.preferences.timeoutMinutes);
if (parsed === undefined || parsed === 0) {
render(preferences.snapshot());
return;
}
custom = parsed;
preferences.setMinutes(parsed);
showCustom();
} else if (value === '0' || PRESETS.includes(value))
preferences.setMinutes(value);
else render(preferences.snapshot());
}),
ui.on('timeout-input', (value) => {
if (disposed) return;
const parsed = parseMinutes(value);
if (parsed === undefined || parsed === 0) {
render(preferences.snapshot());
return;
}
custom = parsed;
preferences.setMinutes(parsed);
showCustom();
}),
];
render(initial);
return {
dispose() {
if (disposed) return;
disposed = true;
unsubscribe();
for (const off of offs) off();
},
};
}
+276
View File
@@ -0,0 +1,276 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
createVoicePreferences,
bindPreferenceControls,
VOICE_PREVIEW_STORAGE_KEY as P,
VOICE_TIMEOUT_STORAGE_KEY as T,
} from './preferences.js';
import { createTestVoiceUi } from '../../testSupport/voiceUi.mjs';
function storage(entries = {}, mode = 'normal') {
const values = new Map(Object.entries(entries));
const writes = [];
return {
writes,
values,
getItem(key) {
if (mode === 'readthrow') throw Error('blocked');
return values.get(key) ?? null;
},
setItem(key, value) {
writes.push([key, value]);
if (mode === 'throw') throw Error('blocked');
if (mode !== 'noop' && mode !== 'readthrow')
values.set(key, mode === 'mismatch' ? `${value}!` : value);
},
};
}
// A panel control whose timeout field shows `custom` minutes, as a view
// would display before the binder renders.
function controls(custom = '7') {
const ui = createTestVoiceUi({ panel: true });
ui.update('preferences', { timeoutMinutes: custom });
return ui;
}
const fields = (ui) => ui.state.preferences;
const copy = {
previewNote: 'Preview local',
timeoutNote: 'Provider limits still apply',
notSaved: 'Not saved',
};
function bind(ui, preferences, onChange) {
return bindPreferenceControls({ ui, preferences, copy, onChange });
}
test('defaults and restore are passive; legacy None and stored custom restore', () => {
for (const [entries, minutes, previewEnabled] of [
[{}, 5, true],
[{ [P]: 'false', [T]: 'none' }, 0, false],
[{ [T]: '17' }, 17, true],
[{ [T]: 'broken' }, 5, true],
]) {
const disk = storage(entries);
const prefs = createVoicePreferences({ storage: disk });
assert.deepEqual(prefs.snapshot(), {
previewEnabled,
minutes,
previewPersisted: null,
timeoutPersisted: null,
});
const ui = controls();
bind(ui, prefs).dispose();
assert.deepEqual(disk.writes, []);
}
});
test('exact valid values, presets, None and serialized readback', () => {
const disk = storage();
const prefs = createVoicePreferences({ storage: disk });
for (const value of [3, 5, 10, 15, '1', '60', 0]) {
const result = prefs.setMinutes(value);
assert.equal(result.ok, true);
assert.equal(result.applied, true);
assert.equal(result.persisted, true);
assert.equal(result.minutes, Number(value));
assert.deepEqual(disk.writes.at(-1), [
T,
Number(value) === 0 ? 'none' : String(Number(value)),
]);
}
assert.equal(prefs.setPreviewEnabled(false).previewEnabled, false);
assert.deepEqual(disk.writes.at(-1), [P, 'false']);
});
test('fractional, malformed, coercible and out-of-range inputs never write or notify', () => {
const disk = storage();
const prefs = createVoicePreferences({ storage: disk });
let notices = 0;
prefs.subscribe(() => notices++);
const before = prefs.snapshot();
for (const value of [
null,
undefined,
true,
{},
[],
'',
' ',
' 3',
'3 ',
'3.0',
'1e1',
'0x10',
'none',
-1,
61,
3.2,
NaN,
Infinity,
]) {
const result = prefs.setMinutes(value);
assert.equal(result.ok, false, String(value));
assert.equal(result.applied, false);
assert.equal(result.changed, false);
assert.deepEqual(prefs.snapshot(), before);
}
for (const value of [0, 'true', null])
assert.equal(prefs.setPreviewEnabled(value).ok, false);
assert.equal(notices, 0);
assert.deepEqual(disk.writes, []);
});
for (const mode of ['throw', 'noop', 'mismatch', 'readthrow', 'missing']) {
test(`storage ${mode} preserves live value and exposes unsaved state across A → B`, () => {
const disk = mode === 'missing' ? null : storage({}, mode);
const prefs = createVoicePreferences({ storage: disk });
const a = controls();
const bindingA = bind(a, prefs);
a.emit('preview-change', false);
a.emit('timeout-select', 'custom');
a.emit('timeout-input', '19');
assert.deepEqual(prefs.snapshot(), {
previewEnabled: false,
minutes: 19,
previewPersisted: false,
timeoutPersisted: false,
});
bindingA.dispose();
const writeCount = disk?.writes.length;
const b = controls();
const bindingB = bind(b, prefs);
assert.equal(fields(b).previewChecked, false);
assert.equal(fields(b).timeoutMinutes, '19');
assert.match(fields(b).previewNote, /Not saved/);
assert.match(fields(b).timeoutNote, /Not saved/);
assert.equal(disk?.writes.length, writeCount);
a.emit('preview-change', true);
a.emit('timeout-select', '3');
assert.equal(prefs.snapshot().minutes, 19);
assert.equal(prefs.snapshot().previewEnabled, false);
bindingB.dispose();
});
}
test('same-value retry really writes and clears warnings after storage recovers', () => {
const disk = storage();
const original = disk.setItem;
disk.setItem = () => {
throw Error('blocked');
};
const prefs = createVoicePreferences({ storage: disk });
const ui = controls();
bind(ui, prefs);
prefs.setPreviewEnabled(false);
prefs.setMinutes(12);
assert.match(fields(ui).timeoutNote, /Not saved/);
disk.setItem = original;
const timeout = prefs.setMinutes(12);
const preview = prefs.setPreviewEnabled(false);
assert.equal(timeout.changed, false);
assert.equal(preview.changed, false);
assert.equal(timeout.persisted, true);
assert.equal(preview.persisted, true);
assert.deepEqual(disk.writes, [
[T, '12'],
[P, 'false'],
]);
assert.equal(fields(ui).timeoutNote, copy.timeoutNote);
assert.equal(fields(ui).previewNote, copy.previewNote);
});
test('Custom commits displayed valid value immediately, even when equal to preset', () => {
const disk = storage();
const prefs = createVoicePreferences({ storage: disk });
const ui = controls('10');
bind(ui, prefs);
assert.equal(disk.writes.length, 0);
ui.emit('timeout-select', 'custom');
assert.equal(prefs.snapshot().minutes, 10);
assert.equal(fields(ui).timeoutChoice, 'custom');
assert.equal(fields(ui).timeoutInputHidden, false);
assert.deepEqual(disk.writes, [[T, '10']]);
ui.emit('timeout-input', '23');
ui.emit('timeout-select', '5');
assert.equal(fields(ui).timeoutMinutes, '23');
assert.equal(fields(ui).timeoutInputHidden, true);
ui.emit('timeout-select', 'custom');
assert.equal(prefs.snapshot().minutes, 23);
assert.equal(fields(ui).timeoutMinutes, '23');
});
test('invalid control edits restore valid state without writes; None remains distinct', () => {
const disk = storage({ [T]: '17' });
const prefs = createVoicePreferences({ storage: disk });
const ui = controls();
bind(ui, prefs);
assert.equal(fields(ui).timeoutMinutes, '17');
for (const value of ['1.5', '', '61', '0']) {
// Each refused edit republishes the applied value, so the view writes it
// back over what was typed.
const revision = fields(ui).revision;
ui.emit('timeout-input', value);
assert.equal(fields(ui).timeoutMinutes, '17');
assert.ok(fields(ui).revision > revision);
assert.equal(prefs.snapshot().minutes, 17);
}
ui.emit('timeout-select', 'malformed');
assert.equal(fields(ui).timeoutChoice, 'custom');
assert.equal(disk.writes.length, 0);
ui.emit('timeout-select', '0');
assert.equal(prefs.snapshot().minutes, 0);
assert.equal(fields(ui).timeoutInputHidden, true);
// The field shows an invalid value when Custom is chosen.
ui.update('preferences', { timeoutMinutes: 'bad' });
ui.emit('timeout-select', 'custom');
assert.equal(fields(ui).timeoutChoice, '0');
assert.equal(prefs.snapshot().minutes, 0);
assert.deepEqual(disk.writes, [[T, 'none']]);
});
test('model snapshots cannot mutate state; unsubscribe and binder disposal release ownership', () => {
const prefs = createVoicePreferences({ storage: storage() });
const ui = controls();
// Count the binder's field listeners still attached.
const attached = new Set();
const on = ui.on;
ui.on = (name, listener) => {
const off = on(name, listener);
const entry = { name };
attached.add(entry);
return () => {
attached.delete(entry);
off();
};
};
let calls = 0;
const binding = bind(ui, prefs, () => calls++);
assert.deepEqual([...attached].map((entry) => entry.name).sort(), [
'preview-change',
'timeout-input',
'timeout-select',
]);
const snap = prefs.snapshot();
snap.minutes = 41;
assert.equal(prefs.snapshot().minutes, 5);
const unsub = prefs.subscribe((state) => {
state.minutes = 22;
});
prefs.setMinutes(11);
assert.equal(calls, 1);
assert.equal(prefs.snapshot().minutes, 11);
unsub();
binding.dispose();
binding.dispose();
assert.equal(attached.size, 0);
prefs.setMinutes(12);
assert.equal(calls, 1);
assert.equal(fields(ui).timeoutMinutes, '11');
// A disposed binder no longer answers its fields.
ui.emit('timeout-select', '3');
ui.emit('timeout-input', '9');
ui.emit('preview-change', false);
assert.equal(prefs.snapshot().minutes, 12);
assert.equal(prefs.snapshot().previewEnabled, true);
assert.equal(fields(ui).timeoutMinutes, '11');
});
+38
View File
@@ -0,0 +1,38 @@
// src/voice/panel/preferencesView.test.mjs — the rendered minutes field keeps
// a draft across renders and shows a refused edit's applied value again.
import '../../testSupport/registerDom.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
import {
bindPreferenceControls,
createVoicePreferences,
VOICE_TIMEOUT_STORAGE_KEY as T,
} from './preferences.js';
import { createTestVoiceUi } from '../../testSupport/voiceUi.mjs';
import { mountVoiceView } from '../../testSupport/voiceView.mjs';
test('a minutes draft survives unrelated renders; a refused edit is written back', async (t) => {
const { pinia, settle } = mountVoiceView(t);
const ui = createTestVoiceUi({ panel: true, pinia });
ui.update('preferences', { timeoutMinutes: '17' });
const prefs = createVoicePreferences({
storage: { getItem: (key) => (key === T ? '17' : null), setItem() {} },
});
bindPreferenceControls({
ui,
preferences: prefs,
copy: { previewNote: '', timeoutNote: 'Local', notSaved: 'Not saved' },
});
await settle();
const input = document.getElementById('gev-voice-timeout-input');
assert.equal(input.value, '17');
// Typing, before the field commits: the waveform renders every frame.
input.value = '2';
ui.update('status', { text: 'LISTENING' });
await settle();
assert.equal(input.value, '2');
input.value = '0';
input.dispatchEvent(new Event('change', { bubbles: true }));
await settle();
assert.equal(input.value, '17');
});
+22 -25
View File
@@ -1,6 +1,9 @@
import '../testSupport/registerDom.mjs';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import test from 'node:test';
import { nextTick } from 'vue';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import { mountVoiceView } from '../testSupport/voiceView.mjs';
import { GevRealtimeController } from './realtimeController.js';
import {
POINTER_OPEN_MIC_RECENT_MS,
@@ -63,16 +66,7 @@ function controllerFixture(pointer, { pushToTalk = false } = {}) {
const sent = [];
const events = [];
const controller = new GevRealtimeController({
ui: {
root: {
dataset: {},
classList: { remove() {} },
querySelectorAll: () => [],
},
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
},
ui: createTestVoiceUi(),
runner: async () => ({ ok: true }),
debugSink: null,
pointer,
@@ -299,20 +293,23 @@ test('card: the chip says what "this" or "here" resolved to and resets per turn'
assert.equal(unchanged, state, 'a stale pointer on an empty chip is a no-op');
});
test('markup: the chip sits under the user caption with a screen-reader separator', () => {
const control = readFileSync(
new URL('./control.js', import.meta.url),
'utf8',
);
const user = control.indexOf('id="gev-voice-card-user"');
const chip = control.indexOf('id="gev-voice-card-pointer"');
const assistant = control.indexOf('id="gev-voice-card-assistant"');
assert.ok(user < chip && chip < assistant);
assert.match(
control,
/id="gev-voice-card-pointer" class="gev-voice-card-pointer" hidden/,
);
assert.match(control, /<span class="gev-voice-card-sr">: <\/span>/);
test('markup: the chip sits under the user caption with a screen-reader separator', async (t) => {
const { pinia } = mountVoiceView(t);
const ui = createTestVoiceUi({ pinia });
t.after(() => ui.remove());
await nextTick();
const card = document.getElementById('gev-voice-card');
const order = [...card.children].map((node) => node.id);
const user = order.indexOf('gev-voice-card-user');
const chip = order.indexOf('gev-voice-card-pointer');
const assistant = order.indexOf('gev-voice-card-assistant');
assert.ok(user >= 0 && user < chip && chip < assistant);
const pointer = document.getElementById('gev-voice-card-pointer');
assert.equal(pointer.className, 'gev-voice-card-pointer');
assert.equal(pointer.hidden, true);
const separator = pointer.querySelector('.gev-voice-card-sr');
assert.equal(separator.textContent, ': ');
assert.equal(separator.previousElementSibling.id, 'gev-voice-card-pointer-kind');
});
test('deixis prompt lint: one short rule, and every sentinel it names exists in the schema', () => {
+69 -5
View File
@@ -22,15 +22,15 @@
import { isGlobeEngine } from '../globe/contract.js';
/** Movement below this many CSS pixels is still the same dwell. */
export const POINTER_DWELL_PX = 12;
const POINTER_DWELL_PX = 12;
/** Stillness that makes a dwell point. */
export const POINTER_DWELL_MS = 250;
/** A resting pointer stays a referent this long after it last moved. */
export const POINTER_FRESH_MS = 30000;
/** Pixels from the canvas edge that count as "parked", never a referent. */
export const POINTER_EDGE_PX = 8;
const POINTER_EDGE_PX = 8;
/** A dwell pick is reused by a snapshot only while this recent (contacts move). */
export const POINTER_DWELL_REUSE_MS = 1000;
const POINTER_DWELL_REUSE_MS = 1000;
/**
* Open mic has no gesture, so the pointer only counts when it moved or
* settled on the map this recently: a cursor resting there for minutes is
@@ -83,7 +83,7 @@ export function cameraPoseKey(globe) {
}
/** Bound and de-control a label before it goes to the model or the card. */
export function pointerLabel(value, max = LABEL_MAX) {
function pointerLabel(value, max = LABEL_MAX) {
const clean = String(value ?? '')
.replace(/[\p{Cc}\p{Cf}]+/gu, ' ')
.replace(/\s+/g, ' ')
@@ -189,6 +189,63 @@ export function pointerChip(snapshot) {
};
}
/** Dedupe key of the clearing pointer_context item. */
export const POINTER_NONE_KEY = 'none';
/**
* The voice-card event for a turn's pointer: a gesture reveals its target at
* once; open-mic and typed turns reveal it only through announcePointer.
* @param {object|null} snapshot
* @param {boolean} gesture
* @returns {{announced: object|null, pointer: object|null}}
*/
export function pointerTurnEvent(snapshot, gesture) {
const chip = gesture ? pointerChip(snapshot) : null;
return chip
? { announced: snapshot, pointer: { ...chip, screenPx: snapshot.screenPx } }
: { announced: null, pointer: null };
}
/**
* The pointer_context item a turn should send, or null when there is nothing
* new. Without a gesture the model learns only what kind of thing is pointed
* at, never which. Losing a sent pointer produces one clearing item.
* @param {object|null} snapshot
* @param {{at: string, gesture: boolean, sentKey: string|null}} turn
* @returns {{item: object, key: string}|null}
*/
export function pointerTurnContext(snapshot, { at, gesture, sentKey }) {
const chip = pointerChip(snapshot);
const item = !chip
? sentKey && sentKey !== POINTER_NONE_KEY
? { type: 'pointer_context', at, target: 'none' }
: null
: gesture
? pointerContextItem(snapshot)
: {
type: 'pointer_context',
at,
pointing: snapshot.entity?.kind || 'ground',
};
if (!item) return null;
const { at: _at, ...content } = item;
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
return key === sentKey ? null : { item, key };
}
/**
* The voice-card event revealing a pointer a tool resolved, once per snapshot.
* @param {object|null} snapshot
* @param {object|null} announced Snapshot already shown.
* @returns {object|null}
*/
export function pointerAnnouncement(snapshot, announced) {
const chip = pointerChip(snapshot);
return chip && snapshot !== announced
? { ...chip, screenPx: snapshot.screenPx }
: null;
}
/**
* Track the pointer over one element (the globe canvas).
* @param {object} options
@@ -337,7 +394,14 @@ export function createPointerTracker({
* (open mic only when it moved recently: `recentMs`). The result becomes
* the turn's referent (null when stale).
*/
beginTurn(at, { recentMs = POINTER_FRESH_MS } = {}) {
beginTurn(at, { recentMs = POINTER_FRESH_MS, capturedSnapshot } = {}) {
// Delayed admitted audio keeps its capture's pointer without replacing a
// newer physical hold. Undefined preserves existing caller behavior.
if (capturedSnapshot !== undefined) {
const snapshot = destroyed ? { fresh: false, at } : capturedSnapshot;
active = snapshot?.fresh ? snapshot : null;
return snapshot;
}
const reuse =
at !== 'text' && held && now() - held.heldAt <= POINTER_HOLD_MS;
const snapshot = reuse ? held.snapshot : capture(at, { recentMs });
+112
View File
@@ -6,8 +6,13 @@ import {
POINTER_HOLD_MS,
buildPointerSnapshot,
createPointerTracker,
POINTER_NONE_KEY,
formatPointerContext,
pointerAnnouncement,
pointerChip,
pointerContextItem,
pointerTurnContext,
pointerTurnEvent,
} from './pointerContext.js';
function harness({ pick } = {}) {
@@ -151,6 +156,51 @@ test('destroy removes listeners and ignores later moves', () => {
assert.equal(h.tracker.capture('keydown').fresh, false);
});
test('delayed speech uses its released capture without replacing a newer physical hold', () => {
const h = harness();
h.move(700, 250);
const capturedA = h.tracker.hold();
h.tracker.releaseHold();
h.move(200, 100);
h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA });
assert.equal(h.tracker.activeSnapshot(), capturedA);
const heldB = h.tracker.hold();
h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA });
assert.equal(h.tracker.activeSnapshot(), capturedA);
assert.equal(h.tracker.isHeld(), true);
assert.equal(h.tracker.beginTurn('speech_start'), heldB);
h.tracker.destroy();
});
test('explicit stale captures clear the referent while a silent hold leaves it alone', () => {
const h = harness();
h.move(700, 250);
const current = h.tracker.beginTurn('text');
h.move(200, 100);
h.tracker.hold();
h.tracker.releaseHold();
assert.equal(h.tracker.activeSnapshot(), current);
for (const capturedSnapshot of [null, { fresh: false, at: 'keydown' }]) {
h.tracker.beginTurn('text');
h.tracker.beginTurn('speech_start', { capturedSnapshot });
assert.equal(h.tracker.activeSnapshot(), null);
}
h.tracker.destroy();
});
test('a capture arriving after destruction cannot restore an active referent', () => {
const h = harness();
h.move(700, 250);
const capturedSnapshot = h.tracker.hold();
assert.equal(capturedSnapshot.fresh, true);
h.tracker.destroy();
const result = h.tracker.beginTurn('speech_start', { capturedSnapshot });
assert.equal(result.fresh, false);
assert.equal(h.tracker.activeSnapshot(), null);
assert.equal(h.tracker.isHeld(), false);
assert.equal(h.tracker.beginTurn('speech_start').fresh, false);
});
test('format: the model item is compact inert JSON; labels are bounded data', () => {
const snapshot = buildPointerSnapshot({
at: 'speech_start',
@@ -182,3 +232,65 @@ test('chip: "this" names an entity, "here" gives the ground coordinate', () => {
assert.deepEqual(pointerChip(ground), { kind: 'here', label: '30.27, -97.74' });
assert.equal(pointerChip(null), null);
});
test('pointer turn helpers reveal gesture targets, describe implicit ones by kind and dedupe repeats', () => {
const snapshot = {
fresh: true,
at: 'keydown',
entity: { kind: 'aircraft', layerId: 'flights', label: 'UAL1' },
lat: 10,
lon: 20,
screen: { x: 5, y: 6 },
screenPx: { x: 5, y: 6 },
};
const shown = pointerTurnEvent(snapshot, true);
assert.equal(shown.announced, snapshot);
assert.deepEqual(shown.pointer, {
kind: 'this',
label: 'UAL1',
screenPx: { x: 5, y: 6 },
});
assert.deepEqual(pointerTurnEvent(snapshot, false), {
announced: null,
pointer: null,
});
const gesture = pointerTurnContext(snapshot, {
at: 'keydown',
gesture: true,
sentKey: null,
});
assert.deepEqual(gesture.item, pointerContextItem(snapshot));
assert.equal(
pointerTurnContext(snapshot, {
at: 'keydown',
gesture: true,
sentKey: gesture.key,
}),
null,
);
assert.deepEqual(
pointerTurnContext(snapshot, { at: 'text', gesture: false, sentKey: null })
.item,
{ type: 'pointer_context', at: 'text', pointing: 'aircraft' },
);
const cleared = pointerTurnContext(null, {
at: 'text',
gesture: false,
sentKey: gesture.key,
});
assert.deepEqual(cleared, {
item: { type: 'pointer_context', at: 'text', target: 'none' },
key: POINTER_NONE_KEY,
});
for (const sentKey of [null, POINTER_NONE_KEY])
assert.equal(
pointerTurnContext(null, { at: 'text', gesture: false, sentKey }),
null,
);
assert.deepEqual(pointerAnnouncement(snapshot, null), shown.pointer);
assert.equal(pointerAnnouncement(snapshot, snapshot), null);
assert.equal(pointerAnnouncement(null, null), null);
});
+1 -1
View File
@@ -4,7 +4,7 @@ import { cameraPoseKey } from './pointerContext.js';
import * as defaultAnnotationResolver from '../annotations/annotationResolver.js';
/** "Around here" radius as a share of camera height, clamped to sane bounds. */
export function pointerRadiusKm(cameraHeightM) {
function pointerRadiusKm(cameraHeightM) {
const heightKm = Number(cameraHeightM) / 1000;
if (!Number.isFinite(heightKm) || heightKm <= 0) return 50;
return Math.round(Math.min(400, Math.max(5, heightKm * 0.75)));
+31 -30
View File
@@ -3,18 +3,22 @@
* live so the user sees what was understood before the answer.
*
* It is a fixed-position overlay (pointer-events: none) placed from the
* snapshot's canvas pixels, so it never touches the Cesium scene. The reticle
* marks where the user pointed, not a tracked world position: it leaves as
* soon as a tool moves the camera, when the reply completes, when the session
* ends, and after {@link RETICLE_MAX_MS} at most. Clean-UI and recording mode
* hide it through CSS.
* snapshot's canvas pixels, so it never touches the Cesium scene. It is
* presented to the pointer reticle store, which the command dock renders.
* The reticle marks where the user pointed, not a tracked world position: it
* leaves as soon as a tool moves the camera, when the reply completes, when
* the session ends, and after {@link RETICLE_MAX_MS} at most. Clean-UI and
* recording mode hide it through CSS.
*/
import { usePointerReticleStore } from '../ui/stores/pointerReticle.js';
import { applicationPinia } from '../ui/vueRuntime.js';
export const RETICLE_MAX_MS = 10000;
export const RETICLE_LINGER_MS = 1500;
/** Tools that move the camera away from the pointed spot. */
export const CAMERA_TOOLS = new Set([
const CAMERA_TOOLS = new Set([
'fly_to_location',
'track_entity',
'zoom_to_globe',
@@ -26,23 +30,28 @@ export const CAMERA_TOOLS = new Set([
'control_cockpit',
]);
// Keys each reticle's element in the view.
let reticleKeys = 0;
export class PointerReticle {
/**
* @param {object} options
* @param {Document} options.doc Document that owns the overlay.
* @param {() => {left: number, top: number}|null} options.rectOf Canvas
* client rect, read at show time.
* @param {object} [options.store] The pointer reticle store.
* @param {Function} [options.setTimer]
* @param {Function} [options.clearTimer]
*/
constructor({
doc,
rectOf,
store = usePointerReticleStore(applicationPinia()),
setTimer = (fn, ms) => setTimeout(fn, ms),
clearTimer = (id) => clearTimeout(id),
}) {
Object.assign(this, { doc, rectOf, setTimer, clearTimer });
this.node = null;
Object.assign(this, { rectOf, store, setTimer, clearTimer });
this.key = ++reticleKeys;
// What this reticle last presented; null until it first shows.
this.view = null;
this.timer = null;
// Tools running in this turn, keyed by call id (name when absent).
this.running = new Set();
@@ -50,17 +59,6 @@ export class PointerReticle {
this.destroyed = false;
}
ensureNode() {
if (this.node || !this.doc?.createElement) return this.node;
const node = this.doc.createElement('div');
node.className = 'gev-pointer-reticle';
node.setAttribute?.('aria-hidden', 'true');
node.hidden = true;
this.doc.body?.appendChild?.(node);
this.node = node;
return node;
}
schedule(ms) {
this.cancelTimer();
this.timer = this.setTimer(() => {
@@ -79,12 +77,14 @@ export class PointerReticle {
show(pointer) {
if (this.destroyed || !pointer?.screenPx) return;
const rect = this.rectOf?.();
const node = this.ensureNode();
if (!rect || !node) return;
node.style.left = `${Math.round(rect.left + pointer.screenPx.x)}px`;
node.style.top = `${Math.round(rect.top + pointer.screenPx.y)}px`;
node.dataset.target = pointer.kind === 'this' ? 'entity' : 'ground';
node.hidden = false;
if (!rect) return;
this.view = {
visible: true,
left: Math.round(rect.left + pointer.screenPx.x),
top: Math.round(rect.top + pointer.screenPx.y),
target: pointer.kind === 'this' ? 'entity' : 'ground',
};
this.store.publish(this, this.view);
this.visible = true;
this.running.clear();
this.schedule(RETICLE_MAX_MS);
@@ -95,7 +95,8 @@ export class PointerReticle {
this.running.clear();
if (!this.visible) return;
this.visible = false;
if (this.node) this.node.hidden = true;
this.view = { ...this.view, visible: false };
this.store.publish(this, this.view);
}
/** Follow provider-neutral session events. */
@@ -129,7 +130,7 @@ export class PointerReticle {
if (this.destroyed) return;
this.hide();
this.destroyed = true;
this.node?.remove?.();
this.node = null;
if (this.view) this.store.publish(this, null);
this.view = null;
}
}
+43 -38
View File
@@ -5,34 +5,27 @@ import { PointerReticle, RETICLE_LINGER_MS, RETICLE_MAX_MS } from './pointerReti
const css = readStylesheet(new URL('../../style.css', import.meta.url));
function fakeDoc() {
const appended = [];
function fakeStore() {
return {
appended,
body: { appendChild: (node) => appended.push(node) },
createElement: (tag) => ({
tag,
hidden: false,
style: {},
dataset: {},
attributes: {},
setAttribute(name, value) {
this.attributes[name] = value;
},
removed: false,
remove() {
this.removed = true;
},
}),
reticles: new Map(),
publishes: 0,
get reticle() {
return [...this.reticles.values()].at(-1) ?? null;
},
publish(owner, view) {
if (view) this.reticles.set(owner, view);
else this.reticles.delete(owner);
this.publishes += 1;
},
};
}
function harness() {
const doc = fakeDoc();
const store = fakeStore();
const timers = new Map();
let next = 1;
const reticle = new PointerReticle({
doc,
store,
rectOf: () => ({ left: 10, top: 20 }),
setTimer: (fn, ms) => {
const id = next++;
@@ -46,31 +39,26 @@ function harness() {
timers.delete(id);
timer.fn();
};
return { doc, timers, reticle, fire };
return { store, timers, reticle, fire };
}
const pointer = { kind: 'this', label: 'UPS793', screenPx: { x: 600, y: 200 } };
test('reticle: shows at the pointed pixel and caps its lifetime', () => {
const { doc, timers, reticle, fire } = harness();
const { store, timers, reticle, fire } = harness();
assert.equal(store.reticle, null);
reticle.handle({ type: 'pointer', pointer });
const node = doc.appended[0];
assert.equal(node.className, 'gev-pointer-reticle');
assert.equal(node.attributes['aria-hidden'], 'true');
assert.equal(node.hidden, false);
assert.equal(node.style.left, '610px');
assert.equal(node.style.top, '220px');
assert.equal(node.dataset.target, 'entity');
assert.deepEqual(store.reticle, { visible: true, left: 610, top: 220, target: 'entity' });
assert.equal([...timers.values()][0].ms, RETICLE_MAX_MS);
fire();
assert.equal(node.hidden, true);
assert.deepEqual(store.reticle, { visible: false, left: 610, top: 220, target: 'entity' });
});
test('reticle: leaves when a tool moves the camera, lingers after a reply', () => {
const { doc, timers, reticle, fire } = harness();
const { store, timers, reticle, fire } = harness();
reticle.handle({ type: 'pointer', pointer: { ...pointer, kind: 'here' } });
const node = doc.appended[0];
assert.equal(node.dataset.target, 'ground');
const hidden = () => !store.reticle.visible;
assert.equal(store.reticle.target, 'ground');
reticle.handle({ type: 'action-call', name: 'analyst_query', callId: 'c1' });
reticle.handle({ type: 'action-call', name: 'analyst_query', callId: 'c2' });
reticle.handle({ type: 'action-result', name: 'analyst_query', callId: 'c1' });
@@ -80,19 +68,36 @@ test('reticle: leaves when a tool moves the camera, lingers after a reply', () =
reticle.handle({ type: 'completion' });
assert.equal([...timers.values()][0].ms, RETICLE_LINGER_MS);
fire();
assert.equal(node.hidden, true);
assert.equal(hidden(), true);
reticle.handle({ type: 'pointer', pointer });
reticle.handle({ type: 'action-call', name: 'track_entity' });
assert.equal(node.hidden, true, 'camera tools hide it at once');
assert.equal(hidden(), true, 'camera tools hide it at once');
reticle.handle({ type: 'pointer', pointer });
reticle.handle({ type: 'state', state: 'idle' });
assert.equal(node.hidden, true, 'session end hides it');
assert.equal(hidden(), true, 'session end hides it');
assert.equal(timers.size, 0);
reticle.destroy();
reticle.destroy();
assert.equal(node.removed, true);
assert.equal(store.reticle, null, 'destroy withdraws the reticle');
const publishes = store.publishes;
reticle.handle({ type: 'pointer', pointer });
assert.equal(doc.appended.length, 1, 'destroyed reticle ignores events');
assert.equal(store.publishes, publishes, 'destroyed reticle ignores events');
});
test('reticle: each reticle keeps its own point', () => {
const store = fakeStore();
const rectOf = () => ({ left: 0, top: 0 });
const older = new PointerReticle({ store, rectOf, setTimer: () => 1, clearTimer() {} });
const newer = new PointerReticle({ store, rectOf, setTimer: () => 2, clearTimer() {} });
older.handle({ type: 'pointer', pointer });
newer.handle({ type: 'pointer', pointer: { ...pointer, screenPx: { x: 5, y: 6 } } });
newer.handle({ type: 'state', state: 'idle' });
assert.equal(store.reticles.get(older).visible, true, 'hiding the newer one leaves the older one');
newer.destroy();
assert.deepEqual([...store.reticles.keys()], [older]);
assert.notEqual(older.key, newer.key);
older.destroy();
assert.equal(store.reticles.size, 0);
});
test('reticle css: overlay is inert and hidden in Clean-UI and recording', () => {
+83
View File
@@ -0,0 +1,83 @@
// src/voice/pointerReticleView.test.mjs — the command dock renders each
// voice pointer reticle where its PointerReticle presents it.
import '../testSupport/registerDom.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
import { createApp, nextTick } from 'vue';
import { createPinia } from 'pinia';
import { usePointerReticleStore } from '../ui/stores/pointerReticle.js';
import CommandDockView from '../ui/views/CommandDockView.vue';
import { PointerReticle } from './pointerReticle.js';
test('the reticle shows at the pointed pixel, hides, and leaves on destroy', async (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
document.body.replaceChildren();
const pinia = createPinia();
const app = createApp(CommandDockView).use(pinia);
app.mount(document.body.appendChild(document.createElement('div')));
t.after(() => app.unmount());
const node = () => document.querySelector('.gev-pointer-reticle');
const reticle = new PointerReticle({
rectOf: () => ({ left: 10, top: 20 }),
store: usePointerReticleStore(pinia),
});
await nextTick();
assert.equal(node(), null, 'nothing until the first point');
reticle.handle({
type: 'pointer',
pointer: { kind: 'this', screenPx: { x: 600, y: 200 } },
});
await nextTick();
const element = node();
assert.ok(element.parentElement === document.body);
assert.equal(element.getAttribute('aria-hidden'), 'true');
assert.equal(element.hidden, false);
assert.equal(element.style.left, '610px');
assert.equal(element.style.top, '220px');
assert.equal(element.dataset.target, 'entity');
reticle.handle({ type: 'state', state: 'idle' });
await nextTick();
assert.ok(node() === element, 'hiding keeps the element');
assert.equal(element.hidden, true);
reticle.handle({
type: 'pointer',
pointer: { kind: 'here', screenPx: { x: 1, y: 2 } },
});
await nextTick();
assert.equal(element.hidden, false);
assert.equal(element.dataset.target, 'ground');
reticle.destroy();
await nextTick();
assert.equal(node(), null);
});
test('two reticles show two elements', async (t) => {
document.body.replaceChildren();
const pinia = createPinia();
const app = createApp(CommandDockView).use(pinia);
app.mount(document.body.appendChild(document.createElement('div')));
t.after(() => app.unmount());
const store = usePointerReticleStore(pinia);
const rectOf = () => ({ left: 0, top: 0 });
const one = new PointerReticle({ rectOf, store });
const two = new PointerReticle({ rectOf, store });
t.after(() => {
one.destroy();
two.destroy();
});
const at = (x) => ({
type: 'pointer',
pointer: { kind: 'this', screenPx: { x, y: 0 } },
});
one.handle(at(1));
two.handle(at(2));
await nextTick();
const lefts = () =>
[...document.querySelectorAll('.gev-pointer-reticle')].map(
(node) => node.style.left,
);
assert.deepEqual(lefts(), ['1px', '2px']);
two.destroy();
await nextTick();
assert.deepEqual(lefts(), ['1px']);
});
+21
View File
@@ -0,0 +1,21 @@
export const VOICE_PROVIDERS = Object.freeze({
openai: Object.freeze({ label: 'OpenAI' }), // i18n-ignore: provider product name
gemini: Object.freeze({ label: 'Gemini' }), // i18n-ignore: provider product name
});
/** Optional providers require an explicit host opt-in and an available factory. */
export function supportedVoiceProviders(
requested = ['openai'],
factories = {},
) {
if (!Array.isArray(requested))
throw new TypeError('supportedProviders must be an array');
const selected = [...new Set(requested)].filter(
(name) =>
Object.hasOwn(VOICE_PROVIDERS, name) &&
typeof factories[name] === 'function',
);
if (!selected.length)
throw new Error('No supported voice provider is configured');
return Object.freeze(selected);
}
+162
View File
@@ -0,0 +1,162 @@
import { supportedVoiceProviders } from './providerCapabilities.js';
const STORAGE_KEY = 'gev.voice.provider';
export function readVoiceProvider(storage, supported = ['openai', 'gemini']) {
let saved;
try {
saved = (storage ?? globalThis.localStorage)?.getItem(STORAGE_KEY);
} catch {
/* Storage is optional. */
}
return supported.includes(saved)
? saved
: supported.includes('openai')
? 'openai'
: supported[0];
}
/** Keep one public controller identity while replacing each provider's lifetime. */
export function createProviderCommands(options, { bind, factories, storage }) {
const supportedProviders = supportedVoiceProviders(
options.supportedProviders ?? Object.keys(factories),
factories,
);
window.__gevVoiceCommands?.stop?.({ removeUi: true });
let provider = readVoiceProvider(storage, supportedProviders);
let current;
let disposed = false;
const lifetime = new AbortController();
let mountGeneration = 0;
let voiceEnabled = false;
const stored = () => {
try {
return storage ?? globalThis.localStorage;
} catch {
return null;
}
};
const stop = (settings = {}) => {
if (disposed) return;
voiceEnabled = false;
if (settings.removeUi) {
disposed = true;
options.signal?.removeEventListener('abort', destroy);
}
try {
current?.session.stop(settings);
} finally {
if (settings.removeUi) {
lifetime.abort();
options.runner.dispose?.();
}
}
};
const destroy = () => stop({ removeUi: true });
const selectProvider = (next) => {
if (disposed || !supportedProviders.includes(next) || next === provider)
return false;
const resume = voiceEnabled && current.session.isActive();
// Carry the input mode, never a held key, microphone or conversation.
const pushToTalk = Boolean(
current.session.adapter.getStartOptions?.().pushToTalk,
);
const generation = ++mountGeneration;
voiceEnabled = false;
current.session.destroy();
// Disposal suppresses the retired adapter's idle event. Clear the shared
// conversation explicitly before the new mount can use these owners.
if (disposed || generation !== mountGeneration) return false;
options.pointer?.clear();
options.referents?.clear();
options.runner.resetConversation?.();
if (disposed || generation !== mountGeneration) return false;
provider = next;
try {
stored()?.setItem(STORAGE_KEY, provider);
} catch {
/* Storage is optional. */
}
mount();
if (resume && !disposed && provider === next)
void current.session.start({ pushToTalk });
// The new control renders first; focus stays on the provider choice.
void current.ui?.focus?.(['provider'], { anyVisibility: true });
return true;
};
const handleSessionEvent = (event) => {
if (event?.type === 'state' && event.state === 'connecting')
voiceEnabled = true;
if (event?.type === 'state' && ['idle', 'error'].includes(event.state)) {
voiceEnabled = false;
}
// The host still observes the mounted provider's neutral session events.
options.onSessionEvent?.(event);
};
const sendTextCommand = (text) => {
if (disposed) return false;
return current.session.sendText(text);
};
const api = {
// The shared action surface is independent of an active voice connection.
runner: options.runner,
supportedProviders,
signal: lifetime.signal,
get provider() {
return provider;
},
get session() {
return current?.session;
},
get status() {
return current?.status ?? current?.session.state ?? 'idle';
},
get state() {
return current?.session.state ?? 'idle';
},
get disposed() {
return disposed;
},
start: (settings) => !disposed && current.session.start(settings),
stop,
destroy,
isActive: () => !disposed && current.session.isActive(),
sendTextCommand,
sendText: sendTextCommand,
notifyMapEvent: (event) => !disposed && current.session.sendMapEvent(event),
};
// Existing diagnostic and controller accessors continue to reflect the active provider.
const facade = new Proxy(api, {
get(target, name, receiver) {
if (Reflect.has(target, name)) return Reflect.get(target, name, receiver);
const value = current?.[name];
return typeof value === 'function' ? value.bind(current) : value;
},
});
function mount() {
voiceEnabled = false;
const generation = ++mountGeneration;
current = bind({
...options,
signal: lifetime.signal,
resetExisting: false,
disposeRunner: false,
provider,
supportedProviders,
onProviderChange: selectProvider,
onSessionEvent: (event) => {
if (!disposed && generation === mountGeneration)
handleSessionEvent(event);
},
createSession: factories[provider],
});
window.__gevVoiceCommands = facade;
}
mount();
options.signal?.addEventListener('abort', destroy, { once: true });
if (options.signal?.aborted) destroy();
return facade;
}
+860
View File
@@ -0,0 +1,860 @@
import { createCesiumEngine } from '../globe/cesium/createCesiumEngine.js';
// Imports intentionally target the candidate, never the immutable reference.
import test from 'node:test';
import assert from 'node:assert/strict';
import {
createProviderCommands,
readVoiceProvider,
} from './providerCommands.js';
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
import { createVoiceCommands as createCommands } from './commands.js';
import { createGevActionRunner } from './gevActions.js';
import { createPointerTracker } from './pointerContext.js';
import { createReferentRegistry } from './referents.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
const PROVIDER_KEY = 'gev.voice.provider';
const settle = () => new Promise((resolve) => setImmediate(resolve));
function deferred() {
let resolve;
let reject;
const promise = new Promise((yes, no) => {
resolve = yes;
reject = no;
});
return { promise, resolve, reject };
}
// A real control whose listeners, removals and provider focus are counted.
// Removal is counted but keeps the control, so an interaction after teardown
// still reaches any listener the binder failed to release.
function countedControl(options, onRemove = () => {}) {
const ui = createTestVoiceUi(options);
const counts = new Map();
const on = ui.on;
ui.on = (name, listener) => {
counts.set(name, (counts.get(name) || 0) + 1);
const off = on(name, listener);
let live = true;
return () => {
if (live) counts.set(name, counts.get(name) - 1);
live = false;
off();
};
};
ui.listenerCount = (name) => counts.get(name) || 0;
ui.removeCount = 0;
ui.remove = () => {
ui.removeCount++;
onRemove();
};
ui.focusCount = 0;
return ui;
}
for (const custom of [false, true]) {
test(`public composition preserves the ${custom ? 'custom adapter' : 'default OpenAI adapter'} path`, async (t) => {
const prior = globalThis.window;
globalThis.window = new EventTarget();
let controls;
t.after(() => {
controls?.session.destroy();
globalThis.window = prior;
});
const ui = countedControl();
let defaultCreated = 0;
let customCreated = 0;
const options = {
runner: async () => ({ ok: true }),
createControl: () => ui,
createController: ({ onSessionEvent }) => {
defaultCreated++;
return {
ui,
start({ pushToTalk }) {
this.pushToTalkMode = pushToTalk;
onSessionEvent({ type: 'state', state: 'listening' });
},
stop() {},
syncCostUi() {},
bindPushToTalkShortcut() {},
sendTextCommand: (text) => `default:${text}`,
};
},
};
if (custom) {
options.supportedProviders = ['gemini'];
options.createSession = ({ emit }) => {
customCreated++;
return {
capabilities: { pushToTalk: false },
start() {
emit({ type: 'state', state: 'listening' });
},
stop() {},
sendText: (text) => `custom:${text}`,
sendMapEvent: () => true,
};
};
}
controls = createCommands(options);
await controls.session.start({ pushToTalk: true });
assert.equal(defaultCreated, custom ? 0 : 1);
assert.equal(customCreated, custom ? 1 : 0);
assert.equal(
controls.session.sendText('test'),
`${custom ? 'custom' : 'default'}:test`,
);
assert.equal(ui.state.provider.hidden, true);
assert.equal(ui.state.root.providerChoice, 'false');
if (!custom)
assert.deepEqual(controls.session.adapter.getStartOptions(), {
pushToTalk: true,
});
});
}
/** Real facade -> real binder -> real common session and real action runner.
* Adapters and passive UI/viewer boundaries are fixtures. Wrappers only count
* calls to real pointer/referent/conversation cleanup; they do not replace it.
*/
function compositionFixture(
t,
{
supportedProviders = ['openai', 'gemini'],
factoryProviders = ['openai', 'gemini'],
savedProvider = null,
storageDenied = false,
gateStarts = () => false,
onSessionEvent = () => {},
} = {},
) {
const original = new Map(
['window', 'document'].map((key) => [
key,
Object.getOwnPropertyDescriptor(globalThis, key),
]),
);
const lifetime = new AbortController();
const views = [];
const instances = [];
const mounts = [];
const events = [];
const order = [];
const annotations = new Set();
const storageWrites = [];
const counters = { pointer: 0, referents: 0, conversation: 0, dispose: 0 };
const state = { height: 1000, zooms: 0, renders: 0 };
let facade;
let pointer;
globalThis.window = new EventTarget();
globalThis.document = {
getElementById: (id) =>
id === 'gev-voice-provider'
? {
focus() {
views.at(-1).focusCount++;
},
}
: null,
};
t.after(async () => {
try {
facade?.destroy();
pointer?.destroy();
for (const instance of instances)
for (const gate of instance.gates) gate.resolve();
await Promise.allSettled(instances.flatMap((instance) => instance.jobs));
} finally {
for (const [key, descriptor] of original) {
if (descriptor) Object.defineProperty(globalThis, key, descriptor);
else delete globalThis[key];
}
}
});
const canvas = new EventTarget();
Object.assign(canvas, { clientWidth: 800, clientHeight: 600 });
const position = { x: 6379137, y: 0, z: 0 };
const viewer = {
clock: { onTick: { addEventListener: () => () => {} } },
camera: {
positionWC: position,
positionCartographic: { height: state.height },
moveEnd: { addEventListener() {} },
cancelFlight() {},
zoomIn(metres) {
position.x -= metres;
state.height -= metres;
this.positionCartographic.height = state.height;
state.zooms++;
},
zoomOut(metres) {
position.x += metres;
state.height += metres;
this.positionCartographic.height = state.height;
state.zooms++;
},
},
scene: {
canvas,
requestRender() {
state.renders++;
},
},
};
const dataManager = { layers: new Map() };
pointer = createPointerTracker({ element: null, now: () => 1000 });
const referents = createReferentRegistry();
const runner = createGevActionRunner({
engine: createCesiumEngine(viewer),
viewer,
dataManager,
styleManager: {},
deixis: { pointer, referents },
});
for (const [object, method, counter] of [
[pointer, 'clear', 'pointer'],
[referents, 'clear', 'referents'],
[runner, 'resetConversation', 'conversation'],
[runner, 'dispose', 'dispose'],
]) {
const originalMethod = object[method];
object[method] = (...args) => {
counters[counter]++;
order.push(counter);
return originalMethod.apply(object, args);
};
}
const factories = Object.fromEntries(
factoryProviders.map((provider) => [
provider,
(hooks) => {
const id = instances.length;
let generation = 0;
const instance = {
id,
provider,
hooks,
starts: [],
stops: [],
jobs: [],
gates: [],
acquired: 0,
microphoneEnabled: false,
bound: 0,
input: { pushToTalk: false, held: false },
texts: [],
mapEvents: [],
};
instances.push(instance);
order.push(`factory:${id}`);
const adapter = {
identity: `${provider}:${id}`,
capabilities: {
pushToTalk: true,
costControls: provider === 'openai',
},
controller: {
providerDiagnostic() {
return this.session.adapter.identity;
},
},
start(settings = {}) {
const attempt = ++generation;
instance.starts.push({ ...settings });
instance.input.pushToTalk = Boolean(settings.pushToTalk);
hooks.emit({
type: 'state',
state: 'connecting',
detail: `adapter:${provider}`,
});
const job = (async () => {
if (attempt !== generation || hooks.signal.aborted) return;
if (gateStarts(instance)) {
const gate = deferred();
instance.gates.push(gate);
await gate.promise;
}
// Adapters own cancellation of their asynchronous acquisition.
if (attempt !== generation || hooks.signal.aborted) return;
instance.acquired++;
instance.microphoneEnabled =
!instance.input.pushToTalk || instance.input.held;
hooks.emit({
type: 'state',
state: 'listening',
detail: provider,
});
})();
instance.jobs.push(job);
return job;
},
stop(settings = {}) {
generation++;
instance.stops.push({ ...settings });
order.push(`stop:${id}`);
instance.input.pushToTalk = instance.input.held = false;
instance.microphoneEnabled = false;
},
getStartOptions: () => ({ pushToTalk: instance.input.pushToTalk }),
bindControls() {
instance.bound++;
},
sendText(text) {
instance.texts.push(text);
return `${provider}:${text}`;
},
sendMapEvent(event) {
instance.mapEvents.push(event);
return true;
},
};
instance.adapter = adapter;
return adapter;
},
]),
);
function createControl(options) {
const id = views.length;
order.push(`control:${id}`);
const view = countedControl(options, () => order.push(`remove:${id}`));
views.push(view);
return view;
}
function passiveView() {
return {
events: [],
outlines: [],
destroyed: 0,
handle(event) {
this.events.push(event);
},
outline(event) {
this.outlines.push(event);
},
destroy() {
this.destroyed++;
},
};
}
facade = createProviderCommands(
{
runner,
pointer,
referents,
dataManager,
signal: lifetime.signal,
supportedProviders,
createControl,
createCard() {
const card = passiveView();
views.at(-1).cardView = card;
return card;
},
createReticle() {
const reticle = passiveView();
views.at(-1).reticle = reticle;
return reticle;
},
annotations: {
onOutlineEvent(callback) {
annotations.add(callback);
return () => annotations.delete(callback);
},
},
onSessionEvent(event) {
events.push(event);
onSessionEvent(event, facade);
},
},
{
factories,
storage: {
getItem() {
if (storageDenied) throw Error('Storage denied');
return savedProvider;
},
setItem(key, value) {
if (storageDenied) throw Error('Storage denied');
storageWrites.push([key, value]);
savedProvider = value;
},
},
bind(options) {
// Keep the real callback so stale-event suppression tests exercise the
// facade's mount generation as well as the common session's disposal.
const mount = { options, callback: options.onSessionEvent };
mounts.push(mount);
mount.controls = bindVoiceCommands(options);
return mount.controls;
},
},
);
return {
facade,
runner,
pointer,
referents,
lifetime,
instances,
views,
mounts,
events,
order,
annotations,
counters,
storageWrites,
state,
select(provider) {
views.at(-1).emit('provider-change', provider);
},
seedConversation() {
pointer.beginTurn('text', {
capturedSnapshot: { fresh: true, at: 'text' },
});
referents.recordResult('frame_overhead', {
ok: true,
display: { title: 'One aircraft' },
referents: [
{ n: 1, id: 'aaa111', label: 'Fixture', layerId: 'flights' },
],
});
assert.ok(pointer.activeSnapshot());
assert.equal(referents.list().length, 1);
},
};
}
test('provider preference uses only the supplied host capability list', () => {
const stored = (value) => ({ getItem: () => value });
assert.equal(readVoiceProvider(stored('gemini'), ['openai']), 'openai');
assert.equal(readVoiceProvider(stored('openai'), ['gemini']), 'gemini');
assert.equal(
readVoiceProvider(stored('unknown'), ['gemini', 'openai']),
'openai',
);
assert.equal(
readVoiceProvider(stored('gemini'), ['openai', 'gemini']),
'gemini',
);
assert.equal(
readVoiceProvider(
{
getItem() {
throw Error('denied');
},
},
['gemini'],
),
'gemini',
);
});
for (const scenario of [
{
title: 'OpenAI-only host ignores saved Gemini',
requested: ['openai'],
saved: 'gemini',
expected: ['openai'],
provider: 'openai',
},
{
title: 'Gemini-only host falls back from saved OpenAI',
requested: ['gemini'],
saved: 'openai',
expected: ['gemini'],
provider: 'gemini',
},
{
title: 'dual host restores Gemini',
requested: ['openai', 'gemini'],
saved: 'gemini',
expected: ['openai', 'gemini'],
provider: 'gemini',
},
{
title: 'unknown, duplicate and unavailable factories are excluded',
requested: ['unknown', 'gemini', 'openai', 'gemini'],
factories: ['gemini'],
saved: 'openai',
expected: ['gemini'],
provider: 'gemini',
},
]) {
test(scenario.title, (t) => {
const f = compositionFixture(t, {
supportedProviders: scenario.requested,
factoryProviders: scenario.factories,
savedProvider: scenario.saved,
});
assert.deepEqual(f.facade.supportedProviders, scenario.expected);
assert.equal(Object.isFrozen(f.facade.supportedProviders), true);
assert.equal(f.facade.provider, scenario.provider);
assert.deepEqual(
f.views[0].state.provider.options.map((option) => option.value),
scenario.expected,
);
assert.equal(
f.views[0].state.provider.hidden,
scenario.expected.length === 1,
);
assert.equal(
f.views[0].state.root.providerChoice,
String(scenario.expected.length > 1),
);
assert.equal(f.instances.length, 1);
assert.equal(
f.instances[0].starts.length,
0,
'discovery must not start a provider',
);
assert.equal(f.facade.state, 'idle');
});
}
test('invalid or empty host support fails before mounting a control', () => {
let mounts = 0;
const dependencies = {
bind() {
mounts++;
},
factories: { openai() {} },
};
assert.throws(
() =>
createProviderCommands({ supportedProviders: 'gemini' }, dependencies),
/must be an array/,
);
assert.throws(
() =>
createProviderCommands(
{ supportedProviders: ['gemini', 'unknown'] },
dependencies,
),
/No supported voice provider/,
);
assert.equal(mounts, 0);
});
test('active switch keeps PTT mode, retires the held key, and disposes before mounting', async (t) => {
const f = compositionFixture(t);
const firstSession = f.facade.session;
const outerSignal = f.facade.signal;
const first = f.instances[0];
// PTT controls can activate the adapter directly; the facade must learn
// activity from its common events, not only from facade.start().
first.input.held = true;
await first.adapter.start({ pushToTalk: true });
assert.equal(first.microphoneEnabled, true);
assert.equal(f.facade.isActive(), true);
f.seedConversation();
const before = { ...f.counters };
f.select('gemini');
await settle();
const second = f.instances[1];
assert.equal(window.__gevVoiceCommands, f.facade);
assert.equal(f.facade.runner, f.runner);
assert.equal(f.facade.signal, outerSignal);
assert.equal(outerSignal.aborted, false);
assert.equal(firstSession.disposed, true);
assert.equal(first.hooks.signal.aborted, true);
assert.deepEqual(first.stops, [{ removeUi: true }]);
assert.equal(first.input.held, false);
assert.equal(first.microphoneEnabled, false);
assert.deepEqual(second.starts, [{ pushToTalk: true }]);
assert.equal(second.input.held, false);
assert.equal(
second.microphoneEnabled,
false,
'a held key must never migrate to the new adapter',
);
assert.equal(f.facade.state, 'listening');
assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('control:1'));
assert.ok(f.order.indexOf('remove:0') < f.order.indexOf('control:1'));
assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('factory:1'));
assert.equal(f.views[0].listenerCount('button'), 0);
assert.equal(f.views[0].listenerCount('provider-change'), 0);
assert.equal(f.views[1].listenerCount('provider-change'), 1);
assert.equal(f.views[1].focusCount, 1);
assert.equal(f.views[1].state.tier.hidden, true);
assert.equal(f.views[1].state.cost.hidden, true);
assert.equal(f.facade.providerDiagnostic(), 'gemini:1');
assert.equal(f.pointer.activeSnapshot(), null);
assert.deepEqual(f.referents.list(), []);
for (const key of ['pointer', 'referents', 'conversation'])
assert.equal(
f.counters[key],
before[key] + 1,
`one explicit replacement reset: ${key}`,
);
assert.equal(
f.counters.dispose,
0,
'provider replacement keeps the page runner',
);
assert.deepEqual(f.storageWrites, [[PROVIDER_KEY, 'gemini']]);
assert.equal(f.facade.sendTextCommand('hello'), 'gemini:hello');
assert.deepEqual(first.texts, []);
assert.deepEqual(second.texts, ['hello']);
const result = await f.facade.runner('adjust_camera_zoom', {
direction: 'in',
amount: 'little',
});
assert.equal(
result.ok,
true,
'the actual shared action runner remains usable after switching',
);
assert.equal(f.state.zooms, 1);
});
for (const initialState of ['idle', 'error']) {
test(`${initialState} selection never autostarts; manual start still works`, async (t) => {
const f = compositionFixture(t, { storageDenied: true });
if (initialState === 'error') {
await f.facade.start({ pushToTalk: true });
f.instances[0].adapter.stop({ preserveStatus: true });
f.instances[0].hooks.emit({
type: 'state',
state: 'error',
detail: 'Fixture failure',
});
}
f.select('gemini');
await settle();
assert.equal(
f.facade.provider,
'gemini',
'denied storage cannot block a supported selection',
);
assert.equal(f.facade.state, 'idle');
assert.deepEqual(f.instances[1].starts, []);
f.select('gemini');
f.select('unknown');
assert.equal(
f.instances.length,
2,
'same and unsupported selections cannot remount',
);
assert.equal(f.facade.provider, 'gemini');
await f.facade.start({ pushToTalk: false });
assert.deepEqual(f.instances[1].starts, [{ pushToTalk: false }]);
assert.equal(f.instances[1].microphoneEnabled, true);
});
}
for (const outcome of ['resolve', 'reject']) {
test(`retired startup ${outcome} and stale events cannot affect the current mount`, async (t) => {
const f = compositionFixture(t, {
gateStarts: (instance) => instance.id === 0,
});
const oldStart = f.facade.start({ pushToTalk: false });
const old = f.instances[0];
assert.equal(f.facade.state, 'connecting');
f.select('gemini');
await settle();
assert.equal(f.facade.state, 'listening');
const current = f.instances[1];
const eventCount = f.events.length;
const currentStops = current.stops.length;
const currentCardCount = f.views[1].cardView.events.length;
old.gates[0][outcome](new Error('Late retired startup failure'));
await oldStart;
for (const state of ['idle', 'error', 'connecting']) {
old.hooks.emit({ type: 'state', state, detail: 'retired adapter' });
f.mounts[0].callback({
type: 'state',
state,
detail: 'retired forward callback',
});
}
assert.equal(f.events.length, eventCount);
assert.equal(f.views[1].cardView.events.length, currentCardCount);
assert.equal(f.facade.state, 'listening');
assert.equal(old.acquired, 0);
assert.equal(current.stops.length, currentStops);
assert.equal(current.starts.length, 1);
f.facade.stop();
f.mounts[0].callback({ type: 'state', state: 'connecting' });
f.select('openai');
assert.deepEqual(
f.instances[2].starts,
[],
'a retired connecting callback cannot revive voice intent',
);
assert.equal(f.facade.state, 'idle');
});
}
test('Stop during replacement startup defeats late settlement and further automatic resume', async (t) => {
const f = compositionFixture(t, {
gateStarts: (instance) => instance.id === 1,
});
await f.facade.start();
f.select('gemini');
const replacement = f.instances[1];
assert.equal(f.facade.state, 'connecting');
f.facade.stop();
replacement.gates[0].resolve();
await Promise.all(replacement.jobs);
assert.equal(replacement.acquired, 0);
assert.equal(f.facade.state, 'idle');
f.select('openai');
assert.deepEqual(f.instances[2].starts, []);
await f.facade.start();
assert.equal(
f.facade.state,
'listening',
'a later explicit start remains healthy',
);
});
for (const emitsIdle of [false, true]) {
test(`Stop and error clear pointer/referents/conversation once: adapter emitsIdle=${emitsIdle}`, async (t) => {
const f = compositionFixture(t);
if (emitsIdle) {
const instance = f.instances[0];
const originalStop = instance.adapter.stop;
instance.adapter.stop = function (settings = {}) {
originalStop.call(this, settings);
if (!settings.preserveStatus && !settings.removeUi)
instance.hooks.emit({ type: 'state', state: 'idle' });
};
}
for (const end of ['stop', 'error']) {
await f.facade.start();
f.seedConversation();
const before = { ...f.counters };
const generation = f.referents.generation();
if (end === 'stop') f.facade.stop();
else {
f.instances[0].adapter.stop({ preserveStatus: true });
f.instances[0].hooks.emit({
type: 'state',
state: 'error',
detail: 'Fixture failure',
});
}
assert.equal(f.pointer.activeSnapshot(), null);
assert.deepEqual(f.referents.list(), []);
assert.equal(f.referents.generation(), generation + 1);
for (const key of ['pointer', 'referents', 'conversation'])
assert.equal(f.counters[key], before[key] + 1, `${end}: ${key}`);
}
});
}
test('outer destruction disposes the runner once and removes each provider binding', async (t) => {
const f = compositionFixture(t);
await f.facade.start();
f.select('gemini');
await settle();
assert.equal(f.annotations.size, 1);
const oldCardEvents = f.views[0].cardView.events.length;
for (const callback of f.annotations) callback({ id: 'fixture-outline' });
assert.equal(f.views[0].cardView.outlines.length, 0);
assert.equal(f.views[1].cardView.outlines.length, 1);
assert.equal(f.instances[1].mapEvents.length, 1);
f.facade.destroy();
f.facade.destroy();
f.facade.stop({ removeUi: true });
f.lifetime.abort();
assert.equal(f.counters.dispose, 1);
assert.equal(f.facade.signal.aborted, true);
assert.equal(f.facade.disposed, true);
assert.equal(f.annotations.size, 0);
const finalEvents = f.events.length;
const starts = f.instances.map((instance) => instance.starts.length);
for (const [index, view] of f.views.entries()) {
assert.equal(view.removeCount, 1);
assert.equal(view.cardView.destroyed, 1);
assert.equal(view.reticle.destroyed, 1);
assert.equal(view.listenerCount('button'), 0);
assert.equal(view.listenerCount('provider-change'), 0);
assert.equal(f.instances[index].hooks.signal.aborted, true);
view.emit('button');
view.emit('provider-change', 'openai');
f.instances[index].hooks.emit({ type: 'state', state: 'connecting' });
f.mounts[index].callback({ type: 'state', state: 'connecting' });
}
assert.equal(f.events.length, finalEvents);
assert.equal(f.views[0].cardView.events.length, oldCardEvents);
assert.deepEqual(
f.instances.map((instance) => instance.starts.length),
starts,
);
assert.equal(f.instances.length, 2);
assert.equal(f.facade.start(), false);
assert.equal(f.facade.sendText('after destroy'), false);
assert.equal(f.facade.notifyMapEvent({}), false);
const result = await f.facade.runner('adjust_camera_zoom', {
direction: 'in',
amount: 'little',
});
assert.equal(result.cancelled, true);
assert.equal(
f.state.zooms,
0,
'the actual runner cannot move the camera after outer disposal',
);
});
for (const action of ['stop', 'destroy']) {
for (const phase of ['common', 'adapter']) {
test(`${action} from replacement ${phase} connecting observer prevents resume`, async (t) => {
let intercepted = 0;
const f = compositionFixture(t, {
onSessionEvent(event, facade) {
if (
facade?.provider !== 'gemini' ||
event.type !== 'state' ||
event.state !== 'connecting'
)
return;
const matching =
phase === 'common'
? event.detail === undefined
: event.detail === 'adapter:gemini';
if (matching && intercepted++ === 0) facade[action]();
},
});
await f.facade.start({ pushToTalk: true });
f.select('gemini');
await settle();
const replacement = f.instances[1];
assert.equal(intercepted, 1);
assert.equal(replacement.starts.length, phase === 'common' ? 0 : 1);
assert.equal(replacement.acquired, 0);
assert.equal(replacement.microphoneEnabled, false);
assert.equal(f.facade.isActive(), false);
assert.equal(f.facade.state, 'idle');
assert.equal(
f.events.filter(
(event) => event.state === 'listening' && event.detail === 'gemini',
).length,
0,
);
assert.equal(f.facade.disposed, action === 'destroy');
assert.equal(f.counters.dispose, action === 'destroy' ? 1 : 0);
if (action === 'stop') {
f.select('openai');
assert.deepEqual(
f.instances[2].starts,
[],
'Stop cleared the intent to resume',
);
await f.facade.start({ pushToTalk: false });
assert.equal(f.facade.state, 'listening');
} else {
f.select('openai');
assert.equal(f.instances.length, 2);
assert.equal(f.facade.start(), false);
}
});
}
}
+2 -5
View File
@@ -1,5 +1,6 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import { createRealtimeBackend } from './realtimeBackend.js';
import { GevRealtimeController } from './realtimeController.js';
@@ -131,11 +132,7 @@ test('controller lifetime stops pending transport and releases resources through
runner: async () => ({}),
signal: lifetime.signal,
debugSink: null,
ui: {
root: { dataset: {}, querySelectorAll: () => [], remove() {} },
status: {},
detail: {},
},
ui: createTestVoiceUi(),
});
controller.connectionAbort = new AbortController();
const connection = controller.connectionAbort.signal;
+7 -5
View File
@@ -1,3 +1,5 @@
import { t } from '../i18n/index.js';
const DISCONNECT_GRACE_MS = 6000;
function releaseStartResources({ localStream = null, localPc = null } = {}) {
@@ -59,7 +61,7 @@ export class RealtimeConnection {
this.input.pushToTalkKeyHeld = pushToTalkKeyHeld;
this.input.spaceKeyHeld = spaceKeyHeld;
if (!window.RTCPeerConnection || !navigator.mediaDevices?.getUserMedia) {
this.setStatus('error', 'WebRTC microphone support unavailable');
this.setStatus('error', t('WebRTC microphone support unavailable'));
return;
}
@@ -78,7 +80,7 @@ export class RealtimeConnection {
// — this is what "applies next session" means.
this.cost.prepareSession();
this.syncCostUi();
this.setStatus('connecting', 'Requesting microphone');
this.setStatus('connecting', t('Requesting microphone'));
this.debugLog('session.starting', {
epoch,
tier: this.cost.voiceTier,
@@ -179,9 +181,9 @@ export class RealtimeConnection {
if (!ownsChannel()) return;
const detail = this.input.pushToTalkMode
? this.input.pushToTalkKeyHeld
? 'Release Space to send'
: 'Hold Space to talk'
: 'Ask or command';
? t('Release Space to send')
: t('Hold Space to talk')
: t('Ask or command');
this.setStatus('listening', detail);
this.debugLog('data_channel.open', {
connection: this.connectionDiagnostics(dataChannel),
+57 -72
View File
@@ -10,14 +10,13 @@ import { TurnMetrics } from './turnMetrics.js';
import { createNarrationScheduler } from './narration.js';
import { createEarcon } from './earcon.js';
import {
POINTER_NONE_KEY,
POINTER_OPEN_MIC_RECENT_MS,
pointerChip,
pointerContextItem,
pointerAnnouncement,
pointerTurnContext,
pointerTurnEvent,
} from './pointerContext.js';
/** Dedupe key of the clearing pointer_context item. */
const POINTER_NONE_KEY = 'none';
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
import { postDebugLog } from './realtimeDiagnostics.js';
@@ -55,13 +54,17 @@ export {
import { createRealtimeBackend } from './realtimeBackend.js';
const STATUS = {
idle: 'OFF',
connecting: 'CONNECTING',
listening: 'LISTENING',
executing: 'EXECUTING',
error: 'ERROR',
};
import { t } from '../i18n/index.js';
import { voiceStatusLabel } from './control.js';
// Machine states (dataset.status, CSS); voiceStatusLabel() translates labels.
const STATUS_STATES = new Set([
'idle',
'connecting',
'listening',
'executing',
'error',
]);
/** Compose voice state owners and coordinate ordered session startup/teardown. */
export class GevRealtimeController extends RealtimeFacade {
@@ -154,7 +157,6 @@ export class GevRealtimeController extends RealtimeFacade {
},
});
this.buttonHandler = null;
this.tierHandler = null;
this.annotationEventUnsubscribe = null;
@@ -283,31 +285,18 @@ export class GevRealtimeController extends RealtimeFacade {
} catch {
snapshot = null;
}
const chip = pointerChip(snapshot);
this.pointerAnnounced = gesture && chip ? snapshot : null;
this.emitSessionEvent({
type: 'pointer',
pointer:
gesture && chip ? { ...chip, screenPx: snapshot.screenPx } : null,
const event = pointerTurnEvent(snapshot, gesture);
this.pointerAnnounced = event.announced;
this.emitSessionEvent({ type: 'pointer', pointer: event.pointer });
const context = pointerTurnContext(snapshot, {
at,
gesture,
sentKey: this.pointerSentKey,
});
const item = !chip
? this.pointerSentKey && this.pointerSentKey !== POINTER_NONE_KEY
? { type: 'pointer_context', at, target: 'none' }
: null
: gesture
? pointerContextItem(snapshot)
: // What kind of thing, never which: no label or coordinates.
{
type: 'pointer_context',
at,
pointing: snapshot.entity?.kind || 'ground',
};
if (!item) return snapshot;
const { at: _at, ...content } = item;
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
if (key === this.pointerSentKey) return snapshot;
if (this._turns.notifyMapEvent(item)) {
this.pointerSentKey = key;
if (!context) return snapshot;
if (this._turns.notifyMapEvent(context.item)) {
this.pointerSentKey = context.key;
const chip = context.key !== POINTER_NONE_KEY;
this.debugLog('pointer.context', {
at,
target: chip ? snapshot.target : 'none',
@@ -323,13 +312,10 @@ export class GevRealtimeController extends RealtimeFacade {
*/
announcePointer() {
const snapshot = this.pointer?.activeSnapshot?.() || null;
const chip = pointerChip(snapshot);
if (!chip || snapshot === this.pointerAnnounced) return false;
const pointer = pointerAnnouncement(snapshot, this.pointerAnnounced);
if (!pointer) return false;
this.pointerAnnounced = snapshot;
this.emitSessionEvent({
type: 'pointer',
pointer: { ...chip, screenPx: snapshot.screenPx },
});
this.emitSessionEvent({ type: 'pointer', pointer });
return true;
}
@@ -392,12 +378,8 @@ export class GevRealtimeController extends RealtimeFacade {
this._radio.clearPendingPlayback();
this._viewport.reset();
this._input.resetSession();
if (removeUi && this.ui?.button && this.buttonHandler) {
this.ui.button.removeEventListener('click', this.buttonHandler);
this.buttonHandler = null;
}
if (removeUi && this.ui?.tierButton && this.tierHandler) {
this.ui.tierButton.removeEventListener('click', this.tierHandler);
if (removeUi && this.tierHandler) {
this.tierHandler();
this.tierHandler = null;
}
if (removeUi) this._input.detachBindings();
@@ -408,11 +390,9 @@ export class GevRealtimeController extends RealtimeFacade {
this.annotationEventUnsubscribe = null;
}
if (removeUi) this._radio.detachObservers();
if (removeUi && this.ui?.root) {
this.ui.root.remove();
}
if (removeUi) this.ui?.remove();
if (!preserveStatus && !removeUi) {
this.setStatus('idle', 'Voice off');
this.setStatus('idle', t('Voice off'));
}
this.setRadioVoiceDucking(false);
if (removeUi) this.emitSessionEvent({ type: 'disposed' });
@@ -429,29 +409,34 @@ export class GevRealtimeController extends RealtimeFacade {
setStatus(status, detail) {
this.status = status;
this.emitSessionEvent({ type: 'state', state: status, detail });
this.ui.root.dataset.status = status;
if (status === 'error') this.ui.root.classList.remove('error-dismissed');
this.ui.update('root', {
status,
...(status === 'error' && { errorDismissed: false }),
});
this.updateVoiceButtonLabel();
this.ui.status.textContent = STATUS[status] || STATUS.idle;
const resolvedDetail =
status === 'listening' && this.pushToTalkMode
? this.pushToTalkKeyHeld
? 'Release Space to send'
: 'Hold Space to talk'
const shownStatus = STATUS_STATES.has(status) ? status : 'idle';
this.ui.update('status', { text: () => voiceStatusLabel(shownStatus) });
// Words are given as functions, so a locale change words them again.
const holdToTalk = status === 'listening' && this.pushToTalkMode;
const held = this.pushToTalkKeyHeld;
const resolvedDetail = () =>
holdToTalk
? held
? t('Release Space to send')
: t('Hold Space to talk')
: detail;
const primaryDetail =
const primaryDetail = () =>
status === 'error'
? 'VOICE UNAVAILABLE'
: resolvedDetail ||
(status === 'idle' ? 'VOICE STANDBY' : 'VOICE ACTIVE');
this.ui.detail.textContent = primaryDetail;
this.ui.detail.title = primaryDetail;
if (this.ui.errorDetail) {
this.ui.errorDetail.textContent =
? t('VOICE UNAVAILABLE')
: resolvedDetail() ||
(status === 'idle' ? t('VOICE STANDBY') : t('VOICE ACTIVE'));
this.ui.update('detail', { text: primaryDetail, title: primaryDetail });
this.ui.update('error', {
text: () =>
status === 'error'
? resolvedDetail || 'Voice session could not be started.'
: '';
}
? resolvedDetail() || t('Voice session could not be started.')
: '',
});
if (status === 'idle' || status === 'connecting' || status === 'error') {
this.setVoiceSpeaker('idle');
}
+85 -40
View File
@@ -1,3 +1,4 @@
import { t } from '../i18n/index.js';
import {
readStoredVoiceTier,
readStoredVoiceLimits,
@@ -45,45 +46,84 @@ export class RealtimeCost {
*/
syncCostUi() {
const state = this.costTracker.state();
// The panel's warning outside the closed panel mirrors the readout.
const showCost = (cost) => {
this.ui.update('cost', { hidden: false, ...cost });
const hidden = !['warn', 'cap', 'unknown'].includes(cost.level);
this.ui.update('costWarning', {
hidden,
text: hidden ? '' : cost.text,
title: cost.title,
level: cost.level,
});
};
const pendingTier = resolveVoiceModel(this.voiceTier).tier;
const isMini = pendingTier === 'mini';
if (this.ui?.tierButton) {
this.ui.tierButton.textContent = isMini ? 'MINI' : 'STD';
this.ui.tierButton.setAttribute(
'aria-pressed',
isMini ? 'true' : 'false',
);
const pendingId = resolveVoiceModel(pendingTier).id;
this.ui.tierButton.title =
if (!this.ui) return;
const pendingId = resolveVoiceModel(pendingTier).id;
this.ui.update('tier', {
text: isMini ? t('MINI') : t('STD'),
pressed: isMini,
title:
this.isActive() && state.modelId !== pendingId
? `Next session: ${pendingId} — this session stays on ${state.modelId}`
: `Voice model: ${pendingId} — click to switch to ${
isMini ? 'standard' : 'mini'
}; applies next session`;
}
if (this.ui?.costValue) {
const liveAuth = this.sessionCloudVoiceAuth || this.cloudVoiceAuth;
if (liveAuth === 'oauth') {
const { responses, input, output, captions } = this.oauthUsage;
this.ui.costValue.hidden = false;
this.ui.costValue.textContent = 'COST UNKNOWN';
this.ui.costValue.dataset.level = 'unknown';
this.ui.costValue.title =
`ChatGPT OAuth session: ${responses} response(s), ${input} input and ${output} output tokens, ${captions} caption transcription(s) reported. ` +
'USD cost and the API spend cap are unavailable for this auth mode.' +
? t('Next session: {pendingId} — this session stays on {modelId}', {
pendingId,
modelId: state.modelId,
})
: t(
'Voice model: {pendingId} — click to switch to {target}; applies next session',
{
pendingId,
target: isMini ? t('standard') : t('mini'),
},
),
});
const liveAuth = this.sessionCloudVoiceAuth || this.cloudVoiceAuth;
if (liveAuth === 'oauth') {
const { responses, input, output, captions } = this.oauthUsage;
showCost({
text: t('COST UNKNOWN'),
level: 'unknown',
title:
t(
'ChatGPT OAuth session: {responses} response(s), {input} input and {output} output tokens, {captions} caption transcription(s) reported.',
{ responses, input, output, captions },
) +
' ' +
t(
'USD cost and the API spend cap are unavailable for this auth mode.',
) +
(state.incomplete
? ' Usage is incomplete because a response was still in flight when the session ended.'
: '');
return;
}
this.ui.costValue.hidden = false;
this.ui.costValue.textContent = state.display;
this.ui.costValue.dataset.level = state.level;
this.ui.costValue.title =
`Estimated session cost on ${state.modelId} — ${state.responses} response(s). ` +
`Warns at ${formatCostUsd(state.warnUsd)}, ends the session at ${formatCostUsd(state.capUsd)}.` +
(state.note ? ` ${state.note}` : '');
? ' ' +
t(
'Usage is incomplete because a response was still in flight when the session ended.',
)
: ''),
});
return;
}
showCost({
text: state.display,
level: state.level,
title:
t(
'Estimated session cost on {modelId} — {responses} response(s). Warns at {warn}, ends the session at {cap}.',
{
modelId: state.modelId,
responses: state.responses,
warn: formatCostUsd(state.warnUsd),
cap: formatCostUsd(state.capUsd),
},
) +
// `state.note` is the tracker's English diagnostic; the tooltip says
// the same in the interface language.
(state.incomplete
? ' ' +
t(
'Estimate is incomplete — a response was still in flight when the session ended, so its usage was never reported.',
)
: ''),
});
}
/**
@@ -122,11 +162,13 @@ export class RealtimeCost {
});
}
this.syncCostUi();
if (this.isActive() && this.ui?.detail) {
this.setStatus(
this.status,
`${this.voiceTier.toUpperCase()} applies next session`,
);
if (this.isActive() && this.ui) {
// The tier as the interface names it; the stored value stays English.
const tier =
resolveVoiceModel(this.voiceTier).tier === 'mini'
? t('MINI')
: t('STANDARD');
this.setStatus(this.status, t('{tier} applies next session', { tier }));
}
return this.voiceTier;
}
@@ -240,7 +282,10 @@ export class RealtimeCost {
try {
this.stop({ preserveStatus: true });
} finally {
this.setStatus('idle', `Session ended — cost cap ${state.display}`);
this.setStatus(
'idle',
t('Session ended — cost cap {cost}', { cost: state.display }),
);
this.syncCostUi();
}
}
@@ -0,0 +1,51 @@
// The API-key cost tooltip states an incomplete estimate in the
// interface language; the tracker keeps its English note for diagnostics.
import test from 'node:test';
import assert from 'node:assert/strict';
import { setLocale } from '../i18n/index.js';
import { RealtimeCost } from './realtimeCost.js';
import { createVoiceCostTracker } from './voiceCost.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
for (const locale of ['es', 'ru']) {
test(`API-key cost tooltip for an incomplete session has no English (${locale})`, () => {
setLocale(locale);
try {
const costTracker = createVoiceCostTracker({ tier: 'standard' });
costTracker.markIncomplete();
const host = {
costTracker,
voiceTier: 'standard',
cloudVoiceAuth: 'api-key',
sessionCloudVoiceAuth: 'api-key',
isActive: () => false,
ui: createTestVoiceUi(),
};
RealtimeCost.prototype.syncCostUi.call(host);
const title = host.ui.state.cost.title;
assert.ok(title.length > 0);
assert.doesNotMatch(
title,
/Estimate is incomplete|still in flight/,
title,
);
} finally {
setLocale('en');
}
});
}
test('API-key cost tooltip in English still ends with the tracker note', () => {
const costTracker = createVoiceCostTracker({ tier: 'standard' });
costTracker.markIncomplete();
const host = {
costTracker,
voiceTier: 'standard',
cloudVoiceAuth: 'api-key',
sessionCloudVoiceAuth: 'api-key',
isActive: () => false,
ui: createTestVoiceUi(),
};
RealtimeCost.prototype.syncCostUi.call(host);
assert.ok(host.ui.state.cost.title.endsWith(` ${costTracker.state().note}`));
});
+132
View File
@@ -0,0 +1,132 @@
import '../testSupport/registerDom.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
import { createApp, h, nextTick } from 'vue';
import { createPinia } from 'pinia';
import { RealtimeCost } from './realtimeCost.js';
import { createVoiceCostTracker, resolveVoiceModel } from './voiceCost.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import { useVoiceStore } from '../ui/stores/voice.js';
import VoiceControl from '../ui/views/panels/VoiceControl.vue';
const $ = (selector) => document.querySelector(selector);
// Shown unless it or an ancestor is hidden.
const visible = (selector) => {
const node = $(selector);
return Boolean(node) && !node.closest('[hidden]');
};
function harness(t) {
document.body.replaceChildren();
const pinia = createPinia();
const ui = createTestVoiceUi({ panel: true, pinia });
const store = useVoiceStore(pinia);
// The dock renders the presented control; this renders only that.
const app = createApp({
render: () => store.view && h(VoiceControl, { view: store.view }),
}).use(pinia);
app.mount(document.body.appendChild(document.createElement('div')));
t.after(() => app.unmount());
const stops = [];
const statuses = [];
const cost = new RealtimeCost({
readUi: () => ui,
readStatus: () => 'listening',
operations: {
isActive: () => true,
stop: (options) => stops.push(options),
setStatus: (...args) => statuses.push(args),
debugLog() {},
},
});
cost.cloudVoiceAuth = 'api-key';
cost.costTracker = createVoiceCostTracker({
tier: 'standard',
limits: { warnUsd: 2, capUsd: 5 },
});
const recordThreeDollars = () => {
const tokens = Math.ceil(
(3 * 1_000_000) / resolveVoiceModel('standard').rates.textOutput,
);
const state = cost.costTracker.record({
output_tokens: tokens,
output_token_details: { text_tokens: tokens },
});
cost.applyCost(state);
};
t.mock.method(console, 'warn', () => {});
return { cost, ui, stops, statuses, recordThreeDollars };
}
test('soft cost warning is visible outside closed Settings, while ordinary spend stays inside', async (t) => {
const { cost, ui, recordThreeDollars } = harness(t);
cost.syncCostUi();
await nextTick();
assert.equal(ui.state.voicePanel.hidden, true);
assert.equal($('#gev-voice-panel').hidden, true);
assert.equal(visible('#gev-voice-cost-value'), false);
assert.equal($('#gev-voice-cost-warning').hidden, true);
assert.equal(
$('#gev-voice-panel').contains($('#gev-voice-cost-warning')),
false,
);
assert.equal(
$('#gev-voice-control').querySelectorAll('#gev-voice-cost-value').length,
1,
);
recordThreeDollars();
await nextTick();
const warning = $('#gev-voice-cost-warning');
const value = $('#gev-voice-cost-value');
assert.equal(visible('#gev-voice-cost-warning'), true);
assert.equal(warning.dataset.level, 'warn');
assert.equal(warning.textContent, value.textContent);
assert.equal(warning.title, value.title);
assert.match(warning.title, /Warns at/);
assert.equal(
$('#gev-voice-panel').hidden,
true,
'warning never forces Settings open',
);
cost.prepareSession();
cost.syncCostUi();
await nextTick();
assert.equal(
$('#gev-voice-cost-warning').hidden,
true,
'new session clears the old warning',
);
assert.equal($('#gev-voice-cost-warning').textContent, '');
});
test('closed Settings cannot hide the hard cap or suppress its ordinary stop and status', async (t) => {
const { stops, statuses, recordThreeDollars } = harness(t);
recordThreeDollars();
assert.equal(stops.length, 0);
recordThreeDollars();
await nextTick();
assert.equal(visible('#gev-voice-cost-warning'), true);
assert.equal($('#gev-voice-cost-warning').dataset.level, 'cap');
assert.deepEqual(stops, [{ preserveStatus: true }]);
assert.equal(statuses.at(-1)[0], 'idle');
assert.match(statuses.at(-1)[1], /Session ended — cost cap/);
});
test('unknown OAuth cost remains visible outside Settings and clears on a fresh API meter', async (t) => {
const { cost } = harness(t);
cost.sessionCloudVoiceAuth = 'oauth';
cost.syncCostUi();
await nextTick();
const warning = $('#gev-voice-cost-warning');
assert.equal(visible('#gev-voice-cost-warning'), true);
assert.equal(warning.dataset.level, 'unknown');
assert.equal(warning.textContent, 'COST UNKNOWN');
assert.match(warning.title, /USD cost and the API spend cap are unavailable/);
assert.equal($('#gev-voice-panel').hidden, true);
cost.sessionCloudVoiceAuth = 'api-key';
cost.syncCostUi();
await nextTick();
assert.equal($('#gev-voice-cost-warning').hidden, true);
});
+53
View File
@@ -0,0 +1,53 @@
// The status after a tier switch names the tier in the interface
// language, as the tier button does; the stored tier value stays English.
import test from 'node:test';
import assert from 'node:assert/strict';
import { setLocale, t } from '../i18n/index.js';
import { RealtimeCost } from './realtimeCost.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
function harness() {
const ui = createTestVoiceUi();
const statusCalls = [];
const cost = new RealtimeCost({
readUi: () => ui,
readStatus: () => 'listening',
operations: {
isActive: () => true,
isVoiceSessionSettled: () => false,
setStatus: (status, detail) => statusCalls.push(detail),
},
});
return { cost, ui, statusCalls };
}
const EXPECTED = {
en: {
mini: 'MINI applies next session',
standard: 'STANDARD applies next session',
},
ru: {
mini: 'МИНИ — со следующей сессии',
standard: 'СТАНДАРТ — со следующей сессии',
},
es: {
mini: 'MINI desde la próxima sesión',
standard: 'ESTÁNDAR desde la próxima sesión',
},
};
for (const locale of Object.keys(EXPECTED)) {
test(`${locale}: tier status names the tier in the interface language`, (ctx) => {
setLocale(locale);
ctx.after(() => setLocale('en'));
for (const tier of ['mini', 'standard']) {
const { cost, ui, statusCalls } = harness();
assert.equal(cost.setVoiceTier(tier), tier, 'stored tier stays English');
assert.equal(statusCalls.at(-1), EXPECTED[locale][tier]);
assert.equal(
ui.state.tier.text,
tier === 'mini' ? t('MINI') : t('STD'),
);
}
});
}
+11 -18
View File
@@ -1,10 +1,10 @@
export const ERROR_LOG_LIMIT = 30;
const ERROR_LOG_LIMIT = 30;
export const ERROR_STORAGE_KEY = 'gev-realtime-errors';
const ERROR_STORAGE_KEY = 'gev-realtime-errors';
export const DEBUG_LOG_URL = '/api/realtime/debug-log';
const DEBUG_LOG_URL = '/api/realtime/debug-log';
export function createDebugSessionId() {
function createDebugSessionId() {
const randomPart = Math.random().toString(36).slice(2, 10);
return `gev-${Date.now().toString(36)}-${randomPart}`;
}
@@ -47,7 +47,7 @@ export function sanitizeDebugValue(value, depth = 0) {
return output;
}
export function sanitizeDebugString(value) {
function sanitizeDebugString(value) {
if (value.startsWith('data:image/')) {
return `[Redacted image data URL, ${value.length} chars]`;
}
@@ -65,13 +65,13 @@ export function sanitizeDebugString(value) {
: redacted;
}
export function isSecretLikeKey(key) {
function isSecretLikeKey(key) {
return /(?:api[_-]?key|authorization|bearer|client[_-]?secret|token|secret|password)/i.test(
key,
);
}
export function createErrorRecord(source, error, extra = {}) {
function createErrorRecord(source, error, extra = {}) {
const rtcError = error?.error || error;
return {
timestamp: new Date().toISOString(),
@@ -90,7 +90,7 @@ export function createErrorRecord(source, error, extra = {}) {
};
}
export function formatErrorForDisplay(record) {
function formatErrorForDisplay(record) {
const primary = [record.source, record.message].filter(Boolean).join(': ');
const state = [
record.errorDetail && `detail=${record.errorDetail}`,
@@ -105,7 +105,7 @@ export function formatErrorForDisplay(record) {
return state ? `${primary}\n${state}` : primary;
}
export function removeEmptyValues(value) {
function removeEmptyValues(value) {
return Object.fromEntries(
Object.entries(value || {}).filter(
([, item]) => item !== null && item !== undefined && item !== '',
@@ -113,14 +113,7 @@ export function removeEmptyValues(value) {
);
}
export function compactText(value, maxLength) {
const text = String(value || '')
.replace(/\s+/g, ' ')
.trim();
return text.length <= maxLength ? text : `${text.slice(0, maxLength - 1)}…`;
}
export function loadStoredErrors() {
function loadStoredErrors() {
try {
const value = JSON.parse(localStorage.getItem(ERROR_STORAGE_KEY) || '[]');
return Array.isArray(value) ? value.slice(0, ERROR_LOG_LIMIT) : [];
@@ -129,7 +122,7 @@ export function loadStoredErrors() {
}
}
export function storeErrors(errors) {
function storeErrors(errors) {
try {
localStorage.setItem(
ERROR_STORAGE_KEY,
+35 -40
View File
@@ -1,3 +1,4 @@
import { t } from '../i18n/index.js';
import {
PUSH_TO_TALK_HOLD_DELAY_MS,
MICROPHONE_VISUALIZER_GATE,
@@ -10,8 +11,8 @@ import {
resolveVoiceVisualizerSpeaker,
resolveVoiceControlHint,
gateVoiceVisualizerLevel,
resetVoiceVisualizerBars,
} from './realtimeInputPolicy.js';
import { VOICE_VISUALIZER_BARS } from './control.js';
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
/** Own physical push-to-talk gestures, microphone controls and audio meters. */
@@ -115,15 +116,16 @@ export class RealtimeInput {
// off if it is talking. Short taps never reach this point.
if (mayInterrupt) this.bargeIn?.();
this.beginPointerTurn?.('keydown');
this.ui.root.dataset.pushToTalk = 'held';
this.ui.update('root', { pushToTalk: 'held' });
this.setMicrophoneEnabled(true);
if (this.status === 'listening')
this.setStatus('listening', 'Release Space to send');
this.setStatus('listening', t('Release Space to send'));
} else {
this.start({ pushToTalk: true });
// start() performs a controlled stop() before connecting. Restore the
// marker it clears so the UI reflects this still-held gesture.
if (this.pushToTalkKeyHeld) this.ui.root.dataset.pushToTalk = 'held';
if (this.pushToTalkKeyHeld)
this.ui.update('root', { pushToTalk: 'held' });
}
}, PUSH_TO_TALK_HOLD_DELAY_MS);
};
@@ -190,11 +192,11 @@ export class RealtimeInput {
this.releasePointerHold?.();
if (!this.pushToTalkKeyHeld) return;
this.pushToTalkKeyHeld = false;
delete this.ui.root.dataset.pushToTalk;
this.ui.update('root', { pushToTalk: null });
if (!this.pushToTalkMode) return;
this.setMicrophoneEnabled(false);
if (this.status === 'listening')
this.setStatus('listening', 'Hold Space to talk');
this.setStatus('listening', t('Hold Space to talk'));
else this.updateVoiceButtonLabel();
}
@@ -204,8 +206,7 @@ export class RealtimeInput {
* @returns {void}
*/
setMicrophoneEnabled(enabled) {
if (this.ui?.root)
this.ui.root.dataset.microphone = enabled ? 'active' : 'muted';
this.ui?.update('root', { microphone: enabled ? 'active' : 'muted' });
this.stream?.getAudioTracks?.().forEach((track) => {
track.enabled = Boolean(enabled);
});
@@ -219,10 +220,7 @@ export class RealtimeInput {
startVoiceVisualizer(stream) {
this.stopVoiceVisualizer();
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
const bars = Array.from(
this.ui.root.querySelectorAll('.gev-voice-visualizer span'),
);
if (!AudioContextClass || !stream || !bars.length) return;
if (!AudioContextClass || !stream || !this.ui) return;
try {
const context = new AudioContextClass();
this.visualizerAudioContext = context;
@@ -251,17 +249,18 @@ export class RealtimeInput {
},
);
if (!signal) {
resetVoiceVisualizerBars(bars);
this.restVoiceVisualizer();
this.visualizerFrame = requestAnimationFrame(render);
return;
}
signal.analyser.getByteFrequencyData(signal.data);
const binCount = signal.data.length;
bars.forEach((bar, index) => {
const start = Math.floor((index / bars.length) * binCount);
const count = VOICE_VISUALIZER_BARS;
const bars = Array.from({ length: count }, (_, index) => {
const start = Math.floor((index / count) * binCount);
const end = Math.max(
start + 1,
Math.floor(((index + 1) / bars.length) * binCount),
Math.floor(((index + 1) / count) * binCount),
);
let energy = 0;
for (let bin = start; bin < end; bin++) energy += signal.data[bin];
@@ -274,15 +273,12 @@ export class RealtimeInput {
gateVoiceVisualizerLevel(normalized, gate),
0.72,
);
bar.style.setProperty(
'--audio-level',
`${Math.round(5 + shaped * 29)}px`,
);
bar.style.setProperty(
'--audio-opacity',
`${(0.5 + shaped * 0.5).toFixed(2)}`,
);
return {
level: `${Math.round(5 + shaped * 29)}px`,
opacity: `${(0.5 + shaped * 0.5).toFixed(2)}`,
};
});
this.ui?.setBars(bars);
this.visualizerFrame = requestAnimationFrame(render);
};
render();
@@ -352,9 +348,12 @@ export class RealtimeInput {
this.visualizerAudioContext.close().catch(() => {});
this.visualizerAudioContext = null;
}
resetVoiceVisualizerBars(
this.ui?.root?.querySelectorAll('.gev-voice-visualizer span'),
);
this.restVoiceVisualizer();
}
/** Return the waveform to its CSS standby baseline. */
restVoiceVisualizer() {
this.ui?.setBars(null);
}
/**
@@ -362,14 +361,13 @@ export class RealtimeInput {
* @returns {void}
*/
updateVoiceButtonLabel() {
if (!this.ui.buttonLabel) return;
this.ui.buttonLabel.textContent = 'MIC';
if (this.ui.helpDetail) {
this.ui.helpDetail.textContent = resolveVoiceControlHint(
this.pushToTalkMode,
this.pushToTalkKeyHeld,
);
}
if (!this.ui) return;
// The mic button's caption; the first-run tip names it the same way.
this.ui.update('mic', { text: () => t('MIC') });
const { pushToTalkMode, pushToTalkKeyHeld } = this;
this.ui.update('help', {
text: () => resolveVoiceControlHint(pushToTalkMode, pushToTalkKeyHeld),
});
}
setVoiceSpeaker(speaker, { keepVisualizerSpeaker = false } = {}) {
@@ -380,7 +378,7 @@ export class RealtimeInput {
nextSpeaker,
keepVisualizerSpeaker,
);
this.ui.root.dataset.speaker = nextSpeaker;
this.ui.update('root', { speaker: nextSpeaker });
if (shouldPauseRadioForVoice({ speaker: nextSpeaker }))
this.pauseRadioForVoice();
}
@@ -390,10 +388,7 @@ export class RealtimeInput {
this.pushToTalkKeyHeld = false;
this.spaceKeyHeld = false;
this.resetPushToTalkGesture();
if (this.ui?.root) {
delete this.ui.root.dataset.pushToTalk;
delete this.ui.root.dataset.microphone;
}
this.ui?.update('root', { pushToTalk: null, microphone: null });
}
detachBindings() {
+4 -15
View File
@@ -1,3 +1,5 @@
import { t } from '../i18n/index.js';
export const PUSH_TO_TALK_HOLD_DELAY_MS = 500;
// The input meter is intentionally stricter than the assistant-output meter:
@@ -150,8 +152,8 @@ export function resolveVoiceVisualizerSpeaker(
*/
export function resolveVoiceControlHint(pushToTalkMode, pushToTalkKeyHeld) {
return pushToTalkMode && pushToTalkKeyHeld
? 'Release Space to send'
: 'Hold Space to speak · tap Space to activate focused controls';
? t('Release Space to send')
: t('Hold Space to speak · tap Space to activate focused controls');
}
/**
@@ -170,16 +172,3 @@ export function gateVoiceVisualizerLevel(level, threshold) {
if (cleanLevel <= cleanThreshold) return 0;
return (cleanLevel - cleanThreshold) / (1 - cleanThreshold);
}
/**
* Restores the CSS-owned standby baseline for every visualizer bar.
* @param {Iterable<HTMLElement>|null|undefined} bars
* @returns {void}
*/
export function resetVoiceVisualizerBars(bars) {
if (!bars) return;
for (const bar of bars) {
bar.style.removeProperty('--audio-level');
bar.style.removeProperty('--audio-opacity');
}
}
+3 -7
View File
@@ -1,4 +1,5 @@
import assert from 'node:assert/strict';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import test from 'node:test';
import { GevRealtimeController } from './realtimeController.js';
import { PROGRESS_MAX_OUTPUT_TOKENS } from './realtimeTurns.js';
@@ -8,12 +9,7 @@ function controllerFixture({ runner = async () => ({ ok: true }) } = {}) {
const statuses = [];
const usage = [];
const controller = new GevRealtimeController({
ui: {
root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] },
status: { textContent: '' },
detail: { textContent: '', title: '' },
errorDetail: { textContent: '' },
},
ui: createTestVoiceUi(),
runner,
debugSink: null,
});
@@ -135,7 +131,7 @@ test('an old progress completion leaves the active in-band response alone', asyn
});
test('user speech stops an audible progress line: cancel and clear its audio', async () => {
const { controller, sent, event, progressLine } = controllerFixture();
const { sent, event, progressLine } = controllerFixture();
await progressLine('resp_p');
await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' });
sent.length = 0;
+13 -8
View File
@@ -1,4 +1,5 @@
import assert from 'node:assert/strict';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import test from 'node:test';
import { GevRealtimeController } from './realtimeController.js';
import { RealtimeInput } from './realtimeInput.js';
@@ -60,7 +61,7 @@ test('retained peer and channel callbacks cannot act after stop or enter a repla
async negotiate() { return 'answer'; },
},
debugSink: null,
ui: { root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] }, status: {}, detail: {} },
ui: createTestVoiceUi(),
});
await controller.start();
const old = peers[0];
@@ -106,7 +107,7 @@ test('an offer resolved after restart cannot change the replacement peer descrip
async negotiate() { return 'answer'; },
},
debugSink: null,
ui: { root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] }, status: {}, detail: {} },
ui: createTestVoiceUi(),
});
const firstStart = controller.start();
while (!finishOldOffer) await new Promise(resolve => setImmediate(resolve));
@@ -170,8 +171,9 @@ test('audio meter initialization failure releases its newly acquired AudioContex
close() { closes++; return Promise.resolve(); }
}
installGlobals(t, { window: { AudioContext } });
const ui = createTestVoiceUi();
const owner = new RealtimeInput({
readUi: () => ({ root: { querySelectorAll: () => [{ style: { setProperty() {}, removeProperty() {} } }] } }),
readUi: () => ui,
readStatus: () => 'listening', readStream: () => null, operations: {},
});
owner.startVoiceVisualizer({});
@@ -199,10 +201,13 @@ test('stopping one audio meter revokes retained frames without stopping another
requestAnimationFrame: callback => { frames.push(callback); return frames.length; },
cancelAnimationFrame() {},
});
const create = () => new RealtimeInput({
readUi: () => ({ root: { querySelectorAll: () => [{ style: { setProperty() {}, removeProperty() {} } }] } }),
readStatus: () => 'listening', readStream: () => null, operations: {},
});
const create = () => {
const ui = createTestVoiceUi();
return new RealtimeInput({
readUi: () => ui,
readStatus: () => 'listening', readStream: () => null, operations: {},
});
};
const first = create();
const second = create();
first.startVoiceVisualizer({}); second.startVoiceVisualizer({});
@@ -236,7 +241,7 @@ test('late action or viewport completion cannot resume a stopped or replacement
async negotiate() { return 'answer'; },
},
debugSink: null,
ui: { root: { dataset: {}, classList: { remove() {} }, querySelectorAll: () => [] }, status: {}, detail: {} },
ui: createTestVoiceUi(),
});
if (phase === 'viewport') controller._viewport.capture = () => new Promise(resolve => {
finish = () => resolve('data:image/jpeg;base64,abc');
+3 -3
View File
@@ -8,12 +8,12 @@ import {
// Voice cost control (repo-wide `godsEyeView.<feature>.<field>` convention;
// the neighbouring ERROR_STORAGE_KEY predates it).
export const VOICE_TIER_STORAGE_KEY = 'godsEyeView.voiceCost.tier';
const VOICE_TIER_STORAGE_KEY = 'godsEyeView.voiceCost.tier';
export const VOICE_LIMITS_STORAGE_KEY = 'godsEyeView.voiceCost.limits';
const VOICE_LIMITS_STORAGE_KEY = 'godsEyeView.voiceCost.limits';
/** Best-effort localStorage handle; absent in tests and locked-down browsers. */
export function voiceStorage(storage) {
function voiceStorage(storage) {
if (storage) return storage;
try {
return typeof localStorage === 'undefined' ? null : localStorage;
+26
View File
@@ -99,6 +99,18 @@ export function hasStructuredViewIdentity(result) {
);
}
export function isGlobeViewRefusal(result) {
return (
result?.action === 'move_camera' &&
result.ok === false &&
[
'globe-view-required',
'globe-reset-unavailable',
'globe-reset-failed',
].includes(result.reason)
);
}
export function responseInstructionForToolResult(result) {
if (result?.action === 'control_radio' && result.radioPlaybackSuppressed) {
if (result.audioState === 'paused') {
@@ -123,6 +135,20 @@ export function responseInstructionForToolResult(result) {
: 'Be concise: useful names, place, layer and one or two notable properties.',
].join(' ');
}
if (isGlobeViewRefusal(result)) {
// A refusal is not a completed action. Keep tool text out of instructions.
if (result.reason === 'globe-reset-failed')
return 'The globe did not rotate after the Globe View reset attempt. Say so briefly. Do not retry, offer or perform another reset for this request.';
if (result.reason === 'globe-reset-unavailable')
return 'The globe did not rotate. Ask the user to zoom out further; the Globe View preset does not fit this viewport. Do not offer or perform a reset.';
return 'The globe did not rotate. Say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" The suggested command is not a user request. Do not ask a question, perform a reset, or call tools in this reply.';
}
if (
(result?.action ?? result?.tool) === 'zoom_to_globe' &&
(result.ok !== true || result.cancelled)
) {
return 'The Globe View reset did not finish. Say so briefly; do not claim Globe View was reached. Do not retry the reset or rotate the globe for this request.';
}
if (result?.action === 'adjust_camera_zoom') {
return result.ok
? `Confirm once that the camera zoomed ${result.direction}. Do not claim any other change.`
+24 -2
View File
@@ -1,3 +1,4 @@
import { t } from '../i18n/index.js';
import {
silenceRadioForVoice,
startPreparedRadioAfterPlaybackReady,
@@ -246,7 +247,9 @@ export class RealtimeRadio {
handoffChannel?.readyState === 'open',
},
);
const stillCurrent = handoffEpoch === this.radioHandoffEpoch;
const stillCurrent =
handoffEpoch === this.radioHandoffEpoch &&
this.radioHandoffAttemptId === handoffAttemptId;
if (this.radioHandoffAttemptId === handoffAttemptId) {
this.radioHandoffInFlight = false;
this.radioHandoffAttemptId = null;
@@ -258,13 +261,32 @@ export class RealtimeRadio {
if (radioHandoff.result?.ok || radioHandoff.cancelled || !stillCurrent)
return;
if (this.dc?.readyState === 'open' && !this.userTurnPending) {
this.setStatus('listening', 'Radio did not start');
this.setStatus('listening', t('Radio did not start'));
this.queueResponseCreate(
'Say exactly one short correction: “The Radio station could not start. Voice is still on.”',
);
}
}
/** Revoke one call's prepared/preflight result, preserving other owners. */
cancelPlaybackResult(result) {
if (!result) return;
if (this.pendingRadioPlaybackResult === result) {
this.pendingRadioPlaybackResult = null;
this.radioHandoffDeferredByReservation = false;
}
if (this.radioHandoffInFlightResult !== result) return;
const attemptId = this.radioHandoffAttemptId;
const shouldStopPlayback = this.radioHandoffInFlight;
this.radioHandoffInFlightResult = null;
this.radioHandoffInFlight = false;
this.radioHandoffAttemptId = null;
// Keep the group epoch and sibling reservations; release ownership before
// Stop can synchronously notify playback observers.
if (shouldStopPlayback)
this.radioLayer?.stopPlayback?.({ origin: 'voice-cleanup', attemptId });
}
/** Invalidate delayed Radio work inside the requested authority scope. */
abortRadioSiblingTools({ responseId = null, scope = 'all' } = {}) {
for (const [controller, metadata] of this.activeRadioToolControllers) {
+111
View File
@@ -0,0 +1,111 @@
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
import { GevRealtimeController } from './realtimeController.js';
const tick = () => new Promise((resolve) => setImmediate(resolve));
const playback = (id) => ({
ok: true,
action: 'control_radio',
radioAction: 'play',
radioPlaybackRequested: true,
id,
});
function fixture(t) {
const attempts = [],
stopped = [],
corrections = [],
states = [];
const controller = new GevRealtimeController({
ui: createTestVoiceUi(),
runner: async () => ({ ok: false }),
debugSink: null,
onSessionEvent: (event) => states.push(event),
radioLayer: {
setVoiceDucked() {},
playForVoice: ({ attemptId }) =>
new Promise((resolve, reject) =>
attempts.push({ attemptId, resolve, reject }),
),
stopPlayback: (options) => stopped.push(options),
},
});
controller.queueResponseCreate = (message) => corrections.push(message);
controller.dc = { readyState: 'open', send() {}, close() {} };
t.after(() => controller.stop({ removeUi: true }));
return { controller, attempts, stopped, corrections, states };
}
test('default OpenAI reservation silences rejected frozen preflight and resumes retained result after failure', async (t) => {
const h = fixture(t),
c = h.controller;
const a = playback('A');
c.pendingRadioPlaybackResult = a;
const pending = c.startPendingRadioHandoff();
const reservation = c.reserveRadioToolHandoff();
assert.equal(c.pendingRadioPlaybackResult, a);
assert.equal(c.radioHandoffInFlight, false);
h.attempts[0].reject(new Error('obsolete frozen preflight'));
await pending;
assert.deepEqual(h.corrections, []);
assert.equal(
h.states.some((event) => event.detail === 'Radio did not start'),
false,
);
c.settleRadioToolHandoffReservation(reservation, { commit: false });
await tick();
assert.equal(h.attempts.length, 2);
assert.equal(c.radioHandoffInFlightResult, a);
assert.notEqual(h.attempts[0].attemptId, h.attempts[1].attemptId);
h.attempts[1].reject(new Error('current preflight failed'));
await tick();
assert.equal(h.corrections.length, 1);
assert.equal(c.status, 'listening');
});
test('default OpenAI group cancellation silences rejected preflight without blocking the next failure correction', async (t) => {
const h = fixture(t),
c = h.controller;
c.pendingRadioPlaybackResult = playback('A');
const old = c.startPendingRadioHandoff();
c.cancelRadioHandoff();
h.attempts[0].reject(new Error('cancelled preflight'));
await old;
assert.deepEqual(h.corrections, []);
c.pendingRadioPlaybackResult = playback('B');
const current = c.startPendingRadioHandoff();
h.attempts[1].resolve(false);
await current;
assert.equal(h.corrections.length, 1);
});
test('per-result cancellation retains sibling result and epoch and ignores late failure or duplicate cancellation', async (t) => {
const h = fixture(t),
c = h.controller;
const a = playback('A'),
b = playback('B');
c.pendingRadioPlaybackResult = a;
const old = c.startPendingRadioHandoff();
const siblingEpoch = c.radioHandoffEpoch;
c.pendingRadioPlaybackResult = b;
c._radio.cancelPlaybackResult(a);
c._radio.cancelPlaybackResult(a);
c._radio.cancelPlaybackResult(playback('unknown'));
assert.equal(c.radioHandoffEpoch, siblingEpoch);
assert.equal(c.pendingRadioPlaybackResult, b);
assert.equal(h.stopped.length, 1);
h.attempts[0].reject(new Error('late cancelled preflight'));
await old;
assert.deepEqual(h.corrections, []);
const current = c.startPendingRadioHandoff();
const reservation = c.reserveRadioToolHandoff();
c._radio.cancelPlaybackResult(b);
assert.equal(c._radio.radioToolHandoffReservations.has(reservation), true);
h.attempts[1].resolve(false);
await current;
assert.deepEqual(h.corrections, []);
c.settleRadioToolHandoffReservation(reservation, { commit: false });
await tick();
assert.equal(h.attempts.length, 2, 'a cancelled result cannot be resumed');
});
+13 -7
View File
@@ -16,18 +16,24 @@ export function createRealtimeSession({
controller,
capabilities: { costControls: true, pushToTalk: true },
start: (settings) => controller.start(settings),
getStartOptions: () => ({ pushToTalk: Boolean(controller.pushToTalkMode) }),
stop: (settings) => controller.stop(settings),
sendText: (text) => controller.sendTextCommand(text),
sendMapEvent: (event) => controller.notifyMapEvent(event),
ignoreButtonClick: () => Boolean(controller.spaceKeyHeld),
bindControls() {
if (controller.ui.tierButton) {
controller.tierHandler = () => controller.toggleVoiceTier();
controller.ui.tierButton.addEventListener(
'click',
controller.tierHandler,
);
}
// Kept as the remover; a full stop releases it. The cost readout and
// tier toggle are worded again in a new locale.
const offTier = controller.ui.on('tier', () =>
controller.toggleVoiceTier(),
);
const offLocale = controller.ui.on('locale', () =>
controller.syncCostUi(),
);
controller.tierHandler = () => {
offTier();
offLocale();
};
controller.syncCostUi();
controller.bindPushToTalkShortcut();
},
+236 -16
View File
@@ -1,10 +1,12 @@
import { realtimeSessionEvent } from './realtimeEvents.js';
import { t } from '../i18n/index.js';
import { readLayerLifecycleSummary } from './layerSummary.js';
import {
CALL_DEDUPE_MS,
SUPERSEDED_RESPONSE_MEMORY,
shouldStopVoiceAfterRadioTool,
responseInstructionForToolResult,
isGlobeViewRefusal,
extractFunctionCalls,
callDedupeKeys,
parseArguments,
@@ -12,7 +14,7 @@ import {
import { narrationLabel } from './speech.js';
/** Out-of-band progress responses carry this metadata tag. */
export const PROGRESS_RESPONSE_TAG = 'progress';
const PROGRESS_RESPONSE_TAG = 'progress';
/**
* Speaks one code-authored progress line out of band. The line travels as
@@ -65,6 +67,11 @@ export class RealtimeTurns {
this.pendingUserTextResponse = false;
this.activeResponseId = null;
this.supersededResponseIds = new Set();
this.globeRefusals = new Map();
this.globeAdmissionGates = new Map();
this.globeResets = new Map();
this.globeResponseRequests = new Map();
this.globeRequest = { failed: false, pending: new Set() };
this.activeToolAbortControllers = new Set();
this.narration = null;
this.narrationSequence = 0;
@@ -136,7 +143,8 @@ export class RealtimeTurns {
content: [{ type: 'input_text', text: cleanText }],
},
};
this.sendRealtimeEvent(itemEvent, 'client.user_text');
if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return;
this.globeRequest = { failed: false, pending: new Set() };
this.requestUserTextResponse();
}
@@ -277,7 +285,7 @@ export class RealtimeTurns {
eventId: payload.event_id,
activeResponseMessage: payload.error?.message || null,
});
this.setStatus('listening', 'Ask or command');
this.setStatus('listening', t('Ask or command'));
return;
}
// A conversation.item.delete for a stale viewport screenshot can land
@@ -329,6 +337,7 @@ export class RealtimeTurns {
this.userTurnPending = true;
this.pendingResponseInstructions = null;
this.cancelRadioHandoff({ abortTools: true });
this.globeRequest = { failed: false, pending: new Set() };
this.setVoiceSpeaker('user');
// Point-and-ask: read the pointer as speech starts (open mic), or reuse
// the push-to-talk keydown snapshot, before the user audio commits.
@@ -383,6 +392,27 @@ export class RealtimeTurns {
}
const toolResponseId = payload.response_id || payload.response?.id || null;
// Follow-up responses retain reset ownership until fresh admitted input.
const arrivalRequest = this.globeRequest;
const globeRequest =
this.globeResponseRequests.get(toolResponseId) ?? arrivalRequest;
// Later siblings must wait for globe admission before they can mutate state.
// Other tool responses keep their existing concurrency.
let heldForAdmission = false;
while (toolResponseId && this.globeAdmissionGates.has(toolResponseId)) {
// Drop claimed duplicates before every wait, before their keys can expire.
if (
calls.every((call) =>
callDedupeKeys(call).some((key) => this.processedCalls.has(key)),
)
)
return;
heldForAdmission = true;
await this.globeAdmissionGates.get(toolResponseId).promise;
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
return;
}
// A newer typed command superseded the response these calls belong to.
// They are stale intent — dispatching one would let the old turn mutate
// the map after the operator asked for something else.
@@ -393,7 +423,10 @@ export class RealtimeTurns {
// hazard `callDedupeKeys` is written to avoid). So each refused call gets
// a terminal output saying plainly that the turn moved on. No
// `response.create` follows — the deferred typed turn is the single answer.
if (this.isSupersededResponse(toolResponseId)) {
if (
this.isSupersededResponse(toolResponseId) ||
(heldForAdmission && arrivalRequest !== this.globeRequest)
) {
this.pruneProcessedCalls();
for (const call of calls) {
const keys = callDedupeKeys(call);
@@ -414,7 +447,7 @@ export class RealtimeTurns {
return;
}
this.setStatus('executing', 'Running command');
this.setStatus('executing', t('Running command'));
this.pruneProcessedCalls();
let sentOutput = false;
let lastResult = null;
@@ -428,7 +461,19 @@ export class RealtimeTurns {
if (keys.some((key) => this.processedCalls.has(key))) continue;
keys.forEach((key) => this.processedCalls.set(key, performance.now()));
if (this.globeRefusals.get(toolResponseId)?.blocking) {
this.sendToolOutput(call.call_id || call.id, {
ok: false,
action: call.name,
skipped: true,
error: 'Not run: this response must wait for the Globe View reply.',
});
continue;
}
let result;
let globeAdmissionGate = null;
let globeResetGate = null;
const resultChannel = this.dc;
let radioHandoffEpochAtStart = this.radio.radioHandoffEpoch;
let toolController = null;
@@ -442,6 +487,80 @@ export class RealtimeTurns {
this.narration?.preamble();
try {
const parsedArguments = parseArguments(call.arguments);
if (
toolResponseId &&
call.name === 'move_camera' &&
String(parsedArguments.motion || '').toLowerCase() === 'rotate_globe'
) {
let resolve;
const promise = new Promise((settle) => {
resolve = settle;
});
globeAdmissionGate = { promise, resolve };
this.globeAdmissionGates.set(toolResponseId, globeAdmissionGate);
// OpenAI delivers sibling calls concurrently. A reset dispatched
// first must finish before rotation evaluates the camera's fit.
// Hold later siblings at admission while this rotation waits.
const resets = this.globeResets.get(toolResponseId);
if (
resets ||
globeRequest.failed ||
globeRequest.pending.size ||
arrivalRequest !== this.globeRequest
) {
while (resets?.pending.size || globeRequest.pending.size)
await Promise.all(
[
...new Set([
...(resets?.pending || []),
...globeRequest.pending,
]),
].map((gate) => gate.promise),
);
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
return;
keys.forEach((key) =>
this.processedCalls.set(key, performance.now()),
);
const superseded =
this.isSupersededResponse(toolResponseId) ||
arrivalRequest !== this.globeRequest;
if (superseded || resets?.failed || globeRequest.failed) {
result = {
ok: false,
action: call.name,
reason: 'globe-reset-failed',
skipped: true,
...(superseded ? { superseded: true } : {}),
error: superseded
? 'Superseded by a newer command from the operator — this call was not run.'
: 'Not run: the Globe View reset/rotation sequence failed or was cancelled.',
};
}
}
}
if (toolResponseId && call.name === 'zoom_to_globe') {
let resolve;
const promise = new Promise((settle) => {
resolve = settle;
});
let resets = this.globeResets.get(toolResponseId);
if (!resets) {
resets = { pending: new Set(), failed: false };
this.globeResets.set(toolResponseId, resets);
// Retain settled outcomes for later siblings, without evicting an
// active reset that another response still needs to await.
for (const [id, record] of this.globeResets) {
if (this.globeResets.size <= SUPERSEDED_RESPONSE_MEMORY) break;
if (!record.pending.size && id !== toolResponseId)
this.globeResets.delete(id);
}
}
globeResetGate = { promise, resolve, resets, request: globeRequest };
resets.pending.add(globeResetGate);
globeRequest.pending.add(globeResetGate);
}
const isRadioControlCall = call.name === 'control_radio';
const isRadioVisibilityCall =
call.name === 'set_layer_visibility' &&
@@ -498,7 +617,7 @@ export class RealtimeTurns {
authorityDomain: radioAuthorityDomain,
});
}
result = await (this.actionExecutor || this.runner)(
result ??= await (this.actionExecutor || this.runner)(
call.name,
parsedArguments,
{
@@ -514,7 +633,28 @@ export class RealtimeTurns {
radioHandoffEpochAtStart === this.radio.radioHandoffEpoch),
},
);
if (
globeAdmissionGate &&
this.globeResets.has(toolResponseId) &&
result?.reason === 'globe-view-required'
)
result = { ...result, reason: 'globe-reset-failed' };
if (!this.ownsConversation(eventChannel)) return;
if (
isGlobeViewRefusal(result) &&
toolResponseId &&
arrivalRequest === this.globeRequest &&
!this.isSupersededResponse(toolResponseId)
) {
this.globeRefusals.set(toolResponseId, {
result,
followupQueued: false,
blocking: !result.skipped,
});
while (this.globeRefusals.size > SUPERSEDED_RESPONSE_MEMORY) {
this.globeRefusals.delete(this.globeRefusals.keys().next().value);
}
}
if (result?.ok && result.radioPlaybackRequested) {
const sessionIsCurrent =
this.activeToolAbortControllers.has(toolController) &&
@@ -594,6 +734,23 @@ export class RealtimeTurns {
: {}),
};
} finally {
if (globeResetGate) {
const { resets, request } = globeResetGate;
if (result?.ok !== true || result.cancelled) {
resets.failed = true;
request.failed = true;
}
resets.pending.delete(globeResetGate);
request.pending.delete(globeResetGate);
globeResetGate.resolve();
}
if (globeAdmissionGate) {
if (
this.globeAdmissionGates.get(toolResponseId) === globeAdmissionGate
)
this.globeAdmissionGates.delete(toolResponseId);
globeAdmissionGate.resolve();
}
if (toolController) {
this.activeToolAbortControllers.delete(toolController);
this.radio.releaseTool(toolController);
@@ -670,15 +827,40 @@ export class RealtimeTurns {
});
}
if (!this.ownsConversation(eventChannel)) return;
// Keep the Radio handoff wording authoritative even when another tool
// result follows Radio in the same multi-intent response.
this.queueResponseCreate(
responseInstructionForToolResult(
this.radio.pendingRadioPlaybackResult || lastResult,
),
);
const refusal = this.globeRefusals.get(toolResponseId);
// A newer turn may arrive during admission or the viewport-image await.
// Preserve its reply; neither the refusal map nor lastResult may revive
// an old refusal follow-up after supersession.
if (
(this.isSupersededResponse(toolResponseId) ||
arrivalRequest !== this.globeRequest) &&
(refusal || isGlobeViewRefusal(lastResult) || lastResult?.superseded)
) {
// Complete an idle turn below, but leave a newer running tool's status
// authoritative while suppressing this obsolete refusal follow-up.
if (this.activeToolAbortControllers.size) return;
} else if (refusal) {
// Even an earlier sibling finishing late cannot replace the reset instruction.
if (!refusal.followupQueued) {
refusal.followupQueued = true;
this.queueResponseCreate({
instructions: responseInstructionForToolResult(refusal.result),
tool_choice: 'none',
});
}
} else {
// Keep Radio handoff wording authoritative for ordinary multi-tool replies.
const followupResult =
this.radio.pendingRadioPlaybackResult || lastResult;
const instructions = responseInstructionForToolResult(followupResult);
this.queueResponseCreate(
isGlobeViewRefusal(followupResult)
? { instructions, tool_choice: 'none' }
: instructions,
);
}
}
this.setStatus('listening', 'Ask or command');
this.setStatus('listening', t('Ask or command'));
}
/**
@@ -965,6 +1147,22 @@ export class RealtimeTurns {
this.userTurnPending = false;
this.activeResponseId =
payload.response?.id || payload.response_id || null;
// Late calls retain only their globe reset/request ownership. Ordinary
// tool admission and concurrency continue to use their existing policy.
if (
this.activeResponseId &&
!this.globeResponseRequests.has(this.activeResponseId)
) {
this.globeResponseRequests.set(
this.activeResponseId,
this.globeRequest,
);
while (this.globeResponseRequests.size > SUPERSEDED_RESPONSE_MEMORY)
this.globeResponseRequests.delete(
this.globeResponseRequests.keys().next().value,
);
}
this.setVoiceSpeaker('ai');
return;
}
@@ -997,7 +1195,7 @@ export class RealtimeTurns {
// connection is still live. Recover to listening so the user can retry
// (mirrors the transient-blip philosophy, H8).
if (this.dc?.readyState === 'open') {
this.setStatus('listening', 'Ask or command');
this.setStatus('listening', t('Ask or command'));
}
}
if (!this.radio.pendingRadioPlaybackResult) {
@@ -1044,7 +1242,8 @@ export class RealtimeTurns {
const sent = this.sendRealtimeEvent(
{
type: 'response.create',
response: { instructions },
response:
typeof instructions === 'string' ? { instructions } : instructions,
},
'client.response_create.tool_followup',
);
@@ -1056,6 +1255,20 @@ export class RealtimeTurns {
controller.abort();
this.activeToolAbortControllers.clear();
this.radio.clearTools();
this.releaseGlobeResets();
}
releaseGlobeResets() {
for (const resets of this.globeResets.values()) {
if (!resets.pending.size) continue;
resets.failed = true;
for (const gate of resets.pending) {
gate.request.failed = true;
gate.request.pending.delete(gate);
gate.resolve();
}
resets.pending.clear();
}
}
reset() {
@@ -1067,6 +1280,13 @@ export class RealtimeTurns {
this.pendingUserTextResponse = false;
this.activeResponseId = null;
this.supersededResponseIds.clear();
this.globeRefusals.clear();
for (const gate of this.globeAdmissionGates.values()) gate.resolve();
this.globeAdmissionGates.clear();
this.releaseGlobeResets();
this.globeResets.clear();
this.globeResponseRequests.clear();
this.globeRequest = { failed: false, pending: new Set() };
this.narration?.cancel();
this.assistantAudioPlaying = false;
this.speechResponseIds.clear();
+261
View File
@@ -0,0 +1,261 @@
import assert from 'node:assert/strict';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import test from 'node:test';
import { GevRealtimeController } from './realtimeController.js';
function fixture({ pointer = null, onUserSend = () => {} } = {}) {
const sent = [];
const executed = [];
const rejectedTexts = new Set();
let rejectResponse = false;
const controller = new GevRealtimeController({
runner: async (name) => {
executed.push(name);
return { ok: true };
},
backend: {},
debugSink: null,
pointer,
ui: createTestVoiceUi(),
});
controller.dc = {
readyState: 'open',
send(body) {
const value = JSON.parse(body);
const text = value.item?.role === 'user' && value.item.content?.[0]?.text;
if (value.item?.role === 'user') onUserSend();
if (
rejectedTexts.has(text) ||
(rejectResponse && value.type === 'response.create')
)
throw new Error('Synthetic channel rejection');
sent.push(value);
},
close() {
this.readyState = 'closed';
},
};
controller.status = 'listening';
controller._turns.narration = null;
const event = (payload) =>
controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
const created = (id) => event({ type: 'response.created', response: { id } });
const done = (id) =>
event({
type: 'response.done',
response: { id, status: 'completed', output: [] },
});
const creates = () =>
sent.filter(
(x) =>
x.type === 'response.create' &&
x.response?.metadata?.gev !== 'progress',
);
const texts = () =>
sent
.filter((x) => x.item?.role === 'user')
.map((x) => x.item.content[0].text);
return {
controller,
sent,
executed,
rejectedTexts,
event,
created,
done,
creates,
texts,
rejectResponses(value) {
rejectResponse = value;
},
};
}
test('rejected idle text schedules no answer, and a successful successor answers once', async () => {
const f = fixture();
try {
f.rejectedTexts.add('rejected');
assert.equal(f.controller.sendTextCommand('rejected'), undefined);
assert.equal(f.creates().length, 0);
assert.equal(f.controller.pendingUserTextResponse, false);
assert.equal(f.controller.responseCreatePending, false);
assert.equal(f.controller.sendTextCommand('successor'), undefined);
assert.deepEqual(f.texts(), ['successor']);
assert.equal(f.creates().length, 1);
await f.created('successor');
await f.done('successor');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
test('rejected text behind an active response cannot queue an answer', async () => {
const f = fixture();
try {
await f.created('active');
f.controller.queueResponseCreate('Old tool follow-up');
assert.equal(
f.controller.pendingResponseInstructions,
'Old tool follow-up',
);
const runningTool = new AbortController();
f.controller._turns.activeToolAbortControllers.add(runningTool);
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.equal(f.controller.pendingUserTextResponse, false);
assert.equal(f.controller.pendingResponseInstructions, null);
assert.equal(f.controller._turns.isSupersededResponse('active'), true);
assert.equal(runningTool.signal.aborted, true);
const staleCall = {
type: 'response.function_call_arguments.done',
response_id: 'active',
call_id: 'stale-call',
name: 'fly_to_location',
arguments: '{"query":"Paris"}',
};
await f.event(staleCall);
await f.event(staleCall);
assert.deepEqual(f.executed, []);
const outputs = f.sent.filter(
(x) => x.item?.type === 'function_call_output',
);
assert.equal(outputs.length, 1);
assert.equal(outputs[0].item.call_id, 'stale-call');
assert.equal(JSON.parse(outputs[0].item.output).superseded, true);
await f.done('active');
assert.equal(f.creates().length, 0);
} finally {
f.controller.stop();
}
});
test('rejected text still prepares narration and pointer context before sending', () => {
const steps = [];
const f = fixture({
pointer: {
beginTurn() {
steps.push('pointer');
return null;
},
clear() {},
},
onUserSend() {
steps.push('send');
},
});
f.controller._turns.narration = {
cancel() {
steps.push('cancel');
},
userTurnEnded() {
steps.push('user-turn-ended');
},
};
try {
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.deepEqual(steps, ['cancel', 'user-turn-ended', 'pointer', 'send']);
assert.equal(f.creates().length, 0);
} finally {
f.controller.stop();
}
});
test('earlier accepted deferred input survives a newer failed send and answers once', async () => {
const f = fixture();
try {
await f.created('active');
f.controller.sendTextCommand('accepted');
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.deepEqual(f.texts(), ['accepted']);
assert.equal(f.controller.pendingUserTextResponse, true);
assert.equal(f.creates().length, 0);
await f.done('active');
assert.equal(f.creates().length, 1);
await f.done('active');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
test('rejected text preserves an outstanding create without adding deferred input', async () => {
const f = fixture();
try {
f.controller.sendTextCommand('accepted');
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.equal(f.controller.responseCreatePending, true);
assert.equal(f.controller.pendingUserTextResponse, false);
await f.created('accepted');
await f.done('accepted');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
for (const acceptedFirst of [false, true]) {
test(`progress completion answers only accepted deferred input (accepted first=${acceptedFirst})`, async () => {
const f = fixture();
try {
assert.equal(
f.controller._turns.speakProgressLine('Checking the map.'),
true,
);
await f.event({
type: 'response.created',
response: { id: 'progress', metadata: { gev: 'progress' } },
});
if (acceptedFirst) f.controller.sendTextCommand('accepted');
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.equal(f.controller.pendingUserTextResponse, acceptedFirst);
await f.event({
type: 'response.done',
response: {
id: 'progress',
status: 'cancelled',
metadata: { gev: 'progress' },
},
});
assert.equal(f.creates().length, acceptedFirst ? 1 : 0);
} finally {
f.controller.stop();
}
});
}
test('accepted text with a rejected response.create releases the pending latch for a successor', async () => {
const f = fixture();
try {
f.rejectResponses(true);
f.controller.sendTextCommand('accepted without response');
assert.equal(f.controller.responseCreatePending, false);
assert.equal(f.creates().length, 0);
f.rejectResponses(false);
f.controller.sendTextCommand('successor');
assert.deepEqual(f.texts(), ['accepted without response', 'successor']);
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
test('successful deferred typed burst still retains both items and coalesces one answer', async () => {
const f = fixture();
try {
await f.created('active');
f.controller.sendTextCommand('first');
f.controller.sendTextCommand('second');
assert.deepEqual(f.texts(), ['first', 'second']);
assert.equal(f.creates().length, 0);
await f.done('active');
await f.done('active');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
+6 -6
View File
@@ -24,10 +24,10 @@ export function readViewerCameraKey() {
// tall portrait window produced an oversized capture whose dc.send could throw.
// Cap total pixels (clamps both dimensions) and drop the image entirely if the
// encoded data URL is still too big for the data channel.
export const VIEWPORT_MAX_PIXELS = 1200 * 900;
const VIEWPORT_MAX_PIXELS = 1200 * 900;
// ~1.08 MP, matches the old 1200px-wide landscape budget
export const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
export async function captureViewportImage() {
// Both engines draw with preserveDrawingBuffer (Cesium's contextOptions;
@@ -80,11 +80,11 @@ export async function captureViewportImage() {
}
/** Side of the point-and-ask crop, in CSS pixels, and its output size. */
export const POINTER_CROP_SIZE = 512;
const POINTER_CROP_SIZE = 512;
/** A pointer snapshot older than this no longer matches the frame. */
export const POINTER_CROP_MAX_AGE_MS = 20000;
const POINTER_CROP_MAX_AGE_MS = 20000;
/** View scales where a crop shows buildings and terrain, not a smear. */
export const POINTER_CROP_SCALES = new Set(['local', 'city']);
const POINTER_CROP_SCALES = new Set(['local', 'city']);
/**
* Square source rectangle around a canvas point, clamped inside the canvas,
@@ -279,7 +279,7 @@ export async function renderFreshFrame(globe) {
}
}
export function isNearlyBlackFrame(ctx, width, height) {
function isNearlyBlackFrame(ctx, width, height) {
const sampleWidth = Math.min(48, width);
const sampleHeight = Math.min(32, height);
if (!sampleWidth || !sampleHeight) return true;
+337 -12
View File
@@ -9,11 +9,27 @@
* referents the card lists (the referent registry reads the same answer, so
* "the second one" always means the second row on screen).
*
* What the model receives stays English (`analystHeadline` feeds the spoken
* answer); the card shows the same answer in the active language.
*
* Pure: no DOM, no Cesium.
* @module voice/resultDisplay
*/
import { voiceLayer } from './layerManifest.js';
import {
formatDate,
formatNumber,
getLocale,
msg,
t,
tn,
translateKnown,
} from '../i18n/index.js';
import { layerNameForId, sourceLabel } from '../ui/layerNames.js';
import { localizePlaceName } from '../ui/placeNames.js';
import { translateStatusLine } from '../ui/statusText.js';
import { formatQuantity } from '../ui/unitLabels.js';
const LAYER_NOUNS = Object.freeze({
flights: 'aircraft',
@@ -43,7 +59,7 @@ function notesOf(...values) {
];
}
export function analystNounFor(layerKeys) {
function analystNounFor(layerKeys) {
const keys = [...new Set(layerKeys.filter(Boolean))];
if (keys.length !== 1) return 'results';
const key = keys[0];
@@ -52,13 +68,17 @@ export function analystNounFor(layerKeys) {
/** The count/scope headline shared by the analyst card and spoken answer. */
const ANALYST_HEADLINE_MAX = 64;
// The card's headline wraps, and a translation runs longer than the English:
// "Al menos 12 aeronaves militares en un radio de 40 km de Los Ángeles" is 67.
// English keeps the spoken bound, so its card title still equals `say`.
const SHOWN_HEADLINE_MAX = 120;
function boundedHeadline(value) {
function boundedHeadline(value, max = ANALYST_HEADLINE_MAX) {
const text = String(value || '')
.replace(/\s+/g, ' ')
.trim();
if (text.length <= ANALYST_HEADLINE_MAX) return text;
return `${text.slice(0, ANALYST_HEADLINE_MAX - 1).trimEnd()}…`;
if (text.length <= max) return text;
return `${text.slice(0, max - 1).trimEnd()}…`;
}
export function analystHeadline(result) {
@@ -72,6 +92,307 @@ export function analystHeadline(result) {
);
}
/* ---------------- the analyst card in the active language ---------------- */
/**
* The count and its noun as the card shows them. The English matches
* `analystNounFor`, one form for any count as the spoken headline has it.
*/
function shownCount(count, noun) {
const values = { n: formatNumber(count) };
switch (noun) {
case 'aircraft':
return tn(count, '{n} aircraft', '{n} aircraft', values);
case 'military aircraft':
return tn(
count,
'{n} military aircraft',
'{n} military aircraft',
values,
);
case 'ships':
return tn(count, '{n} ships', '{n} ships', values);
case 'satellites':
return tn(count, '{n} satellites', '{n} satellites', values);
case 'launches':
return tn(count, '{n} launches', '{n} launches', values);
case 'earthquakes':
return tn(count, '{n} earthquakes', '{n} earthquakes', values);
case 'fires':
return tn(count, '{n} fires', '{n} fires', values);
case 'fire perimeters':
return tn(count, '{n} fire perimeters', '{n} fire perimeters', values);
case 'cyclones':
return tn(count, '{n} cyclones', '{n} cyclones', values);
case 'buses':
return tn(count, '{n} buses', '{n} buses', values);
case 'bike share':
return tn(count, '{n} bike share', '{n} bike share', values);
case 'flock cameras':
return tn(count, '{n} flock cameras', '{n} flock cameras', values);
case 'military bases':
return tn(count, '{n} military bases', '{n} military bases', values);
case 'datacenters':
return tn(count, '{n} datacenters', '{n} datacenters', values);
case 'dams':
return tn(count, '{n} dams', '{n} dams', values);
case 'air quality':
return tn(count, '{n} air quality', '{n} air quality', values);
case 'results':
return tn(count, '{n} results', '{n} results', values);
default:
return `${values.n} ${noun}`;
}
}
const shownRadius = (km) => formatQuantity(Number(km), 'km');
const FIXED_SCOPES = new Map(
[msg('anywhere in the loaded data'), msg('in view')].map((text) => [
text,
text,
]),
);
/**
* The engine's scope phrases (analystEngine, the Contacts window in
* gevActions) as the card shows them. The English is exactly the engine's;
* a phrase not listed shows as given.
*/
const SCOPE_PATTERNS = [
[
/^within (\d+(?:\.\d+)?) km of the view centre$/,
([, km]) =>
t('within {radius} of the view centre', { radius: shownRadius(km) }),
],
[
/^within (\d+(?:\.\d+)?) km of (.+) \(Contacts window\)$/,
([, km, place]) =>
t('within {radius} of {place} (Contacts window)', {
radius: shownRadius(km),
place: localizePlaceName(place),
}),
],
[
/^within (\d+(?:\.\d+)?) km of (.+)$/,
([, km, place]) =>
t('within {radius} of {place}', {
radius: shownRadius(km),
place: localizePlaceName(place),
}),
],
[
/^within (\d+(?:\.\d+)?) km$/,
([, km]) => t('within {radius}', { radius: shownRadius(km) }),
],
[/^over (.+)$/, ([, region]) => t('over {region}', { region })],
];
function shownScope(text) {
const scope = String(text || '').trim();
if (!scope) return '';
if (FIXED_SCOPES.has(scope)) return t(FIXED_SCOPES.get(scope));
for (const [pattern, show] of SCOPE_PATTERNS) {
const match = pattern.exec(scope);
if (match) return show(match);
}
return translateKnown(scope);
}
/** The analyst headline in the active language, for the card. */
function shownAnalystHeadline(result) {
const keys = (result?.coverage?.layersQueried || []).map(
(layer) => layer.layerKey,
);
const count = Number(result?.count) || 0;
const counted = shownCount(count, analystNounFor(keys));
const items =
result?.complete === false
? t('At least {items}', { items: counted })
: counted;
const scope = shownScope(result?.scopeLabel);
return boundedHeadline(
scope ? t('{items} {scope}', { items, scope }) : items,
getLocale() === 'en' ? ANALYST_HEADLINE_MAX : SHOWN_HEADLINE_MAX,
);
}
/** A layer key as the card names it. */
function shownLayerKey(key) {
const text = String(key || '').trim();
return layerNameForId(text) || text;
}
const shownLayerKeys = (list) =>
String(list || '')
.split(/,\s*/)
.map(shownLayerKey)
.filter(Boolean)
.join(', ');
/** The feed windows layers declare in the voice manifest. */
const FIXED_WINDOWS = new Map(
[msg('last 24 h'), msg('past 30 d and upcoming')].map((text) => [text, text]),
);
function shownWindow(text) {
return String(text || '')
.split(/;\s*/)
.map((part) => part.trim())
.filter(Boolean)
.map((part) =>
FIXED_WINDOWS.has(part)
? t(FIXED_WINDOWS.get(part))
: translateKnown(part),
)
.join('; ');
}
const UTC_CLOCK = Object.freeze({
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
hourCycle: 'h23',
timeZone: 'UTC',
});
const UTC_DATE = Object.freeze({
day: '2-digit',
month: '2-digit',
year: 'numeric',
timeZone: 'UTC',
});
/**
* The note's ISO snapshot time as the card shows it: "14:32:05 UTC", dated
* ("15.10.2026, 14:32:05 UTC") when not from today (UTC). English and
* non-time values show as given.
*/
export function shownSnapshotTime(iso, now = Date.now()) {
const time = Date.parse(iso);
if (getLocale() === 'en' || !Number.isFinite(time)) return iso;
const clock = `${formatDate(time, UTC_CLOCK)} UTC`;
const today = new Date(now).toISOString().slice(0, 10);
return new Date(time).toISOString().slice(0, 10) === today
? clock
: `${formatDate(time, UTC_DATE)}, ${clock}`;
}
/** Coverage notes and caveats the engine and the action runner write. */
const FIXED_NOTES = new Map(
[
msg('the flights layer loads by viewport'),
msg('distance is ground distance, not overhead'),
msg(
'the count is a lower bound because at least one layer reached its retained limit.',
),
].map((text) => [text, text]),
);
const LAYER_STATUS_NOTES = Object.freeze({
off: (label) => t('{label} off', { label }),
loading: (label) => t('{label} loading', { label }),
error: (label) => t('{label} error', { label }),
});
const NOTE_PATTERNS = [
[
/^counted the first ([\d,]+) loaded records$/,
([, returned]) =>
t('counted the first {returned} loaded records', {
returned: formatNumber(Number(returned.replace(/,/g, ''))),
}),
],
[
/^counted ([\d,]+) of ([\d,]+) loaded records$/,
([, returned, total]) =>
t('counted {returned} of {total} loaded records', {
returned: formatNumber(Number(returned.replace(/,/g, ''))),
total: formatNumber(Number(total.replace(/,/g, ''))),
}),
],
[
/^(.+) just turned on — still loading, counts will rise$/,
([, layers]) =>
t('{layers} just turned on — still loading, counts will rise', {
layers: shownLayerKeys(layers),
}),
],
[
// The note ends in a period unless a lower-bound clause follows it.
/^Contacts panel snapshot(?: at (\S+))? — the same retained cohort supplies the card, speech, and follow-up memory(\.?)$/,
([, time, period]) =>
(time
? t(
'Contacts panel snapshot at {time} — the same retained cohort supplies the card, speech, and follow-up memory',
{ time: shownSnapshotTime(time) },
)
: t(
'Contacts panel snapshot — the same retained cohort supplies the card, speech, and follow-up memory',
)) + period,
],
// Transit's empty note when no feed covers the view (transit/queries.js);
// the model gets the English.
[
/^Fly below ([\d,]+) km to a covered region$/,
([, km]) =>
t('Fly below {km} km to a covered region', {
km: formatNumber(Number(km.replace(/,/g, ''))),
}),
],
[
/^No feed here yet · ([\d,]+) regions available$/,
([, regions]) => {
const count = Number(regions.replace(/,/g, ''));
return tn(
count,
'No feed here yet · {n} region available',
'No feed here yet · {n} regions available',
{ n: formatNumber(count) },
);
},
],
[
/^([a-z0-9-]+) (off|loading|error)$/,
([, key, status]) =>
layerNameForId(key)
? LAYER_STATUS_NOTES[status](shownLayerKey(key))
: null,
],
[
/^([a-z0-9-]+): (.+)$/,
([, key, note]) =>
layerNameForId(key) ? `${shownLayerKey(key)}: ${shownNote(note)}` : null,
],
];
function shownNote(text) {
if (FIXED_NOTES.has(text)) return t(FIXED_NOTES.get(text));
for (const [pattern, show] of NOTE_PATTERNS) {
const match = pattern.exec(text);
const shown = match ? show(match) : null;
if (shown) return shown;
}
// A layer's empty note can be its status line, English pieces joined by
// " · " ("feeds 1090, 978 stale · 0 heard", "TTC 0 (95s old)").
const known = translateKnown(text);
return known !== text ? known : translateStatusLine(text);
}
/** Feed states as the card's chips show them; the values stay English. */
const FEED_STATE_LABELS = Object.freeze({
loading: msg('loading'),
degraded: msg('degraded'),
stale: msg('stale'),
partial: msg('partial'),
fallback: msg('fallback'),
unavailable: msg('unavailable'),
});
function shownFeedState(state) {
const key = FEED_STATE_LABELS[state];
return key ? t(key) : state;
}
function analystCard(result) {
if (!result?.ok) return null;
const display = result.display || {};
@@ -91,28 +412,32 @@ function analystCard(result) {
].map((label) => ({ label }));
return {
display: {
title: analystHeadline(result),
title: shownAnalystHeadline(result),
// The precise scope, unless the title already says exactly that.
lines: [
display.scope !== result.scopeLabel ? display.scope : null,
display.window,
display.scope !== result.scopeLabel ? shownScope(display.scope) : null,
shownWindow(display.window),
].filter(Boolean),
chips: [
...(result.feedState && result.feedState !== 'nominal'
? [{ label: result.feedState }]
? [{ label: shownFeedState(result.feedState) }]
: []),
...(result.partial ? [{ label: 'partial' }] : []),
...(result.partial ? [{ label: t('partial') }] : []),
],
// Coverage qualifications lead the feed detail. The bounded card groups
// overflow in its existing Notes disclosure rather than dropping them.
preserveNotes: true,
notes: [
...notesOf(result.coverage?.note, display.caveat),
...notesOf(result.coverage?.note, display.caveat).map(shownNote),
...(unanswered.length
? [`Not answered: ${unanswered.join(', ')}`]
? [
t('Not answered: {layers}', {
layers: unanswered.map(shownLayerKey).join(', '),
}),
]
: []),
],
sources,
sources: sources.map(({ label }) => ({ label: sourceLabel(label) })),
},
referents: Array.isArray(result.items) ? result.items : [],
resultSet: true,
+9 -3
View File
@@ -147,8 +147,9 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
},
async start(options) {
if (disposed || !['idle', 'error'].includes(state)) return;
emit({ type: 'state', state: 'connecting' });
const attempt = ++startAttempt;
emit({ type: 'state', state: 'connecting' });
if (disposed || attempt !== startAttempt) return;
try {
await adapter.start(options);
} catch (error) {
@@ -164,12 +165,17 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
stop(options = {}) {
if (options.removeUi) return dispose();
if (disposed) return;
startAttempt++;
const attempt = ++startAttempt;
cancelActions();
try {
adapter.stop(options);
} finally {
if (!options.preserveStatus)
if (
!options.preserveStatus &&
!disposed &&
attempt === startAttempt &&
state !== 'idle'
)
emit({ type: 'state', state: 'idle', detail: 'Voice off' });
}
},
+196 -24
View File
@@ -6,8 +6,10 @@ import {
voiceCardView,
} from './voiceCardPresentation.js';
import { createVoiceSession } from './session.js';
import { createRealtimeSession } from './realtimeSession.js';
import { realtimeSessionEvent } from './realtimeEvents.js';
import { createVoiceCommands } from './sessionCommands.js';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
function deferred() {
let resolve;
@@ -240,26 +242,17 @@ test('common button and annotation bindings work with an alternate adapter and c
const previous = globalThis.window;
globalThis.window = {};
try {
const button = new EventTarget();
button.setAttribute = () => {};
let removed = 0;
let subscribed = null;
let calls = 0;
const events = [];
const ui = {
button,
root: {
dataset: {},
remove() {
removed++;
},
},
status: {},
detail: {},
tierButton: {},
costValue: {},
helpDetail: {},
const ui = createTestVoiceUi();
// Count removals but keep the control, so a click after cleanup still
// reaches any listener the binder failed to release.
ui.remove = () => {
removed++;
};
const click = () => ui.emit('button');
const lifetime = new AbortController();
const controls = createVoiceCommands({
runner: async () => {
@@ -292,25 +285,27 @@ test('common button and annotation bindings work with an alternate adapter and c
};
},
});
button.dispatchEvent(new Event('click'));
click();
await new Promise((done) => setTimeout(done, 0));
assert.equal(calls, 1);
assert.equal(ui.tierButton.hidden, true);
assert.equal(ui.costValue.hidden, true);
assert.equal(ui.helpDetail.textContent, 'Activate to toggle voice');
assert.equal(ui.status.textContent, 'LISTENING');
assert.equal(ui.detail.textContent, 'Ready');
assert.equal(ui.state.tier.hidden, true);
assert.equal(ui.state.cost.hidden, true);
assert.equal(ui.state.help.text, 'Activate to toggle voice');
assert.equal(ui.state.status.text, 'LISTENING');
assert.equal(ui.state.detail.text, 'Ready');
subscribed({ status: 'resolved' });
assert.deepEqual(events, [
{ type: 'map_annotation_outline', status: 'resolved' },
]);
button.dispatchEvent(new Event('click'));
click();
assert.equal(controls.state, 'idle');
click();
await new Promise((done) => setTimeout(done, 0));
lifetime.abort();
assert.equal(subscribed, null);
assert.equal(removed, 1);
button.dispatchEvent(new Event('click'));
assert.equal(calls, 1);
click();
assert.equal(calls, 2);
} finally {
globalThis.window = previous;
}
@@ -433,3 +428,180 @@ test('a late rejection cannot settle a newer turn card', async () => {
);
f.session.destroy();
});
function startReentrancyFixture() {
let starts = 0;
let stops = 0;
const settings = [];
const session = createVoiceSession({
runner: async () => ({ ok: true }),
createAdapter: ({ emit }) => ({
async start(options) {
starts++;
settings.push(options);
emit({ type: 'state', state: 'listening' });
},
stop() {
stops++;
},
sendText() {},
sendMapEvent() {},
}),
});
return { session, settings, starts: () => starts, stops: () => stops };
}
for (const action of ['stop', 'destroy']) {
test(`initial connecting observer ${action} prevents adapter acquisition`, async () => {
const f = startReentrancyFixture();
const unsubscribe = f.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'connecting') {
unsubscribe();
f.session[action]();
}
});
await f.session.start();
assert.equal(
f.starts(),
0,
'cancelled attempt must not call adapter.start',
);
assert.equal(f.stops(), 1);
assert.equal(f.session.state, 'idle');
if (action === 'stop') {
await f.session.start();
assert.equal(f.starts(), 1, 'later explicit Start remains usable');
assert.equal(f.session.state, 'listening');
}
f.session.destroy();
});
}
test('preserveStatus Stop still prevents adapter acquisition', async () => {
const f = startReentrancyFixture();
const unsubscribe = f.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'connecting') {
unsubscribe();
f.session.stop({ preserveStatus: true });
}
});
await f.session.start();
assert.equal(f.starts(), 0);
assert.equal(
f.session.state,
'connecting',
'preserved state is not ownership',
);
f.session.stop();
await f.session.start();
assert.equal(f.starts(), 1);
f.session.destroy();
});
test('Stop and explicit reentrant successor start acquire only for the successor', async () => {
const f = startReentrancyFixture();
let successor;
const unsubscribe = f.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'connecting') {
unsubscribe();
f.session.stop();
successor = f.session.start();
}
});
await f.session.start();
await successor;
assert.equal(f.starts(), 1);
assert.equal(f.session.state, 'listening');
f.session.destroy();
});
test('ordinary start forwards caller settings once and remains stoppable', async () => {
const f = startReentrancyFixture();
const settings = { pushToTalk: true };
await f.session.start(settings);
assert.equal(f.settings.length, 1);
assert.equal(f.settings[0], settings);
assert.equal(f.starts(), 1);
assert.equal(f.session.state, 'listening');
f.session.stop();
assert.equal(f.session.state, 'idle');
f.session.destroy();
});
for (const emitsIdle of [false, true]) {
for (const outcome of ['normal', 'preserveStatus', 'throw']) {
test(`stop preserves custom adapter contract: emitsIdle=${emitsIdle}, ${outcome}`, async (t) => {
let adapter;
const calls = [];
const events = [];
const failure = new Error('Controlled stop failure');
const session = createVoiceSession({
runner: async () => ({}),
createAdapter: ({ emit }) =>
(adapter = {
async start() {
emit({ type: 'state', state: 'listening' });
},
stop(settings = {}) {
calls.push({ receiver: this, settings });
if (settings.removeUi) return;
if (emitsIdle && !settings.preserveStatus)
emit({ type: 'state', state: 'idle' });
if (outcome === 'throw') throw failure;
return 'adapter-only return';
},
sendText() {},
sendMapEvent() {},
}),
});
t.after(() => session.destroy());
session.subscribe((event) => events.push(event));
await session.start();
events.length = 0;
const settings =
outcome === 'preserveStatus' ? { preserveStatus: true } : {};
if (outcome === 'throw')
assert.throws(
() => session.stop(settings),
(error) => error === failure,
);
else assert.equal(session.stop(settings), undefined);
assert.equal(calls[0].receiver, adapter);
assert.equal(calls[0].settings, settings);
assert.equal(
session.state,
outcome === 'preserveStatus' ? 'listening' : 'idle',
);
assert.equal(
events.filter(
(event) => event.type === 'state' && event.state === 'idle',
).length,
outcome === 'preserveStatus' ? 0 : 1,
);
});
}
}
test('actual default OpenAI adapter stop publishes one common idle', (t) => {
const events = [];
const ui = createTestVoiceUi();
const runner = async () => ({});
const session = createVoiceSession({
runner,
createAdapter: (hooks) =>
createRealtimeSession({ ...hooks, runner, ui, debugSink() {} }),
});
t.after(() => session.destroy());
session.subscribe((event) => events.push(event));
// Exercise the real stop path without acquiring microphone/WebRTC resources.
session.adapter.controller.setStatus('listening', 'Controlled active state');
events.length = 0;
session.stop();
assert.equal(
events.filter((event) => event.type === 'state' && event.state === 'idle')
.length,
1,
);
assert.equal(session.state, 'idle');
assert.equal(session.adapter.controller.status, 'idle');
});
+121 -51
View File
@@ -1,14 +1,14 @@
import { createVoiceControl } from './control.js';
import { createVoiceControl, voiceStatusLabel } from './control.js';
import { createVoiceSession } from './session.js';
import { VoiceCardControls } from './voiceCard.js';
import { PointerReticle } from './pointerReticle.js';
import { VOICE_PROVIDERS } from './providerCapabilities.js';
import { t } from '../i18n/index.js';
function createDefaultReticle(pointer) {
const element = pointer?.element;
const doc = element?.ownerDocument;
if (!doc) return null;
if (!element?.ownerDocument) return null;
return new PointerReticle({
doc,
rectOf: () => element.getBoundingClientRect?.() || null,
});
}
@@ -24,19 +24,31 @@ export function createVoiceCommands({
signal,
debugSink,
createControl = createVoiceControl,
resetExisting = true,
disposeRunner = true,
provider,
supportedProviders,
onProviderChange,
onSessionEvent,
createCard = (options) => new VoiceCardControls(options),
pointer = null,
referents = null,
createReticle = createDefaultReticle,
panelHost = null,
nativePanelProvider = null,
}) {
window.__gevVoiceCommands?.stop?.({ removeUi: true });
const ui = createControl({ reset: true });
if (resetExisting) window.__gevVoiceCommands?.stop?.({ removeUi: true });
const ui = createControl(panelHost ? { panel: true } : {});
let actionObserver;
let panelMount;
const session = createVoiceSession({
runner,
signal,
createAdapter: (hooks) =>
createSession({
createAdapter: (hooks) => {
if (panelHost) actionObserver = panelHost.observeActions(hooks);
return createSession({
...hooks,
...(actionObserver ? { runAction: actionObserver.runAction } : {}),
runner,
ui,
dataManager,
@@ -45,41 +57,96 @@ export function createVoiceCommands({
createController,
pointer,
radioLayer: dataManager?.layers?.get('radio')?.module || null,
}),
});
},
});
const adapter = session.adapter;
const capabilities = adapter.capabilities || {};
if (ui.tierButton) ui.tierButton.hidden = !capabilities.costControls;
if (ui.costValue) ui.costValue.hidden = !capabilities.costControls;
const offProvider = onProviderChange
? ui.on('provider-change', (value) => onProviderChange(value))
: null;
if (onProviderChange) {
ui.update('provider', {
...(supportedProviders && {
options: supportedProviders.map((name) => ({
value: name,
label: VOICE_PROVIDERS[name].label,
})),
}),
value: provider,
title: () =>
[
t('Changing provider reconnects voice if it is on.'),
capabilities.advisoryLimit,
]
.filter(Boolean)
.join(' '),
});
ui.update('root', { provider });
}
const hasProviderChoice = Boolean(
onProviderChange && supportedProviders?.length > 1,
);
ui.update('provider', { hidden: !hasProviderChoice });
ui.update('root', { providerChoice: String(hasProviderChoice) });
ui.update('tier', { hidden: !capabilities.costControls });
ui.update('cost', { hidden: !capabilities.costControls });
if (!capabilities.pushToTalk) {
ui.button.setAttribute('aria-label', 'Toggle voice control');
if (ui.helpDetail) ui.helpDetail.textContent = 'Activate to toggle voice';
ui.update('button', { label: () => t('Toggle voice control') });
ui.update('help', { text: () => t('Activate to toggle voice') });
}
// Retain the existing controller's inspection surface for browser tools.
const controls = adapter.controller || session;
controls.session = session;
controls.ui = ui;
const updateStatus = session.subscribe((event) => {
if (event.type !== 'state') return;
ui.root.dataset.status = event.state;
ui.status.textContent =
event.state === 'idle' ? 'OFF' : event.state.toUpperCase();
ui.detail.textContent =
event.detail || (event.state === 'idle' ? 'Voice off' : 'Voice active');
ui.button.setAttribute('aria-pressed', String(session.isActive()));
if (ui.errorDetail)
ui.errorDetail.textContent =
ui.update('root', {
status: event.state,
...(event.state === 'error' && { errorDismissed: false }),
});
// States are machine values; only the label is translated.
ui.update('status', { text: () => voiceStatusLabel(event.state) });
// The portable session reports 'Voice off' in English: show the idle
// label instead. Other details are provider messages, shown as they come.
const idle = event.state === 'idle';
const detail = event.detail === 'Voice off' ? '' : event.detail;
ui.update('detail', {
text: () => detail || (idle ? t('Voice off') : t('Voice active')),
});
ui.update('button', { pressed: session.isActive() });
ui.update('error', {
text: () =>
event.state === 'error'
? event.detail || 'Voice could not be started.'
: '';
if (event.state === 'error') ui.root.classList?.remove('error-dismissed');
? event.detail || t('Voice could not be started.')
: '',
});
});
// Captions, plan and result card nested in the control; it only reads
// provider-neutral session events.
const card = ui.card ? createCard({ root: ui.root }) : null;
const updateCard = card
? session.subscribe((event) => card.handle(event))
: null;
const card = createCard({
ui,
...(panelHost ? { onRender: () => panelMount?.panel.sync() } : {}),
});
panelMount = panelHost?.mount({
ui,
card,
session,
observer: actionObserver,
nativeProvider: nativePanelProvider,
});
const updatePanel = panelMount ? session.subscribe(panelMount.handle) : null;
const updateCard =
card && !panelMount
? session.subscribe((event) => card.handle(event))
: null;
controls.voiceCard = card;
if (panelMount) {
controls.voicePanel = panelMount.panel;
controls.captureVoicePanelOperation = panelMount.captureOperation;
controls.getVoicePanelState = panelMount.snapshot;
controls.getVoicePreferences = panelHost.snapshot;
}
// Point-and-ask: the reticle follows turn events; pointer and referents are
// session-scoped, so a session that ends forgets both.
const reticle = pointer ? createReticle(pointer) : null;
@@ -95,42 +162,45 @@ export function createVoiceCommands({
});
controls.pointer = pointer;
controls.referents = referents;
// Forward last: a host observer may synchronously replace or stop this mount.
const forwardEvents = onSessionEvent
? session.subscribe(onSessionEvent)
: null;
const annotationUnsubscribe = annotations?.onOutlineEvent?.((event) => {
card?.outline(event);
session.sendMapEvent({ type: 'map_annotation_outline', ...event });
});
const buttonHandler = () => {
if (adapter.ignoreButtonClick?.()) return;
if (session.isActive()) session.stop();
else void session.start({ pushToTalk: false });
if (session.isActive()) {
const ticket = panelMount?.stopTicket();
session.stop();
panelMount?.closeAfterStop(ticket);
} else void session.start({ pushToTalk: false });
};
ui.button.addEventListener('click', buttonHandler);
session.signal.addEventListener(
'abort',
() => {
ui.button.removeEventListener('click', buttonHandler);
annotationUnsubscribe?.();
updateStatus();
updateCard?.();
updatePointer();
reticle?.destroy();
card?.destroy();
runner.dispose?.();
ui.root.remove();
},
{ once: true },
);
if (session.disposed) {
ui.button.removeEventListener('click', buttonHandler);
const offButton = ui.on('button', buttonHandler);
let cleaned = false;
const cleanup = () => {
if (cleaned) return;
cleaned = true;
offButton();
offProvider?.();
annotationUnsubscribe?.();
updateStatus();
updateCard?.();
updatePanel?.();
panelMount?.destroy();
actionObserver?.dispose();
updatePointer();
forwardEvents?.();
reticle?.destroy();
card?.destroy();
runner.dispose?.();
ui.root.remove();
} else adapter.bindControls?.();
if (disposeRunner) runner.dispose?.();
ui.remove();
};
session.signal.addEventListener('abort', cleanup, { once: true });
if (session.disposed) cleanup();
else adapter.bindControls?.();
window.__gevVoiceCommands = controls;
return controls;
}
+486 -64
View File
@@ -16,9 +16,29 @@
*
* Builders are small pure functions keyed by tool name. Tools without one are
* returned unchanged.
*
* Only `display` and plan/progress labels are UI text (t()). `say`,
* `identityLine`, spoken progress lines and the tool result stay English for
* the model; the card shows them through the shown* helpers.
*/
import { analystHeadline } from './resultDisplay.js';
import {
formatNumber,
getLocale,
msg,
t,
tn,
translateKnown,
} from '../i18n/index.js';
import {
layerNameForId,
shownLayerName,
sourceLabel,
} from '../ui/layerNames.js';
import { localizePlaceName } from '../ui/placeNames.js';
import { CITY_POIS } from '../cityPois.js';
import { formatQuantity } from '../ui/unitLabels.js';
const LAYER_NOUNS = Object.freeze({
flights: 'aircraft',
@@ -53,17 +73,180 @@ function listPhrase(items) {
return `${list.slice(0, -1).join(', ')} and ${list.at(-1)}`;
}
function feetRounded(meters) {
// A missing altitude is unknown, not sea level: Number(null) is 0.
return Number.isFinite(meters)
? Math.round((meters * 3.28084) / 100) * 100
: null;
}
function feet(meters) {
const value = Number(meters);
if (!Number.isFinite(value)) return null;
const rounded = Math.round((value * 3.28084) / 100) * 100;
const rounded = feetRounded(meters);
if (rounded === null) return null;
return `${rounded.toLocaleString('en-US')} feet`;
}
function kilometers(km) {
/** The card's altitude, in the active language. */
function shownFeet(meters) {
const rounded = feetRounded(meters);
if (rounded === null) return null;
return t('{value} feet', { value: formatNumber(rounded) });
}
function roundedKm(km) {
const value = Number(km);
if (!Number.isFinite(value)) return null;
return `${value < 10 ? Number(value.toFixed(1)) : Math.round(value)} km`;
return value < 10 ? Number(value.toFixed(1)) : Math.round(value);
}
function kilometers(km) {
const value = roundedKm(km);
return value === null ? null : `${value} km`;
}
/** The card's distance, in the active language: "2,5 км". */
function shownKm(km) {
const value = roundedKm(km);
return value === null ? null : formatQuantity(value, 'km');
}
/** Sources as the card lists them. */
function shownSources(source) {
return source ? [{ label: sourceLabel(source) }] : [];
}
// Entity labels that layers register as English data (firms/, alpr/); keep in
// step with the producers. Other names (callsigns, places) show as they came.
const ENTITY_LABELS = new Set([msg('ALPR camera')]);
const FIRE_LABEL = /^Fire · FRP (\S+) MW$/;
/** An entity label as the card shows it, in the active language. */
export function shownEntityLabel(label) {
if (ENTITY_LABELS.has(label)) return t(label);
const fire = FIRE_LABEL.exec(label ?? '');
return fire ? t('Fire · FRP {frp} MW', { frp: fire[1] }) : label;
}
/*
* Tool errors stay English in the result (the model reads them). The card
* translates the known ones; the English must match gevActions exactly.
*/
const KNOWN_ERRORS = new Map(
[
msg('The flights layer is not enabled'),
msg('The military layer is not enabled'),
msg('The satellites layer is not enabled'),
msg('The ais-live-vessels layer is not enabled'),
msg('Could not enable the requested layer'),
msg('Could not disable the requested layer'),
msg('Radio layer unavailable'),
msg('Camera navigation is unavailable in the current view'),
msg('Camera navigation policy unavailable'),
msg('Nearest-aircraft selection supports Flights or Military Flights only'),
msg(
'Nearest-aircraft selection needs a preset, place name, or latitude and longitude',
),
msg('Could not arrive at the requested place'),
msg('The nearest airborne aircraft could not be selected'),
].map((text) => [text, text]),
);
function noneWithinView(target, radius) {
switch (target) {
case 'flights':
return t('No flights within {radius} of the current view', { radius });
case 'military':
return t('No military within {radius} of the current view', { radius });
case 'satellites':
return t('No satellites within {radius} of the current view', {
radius,
});
case 'vessels':
return t('No vessels within {radius} of the current view', { radius });
default:
return null;
}
}
const ERROR_PATTERNS = [
[
/^(.+) is enabled on the (.+) feed, but no airborne aircraft is loaded in the (.+) view yet$/,
([, layer, source, place]) =>
t(
'{layer} is enabled on the {source} feed, but no airborne aircraft is loaded in the {place} view yet',
{
layer: shownLayerName(layer),
source: sourceLabel(source),
place: localizePlaceName(place),
},
),
],
[
/^(.+) is enabled, but no airborne aircraft is loaded in the (.+) view yet$/,
([, layer, place]) =>
t(
'{layer} is enabled, but no airborne aircraft is loaded in the {place} view yet',
{ layer: shownLayerName(layer), place: localizePlaceName(place) },
),
],
[
/^(.+) is enabled, but its aircraft feed is unavailable$/,
([, layer]) =>
t('{layer} is enabled, but its aircraft feed is unavailable', {
layer: shownLayerName(layer),
}),
],
[
/^(.+) is enabled, but its destination refresh did not complete$/,
([, layer]) =>
t('{layer} is enabled, but its destination refresh did not complete', {
layer: shownLayerName(layer),
}),
],
[
/^(.+) is enabled, but (.+) is unavailable$/,
([, layer, source]) =>
t('{layer} is enabled, but {source} is unavailable', {
layer: shownLayerName(layer),
source: sourceLabel(source),
}),
],
[
/^Could not arrive at (.+)$/,
([, place]) =>
t('Could not arrive at {place}', { place: localizePlaceName(place) }),
],
[
/^(.+) is no longer loaded$/,
([, name]) => t('{name} is no longer loaded', { name }),
],
[
/^Unknown target layer: (.*)$/,
([, target]) => t('Unknown target layer: {target}', { target }),
],
[
/^No (\S+) within (\d+(?:\.\d+)?) km of the current view$/,
([, target, km]) => noneWithinView(target, shownKm(km)),
],
];
/** A tool error as the card shows it. */
function shownError(error) {
if (typeof error !== 'string' || !error) return null;
const known = KNOWN_ERRORS.get(error);
if (known) return t(known);
for (const [pattern, show] of ERROR_PATTERNS) {
const match = pattern.exec(error);
const shown = match ? show(match) : null;
if (shown) return shown;
}
return translateKnown(error);
}
// The card wraps a failed result's lines rather than cutting them: a
// translated error runs longer than the English (voiceCardPresentation).
function errorLines(result) {
return [shownError(result?.error)].filter(Boolean);
}
function sentence(text) {
@@ -76,9 +259,110 @@ function feedTag(state) {
return MATERIAL_FEED_TAGS[state] || null;
}
/** Feed states as the card shows them; the state values stay English. */
const FEED_STATE_LABELS = Object.freeze({
nominal: msg('nominal'),
loading: msg('loading'),
degraded: msg('degraded'),
stale: msg('stale'),
partial: msg('partial'),
fallback: msg('fallback'),
unavailable: msg('unavailable'),
});
function feedStateLabel(state) {
const key = FEED_STATE_LABELS[state];
return key ? t(key) : state;
}
/** A feed chip only when the state is worth seeing; nominal is the default. */
function feedChips(state) {
return state && state !== 'nominal' ? [{ label: state }] : [];
return state && state !== 'nominal' ? [{ label: feedStateLabel(state) }] : [];
}
/** '{label} on' / '{label} off' as the card shows it. */
function switchedTitle(label, on) {
return on ? t('{label} on', { label }) : t('{label} off', { label });
}
// Card titles name the layer's things in whole phrases, so each language can
// inflect the noun; the English matches LAYER_NOUNS.
function frameTitle(layerId) {
switch (layerId) {
case 'flights':
return t('Frame aircraft');
case 'military':
return t('Frame military aircraft');
case 'ais-live-vessels':
return t('Frame ships');
case 'satellites':
return t('Frame satellites');
default:
return t('Frame contacts');
}
}
function inFrameTitle(layerId, count) {
switch (layerId) {
case 'flights':
return tn(
count,
'{count} aircraft in frame',
'{count} aircraft in frame',
);
case 'military':
return tn(
count,
'{count} military aircraft in frame',
'{count} military aircraft in frame',
);
case 'ais-live-vessels':
return tn(count, '{count} ships in frame', '{count} ships in frame');
case 'satellites':
return tn(
count,
'{count} satellites in frame',
'{count} satellites in frame',
);
default:
return tn(
count,
'{count} contacts in frame',
'{count} contacts in frame',
);
}
}
function nearestTitle(layerId) {
switch (layerId) {
case 'military':
return t('Nearest military aircraft');
case 'ais-live-vessels':
return t('Nearest ships');
case 'satellites':
return t('Nearest satellites');
default:
return t('Nearest aircraft');
}
}
/** Two shown items joined, or the one item. */
function shownPair(items) {
return items.length === 2
? t('{first} and {second}', { first: items[0], second: items[1] })
: items[0] || '';
}
/** A preset city id (`sf`, `nyc`) as its English name; any other value unchanged. */
function presetPlaceName(value) {
const id = typeof value === 'string' ? value.trim() : '';
return Object.hasOwn(CITY_POIS, id) ? CITY_POIS[id].name : value;
}
// A place as the card shows it: a preset city in the interface language,
// whether the call named it ("Tokyo") or gave its id ("tokyo", "nyc").
function shownPlace(value, max) {
return localizePlaceName(displayName(presetPlaceName(value), max));
}
/** Ids such as `austin` or `local-firms` read as names on screen. */
@@ -96,6 +380,12 @@ function layerLabel(result) {
return spokenLabel(result?.label || result?.layerId || 'Layer');
}
/** The layer as the card names it, in the active language. */
function shownLayerLabel(result) {
const name = shownLayerName(result?.label, result?.layerId);
return spokenLabel(name || t('Layer'));
}
/* ---------------- per-tool builders ---------------- */
function setLayerVisibility(result, args = {}) {
@@ -112,7 +402,7 @@ function setLayerVisibility(result, args = {}) {
};
return {
say: `Couldn't turn ${wanted} ${label}.`,
display: { title: label, lines: [result?.error].filter(Boolean) },
display: { title: shownLayerLabel(result), lines: errorLines(result) },
};
}
const on = result.enabled !== false;
@@ -124,28 +414,35 @@ function setLayerVisibility(result, args = {}) {
return {
say,
display: {
title: `${label} ${on ? 'on' : 'off'}`,
title: switchedTitle(shownLayerLabel(result), on),
chips: on ? feedChips(result.feedState) : [],
sources: result.source ? [{ label: result.source }] : [],
sources: shownSources(result.source),
},
};
}
function flyToLocation(result) {
const label = spokenLabel(result?.label || result?.query || 'there');
const title =
spokenLabel(localizePlaceName(result?.label || result?.query)) ||
t('there');
if (result?.cancelled) return null;
if (!result?.ok) {
return {
say: sentence(`Couldn't find ${label}`),
display: { title: label, lines: [result?.error].filter(Boolean) },
display: { title, lines: errorLines(result) },
};
}
return {
say: sentence(result.arrived ? `Over ${label}` : `Flying to ${label}`),
display: {
title: label,
title,
lines: result.rangeM
? [`Camera range ${kilometers(result.rangeM / 1000)}`]
? [
t('Camera range {range}', {
range: shownKm(result.rangeM / 1000),
}),
]
: [],
},
};
@@ -169,8 +466,8 @@ function frameOverhead(result) {
return {
say,
display: {
title: `Frame ${noun}`,
lines: [result?.error].filter(Boolean),
title: frameTitle(result?.layerId),
lines: errorLines(result),
},
};
}
@@ -188,12 +485,15 @@ function frameOverhead(result) {
return {
say: sentence(`Framed ${countText} ${noun} within ${radius}`),
display: {
title: `${count} ${noun} in frame`,
title: inFrameTitle(result.layerId, count),
lines: [
`Within ${radius} of the view`,
result.detectionEnabled ? 'Labels on' : null,
t('Within {radius} of the view', {
radius: shownKm(result?.radiusKm),
}),
result.detectionEnabled ? t('Labels on') : null,
].filter(Boolean),
notes: count >= 80 ? ['Framing uses the nearest 80 loaded contacts'] : [],
notes:
count >= 80 ? [t('Framing uses the nearest 80 loaded contacts')] : [],
},
referents,
};
@@ -204,8 +504,12 @@ function selectNearestAircraft(result) {
result?.location?.label || result?.location || 'there',
);
const noun = LAYER_NOUNS[result?.layerId] || 'aircraft';
const shownLocation =
spokenLabel(
localizePlaceName(result?.location?.label || result?.location),
) || t('there');
const feed = result?.feed || {};
const sources = feed.source ? [{ label: feed.source }] : [];
const sources = shownSources(feed.source);
if (result?.cancelled) return null;
if (!result?.ok) {
let say;
@@ -219,8 +523,8 @@ function selectNearestAircraft(result) {
return {
say,
display: {
title: `Nearest ${noun}`,
lines: [result?.error].filter(Boolean),
title: nearestTitle(result?.layerId),
lines: errorLines(result),
chips: feedChips(feed.state),
sources,
},
@@ -246,8 +550,17 @@ function selectNearestAircraft(result) {
display: {
title: name,
lines: [
distance ? `${distance} from ${location}` : null,
altitude ? `Altitude ${altitude}` : null,
distance
? t('{distance} from {location}', {
distance: shownKm(aircraft.distanceKm),
location: shownLocation,
})
: null,
altitude
? t('Altitude {altitude}', {
altitude: shownFeet(aircraft.altitudeM),
})
: null,
].filter(Boolean),
chips: feedChips(feed.state),
sources,
@@ -258,14 +571,30 @@ function selectNearestAircraft(result) {
};
}
function routeText(properties) {
const SPOKEN_ROUTE = Object.freeze({
between: (origin, destination) => `${origin} to ${destination}`,
from: (origin) => `from ${origin}`,
to: (destination) => `to ${destination}`,
});
const SHOWN_ROUTE = Object.freeze({
between: (origin, destination) =>
t('{origin} to {destination}', { origin, destination }),
from: (origin) => t('from {origin}', { origin }),
to: (destination) => t('to {destination}', { destination }),
});
function routeText(properties, words = SPOKEN_ROUTE) {
const route = spokenLabel(properties.route || '', 40);
if (route) return route.replace(/\s*(?:-|→|>)\s*/g, ' to ');
if (route)
return route
.split(/\s*(?:-|→|>)\s*/)
.reduce((origin, destination) => words.between(origin, destination));
const origin = spokenLabel(properties.routeOrigin || '', 12);
const destination = spokenLabel(properties.routeDestination || '', 12);
if (origin && destination) return `${origin} to ${destination}`;
if (origin) return `from ${origin}`;
if (destination) return `to ${destination}`;
if (origin && destination) return words.between(origin, destination);
if (origin) return words.from(origin);
if (destination) return words.to(destination);
return null;
}
@@ -282,9 +611,13 @@ function getEntityContext(result) {
return {
say: null,
display: {
title,
lines: [selected.layerName].filter(Boolean),
sources: selected.source ? [{ label: selected.source }] : [],
title: spokenLabel(shownEntityLabel(title)),
lines: [
selected.layerName
? shownLayerName(selected.layerName, selected.layerId)
: null,
].filter(Boolean),
sources: shownSources(selected.source),
},
};
}
@@ -308,11 +641,15 @@ function getEntityContext(result) {
title,
lines: [
properties.registration
? `Registration ${spokenLabel(properties.registration, 16)}`
? t('Registration {registration}', {
registration: spokenLabel(properties.registration, 16),
})
: null,
route
? t('Route {route}', { route: routeText(properties, SHOWN_ROUTE) })
: null,
route ? `Route ${route}` : null,
].filter(Boolean),
sources: selected.source ? [{ label: selected.source }] : [],
sources: shownSources(selected.source),
},
referents: [
{
@@ -357,10 +694,10 @@ function getCurrentViewState(result) {
.map((part) => sentence(part[0].toUpperCase() + part.slice(1)))
.join(' '),
display: {
title: 'View state',
title: t('View state'),
lines: enabled.map(
(layer) =>
`${spokenLabel(layer.name || layer.id, 32)} · ${layer.count ?? 0} · ${layer.feedState || 'nominal'}`,
`${spokenLabel(shownLayerName(layer.name, layer.id), 32)} · ${layer.count ?? 0} · ${feedStateLabel(layer.feedState || 'nominal')}`,
),
},
schedule: 'silent',
@@ -394,13 +731,15 @@ function annotateMap(result) {
say: parts.length ? parts.join(' ') : null,
display: {
title: result?.ok
? `Marked ${drawn.length} ${drawn.length === 1 ? 'place' : 'places'}`
: 'Nothing marked',
? tn(drawn.length, 'Marked {count} place', 'Marked {count} places')
: t('Nothing marked'),
// The numbered referents list the marks; lines stay for extra facts.
lines: [],
notes: [
...(failed.length ? [`Not found: ${failed.join(', ')}`] : []),
...(result?.outlinePending ? ['Tracing outlines'] : []),
...(failed.length
? [t('Not found: {places}', { places: failed.join(', ') })]
: []),
...(result?.outlinePending ? [t('Tracing outlines')] : []),
],
},
referents,
@@ -494,29 +833,99 @@ function annotationTargets(args) {
.filter(Boolean);
}
// frame_overhead targets as the schema names them.
function frameStepLabel(target) {
switch (target) {
case 'flights':
return t('Frame flights');
case 'military':
return t('Frame military');
case 'satellites':
return t('Frame satellites');
case 'vessels':
return t('Frame vessels');
default:
return t('Frame {target}', { target });
}
}
// Plan-step labels for the other tools in actionSchemas. A tool missing here
// shows its raw name in English only.
const STEP_TITLES = Object.freeze({
zoom_to_globe: msg('Zoom out to globe'),
show_data_layers_menu: msg('Open data layers'),
set_panel_open: msg('Open or close a panel'),
set_voice_inactivity_timeout: msg('Voice inactivity timeout'),
set_context_mode: msg('Change context mode'),
control_cockpit: msg('Control cockpit'),
set_visual_style: msg('Change visual style'),
set_hud: msg('Adjust HUD'),
set_cyber_sonar: msg('Adjust cyber sonar'),
set_detection: msg('Adjust detection overlay'),
set_map_stack: msg('Change map stack'),
set_post_processing: msg('Adjust post-processing'),
control_scene: msg('Control scene'),
control_cctv: msg('Control CCTV'),
control_radio: msg('Control radio'),
track_entity: msg('Track contact'),
stop_tracking: msg('Stop tracking'),
clear_annotations: msg('Clear marks'),
move_camera: msg('Move camera'),
fly_route: msg('Fly route'),
analyst_query: msg('Analyst query'),
next_iss_pass: msg('Next ISS pass'),
next_satellite_pass: msg('Next satellite pass'),
});
/** A short on-screen label for one tool call in the voice card's plan. */
export function planStepLabel(name, args = {}) {
if (Object.hasOwn(STEP_TITLES, name)) return t(STEP_TITLES[name]);
switch (name) {
case 'annotate_map': {
const targets = annotationTargets(args);
return targets.length
? `Mark ${listPhrase(targets.slice(0, 2))}${targets.length > 2 ? ` +${targets.length - 2}` : ''}`
: 'Mark the map';
if (!targets.length) return t('Mark the map');
const shown = shownPair(targets.slice(0, 2));
return targets.length > 2
? t('Mark {targets} +{more}', {
targets: shown,
more: targets.length - 2,
})
: t('Mark {targets}', { targets: shown });
}
case 'fly_to_location':
return `Fly to ${displayName(args.query || args.locationId || 'location', 40)}`;
case 'select_nearest_aircraft':
return `Nearest aircraft to ${displayName(args.locationQuery || args.locationId || 'here', 32)}`;
case 'set_layer_visibility':
return `${displayName(args.layerId || 'Layer', 32)} ${args.enabled === false ? 'off' : 'on'}`;
case 'fly_to_location': {
const place = args.query || args.locationId;
return place
? t('Fly to {place}', { place: shownPlace(place, 40) })
: t('Fly to Location');
}
case 'select_nearest_aircraft': {
const place = args.locationQuery || args.locationId;
return place
? t('Nearest aircraft to {place}', { place: shownPlace(place, 32) })
: t('Nearest aircraft to Here');
}
case 'set_layer_visibility': {
const on = args.enabled !== false;
const label =
layerNameForId(args.layerId) ||
(args.layerId ? displayName(args.layerId, 32) : '');
// "Layer" alone is a noun the languages inflect the state for.
if (!label) return on ? t('Layer on') : t('Layer off');
return switchedTitle(label, on);
}
case 'adjust_camera_zoom':
return args.direction === 'out' ? t('Zoom out') : t('Zoom in');
case 'get_entity_context':
return 'Read the view';
return t('Read the view');
case 'get_current_view_state':
return 'Check view state';
return t('Check view state');
case 'frame_overhead':
return `Frame ${spokenLabel(args.target || 'flights', 24)}`;
return frameStepLabel(spokenLabel(args.target || 'flights', 24));
default:
return spokenLabel(String(name || 'Action').replace(/_/g, ' '), 40);
// A tool with no label: its name in English, "Action" elsewhere.
return getLocale() === 'en'
? spokenLabel(String(name || 'Action').replace(/_/g, ' '), 40)
: t('Action');
}
}
@@ -529,9 +938,15 @@ export function narrationLabel(name, args = {}) {
case 'annotate_map':
return annotationTargets(args)[0] || '';
case 'fly_to_location':
return displayName(args.query || args.locationId || '', 32);
return displayName(
presetPlaceName(args.query || args.locationId || ''),
32,
);
case 'select_nearest_aircraft':
return displayName(args.locationQuery || args.locationId || '', 32);
return displayName(
presetPlaceName(args.locationQuery || args.locationId || ''),
32,
);
default:
return '';
}
@@ -539,19 +954,26 @@ export function narrationLabel(name, args = {}) {
/** On-screen labels for progress steps. */
const STEP_LABELS = Object.freeze({
resolve: 'Finding places',
outline: 'Tracing outline',
search: 'Looking up place',
fly: 'Flying',
layer: 'Turning on layer',
refresh: 'Loading aircraft',
nearest: 'Picking nearest',
resolve: msg('Finding places'),
outline: msg('Tracing outline'),
search: msg('Looking up place'),
fly: msg('Flying'),
layer: msg('Turning on layer'),
refresh: msg('Loading aircraft'),
nearest: msg('Picking nearest'),
});
/** On-screen label for a progress step. */
export function progressStepLabel(step, label) {
const base = STEP_LABELS[step] || spokenLabel(step || 'Working', 24);
return label ? `${base}: ${spokenLabel(label, 32)}` : base;
const base = STEP_LABELS[step]
? t(STEP_LABELS[step])
: step
? spokenLabel(step, 24)
: t('Working');
// The join is a catalog string so each language spaces its colon.
return label
? t('{name}: {state}', { name: base, state: spokenLabel(label, 32) })
: base;
}
/**
+41
View File
@@ -9,6 +9,7 @@ import {
spokenLabel,
} from './speech.js';
import { countWords, findHedges } from './speechLint.js';
import { setLocale } from '../i18n/index.js';
function envelope(name, result, args) {
return attachVoiceResult(name, result, args);
@@ -217,6 +218,34 @@ test('nearest-aircraft results name the aircraft, distance and altitude briefly'
assert.equal(stale.say, 'Selected AAL1, 3.3 km from Austin, stale.');
});
test('nearest-aircraft results leave out an altitude the feed did not report', () => {
// gevActions sends `altitudeM: null` for an airborne military contact
// without a barometric altitude; that is unknown, not sea level.
const unknown = envelope('select_nearest_aircraft', {
ok: true,
location: 'Ramstein',
layerId: 'military',
feed: { state: 'nominal' },
aircraft: { callsign: 'RCH123', altitudeM: null, distanceKm: 8.2 },
});
assert.equal(unknown.say, 'Selected RCH123, 8.2 km from Ramstein.');
assert.deepEqual(unknown.display.lines, ['8.2\u00a0km from Ramstein']);
// A reported sea-level altitude is still spoken.
const seaLevel = envelope('select_nearest_aircraft', {
ok: true,
location: 'Ramstein',
layerId: 'military',
feed: { state: 'nominal' },
aircraft: { callsign: 'RCH123', altitudeM: 0, distanceKm: 8.2 },
});
assert.equal(seaLevel.say, 'Selected RCH123, 8.2 km from Ramstein, 0 feet.');
assert.deepEqual(seaLevel.display.lines, [
'8.2\u00a0km from Ramstein',
'Altitude 0 feet',
]);
});
test('frame_overhead keeps the essential scope audible and lists referents', () => {
const framed = envelope('frame_overhead', {
ok: true,
@@ -459,3 +488,15 @@ test('labels are bounded and cleaned before they are spoken', () => {
'background steps stay on screen',
);
});
test('progress step labels join through the catalog colon', () => {
setLocale('fr');
try {
assert.equal(
progressStepLabel('resolve', 'Capitol'),
'Recherche de lieux\u202f: Capitol',
);
} finally {
setLocale('en');
}
});
+1 -1
View File
@@ -7,7 +7,7 @@
* Phrases that add length without adding an answer. Tool numbers are spoken
* verbatim, so approximations are hedges too.
*/
export const HEDGE_PHRASES = Object.freeze([
const HEDGE_PHRASES = Object.freeze([
'keep in mind',
"it's worth noting",
'it is worth noting',
+1 -1
View File
@@ -1,7 +1,7 @@
import { countWords, findHedges } from './speechLint.js';
/** Quiet period after the last audio or response before a turn is closed. */
export const TURN_SETTLE_MS = 2500;
const TURN_SETTLE_MS = 2500;
/**
* Per-turn latency and speech spans for one voice session.
+73 -125
View File
@@ -5,115 +5,109 @@ import {
} from './voiceCardPresentation.js';
/** Hide an idle card this long after its last update. */
export const VOICE_CARD_IDLE_MS = 14000;
const ELEMENT_IDS = Object.freeze({
card: 'gev-voice-card',
phase: 'gev-voice-card-phase',
dismiss: 'gev-voice-card-dismiss',
user: 'gev-voice-card-user',
userText: 'gev-voice-card-user-text',
pointer: 'gev-voice-card-pointer',
pointerKind: 'gev-voice-card-pointer-kind',
pointerText: 'gev-voice-card-pointer-text',
assistant: 'gev-voice-card-assistant',
assistantText: 'gev-voice-card-assistant-text',
plan: 'gev-voice-card-plan',
result: 'gev-voice-card-result',
resultTitle: 'gev-voice-card-result-title',
chips: 'gev-voice-card-chips',
lines: 'gev-voice-card-lines',
referents: 'gev-voice-card-referents',
notes: 'gev-voice-card-notes',
noteList: 'gev-voice-card-note-list',
live: 'gev-voice-card-live',
});
function setText(element, text) {
if (element && element.textContent !== text) element.textContent = text;
}
function setHidden(element, hidden) {
if (element && element.hidden !== hidden) element.hidden = hidden;
}
function fillList(list, items, render) {
if (!list) return;
const doc = list.ownerDocument;
list.replaceChildren(
...items.map((item) => {
const node = doc.createElement('li');
render(node, item, doc);
return node;
}),
);
}
const VOICE_CARD_IDLE_MS = 14000;
/**
* Voice card controls, nested in the voice control's markup (`control.js`)
* Voice card controls. The card is part of the voice control (`control.js`),
* beside its help and error trays, so Clean-UI and recording mode hide it with
* the control. Receives provider-neutral session events through `handle` and
* annotation outline outcomes through `outline`; owns only its listeners and
* idle timer. All text is set through textContent. Captions update silently;
* the polite live region carries final replies and result titles only.
* annotation outline outcomes through `outline`, and presents the card's view
* on the control; owns only its listener and idle timer. Captions update
* silently; the polite live region carries final replies and result titles
* only.
*/
export class VoiceCardControls {
/**
* @param {object} options
* @param {HTMLElement} options.root Voice control root containing the card.
* @param {object} options.ui The voice control presenter.
* @param {number} [options.idleMs]
* @param {Function} [options.setTimer]
* @param {Function} [options.clearTimer]
*/
constructor({
root,
ui,
idleMs = VOICE_CARD_IDLE_MS,
setTimer = (fn, ms) => setTimeout(fn, ms),
clearTimer = (id) => clearTimeout(id),
onRender,
}) {
Object.assign(this, { idleMs, setTimer, clearTimer });
this.elements = Object.fromEntries(
Object.entries(ELEMENT_IDS).map(([key, id]) => [
key,
root?.querySelector?.(`#${id}`) || null,
]),
);
Object.assign(this, { ui, idleMs, setTimer, clearTimer, onRender });
this.state = initialVoiceCardState();
this.enabled = true;
this.presented = true;
this.announcementRevision = 0;
this.consumedPresentationAnnouncement = -1;
this.hideTimer = null;
this.destroyed = false;
this.listeners = new AbortController();
if (!this.elements.card) return;
this.elements.dismiss?.addEventListener(
'click',
() => this.handle({ type: 'dismiss' }),
{ signal: this.listeners.signal },
if (!ui) return;
// The card's labels are worded again in a new locale.
this.offLocale = ui.on('locale', () => {
if (!this.destroyed) this.render();
});
this.offDismiss = ui.on('card-dismiss', () =>
this.handle({ type: 'dismiss' }),
);
this.render();
}
/** Apply one session event (or `outline`/`dismiss`). */
handle(event) {
if (this.destroyed || !this.elements.card) return;
if (this.destroyed || !this.ui) return;
const next = reduceVoiceCard(this.state, event);
if (next === this.state) return;
if (
next.announce !== this.state.announce ||
(next.announce &&
((event.type === 'transcript' &&
event.role === 'assistant' &&
event.final) ||
(event.type === 'action-result' &&
next.result !== this.state.result) ||
(event.type === 'pointer' &&
next.pointer &&
next.pointer !== this.state.pointer)))
)
this.announcementRevision++;
this.state = next;
this.render();
if (this.state.visible) this.scheduleIdle();
else this.cancelIdle();
}
/** A local submission allows future output after dismissal; keep its content. */
rearm() {
if (this.destroyed || !this.ui) return;
this.state = { ...this.state, dismissed: false };
}
/** An annotation outline resolved or failed after its tool result. */
outline(event) {
this.handle({ type: 'outline', id: event?.id, status: event?.status });
}
/** Change presentation only; session events and the idle deadline continue. */
setEnabled(enabled) {
if (this.destroyed || !this.ui) return;
this.enabled = Boolean(enabled);
this.render();
}
/** View selection changes presentation without changing the saved preference. */
setPresented(presented) {
if (this.destroyed || !this.ui || this.presented === presented) return;
if (!presented)
this.consumedPresentationAnnouncement = this.announcementRevision;
this.presented = presented;
this.render();
}
scheduleIdle() {
this.cancelIdle();
this.hideTimer = this.setTimer(() => {
this.hideTimer = null;
if (this.destroyed) return;
// Keep the card while the pointer or focus is inside it.
if (this.elements.card.matches?.(':hover, :focus-within')) {
if (this.ui.element('card')?.matches?.(':hover, :focus-within')) {
this.scheduleIdle();
return;
}
@@ -128,76 +122,30 @@ export class VoiceCardControls {
}
render() {
const el = this.elements;
const view = voiceCardView(this.state);
setHidden(el.card, !view.visible);
el.card.dataset.busy = String(view.busy);
setText(el.phase, view.phase);
setHidden(el.user, !view.user.visible);
setText(el.userText, view.user.text);
setHidden(el.pointer, !view.pointer.visible);
setText(el.pointerKind, view.pointer.kind);
setText(el.pointerText, view.pointer.text);
setHidden(el.assistant, !view.assistant.visible);
setText(el.assistantText, view.assistant.text);
setHidden(el.plan, view.steps.length === 0);
fillList(el.plan, view.steps, (node, step, doc) => {
node.className = 'gev-voice-card-step';
node.dataset.status = step.status;
const label = doc.createElement('span');
label.className = 'gev-voice-card-step-label';
label.textContent = step.label;
const status = doc.createElement('span');
status.className = 'gev-voice-card-sr';
status.textContent = step.statusText ? `, ${step.statusText}` : '';
node.append(label, status);
if (step.detail) {
const detail = doc.createElement('span');
detail.className = 'gev-voice-card-step-detail';
detail.textContent = step.detail;
node.append(detail);
}
});
setHidden(el.result, !view.result.visible);
setText(el.resultTitle, view.result.title);
setHidden(el.chips, view.result.chips.length === 0);
if (el.chips) {
const doc = el.chips.ownerDocument;
el.chips.replaceChildren(
...view.result.chips.map((text) => {
const chip = doc.createElement('span');
chip.className = 'gev-voice-card-chip';
chip.dataset.tone = text;
chip.textContent = text;
return chip;
}),
);
}
fillList(el.lines, view.result.lines, (node, line) => {
node.textContent = line;
});
setHidden(el.referents, view.result.referents.length === 0);
fillList(el.referents, view.result.referents, (node, ref) => {
node.value = ref.n;
node.textContent = ref.label;
});
setHidden(el.notes, view.result.notes.length === 0);
fillList(el.noteList, view.result.notes, (node, note) => {
node.textContent = note;
});
setText(el.live, view.announce);
this.ui.replace('card', this.snapshot());
this.onRender?.();
}
/** Test/diagnostic view of what the card shows. */
snapshot() {
return voiceCardView(this.state);
const view = voiceCardView(this.state);
return this.enabled && this.presented
? {
...view,
announce:
this.consumedPresentationAnnouncement === this.announcementRevision
? ''
: view.announce,
}
: { ...view, visible: false, announce: '' };
}
/** Idempotent: removes listeners and the idle timer; markup stays owned by the control. */
/** Idempotent: removes the listener and the idle timer; the control stays. */
destroy() {
if (this.destroyed) return;
this.destroyed = true;
this.cancelIdle();
this.listeners.abort();
this.offDismiss?.();
this.offLocale?.();
}
}
+497 -113
View File
@@ -1,15 +1,18 @@
import '../testSupport/registerDom.mjs';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import test from 'node:test';
import { nextTick } from 'vue';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import { mountVoiceView } from '../testSupport/voiceView.mjs';
import { readStylesheet } from '../testSupport/readStylesheet.mjs';
import { VoiceCardControls } from './voiceCard.js';
import { getLocale, setLocale } from '../i18n/index.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
const controlSource = readFileSync(new URL('./control.js', import.meta.url), 'utf8');
const css = readStylesheet(new URL('../../style.css', import.meta.url));
function fold(events) {
@@ -19,11 +22,37 @@ function fold(events) {
test('presentation: a slow annotate turn shows captions, plan steps and the result card', () => {
const state = fold([
{ type: 'interruption' },
{ type: 'transcript', role: 'user', text: 'Annotate the Texas State Capitol', final: true },
{ type: 'transcript', role: 'assistant', text: 'Finding the Capitol, ', responseId: 'r1' },
{ type: 'transcript', role: 'assistant', text: 'then outlining it.', responseId: 'r1' },
{ type: 'action-call', name: 'annotate_map', callId: 'call_a', arguments: { annotations: [{ target: 'Texas State Capitol' }] } },
{ type: 'progress', name: 'annotate_map', callId: 'call_a', step: 'resolve', label: 'Texas State Capitol' },
{
type: 'transcript',
role: 'user',
text: 'Annotate the Texas State Capitol',
final: true,
},
{
type: 'transcript',
role: 'assistant',
text: 'Finding the Capitol, ',
responseId: 'r1',
},
{
type: 'transcript',
role: 'assistant',
text: 'then outlining it.',
responseId: 'r1',
},
{
type: 'action-call',
name: 'annotate_map',
callId: 'call_a',
arguments: { annotations: [{ target: 'Texas State Capitol' }] },
},
{
type: 'progress',
name: 'annotate_map',
callId: 'call_a',
step: 'resolve',
label: 'Texas State Capitol',
},
]);
let view = voiceCardView(state);
assert.equal(view.visible, true);
@@ -31,9 +60,16 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
assert.equal(view.phase, 'WORKING');
assert.equal(view.user.text, 'Annotate the Texas State Capitol');
assert.equal(view.assistant.text, 'Finding the Capitol, then outlining it.');
assert.deepEqual(view.steps.map(({ label, status, detail }) => ({ label, status, detail })), [
{ label: 'Mark Texas State Capitol', status: 'running', detail: 'Finding places: Texas State Capitol' },
]);
assert.deepEqual(
view.steps.map(({ label, status, detail }) => ({ label, status, detail })),
[
{
label: 'Mark Texas State Capitol',
status: 'running',
detail: 'Finding places: Texas State Capitol',
},
],
);
const done = [
{
type: 'action-result',
@@ -43,7 +79,12 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
ok: true,
outlinePending: true,
items: [{ ok: true, id: 'anno-1', outlinePending: true }],
display: { title: 'Marked 1 place', lines: ['Texas State Capitol'], notes: ['Tracing outlines'], sources: [{ label: 'OpenStreetMap' }] },
display: {
title: 'Marked 1 place',
lines: ['Texas State Capitol'],
notes: ['Tracing outlines'],
sources: [{ label: 'OpenStreetMap' }],
},
referents: [{ n: 1, id: 'anno-1', label: 'Texas State Capitol' }],
},
},
@@ -53,100 +94,146 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
assert.equal(view.phase, 'DONE');
assert.equal(view.steps[0].detail, 'Outline traced');
assert.equal(view.result.title, 'Marked 1 place');
assert.deepEqual(view.result.referents, [{ n: 1, label: 'Texas State Capitol' }]);
assert.deepEqual(view.result.notes, ['Tracing outlines', 'Source: OpenStreetMap']);
assert.equal(view.announce, 'Marked 1 place', 'the live region carries the result title');
assert.deepEqual(view.result.referents, [
{ n: 1, label: 'Texas State Capitol' },
]);
assert.deepEqual(view.result.notes, [
'Tracing outlines',
'Source: OpenStreetMap',
]);
assert.equal(
view.announce,
'Marked 1 place',
'the live region carries the result title',
);
});
test('presentation: silent lookups, dismissal and session end', () => {
const lookup = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'get_current_view_state', arguments: {} },
{ type: 'action-result', name: 'get_current_view_state', result: { ok: true, schedule: 'silent', display: { title: 'View state' } } },
{
type: 'action-result',
name: 'get_current_view_state',
result: {
ok: true,
schedule: 'silent',
display: { title: 'View state' },
},
},
]);
assert.equal(voiceCardView(lookup).result.visible, false, 'a silent lookup does not take the result card');
assert.equal(
voiceCardView(lookup).result.visible,
false,
'a silent lookup does not take the result card',
);
const dismissed = reduceVoiceCard(lookup, { type: 'dismiss' });
assert.equal(voiceCardView(dismissed).visible, false);
const stillDismissed = reduceVoiceCard(dismissed, { type: 'transcript', role: 'assistant', text: 'Hi', final: true });
assert.equal(voiceCardView(stillDismissed).visible, false, 'dismissed until the next turn');
assert.equal(voiceCardView(reduceVoiceCard(stillDismissed, { type: 'interruption' })).visible, true);
assert.deepEqual(reduceVoiceCard(lookup, { type: 'state', state: 'idle' }), initialVoiceCardState());
assert.equal(reduceVoiceCard(lookup, { type: 'turn-metrics' }), lookup, 'unrelated events are no-ops');
const stillDismissed = reduceVoiceCard(dismissed, {
type: 'transcript',
role: 'assistant',
text: 'Hi',
final: true,
});
assert.equal(
voiceCardView(stillDismissed).visible,
false,
'dismissed until the next turn',
);
assert.equal(
voiceCardView(reduceVoiceCard(stillDismissed, { type: 'interruption' }))
.visible,
true,
);
assert.deepEqual(
reduceVoiceCard(lookup, { type: 'state', state: 'idle' }),
initialVoiceCardState(),
);
assert.equal(
reduceVoiceCard(lookup, { type: 'turn-metrics' }),
lookup,
'unrelated events are no-ops',
);
});
test('markup: the card nests inside the voice control with every element and a polite live region', () => {
const root = controlSource.indexOf("root.id = 'gev-voice-control'");
const card = controlSource.indexOf('id="gev-voice-card"');
assert.ok(root >= 0 && card > root, 'the card is part of the voice control markup');
for (const id of [
'gev-voice-card-phase',
'gev-voice-card-dismiss',
'gev-voice-card-user-text',
'gev-voice-card-assistant-text',
'gev-voice-card-plan',
'gev-voice-card-result-title',
'gev-voice-card-chips',
'gev-voice-card-lines',
'gev-voice-card-referents',
'gev-voice-card-note-list',
]) {
assert.match(controlSource, new RegExp(`id="${id}"`), `${id} is missing`);
test('markup: the card nests inside the voice control with every element and a polite live region', async (t) => {
const { pinia } = mountVoiceView(t);
for (const panel of [false, true]) {
const ui = createTestVoiceUi({ panel, pinia });
await nextTick();
const card = document.getElementById('gev-voice-card');
assert.equal(
document.getElementById('gev-voice-control').contains(card),
true,
'the card is part of the voice control',
);
assert.equal(document.querySelectorAll('#gev-voice-card').length, 1);
if (panel)
assert.equal(card.parentElement.id, 'gev-voice-preview-panel');
for (const id of [
'gev-voice-card-phase',
'gev-voice-card-dismiss',
'gev-voice-card-user-text',
'gev-voice-card-user-who',
'gev-voice-card-assistant-text',
'gev-voice-card-plan',
'gev-voice-card-result-title',
'gev-voice-card-chips',
'gev-voice-card-lines',
'gev-voice-card-referents',
'gev-voice-card-note-list',
])
assert.equal(card.contains(document.getElementById(id)), true, id);
assert.equal(
document.getElementById('gev-voice-card-live').getAttribute('aria-live'),
'polite',
);
assert.equal(
document
.getElementById('gev-voice-card-dismiss')
.getAttribute('aria-label'),
'Dismiss voice card',
);
ui.remove();
}
assert.match(controlSource, /id="gev-voice-card-live"[^>]*aria-live="polite"/);
assert.match(controlSource, /aria-label="Dismiss voice card"/);
assert.match(css, /#gev-voice-control \.gev-voice-card \{/);
assert.match(css, /body\.recording-mode #gev-voice-control/, 'recording mode hides the control, card included');
assert.match(css, /body\.ui-clean-view #command-dock/, 'Clean-UI hides the dock, card included');
assert.match(
css,
/body\.recording-mode #gev-voice-control/,
'recording mode hides the control, card included',
);
assert.match(
css,
/body\.ui-clean-view #command-dock/,
'Clean-UI hides the dock, card included',
);
});
function fakeRoot() {
const make = (id) => {
const listeners = new Map();
const node = {
id,
hidden: false,
textContent: '',
dataset: {},
children: [],
ownerDocument: null,
addEventListener(type, handler, options) {
listeners.set(type, handler);
options?.signal?.addEventListener('abort', () => listeners.delete(type));
},
fire: (type) => listeners.get(type)?.({}),
listenerCount: () => listeners.size,
replaceChildren(...nodes) {
this.children = nodes;
},
append(...nodes) {
this.children.push(...nodes);
},
matches: () => false,
// A real control whose listeners are counted, so a test sees teardown
// release them.
function countedUi(options) {
const ui = createTestVoiceUi(options);
const on = ui.on;
const live = new Set();
ui.on = (name, listener) => {
const entry = { name };
live.add(entry);
const off = on(name, listener);
return () => {
live.delete(entry);
off();
};
return node;
};
const nodes = new Map();
const doc = { createElement: (tag) => ({ ...make(tag), tag }) };
const root = {
querySelector(selector) {
const id = selector.slice(1);
if (!nodes.has(id)) {
const node = make(id);
node.ownerDocument = doc;
nodes.set(id, node);
}
return nodes.get(id);
},
};
return { root, nodes };
return { ui, listenerCount: () => live.size };
}
test('controls: render, idle hide, dismiss and idempotent teardown', () => {
const { root, nodes } = fakeRoot();
const { ui, listenerCount } = countedUi();
const shown = () => ui.state.card.visible;
const timers = new Map();
let nextId = 1;
const card = new VoiceCardControls({
root,
ui,
idleMs: 1000,
setTimer: (fn) => {
const id = nextId++;
@@ -158,30 +245,41 @@ test('controls: render, idle hide, dismiss and idempotent teardown', () => {
},
clearTimer: (id) => timers.delete(id),
});
assert.equal(nodes.get('gev-voice-card').hidden, true, 'starts hidden');
assert.equal(shown(), false, 'starts hidden');
card.handle({ type: 'interruption' });
card.handle({ type: 'action-call', name: 'fly_to_location', arguments: { query: 'Tokyo' } });
card.handle({ type: 'action-result', name: 'fly_to_location', result: { ok: true, display: { title: 'Tokyo', lines: [], chips: [{ label: 'stale' }] } } });
assert.equal(nodes.get('gev-voice-card').hidden, false);
assert.equal(nodes.get('gev-voice-card-result-title').textContent, 'Tokyo');
assert.equal(nodes.get('gev-voice-card-plan').children.length, 1);
assert.equal(nodes.get('gev-voice-card-chips').children[0].textContent, 'stale');
card.handle({
type: 'action-call',
name: 'fly_to_location',
arguments: { query: 'Tokyo' },
});
card.handle({
type: 'action-result',
name: 'fly_to_location',
result: {
ok: true,
display: { title: 'Tokyo', lines: [], chips: [{ label: 'stale' }] },
},
});
assert.equal(shown(), true);
assert.equal(ui.state.card.result.title, 'Tokyo');
assert.equal(ui.state.card.steps.length, 1);
assert.equal(ui.state.card.result.chips[0], 'stale');
assert.equal(timers.size, 1, 'one idle timer at a time');
[...timers.values()][0]();
assert.equal(nodes.get('gev-voice-card').hidden, true, 'hides when idle');
assert.equal(shown(), false, 'hides when idle');
card.handle({ type: 'interruption' });
nodes.get('gev-voice-card-dismiss').fire('click');
assert.equal(nodes.get('gev-voice-card').hidden, true, 'dismiss hides it');
ui.emit('card-dismiss');
assert.equal(shown(), false, 'dismiss hides it');
card.destroy();
card.destroy();
assert.equal(timers.size, 0);
assert.equal(nodes.get('gev-voice-card-dismiss').listenerCount(), 0);
assert.equal(listenerCount(), 0);
card.handle({ type: 'interruption' });
assert.equal(nodes.get('gev-voice-card').hidden, true, 'a destroyed card ignores late events');
assert.equal(shown(), false, 'a destroyed card ignores late events');
});
test('controls: a control without card markup is inert', () => {
const card = new VoiceCardControls({ root: { querySelector: () => null } });
test('controls: card controls without a voice control are inert', () => {
const card = new VoiceCardControls({});
assert.doesNotThrow(() => card.handle({ type: 'interruption' }));
card.destroy();
});
@@ -189,17 +287,41 @@ test('controls: a control without card markup is inert', () => {
test('presentation: concurrent calls of one tool update their own steps', () => {
const state = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'fly_to_location', callId: 'c1', arguments: { query: 'Tokyo' } },
{ type: 'action-call', name: 'fly_to_location', callId: 'c2', arguments: { query: 'Paris' } },
{
type: 'action-call',
name: 'fly_to_location',
callId: 'c1',
arguments: { query: 'Tokyo' },
},
{
type: 'action-call',
name: 'fly_to_location',
callId: 'c2',
arguments: { query: 'Paris' },
},
// Out of order: the first call finishes last.
{ type: 'action-result', name: 'fly_to_location', callId: 'c2', result: { ok: false, error: 'x' } },
{ type: 'progress', name: 'fly_to_location', callId: 'c1', step: 'search', label: 'Tokyo' },
{
type: 'action-result',
name: 'fly_to_location',
callId: 'c2',
result: { ok: false, error: 'x' },
},
{
type: 'progress',
name: 'fly_to_location',
callId: 'c1',
step: 'search',
label: 'Tokyo',
},
]);
const steps = voiceCardView(state).steps;
assert.deepEqual(steps.map(({ label, status }) => [label, status]), [
['Fly to Tokyo', 'running'],
['Fly to Paris', 'failed'],
]);
assert.deepEqual(
steps.map(({ label, status }) => [label, status]),
[
['Fly to Tokyo', 'running'],
['Fly to Paris', 'failed'],
],
);
assert.equal(steps[0].detail, 'Looking up place: Tokyo');
});
@@ -207,27 +329,289 @@ test('presentation: stale and unmatched events never reopen the card', () => {
const ended = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'annotate_map', callId: 'c1', arguments: {} },
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, outlinePending: true, items: [{ ok: true, id: 'anno-9', outlinePending: true }] } },
{
type: 'action-result',
name: 'annotate_map',
callId: 'c1',
result: {
ok: true,
outlinePending: true,
items: [{ ok: true, id: 'anno-9', outlinePending: true }],
},
},
{ type: 'state', state: 'idle' },
]);
for (const late of [
{ type: 'outline', id: 'anno-9', status: 'resolved' },
{ type: 'outline', status: 'resolved' },
{ type: 'progress', name: 'annotate_map', callId: 'c1', step: 'resolve' },
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, display: { title: 'Marked 1 place' } } },
{
type: 'action-result',
name: 'annotate_map',
callId: 'c1',
result: { ok: true, display: { title: 'Marked 1 place' } },
},
]) {
assert.equal(reduceVoiceCard(ended, late), ended, `${late.type} after stop is a no-op`);
assert.equal(
reduceVoiceCard(ended, late),
ended,
`${late.type} after stop is a no-op`,
);
}
const live = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'annotate_map', callId: 'c1', arguments: {} },
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, outlinePending: true, items: [{ ok: true, id: 'anno-1', outlinePending: true }, { ok: true, id: 'anno-2', outlinePending: true }] } },
{
type: 'action-result',
name: 'annotate_map',
callId: 'c1',
result: {
ok: true,
outlinePending: true,
items: [
{ ok: true, id: 'anno-1', outlinePending: true },
{ ok: true, id: 'anno-2', outlinePending: true },
],
},
},
{ type: 'dismiss' },
]);
assert.equal(reduceVoiceCard(live, { type: 'outline', id: 'anno-7', status: 'resolved' }), live, 'another mark is ignored');
const first = reduceVoiceCard(live, { type: 'outline', id: 'anno-1', status: 'resolved' });
assert.equal(voiceCardView(first).steps[0].detail, 'Tracing outline', 'one of two outlines still pending');
assert.equal(voiceCardView(first).visible, false, 'a dismissed card stays hidden');
const both = reduceVoiceCard(first, { type: 'outline', id: 'anno-2', status: 'failed' });
assert.equal(
reduceVoiceCard(live, {
type: 'outline',
id: 'anno-7',
status: 'resolved',
}),
live,
'another mark is ignored',
);
const first = reduceVoiceCard(live, {
type: 'outline',
id: 'anno-1',
status: 'resolved',
});
assert.equal(
voiceCardView(first).steps[0].detail,
'Tracing outline',
'one of two outlines still pending',
);
assert.equal(
voiceCardView(first).visible,
false,
'a dismissed card stays hidden',
);
const both = reduceVoiceCard(first, {
type: 'outline',
id: 'anno-2',
status: 'failed',
});
assert.equal(voiceCardView(both).steps[0].detail, 'Outline unavailable');
});
test('presentation controls preserve real reducer content and the existing 14-second idle deadline', () => {
const ui = createTestVoiceUi();
const timers = new Map();
let next = 0,
renders = 0;
const card = new VoiceCardControls({
ui,
setTimer(fn, ms) {
const id = ++next;
timers.set(id, { fn, ms });
return id;
},
clearTimer(id) {
timers.delete(id);
},
onRender() {
renders++;
},
});
card.handle({ type: 'interruption' });
card.handle({
type: 'transcript',
role: 'assistant',
text: 'A current answer',
final: true,
responseId: 'r1',
});
const content = card.state;
const timer = [...timers.keys()][0];
assert.equal(timers.get(timer).ms, 14000);
card.setPresented(false);
assert.equal(card.snapshot().visible, false);
assert.equal(ui.state.card.announce, '');
card.setPresented(true);
assert.equal(card.snapshot().visible, true);
assert.equal(card.snapshot().assistant.text, 'A current answer');
assert.equal(
card.snapshot().announce,
'',
'returning to a tab does not repeat an already presented announcement',
);
assert.equal(card.state, content);
assert.equal(
[...timers.keys()][0],
timer,
'presentation switches do not reset idle',
);
card.setEnabled(false);
card.handle({
type: 'transcript',
role: 'assistant',
text: 'A later answer',
final: true,
responseId: 'r2',
});
assert.equal(card.snapshot().visible, false);
card.setEnabled(true);
assert.equal(card.snapshot().assistant.text, 'A later answer');
assert.equal(card.snapshot().announce, 'A later answer');
assert.ok(renders > 4);
const idle = [...timers.values()][0];
idle.fn();
assert.equal(
card.snapshot().visible,
false,
'original idle still ends content presentation',
);
card.destroy();
const lastRenders = renders;
card.setPresented(false);
card.setEnabled(false);
assert.equal(renders, lastRenders);
});
test('a new hidden reply with identical words is still a fresh announcement', () => {
const ui = createTestVoiceUi();
const card = new VoiceCardControls({
ui,
setTimer: () => 1,
clearTimer() {},
});
card.handle({ type: 'interruption' });
card.handle({
type: 'transcript',
role: 'assistant',
text: 'Same words',
final: true,
responseId: 'r1',
});
card.setPresented(false);
card.handle({
type: 'transcript',
role: 'assistant',
text: 'Same words',
final: true,
responseId: 'r2',
});
card.setPresented(true);
assert.equal(card.snapshot().announce, 'Same words');
card.destroy();
});
test('additive presentation gates retain the current localized reducer labels', () => {
const previous = getLocale();
try {
setLocale('es');
const ui = createTestVoiceUi();
const card = new VoiceCardControls({
ui,
setTimer: () => 1,
clearTimer() {},
});
card.handle({ type: 'interruption' });
card.handle({
type: 'action-call',
name: 'annotate_map',
callId: 'a',
arguments: {},
});
assert.equal(card.snapshot().phase, 'EN CURSO');
const steps = card.state.steps;
const result = card.state.result;
const who = () => ui.state.card.user.label;
card.handle({ type: 'transcript', role: 'user', text: 'Same words', attribution: 'unverified' });
assert.equal(card.state.userAttribution, 'unverified');
assert.equal(card.snapshot().user.label, 'Habla reciente');
assert.equal(who(), 'Habla reciente');
card.handle({ type: 'transcript', role: 'user', text: 'Same words' });
assert.equal(card.state.userAttribution, null);
assert.equal(card.snapshot().user.label, 'TÚ');
assert.equal(who(), 'TÚ');
assert.equal(card.state.steps, steps);
assert.equal(card.state.result, result);
card.setPresented(false);
card.setPresented(true);
assert.equal(card.snapshot().phase, 'EN CURSO');
card.setEnabled(false);
card.setEnabled(true);
assert.equal(card.snapshot().phase, 'EN CURSO');
card.destroy();
} finally {
setLocale(previous);
}
});
test('manual panel and preference compose with the actual card without changing voice content or enabling Preview', async () => {
const { createManualVoicePanel } = await import('./panel/manualPanel.js');
const { createVoicePreferences, bindPreferenceControls } =
await import('./panel/preferences.js');
const ui = createTestVoiceUi({ panel: true });
let panel;
const card = new VoiceCardControls({
ui,
setTimer: () => 1,
clearTimer() {},
onRender: () => panel?.sync(),
});
const preference = createVoicePreferences({ storage: null });
const fields = bindPreferenceControls({
ui,
preferences: preference,
copy: {
previewNote: 'Mostrar respuestas.',
timeoutNote: '',
notSaved: 'No guardado.',
},
onChange: (state) => {
card.setEnabled(state.previewEnabled);
panel?.sync();
},
});
card.setEnabled(preference.snapshot().previewEnabled);
panel = createManualVoicePanel({
ui,
card,
getPreviewEnabled: () => preference.snapshot().previewEnabled,
copy: { previewEmpty: 'Sin respuesta.', previewDisabled: 'Desactivada.' },
});
card.handle({ type: 'interruption' });
card.handle({
type: 'transcript',
role: 'assistant',
text: 'Actual current answer',
final: true,
});
const content = card.state;
assert.equal(
card.snapshot().visible,
false,
'manual panel stays closed despite content',
);
panel.select('preview');
assert.equal(card.snapshot().assistant.text, 'Actual current answer');
assert.equal(card.snapshot().visible, true);
preference.setPreviewEnabled(false);
assert.equal(card.snapshot().visible, false);
assert.equal(ui.state.previewEmpty.text, 'Desactivada.');
panel.select('settings');
panel.select('preview');
assert.equal(preference.snapshot().previewEnabled, false);
assert.equal(card.snapshot().visible, false);
assert.equal(card.state, content);
fields.dispose();
panel.destroy();
card.destroy();
});
+109
View File
@@ -0,0 +1,109 @@
// A failed tool step: the card shows the tool error in the UI language; the
// result keeps gevActions' English for the model.
import assert from 'node:assert/strict';
import test from 'node:test';
import { setLocale } from '../i18n/index.js';
import { attachVoiceResult, spokenLabel } from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
// Error texts as gevActions.js builds them (frameOverhead, select_nearest_aircraft).
const CASES = [
{
name: 'frame_overhead',
args: { target: 'military' },
result: {
ok: false,
action: 'frame_overhead',
layerId: 'military',
error: 'The military layer is not enabled',
},
},
{
name: 'select_nearest_aircraft',
args: { locationQuery: 'Madrid' },
result: {
ok: false,
action: 'select_nearest_aircraft',
stage: 'nearest',
cancelled: false,
error:
'Live Flights is enabled, but no airborne aircraft is loaded in the Madrid view yet',
location: { label: 'Madrid' },
layer: { label: 'Live Flights' },
feed: { state: 'nominal', source: 'OpenSky' },
count: 0,
},
},
{
name: 'frame_overhead',
args: { target: 'flights' },
result: {
ok: false,
action: 'frame_overhead',
layerId: 'flights',
radiusKm: 150,
count: 0,
error: 'No flights within 150 km of the current view',
},
},
];
function cardFor(locale, { name, args, result }) {
setLocale(locale);
let state = reduceVoiceCard(initialVoiceCardState(), {
type: 'action-call',
id: 'c1',
name,
arguments: args,
});
state = reduceVoiceCard(state, {
type: 'action-result',
id: 'c1',
name,
result: attachVoiceResult(name, result, args),
});
return voiceCardView(state);
}
for (const locale of ['es', 'ru']) {
for (const c of CASES) {
test(`${locale}: ${c.name} failure shows the card lines in the UI language`, () => {
const view = cardFor(locale, c);
try {
assert.ok(view.result.visible, 'card shows a result');
assert.notEqual(view.result.title, '', 'card has a title');
// Title and step status are translated.
assert.notEqual(
view.steps[0].statusText,
'failed',
'step status translated',
);
// The error line is translated, not the raw English.
for (const line of view.result.lines) {
assert.ok(
!/\b(is not enabled|no airborne aircraft|of the current view)\b/.test(
line,
),
`English tool error shown under ${locale} title "${view.result.title}" / step "${view.steps[0].label}: ${view.steps[0].statusText}": "${line}"`,
);
}
} finally {
setLocale('en');
}
});
}
}
test('en: the card shows the tool error as the model got it', () => {
for (const c of CASES) {
const view = cardFor('en', c);
// Whole, not cut at 80 characters: the card wraps an error line.
assert.deepEqual(view.result.lines, [
spokenLabel(c.result.error, Infinity),
]);
}
});
+142
View File
@@ -0,0 +1,142 @@
// The voice card names layers, sources and distances in the interface
// language; what the model receives stays English.
import test from 'node:test';
import assert from 'node:assert/strict';
import { setLocale } from '../i18n/index.js';
import {
attachVoiceResult,
planStepLabel,
shownEntityLabel,
} from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
for (const locale of ['ru', 'es']) {
test(`voice card is in ${locale} for a layer switch`, (t) => {
t.after(() => setLocale('en'));
setLocale(locale);
const out = attachVoiceResult(
'set_layer_visibility',
{
ok: true,
layerId: 'flights',
label: 'Live Flights',
enabled: true,
source: 'Flights',
},
{ layerId: 'flights', enabled: true },
);
const view = voiceCardView({ steps: [], result: out });
assert.doesNotMatch(out.display.title, /Live Flights/);
assert.doesNotMatch(JSON.stringify(view), /Flights/);
});
test(`voice card distance is in ${locale}`, (t) => {
t.after(() => setLocale('en'));
setLocale(locale);
const out = attachVoiceResult('fly_to_location', {
ok: true,
label: 'Paris',
rangeM: 2500,
});
assert.doesNotMatch(out.display.lines[0], /2\.5 km/);
});
test(`plan label is in ${locale}`, (t) => {
t.after(() => setLocale('en'));
setLocale(locale);
const a = planStepLabel('select_nearest_aircraft', {});
const b = planStepLabel('set_layer_visibility', {
layerId: 'earthquakes',
enabled: true,
});
assert.doesNotMatch(a, /Here/);
assert.doesNotMatch(b, /Earthquakes/);
});
}
test('en: layer switch, distance and plan labels in English', () => {
const out = attachVoiceResult(
'set_layer_visibility',
{
ok: true,
layerId: 'flights',
label: 'Live Flights',
enabled: true,
source: 'Flights',
},
{ layerId: 'flights', enabled: true },
);
assert.equal(out.say, 'Live Flights on.');
assert.equal(out.display.title, 'Live Flights on');
assert.deepEqual(out.display.sources, [{ label: 'Flights' }]);
const fly = attachVoiceResult('fly_to_location', {
ok: true,
label: 'Paris',
rangeM: 2500,
});
assert.equal(fly.display.lines[0].replace(/\s/g, ' '), 'Camera range 2.5 km');
assert.equal(
planStepLabel('select_nearest_aircraft', {}),
'Nearest aircraft to Here',
);
assert.equal(
planStepLabel('frame_overhead', { target: 'vessels' }),
'Frame vessels',
);
// The plan names the layer as its result card does, not by its id.
assert.equal(
planStepLabel('set_layer_visibility', {
layerId: 'earthquakes',
enabled: true,
}),
'Earthquakes (24h) on',
);
});
// FIRMS and ALPR register English entity labels: the model reads them as data
// (get_entity_context, pointer_context), the card shows them translated.
const ENTITY_LABELS = {
ru: { fire: 'Пожар · FRP 12 МВт', alpr: 'Камера ALPR' },
es: { fire: 'Incendio · FRP 12 MW', alpr: 'Cámara ALPR' },
};
for (const [locale, shown] of Object.entries(ENTITY_LABELS)) {
test(`entity labels stay English in the result and show in ${locale}`, (t) => {
t.after(() => setLocale('en'));
setLocale(locale);
const out = attachVoiceResult('get_entity_context', {
ok: true,
selected: {
id: 'fire-1',
name: 'Fire · FRP 12 MW',
layerId: 'local-firms',
layerName: 'NASA FIRMS',
},
});
assert.equal(out.selected.name, 'Fire · FRP 12 MW');
assert.equal(out.display.title, shown.fire);
const state = reduceVoiceCard(initialVoiceCardState(), {
type: 'pointer',
pointer: { kind: 'this', label: 'ALPR camera' },
});
assert.equal(state.pointer.label, 'ALPR camera');
assert.equal(voiceCardView(state).pointer.text, shown.alpr);
assert.equal(state.announce, `This: ${shown.alpr}`);
});
}
test('only the registered entity labels are translated', () => {
assert.equal(shownEntityLabel('Fire · FRP 4.9 MW'), 'Fire · FRP 4.9 MW');
assert.equal(shownEntityLabel('ALPR camera'), 'ALPR camera');
setLocale('ru');
try {
assert.equal(shownEntityLabel('Fire · FRP 4.9 MW'), 'Пожар · FRP 4.9 МВт');
assert.equal(shownEntityLabel('UPS793'), 'UPS793');
assert.equal(shownEntityLabel('Fire Station 5'), 'Fire Station 5');
} finally {
setLocale('en');
}
});
@@ -0,0 +1,48 @@
// The voice card names a preset city in the interface language in the
// fly-to headline and the nearest-aircraft distance line; what the model
// says stays English.
import assert from 'node:assert/strict';
import test from 'node:test';
import { setLocale } from '../i18n/index.js';
import { SPEECH_BUILDERS } from './speech.js';
function inLocale(locale, run) {
setLocale(locale);
try {
return run();
} finally {
setLocale('en');
}
}
const FLY = { ok: true, arrived: true, label: 'Tokyo' };
const NEAREST = {
ok: true,
layerId: 'flights',
location: { label: 'Tokyo' },
aircraft: { callsign: 'JAL123', distanceKm: 12, altitudeM: 3000 },
feed: {},
};
test('the fly-to headline shows the city in the interface language', () => {
assert.equal(SPEECH_BUILDERS.fly_to_location(FLY).display.title, 'Tokyo');
const ru = inLocale('ru', () => SPEECH_BUILDERS.fly_to_location(FLY));
assert.equal(ru.display.title, 'Токио');
assert.match(ru.say, /Tokyo/);
const es = inLocale('es', () => SPEECH_BUILDERS.fly_to_location(FLY));
assert.equal(es.display.title, 'Tokio');
});
test('the nearest-aircraft distance line shows the city in the interface language', () => {
const en = SPEECH_BUILDERS.select_nearest_aircraft(NEAREST);
assert.match(en.display.lines[0], /from Tokyo$/);
const ru = inLocale('ru', () =>
SPEECH_BUILDERS.select_nearest_aircraft(NEAREST),
);
assert.match(ru.display.lines[0], /Токио/);
assert.match(ru.say, /Tokyo/);
const es = inLocale('es', () =>
SPEECH_BUILDERS.select_nearest_aircraft(NEAREST),
);
assert.match(es.display.lines[0], /Tokio/);
});
+282
View File
@@ -0,0 +1,282 @@
// Voice card in the UI language: every tool's plan step, the bare "Layer"
// step, whole error lines and long headlines, the Contacts snapshot clock.
// What the model receives stays English.
import assert from 'node:assert/strict';
import test from 'node:test';
import { setLocale } from '../i18n/index.js';
import { GEV_ACTION_SCHEMAS } from './actionSchemas.js';
import { presentResult, shownSnapshotTime } from './resultDisplay.js';
import { CITY_POIS } from '../cityPois.js';
import { attachVoiceResult, narrationLabel, planStepLabel } from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
const TOOLS = GEV_ACTION_SCHEMAS.map((schema) => schema.name);
function inLocale(locale, run) {
setLocale(locale);
try {
return run();
} finally {
setLocale('en');
}
}
test('every tool has a plan label, never its raw name', () => {
for (const name of TOOLS) {
const label = planStepLabel(name, {});
assert.notEqual(label, name.replace(/_/g, ' '), name);
assert.doesNotMatch(label, /_/, name);
}
assert.equal(planStepLabel('analyst_query', {}), 'Analyst query');
assert.equal(
planStepLabel('adjust_camera_zoom', { direction: 'in' }),
'Zoom in',
);
assert.equal(
planStepLabel('adjust_camera_zoom', { direction: 'out' }),
'Zoom out',
);
// A tool the card does not know keeps its name in English.
assert.equal(planStepLabel('brand_new_tool', {}), 'brand new tool');
});
for (const locale of ['ru', 'es']) {
test(`${locale} plan labels are all translated`, () => {
const english = Object.fromEntries(
TOOLS.map((name) => [name, planStepLabel(name, { direction: 'out' })]),
);
inLocale(locale, () => {
for (const name of TOOLS) {
// A layer switch without a layer is checked below.
const label = planStepLabel(name, { direction: 'out' });
assert.notEqual(label, english[name], `${name}: "${label}"`);
}
});
});
}
test('analyst query and an unknown tool in ru and es', () => {
inLocale('ru', () => {
assert.equal(planStepLabel('analyst_query', {}), 'Аналитический запрос');
assert.equal(planStepLabel('brand_new_tool', {}), 'Действие');
});
inLocale('es', () => {
assert.equal(planStepLabel('analyst_query', {}), 'Consulta analítica');
assert.equal(planStepLabel('brand_new_tool', {}), 'Acción');
});
});
test('a layer step with no layer agrees with "Capa"', () => {
const step = (enabled) => planStepLabel('set_layer_visibility', { enabled });
assert.equal(step(true), 'Layer on');
assert.equal(step(false), 'Layer off');
inLocale('es', () => {
assert.equal(step(true), 'Capa: activada');
assert.equal(step(false), 'Capa: desactivada');
// A named layer keeps the shared "{label} on" wording.
assert.match(
planStepLabel('set_layer_visibility', {
layerId: 'earthquakes',
enabled: true,
}),
/: activado$/,
);
});
inLocale('ru', () => {
assert.equal(step(true), 'Слой: вкл.');
assert.equal(step(false), 'Слой: выкл.');
});
});
function cardAfter(name, result, args = {}) {
let state = reduceVoiceCard(initialVoiceCardState(), {
type: 'action-call',
id: 'c1',
name,
arguments: args,
});
state = reduceVoiceCard(state, {
type: 'action-result',
id: 'c1',
name,
result: attachVoiceResult(name, result, args),
});
return voiceCardView(state);
}
const NO_PLACE = {
ok: false,
action: 'select_nearest_aircraft',
stage: 'location',
error:
'Nearest-aircraft selection needs a preset, place name, or latitude and longitude',
feed: {},
};
test('an error line shows whole, in every language', () => {
for (const [locale, expected] of [
['en', NO_PLACE.error],
[
'ru',
'Для выбора ближайшего самолёта нужен пресет, название места или широта и долгота',
],
[
'es',
'Para seleccionar la aeronave más cercana se necesita un preajuste, un nombre de lugar o la latitud y la longitud',
],
]) {
const view = inLocale(locale, () =>
cardAfter('select_nearest_aircraft', NO_PLACE),
);
assert.deepEqual(view.result.lines, [expected], locale);
}
});
test('a successful line keeps its 80-character cap', () => {
const long = 'x'.repeat(120);
const view = voiceCardView({
...initialVoiceCardState(),
result: { display: { title: 'T', lines: [long] } },
});
assert.equal(view.result.lines[0].length, 80);
});
test('a translated analyst headline fits the card title whole', () => {
const title =
'Al menos 12 aeronaves militares en un radio de 40 km de Los Angeles';
const card = () =>
voiceCardView({
...initialVoiceCardState(),
result: { display: { title, lines: [] } },
}).result.title;
assert.equal(inLocale('es', card), title);
// English keeps its 64-character title, equal to the spoken headline.
assert.equal(card().length, 64);
});
test('the snapshot time reads as a UTC clock, dated when not today', () => {
const iso = '2026-10-15T14:32:05.000Z';
const sameDay = Date.parse('2026-10-15T23:00:00Z');
const nextDay = Date.parse('2026-10-16T00:10:00Z');
assert.equal(shownSnapshotTime(iso, sameDay), iso, 'English keeps ISO');
inLocale('ru', () => {
assert.equal(shownSnapshotTime(iso, sameDay), '14:32:05 UTC');
assert.equal(shownSnapshotTime(iso, nextDay), '15.10.2026, 14:32:05 UTC');
});
inLocale('es', () => {
assert.equal(shownSnapshotTime(iso, sameDay), '14:32:05 UTC');
assert.equal(shownSnapshotTime(iso, nextDay), '15/10/2026, 14:32:05 UTC');
assert.equal(shownSnapshotTime('not a time'), 'not a time');
});
});
test('the Contacts note shows the clock; the model note stays ISO', () => {
const evaluatedAt = new Date();
evaluatedAt.setUTCHours(14, 32, 5, 0);
const iso = evaluatedAt.toISOString();
const note = `Contacts panel snapshot at ${iso} — the same retained cohort supplies the card, speech, and follow-up memory.`;
const result = {
ok: true,
count: 12,
complete: true,
scopeLabel: 'in view',
display: { scope: 'in view' },
coverage: { note },
items: [],
};
const notes = inLocale('es', () =>
presentResult('analyst_query', result).display.notes.join(' | '),
);
assert.match(notes, /\(14:32:05 UTC\)/);
assert.doesNotMatch(notes, /T14:32/);
assert.equal(result.coverage.note, note, 'the result is not changed');
});
test('plan steps and errors name a preset place in the UI language', () => {
const failed = {
ok: false,
action: 'select_nearest_aircraft',
stage: 'nearest',
error:
'Live Flights is enabled, but no airborne aircraft is loaded in the Tokyo view yet',
location: { label: 'Tokyo' },
feed: { state: 'nominal' },
};
assert.equal(
planStepLabel('fly_to_location', { query: 'Tokyo' }),
'Fly to Tokyo',
);
inLocale('es', () => {
assert.match(planStepLabel('fly_to_location', { query: 'Tokyo' }), /Tokio/);
assert.match(
planStepLabel('select_nearest_aircraft', { locationId: 'tokyo' }),
/Tokio/,
);
const view = cardAfter('select_nearest_aircraft', failed);
assert.match(view.result.lines[0], /Tokio/);
});
inLocale('ru', () => {
assert.match(planStepLabel('fly_to_location', { query: 'Tokyo' }), /Токио/);
});
assert.match(failed.error, /Tokyo view/, 'the result stays English');
});
test('the analyst scope names Los Angeles in the UI language', () => {
const result = {
ok: true,
count: 12,
complete: false,
scopeLabel: 'within 40 km of Los Angeles',
display: { scope: 'within 40 km of Los Angeles' },
coverage: { layersQueried: [{ layerKey: 'military' }] },
items: [],
};
const title = inLocale(
'es',
() => presentResult('analyst_query', result).display.title,
);
assert.match(title, /Los Ángeles$/, 'the headline is not cut at 64');
assert.equal(result.scopeLabel, 'within 40 km of Los Angeles');
});
// A preset city id from the tool schema (sf, nyc, dc) reads as its city name.
test('a preset city id names the city in plan steps and narration', () => {
const step = (id, action = 'fly_to_location') =>
planStepLabel(action, { locationId: id });
assert.equal(step('sf'), 'Fly to San Francisco');
assert.equal(
narrationLabel('fly_to_location', { locationId: 'dc' }),
'Washington DC',
);
assert.equal(
planStepLabel('fly_to_location', { query: 'Ulaanbaatar' }),
'Fly to Ulaanbaatar',
'other places are shown as given',
);
inLocale('es', () => {
assert.equal(step('tokyo'), 'Volar a Tokio');
assert.equal(step('nyc'), 'Volar a Nueva York');
for (const id of ['sf', 'nyc', 'dc'])
assert.doesNotMatch(step(id), /\b(Sf|Nyc|Dc)$/, id);
});
inLocale('ru', () => {
assert.match(step('dc', 'select_nearest_aircraft'), /Вашингтон/);
});
});
test('every locationId the voice tools accept is a preset city', () => {
const ids = new Set();
const visit = (node) => {
if (!node || typeof node !== 'object') return;
if (Array.isArray(node.locationId?.enum))
node.locationId.enum.forEach((id) => ids.add(id));
Object.values(node).forEach(visit);
};
visit(GEV_ACTION_SCHEMAS);
assert.ok(ids.size > 0, 'found the locationId enums');
for (const id of ids) assert.ok(Object.hasOwn(CITY_POIS, id), id);
});
+50 -19
View File
@@ -1,7 +1,13 @@
import { planStepLabel, progressStepLabel, spokenLabel } from './speech.js';
import {
planStepLabel,
progressStepLabel,
shownEntityLabel,
spokenLabel,
} from './speech.js';
import { presentResult } from './resultDisplay.js';
import { MAX_DISPLAYED_REFERENTS, normalizeReferents } from './referents.js';
import { getLocale, msg, t } from '../i18n/index.js';
/**
* Pure presentation of the voice card nested in the voice control: captions,
* the current plan, the latest result and its numbered referents.
@@ -16,12 +22,20 @@ const MAX_STEPS = 6;
const MAX_LINES = 6;
const MAX_NOTES = 6;
const MAX_CAPTION = 220;
// Titles and lines wrap; the caps only stop runaway values. Translations run
// longer (resultDisplay.js), so they cap at 120. Error lines are never cut.
const MAX_TITLE = 64;
const MAX_TRANSLATED_TITLE = 120;
const MAX_LINE = 80;
const titleMax = () =>
getLocale() === 'en' ? MAX_TITLE : MAX_TRANSLATED_TITLE;
const STATUS_TEXT = Object.freeze({
running: 'in progress',
done: 'done',
failed: 'failed',
cancelled: 'cancelled',
running: msg('in progress'),
done: msg('done'),
failed: msg('failed'),
cancelled: msg('cancelled'),
});
/** An empty, hidden card. */
@@ -30,6 +44,7 @@ export function initialVoiceCardState() {
visible: false,
dismissed: false,
user: null,
userAttribution: null,
pointer: null,
assistant: null,
assistantResponse: null,
@@ -55,6 +70,7 @@ function newTurn(state) {
visible: true,
dismissed: false,
user: '',
userAttribution: null,
pointer: null,
assistant: null,
assistantResponse: null,
@@ -123,13 +139,20 @@ export function reduceVoiceCard(state, event) {
return reveal({
...state,
pointer,
announce: `${kind}: ${spokenLabel(pointer.label, 32)}`,
announce: `${kind}: ${spokenLabel(shownEntityLabel(pointer.label), 32)}`,
});
}
case 'transcript': {
const text = event.text || '';
if (event.role === 'user') {
return text ? reveal({ ...state, user: caption(text) }) : state;
return text
? reveal({
...state,
user: caption(text),
userAttribution:
event.attribution === 'unverified' ? 'unverified' : null,
})
: state;
}
if (event.role !== 'assistant' || !text) return state;
const sameResponse =
@@ -190,7 +213,7 @@ export function reduceVoiceCard(state, event) {
: result.ok
? 'done'
: 'failed',
detail: result.outlinePending ? 'Tracing outline' : '',
detail: result.outlinePending ? t('Tracing outline') : '',
annotationIds: result.outlinePending ? annotationIds(result) : [],
}),
);
@@ -204,12 +227,14 @@ export function reduceVoiceCard(state, event) {
...next,
result: {
display: shown.display,
// A failed result's lines are its error: shown whole, wrapped.
wholeLines: result.ok === false,
referents: normalizeReferents(
shown.referents,
MAX_DISPLAYED_REFERENTS,
),
},
announce: spokenLabel(shown.display.title, 64),
announce: spokenLabel(shown.display.title, titleMax()),
});
}
case 'outline': {
@@ -224,10 +249,10 @@ export function reduceVoiceCard(state, event) {
...step,
annotationIds: remaining,
detail: remaining.length
? 'Tracing outline'
? t('Tracing outline')
: event.status === 'resolved'
? 'Outline traced'
: 'Outline unavailable',
? t('Outline traced')
: t('Outline unavailable'),
};
return reveal({ ...state, steps });
}
@@ -244,7 +269,9 @@ export function voiceCardView(state) {
const display = state.result?.display || null;
const allNotes = [
...(display?.notes || []),
...(display?.sources || []).map((source) => `Source: ${source.label}`),
...(display?.sources || []).map((source) =>
t('Source: {label}', { label: source.label }),
),
];
// Analyst notes carry count/scope/provenance qualifications. Preserve their
// full sanitized text in at most six existing rows, joining overflow into
@@ -267,15 +294,17 @@ export function voiceCardView(state) {
return {
visible: Boolean(state.visible),
busy: running,
phase: running ? 'WORKING' : state.steps.length ? 'DONE' : '',
phase: running ? t('WORKING') : state.steps.length ? t('DONE') : '',
user: {
label:
state.userAttribution === 'unverified' ? t('Recent speech') : t('YOU'),
visible: state.user !== null,
text: state.user || '…',
},
pointer: {
visible: Boolean(state.pointer),
kind: state.pointer?.kind === 'here' ? 'HERE' : 'THIS',
text: spokenLabel(state.pointer?.label || '', 32),
kind: state.pointer?.kind === 'here' ? t('HERE') : t('THIS'),
text: spokenLabel(shownEntityLabel(state.pointer?.label || ''), 32),
},
assistant: {
visible: Boolean(state.assistant),
@@ -285,15 +314,17 @@ export function voiceCardView(state) {
key: String(step.id),
label: step.label,
status: step.status,
statusText: STATUS_TEXT[step.status] || '',
statusText: STATUS_TEXT[step.status] ? t(STATUS_TEXT[step.status]) : '',
detail: step.detail || '',
})),
result: {
visible: Boolean(display?.title),
title: spokenLabel(display?.title || '', 64),
title: spokenLabel(display?.title || '', titleMax()),
chips: (display?.chips || []).map((chip) => spokenLabel(chip.label, 24)),
lines: (display?.lines || [])
.map((line) => spokenLabel(line, 80))
.map((line) =>
spokenLabel(line, state.result?.wholeLines ? Infinity : MAX_LINE),
)
.slice(0, MAX_LINES),
referents: (state.result?.referents || [])
.slice(0, MAX_DISPLAYED_REFERENTS)
+132
View File
@@ -0,0 +1,132 @@
// src/voice/voiceControlView.test.mjs — the voice control renders the card's
// lists and the waveform from its presenter, over happy-dom.
import '../testSupport/registerDom.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
import { createTestVoiceUi } from '../testSupport/voiceUi.mjs';
import { mountVoiceView } from '../testSupport/voiceView.mjs';
import { useVoiceStore } from '../ui/stores/voice.js';
import { createManualVoicePanel } from './panel/manualPanel.js';
const $ = (selector) => document.querySelector(selector);
const $$ = (selector) => [...document.querySelectorAll(selector)];
test('the card renders its plan, chips, lines, referents and notes', async (t) => {
const { pinia, settle } = mountVoiceView(t);
const ui = createTestVoiceUi({ pinia });
ui.replace('card', {
...ui.state.card,
visible: true,
busy: true,
phase: 'WORKING',
steps: [
{
key: 'a',
label: 'Fly to Austin',
status: 'done',
statusText: 'done',
detail: '',
},
{
key: 'b',
label: 'Show traffic',
status: 'running',
statusText: '',
detail: 'Loading',
},
],
result: {
visible: true,
title: 'Austin',
chips: ['live'],
lines: ['Line one', 'Line two'],
referents: [{ n: 3, label: 'Tower' }],
notes: ['Source: Example'],
},
announce: 'Austin',
});
await settle();
assert.equal($('#gev-voice-card').hidden, false);
assert.equal($('#gev-voice-card').dataset.busy, 'true');
assert.equal($('#gev-voice-card-phase').textContent, 'WORKING');
const steps = $$('#gev-voice-card-plan .gev-voice-card-step');
assert.deepEqual(
steps.map((step) => [step.dataset.status, step.textContent]),
[
['done', 'Fly to Austin, done'],
['running', 'Show trafficLoading'],
],
);
assert.equal($('#gev-voice-card-plan').hidden, false);
const chip = $('#gev-voice-card-chips .gev-voice-card-chip');
assert.equal(chip.dataset.tone, 'live');
assert.equal(chip.textContent, 'live');
assert.deepEqual(
$$('#gev-voice-card-lines li').map((li) => li.textContent),
['Line one', 'Line two'],
);
const referent = $('#gev-voice-card-referents li');
assert.equal(referent.value, 3);
assert.equal(referent.textContent, 'Tower');
assert.equal($('#gev-voice-card-notes').hidden, false);
assert.equal($('#gev-voice-card-live').textContent, 'Austin');
ui.replace('card', {
...ui.state.card,
steps: [],
result: { ...ui.state.card.result, chips: [], referents: [], notes: [] },
});
await settle();
assert.equal($('#gev-voice-card-plan').hidden, true);
assert.equal($('#gev-voice-card-chips').hidden, true);
assert.equal($('#gev-voice-card-referents').hidden, true);
assert.equal($('#gev-voice-card-notes').hidden, true);
});
test('the waveform shows bars as they change and rests to the CSS baseline', async (t) => {
const { pinia, settle } = mountVoiceView(t);
const ui = createTestVoiceUi({ pinia });
await settle();
const bars = () => $$('.gev-voice-visualizer span');
assert.equal(bars().length, 15);
assert.equal(bars()[4].style.getPropertyValue('--bar'), '4');
assert.equal(bars()[4].style.getPropertyValue('--audio-level'), '');
const frame = Array.from({ length: 15 }, (_, i) => ({
level: `${5 + i}px`,
opacity: '0.80',
}));
ui.setBars(frame);
await settle();
assert.equal(bars()[4].style.getPropertyValue('--audio-level'), '9px');
assert.equal(bars()[4].style.getPropertyValue('--audio-opacity'), '0.80');
// An identical frame publishes nothing, and bars never replace the
// control's view, so the rest of the dock does not re-render for them.
const store = useVoiceStore(pinia);
const view = store.view;
const published = store.bars;
ui.setBars(frame.map((bar) => ({ ...bar })));
assert.equal(store.bars === published, true);
assert.equal(store.view === view, true);
ui.setBars(null);
await settle();
assert.equal(bars()[4].style.getPropertyValue('--audio-level'), '');
assert.equal(bars()[4].style.getPropertyValue('--bar'), '4');
});
test('Preview is not reported visible while the error tray replaces the card', async (t) => {
const { pinia, settle } = mountVoiceView(t);
const ui = createTestVoiceUi({ panel: true, pinia });
ui.replace('card', { ...ui.state.card, visible: true });
const panel = createManualVoicePanel({
ui,
card: { setPresented() {} },
getPreviewEnabled: () => true,
copy: { previewEmpty: 'Empty', previewDisabled: 'Off' },
});
t.after(() => panel.destroy());
panel.select('preview');
await settle();
assert.equal(panel.snapshot().previewVisible, true);
ui.update('root', { status: 'error' });
assert.equal(panel.snapshot().previewVisible, false);
});
+2 -2
View File
@@ -50,7 +50,7 @@ export const VOICE_MODELS = Object.freeze({
standard: Object.freeze({
tier: 'standard',
id: 'gpt-realtime-2',
label: 'STANDARD',
label: 'STANDARD', // i18n-ignore: model data; the tier button has its own text
/** USD per 1M tokens — gpt-realtime-2. */
rates: Object.freeze({
textInput: 4,
@@ -66,7 +66,7 @@ export const VOICE_MODELS = Object.freeze({
mini: Object.freeze({
tier: 'mini',
id: 'gpt-realtime-2.1-mini',
label: 'MINI',
label: 'MINI', // i18n-ignore: model data; the tier button has its own text
/** USD per 1M tokens — gpt-realtime-2.1-mini (~3.2× cheaper on audio). */
rates: Object.freeze({
textInput: 0.6,