Merge upstream/main into fix/drone-airspace

Move the share-link token to 6 (3 is gnss-integrity), build altitude
ranges from LOWER/UPPER VAL, UOM and CODE, mark card and legend strings
for translation, rename the layer to Prohibited & Restricted Airspace,
move the changelog entry to changelog.d, and keep the layer voice-off
while the Realtime prompt is at its budget.
This commit is contained in:
Matt Van Horn committed 2026-10-27 05:06:05 -07:00
commit 4c2159a70a
1387 files changed
+160157 -23954

No files matched your search

+14 -1
View File
@@ -178,6 +178,9 @@ const schemas = [
'global-context-panel',
'weather-panel',
'recent-imagery-panel',
'voice-settings',
'voice-preview',
'provider-settings',
],
},
open: {
@@ -187,6 +190,15 @@ const schemas = [
required: ['panelId', 'open'],
},
},
{
name: 'set_voice_inactivity_timeout',
parameters: {
type: 'object',
additionalProperties: false,
properties: { minutes: { type: 'integer', minimum: 0, maximum: 60 } },
required: ['minutes'],
},
},
{
name: 'set_context_mode',
parameters: {
@@ -223,6 +235,7 @@ const schemas = [
aircraftClass: {
type: 'string',
},
voicePanel: { type: 'string', enum: ['preview', 'settings'] },
},
required: ['action'],
},
@@ -729,7 +742,7 @@ const schemas = [
properties: {
motion: {
type: 'string',
enum: ['orbit', 'pan', 'tilt', 'rotate', 'stop'],
enum: ['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe', 'stop'],
},
direction: {
type: 'string',
+77 -6
View File
@@ -1,3 +1,4 @@
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
import test from 'node:test';
import assert from 'node:assert/strict';
import { createHash } from 'node:crypto';
@@ -31,14 +32,20 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
.digest('hex');
assert.equal(
digest,
// Re-derived for the voice layer manifest (generated layer enums and
// Re-derived for Provider Settings and explicit Visual Presets/Scenes routing,
// plus the deliberate direct Panel tool/target additions and voice layer manifest (generated layer enums and
// aliases), point-and-ask (pointer sentinels, referent args) and the
// consolidated tool wording (each policy stated once); model-facing tools
// omit top-level anyOf (OpenAI rejects it). Contacts list routing preserves
// requested radius and center authority in descriptions only.
// Street Level adds its toggle enum values and generated alias hint.
// Drone Restrictions adds toggle/query enum values and spoken aliases.
'3ceef7975c1437eacb2c602bf9728829b299532b7dd95534cf7d9eb47115fa54',
// Street Level and GNSS extend the manifest; tracked and analyst aircraft describe broadcasts.
// Air quality (ECCC AQHI) extends the manifest.
// Globe rotation adds one motion; visual keyhole fit and explicit
// reset guidance/defaults update only move_camera and mode descriptions.
// S2 GhostMaps extends the generated layer enums.
// Panel adds named voice targets and timeout; shared Panel/Cockpit guidance
// is consolidated without changing either feature's arguments.
'9ee82c4214499280c208c93751b19d7b724c81b7727a2d0c6dab0184749fbe15',
);
});
@@ -96,19 +103,43 @@ test('metadata cannot add tools, fields, types or enum values', () => {
test('all legacy action arguments are byte-identical after removing the deliberate additions', () => {
const legacy = structuredClone(GEV_ACTION_SCHEMAS).filter(
(tool) => !['next_satellite_pass', 'set_cyber_sonar'].includes(tool.name),
(tool) =>
![
'next_satellite_pass',
'set_cyber_sonar',
'set_voice_inactivity_timeout',
].includes(tool.name),
);
// Layer enums are generated from the voice layer manifest and pinned by
// layerManifest.test.mjs; the two shipped right-rail panels are additive.
const property = (name) =>
legacy.find((tool) => tool.name === name).parameters.properties;
delete property('control_cockpit').voicePanel;
delete property('set_layer_visibility').layerId.enum;
delete property('show_data_layers_menu').layerId.enum;
delete property('get_entity_context').layerId.enum;
delete property('analyst_query').layers.items.enum;
// Earth-centered rotation is additive; local camera yaw keeps its contract.
const motion = property('move_camera').motion;
assert.deepEqual(motion.enum, [
'orbit',
'pan',
'tilt',
'rotate',
'rotate_globe',
'stop',
]);
motion.enum = motion.enum.filter((value) => value !== 'rotate_globe');
const panels = property('set_panel_open').panelId;
panels.enum = panels.enum.filter(
(id) => !['weather-panel', 'recent-imagery-panel'].includes(id),
(id) =>
![
'weather-panel',
'recent-imagery-panel',
'voice-settings',
'voice-preview',
'provider-settings',
].includes(id),
);
for (const tool of legacy) {
// Point-and-ask adds `referent` arguments and 'pointer' enum values.
@@ -145,6 +176,24 @@ test('all legacy action arguments are byte-identical after removing the delibera
);
});
test('globe rotation is an additive camera motion with the existing control arguments', () => {
const schema = GEV_ACTION_SCHEMAS.find(
(tool) => tool.name === 'move_camera',
).parameters;
for (const motion of ['rotate_globe', 'rotate', 'orbit', 'pan', 'tilt']) {
assert.deepEqual(
validateArgs(schema, {
motion,
direction: 'right',
speed: 'slow',
mode: 'continuous',
}),
[],
);
}
assert.ok(validateArgs(schema, { motion: 'spin_camera' }).length > 0);
});
test('track_entity accepts a referent alone but rejects an empty target', () => {
const schema = GEV_ACTION_SCHEMAS.find(
(tool) => tool.name === 'track_entity',
@@ -183,3 +232,25 @@ test('the benchmark still rejects an empty track_entity although the model-facin
[],
);
});
test('globe reset guidance requires a separate explicit command after refusal', () => {
const paragraph = realtimeInstructions()
.split('\n')
.find((line) => line.includes('globe-view-required'));
assert.ok(paragraph);
assert.match(paragraph, /calls only move_camera motion rotate_globe/);
assert.match(paragraph, /explicit user Reset\/Show Globe View/);
assert.match(paragraph, /never from your own plan/);
assert.match(paragraph, /bare yes\/ok/);
assert.match(paragraph, /Do not ask a question/);
});
test('direct voice timeout schema accepts None and exact integers, rejects malformed duration', () => {
const tool = GEV_REALTIME_TOOLS.find(
(tool) => tool.name === 'set_voice_inactivity_timeout',
);
for (const minutes of [0, 1, 5, 60])
assert.equal(validateArgs(tool.parameters, { minutes }).length, 0);
for (const minutes of [-1, 61, 1.5, '5'])
assert.ok(validateArgs(tool.parameters, { minutes }).length > 0);
});
+102
View File
@@ -0,0 +1,102 @@
// The analyst card shows its headline, scope, chips and caveats in the UI
// language; the spoken headline stays English for the model.
import assert from 'node:assert/strict';
import test from 'node:test';
import { setLocale } from '../i18n/index.js';
import { attachVoiceResult as withSpeech } from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
function cardFor(name, result, args = {}) {
const shaped = withSpeech(name, result, args);
return {
shaped,
view: voiceCardView(
[
{ type: 'action-call', callId: 'c1', name, arguments: args },
{ type: 'action-result', callId: 'c1', name, result: shaped },
].reduce(reduceVoiceCard, initialVoiceCardState()),
),
};
}
const analyst = {
ok: true,
count: 3,
complete: true,
scopeLabel: 'anywhere in the loaded data',
partial: true,
feedState: 'degraded',
unanswered: ['earthquakes'],
coverage: { layersQueried: [{ layerKey: 'flights', source: 'OpenSky' }] },
display: { scope: 'anywhere in the loaded data' },
items: [],
};
const radius = {
...analyst,
count: 1234,
complete: false,
partial: false,
feedState: 'nominal',
unanswered: [],
scopeLabel: 'in view',
display: {
scope: 'within 42 km of the view centre',
window: 'last 24 h',
caveat:
'counted 1,000 of 2,345 loaded records; earthquakes loading; flights: distance is ground distance, not overhead',
},
coverage: {
layersQueried: [{ layerKey: 'flights', source: 'OpenSky' }],
note: 'the flights layer loads by viewport',
},
};
test('analyst card in es has no English text', () => {
setLocale('es');
try {
const { view } = cardFor('analyst_query', analyst);
assert.doesNotMatch(view.result.title, /anywhere in the loaded data/);
assert.ok(!view.result.chips.includes('partial'));
assert.ok(!view.result.chips.includes('degraded'));
assert.ok(!view.result.notes.some((n) => n.startsWith('Not answered')));
} finally {
setLocale('en');
}
});
test('analyst scope, window and caveats in ru', () => {
setLocale('ru');
try {
const { shaped, view } = cardFor('analyst_query', radius);
assert.match(shaped.say, /^At least 1,234 aircraft in view/);
assert.doesNotMatch(
JSON.stringify(view.result),
/aircraft|within|view centre|last 24 h|counted|loading|ground distance|loads by viewport/,
);
assert.match(view.result.title, /^1\s234 /);
} finally {
setLocale('en');
}
});
test('en: the card keeps its English text and names layers', () => {
const { shaped, view } = cardFor('analyst_query', analyst);
assert.equal(view.result.title, '3 aircraft anywhere in the loaded data');
assert.deepEqual(view.result.chips, ['degraded', 'partial']);
assert.ok(view.result.notes.includes('Not answered: Earthquakes (24h)'));
// The model still gets the layer key.
assert.match(shaped.say, /earthquakes not answered/);
const capped = cardFor('analyst_query', radius).view.result;
assert.equal(capped.title, 'At least 1,234 aircraft in view');
assert.deepEqual(capped.lines, [
'within 42 km of the view centre',
'last 24 h',
]);
assert.ok(capped.notes.includes('counted 1,000 of 2,345 loaded records'));
assert.ok(capped.notes.includes('the flights layer loads by viewport'));
});
+114
View File
@@ -0,0 +1,114 @@
// The analyst card's empty notes can be a layer's status line: Local ADS-B
// with every decoder feed stale, Transit with a quiet feed ("(95s old)") or
// its getStats().coverage text. The card shows them in the UI language; the
// model gets the English.
import assert from 'node:assert/strict';
import test from 'node:test';
import { setLocale } from '../i18n/index.js';
import { attachVoiceResult } from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
function viewFor(layer, caveat) {
const shaped = attachVoiceResult(
'analyst_query',
{
ok: true,
count: 0,
complete: true,
scopeLabel: 'in view',
feedState: 'nominal',
unanswered: [],
coverage: { layersQueried: [{ ...layer, status: 'empty' }] },
display: { scope: 'in view', caveat },
items: [],
},
{},
);
return voiceCardView(
[
{
type: 'action-call',
callId: 'c1',
name: 'analyst_query',
arguments: {},
},
{
type: 'action-result',
callId: 'c1',
name: 'analyst_query',
result: shaped,
},
].reduce(reduceVoiceCard, initialVoiceCardState()),
);
}
function notesFor(layerKey, caveat) {
return viewFor({ layerKey }, caveat).result.notes;
}
function inLocale(locale, run) {
setLocale(locale);
try {
run();
} finally {
setLocale('en');
}
}
// Caveat = `${layerKey}: ${getEmptyNote(layerKey)}` (data/analystEngine.js;
// gevActions.js getEmptyNote → stats.coverage || stats.statusMessage).
const cases = [
['local-adsb', 'local-adsb: feeds 1090, 978 stale · 0 heard'],
['local-adsb', 'local-adsb: feed 1090 stale · 0 heard'],
['transit', 'transit: TTC 0 (95s old)'],
['transit', 'transit: TTC 0 · GO Transit 0 (95s old)'],
];
// Transit's coverage text when nothing is in view (transit/queries.js).
const transitCoverage = [
'transit: Fly below 3,000 km to a covered region',
'transit: No feed here yet · 12 regions available',
];
for (const locale of ['es', 'ru']) {
for (const [layerKey, caveat] of cases) {
test(`${locale}: analyst note "${caveat}" is translated`, () => {
inLocale(locale, () => {
const notes = notesFor(layerKey, caveat);
assert.ok(notes.length > 0, 'the note is shown');
assert.doesNotMatch(notes.join(' | '), /\b(stale|heard|old)\b/);
});
});
}
test(`${locale}: Transit's empty note is translated on the analyst card`, () => {
inLocale(locale, () => {
for (const caveat of transitCoverage) {
const view = viewFor(
{ layerKey: 'transit', source: 'GTFS-RT' },
caveat,
);
const text = JSON.stringify(view.result);
assert.doesNotMatch(
text,
/Fly below|No feed here yet|regions available/,
text,
);
}
});
});
}
for (const [layerKey, caveat] of cases) {
test(`en: analyst note "${caveat}" keeps its status line`, () => {
setLocale('en');
const line = caveat.slice(caveat.indexOf(': ') + 2);
assert.ok(
notesFor(layerKey, caveat).some((note) => note.endsWith(`: ${line}`)),
);
});
}
+496
View File
@@ -0,0 +1,496 @@
import { after, before, test } from 'node:test';
import assert from 'node:assert/strict';
import { DataLayerManager } from '../data/manager.js';
import { CCTV_FOCUS_RESULT } from '../layers/cctv/index.js';
import { controlCctv, createGevActionRunner } from './gevActions.js';
const originalWindow = globalThis.window;
before(() => {
globalThis.window ??= { clearTimeout, setTimeout, requestIdleCallback: null };
});
after(() => {
if (originalWindow === undefined) delete globalThis.window;
else globalThis.window = originalWindow;
});
function viewerFixture() {
return {
clock: { onTick: { addEventListener: () => () => {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
camera: { moveEnd: { addEventListener() {} } },
};
}
function deferred() {
let resolve;
const promise = new Promise((done) => {
resolve = done;
});
return { promise, resolve };
}
function runnerFixture(t, dataManager, options = {}) {
const runner = createGevActionRunner({
viewer: viewerFixture(),
styleManager: {},
dataManager,
...options,
});
t.after(() => runner.dispose());
return runner;
}
// Only the inert module's lifecycle completion is controlled. The manager's
// intent queue, cancellation, compensation, events, and runner are production.
async function lifecycleFixture(t, targetEnabled) {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const calls = [];
const gates = [];
const pending = [];
let nextGate = null;
let moduleEnabled = false;
const phase = targetEnabled ? 'enable' : 'disable';
const lifecycle = async (name, options = {}) => {
calls.push(name);
if (name === phase && nextGate) {
const gate = nextGate;
nextGate = null;
gate.entered.resolve(options.signal);
await gate.release.promise;
if (gate.outcome instanceof Error) throw gate.outcome;
if (gate.outcome === false) return false;
}
moduleEnabled = name === 'enable';
return true;
};
manager.register({
id: 'cctv',
name: 'CCTV',
source: 'test',
updateInterval: -1,
init: async () => true,
enable: (_viewer, options) => lifecycle('enable', options),
disable: (_viewer, options) => lifecycle('disable', options),
update: async () => true,
getStats: () => ({ count: 0, lastUpdate: 0 }),
});
if (!targetEnabled)
assert.equal(await manager.setEnabled('cctv', true), true);
calls.length = 0;
const events = [];
const unsubscribe = manager.subscribe((event) => events.push(event));
const runner = runnerFixture(t, manager, { viewer });
t.after(async () => {
for (const gate of gates) gate.release.resolve();
await Promise.allSettled(pending);
unsubscribe();
await manager.destroyAll();
});
return {
manager,
events,
calls,
get moduleEnabled() {
return moduleEnabled;
},
gate(outcome = true) {
const gate = { entered: deferred(), release: deferred(), outcome };
gates.push(gate);
nextGate = gate;
return gate;
},
run(action, options = {}) {
const work = runner('control_cctv', { action }, options);
pending.push(work);
return work;
},
};
}
function assertLifecycle(result, manager, enabled, uncertain = false) {
assert.equal(result.action, 'control_cctv');
assert.equal(result.layerId, 'cctv');
assert.equal(result.enabled, enabled);
assert.equal(result.lifecycleState, enabled ? 'enabled' : 'disabled');
assert.equal(result.lifecycleUncertain, uncertain);
assert.deepEqual(manager.getLayerLifecycleState('cctv'), {
enabled,
lifecycleState: result.lifecycleState,
uncertain,
});
}
const terminalTypes = new Set([
'visibility',
'visibility-cancelled',
'visibility-failed',
'visibility-blocked',
]);
for (const action of ['enable', 'disable']) {
const targetEnabled = action === 'enable';
for (const outcome of [
'success',
'false',
'throw',
'caller-abort',
'resource-abort',
'late-abort',
]) {
test(
`CCTV ${action}: ${outcome} preserves the manager outcome`,
{ timeout: 2000 },
async (t) => {
const h = await lifecycleFixture(t, targetEnabled);
const controller = new AbortController();
const failure =
outcome === 'resource-abort'
? Object.assign(new Error('fixture resource ended'), {
name: 'AbortError',
})
: outcome === 'throw'
? new Error('fixture lifecycle failed')
: null;
const gate = h.gate(failure || (outcome === 'false' ? false : true));
let lateAbortObserved = false;
if (outcome === 'late-abort') {
t.after(
h.manager.subscribe((event) => {
if (event.type === 'visibility' && event.origin === 'voice') {
lateAbortObserved = true;
controller.abort();
}
}),
);
}
let settled = false;
// Uppercase exercises the existing case-insensitive action contract.
const work = h.run(
outcome === 'success' ? action.toUpperCase() : action,
{
signal: controller.signal,
isCurrent: () => !controller.signal.aborted,
},
);
void work.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
const moduleSignal = await gate.entered.promise;
assert.equal(
settled,
false,
'the tool must wait for lifecycle completion',
);
if (outcome === 'caller-abort') {
controller.abort();
assert.ok(
moduleSignal,
'the manager must receive a lifecycle signal',
);
assert.equal(
moduleSignal.aborted,
true,
'caller abort must reach the manager lifecycle',
);
}
gate.release.resolve();
const result = await work;
const committed = outcome === 'success' || outcome === 'late-abort';
const cancelled =
outcome === 'caller-abort' || outcome === 'resource-abort';
assert.equal(result.ok, committed);
assert.equal(result.cancelled, cancelled ? true : undefined);
assertLifecycle(
result,
h.manager,
committed ? targetEnabled : !targetEnabled,
!targetEnabled && (outcome === 'false' || outcome === 'throw'),
);
assert.equal(
h.moduleEnabled,
committed ? targetEnabled : !targetEnabled,
'the module must match the committed or compensated manager state',
);
if (cancelled) {
assert.equal(result.cancellationReason, outcome);
assert.equal(result.phase, action);
assert.equal(result.successorIntentEpoch, null);
assert.equal(result.successorEnabled, null);
assert.equal(result.successorOrigin, null);
} else if (!committed) {
assert.equal(result.error, `Could not ${action} the requested layer`);
}
const terminal = h.events.filter(
(event) => event.origin === 'voice' && terminalTypes.has(event.type),
);
assert.equal(
terminal.length,
1,
'each voice intent has one terminal manager event',
);
assert.equal(
terminal[0].type,
committed
? 'visibility'
: cancelled
? 'visibility-cancelled'
: 'visibility-failed',
);
if (outcome === 'resource-abort')
assert.equal(controller.signal.aborted, false);
if (outcome === 'late-abort') assert.equal(lateAbortObserved, true);
},
);
}
test(
`CCTV ${action}: already-aborted caller cannot enter lifecycle work`,
{ timeout: 2000 },
async (t) => {
const h = await lifecycleFixture(t, targetEnabled);
const controller = new AbortController();
controller.abort();
const result = await h.run(action, { signal: controller.signal });
assert.equal(result.ok, false);
assert.equal(result.cancelled, true);
assert.equal(result.cancellationReason, 'caller-abort');
assert.equal(result.phase, 'queued');
assertLifecycle(result, h.manager, !targetEnabled);
assert.equal(h.moduleEnabled, !targetEnabled);
assert.deepEqual(h.calls, []);
assert.equal(
h.events.filter((event) => terminalTypes.has(event.type)).length,
1,
);
},
);
}
test(
'CCTV toggles retain missing-module errors without calling visibility',
{ timeout: 2000 },
async (t) => {
for (const layers of [new Map(), new Map([['cctv', {}]])]) {
const runner = runnerFixture(t, {
layers,
getAll: () => [],
setEnabled: () =>
assert.fail('missing CCTV must not dispatch visibility'),
});
for (const action of ['enable', 'disable']) {
assert.deepEqual(await runner('control_cctv', { action }), {
ok: false,
action: 'control_cctv',
error: 'CCTV layer unavailable',
});
}
}
},
);
test('exported controlCctv refuses toggles without calling the manager', async () => {
const manager = {
layers: new Map([['cctv', { module: {} }]]),
setEnabled: () => assert.fail('direct helper must not change visibility'),
isEnabled: () => assert.fail('direct toggle must not read enabled state'),
};
for (const action of ['ENABLE', 'disable']) {
const result = await controlCctv(manager, { action });
assert.equal(result.ok, false);
assert.equal(result.action, 'control_cctv');
assert.match(result.error, /set_layer_visibility/);
}
});
test(
'CCTV fallback managers report rejection, transitional state, and uncertainty',
{ timeout: 2000 },
async (t) => {
for (const enabled of [true, false]) {
const action = enabled ? 'enable' : 'disable';
for (const mode of [
'rejection',
'mismatch',
'transitional',
'uncertain',
]) {
const actualEnabled =
mode === 'rejection' || mode === 'mismatch' ? !enabled : enabled;
const lifecycleState =
mode === 'transitional'
? enabled
? 'enabling'
: 'disabling'
: actualEnabled
? 'enabled'
: 'disabled';
const manager = {
layers: new Map([['cctv', { module: {} }]]),
getAll: () => [{ id: 'cctv', name: 'CCTV' }],
isEnabled: () => actualEnabled,
getLayerLifecycleState: () => ({
enabled: actualEnabled,
lifecycleState,
uncertain: mode === 'uncertain',
}),
async setEnabled() {
if (mode === 'rejection') throw new Error('adapter rejected');
return true;
},
};
const runner = runnerFixture(t, manager);
const result = await runner('control_cctv', { action });
assert.equal(result.ok, false, `${action}: ${mode}`);
assert.equal(result.action, 'control_cctv');
assert.equal(result.enabled, actualEnabled);
assert.equal(result.lifecycleState, lifecycleState);
assert.equal(result.lifecycleUncertain, mode === 'uncertain');
assert.equal(
result.error,
mode === 'rejection'
? 'adapter rejected'
: `Could not ${action} the requested layer`,
);
}
}
},
);
test(
'CCTV visibility composes only its outer speech envelope',
{ timeout: 2000 },
async (t) => {
let enabled = false;
const calls = [];
const runner = runnerFixture(
t,
{
layers: new Map([['cctv', { module: {} }]]),
getAll: () => [{ id: 'cctv', name: 'CCTV' }],
isEnabled: () => enabled,
async setEnabled(_id, value) {
enabled = value;
return true;
},
},
{
speechBuilders: {
control_cctv(result, args) {
calls.push(['cctv', result.action, args.action]);
return {
say: 'CCTV response',
display: { title: 'CCTV', lines: [] },
};
},
set_layer_visibility() {
calls.push(['generic']);
return { say: 'Layer response' };
},
},
},
);
const result = await runner('control_cctv', { action: 'enable' });
assert.equal(result.ok, true);
assert.equal(result.action, 'control_cctv');
assert.equal(result.say, 'CCTV response');
assert.deepEqual(result.display, { title: 'CCTV', lines: [] });
assert.deepEqual(calls, [['cctv', 'control_cctv', 'enable']]);
},
);
test(
'CCTV non-toggle actions retain camera navigation and durable coverage',
{ timeout: 2000 },
async (t) => {
const calls = [];
const cameras = [
{ id: 'a', name: 'Camera A' },
{ id: 'b', name: 'Camera B' },
];
let activeCameraId = 'a';
let coverageMode = 'viewshed';
const module = {
getUIState: () => ({
cameras,
activeCameraId,
activeCamera: cameras.find((camera) => camera.id === activeCameraId),
coverageMode,
showCoverage: coverageMode !== 'off',
}),
selectCamera(id) {
calls.push(['select', id]);
activeCameraId = id;
return true;
},
cycleCamera(step) {
calls.push(['cycle', step]);
activeCameraId = step > 0 ? 'b' : 'a';
return activeCameraId;
},
focusNearest(options) {
calls.push(['nearest', options]);
activeCameraId = 'a';
return 'a';
},
focusCamera(id, duration) {
calls.push(['focus', id, duration]);
return CCTV_FOCUS_RESULT.TRACKING_HOLDS_VIEW;
},
};
let superseded = 0;
const runner = runnerFixture(
t,
{
layers: new Map([['cctv', { module }]]),
isEnabled: () => true,
setEnabled: () =>
assert.fail('non-toggle action must not dispatch visibility'),
setLayerParams(id, params, options) {
calls.push(['params', id, params, options]);
coverageMode = params.coverageMode;
},
},
{
styleManager: {
supersedeDeferredNavigation() {
superseded += 1;
},
},
},
);
for (const args of [
{ action: 'select', cameraQuery: 'Camera B' },
{ action: 'next' },
{ action: 'prev' },
{ action: 'nearest' },
{ action: 'focus' },
]) {
const result = await runner('control_cctv', args);
assert.equal(result.ok, false);
assert.equal(result.action, 'control_cctv');
assert.match(result.error, /tracking holds the view/);
}
const coverage = await runner('control_cctv', { action: 'coverage' });
assert.equal(coverage.ok, true);
assert.equal(coverage.coverageMode, 'off');
assert.equal(superseded, 5, 'coverage must not acquire camera navigation');
assert.deepEqual(calls, [
['select', 'b'],
['focus', 'b', 1.8],
['cycle', 1],
['focus', 'b', 1.8],
['cycle', -1],
['focus', 'a', 1.8],
['nearest', { focus: false }],
['focus', 'a', 1.8],
['focus', 'a', undefined],
['params', 'cctv', { coverageMode: 'off' }, { origin: 'voice' }],
]);
},
);
+3 -3
View File
@@ -1,5 +1,5 @@
export const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']);
export const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key';
const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']);
const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key';
export const CLOUD_VOICE_AUTH_STORAGE_KEY = 'godsEyeView.voice.cloudAuth';
function storageRef(storage) {
@@ -11,7 +11,7 @@ function storageRef(storage) {
}
}
export function normalizeCloudVoiceAuthMode(mode) {
function normalizeCloudVoiceAuthMode(mode) {
const normalized = String(mode || '')
.trim()
.toLowerCase();
+46 -5
View File
@@ -1,10 +1,51 @@
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
import { createRealtimeSession } from './realtimeSession.js';
import { createGeminiSession } from './geminiSession.js';
import { createProviderCommands } from './providerCommands.js';
import { createVoicePanelHost } from './panel/host.js';
/** Default composition; callers may supply another session adapter factory. */
export function createVoiceCommands(options) {
return bindVoiceCommands({
createSession: createRealtimeSession,
...options,
});
export function createVoiceCommands(options = {}) {
const panelHost =
options.panel === true
? createVoicePanelHost({
storage: options.panelStorage,
clock: options.panelClock,
readCockpitState: options.styleManager?.getCockpitState
? () => options.styleManager.getCockpitState()
: undefined,
})
: null;
const bind = (settings) =>
bindVoiceCommands({
...settings,
...(panelHost
? {
panelHost,
nativePanelProvider: settings.createController
? null
: settings.createSession === createRealtimeSession
? 'openai'
: settings.createSession === createGeminiSession
? 'gemini'
: null,
}
: {}),
});
let commands;
// Generic hosts retain the existing default; standalone opts in explicitly.
if (!options.createSession && options.supportedProviders !== undefined)
commands = createProviderCommands(options, {
bind,
factories: { openai: createRealtimeSession, gemini: createGeminiSession },
});
else commands = bind({ createSession: createRealtimeSession, ...options });
if (panelHost) {
const lifetime = commands.signal ?? commands.session?.signal;
lifetime?.addEventListener('abort', () => panelHost.destroy(), {
once: true,
});
if (lifetime?.aborted) panelHost.destroy();
}
return commands;
}
+130 -26
View File
@@ -1,5 +1,33 @@
import { t } from '../i18n/index.js';
// The assistant's name on its card. i18n-ignore: product name
const ASSISTANT_NAME = 'GEV';
/**
* Short state label for the narrow voice heading. The `voice` context lets
* languages use shorter words; unknown states show upper-cased.
* @param {string} status machine state (dataset.status)
* @returns {string}
*/
export function voiceStatusLabel(status) {
switch (status) {
case 'idle':
return t('OFF', { context: 'voice' });
case 'connecting':
return t('CONNECTING', { context: 'voice' });
case 'listening':
return t('LISTENING', { context: 'voice' });
case 'executing':
return t('EXECUTING', { context: 'voice' });
case 'error':
return t('ERROR', { context: 'voice' });
default:
return String(status || '').toUpperCase();
}
}
/** Build the voice control independently of its connection backend. */
export function createVoiceControl({ reset = false } = {}) {
export function createVoiceControl({ reset = false, panel = false } = {}) {
let root = document.getElementById('gev-voice-control');
if (root && reset) {
root.remove();
@@ -11,57 +39,116 @@ export function createVoiceControl({ reset = false } = {}) {
root.dataset.status = 'idle';
root.dataset.speaker = 'idle';
root.innerHTML = `
<label id="gev-voice-provider-field" class="gev-voice-provider-field" hidden>
<span>${t('Voice provider')}</span>
<select id="gev-voice-provider" aria-label="${t('Voice provider')}" title="${t('Changing provider reconnects voice if it is on.')}"></select>
</label>
${panel ? `<button id="gev-voice-settings-button" class="gev-voice-settings-button" type="button" aria-label="${t('Voice Settings')}" aria-controls="gev-voice-panel" aria-expanded="false"><span class="material-symbols-outlined gev-voice-settings-icon" aria-hidden="true">settings</span></button>` : ''}
<div class="gev-voice-heading">
<div class="gev-voice-kicker">AI AGENT</div>
<div id="gev-voice-status">OFF</div>
<div class="gev-voice-cost">
<button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="Voice model tier — applies next session">STD</button>
<span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="Estimated session cost">~$0.00</span>
</div>
<div class="gev-voice-kicker">${t('AI AGENT')}</div>
<div id="gev-voice-status">${voiceStatusLabel('idle')}</div>
${
panel
? `<span id="gev-voice-cost-warning" class="gev-voice-cost-value gev-voice-cost-warning" hidden role="status" aria-live="polite"></span>`
: `<div class="gev-voice-cost">
<button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="${t('Voice model tier — applies next session')}">${t('STD')}</button>
<span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="${t('Estimated session cost')}">~$0.00</span>
</div>`
}
</div>
<button id="gev-voice-button" type="button" aria-label="Voice control — activate to toggle voice; hold Space to speak" aria-describedby="gev-voice-help">
<button id="gev-voice-button" type="button" aria-label="${t('Voice control — activate to toggle voice; hold Space to speak')}" aria-describedby="gev-voice-help">
<span class="gev-mic-orbit"><img src="/mic.svg" alt="" /></span>
<span class="gev-mic-label">ON/OFF</span>
<span class="gev-mic-label">${t('ON/OFF')}</span>
</button>
<div class="gev-voice-visualizer" aria-hidden="true">
${Array.from({ length: 15 }, (_, index) => `<span style="--bar:${index}"></span>`).join('')}
</div>
<div class="gev-voice-readout">
<div id="gev-voice-detail">VOICE STANDBY</div>
<div id="gev-voice-detail">${t('VOICE STANDBY')}</div>
</div>
<div id="gev-voice-help" class="gev-voice-help-tray" role="tooltip">
<span class="gev-voice-help-kicker">VOICE CONTROL</span>
<span class="gev-voice-help-detail">Hold Space to speak · tap Space to activate focused controls</span>
<span class="gev-voice-help-kicker">${t('VOICE CONTROL')}</span>
<span class="gev-voice-help-detail">${t('Hold Space to speak · tap Space to activate focused controls')}</span>
</div>
<section id="gev-voice-card" class="gev-voice-card" hidden aria-label="Voice assistant">
<div class="gev-voice-card-header">
<span class="gev-voice-card-kicker">VOICE</span>
<span id="gev-voice-card-phase" class="gev-voice-card-phase"></span>
<button id="gev-voice-card-dismiss" class="gev-voice-card-dismiss" type="button" aria-label="Dismiss voice card">×</button>
${
panel
? `<section id="gev-voice-panel" class="gev-voice-panel" hidden aria-label="${t('Voice Settings')}">
<div class="gev-voice-panel-header">
<span class="gev-voice-panel-title">${t('VOICE')}</span>
<div class="gev-voice-panel-tabs" role="tablist" aria-label="${t('Voice Settings')}">
<button id="gev-voice-preview-tab" class="gev-voice-panel-tab" type="button" role="tab" aria-label="${t('Preview')}" title="${t('Preview')}" aria-selected="true" aria-controls="gev-voice-preview-panel"><span class="material-symbols-outlined gev-voice-panel-tab-icon" aria-hidden="true">visibility</span></button>
<button id="gev-voice-settings-tab" class="gev-voice-panel-tab" type="button" role="tab" aria-label="${t('Settings')}" title="${t('Settings')}" aria-selected="false" tabindex="-1" aria-controls="gev-voice-settings-panel"><span class="material-symbols-outlined gev-voice-panel-tab-icon" aria-hidden="true">settings</span></button>
</div>
<button id="gev-voice-settings-close" type="button" aria-label="${t('Close voice panel')}">×</button>
</div>
<p id="gev-voice-card-user" class="gev-voice-card-caption" data-role="user" hidden><span class="gev-voice-card-who">YOU</span><span id="gev-voice-card-user-text" class="gev-voice-card-text"></span></p>
<p id="gev-voice-card-pointer" class="gev-voice-card-pointer" hidden><span id="gev-voice-card-pointer-kind" class="gev-voice-card-who">THIS</span><span class="gev-voice-card-sr">: </span><span id="gev-voice-card-pointer-text" class="gev-voice-card-pointer-label"></span></p>
<p id="gev-voice-card-assistant" class="gev-voice-card-caption" data-role="assistant" hidden><span class="gev-voice-card-who">GEV</span><span id="gev-voice-card-assistant-text" class="gev-voice-card-text"></span></p>
<ol id="gev-voice-card-plan" class="gev-voice-card-plan" aria-label="Plan" hidden></ol>
<div class="gev-voice-panel-body">
<section id="gev-voice-preview-panel" role="tabpanel" aria-labelledby="gev-voice-preview-tab">
<p id="gev-voice-preview-empty">${t('Voice replies appear here.')}</p>`
: ''
}
<section id="gev-voice-card" class="gev-voice-card" hidden aria-label="${t('Voice assistant')}">
<div class="gev-voice-card-header">
<span class="gev-voice-card-kicker">${t('VOICE')}</span>
<span id="gev-voice-card-phase" class="gev-voice-card-phase"></span>
<button id="gev-voice-card-dismiss" class="gev-voice-card-dismiss" type="button" aria-label="${t('Dismiss voice card')}">×</button>
</div>
<p id="gev-voice-card-user" class="gev-voice-card-caption" data-role="user" hidden><span id="gev-voice-card-user-who" class="gev-voice-card-who">${t('YOU')}</span><span id="gev-voice-card-user-text" class="gev-voice-card-text"></span></p>
<p id="gev-voice-card-pointer" class="gev-voice-card-pointer" hidden><span id="gev-voice-card-pointer-kind" class="gev-voice-card-who">${t('THIS')}</span><span class="gev-voice-card-sr">: </span><span id="gev-voice-card-pointer-text" class="gev-voice-card-pointer-label"></span></p>
<p id="gev-voice-card-assistant" class="gev-voice-card-caption" data-role="assistant" hidden><span class="gev-voice-card-who">${ASSISTANT_NAME}</span><span id="gev-voice-card-assistant-text" class="gev-voice-card-text"></span></p>
<ol id="gev-voice-card-plan" class="gev-voice-card-plan" aria-label="${t('Plan')}" hidden></ol>
<div id="gev-voice-card-result" class="gev-voice-card-result" hidden>
<div id="gev-voice-card-result-title" class="gev-voice-card-result-title"></div>
<div id="gev-voice-card-chips" class="gev-voice-card-chips" hidden></div>
<ul id="gev-voice-card-lines" class="gev-voice-card-lines"></ul>
<ol id="gev-voice-card-referents" class="gev-voice-card-referents" aria-label="Referenced items" hidden></ol>
<ol id="gev-voice-card-referents" class="gev-voice-card-referents" aria-label="${t('Referenced items')}" hidden></ol>
<details id="gev-voice-card-notes" class="gev-voice-card-notes" hidden>
<summary>NOTES &amp; SOURCES</summary>
<summary>${t('NOTES & SOURCES')}</summary>
<ul id="gev-voice-card-note-list" class="gev-voice-card-lines"></ul>
</details>
</div>
<div id="gev-voice-card-live" class="gev-voice-card-sr" aria-live="polite" aria-atomic="true"></div>
</section>
${
panel
? `</section>
<section id="gev-voice-settings-panel" role="tabpanel" aria-labelledby="gev-voice-settings-tab" hidden>
<label class="gev-voice-preference gev-voice-preference-toggle"><input id="gev-voice-preview-enabled" type="checkbox" checked aria-describedby="gev-voice-preview-note" /><span>${t('Show Voice Preview')}</span></label>
<p id="gev-voice-preview-note" class="gev-voice-settings-note"></p>
<div class="gev-voice-timeout-group">
<label class="gev-voice-preference gev-voice-setting-title" for="gev-voice-timeout-select">${t('Voice inactivity timeout')}</label>
<div class="gev-voice-timeout-controls">
<select id="gev-voice-timeout-select" aria-describedby="gev-voice-timeout-note gev-voice-timeout-status">
<option value="3">3 ${t('Minutes')}</option><option value="5" selected>5 ${t('Minutes')}</option><option value="10">10 ${t('Minutes')}</option><option value="15">15 ${t('Minutes')}</option>
<option value="custom">${t('Custom')}</option><option value="0">${t('None')}</option>
</select>
<label class="gev-voice-timeout-unit" for="gev-voice-timeout-input">${t('Minutes')}</label>
<input id="gev-voice-timeout-input" type="number" min="1" max="60" step="1" value="5" aria-describedby="gev-voice-timeout-note" />
</div>
<p id="gev-voice-timeout-note" class="gev-voice-settings-note">${t('These preferences stay in this browser.')}</p>
<p id="gev-voice-timeout-status" class="gev-voice-settings-note" role="status" aria-live="polite"></p>
</div>
<div class="gev-voice-cost-settings">
<div class="gev-voice-cost-setting-row">
<span>${t('Voice model tier — applies next session')}</span>
<button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="${t('Voice model tier — applies next session')}">${t('STD')}</button>
</div>
<div class="gev-voice-cost-setting-row">
<span>${t('Estimated session cost')}</span>
<span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="${t('Estimated session cost')}">~$0.00</span>
</div>
</div>
</section>
</div>
</section>`
: ''
}
<div class="gev-voice-error-tray" role="alert" aria-live="assertive">
<div class="gev-voice-error-header">
<span>VOICE SYSTEM ERROR</span>
<button class="gev-voice-error-dismiss" type="button">DISMISS</button>
<span>${t('VOICE SYSTEM ERROR')}</span>
<button class="gev-voice-error-dismiss" type="button">${t('DISMISS')}</button>
</div>
<div id="gev-voice-error-detail"></div>
<div class="gev-voice-error-hint">Check microphone permission and network access, then try again.</div>
<div class="gev-voice-error-hint">${t('Check microphone permission and network access, then try again.')}</div>
</div>
`;
const commandDock = document.getElementById('command-dock');
@@ -82,6 +169,22 @@ export function createVoiceControl({ reset = false } = {}) {
}
return {
root,
voicePanel: root.querySelector('#gev-voice-panel'),
previewPanel: root.querySelector('#gev-voice-preview-panel'),
voiceSettingsPanel: root.querySelector('#gev-voice-settings-panel'),
previewTab: root.querySelector('#gev-voice-preview-tab'),
settingsTab: root.querySelector('#gev-voice-settings-tab'),
voiceSettingsButton: root.querySelector('#gev-voice-settings-button'),
voiceSettingsClose: root.querySelector('#gev-voice-settings-close'),
previewEmpty: root.querySelector('#gev-voice-preview-empty'),
voicePreviewInput: root.querySelector('#gev-voice-preview-enabled'),
voicePreviewNote: root.querySelector('#gev-voice-preview-note'),
voiceTimeoutSelect: root.querySelector('#gev-voice-timeout-select'),
voiceTimeoutInput: root.querySelector('#gev-voice-timeout-input'),
voiceTimeoutNote: root.querySelector('#gev-voice-timeout-note'),
voiceTimeoutStatus: root.querySelector('#gev-voice-timeout-status'),
providerField: root.querySelector('#gev-voice-provider-field'),
providerSelect: root.querySelector('#gev-voice-provider'),
button: root.querySelector('#gev-voice-button'),
buttonLabel: root.querySelector('.gev-mic-label'),
status: root.querySelector('#gev-voice-status'),
@@ -90,6 +193,7 @@ export function createVoiceControl({ reset = false } = {}) {
errorDetail: root.querySelector('#gev-voice-error-detail'),
tierButton: root.querySelector('#gev-voice-tier'),
costValue: root.querySelector('#gev-voice-cost-value'),
costWarning: root.querySelector('#gev-voice-cost-warning'),
card: root.querySelector('#gev-voice-card'),
};
}
+1 -1
View File
@@ -18,7 +18,7 @@
import { finiteOrNull } from './pointerContext.js';
export const POINTER_SENTINEL = 'pointer';
const POINTER_SENTINEL = 'pointer';
const TRACKABLE_LAYERS = new Set([
'flights',
+142
View File
@@ -0,0 +1,142 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
deferred,
tick,
} from '../testSupport/geminiNativeHarness.mjs';
function unreadResponse(bytes = null) {
const state = { reads: 0, cancelled: 0 };
const response = new Response(
new ReadableStream(
{
pull(controller) {
state.reads++;
if (bytes) controller.enqueue(bytes);
},
cancel() {
state.cancelled++;
},
},
{ highWaterMark: 0 },
),
);
return { response, state };
}
test('already aborted token request does not acquire or read a response', async (t) => {
let fetches = 0;
const h = harness({
fetchImpl: () => {
fetches++;
throw Error('must not fetch');
},
});
t.after(() => h.c.stop());
const abort = new AbortController();
abort.abort();
await assert.rejects(h.c.requestToken(abort.signal), { name: 'AbortError' });
assert.equal(fetches, 0);
});
for (const phase of [
'before response',
'between resolution and continuation',
]) {
test(`aborted token acquisition closes the unread body ${phase}`, async (t) => {
const gate = deferred();
const h = harness({ fetchImpl: () => gate.promise });
t.after(() => h.c.stop());
const abort = new AbortController();
const { response, state } = unreadResponse();
const result = assert.rejects(h.c.requestToken(abort.signal), {
name: 'AbortError',
});
if (phase === 'before response') abort.abort();
gate.resolve(response);
if (phase !== 'before response') queueMicrotask(() => abort.abort());
await result;
await tick();
assert.deepEqual(state, { reads: 0, cancelled: 1 });
});
}
test('token body cap cancels a native response stream', async (t) => {
const { response, state } = unreadResponse(new Uint8Array(256 * 1024 + 1));
const h = harness({ fetchImpl: async () => response });
t.after(() => h.c.stop());
await assert.rejects(
h.c.requestToken(new AbortController().signal),
/too large|exceed|limit/i,
);
assert.equal(state.cancelled, 1);
assert.equal(state.reads, 1);
});
test('token body read ends promptly on caller abort and releases the reader', async (t) => {
const { response, state } = unreadResponse();
const h = harness({ fetchImpl: async () => response });
t.after(() => h.c.stop());
const abort = new AbortController();
const result = assert.rejects(h.c.requestToken(abort.signal), {
name: 'AbortError',
});
await tick();
assert.equal(state.reads, 1);
abort.abort();
await result;
assert.equal(state.cancelled, 1);
assert.equal(response.body.locked, false);
});
test('permission grant resolved just before Stop releases its tracks and preserves a fresh start', async (t) => {
const gate = deferred();
const h = harness({ getUserMedia: () => gate.promise });
t.after(() => h.c.stop());
const stream = h.makeStream();
const start = h.c.start();
gate.resolve(stream);
queueMicrotask(() => h.c.stop());
await start;
await tick();
assert.equal(h.tracks[0].readyState, 'ended');
assert.equal(h.requests.length, 0);
assert.equal(h.sockets.length, 0);
assert.equal(h.captures[0].closed, true);
h.c.getUserMedia = async () => h.makeStream();
await h.c.start();
assert.equal(h.c.status, 'listening');
assert.equal(h.tracks[1].readyState, 'live');
assert.equal(h.captures[1].closed, false);
});
test('late audio initialization cannot connect a retired microphone or close successor audio', async (t) => {
const gate = deferred();
const h = harness();
t.after(() => h.c.stop());
const createAudio = h.c.createAudio;
let first = true;
h.c.createAudio = (options) => {
const audio = createAudio(options);
if (first) {
first = false;
audio.initialize = () => gate.promise;
audio.connectMicrophone = () =>
assert.fail('retired initialization must not connect');
}
return audio;
};
const oldStart = h.c.start();
await tick();
h.c.stop();
await h.c.start();
gate.resolve();
await oldStart;
await tick();
assert.equal(h.c.status, 'listening');
assert.equal(h.captures[0].closed, true);
assert.equal(h.captures[1].closed, false);
assert.equal(h.tracks[0].readyState, 'ended');
assert.equal(h.tracks[1].readyState, 'live');
});
+380
View File
@@ -0,0 +1,380 @@
import { msg } from '../i18n/marker.js';
const MAX_AUDIO_BYTES = 2 * 1024 * 1024;
const MAX_QUEUED_SECONDS = 60;
const MAX_QUEUED_SOURCES = 256;
const PTT_WINDOW_SAMPLES = 160; // 10 ms at the worklet's fixed 16 kHz.
const PTT_ENERGY_FLOOR = 0.002;
const PTT_PRE_ROLL_BYTES = 16000 * 2 * 0.4;
/** Admit manual input only after sustained energy; not an utterance detector. */
function pttCapture(captureId) {
return {
captureId,
admitted: false,
buffers: [],
bytes: 0,
samples: 0,
energy: 0,
aboveFloor: 0,
voicedWindows: 0,
};
}
function hasPttEnergy(state, pcm) {
for (let index = 0; index + 1 < pcm.byteLength; index += 2) {
const sample = pcm.getInt16(index, true) / 32768;
state.energy += sample * sample;
if (Math.abs(sample) >= PTT_ENERGY_FLOOR) state.aboveFloor++;
if (++state.samples !== PTT_WINDOW_SAMPLES) continue;
// Requiring occupancy as well as RMS rejects isolated high-peak key clicks.
const voiced =
state.energy / state.samples >= PTT_ENERGY_FLOOR ** 2 &&
state.aboveFloor >= PTT_WINDOW_SAMPLES / 8;
state.voicedWindows = voiced ? state.voicedWindows + 1 : 0;
state.samples = state.energy = state.aboveFloor = 0;
if (state.voicedWindows >= 2) return true;
}
return false;
}
export function pcmBytesToBase64(buffer) {
const bytes = new Uint8Array(buffer);
let binary = '';
for (let offset = 0; offset < bytes.length; offset += 8192) {
binary += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
}
return btoa(binary);
}
export function decodeGeminiPcm(data, mimeType = 'audio/pcm;rate=24000') {
if (
typeof data !== 'string' ||
data.length > 4 * Math.ceil(MAX_AUDIO_BYTES / 3)
)
throw new Error(msg('Gemini returned an oversized audio chunk'));
const match = /^audio\/pcm(?:;rate=(\d+))?$/.exec(mimeType);
if (!match)
throw new Error(msg('Gemini returned an unsupported audio format'));
const sampleRate = Number(match[1] || 24000);
if (!Number.isInteger(sampleRate) || sampleRate < 8000 || sampleRate > 96000)
throw new Error(msg('Gemini returned an invalid audio sample rate'));
const binary = atob(data);
if (binary.length > MAX_AUDIO_BYTES)
throw new Error(msg('Gemini returned an oversized audio chunk'));
if (!binary.length || binary.length % 2)
throw new Error(msg('Gemini returned incomplete PCM audio'));
const bytes = new Uint8Array(binary.length);
for (let index = 0; index < binary.length; index++)
bytes[index] = binary.charCodeAt(index);
const view = new DataView(bytes.buffer);
const samples = new Float32Array(bytes.length / 2);
for (let index = 0; index < samples.length; index++)
samples[index] = view.getInt16(index * 2, true) / 32768;
return { samples, sampleRate };
}
/** Own capture and playback independently from the Live protocol. */
export function createGeminiAudio({
pushToTalk = false,
onAudioStart,
onAudio,
onAudioEnd,
onAudioDiscard,
onDrain,
onError,
AudioContextClass = globalThis.AudioContext || globalThis.webkitAudioContext,
AudioWorkletNodeClass = globalThis.AudioWorkletNode,
} = {}) {
if (!AudioContextClass || !AudioWorkletNodeClass)
throw new Error(
msg('This browser does not support Gemini microphone audio'),
);
let captureContext;
let playbackContext;
let output;
let meter;
let resumed;
const closeContext = (context) => {
try {
return Promise.resolve(context?.close());
} catch (error) {
return Promise.reject(error);
}
};
try {
// Capture at the device rate; the worklet explicitly resamples to 16 kHz.
captureContext = new AudioContextClass();
playbackContext = new AudioContextClass({ sampleRate: 24000 });
// Resume both synchronously within the mic gesture, before any awaits.
const resume = (context) => {
try {
return Promise.resolve(context.resume());
} catch (error) {
return Promise.reject(error);
}
};
resumed = Promise.all([resume(captureContext), resume(playbackContext)]);
// Permission may remain pending before initialize observes this failure.
void resumed.catch(() => {});
output = playbackContext.createGain();
meter = playbackContext.createMediaStreamDestination();
output.connect(playbackContext.destination);
output.connect(meter);
} catch (error) {
output?.disconnect();
meter?.disconnect();
meter?.stream.getTracks?.().forEach((track) => track.stop());
void Promise.allSettled([
closeContext(captureContext),
closeContext(playbackContext),
]);
throw error;
}
let closed = false;
let closeResult;
let capture = null;
let microphone = null;
let mute = null;
let nextTime = 0;
let inputChunks = 0;
let inputPeak = 0;
let nonSilentInputChunks = 0;
let outputChunks = 0;
let outputSamples = 0;
let completedOutputSamples = 0;
let discardedOutputSamples = 0;
let pendingPtt = null;
let latestPttCaptureId = 0;
let suppressedPttCaptures = 0;
let admittedPttCaptures = 0;
const sources = new Map();
const emitAudio = (buffer, metadata) =>
onAudio?.(
{ data: pcmBytesToBase64(buffer), mimeType: 'audio/pcm;rate=16000' },
metadata,
);
function clearPlayback() {
const oldSources = [...sources];
sources.clear();
nextTime = playbackContext.currentTime;
for (const [source, sampleCount] of oldSources) {
discardedOutputSamples += sampleCount;
source.onended = null;
try {
source.stop();
} catch {
/* Already ended. */
}
source.disconnect();
}
}
return {
outputStream: meter.stream,
get pending() {
return sources.size > 0;
},
async initialize(signal) {
try {
await Promise.all([
resumed,
captureContext.audioWorklet.addModule(
new URL('./geminiCaptureWorklet.js', import.meta.url),
),
]);
signal?.throwIfAborted();
if (closed) throw new DOMException('Voice stopped', 'AbortError');
} catch (error) {
await this.close();
throw error;
}
},
connectMicrophone(stream, enabled, metadata) {
if (closed) throw new DOMException('Voice stopped', 'AbortError');
capture = new AudioWorkletNodeClass(
captureContext,
'gev-gemini-capture',
{
numberOfInputs: 1,
numberOfOutputs: 1,
outputChannelCount: [1],
},
);
capture.port.onmessage = ({ data }) => {
if (closed) return;
// Capture ownership is transport metadata, never part of Live audio.
if (!Number.isSafeInteger(data?.captureId) || data.captureId <= 0)
return;
const metadata = { captureId: data.captureId };
try {
if (data?.type === 'audio' && data.buffer instanceof ArrayBuffer) {
inputChunks++;
const pcm = new DataView(data.buffer);
let peak = 0;
for (let index = 0; index + 1 < pcm.byteLength; index += 2)
peak = Math.max(
peak,
Math.abs(pcm.getInt16(index, true)) / 32768,
);
inputPeak = Math.max(inputPeak, peak);
if (peak >= 0.01) nonSilentInputChunks++;
if (!pushToTalk) emitAudio(data.buffer, metadata);
else {
const state = pendingPtt;
if (state?.captureId !== data.captureId) return;
if (state.admitted) emitAudio(data.buffer, metadata);
else {
state.buffers.push(data.buffer);
state.bytes += data.buffer.byteLength;
if (hasPttEnergy(state, pcm)) {
state.admitted = true;
admittedPttCaptures++;
// Context and activityStart are gated together with PCM.
onAudioStart?.(metadata);
for (const buffer of state.buffers) {
if (closed || pendingPtt !== state) break;
emitAudio(buffer, metadata);
}
state.buffers = [];
state.bytes = 0;
} else {
while (state.bytes > PTT_PRE_ROLL_BYTES) {
state.bytes -= state.buffers.shift().byteLength;
}
}
}
}
} else if (data?.type === 'start') {
if (!pushToTalk) onAudioStart?.(metadata);
else if (data.captureId > latestPttCaptureId) {
latestPttCaptureId = data.captureId;
pendingPtt = pttCapture(data.captureId);
}
} else if (data?.type === 'end') {
if (!pushToTalk) onAudioEnd?.(metadata);
else if (pendingPtt?.captureId === data.captureId) {
const admitted = pendingPtt.admitted;
pendingPtt = null;
if (admitted) onAudioEnd?.(metadata);
else {
suppressedPttCaptures++;
onAudioDiscard?.(metadata);
}
}
}
} catch (error) {
onError?.(error);
}
};
capture.onprocessorerror = () => {
if (!closed)
onError?.(new Error(msg('Microphone audio processing failed')));
};
microphone = captureContext.createMediaStreamSource(stream);
mute = captureContext.createGain();
mute.gain.value = 0;
capture.port.postMessage({
enabled: Boolean(enabled),
captureId: metadata?.captureId,
});
microphone.connect(capture);
capture.connect(mute);
mute.connect(captureContext.destination);
},
setMicrophoneEnabled(enabled, metadata) {
if (closed) return;
capture?.port.postMessage({
enabled: Boolean(enabled),
captureId: metadata?.captureId,
});
},
play(data, mimeType) {
if (closed) return;
// Tiny chunks can exhaust source nodes long before the duration limit.
if (sources.size >= MAX_QUEUED_SOURCES)
throw new Error(
msg('Gemini audio playback fell behind. Reconnect to continue.'),
);
const { samples, sampleRate } = decodeGeminiPcm(data, mimeType);
const start = Math.max(playbackContext.currentTime + 0.02, nextTime);
if (
start + samples.length / sampleRate - playbackContext.currentTime >
MAX_QUEUED_SECONDS
)
throw new Error(
msg('Gemini audio playback fell behind. Reconnect to continue.'),
);
const buffer = playbackContext.createBuffer(
1,
samples.length,
sampleRate,
);
buffer.copyToChannel(samples, 0);
const source = playbackContext.createBufferSource();
source.buffer = buffer;
source.connect(output);
sources.set(source, samples.length);
source.onended = () => {
if (!sources.delete(source)) return;
completedOutputSamples += samples.length;
source.disconnect();
if (!closed && !sources.size) onDrain?.();
};
nextTime = start + buffer.duration;
outputChunks++;
outputSamples += samples.length;
source.start(start);
},
clearPlayback,
diagnostics() {
return {
inputChunks,
inputPeak,
nonSilentInputChunks,
outputChunks,
outputSamples,
completedOutputSamples,
discardedOutputSamples,
queuedSeconds: Math.max(0, nextTime - playbackContext.currentTime),
contextState:
captureContext.state === playbackContext.state
? captureContext.state
: 'suspended',
captureContextState: captureContext.state,
playbackContextState: playbackContext.state,
captureSampleRate: captureContext.sampleRate,
playbackSampleRate: playbackContext.sampleRate,
inputSampleRate: 16000,
suppressedPttCaptures,
admittedPttCaptures,
pttBufferedBytes: pendingPtt?.bytes || 0,
};
},
close() {
if (closed) return closeResult;
closed = true;
pendingPtt = null;
clearPlayback();
if (capture) {
capture.port.onmessage = null;
capture.onprocessorerror = null;
capture.port.close();
capture.disconnect();
}
microphone?.disconnect();
mute?.disconnect();
output.disconnect();
meter.disconnect();
meter.stream.getTracks?.().forEach((track) => track.stop());
closeResult = Promise.allSettled([
closeContext(captureContext),
closeContext(playbackContext),
]).then((results) =>
results.every((result) => result.status === 'fulfilled'),
);
return closeResult;
},
};
}
File diff suppressed because it is too large. Load diff
+791
View File
@@ -0,0 +1,791 @@
// Native automatic activity, reply ownership and expiry lifecycle regressions.
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
const vad = (ws, type) => ws.message({ voiceActivity: { type } });
const onset = (ws) => {
vad(ws, 'ACTIVITY_START');
vad(ws, 'ACTIVITY_END');
};
const split = (ws) => {
onset(ws);
onset(ws);
};
const audio = {
serverContent: {
modelTurn: {
parts: [
{ inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } },
],
},
},
};
const done = (ws) => ws.message({ serverContent: { turnComplete: true } });
const tools = (ws, id) =>
ws.message({
toolCall: {
functionCalls: [
{
id,
name: 'adjust_camera_zoom',
args: { direction: 'out', amount: 'little' },
},
],
},
});
const responses = (ws) =>
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
function close(h) {
h.c.stop();
assert.equal(h.timers.size, 0, 'owned timers released');
}
test('N1 VAD-only split admits the first real answer and completes after playback', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
ws.message(audio);
ws.message({ serverContent: { outputTranscription: { text: 'Answer' } } });
done(ws);
assert.equal(h.captures[0].played.length, 1);
h.captures[0].drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
assert.equal(h.c.retiredReplyPending, false);
} finally {
close(h);
}
});
test('N2 VAD-only split does not arm a nonexistent reply timeout', async () => {
const h = harness();
try {
await h.c.start();
split(h.sockets[0]);
h.advance(5001);
assert.equal(h.c.status, 'listening');
assert.equal(h.c.retiredReplyPending, false);
assert.equal(h.c.retiredBoundaryWait, null);
} finally {
close(h);
}
});
test('N3 server interruption before output still fences until ordered turnComplete', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
ws.message({ serverContent: { interrupted: true } });
assert.equal(h.c.retiredReplyPending, true);
ws.message(audio);
tools(ws, 'retired-before-boundary');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 0);
assert.equal(h.calls.length, 0);
assert.equal(responses(ws).length, 0);
ws.message({
serverContent: { ...audio.serverContent, turnComplete: true },
});
assert.equal(
h.captures[0].played.length,
0,
'coalesced retired content is discarded',
);
ws.message(audio);
assert.equal(h.captures[0].played.length, 1);
done(ws);
h.captures[0].drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
} finally {
close(h);
}
});
test('N4 generationComplete is server evidence and next onset retires it', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message({ serverContent: { generationComplete: true } });
onset(ws);
assert.equal(h.c.retiredReplyPending, true);
assert.ok(h.c.retiredBoundaryWait);
ws.message(audio);
tools(ws, 'old-generation');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 0);
assert.equal(h.calls.length, 0);
done(ws);
assert.equal(h.c.retiredReplyPending, false);
} finally {
close(h);
}
});
test('N5 goAway during VAD-only activity waits for actual reply and playback', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
assert.ok(h.c.goAway);
assert.equal(ws.closed, undefined);
h.advance(5001);
assert.equal(h.c.isActive(), true);
ws.message(audio);
done(ws);
assert.equal(
h.c.isActive(),
true,
'pending actual PCM drains before expiry',
);
h.captures[0].drain();
assert.equal(h.c.status, 'error');
assert.equal(ws.closed, true);
} finally {
close(h);
}
});
test('N6 first server tool after VAD-only split executes exactly once', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
split(ws);
tools(ws, 'new-generation');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(
responses(ws).filter((x) => x.id === 'new-generation').length,
1,
);
assert.equal(responses(ws).at(-1).response.ok, true);
tools(ws, 'new-generation');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(
responses(ws).filter((x) => x.id === 'new-generation').length,
1,
);
} finally {
close(h);
}
});
test('C1 actual old audio followed by onset fences late audio and tool until boundary', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
onset(ws);
assert.equal(h.c.retiredReplyPending, true);
ws.message(audio);
tools(ws, 'late-old');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 1);
assert.equal(h.calls.length, 0);
ws.message({ serverContent: { interrupted: true } });
ws.message(audio);
assert.equal(h.captures[0].played.length, 1);
done(ws);
ws.message(audio);
assert.equal(h.captures[0].played.length, 2);
} finally {
close(h);
}
});
test('C2 first server tool in flight is aborted and never answered after new onset', async () => {
const gate = deferred(),
entered = [];
const h = harness({
runAction: async (_n, _a, o) => {
entered.push(o);
await gate.promise;
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
tools(ws, 'old-running');
await tick();
assert.equal(entered.length, 1);
onset(ws);
assert.equal(entered[0].signal.aborted, true);
gate.resolve();
await h.c.toolQueue;
assert.equal(responses(ws).filter((x) => x.id === 'old-running').length, 0);
assert.equal(h.c.retiredReplyPending, true);
} finally {
gate.resolve();
close(h);
}
});
test('C5 typed request before first server output still retires on next onset', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
assert.equal(h.c.sendTextCommand('Zoom in'), true);
onset(ws);
assert.equal(h.c.retiredReplyPending, true);
ws.message(audio);
tools(ws, 'late-typed');
await h.c.toolQueue;
assert.equal(h.captures[0].played.length, 0);
assert.equal(h.calls.length, 0);
} finally {
close(h);
}
});
test('N7 old retired boundary preserves the newer automatic owner for goAway', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
const oldOwner = h.c.automaticActivityOwner;
ws.message(audio);
onset(ws);
const successor = h.c.automaticActivityOwner;
assert.notEqual(successor, oldOwner);
assert.equal(h.c.retiredAutomaticActivityOwner, oldOwner);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.automaticActivityOwner, successor);
assert.equal(h.c.automaticTurnOpen, true);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message(audio);
done(ws);
h.captures[0].drain();
assert.equal(h.c.status, 'error');
} finally {
close(h);
}
});
test('N8 old automatic owner with no successor is cleared by its retired boundary', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
const owner = h.c.automaticActivityOwner;
ws.message(audio);
ws.message({ serverContent: { interrupted: true } });
assert.equal(h.c.retiredAutomaticActivityOwner, owner);
done(ws);
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), false, 'no invented successor wait');
} finally {
close(h);
}
});
test('N9 coalesced old completion and new VAD preserves the new owner', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
vad(ws, 'ACTIVITY_END');
ws.message({
voiceActivity: { type: 'ACTIVITY_START' },
serverContent: { interrupted: true, turnComplete: true },
});
assert.equal(h.c.automaticTurnOpen, true);
assert.ok(h.c.automaticActivityOwner);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
} finally {
close(h);
}
});
test('N10 stop, restart and destroy release automatic owner identities and timers', async () => {
const h = harness();
try {
await h.c.start();
onset(h.sockets[0]);
const first = h.c.automaticActivityOwner;
h.c.stop();
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
assert.equal(h.c.automaticActivitySequence, 0);
await h.c.start();
onset(h.sockets[1]);
assert.notEqual(h.c.automaticActivityOwner, first);
assert.notEqual(h.c.automaticActivityOwner.epoch, first.epoch);
h.c.stop({ removeUi: true });
assert.equal(h.c.disposed, true);
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(h.c.retiredAutomaticActivityOwner, null);
assert.equal(h.timers.size, 0);
} finally {
close(h);
}
});
test('N11 deferred typed ownership counts discarded open-mic PCM without queuing or replay', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
assert.equal(h.c.sendTextCommand('Zoom in'), true);
assert.equal(h.c.sendTextCommand('Zoom out'), true);
assert.equal(h.c.deferredInput.kind, 'user-text');
const before = ws.sent.filter((x) => x.realtimeInput?.audio).length;
const metadata = { captureId: h.c.captureOwner.captureId };
h.captures[0].callbacks.onAudio(
{ data: 'AAAA', mimeType: 'audio/pcm;rate=16000' },
metadata,
);
assert.equal(ws.sent.filter((x) => x.realtimeInput?.audio).length, before);
assert.equal(
h.c.deferredInput.messages.some((x) => x.realtimeInput?.audio),
false,
);
assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1);
assert.deepEqual(h.c.getDiagnostics().deferredAudioDrops, {
nonPttOwner: 1,
closedActivity: 0,
});
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
done(ws);
assert.equal(
ws.sent.filter((x) => x.realtimeInput?.audio).length,
before,
'no wrong-owner PCM replay after boundaries',
);
} finally {
close(h);
}
});
test('N12 goAway before old completion still waits for the newer automatic owner', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
onset(ws);
const successor = h.c.automaticActivityOwner;
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.automaticActivityOwner, successor);
assert.equal(
h.c.isActive(),
true,
'old boundary during goAway cannot end successor wait',
);
ws.message(audio);
done(ws);
h.captures[0].drain();
assert.equal(h.c.status, 'error');
} finally {
close(h);
}
});
for (const interruptedBeforeExpiry of [false, true])
test(`N13 no successor clears retired ownership with goAway ${interruptedBeforeExpiry ? 'after interruption' : 'before interruption'}`, async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
if (interruptedBeforeExpiry)
ws.message({ serverContent: { interrupted: true } });
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.automaticActivityOwner, null);
assert.equal(
h.c.isActive(),
false,
'no successor finishes at its actual boundary',
);
} finally {
close(h);
}
});
test('N14 native interruption during goAway aborts pending old tool and suppresses late response', async () => {
const gate = deferred(),
entered = [];
const h = harness({
runAction: async (_n, _a, o) => {
entered.push(o);
await gate.promise;
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
tools(ws, 'expiring-old-tool');
await tick();
assert.equal(entered.length, 1);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(entered[0].signal.aborted, true);
gate.resolve();
await h.c.toolQueue;
assert.equal(
responses(ws).filter((x) => x.id === 'expiring-old-tool').length,
0,
);
assert.equal(h.c.isActive(), false);
} finally {
gate.resolve();
close(h);
}
});
test('N15 goAway still refuses local PTT and typed replacement without reopening microphone', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
ws.message({ goAway: { timeLeft: '30s' } });
const turn = h.c.turn,
sent = ws.sent.length;
assert.equal(h.c.bargeIn(), false);
assert.equal(h.c.sendTextCommand('Zoom out'), false);
h.c.interrupt('push-to-talk');
assert.equal(h.c.turn, turn);
assert.equal(ws.sent.length, sent);
assert.equal(
h.tracks.every((t) => !t.enabled),
true,
);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.isActive(), false);
assert.equal(
h.tracks.every((t) => t.readyState === 'ended'),
true,
);
} finally {
close(h);
}
});
test('T-F1a PTT successor PCM survives retired boundary and a second input fences its late output', async () => {
const h = harness();
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
a = h.captures[0];
ws.message(audio);
h.c.input.pushToTalkKeyHeld = true;
assert.equal(h.c.input.bargeIn(), true);
h.c.input.setMicrophoneEnabled(true);
a.callbacks.onAudio(
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
a.metadata,
);
h.c.input.setMicrophoneEnabled(false);
assert.equal(h.c.replyPending, true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(
h.c.replyPending,
true,
'old boundary cannot clear submitted successor',
);
assert.equal(h.c.sendTextCommand('Zoom out second request'), true);
assert.ok(h.c.currentTypedInput());
const played = a.played.length;
ws.message(audio);
tools(ws, 'late-ptt-predecessor');
await h.c.toolQueue;
assert.equal(a.played.length, played);
assert.equal(
h.calls.length,
0,
'no old tool under newer local typed input',
);
done(ws);
tools(ws, 'current-typed');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
assert.equal(h.c.currentTypedInput().text, 'Zoom out second request');
} finally {
close(h);
}
});
for (const expiryBeforeBoundary of [false, true])
test(`T-F1b VAD then submitted text survives goAway ${expiryBeforeBoundary ? 'before' : 'after'} retired boundary`, async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
onset(ws);
assert.equal(h.c.sendTextCommand('Zoom out successor'), true);
const typed = h.c.currentTypedInput();
assert.ok(typed);
assert.equal(h.c.replyPending, true);
if (expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } });
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
assert.equal(h.c.replyPending, true);
if (!expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true, 'submitted text must finish');
tools(ws, 'valid-successor');
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
assert.equal(h.c.currentTypedInput(), typed);
ws.message(audio);
done(ws);
assert.equal(h.c.isActive(), true);
h.captures[0].drain();
assert.equal(h.c.isActive(), false);
} finally {
close(h);
}
});
for (const repeat of [false, true])
test(`T-F2 forwarded A superseded by deferred B cannot run after goAway${repeat ? ' shortened twice' : ''}`, async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
assert.equal(h.c.sendTextCommand('Zoom out A'), true);
assert.equal(h.c.replyPending, true);
assert.equal(h.c.sendTextCommand('Zoom out B'), true);
assert.equal(h.c.deferredInput.kind, 'user-text');
assert.ok(h.c.currentTypedInput());
onset(ws);
assert.equal(h.c.automaticTurnOpen, true);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.deferredInput, null);
if (repeat) {
ws.message({ goAway: { timeLeft: '20s' } });
ws.message({ goAway: { timeLeft: '10s' } });
}
assert.equal(h.c.goAway.supersededSuccessor, true);
assert.equal(h.c.isActive(), true);
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
tools(ws, 'superseded-A');
await h.c.toolQueue;
assert.equal(h.calls.length, 0, 'A cannot run with dropped B authority');
assert.equal(
h.c.isActive(),
false,
'ordered boundary closes superseded forwarded predecessor',
);
} finally {
close(h);
}
});
test('T-F3 VAD-only no-generation expiry waits to bounded native deadline and releases resources', async () => {
const h = harness();
try {
await h.c.start();
onset(h.sockets[0]);
h.sockets[0].message({ goAway: { timeLeft: '30s' } });
h.advance(29999);
assert.equal(h.c.isActive(), true);
h.advance(1);
assert.equal(h.c.isActive(), false);
assert.equal(h.timers.size, 0);
assert.equal(
h.tracks.every((t) => t.readyState === 'ended'),
true,
);
} finally {
close(h);
}
});
test('T-C1 coalesced new START and natural old completion preserve newer automatic activity', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
ws.message({
voiceActivity: { type: 'ACTIVITY_START' },
serverContent: { turnComplete: true },
});
assert.equal(h.c.retiredReplyPending, false);
assert.equal(h.c.automaticTurnOpen, true);
ws.message({ goAway: { timeLeft: '30s' } });
assert.equal(h.c.isActive(), true);
} finally {
close(h);
}
});
test('T-C2 generationComplete alone followed by natural completion emits completion with no retirement timer', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message({ serverContent: { generationComplete: true } });
assert.equal(h.c.replyPending, true);
done(ws);
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
assert.equal(h.c.replyPending, false);
assert.equal(h.c.automaticTurnOpen, false);
assert.equal(h.c.retiredBoundaryWait, null);
assert.equal(h.timers.size, 0);
} finally {
close(h);
}
});
test('T-C3 silent PTT terminal completion emits spoken false without fabricated audio', async () => {
const h = harness();
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
a = h.captures[0];
h.c.input.setMicrophoneEnabled(true);
a.callbacks.onAudio(
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
a.metadata,
);
h.c.input.setMicrophoneEnabled(false);
assert.equal(h.c.userTurnPending, true);
ws.message({ serverContent: { generationComplete: true } });
done(ws);
const event = h.events.findLast((e) => e.type === 'completion');
assert.equal(event.status, 'completed');
assert.equal(event.spoken, false);
assert.equal(a.played.length, 0);
assert.equal(h.c.userTurnPending, false);
} finally {
close(h);
}
});
test('T-C4 silent terminal boundary waits for pending tool and native follow-up completion', async () => {
const gate = deferred(),
entered = [];
const h = harness({
runAction: async (_n, _a, o) => {
entered.push(o);
await gate.promise;
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
tools(ws, 'pending-terminal-tool');
await tick();
assert.equal(entered.length, 1);
done(ws);
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
assert.equal(h.c.userTurnPending, false);
gate.resolve();
await h.c.toolQueue;
h.c.finishPlayback();
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
assert.equal(h.c.replyPending, true, 'tool result awaits native follow-up');
done(ws);
assert.equal(
h.events.findLast((e) => e.type === 'completion')?.status,
'completed',
);
} finally {
gate.resolve();
close(h);
}
});
test('T-C5 terminal boundary with queued PCM waits for drain and preserves real spoken state', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
ws.message(audio);
ws.message({ serverContent: { generationComplete: true } });
done(ws);
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
h.captures[0].drain();
const event = h.events.findLast((e) => e.type === 'completion');
assert.equal(event.status, 'completed');
assert.equal(event.spoken, true);
} finally {
close(h);
}
});
test('T-C6 old retired terminal boundary cannot clear successor userTurnPending', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(audio);
assert.equal(h.c.sendTextCommand('Zoom out successor'), true);
assert.equal(h.c.userTurnPending, true);
done(ws);
assert.equal(h.c.userTurnPending, true);
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
} finally {
close(h);
}
});
test('T-C7 reentrant silent completion observer keeps new input pending', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
const emit = h.c.emit.bind(h.c);
h.c.emit = (event) => {
emit(event);
if (event.type === 'completion')
h.c.sendTextCommand('Zoom out reentrant');
};
onset(ws);
ws.message({ serverContent: { generationComplete: true } });
done(ws);
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
assert.equal(h.c.replyPending, true);
assert.equal(h.c.userTurnPending, true);
assert.ok(h.c.currentTypedInput());
assert.equal(h.c.modelComplete, false);
} finally {
close(h);
}
});
+125
View File
@@ -0,0 +1,125 @@
const OUTPUT_RATE = 16000;
const FILTER_TAPS = 64;
const FILTER_PHASES = 128;
// A causal, windowed-sinc low-pass filter avoids folding device frequencies
// above 8 kHz into speech. Precompute phases; process allocates no filter data.
function resamplingFilters(inputRate) {
const cutoff = 0.45 * Math.min(1, OUTPUT_RATE / inputRate);
const filters = new Float32Array(FILTER_TAPS * FILTER_PHASES);
for (let phase = 0; phase < FILTER_PHASES; phase++) {
let total = 0;
for (let tap = 0; tap < FILTER_TAPS; tap++) {
const distance = tap - (FILTER_TAPS - 1) / 2 - phase / FILTER_PHASES;
const sinc =
distance === 0
? 2 * cutoff
: Math.sin(2 * Math.PI * cutoff * distance) / (Math.PI * distance);
const window =
0.5 - 0.5 * Math.cos((2 * Math.PI * tap) / (FILTER_TAPS - 1));
const weight = sinc * window;
filters[phase * FILTER_TAPS + tap] = weight;
total += weight;
}
for (let tap = 0; tap < FILTER_TAPS; tap++)
filters[phase * FILTER_TAPS + tap] /= total;
}
return filters;
}
/** PCM microphone capture. Only explicitly enabled frames leave this processor. */
class GeminiCaptureProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.enabled = false;
this.captureId = null;
this.samples = new Int16Array(1024);
this.length = 0;
this.inputRate = sampleRate;
this.filters =
this.inputRate === OUTPUT_RATE ? null : resamplingFilters(this.inputRate);
this.history = new Float32Array(FILTER_TAPS);
this.resetResampler();
this.port.onmessage = ({ data }) => {
if (data?.enabled === true) {
if (
this.enabled ||
!Number.isSafeInteger(data.captureId) ||
data.captureId <= 0
)
return;
this.captureId = data.captureId;
this.resetResampler();
this.enabled = true;
this.port.postMessage({ type: 'start', captureId: this.captureId });
} else if (
data?.enabled === false &&
this.enabled &&
data.captureId === this.captureId
) {
this.flush();
this.port.postMessage({ type: 'end', captureId: this.captureId });
this.enabled = false;
this.captureId = null;
this.length = 0;
// Do not drain filter tails into a later physical microphone capture.
this.resetResampler();
}
};
}
resetResampler() {
this.phase = 0;
this.historyIndex = 0;
this.history.fill(0);
}
appendSample(sample) {
const value = Math.max(-1, Math.min(1, sample));
this.samples[this.length++] = Math.round(
value * (value < 0 ? 32768 : 32767),
);
if (this.length === this.samples.length) this.flush();
}
flush() {
if (!this.length) return;
const buffer = new ArrayBuffer(this.length * 2);
const view = new DataView(buffer);
for (let i = 0; i < this.length; i++)
view.setInt16(i * 2, this.samples[i], true);
this.port.postMessage(
{ type: 'audio', buffer, captureId: this.captureId },
[buffer],
);
this.length = 0;
}
process(inputs) {
if (!this.enabled) return true;
const channel = inputs[0]?.[0];
if (!channel) return true;
for (const sample of channel) {
if (!this.filters) {
this.appendSample(sample);
continue;
}
this.history[this.historyIndex] = sample;
this.historyIndex = (this.historyIndex + 1) % FILTER_TAPS;
this.phase += OUTPUT_RATE;
while (this.phase >= this.inputRate) {
this.phase -= this.inputRate;
const phase = Math.min(
FILTER_PHASES - 1,
Math.floor((this.phase * FILTER_PHASES) / OUTPUT_RATE),
);
let value = 0;
for (let tap = 0; tap < FILTER_TAPS; tap++) {
const index =
(this.historyIndex - 1 - tap + FILTER_TAPS) % FILTER_TAPS;
value +=
this.history[index] * this.filters[phase * FILTER_TAPS + tap];
}
this.appendSample(value);
}
}
return true;
}
}
registerProcessor('gev-gemini-capture', GeminiCaptureProcessor);
+285
View File
@@ -0,0 +1,285 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
nativeIntegrationFixture,
assertSuccessfulZoom,
} from '../testSupport/geminiIntegrationFixture.mjs';
const options = { concurrency: false, timeout: 5000 };
const OLD_PCM = 'BAAFAA==';
const TEXT = 'Zoom in for the owned typed successor';
function exposeCapture(h) {
const createAudio = h.controller.createAudio;
h.controller.createAudio = (callbacks) => {
const audio = createAudio(callbacks);
audio.captureCallbacks = callbacks;
return audio;
};
}
async function startManual(h) {
await h.session.start({ pushToTalk: true });
assert.equal(h.session.state, 'listening');
assert.equal(h.controller.input.pushToTalkMode, true);
assert.equal(Boolean(h.controller.input.pushToTalkKeyHeld), false);
return h.sockets.at(-1);
}
// Explicit fake worklet callbacks retain the native physical-capture path;
// this helper does not populate controller queues or ownership fields itself.
function speakAndRelease(h, data) {
const c = h.controller;
const audio = h.audios.at(-1);
c.input.pushToTalkKeyHeld = true;
c.input.setMicrophoneEnabled(true);
const metadata = audio.metadata;
assert.ok(Number.isSafeInteger(metadata?.captureId));
audio.captureCallbacks.onAudioStart(metadata);
audio.captureCallbacks.onAudio(
{ data, mimeType: 'audio/pcm;rate=16000' },
metadata,
);
c.input.pushToTalkKeyHeld = false;
c.input.setMicrophoneEnabled(false);
audio.captureCallbacks.onAudioEnd(metadata);
return metadata;
}
async function prepareDeferredPtt(h) {
exposeCapture(h);
const socket = await startManual(h);
speakAndRelease(h, 'AAABAA=='); // A is submitted and awaits its reply.
speakAndRelease(h, 'AgADAA=='); // B is submitted, retiring A.
const wireBeforeC = socket.sent.length;
const metadata = speakAndRelease(h, OLD_PCM); // C waits behind A's boundary.
const owner = h.controller.deferredInput;
assert.equal(owner?.kind, 'push-to-talk');
assert.equal(owner.captureId, metadata.captureId);
assert.equal(
socket.sent.length,
wireBeforeC,
'C must still be wholly deferred',
);
assert.deepEqual(
owner.messages.map((message) =>
message.realtimeInput?.activityStart
? 'start'
: message.realtimeInput?.audio
? 'audio'
: message.realtimeInput?.activityEnd
? 'end'
: 'other',
),
['start', 'audio', 'end'],
);
return { socket, owner, metadata };
}
async function assertHealthySuccessor(h, socket, callId) {
const beforeHeight = h.state.height;
await h.tool(callId, socket);
assertSuccessfulZoom(h, socket, callId, beforeHeight);
assert.equal(h.controller.pendingTools, 0);
h.complete(socket); // Native post-tool response boundary, with no output PCM.
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.session.state, 'listening');
}
for (const mode of ['typed replacement', 'Stop', 'Stop+restart']) {
test(
`F04-5 deferred PTT flush loses its owner on ${mode}`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const { socket: oldSocket, owner } = await prepareDeferredPtt(h);
const oldAudio = h.audios.at(-1);
let observed = 0;
let sentAtObserverReturn;
let restart;
let stateAtObserverReturn;
const unsubscribe = h.session.subscribe((event) => {
if (
event.type !== 'transcript' ||
event.role !== 'user' ||
event.text !== '' ||
observed
)
return;
observed++;
if (mode === 'typed replacement') {
assert.equal(h.session.sendText(TEXT), true);
} else {
h.session.stop();
if (mode === 'Stop+restart')
restart = h.session.start({ pushToTalk: true });
}
sentAtObserverReturn = oldSocket.sent.length;
stateAtObserverReturn = {
epoch: h.controller.epoch,
turn: h.controller.turn,
replyPending: h.controller.replyPending,
forwardedActivityOpen: h.controller.forwardedActivityOpen,
};
});
t.after(unsubscribe);
h.complete(oldSocket); // A's boundary starts flushing C. C's start emits.
assert.equal(
observed,
1,
'must enter the actual flush/onset observer boundary',
);
assert.ok(
stateAtObserverReturn,
'observer must complete its replacement action',
);
assert.deepEqual(
oldSocket.sent.slice(sentAtObserverReturn),
[],
'retired C must send neither queued PCM nor an extra end after the observer returns',
);
assert.deepEqual(
{
epoch: h.controller.epoch,
turn: h.controller.turn,
replyPending: h.controller.replyPending,
forwardedActivityOpen: h.controller.forwardedActivityOpen,
},
stateAtObserverReturn,
'old flush must not write successor state',
);
assert.equal(h.controller.deferredInput, null);
assert.equal(
h.controller.typedIntent?.text ?? null,
mode === 'typed replacement' ? TEXT : null,
);
let successorSocket;
if (mode === 'typed replacement') {
assert.equal(h.controller.epoch, owner.epoch);
assert.notEqual(h.controller.turn, owner.turn);
assert.equal(
oldSocket.sent.some(
(message) => message.realtimeInput?.audio?.data === OLD_PCM,
),
false,
);
const typedIndex = oldSocket.sent.findIndex((message) =>
message.clientContent?.turns?.some((turn) =>
turn.parts?.some((part) => part.text === TEXT),
),
);
assert.ok(
typedIndex >= 0,
'owned replacement must actually reach transport',
);
// The replacement legitimately closes the already-forwarded C start once.
assert.ok(oldSocket.sent[typedIndex - 1]?.realtimeInput?.activityEnd);
h.complete(oldSocket); // B's remaining retired boundary, deliberately separate.
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(h.controller.replyPending, true);
successorSocket = oldSocket;
} else {
assert.equal(oldAudio.closed, true);
assert.equal(oldSocket.closed, true);
if (!restart) restart = h.session.start({ pushToTalk: true });
await restart;
successorSocket = h.sockets.at(-1);
assert.notEqual(successorSocket, oldSocket);
assert.equal(
successorSocket.sent.some((message) => message.realtimeInput?.audio),
false,
);
assert.equal(
successorSocket.sent.some(
(message) => message.realtimeInput?.activityEnd,
),
false,
);
assert.equal(h.session.sendText(TEXT), true);
}
await assertHealthySuccessor(
h,
successorSocket,
`healthy-after-deferred-${mode}`,
);
},
);
}
test(
'F04-5 deferred typed onSent cannot resurrect replyPending after Stop+restart',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const oldSocket = await h.start();
assert.equal(h.session.sendText('A'), true);
assert.equal(h.session.sendText('B'), true);
assert.equal(h.session.sendText('C deferred text'), true);
assert.equal(h.controller.deferredInput?.kind, 'user-text');
let restart;
let observed = 0;
const unsubscribe = h.session.subscribe((event) => {
if (
event.type !== 'transcript' ||
event.role !== 'user' ||
event.text !== 'C deferred text'
)
return;
observed++;
h.session.stop();
restart = h.session.start();
});
t.after(unsubscribe);
h.complete(oldSocket);
assert.equal(observed, 1);
assert.equal(
h.controller.replyPending,
false,
'old onSent wrapper must not resurrect a stopped reply',
);
await restart;
assert.equal(h.controller.replyPending, false);
const socket = h.sockets.at(-1);
assert.equal(
socket.sent.length,
1,
'replacement connection should contain setup only',
);
assert.equal(h.session.sendText(TEXT), true);
await assertHealthySuccessor(h, socket, 'healthy-after-deferred-text');
},
);
test(
'F04-5 ordinary deferred PTT flush preserves start, PCM and end exactly once',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const { socket } = await prepareDeferredPtt(h);
const before = socket.sent.length;
h.complete(socket);
assert.deepEqual(
socket.sent
.slice(before)
.map((message) =>
message.realtimeInput?.activityStart
? 'start'
: message.realtimeInput?.audio?.data === OLD_PCM
? 'audio'
: message.realtimeInput?.activityEnd
? 'end'
: 'other',
),
['start', 'audio', 'end'],
);
assert.equal(h.controller.deferredInput, null);
assert.equal(h.controller.replyPending, true);
assert.equal(h.controller.retiredReplyPending, true);
h.complete(socket); // B's boundary; C remains the owned submitted input.
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(h.controller.replyPending, true);
await assertHealthySuccessor(h, socket, 'healthy-ordinary-deferred');
},
);
+287
View File
@@ -0,0 +1,287 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
nativeIntegrationFixture,
assertSuccessfulZoom,
turnOfEventLoop,
} from '../testSupport/geminiIntegrationFixture.mjs';
const options = { concurrency: false, timeout: 5000 };
const resultFor = (h, id) =>
h.events.filter(
(event) => event.type === 'action-result' && event.callId === id,
);
const callFor = (h, id) =>
h.events.filter(
(event) => event.type === 'action-call' && event.callId === id,
);
test(
'native tool reaches actual common session and camera runner without a caption',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
h.activity(ws); // Native activity is input framing, not transcript admission.
await h.tool('ordinary', ws);
assertSuccessfulZoom(h, ws, 'ordinary', 1000);
assert.equal(h.runnerCalls.length, 1);
assert.equal(h.runnerCalls[0].options.callId, 'ordinary');
assert.equal(h.runnerCalls[0].options.signal.aborted, false);
assert.equal(h.runnerCalls[0].options.isCurrent(), true);
h.complete(ws);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
},
);
for (const cancellation of ['stop', 'native interruption']) {
test(
`${cancellation} discards a stale completed action result; successor changes camera once`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const oldSocket = await h.start();
h.activity(oldSocket);
const gate = h.holdResult('old');
const oldDispatch = h.tool('old', oldSocket);
const committedResult = await gate.ready;
assert.equal(committedResult.ok, true);
assert.equal(
h.state.height,
750,
'the legitimate first action committed before cancellation',
);
assert.equal(callFor(h, 'old').length, 1);
assert.deepEqual(resultFor(h, 'old'), []);
assert.deepEqual(h.outputs(oldSocket), []);
if (cancellation === 'stop') h.session.stop();
else
oldSocket.message({
serverContent: { interrupted: true, turnComplete: true },
});
assert.equal(h.runnerCalls[0].options.signal.aborted, true);
assert.equal(h.runnerCalls[0].options.isCurrent(), false);
await oldDispatch; // Cancellation must settle before the held result returns.
gate.release();
await turnOfEventLoop();
assert.equal(
h.state.height,
750,
'settling a cancelled result adds no camera mutation',
);
assert.equal(h.state.zooms.length, 1);
assert.deepEqual(resultFor(h, 'old'), []);
assert.deepEqual(h.outputs(oldSocket), []);
if (cancellation === 'native interruption')
assert.ok(h.events.some((event) => event.type === 'interruption'));
const successorSocket =
cancellation === 'stop' ? await h.start() : oldSocket;
h.activity(successorSocket);
await h.tool('successor', successorSocket);
assertSuccessfulZoom(h, successorSocket, 'successor', 750);
assert.equal(h.state.zooms.length, 2);
assert.equal(h.runnerCalls.length, 2);
assert.equal(
h.outputs(successorSocket).some((output) => output.id === 'old'),
false,
);
h.complete(successorSocket);
},
);
}
for (const phase of ['common connecting', 'adapter connecting']) {
test(
`Stop from ${phase} observer prevents acquisition and permits fresh Start`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
let stopped = false;
const remove = h.session.subscribe((event) => {
const matchingDetail =
phase === 'common connecting'
? event.detail === undefined
: typeof event.detail === 'string' && event.detail.length > 0;
if (
!stopped &&
event.type === 'state' &&
event.state === 'connecting' &&
matchingDetail
) {
stopped = true;
h.session.stop();
}
});
await h.session.start();
remove();
assert.equal(
stopped,
true,
'the chosen production observer boundary was exercised',
);
assert.deepEqual(h.acquisitions, {
microphone: 0,
audio: 0,
token: 0,
socket: 0,
});
assert.equal(h.session.state, 'idle');
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
const ws = await h.start();
h.activity(ws);
await h.tool('fresh-after-start-cancel', ws);
assertSuccessfulZoom(h, ws, 'fresh-after-start-cancel', 1000);
h.complete(ws);
},
);
}
test(
'Stop from common action-call observer prevents runner and leaves no unhandled rejection',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
const remove = h.session.subscribe((event) => {
if (event.type === 'action-call' && event.callId === 'cancel-at-call')
h.session.stop();
});
h.activity(ws);
await h.tool('cancel-at-call', ws);
// Do not install a swallowing process rejection listener. node:test records
// an unhandled rejection as a failure; these checkpoints let it surface here.
await turnOfEventLoop();
await turnOfEventLoop();
remove();
assert.equal(callFor(h, 'cancel-at-call').length, 1);
assert.deepEqual(resultFor(h, 'cancel-at-call'), []);
assert.equal(h.runnerCalls.length, 0);
assert.equal(h.state.height, 1000);
assert.deepEqual(h.outputs(ws), []);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
const successor = await h.start();
h.activity(successor);
await h.tool('after-call-cancel', successor);
assertSuccessfulZoom(h, successor, 'after-call-cancel', 1000);
h.complete(successor);
},
);
test(
'Stop from executing observer cannot recreate narration or dispatch the runner',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
let stopped = false;
const remove = h.session.subscribe((event) => {
if (!stopped && event.type === 'state' && event.state === 'executing') {
stopped = true;
h.session.stop();
}
});
h.activity(ws);
await h.tool('cancel-at-executing', ws);
remove();
assert.equal(stopped, true);
assert.equal(h.runnerCalls.length, 0);
assert.deepEqual(callFor(h, 'cancel-at-executing'), []);
assert.deepEqual(resultFor(h, 'cancel-at-executing'), []);
assert.equal(h.state.height, 1000);
assert.deepEqual(h.outputs(ws), []);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
assert.equal(ws.closed, true);
assert.ok(h.audios.every((audio) => audio.closed));
assert.ok(h.tracks.every((track) => track.readyState === 'ended'));
const before = {
events: h.events.length,
wire: ws.sent.length,
earcon: h.state.earconStarts,
};
h.clock.advance(60_000);
assert.deepEqual(
{
events: h.events.length,
wire: ws.sent.length,
earcon: h.state.earconStarts,
},
before,
);
const successor = await h.start();
h.activity(successor);
await h.tool('after-executing-cancel', successor);
assertSuccessfulZoom(h, successor, 'after-executing-cancel', 1000);
h.complete(successor);
},
);
test(
'native idle observer restart remains connecting and public Stop cancels its pending permission',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
await h.start();
let grant;
const permission = new Promise((resolve) => {
grant = resolve;
});
h.controller.getUserMedia = () => permission;
const states = [];
let restarted = false;
let startup;
const remove = h.session.subscribe((event) => {
if (event.type === 'state') states.push(event.state);
if (!restarted && event.type === 'state' && event.state === 'idle') {
restarted = true;
startup = h.session.start();
}
});
h.session.stop();
remove();
assert.equal(restarted, true);
assert.equal(states.filter((state) => state === 'idle').length, 1);
assert.equal(h.session.state, 'connecting');
assert.equal(h.controller.status, 'connecting');
assert.equal(
h.session.isActive(),
true,
'the common button/facade must offer Stop',
);
h.session.stop();
assert.equal(h.session.state, 'idle');
assert.equal(h.controller.status, 'idle');
const lateTrack = {
readyState: 'live',
stopCalls: 0,
stop() {
this.stopCalls++;
this.readyState = 'ended';
},
};
h.tracks.push(lateTrack);
grant({ getTracks: () => [lateTrack], getAudioTracks: () => [lateTrack] });
await startup;
await turnOfEventLoop();
assert.equal(lateTrack.stopCalls, 1);
assert.equal(
h.requests.length,
1,
'cancelled restart must not mint another token',
);
assert.equal(
h.sockets.length,
1,
'cancelled restart must not open another socket',
);
assert.equal(h.session.state, 'idle');
},
);
@@ -0,0 +1,85 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { harness } from '../testSupport/geminiNativeHarness.mjs';
const caption = (socket, text) =>
socket.message({ serverContent: { inputTranscription: { text } } });
const userEvents = (events) =>
events.filter(
(event) => event.type === 'transcript' && event.role === 'user',
);
test('native caption events preserve received advisory segments without granting tool authority', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0];
const start = h.events.length;
for (const text of ['little. ', 'Zoom in a ']) caption(ws, text);
const received = userEvents(h.events.slice(start));
assert.deepEqual(
received.map((event) => event.text),
['little. ', 'Zoom in a '],
);
assert.equal(
received.every(
(event) =>
event.delta === true &&
event.final === false &&
event.attribution === 'unverified' &&
event.responseId === null,
),
true,
);
assert.equal(h.c.currentTypedInput(), null);
ws.message({
toolCall: {
functionCalls: [
{
id: 'ordinary-native',
name: 'get_current_view_state',
args: {},
},
],
},
});
await h.c.toolQueue;
assert.equal(h.calls.length, 1);
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
ws.message({ serverContent: { turnComplete: true } });
assert.equal(
userEvents(h.events.slice(start)).some((event) => event.final),
false,
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
test('native old socket captions cannot enter a restarted controller event stream', async () => {
const h = harness();
try {
await h.c.start();
const old = h.sockets[0];
caption(old, 'old partial');
h.c.stop();
await h.c.start();
const start = h.events.length;
const current = h.sockets[1];
caption(current, 'Zoom ');
caption(old, 'stale old socket');
caption(current, 'in a little.');
current.message({ serverContent: { turnComplete: true } });
assert.deepEqual(
userEvents(h.events.slice(start)).map((event) => event.text),
['Zoom ', 'in a little.'],
);
assert.equal(h.tracks[0].readyState, 'ended');
assert.equal(h.captures[0].closed, true);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
+667
View File
@@ -0,0 +1,667 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
import { withToolCatalog } from './gevRealtime.js';
const audioContent = (data) => ({
modelTurn: {
parts: [{ inlineData: { data, mimeType: 'audio/pcm;rate=24000' } }],
},
});
const request = (ws, id, name = 'get_current_view_state', args = {}) =>
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
const responses = (ws) =>
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
const onset = (ws) => {
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
};
const caption = (ws, text) =>
ws.message({ serverContent: { inputTranscription: { text } } });
for (const [id, name, args] of [
['H01', 'get_entity_context', {}],
['H02', 'control_radio', { action: 'status' }],
]) {
test(`${id} native: cancelling a current ${name} query preserves unrelated Radio preflight`, async () => {
const ready = deferred(),
query = deferred(),
executions = [];
const h = harness({
runAction: (name, args, options) => {
executions.push({ name, args, options });
return args.action === 'play'
? Promise.resolve({
ok: true,
action: 'control_radio',
radioAction: 'play',
radioPlaybackRequested: true,
})
: query.promise;
},
});
h.c.radio.radioLayer.playForVoice = () => ready.promise;
try {
await h.c.start();
h.c.sendTextCommand('Play radio');
const ws = h.sockets[0];
request(ws, 'owner', 'control_radio', { action: 'play' });
await h.c.toolQueue;
ws.message({
serverContent: { ...audioContent('confirmation'), turnComplete: true },
});
h.captures[0].drain();
assert.equal(h.c.radio.radioHandoffInFlight, true);
request(ws, 'query', name, args);
await tick();
assert.equal(
executions.length,
2,
'current native query may start after provider completion',
);
const epoch = h.c.radio.radioHandoffEpoch;
const pending = h.c.radio.pendingRadioPlaybackResult;
const attempt = h.c.radio.radioHandoffAttemptId;
const ownerCurrent = executions[0].options.isCurrent();
ws.message({ toolCallCancellation: { ids: ['unknown', 'query'] } });
await h.c.toolQueue;
assert.equal(executions[1].options.signal.aborted, true);
assert.equal(executions[1].options.isCurrent(), false);
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
assert.equal(executions[0].options.signal.aborted, false);
assert.equal(executions[0].options.isCurrent(), ownerCurrent);
assert.equal(h.c.radio.pendingRadioPlaybackResult, pending);
assert.equal(h.c.radio.radioHandoffAttemptId, attempt);
assert.equal(h.c.radio.radioHandoffInFlight, true);
assert.equal(h.state.stopped, 0);
assert.equal(h.c.pendingTools, 0);
assert.deepEqual(
responses(ws).map((r) => r.id),
['owner'],
);
assert.equal(responses(ws)[0].response.ok, true);
query.resolve({ ok: true });
ready.resolve(true);
await tick();
assert.equal(h.c.status, 'idle');
assert.equal(h.state.ducked, false);
assert.deepEqual(
responses(ws).map((r) => r.id),
['owner'],
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const [id, prior] of [
['H03', 'playing'],
['H04', 'before-first-output'],
]) {
test(`${id} native: PTT fences ${prior} reply and recovers the new call after its boundary`, async () => {
const h = harness();
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
audio = h.captures[0];
if (prior === 'playing')
ws.message({ serverContent: audioContent('old') });
else {
h.c.input.setMicrophoneEnabled(true);
audio.callbacks.onAudio(
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
audio.metadata,
);
h.c.input.setMicrophoneEnabled(false);
}
h.c.input.pushToTalkKeyHeld = true;
assert.equal(h.c.input.bargeIn(), true);
h.c.input.setMicrophoneEnabled(true);
h.c.input.setMicrophoneEnabled(false);
const turn = h.c.turn,
played = audio.played.length,
firstEvent = h.events.length;
ws.message({
serverContent: {
...audioContent('late-old'),
outputTranscription: { text: 'retired output' },
inputTranscription: { text: 'new request' },
},
});
request(ws, 'retired-call', 'fly_to_location', { query: 'old target' });
await h.c.toolQueue;
assert.equal(audio.played.length, played);
assert.equal(audio.pending, false);
assert.equal(h.calls.length, 0);
assert.equal(
h.events.slice(firstEvent).some((e) => e.role === 'assistant'),
false,
);
assert.equal(
h.events
.slice(firstEvent)
.some((e) => e.role === 'user' && e.text === 'new request'),
true,
);
ws.message({ serverContent: { interrupted: true } });
assert.equal(h.c.turn, turn);
ws.message({ serverContent: audioContent('after-interrupted') });
assert.equal(
audio.played.length,
played,
'interrupted alone cannot release the fence',
);
ws.message({
serverContent: {
...audioContent('boundary-old'),
outputTranscription: { text: 'still retired' },
turnComplete: true,
},
});
assert.equal(
audio.played.length,
played,
'coalesced old boundary output is discarded',
);
assert.equal(
h.events.slice(firstEvent).some((e) => e.type === 'completion'),
false,
);
caption(ws, 'unrelated advisory words');
ws.message({
toolCall: {
functionCalls: [
{
id: 'retired-call',
name: 'fly_to_location',
args: { query: 'replayed old target' },
},
{
id: 'new-call',
name: 'fly_to_location',
args: { query: 'new target' },
},
],
},
});
await h.c.toolQueue;
assert.deepEqual(
h.calls.map((c) => c.args.query),
['new target'],
);
assert.equal(h.calls[0].options.userTurn, undefined);
ws.message({
serverContent: {
...audioContent('new'),
outputTranscription: { text: 'new reply' },
turnComplete: true,
},
});
assert.equal(audio.played.at(-1).data, 'new');
audio.drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion').status,
'completed',
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
test('H05 native: local typed presentation survives audio drain and new input revokes old call ownership', async () => {
const h = harness();
try {
await h.c.start();
const ws = h.sockets[0],
audio = h.captures[0];
h.c.sendTextCommand('Show the current view');
const response = h.c.currentTypedInput();
assert.equal(response.text, 'Show the current view');
assert.equal(
h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
.text,
response.text,
);
request(ws, 'first');
await h.c.toolQueue;
assert.equal(h.calls[0].options.userTurn, undefined);
assert.equal(h.calls[0].options.isCurrent(), true);
ws.message({
serverContent: {
...audioContent('confirmation'),
outputTranscription: { text: 'Here is the current view' },
turnComplete: true,
},
});
assert.equal(
h.events.some((e) => e.type === 'completion'),
false,
);
assert.equal(
h.events.findLast((e) => e.role === 'assistant').responseId,
response.responseId,
);
audio.drain();
const completion = h.events.findLast((e) => e.type === 'completion');
assert.equal(completion.responseId, response.responseId);
assert.equal(completion.spoken, true);
assert.equal(completion.audioDrained, true);
assert.equal(h.c.currentTypedInput(), response);
assert.equal(response.text, 'Show the current view');
request(ws, 'same-turn');
await h.c.toolQueue;
assert.equal(h.calls[1].options.userTurn, undefined);
assert.equal(h.c.currentTypedInput(), response);
h.c.sendTextCommand('New request');
assert.notEqual(h.c.currentTypedInput(), response);
assert.equal(h.c.currentTypedInput().text, 'New request');
assert.equal(h.calls[0].options.isCurrent(), false);
assert.equal(h.calls[1].options.isCurrent(), false);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
test('H08 native: replacing pending runner work cannot underflow tool counts or block successor', async () => {
const gate = deferred(),
executions = [];
const h = harness({
runAction: (name, args, options) => {
executions.push({ name, args, options });
return options.callId === 'old'
? gate.promise
: Promise.resolve({ ok: true });
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
request(ws, 'old');
await tick();
const oldWork = h.c.toolQueue;
assert.equal(h.c.pendingTools, 1);
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
assert.equal(h.c.pendingTools, 0);
await oldWork;
assert.equal(h.c.pendingTools, 0);
assert.equal(executions[0].options.signal.aborted, true);
assert.equal(executions[0].options.isCurrent(), false);
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
request(ws, 'fresh');
await h.c.toolQueue;
assert.equal(executions.at(-1).options.callId, 'fresh');
assert.equal(h.c.pendingTools, 0);
gate.resolve({ ok: true });
await tick();
assert.equal(h.c.pendingTools, 0);
assert.deepEqual(
responses(ws).map((r) => r.id),
['fresh'],
);
ws.message({
serverContent: { ...audioContent('fresh reply'), turnComplete: true },
});
h.captures[0].drain();
assert.equal(
h.events.findLast((e) => e.type === 'completion').status,
'completed',
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
test('H10 native: held speech dispatches without caption authority and new text revokes its call', async () => {
const gate = deferred(),
executions = [];
const h = harness({
runAction: (name, args, options) => {
executions.push({ name, args, options });
return gate.promise;
},
});
try {
await h.c.start({ pushToTalk: true });
const ws = h.sockets[0],
audio = h.captures[0];
h.c.input.bargeIn();
h.c.input.setMicrophoneEnabled(true);
audio.callbacks.onAudio(
{ data: 'AA==', mimeType: 'audio/pcm;rate=16000' },
audio.metadata,
);
h.c.input.setMicrophoneEnabled(false);
caption(ws, 'Show the current view');
assert.equal(h.c.currentTypedInput(), null);
assert.equal(
h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
.text,
'Show the current view',
);
request(ws, 'held');
await tick();
const oldWork = h.c.toolQueue;
assert.equal(executions.length, 1);
assert.equal(executions[0].options.userTurn, undefined);
assert.equal(executions[0].options.isCurrent(), true);
h.c.sendTextCommand('new intent');
await oldWork;
assert.equal(executions[0].options.signal.aborted, true);
assert.equal(executions[0].options.isCurrent(), false);
gate.resolve({ ok: true });
await tick();
assert.equal(
responses(ws).some((r) => r.id === 'held'),
false,
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
for (const [id, restart] of [
['H11', false],
['H12', true],
]) {
test(`${id} native: pending tool cannot revive voice after Stop${restart ? '/restart' : ''}`, async () => {
const gate = deferred(),
mutations = [],
executions = [];
const h = harness({
runAction: async (name, args, options) => {
executions.push(options);
await gate.promise;
if (options.isCurrent()) mutations.push(name);
return { ok: true };
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
request(ws, 'old');
await tick();
const pending = h.c.toolQueue;
assert.equal(h.c.pendingTools, 1);
h.c.stop();
const boundary = h.events.length;
const successor = restart ? h.c.start() : null;
await pending;
assert.equal(
h.events
.slice(boundary)
.some((e) => e.type === 'state' && e.state === 'listening'),
false,
);
assert.equal(executions[0].signal.aborted, true);
assert.equal(executions[0].isCurrent(), false);
assert.equal(h.c.pendingTools, 0);
if (successor) await successor;
const laterBoundary = h.events.length;
gate.resolve();
caption(ws, 'late old speech');
await tick();
assert.deepEqual(mutations, []);
assert.equal(
h.events.slice(laterBoundary).some((e) => e.type === 'state'),
false,
);
assert.equal(responses(ws).length, 0);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const [id, boundary] of [
['H14', 'interrupted'],
['H15', 'natural'],
]) {
test(`${id} native: shared runner zoom executes promptly after ${boundary} retired boundary`, async () => {
const mutations = [],
observed = [];
const guarded = withToolCatalog(
async (name, args) => {
mutations.push({ name, args });
return { ok: true };
},
async () => null,
);
const h = harness({
runAction: (name, args, options) => {
observed.push(options);
return guarded(name, args, options);
},
});
try {
await h.c.start();
const ws = h.sockets[0];
ws.message({ serverContent: audioContent('old reply') });
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({
serverContent: {
...(boundary === 'interrupted' ? { interrupted: true } : {}),
turnComplete: true,
},
});
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
caption(ws, 'zoom in');
request(ws, 'zoom', 'adjust_camera_zoom', { direction: 'in' });
await tick();
assert.equal(
mutations.length,
1,
'native action executes without a transcript wait',
);
assert.equal(observed[0].userTurn, undefined);
assert.equal(h.c.pendingTools, 0);
assert.equal(
responses(ws).some((r) => r.response.voiceIntentUnverified),
false,
);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const [id, exit] of [
['H18', 'stop'],
['H19', 'replacement'],
]) {
test(`${id} native: ${exit} prevents queued camera dispatch and late captions cannot revive it`, async () => {
const mutations = [],
observed = [];
const guarded = withToolCatalog(
async (name) => {
mutations.push(name);
return { ok: true };
},
async () => null,
);
const h = harness({
runAction: (name, args, options) => {
observed.push(name);
return guarded(name, args, options);
},
});
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
request(ws, 'queued-old', 'adjust_camera_zoom', { direction: 'in' });
const oldWork = h.c.toolQueue;
assert.equal(h.c.pendingTools, 1);
if (exit === 'stop') h.c.stop();
else h.c.sendTextCommand('Show the whole globe');
caption(ws, 'zoom in');
await oldWork;
assert.deepEqual(mutations, []);
assert.deepEqual(observed, []);
assert.equal(h.c.pendingTools, 0);
if (exit === 'replacement') {
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
request(ws, 'fresh-camera', 'zoom_to_globe');
await h.c.toolQueue;
assert.deepEqual(mutations, ['zoom_to_globe']);
assert.deepEqual(observed, ['zoom_to_globe']);
assert.equal(responses(ws).at(-1).response.ok, true);
}
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const settlement of ['queued', 'running']) {
test(`native: cancelled ${settlement} tool settles a completed provider turn`, async () => {
const result = deferred();
let executions = 0;
const h = harness({
runAction: () => {
executions++;
return result.promise;
},
});
try {
await h.c.start();
h.c.sendTextCommand('Read current view');
const ws = h.sockets[0];
request(ws, 'cancelled-query');
if (settlement === 'running') await tick();
ws.message({ toolCallCancellation: { ids: ['cancelled-query'] } });
ws.message({ serverContent: { turnComplete: true } });
await h.c.toolQueue;
assert.equal(h.c.pendingTools, 0);
assert.equal(executions, settlement === 'queued' ? 0 : 1);
assert.deepEqual(responses(ws), []);
assert.equal(
h.events.filter((event) => event.type === 'completion').length,
1,
);
assert.equal(h.c.pendingCompletionStatus, null);
result.resolve({ ok: true });
await tick();
h.captures[0].drain();
assert.equal(
h.events.filter((event) => event.type === 'completion').length,
1,
);
assert.deepEqual(responses(ws), []);
} finally {
result.resolve({ ok: true });
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const order of ['before', 'after']) {
test(`native: cancelling queued output ${order} turnComplete emits one completion`, async () => {
const h = harness();
try {
await h.c.start();
h.c.sendTextCommand('Read current view');
const ws = h.sockets[0];
ws.bufferedAmount = 1e9;
request(ws, 'queued-output');
await h.c.toolQueue;
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.controlQueue.length, 1);
assert.equal(responses(ws).length, 0);
if (order === 'after')
ws.message({ serverContent: { turnComplete: true } });
ws.message({ toolCallCancellation: { ids: ['queued-output'] } });
if (order === 'before')
ws.message({ serverContent: { turnComplete: true } });
assert.equal(h.c.controlQueue.length, 0);
assert.equal(h.c.controlTimer, null);
assert.equal(h.c.pendingCompletionStatus, null);
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
ws.bufferedAmount = 0;
h.advance(1000);
h.captures[0].drain();
assert.deepEqual(responses(ws), []);
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
for (const successor of ['input', 'stop']) {
test(`native: queued-output cancellation completion fences reentrant ${successor}`, async () => {
let h;
let successorTurn;
h = harness({
emit: (event) => {
h.events.push(event);
if (event.type !== 'completion') return;
h.sockets[0].bufferedAmount = 0;
if (successor === 'stop') h.c.stop();
else {
h.c.sendTextCommand('Show the whole globe');
successorTurn = h.c.turn;
}
},
});
try {
await h.c.start();
h.c.sendTextCommand('Read current view');
const ws = h.sockets[0];
ws.bufferedAmount = 1e9;
request(ws, 'queued-output');
await h.c.toolQueue;
ws.message({ serverContent: { turnComplete: true } });
const callCount = h.calls.length;
ws.message({
toolCallCancellation: { ids: ['queued-output'] },
serverContent: {
outputTranscription: { text: 'Retired packet caption' },
turnComplete: true,
},
toolCall: {
functionCalls: [
{ id: 'retired-tool', name: 'get_current_view_state', args: {} },
],
},
});
await h.c.toolQueue;
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
assert.equal(h.calls.length, callCount);
assert.deepEqual(responses(ws), []);
assert.ok(
!h.events.some((e) =>
JSON.stringify(e).includes('Retired packet caption'),
),
);
if (successor === 'input') {
assert.equal(h.c.turn, successorTurn);
assert.equal(h.c.modelComplete, false);
assert.equal(h.c.replyPending, true);
} else assert.equal(h.c.isActive(), false);
} finally {
h.c.stop();
}
assert.equal(h.timers.size, 0);
});
}
+734
View File
@@ -0,0 +1,734 @@
// Native ownership replacements for historical Radio regression cases.
// Real Gemini controller, action runner, DataLayerManager and RealtimeRadio consumers.
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
import { createGevActionRunner } from './gevActions.js';
import { createVoiceSession } from './session.js';
import { DataLayerManager } from '../data/manager.js';
const viewerFixture = () => ({
clock: { onTick: { addEventListener: () => () => {} } },
camera: { moveEnd: { addEventListener() {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
});
const request = (ws, id, name, args = {}) =>
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
const responses = (ws) =>
ws.sent.flatMap((frame) => frame.toolResponse?.functionResponses || []);
const responseFor = (ws, id) =>
responses(ws).findLast((response) => response.id === id)?.response;
const speechTurn = (ws) => {
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
};
function drainRadioReply(h) {
h.sockets.at(-1).message({
serverContent: {
modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] },
turnComplete: true,
},
});
h.captures.at(-1).drain();
}
function radioSiblingFixture() {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const lookupEntered = deferred();
const lookupReleased = deferred();
const state = {
stationCount: 2,
selected: { id: 'existing-station', name: 'Existing Station' },
audioState: 'stopped',
};
const trace = { stops: 0, pauses: 0, selections: 0, options: new Map() };
const radio = {
async init() {},
async enable() {},
async disable() {},
async update() {},
getStats: () => ({ count: 2 }),
getUIState: () => ({ ...state }),
setVoiceDucked() {},
pause({ origin } = {}) {
if (origin === 'voice') trace.pauses++;
state.audioState = 'paused';
return true;
},
stopPlayback() {
trace.stops++;
state.audioState = 'stopped';
return true;
},
selectRequestedStation() {
trace.selections++;
state.selected = { id: 'late-station', name: 'Late Station' };
return state.selected;
},
subscribePlaybackControls: () => () => {},
};
manager.register(
Object.assign(radio, {
id: 'radio',
name: 'Radio',
updateInterval: -1,
}),
);
const runner = createGevActionRunner({
viewer,
dataManager: manager,
placeSearch: {
async geocode(_query, { signal }) {
trace.lookupSignal = signal;
lookupEntered.resolve();
// Deliberately ignore abort: production currentness must still prevent
// this late service completion from committing a station selection.
await lookupReleased.promise;
return { place: { lat: 10, lng: 20, label: 'Deferred Place' } };
},
},
});
const h = harness({
dataManager: manager,
radioLayer: radio,
runAction(name, args, options) {
trace.options.set(options.callId, options);
return runner(name, args, options);
},
});
return {
h,
manager,
runner,
radio,
trace,
state,
lookupEntered,
lookupReleased,
};
}
for (const action of ['select', 'play']) {
for (const stronger of ['stop', 'pause', 'disable', 'off']) {
test(`native same-turn Radio ${stronger} preempts deferred ${action} without cancelling a query sibling`, async () => {
const f = radioSiblingFixture();
const { h, trace, state } = f;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
const turn = h.c.turn;
request(ws, 'slow-selection', 'control_radio', {
action,
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'query-sibling', 'control_radio', { action: 'status' });
request(
ws,
'stronger-radio',
stronger === 'off' ? 'set_layer_visibility' : 'control_radio',
stronger === 'off'
? { layerId: 'radio', enabled: false }
: { action: stronger },
);
await tick();
assert.equal(
responseFor(ws, 'stronger-radio')?.ok,
true,
'stronger control must finish before lookup release',
);
assert.equal(trace.lookupSignal.aborted, true);
await h.c.toolQueue;
assert.equal(
h.c.turn,
turn,
'sibling control cannot invent a new turn',
);
assert.equal(trace.selections, 0);
assert.equal(state.selected.id, 'existing-station');
assert.equal(responseFor(ws, 'slow-selection').ok, false);
assert.match(responseFor(ws, 'slow-selection').error, /cancelled/);
assert.equal(
responseFor(ws, 'slow-selection').enabled,
!['disable', 'off'].includes(stronger),
);
assert.equal(responseFor(ws, 'query-sibling').ok, true);
assert.equal(trace.options.get('query-sibling').signal.aborted, false);
assert.equal(
responseFor(ws, 'stronger-radio').enabled,
!['disable', 'off'].includes(stronger),
);
assert.equal(trace.stops, stronger === 'stop' ? 1 : 0);
assert.equal(trace.pauses, stronger === 'pause' ? 1 : 0);
f.lookupReleased.resolve();
await tick();
assert.equal(trace.selections, 0, 'late lookup cannot commit');
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
for (const id of ['slow-selection', 'query-sibling', 'stronger-radio'])
assert.equal(responses(ws).filter((r) => r.id === id).length, 1);
} finally {
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
}
for (const guard of ['malformed', 'replay', 'invalid-arguments']) {
test(`native stronger Radio ${guard} cannot revoke a same-turn lookup`, async () => {
const f = radioSiblingFixture();
const { h, trace } = f;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
if (guard === 'replay') {
request(ws, 'stronger-radio', 'control_radio', { action: 'stop' });
await h.c.toolQueue;
}
speechTurn(ws);
if (guard === 'replay')
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
const stops = trace.stops;
const handoffEpoch = h.c.radio.radioHandoffEpoch;
request(
ws,
guard === 'malformed' ? '\n' : 'stronger-radio',
'control_radio',
{
action: 'stop',
...(guard === 'invalid-arguments'
? { latitude: 91, longitude: 0 }
: {}),
},
);
await tick();
assert.equal(trace.lookupSignal.aborted, false);
assert.equal(trace.stops, stops);
assert.equal(h.c.radio.radioHandoffEpoch, handoffEpoch);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
if (guard === 'invalid-arguments') {
assert.equal(responseFor(ws, 'stronger-radio').ok, false);
assert.match(responseFor(ws, 'stronger-radio').error, /coordinates/);
}
f.lookupReleased.resolve();
await h.c.toolQueue;
assert.equal(
trace.selections,
1,
'older eligible work retains authority',
);
assert.equal(responseFor(ws, 'slow-selection').ok, true);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
} finally {
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
test('native failed Stop releases its reservation and lets the older selection finish', async () => {
const f = radioSiblingFixture();
const { h, trace } = f;
const stopReleased = deferred();
f.radio.stopPlayback = () => stopReleased.promise;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'failed-stop', 'control_radio', { action: 'stop' });
await tick();
assert.equal(h.c.radio.radioToolHandoffReservations.size, 1);
assert.equal(
trace.lookupSignal.aborted,
false,
'provisional Stop cannot revoke work',
);
assert.equal(h.c.pendingTools, 2);
stopReleased.resolve(false);
await tick();
assert.equal(responseFor(ws, 'failed-stop').ok, false);
assert.match(responseFor(ws, 'failed-stop').error, /could not be stopped/);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
assert.equal(trace.lookupSignal.aborted, false);
f.lookupReleased.resolve();
await h.c.toolQueue;
assert.equal(responseFor(ws, 'slow-selection').ok, true);
assert.equal(trace.selections, 1);
assert.equal(f.state.selected.id, 'late-station');
assert.equal(h.c.pendingTools, 0);
} finally {
stopReleased.resolve(false);
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
test('native pending Stop preserves selection completed before its own successful settlement', async () => {
const f = radioSiblingFixture();
const { h } = f;
const stopReleased = deferred();
f.radio.stopPlayback = () => stopReleased.promise;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'pending-stop', 'control_radio', { action: 'stop' });
await tick();
f.lookupReleased.resolve();
await tick();
assert.equal(responseFor(ws, 'slow-selection').ok, true);
assert.equal(f.state.selected.id, 'late-station');
assert.equal(h.c.radio.radioToolHandoffReservations.size, 1);
assert.ok(h.c.radio.pendingRadioPlaybackResult);
stopReleased.resolve(true);
await h.c.toolQueue;
assert.equal(responseFor(ws, 'pending-stop').ok, true);
assert.equal(responseFor(ws, 'pending-stop').stationId, 'late-station');
assert.equal(f.state.selected.id, 'late-station');
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
assert.equal(h.c.pendingTools, 0);
} finally {
stopReleased.resolve(false);
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
for (const restart of [false, true]) {
test(`native Stop${restart ? '/restart' : ''} retires concurrent stronger Radio work without late output or count underflow`, async () => {
const f = radioSiblingFixture();
const { h } = f;
const stopReleased = deferred();
f.radio.stopPlayback = () => stopReleased.promise;
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'slow-selection', 'control_radio', {
action: 'select',
locationQuery: 'Deferred Place',
});
await f.lookupEntered.promise;
request(ws, 'pending-stop', 'control_radio', { action: 'stop' });
await tick();
const oldDrain = h.c.toolQueue;
const stopOptions = f.trace.options.get('pending-stop');
assert.equal(h.c.pendingTools, 2);
h.c.stop();
assert.equal(stopOptions.signal.aborted, true);
assert.equal(f.trace.lookupSignal.aborted, true);
if (restart) {
await h.c.start();
request(h.sockets[1], 'new-query', 'control_radio', {
action: 'status',
});
await h.c.toolQueue;
assert.equal(responseFor(h.sockets[1], 'new-query').ok, true);
}
stopReleased.resolve(true);
f.lookupReleased.resolve();
await oldDrain;
await tick();
assert.equal(responseFor(ws, 'pending-stop'), undefined);
assert.equal(responseFor(ws, 'slow-selection'), undefined);
assert.equal(f.trace.selections, 0);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
assert.equal(h.c.isActive(), restart);
} finally {
stopReleased.resolve(false);
f.lookupReleased.resolve();
h.c.stop();
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
for (const phase of ['prepared', 'preflight']) {
test(`native same-turn Stop revokes only voice-owned ${phase} playback and preserves completed station`, async () => {
const f = radioSiblingFixture();
const { h } = f;
const ready = deferred();
const stopped = [];
let preflight;
f.radio.playForVoice = () => ready.promise;
f.radio.stopPlayback = (options) => {
stopped.push(options);
return true;
};
try {
await f.manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
request(ws, 'prepared-play', 'control_radio', { action: 'play' });
await h.c.toolQueue;
assert.equal(
responseFor(ws, 'prepared-play').radioPlaybackRequested,
true,
);
if (phase === 'preflight')
preflight = h.c.radio.startPendingRadioHandoff();
const attemptId = h.c.radio.radioHandoffAttemptId;
request(ws, 'stronger-stop', 'control_radio', { action: 'stop' });
await h.c.toolQueue;
assert.equal(responseFor(ws, 'stronger-stop').ok, true);
assert.equal(f.state.selected.id, 'existing-station');
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
assert.equal(h.c.radio.radioHandoffInFlight, false);
const cleanups = stopped.filter((o) => o.origin === 'voice-cleanup');
assert.deepEqual(
cleanups,
phase === 'preflight' ? [{ origin: 'voice-cleanup', attemptId }] : [],
);
ready.resolve(true);
await preflight;
await tick();
assert.equal(
h.c.isActive(),
true,
'late old readiness cannot stop voice',
);
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
assert.equal(h.c.pendingTools, 0);
} finally {
ready.resolve(false);
h.c.stop();
await preflight;
f.runner.dispose();
await f.manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
for (const phase of ['prepared', 'preflight', 'settling']) {
test(`native Radio cancellation: ${phase} preserves selection truth and prevents stale speaker transfer`, async () => {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const prepared = deferred();
const settleResult = deferred();
const playbackReady = deferred();
let playCalls = 0;
let stopCalls = 0;
let selections = 0;
let activeOptions;
const state = {
stationCount: 2,
selected: null,
audioState: 'stopped',
voiceDucked: false,
};
const radio = {
async init() {},
async enable() {},
async disable() {},
async update() {},
getStats: () => ({ count: 2 }),
getUIState: () => ({ ...state }),
setVoiceDucked(value) {
state.voiceDucked = value;
},
pause() {
state.audioState = 'paused';
},
subscribePlaybackControls() {
return () => {};
},
selectRequestedStation(_criteria, options) {
assert.equal(options.autoplay, false);
selections++;
state.selected = { id: 'station-1', name: 'Station One' };
return state.selected;
},
playForVoice() {
playCalls++;
return playbackReady.promise;
},
stopPlayback() {
stopCalls++;
state.audioState = 'stopped';
},
};
manager.register({
id: 'radio',
name: 'Radio',
updateInterval: -1,
...radio,
});
const runner = createGevActionRunner({ viewer, dataManager: manager });
let h;
let preflight;
const session = createVoiceSession({
runner: async (name, args, options) => {
activeOptions = options;
const result = await runner(name, args, options);
prepared.resolve(result);
if (phase === 'settling') await settleResult.promise;
return result;
},
createAdapter: (shared) => {
h = harness({ ...shared, dataManager: manager, radioLayer: radio });
return {
start: (settings) => h.c.start(settings),
stop: (settings) => h.c.stop(settings),
sendText: (text) => h.c.sendTextCommand(text),
sendMapEvent: (event) => h.c.notifyMapEvent(event),
};
},
});
try {
await session.start();
const ws = h.sockets[0];
assert.equal(session.sendText('Select Station One on Radio'), true);
request(ws, 'prepared-radio', 'control_radio', {
action: 'select',
stationQuery: 'Station One',
});
const actual = await prepared.promise;
assert.equal(actual.ok, true);
assert.equal(actual.radioPlaybackRequested, true);
assert.equal(actual.stationId, 'station-1');
assert.equal(manager.isEnabled('radio'), true);
if (phase !== 'settling') {
await h.c.toolQueue;
assert.ok(h.c.radio.pendingRadioPlaybackResult);
}
if (phase === 'preflight') {
preflight = h.c.radio.startPendingRadioHandoff();
assert.equal(h.c.radio.radioHandoffInFlight, true);
assert.equal(playCalls, 1);
}
// Native call ownership revokes only this deferred playback. No caption
// parsing or caption-driven provider policy participates in cancellation.
ws.message({ toolCallCancellation: { ids: ['prepared-radio'] } });
if (phase === 'settling')
assert.equal(activeOptions.signal.aborted, true);
settleResult.resolve();
await h.c.toolQueue;
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
assert.equal(h.c.radio.radioHandoffInFlight, false);
assert.equal(manager.isEnabled('radio'), true);
assert.equal(state.selected.id, 'station-1');
assert.equal(selections, 1);
const modelResult = responseFor(ws, 'prepared-radio');
if (phase === 'settling')
assert.equal(
modelResult,
undefined,
'cancelled active call never sends a late result',
);
else {
assert.equal(modelResult.ok, true);
assert.equal(modelResult.stationId, 'station-1');
}
const stopsAfterCancellation = stopCalls;
assert.equal(stopsAfterCancellation, phase === 'preflight' ? 1 : 0);
drainRadioReply(h);
playbackReady.resolve(true);
await preflight;
await tick();
assert.equal(
playCalls,
phase === 'preflight' ? 1 : 0,
'drain cannot revive cancelled playback',
);
assert.equal(h.c.isActive(), true, 'late readiness cannot stop voice');
assert.equal(h.c.radio.radioHandoffInFlight, false);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.tools.size, 0);
assert.equal(session.sendText('Play radio'), true);
request(ws, 'fresh-radio', 'control_radio', { action: 'play' });
await h.c.toolQueue;
assert.ok(h.c.radio.pendingRadioPlaybackResult);
drainRadioReply(h);
await tick();
assert.equal(playCalls, phase === 'preflight' ? 2 : 1);
assert.equal(
h.c.isActive(),
false,
'fresh typed Radio handoff alone ends voice',
);
} finally {
settleResult.resolve();
playbackReady.resolve(false);
session.destroy();
await preflight;
runner.dispose();
await manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
// A stronger Radio control skips the ordered queue. An older Radio call still
// queued behind a slow sibling must not start after that control succeeded,
// while a failed control or a later call in the same turn keeps its order.
for (const [label, ids, stopOk, expectPending] of [
[
'older queued select is revoked by a later successful stop',
['slow', 'sel', 'stop'],
true,
false,
],
[
'a failed stop leaves the older queued select current',
['slow', 'sel', 'stop'],
false,
true,
],
[
'a select after the stop in the same turn still runs',
['slow', 'stop', 'sel'],
true,
true,
],
]) {
test(`stronger Radio ordering: ${label}`, async () => {
const viewer = viewerFixture();
const manager = new DataLayerManager(viewer);
const state = {
stationCount: 2,
selected: { id: 'existing', name: 'Existing' },
audioState: 'stopped',
};
const radio = {
id: 'radio',
name: 'Radio',
updateInterval: -1,
async init() {},
async enable() {},
async disable() {},
async update() {},
getStats: () => ({ count: 2 }),
getUIState: () => ({ ...state }),
setVoiceDucked() {},
pause() {
state.audioState = 'paused';
return true;
},
stopPlayback() {
state.audioState = 'stopped';
return stopOk;
},
selectRequestedStation() {
state.selected = { id: 'late', name: 'Late' };
return state.selected;
},
async playForVoice() {
return true;
},
subscribePlaybackControls: () => () => {},
};
manager.register(radio);
const runner = createGevActionRunner({
viewer,
dataManager: manager,
placeSearch: {
async geocode() {
return { place: { lat: 10, lng: 20, label: 'Place' } };
},
},
});
const slow = deferred();
const h = harness({
dataManager: manager,
radioLayer: radio,
async runAction(name, args, options) {
if (name === 'slow_sibling') {
await slow.promise;
return { ok: true, action: 'slow_sibling' };
}
if (name === 'control_radio' && args.action === 'stop' && !stopOk)
return {
ok: false,
action: 'control_radio',
error: 'Radio stop failed',
};
return runner(name, args, options);
},
});
const calls = {
slow: { id: 'slow', name: 'slow_sibling', args: {} },
sel: {
id: 'sel',
name: 'control_radio',
args: { action: 'select', locationQuery: 'Place' },
},
stop: { id: 'stop', name: 'control_radio', args: { action: 'stop' } },
};
try {
await manager.setEnabled('radio', true);
await h.c.start();
const ws = h.sockets[0];
speechTurn(ws);
ws.message({ toolCall: { functionCalls: ids.map((id) => calls[id]) } });
await tick();
assert.equal(
responseFor(ws, 'stop')?.ok,
stopOk,
'stop settles without waiting',
);
slow.resolve();
await h.c.toolQueue;
await tick();
assert.equal(
Boolean(h.c.radio.pendingRadioPlaybackResult),
expectPending,
);
assert.equal(responseFor(ws, 'sel')?.ok, expectPending);
} finally {
slow.resolve();
h.c.stop();
runner.dispose();
await manager.setEnabled('radio', false);
}
assert.equal(h.timers.size, 0);
});
}
+280
View File
@@ -0,0 +1,280 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
harness,
tick,
deferred,
} from '../testSupport/geminiNativeHarness.mjs';
import { createGevActionRunner } from './gevActions.js';
const tool = (ws, id) =>
ws.message({
toolCall: {
functionCalls: [
{
id,
name: 'adjust_camera_zoom',
args: { direction: 'in', amount: 'little' },
},
],
},
});
const caption = (ws, text) =>
ws.message({ serverContent: { inputTranscription: { text } } });
const onset = (ws) => {
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
};
const complete = (ws) => ws.message({ serverContent: { turnComplete: true } });
const oldAudio = {
serverContent: {
modelTurn: {
parts: [
{
inlineData: {
data: 'AAAA',
mimeType: 'audio/pcm;rate=24000',
},
},
],
},
},
};
const outputs = (ws) =>
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
// Socket/media are deterministic. Mutations use the production action runner.
function fixture({ holdFirst = false } = {}) {
const state = { height: 1000, renders: 0, providerWrites: [], panels: [] };
const position = { x: 6379137, y: 0, z: 0 };
const viewer = {
clock: { onTick: { addEventListener: () => () => {} } },
camera: {
positionWC: position,
positionCartographic: { height: 1000 },
moveEnd: { addEventListener() {} },
cancelFlight() {},
zoomIn(n) {
position.x -= n;
state.height -= n;
this.positionCartographic.height = state.height;
},
zoomOut(n) {
position.x += n;
state.height += n;
this.positionCartographic.height = state.height;
},
},
scene: {
canvas: {
clientWidth: 800,
clientHeight: 600,
addEventListener() {},
removeEventListener() {},
},
requestRender() {
state.renders++;
},
},
};
const runner = createGevActionRunner({
viewer,
dataManager: { layers: new Map() },
styleManager: {
setVoiceProvider: (p) => state.providerWrites.push(p),
setPanelCollapsed: (...args) => state.panels.push(args),
},
});
const entered = [];
const gate = deferred();
const h = harness({
runAction: async (name, args, options) => {
entered.push({ name, args, options });
if (holdFirst && entered.length === 1) await gate.promise;
if (!options.isCurrent())
return { ok: false, action: name, changed: false };
return runner(name, args, options);
},
});
return {
...h,
state,
entered,
gate,
close() {
gate.resolve();
h.c.stop();
runner.dispose();
assert.equal(h.timers.size, 0);
assert.deepEqual(state.providerWrites, []);
assert.deepEqual(state.panels, []);
},
};
}
for (const [id, order] of [
['H20', 'before-boundary'],
['H21', 'after-boundary'],
]) {
test(`native replacement ${id}: ${order} tool respects retired output and next onset recovers`, async () => {
const h = fixture();
try {
await h.c.start();
const ws = h.sockets[0];
ws.message(oldAudio);
onset(ws);
caption(ws, 'zoom in');
if (order === 'before-boundary') {
tool(ws, 'ambiguous-before');
await tick();
assert.equal(h.state.height, 1000);
assert.equal(h.entered.length, 0);
assert.equal(outputs(ws).length, 0);
}
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
if (order === 'after-boundary') {
tool(ws, 'native-after');
await h.c.toolQueue;
assert.equal(h.state.height, 750);
assert.equal(h.entered.length, 1);
assert.equal(outputs(ws).at(-1).response.ok, true);
complete(ws);
}
onset(ws);
caption(ws, 'zoom in');
tool(ws, 'fresh-clean-onset');
await h.c.toolQueue;
assert.equal(h.state.height, order === 'before-boundary' ? 750 : 562.5);
assert.equal(h.entered.length, order === 'before-boundary' ? 1 : 2);
assert.equal(h.entered.at(-1).options.userTurn, undefined);
assert.equal(h.c.pendingTools, 0);
assert.equal(
outputs(ws).some((o) => o.id === 'ambiguous-before'),
false,
);
assert.equal(
outputs(ws).find((o) => o.id === 'fresh-clean-onset').response.ok,
true,
);
} finally {
h.close();
}
});
}
test('native replacement H30: ordinary tools run promptly and late caption fragments cannot strand siblings', async () => {
const h = fixture();
try {
await h.c.start();
const ws = h.sockets[0];
onset(ws);
caption(ws, 'switch to');
tool(ws, 'ordinary-before-caption-resolution');
await tick();
await h.c.toolQueue;
assert.equal(h.state.height, 750);
assert.equal(h.c.pendingTools, 0);
h.advance(1499);
caption(ws, 'Austin');
h.advance(5000);
tool(ws, 'released-sibling');
await h.c.toolQueue;
assert.equal(h.state.height, 562.5);
assert.equal(h.entered.length, 2);
assert.equal(
h.entered.every((c) => c.options.userTurn === undefined),
true,
);
assert.equal(outputs(ws).length, 2);
assert.equal(
outputs(ws).every((o) => o.response.ok),
true,
);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.status, 'listening');
} finally {
h.close();
}
});
for (const [id, exit] of [
['H32', 'stop'],
['H33', 'restart'],
['H34', 'new-input'],
]) {
test(`native replacement ${id}: ${exit} retires owned in-flight work and late callbacks`, async () => {
const h = fixture({ holdFirst: true });
try {
await h.c.start();
let ws = h.sockets[0];
onset(ws);
caption(ws, 'switch to');
tool(ws, 'old-owned-action');
await tick();
assert.equal(h.entered.length, 1);
assert.equal(h.c.pendingTools, 1);
assert.equal(h.state.height, 1000);
const oldWork = h.c.toolQueue;
const oldMessage = ws.onmessage;
const oldOwner = h.entered[0].options;
if (exit === 'new-input') h.c.sendTextCommand('zoom out');
else h.c.stop();
assert.equal(oldOwner.signal.aborted, true);
assert.equal(oldOwner.isCurrent(), false);
if (exit === 'restart') {
await h.c.start();
ws = h.sockets.at(-1);
}
if (exit !== 'new-input')
oldMessage({
data: JSON.stringify({
toolCall: {
functionCalls: [
{
id: 'late-old-call',
name: 'adjust_camera_zoom',
args: { direction: 'in' },
},
],
},
}),
});
caption(ws, 'Gemini');
h.advance(1500);
h.gate.resolve();
await oldWork;
await tick();
assert.equal(h.state.height, 1000);
assert.equal(h.c.pendingTools, 0);
assert.equal(
outputs(ws).some(
(o) => o.id === 'old-owned-action' || o.id === 'late-old-call',
),
false,
);
if (exit === 'stop') {
assert.equal(h.c.status, 'idle');
assert.equal(h.c.isActive(), false);
} else {
if (exit === 'new-input') {
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
} else h.c.sendTextCommand('zoom out');
tool(ws, 'fresh-after-retirement');
await h.c.toolQueue;
assert.equal(h.state.height, 750);
assert.equal(h.entered.length, 2);
assert.equal(
outputs(ws).find((o) => o.id === 'fresh-after-retirement').response
.ok,
true,
);
assert.equal(h.c.pendingTools, 0);
assert.equal(h.c.isActive(), true);
}
} finally {
h.close();
}
});
}
+529
View File
@@ -0,0 +1,529 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
nativeIntegrationFixture,
assertSuccessfulZoom,
turnOfEventLoop,
} from '../testSupport/geminiIntegrationFixture.mjs';
const options = { concurrency: false, timeout: 5000 };
const OLD_TEXT = 'Old packet caption.';
const OLD_PCM = 'AAABAA==';
const POST_TOOL_PCM = 'AgADAA==';
const OLD_CALL = 'old-coalesced-call';
function coalescedPacket(callId = OLD_CALL) {
return {
serverContent: {
outputTranscription: { text: OLD_TEXT },
modelTurn: {
parts: [
{ inlineData: { data: OLD_PCM, mimeType: 'audio/pcm;rate=24000' } },
],
},
turnComplete: true,
},
toolCall: {
functionCalls: [
{
id: callId,
name: 'adjust_camera_zoom',
args: { direction: 'in', amount: 'little' },
},
],
},
};
}
// Local test observations wrap only injected device fakes and the actual
// scheduler methods, delegating unchanged. No frozen fixture file is modified.
function observePacketEffects(h) {
const played = [];
const narration = { audible: 0, toolStarted: 0, toolFinished: 0 };
const createAudio = h.controller.createAudio;
h.controller.createAudio = (callbacks) => {
const audio = createAudio(callbacks);
const play = audio.play.bind(audio);
audio.play = (data, mimeType) => {
played.push({ audio, data, mimeType });
return play(data, mimeType);
};
return audio;
};
for (const method of Object.keys(narration)) {
const invoke = h.controller.narration[method].bind(h.controller.narration);
h.controller.narration[method] = (...args) => {
narration[method]++;
return invoke(...args);
};
}
return { played, narration };
}
function packetState(h) {
const c = h.controller;
return {
epoch: c.epoch,
inputGeneration: c.inputGeneration,
turn: c.turn,
completedTurns: c.completedTurns,
pendingTools: c.pendingTools,
modelComplete: c.modelComplete,
pendingCompletionStatus: c.pendingCompletionStatus,
postToolOutput: c.postToolOutput,
modelSpoke: c.modelSpoke,
assistant: c.transcripts.assistant,
replyPending: c.replyPending,
retiredReplyPending: c.retiredReplyPending,
typedText: c.typedIntent?.text ?? null,
typedResponseId: c.typedIntent?.responseId ?? null,
narrationTools: c.narration.state().tools,
narrationTimers: c.narration.state().pendingTimers,
ownedTimers: h.clock.timers.size,
};
}
const completions = (h) =>
h.events.filter((event) => event.type === 'completion');
const toolEvents = (h, callId) =>
h.events.filter(
(event) =>
['action-call', 'action-result', 'progress'].includes(event.type) &&
event.callId === callId,
);
function assertOldRemainderAbsent(h, observed) {
assert.equal(h.state.height, 1000);
assert.deepEqual(h.state.zooms, []);
assert.deepEqual(h.runnerCalls, []);
assert.deepEqual(toolEvents(h, OLD_CALL), []);
assert.deepEqual(observed.played, []);
assert.deepEqual(observed.narration, {
audible: 0,
toolStarted: 0,
toolFinished: 0,
});
assert.deepEqual(completions(h), []);
assert.equal(h.controller.completedTurns, 0);
assert.equal(h.controller.pendingTools, 0);
assert.equal(
h.events.filter(
(event) =>
event.type === 'transcript' &&
event.role === 'assistant' &&
event.final &&
event.text === OLD_TEXT,
).length,
0,
);
for (const socket of h.sockets)
assert.equal(
h.outputs(socket).some((output) => output.id === OLD_CALL),
false,
);
}
for (const mode of ['Stop', 'Stop then Start', 'typed successor']) {
test(
`assistant transcript observer ${mode} fences the rest of a coalesced native packet`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const observed = observePacketEffects(h);
const oldSocket = await h.start();
const oldAudio = h.audios[0];
h.activity(oldSocket);
const before = packetState(h);
let reached = false;
let restart;
let typedAccepted;
let afterObserver;
const remove = h.session.subscribe((event) => {
if (
reached ||
event.type !== 'transcript' ||
event.role !== 'assistant' ||
event.final ||
event.text !== OLD_TEXT
)
return;
reached = true;
if (mode === 'typed successor') {
typedAccepted = h.session.sendText('Zoom in for the typed successor');
} else {
h.session.stop();
if (mode === 'Stop then Start') restart = h.session.start();
}
afterObserver = packetState(h);
});
oldSocket.message(coalescedPacket());
remove();
assert.equal(
reached,
true,
'the actual common transcript observer ran synchronously',
);
// A retired reply may consume its own received terminal, but cannot
// complete or overwrite the successor. Stop/restart permits no old writes.
const afterPacketExpected =
mode === 'typed successor'
? {
...afterObserver,
retiredReplyPending: false,
ownedTimers: afterObserver.ownedTimers - 1,
}
: afterObserver;
assert.deepEqual(packetState(h), afterPacketExpected);
await h.controller.toolQueue;
await turnOfEventLoop();
if (restart) await restart;
assertOldRemainderAbsent(h, observed);
let successorSocket;
if (mode === 'typed successor') {
assert.equal(typedAccepted, true);
assert.equal(
h.controller.epoch,
before.epoch,
'typed replacement shares the socket epoch',
);
assert.ok(h.controller.inputGeneration > before.inputGeneration);
assert.ok(h.controller.turn > before.turn);
assert.deepEqual(packetState(h), afterPacketExpected);
assert.equal(
h.controller.typedIntent.text,
'Zoom in for the typed successor',
);
assert.equal(h.controller.replyPending, true);
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(oldAudio.closed, false);
const typedMessages = oldSocket.sent.filter((message) =>
message.clientContent?.turns?.some((turn) =>
turn.parts?.some(
(part) => part.text === 'Zoom in for the typed successor',
),
),
);
assert.equal(typedMessages.length, 1);
// The received packet already carried the old terminal. No extra
// provider boundary may be invented before testing the actual successor.
assert.equal(h.controller.retiredBoundaryWait, null);
assert.equal(h.controller.cancelledCalls.has(OLD_CALL), true);
assert.equal(h.controller.replyPending, true);
assert.equal(h.controller.completedTurns, 0);
assert.deepEqual(completions(h), []);
successorSocket = oldSocket;
} else {
assert.equal(oldSocket.closed, true);
assert.equal(oldAudio.closed, true);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
if (mode === 'Stop') {
assert.equal(h.session.state, 'idle');
successorSocket = await h.start();
} else {
assert.equal(h.session.state, 'listening');
assert.equal(h.sockets.length, 2);
assert.equal(h.audios.length, 2);
assert.notEqual(h.audios[1], oldAudio);
assert.equal(h.audios[1].closed, false);
assert.equal(h.audios[1].pending, false);
successorSocket = h.sockets[1];
}
h.activity(successorSocket);
}
await h.tool('owned-after-packet-cancel', successorSocket);
assertSuccessfulZoom(
h,
successorSocket,
'owned-after-packet-cancel',
1000,
);
assert.equal(h.state.zooms.length, 1);
assert.equal(h.runnerCalls.length, 1);
assert.equal(
h.outputs(successorSocket).some((output) => output.id === OLD_CALL),
false,
);
h.complete(successorSocket);
assert.equal(completions(h).length, 1);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
},
);
}
test(
'ordinary coalesced packet plays and executes once, then completes only after its post-tool reply',
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const observed = observePacketEffects(h);
const ws = await h.start();
h.activity(ws);
ws.message(coalescedPacket('ordinary-coalesced'));
assert.equal(
h.controller.completedTurns,
1,
'first generation terminal is counted once',
);
assert.equal(h.controller.pendingTools, 1);
assert.deepEqual(
observed.played.map(({ data }) => data),
[OLD_PCM],
);
assert.equal(observed.played[0].audio, h.audios[0]);
assert.deepEqual(completions(h), [], 'tool and playback are still pending');
await h.controller.toolQueue;
assertSuccessfulZoom(h, ws, 'ordinary-coalesced', 1000);
assert.equal(h.controller.pendingTools, 0);
assert.equal(observed.narration.toolStarted, 1);
assert.equal(observed.narration.toolFinished, 1);
assert.equal(h.controller.narration.state().tools, 0);
h.audios[0].drain();
assert.equal(
h.controller.modelComplete,
false,
'function response waits for the post-tool generation',
);
assert.equal(h.controller.completedTurns, 1);
assert.deepEqual(
completions(h),
[],
'draining first-generation PCM cannot finish the tool reply',
);
// The accepted contract has two generations: the function response prompts
// a fresh model reply. This boundary is intentional, not a stale-packet fix.
ws.message({
serverContent: {
outputTranscription: { text: 'The requested zoom is complete.' },
modelTurn: {
parts: [
{
inlineData: {
data: POST_TOOL_PCM,
mimeType: 'audio/pcm;rate=24000',
},
},
],
},
turnComplete: true,
},
});
assert.equal(h.controller.completedTurns, 2);
assert.deepEqual(
observed.played.map(({ data }) => data),
[OLD_PCM, POST_TOOL_PCM],
);
assert.deepEqual(
completions(h),
[],
'post-tool playback must drain before completion',
);
h.audios[0].drain();
assert.equal(completions(h).length, 1);
assert.equal(completions(h)[0].status, 'completed');
assert.equal(completions(h)[0].spoken, true);
assert.equal(completions(h)[0].audioDrained, true);
assert.equal(h.controller.pendingCompletionStatus, null);
assert.equal(h.controller.narration.state().tools, 0);
assert.equal(h.controller.narration.state().pendingTimers, 0);
assert.equal(h.clock.timers.size, 0);
assert.equal(observed.narration.audible, 2);
assert.equal(h.runnerCalls.length, 1);
assert.equal(h.state.zooms.length, 1);
assert.equal(h.outputs(ws).length, 1);
h.audios[0].drain();
assert.equal(
completions(h).length,
1,
'a duplicate device drain cannot complete twice',
);
},
);
// Real native message-type fields are separate packets here; no extra old
// completion is inserted before the accepted successor.
const BOUNDARY_OLD = 'Prior reply caption.';
const BOUNDARY_NEXT = 'Zoom in for the typed successor';
const boundaryZoomCall = (id) => ({
toolCall: {
functionCalls: [
{
id,
name: 'adjust_camera_zoom',
args: { direction: 'in', amount: 'little' },
},
],
},
});
function boundarySnapshot(h) {
const c = h.controller;
return {
retiredReplyPending: c.retiredReplyPending,
retiredBoundaryWait: Boolean(c.retiredBoundaryWait),
replyPending: c.replyPending,
typedText: c.typedIntent?.text ?? null,
completions: h.events.filter((event) => event.type === 'completion').length,
};
}
for (const mode of [
'partial assistant transcript',
'input transcript',
'interruption event',
'listening event',
'final assistant transcript control',
]) {
test(
`received native terminal preserves typed successor after ${mode}`,
{ concurrency: false, timeout: 5000 },
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
h.activity(ws);
let oldDispatch, gate;
const interruptionMode =
mode === 'interruption event' || mode === 'listening event';
if (interruptionMode) {
h.session.sendText('Prior zoom request');
gate = h.holdResult('already-committed-old-tool');
oldDispatch = h.tool('already-committed-old-tool', ws);
await gate.ready;
} else if (mode === 'input transcript') {
ws.message({
serverContent: {
outputTranscription: { text: 'Previously emitted output' },
},
});
}
let entered = 0,
accepted;
const remove = h.session.subscribe((event) => {
if (entered) return;
const matches =
mode === 'interruption event'
? event.type === 'interruption' && event.reason === 'user-speech'
: mode === 'listening event'
? event.type === 'state' && event.state === 'listening'
: mode === 'input transcript'
? event.type === 'transcript' &&
event.role === 'user' &&
event.text === 'Old input caption'
: event.type === 'transcript' &&
event.role === 'assistant' &&
event.text === BOUNDARY_OLD &&
event.final ===
(mode === 'final assistant transcript control');
if (!matches) return;
entered++;
accepted = h.session.sendText(BOUNDARY_NEXT);
});
if (interruptionMode)
ws.message({
serverContent: { interrupted: true, turnComplete: true },
});
else
ws.message({
serverContent: {
...(mode === 'input transcript'
? { inputTranscription: { text: 'Old input caption' } }
: {}),
outputTranscription: { text: BOUNDARY_OLD },
turnComplete: true,
},
});
remove();
if (oldDispatch) {
await oldDispatch;
gate.release();
}
await turnOfEventLoop();
const afterPacket = boundarySnapshot(h);
const beforeSuccessorHeight = h.state.height;
// No extra synthetic old boundary. This is the next reply to the accepted text.
ws.message({
serverContent: {
outputTranscription: { text: 'Handling your new zoom.' },
},
});
ws.message(boundaryZoomCall('typed-successor-call'));
await h.controller.toolQueue;
await turnOfEventLoop();
ws.message({ serverContent: { turnComplete: true } });
const afterSuccessor = boundarySnapshot(h);
assert.equal(entered, 1);
assert.equal(accepted, true);
assert.equal(
afterPacket.retiredReplyPending,
false,
'received old terminal must settle its retirement',
);
assert.equal(afterPacket.retiredBoundaryWait, false);
assert.equal(afterPacket.typedText, BOUNDARY_NEXT);
assert.equal(
h.runnerCalls.filter((x) => x.options.callId === 'typed-successor-call')
.length,
1,
'accepted typed successor must reach the actual runner',
);
assert.equal(h.state.height, beforeSuccessorHeight * 0.75);
assert.equal(afterSuccessor.completions, 1);
assert.equal(afterSuccessor.replyPending, false);
assert.equal(
h.controller.cancelledCalls.has('typed-successor-call'),
false,
);
},
);
}
for (const terminalAlreadyApplied of [false, true]) {
test(
`typed reentry retains a genuinely pending boundary when terminal already applied is ${terminalAlreadyApplied}`,
options,
async (t) => {
const h = nativeIntegrationFixture(t);
const ws = await h.start();
h.activity(ws);
let entered = false;
const remove = h.session.subscribe((event) => {
if (
entered ||
event.type !== 'transcript' ||
event.role !== 'assistant' ||
event.text !== 'Boundary control' ||
event.final !== terminalAlreadyApplied
)
return;
entered = true;
assert.equal(h.session.sendText('First typed replacement'), true);
if (terminalAlreadyApplied)
assert.equal(h.session.sendText('Second typed replacement'), true);
});
ws.message({
serverContent: {
outputTranscription: { text: 'Boundary control' },
...(terminalAlreadyApplied ? { turnComplete: true } : {}),
},
});
remove();
assert.equal(entered, true);
// With no old terminal received, or when that terminal was consumed
// before the observer, a real predecessor boundary is still outstanding.
assert.equal(h.controller.retiredReplyPending, true);
assert.ok(h.controller.retiredBoundaryWait);
assert.equal(completions(h).length, 0);
assert.equal(h.runnerCalls.length, 0);
ws.message({ serverContent: { turnComplete: true } });
assert.equal(h.controller.retiredReplyPending, false);
assert.equal(h.controller.retiredBoundaryWait, null);
await h.tool('after-real-pending-boundary', ws);
assertSuccessfulZoom(h, ws, 'after-real-pending-boundary', 1000);
h.complete(ws);
assert.equal(completions(h).length, 1);
},
);
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+244 -251
View File
@@ -1,6 +1,6 @@
import { interruptGlobeRotation } from '../cameraVerbs.js';
import {
layerSnapshot,
layerSnapshots,
feedProvenanceEnvelope,
} from '../data/layerSnapshot.js';
import { readLayerLifecycleSummary } from './layerSummary.js';
@@ -8,9 +8,11 @@ import { SPEECH_BUILDERS, attachVoiceResult } from './speech.js';
import { resolveDeicticArgs } from './deixis.js';
export { readLayerLifecycleSummary } from './layerSummary.js';
import { defaultGeospatial } from '../search/defaults.js';
import { msg } from '../i18n/index.js';
import * as Cesium from 'cesium';
import { CITY_POIS } from '../cityPois.js';
import { greatCircleKm } from '../geoDistance.js';
import {
CITY_POIS,
findPoiByName,
flyToGlobeView,
flyToLandmark,
@@ -25,7 +27,6 @@ import {
isContextRecordActive,
} from '../data/contextStore.js';
import { CCTV_FOCUS_RESULT } from '../layers/cctv/index.js';
import { contextModeWord } from '../contextModePolicy.js';
import {
createAnalystEngine,
normalizeAnalystSpec,
@@ -46,6 +47,12 @@ import { unavailablePlaceSearch } from '../search/placeSearch.js';
import * as defaultAnnotationResolver from '../annotations/annotationResolver.js';
import { normalizeRadioCountryInput } from '../data/radioCountry.js';
import { TR3B_CLASS } from '../data/tr3bRegistry.js';
import {
getWorldPacket,
withContextModeVocabulary,
activeContactsWindow,
TRACKABLE_FAMILIES,
} from '../agent/worldPacket.js';
const ALLOWED_STYLES = new Set([
'normal',
@@ -67,8 +74,17 @@ const PANEL_ALIASES = new Map([
['styles', 'control-panel'],
['filters', 'control-panel'],
['visual styles', 'control-panel'],
['visual preset', 'control-panel'],
['visual presets', 'control-panel'],
['provider settings', 'provider-settings'],
['cctv', 'cctv-panel'],
['cameras', 'cctv-panel'],
['voice settings', 'voice-settings'],
['voice-settings', 'voice-settings'],
['voice preview', 'voice-preview'],
['voice-preview', 'voice-preview'],
['voice control', 'voice-preview'],
['voice controls', 'voice-preview'],
['radio', 'radio-panel'],
['internet radio', 'radio-panel'],
['radio stations', 'radio-panel'],
@@ -103,6 +119,7 @@ const PANEL_IDS = new Set([
'pp-toggles',
'weather-panel',
'recent-imagery-panel',
'provider-settings',
]);
const CONTEXT_MODE_ALIASES = new Map([
['off', 'off'],
@@ -117,64 +134,6 @@ const CONTEXT_MODE_ALIASES = new Map([
['space-missions', 'space-missions'],
['missions', 'space-missions'],
]);
/**
* Every model-readable field that carries a context-mode id, and what an
* absent value means for each.
*
* `mode` always names a mode, so nothing is 'off'. `entering` and `priorMode`
* are absent when there is no such mode at all — calling those 'off' would
* assert a state that does not exist.
*/
const CONTEXT_MODE_RESULT_FIELDS = Object.freeze([
{ field: 'mode', emptyAs: 'off' },
{ field: 'entering', emptyAs: null },
{ field: 'priorMode', emptyAs: null },
]);
/** Nested results that are themselves context-mode payloads the model reads. */
const NESTED_CONTEXT_RESULT_FIELDS = Object.freeze([
'context',
'contextRollback',
]);
/**
* Report a context-mode payload in the tools' own vocabulary.
*
* `set_context_mode` accepts 'contacts' while the mode's internal id is
* 'flights'. Reporting the internal id back made the model read
* `mode:'flights'` as "Contacts is off" and refuse to answer from the Contacts
* window counts sitting in the very same payload (owner field session
* 2026-08-21). Secondary fields and nested transition/rollback results are
* translated too — one leaked internal id is enough to recreate the confusion,
* and a rollback result is exactly what the model reads when something went
* wrong. Each internal id is kept alongside as `<field>Internal` for anything
* reasoning about layers.
* @param {object|null|undefined} state Any payload carrying context-mode fields.
* @returns {object|null|undefined} The same payload, modes translated.
*/
function withContextModeVocabulary(state) {
if (!state || typeof state !== 'object') return state;
let out = state;
const mutable = () => {
if (out === state) out = { ...state };
return out;
};
for (const { field, emptyAs } of CONTEXT_MODE_RESULT_FIELDS) {
if (!(field in state)) continue;
const internal = state[field] ?? null;
const target = mutable();
target[field] = contextModeWord(internal, { emptyAs });
target[`${field}Internal`] = internal;
}
for (const field of NESTED_CONTEXT_RESULT_FIELDS) {
const nested = state[field];
if (!nested || typeof nested !== 'object') continue;
const translated = withContextModeVocabulary(nested);
if (translated !== nested) mutable()[field] = translated;
}
return out;
}
const COCKPIT_ACTION_ALIASES = new Map([
['next', 'next'],
['previous', 'previous'],
@@ -244,14 +203,6 @@ const STACK_ALIASES = new Map([
['road map', 'osm'],
]);
/** Search order for track_entity across entity layer families. */
const TRACKABLE_FAMILIES = [
{ layerId: 'flights', kind: 'aircraft' },
{ layerId: 'military', kind: 'aircraft' },
{ layerId: 'ais-live-vessels', kind: 'vessel' },
{ layerId: 'satellites', kind: 'satellite' },
];
const FRAME_TARGETS = new Map([
['flights', 'flights'],
['planes', 'flights'],
@@ -302,6 +253,7 @@ export function createGevActionRunner({
searchNavigation = searchAndFlyTo,
speechBuilders = null,
deixis = null,
readVoicePanelOperation = null,
}) {
// Voice enable times and analyst follow-up memory belong to this runner.
// Speech builders turn results into say/display envelopes; callers extend
@@ -322,9 +274,86 @@ export function createGevActionRunner({
}),
));
installViewTargetPrewarm(viewer);
initCameraVerbs(viewer, getViewTargetCartesian);
initCameraVerbs(viewer, getViewTargetCartesian, {
isCockpitActive: () => Boolean(styleManager?.getCockpitState?.()?.active),
});
// Nested calls (select_nearest_aircraft composing layer + flight) use the
// raw runner; only the outer call gets a voice envelope.
const voiceUnavailable = (action) => ({
ok: false,
action,
applied: false,
unavailable: true,
error: 'Voice panel controls are unavailable in this host.',
say: 'Voice panel controls are unavailable in this host.',
});
async function showVoicePanel(
view,
open,
options,
operation = readVoicePanelOperation?.(),
) {
const current = () =>
!options.signal?.aborted &&
(!options.isCurrent || options.isCurrent()) &&
operation?.isCurrent();
if (!operation) return voiceUnavailable('set_panel_open');
if (!current())
return {
ok: false,
action: 'set_panel_open',
applied: false,
cancelled: true,
};
let exit = null;
if (open) {
if (typeof styleManager?.getCockpitState !== 'function')
return voiceUnavailable('set_panel_open');
if (styleManager.getCockpitState()?.active) {
if (typeof styleManager.controlCockpit !== 'function')
return voiceUnavailable('set_panel_open');
exit = await styleManager.controlCockpit('exit');
if (styleManager.getCockpitState()?.active)
return {
ok: false,
action: 'set_panel_open',
applied: false,
exit,
error: exit?.error || 'Cockpit did not exit.',
};
}
}
if (!current())
return {
ok: false,
action: 'set_panel_open',
applied: false,
cancelled: true,
exit,
cockpitExited: Boolean(exit && !styleManager.getCockpitState()?.active),
say:
exit && !styleManager.getCockpitState()?.active
? 'Cockpit exited; the voice panel was left as you last set it.'
: 'The voice panel was left as you last set it.',
};
const result = operation.show(view, open);
return {
...result,
action: 'set_panel_open',
panelId: view === 'settings' ? 'voice-settings' : 'voice-preview',
exit,
state: styleManager?.getCockpitState?.() ?? null,
say: !result.ok
? 'The voice panel was not changed.'
: result.settingsOpen
? 'Voice Settings is open.'
: result.panelOpen
? result.previewEnabled
? 'Voice Preview is open.'
: 'Voice control is open; Preview remains off.'
: 'The voice panel is closed.',
};
}
const runGevAction = async function runGevAction(
name,
rawArgs = {},
@@ -358,6 +387,7 @@ export function createGevActionRunner({
// Zoom during an active orbit adjusts the orbit RADIUS (spiral in/out) —
// a straight camera move would be snapped back by the per-frame lookAt.
if (name === 'adjust_camera_zoom') {
interruptGlobeRotation('voice-zoom-intent');
const zoomOut = String(args.direction || '').toLowerCase() === 'out';
const amt = String(args.amount || 'medium').toLowerCase();
const factor = { little: 1.25, medium: 1.6, lot: 2.4 }[amt] || 1.6;
@@ -695,6 +725,25 @@ export function createGevActionRunner({
return { ok: true, action: 'set_visual_style', style };
}
if (name === 'set_voice_inactivity_timeout') {
const operation = readVoicePanelOperation?.();
if (!operation) return voiceUnavailable(name);
if (!current() || !operation.isCurrent())
return { ok: false, action: name, applied: false, cancelled: true };
const result = operation.setMinutes(args.minutes);
const duration =
result.minutes === 0 ? 'disabled' : `${result.minutes} minutes`;
return {
...result,
action: name,
say: !result.applied
? 'Voice inactivity was not changed.'
: result.persisted
? `Voice inactivity is ${duration}, saved in this browser.`
: `Voice inactivity is ${duration} for now, but could not be saved in this browser.`,
};
}
if (name === 'set_panel_open') {
const panelId = normalizePanelId(args.panelId || args.panel);
if (!panelId)
@@ -702,6 +751,70 @@ export function createGevActionRunner({
`Unknown panel: ${args.panelId || args.panel || 'missing'}`,
);
const open = args.open !== false;
if (panelId === 'voice-settings' || panelId === 'voice-preview')
return showVoicePanel(
panelId === 'voice-settings' ? 'settings' : 'preview',
open,
runOptions,
);
if (panelId === 'provider-settings') {
if (!current())
return {
ok: false,
action: name,
panelId,
applied: false,
cancelled: true,
};
const doc = globalThis.document;
const panel = doc?.getElementById?.('key-setup');
const button = open
? doc?.getElementById?.('key-setup-chip')
: panel?.querySelector?.('[data-key-setup-close]');
const concealed =
open &&
[
'ui-clean-view',
'recording-mode',
'cockpit-mode',
'scene-playback-mode',
].some((mode) => doc?.body?.classList?.contains(mode));
if (
concealed ||
panel?.dataset?.ready !== 'true' ||
typeof button?.click !== 'function'
) {
const error = concealed
? 'Provider Settings is hidden in the current view. Leave that view before opening it.'
: 'Provider Settings is unavailable in this runtime.';
return {
ok: false,
action: name,
panelId,
applied: false,
unavailable: true,
error,
say: error,
};
}
button.click();
// Opening unhides immediately; closing removes visible before its fade ends.
const applied = open
? !panel.hidden
: !panel.classList.contains('visible');
const say = applied
? `Provider Settings is ${open ? 'open' : 'closed'}.`
: 'Provider Settings did not change.';
return {
ok: applied,
action: name,
panelId,
applied,
open: applied ? open : !panel.hidden,
say,
...(!applied ? { error: say } : {}),
};
}
setPanelOpen(styleManager, panelId, open);
return { ok: true, action: 'set_panel_open', panelId, open };
}
@@ -790,6 +903,25 @@ export function createGevActionRunner({
error: `Unknown cockpit action: ${args.action || args.command || 'missing'}`,
};
}
if (args.voicePanel !== undefined) {
if (
action !== 'exit' ||
!['preview', 'settings'].includes(args.voicePanel)
)
return {
ok: false,
action: 'control_cockpit',
applied: false,
error: 'Voice panel reveal requires exit and a valid view.',
};
const result = await showVoicePanel(args.voicePanel, true, runOptions);
return {
...result,
action: 'control_cockpit',
cockpitAction: 'exit',
voicePanel: args.voicePanel,
};
}
const inferred = normalizeCockpitNavigationHints(rawAction);
const targetLayer = normalizeCockpitTargetLayer(
args.targetLayer || inferred.targetLayer || args.layer || args.layerId,
@@ -1152,6 +1284,18 @@ export function createGevActionRunner({
}
if (name === 'control_cctv') {
const action = String(args.action || '').toLowerCase();
if (
(action === 'enable' || action === 'disable') &&
dataManager.layers.get('cctv')?.module
) {
const result = await runGevAction(
'set_layer_visibility',
{ layerId: 'cctv', enabled: action === 'enable' },
runOptions,
);
return { ...result, action: 'control_cctv' };
}
return controlCctv(dataManager, args, styleManager);
}
@@ -1442,7 +1586,7 @@ function controlScene(sceneDirector, args = {}) {
};
}
if (sceneAction === 'stop') {
sceneDirector.stopScene('Stopped by voice');
sceneDirector.stopScene(msg('Stopped by voice'));
return { ok: true, action: 'control_scene', running: false };
}
if (sceneAction === 'next') {
@@ -1492,13 +1636,10 @@ export async function controlCctv(dataManager, args = {}, styleManager = null) {
}
if (action === 'enable' || action === 'disable') {
await dataManager.setEnabled('cctv', action === 'enable', {
origin: 'voice',
});
return {
ok: true,
ok: false,
action: 'control_cctv',
enabled: dataManager.isEnabled('cctv'),
error: 'Enable or disable CCTV through set_layer_visibility',
};
}
if (!dataManager.isEnabled('cctv')) {
@@ -1643,14 +1784,18 @@ export async function controlCctv(dataManager, args = {}, styleManager = null) {
}
const RADIO_COUNTRY_CENTERS = new Map([
// i18n-ignore: place name in tool data
['us', { lat: 39.8, lon: -98.6, country: 'US', label: 'United States' }],
// i18n-ignore: place name in tool data
['usa', { lat: 39.8, lon: -98.6, country: 'US', label: 'United States' }],
[
'united states',
// i18n-ignore: place name in tool data
{ lat: 39.8, lon: -98.6, country: 'US', label: 'United States' },
],
[
'united states of america',
// i18n-ignore: place name in tool data
{ lat: 39.8, lon: -98.6, country: 'US', label: 'United States' },
],
]);
@@ -2251,6 +2396,7 @@ async function trackEntity(
action: 'track_entity',
kind: 'fire',
layerId: 'local-firms',
// i18n-ignore: entity name in a tool result for the voice model
label: strongest.label || 'Strongest fire',
latitude: strongest.latitude,
longitude: strongest.longitude,
@@ -2546,26 +2692,6 @@ function runManagedVoiceNavigation(
};
}
/** Gathers tracked/selected entities across layer families for read-back. */
function collectTrackedEntities(dataManager) {
const tracked = [];
for (const family of TRACKABLE_FAMILIES) {
const module = dataManager.layers.get(family.layerId)?.module;
if (!module) continue;
try {
const info =
family.kind === 'vessel'
? module.getSelectedInfo?.()
: module.getTrackedInfo?.();
if (info)
tracked.push({ kind: family.kind, layerId: family.layerId, ...info });
} catch {
// layer not ready
}
}
return tracked;
}
export async function getBasemapLabelContext(
viewer,
service = defaultGeospatial,
@@ -2997,7 +3123,11 @@ function focusDataLayerRow(layerId) {
void row.offsetWidth;
row.classList.add('gev-voice-focus');
window.setTimeout(() => row.classList.remove('gev-voice-focus'), 3000);
const name = row.querySelector('.data-name')?.textContent?.trim() || layerId;
// The English name: tool results stay English whatever the UI language.
const name =
row.dataset.layerName ||
row.querySelector('.data-name')?.textContent?.trim() ||
layerId;
return { id: layerId, name };
}
@@ -3199,60 +3329,12 @@ function getCurrentViewState(
dataManager,
sceneDirector = null,
) {
const cartographic = Cesium.Cartographic.fromCartesian(
viewer.camera.positionWC,
);
return {
ok: true,
action: 'get_current_view_state',
camera: {
latitude: Cesium.Math.toDegrees(cartographic.latitude),
longitude: Cesium.Math.toDegrees(cartographic.longitude),
heightM: cartographic.height,
},
style: styleManager.activeStyle || 'normal',
context:
typeof styleManager.getContextModeState === 'function'
? {
...withContextModeVocabulary(styleManager.getContextModeState()),
// The numbers on the operator's Contacts panel, so a window/count
// question can be answered from what they are looking at.
...(activeContactsWindow(dataManager)
? { contactsWindow: activeContactsWindow(dataManager) }
: {}),
}
: null,
cockpit:
typeof styleManager.getCockpitState === 'function'
? styleManager.getCockpitState()
: null,
controls:
typeof styleManager.getControlState === 'function'
? styleManager.getControlState()
: null,
scenePlayback: sceneDirector?.getPlaybackStatus?.() || null,
tracked: collectTrackedEntities(dataManager),
// Enabled layers only: the full registry cost ~1k tokens per call and
// the model only reasons about what is on.
disabledLayerCount: dataManager.getAll().filter((layer) => !layer.enabled)
.length,
layers: dataManager
.getAll()
.filter((layer) => layer.enabled)
.map((layer) => ({
id: layer.id,
name: layer.name,
enabled: layer.enabled,
count: layer.stats?.count || 0,
error: layer.stats?.error || null,
feedState: layerSnapshot(layer).feedState,
source: layerSnapshot(layer).source,
lastUpdate: layerSnapshot(layer).lastUpdate,
})),
feedProvenance: feedProvenanceEnvelope(
layerSnapshots(dataManager.getAll()).filter((layer) => layer.enabled),
),
};
return getWorldPacket({
viewer,
styleManager,
dataManager,
sceneDirector,
});
}
async function getEntityContext(
@@ -3732,8 +3814,10 @@ function centerMatchesSubject(center, subjectPosition) {
const subjectLat = Cesium.Math.toDegrees(carto.latitude);
const subjectLon = Cesium.Math.toDegrees(carto.longitude);
return (
haversineKm(subjectLat, subjectLon, center.lat, center.lon) <=
SUBJECT_CENTER_TOLERANCE_KM
greatCircleKm(
{ lat: subjectLat, lon: subjectLon },
{ lat: center.lat, lon: center.lon },
) <= SUBJECT_CENTER_TOLERANCE_KM
);
}
@@ -4023,7 +4107,7 @@ function nearbyKnownLandmarks(latitude, longitude, cameraHeightM) {
for (const [cityId, city] of Object.entries(CITY_POIS)) {
for (let poiIndex = 0; poiIndex < city.pois.length; poiIndex++) {
const poi = city.pois[poiIndex];
const distanceKm = haversineKm(latitude, longitude, poi.lat, poi.lon);
const distanceKm = greatCircleKm({ lat: latitude, lon: longitude }, poi);
if (distanceKm > maxDistanceKm) continue;
matches.push({
name: poi.name,
@@ -4038,20 +4122,6 @@ function nearbyKnownLandmarks(latitude, longitude, cameraHeightM) {
return matches.sort((a, b) => a.distanceKm - b.distanceKm).slice(0, 5);
}
function haversineKm(lat1, lon1, lat2, lon2) {
const toRad = (value) => Cesium.Math.toRadians(value);
const radiusKm = 6371.0088;
const dLat = toRad(lat2 - lat1);
const dLon = toRad(lon2 - lon1);
const a =
Math.sin(dLat / 2) * Math.sin(dLat / 2) +
Math.cos(toRad(lat1)) *
Math.cos(toRad(lat2)) *
Math.sin(dLon / 2) *
Math.sin(dLon / 2);
return 2 * radiusKm * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
}
function coarseBasemapPlace(
viewScale,
latitude,
@@ -4491,46 +4561,6 @@ function dominantValue(values) {
};
}
function summarizeEntity(viewer, entity, { includeProperties = false } = {}) {
const now = Cesium.JulianDate.now();
if (entity.__gevContextId) {
const store = window.__gevContextStore;
const record = store?.entities?.get(entity.__gevContextId);
if (record) return summarizeContextRecord(record, { includeProperties });
}
const props = propertyObject(entity);
const layerId = entity.__localLayerId || props.layerId || null;
const tags = props.tags || {};
const label = cleanText(
props.name ||
tags.name ||
tags['name:en'] ||
tags.official_name ||
tags.operator ||
props.operator ||
entity.name ||
layerTitle(layerId),
);
const position =
entity.__localBaseCartesian ||
entity.position?.getValue?.(now) ||
polygonCenter(entity, now);
const carto = position ? Cesium.Cartographic.fromCartesian(position) : null;
return {
id: String(entity.id || ''),
name: label || layerTitle(layerId),
layerId,
layerName: layerTitle(layerId),
latitude: carto
? Number(Cesium.Math.toDegrees(carto.latitude).toFixed(6))
: null,
longitude: carto
? Number(Cesium.Math.toDegrees(carto.longitude).toFixed(6))
: null,
properties: includeProperties ? compactProperties(props) : undefined,
};
}
function summarizeContextRecord(record, { includeProperties = false } = {}) {
return {
id: String(record.id || ''),
@@ -4549,31 +4579,6 @@ function summarizeContextRecord(record, { includeProperties = false } = {}) {
};
}
function polygonCenter(entity, now) {
const hierarchy = entity.polygon?.hierarchy?.getValue?.(now);
const positions = hierarchy?.positions;
if (!positions?.length) return null;
return Cesium.BoundingSphere.fromPoints(positions).center;
}
function propertyObject(entity) {
const raw = entity?.properties?.getValue?.(Cesium.JulianDate.now()) || {};
return unwrapProperties(raw);
}
function unwrapProperties(value) {
if (!value || typeof value !== 'object') return value;
if (Array.isArray(value)) return value.map(unwrapProperties);
const out = {};
for (const [key, entry] of Object.entries(value)) {
out[key] =
entry && typeof entry.getValue === 'function'
? unwrapProperties(entry.getValue(Cesium.JulianDate.now()))
: unwrapProperties(entry);
}
return out;
}
function compactProperties(props) {
const preferredKeys = [
'name',
@@ -4636,23 +4641,6 @@ function clampNumber(value, min, max, fallback) {
* ("which of those is closest?") scoped to the session.
*/
/**
* The Contacts panel's own counts, or null when Contacts has no subject.
* Read through the awareness snapshot the panel renders, so the two cannot
* diverge no matter which surface asks.
* @returns {object|null} Panel-equivalent window counts.
*/
function activeContactsWindow(dataManager) {
try {
const layer = dataManager?.layers?.get('military-awareness')?.module;
return (
layer?.contactsWindowFromSnapshot?.(layer.getContextSnapshot?.()) ?? null
);
} catch {
return null;
}
}
/**
* Most records one layer hands a single query. Above every loaded feed seen
* so far (flights ~8k, vessels ~12k, a 24 h global FIRMS day ~160k), so
@@ -4809,7 +4797,12 @@ function compactAnalystItem(r) {
if (!present(r[key]) || key in compact) continue;
if (spec.type === 'time') {
const ms = Number(r[key]);
if (Number.isFinite(ms)) compact.time = new Date(ms).toISOString();
// A finite value can still be past Date's range, where toISOString throws.
const time = new Date(ms);
if (Number.isFinite(time.getTime())) {
if (key === 'lastContactAtMs') compact.lastContactAtMs = ms;
else compact.time = time.toISOString();
}
} else compact[key] = r[key];
}
if (present(r.ageHours)) compact.ageHours = r.ageHours;
+50 -4
View File
@@ -1079,7 +1079,9 @@ test('generic layer visibility exposes lifecycle truth for every manager phase a
});
// The voice envelope is additive and states the failure plainly.
assert.equal(say, "Couldn't turn on radio.");
assert.equal(display.title, 'radio');
// The card names the layer, not its id; the result keeps the id.
assert.equal(display.title, 'Radio');
assert.deepEqual(display.lines, ['Radio layer unavailable']);
});
test('generic voice visibility maps Space Missions to the explicit mission layer', async () => {
@@ -3180,7 +3182,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl
const requests = [];
installKeyedFetch(t, async (url) => {
requests.push(String(url));
if (String(url).startsWith('https://maps.googleapis.com/')) {
if (String(url).startsWith('/api/google/geocode')) {
return { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) };
}
assert.match(String(url), /^https:\/\/photon\.komoot\.io\/api\/\?/);
@@ -3205,7 +3207,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl
assert.ok(Math.abs(calls[0].criteria.anchor.lat - 20.9101) < 1e-9);
assert.ok(Math.abs(calls[0].criteria.anchor.lon - 107.1839) < 1e-9);
// Google is asked first and exactly once; Photon answers unbiased, in one call.
assert.equal(requests.filter((url) => url.includes('maps.googleapis.com')).length, 1);
assert.equal(requests.filter((url) => url.includes('/api/google/geocode')).length, 1);
assert.equal(requests.filter((url) => url.includes('photon.komoot.io')).length, 1);
assert.match(requests.at(-1), /[?&]q=H%E1%BA%A1\+Long\+Bay/);
assert.doesNotMatch(requests.at(-1), /[?&](lat|lon|bbox)=/, 'a named radio location is not viewport-biased');
@@ -3218,7 +3220,7 @@ test('voice Radio: the keyless path applies no country filter the keyed path wou
// "No Radio station matched" for exactly the places it just resolved, while a
// keyed install placed a station. The label may carry it; the filter may not.
const { calls, dataManager } = radioSelectionHarness();
installKeyedFetch(t, async (url) => (String(url).startsWith('https://maps.googleapis.com/')
installKeyedFetch(t, async (url) => (String(url).startsWith('/api/google/geocode')
? { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) }
: {
ok: true,
@@ -3263,6 +3265,30 @@ test('ALPR common names toggle only the registered camera layer through the norm
}
});
test('GNSS integrity and interference common names toggle only the GNSS layer through the normal voice action', async () => {
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
const viewer = { clock: { onTick: { addEventListener: () => () => {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
camera: { moveEnd: { addEventListener() {} } } };
const calls = [];
let enabled = false;
const dataManager = {
layers: new Map([['gnss-integrity', { module: {} }]]),
getAll: () => [{ id: 'gnss-integrity', name: 'GNSS Integrity' }],
isEnabled: () => enabled,
setEnabled: async (id, value) => { calls.push([id, value]); enabled = value; return true; },
};
const runner = createGevActionRunner({ viewer, styleManager: {}, dataManager });
for (const alias of ['gnss-integrity', 'gnss integrity', 'navigation integrity', 'low navigation accuracy', 'gnss interference', 'gps interference', 'gps jamming', 'gnss']) {
for (const value of [true, false]) {
const result = await runner('set_layer_visibility', { layerId: alias, enabled: value });
assert.equal(result.ok, true);
assert.equal(result.layerId, 'gnss-integrity');
assert.deepEqual(calls.at(-1), ['gnss-integrity', value]);
}
}
});
test('Local ADS-B common names toggle only the receiver layer through the normal voice action', async () => {
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
const viewer = { clock: { onTick: { addEventListener: () => () => {} } },
@@ -3459,6 +3485,26 @@ test('analyst_query refusals carry their code and the allowed values', async ()
assert.match(ok.items[0].time, /^\d{4}-\d\d-\d\dT/);
});
test('analyst_query keeps its answer when one record has an out-of-range time', async () => {
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
const now = Date.now();
const earthquakes = {
getStats: () => ({ count: 2, lastUpdate: now }),
getAnalystRecords: () => [
{ id: 'q', lat: 0, lon: 0, magnitude: 5, timeMs: now },
{ id: 'r', lat: 1, lon: 1, magnitude: 4, timeMs: 8.64e15 + 1 },
],
};
const { runner } = manifestRunner({ earthquakes }, { enabled: ['earthquakes'] });
const result = await runner('analyst_query', { layers: ['earthquakes'], scope: { kind: 'anywhere' } });
assert.equal(result.ok, true);
const q = result.items.find((item) => item.id === 'q');
assert.match(q.time, /^\d{4}-\d\d-\d\dT/);
const r = result.items.find((item) => item.id === 'r');
assert.ok(r, 'the record is still listed');
assert.equal(r.time, undefined, 'an unrepresentable time is omitted');
});
test('spoken layer names from the manifest reach the newer layers', async () => {
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
const stub = () => ({ getStats: () => ({ count: 0 }) });
+10 -7
View File
@@ -50,21 +50,24 @@ export function initGevVoiceCommands(options) {
const referents = createReferentRegistry();
// The controller exists only after the runner, so its hooks bind late.
const deixis = { pointer, referents, cameraKey };
const commands = createVoiceCommands({
let commands;
commands = createVoiceCommands({
...options,
pointer,
referents,
runner: withToolCatalog(
createGevActionRunner({ ...options, deixis }),
createGevActionRunner({
...options,
deixis,
readVoicePanelOperation: () => commands?.captureVoicePanelOperation?.(),
}),
options.toolCatalog,
),
});
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
deixis.onPointerUsed = () => commands.announcePointer?.();
commands.session?.signal?.addEventListener(
'abort',
() => pointer?.destroy(),
{ once: true },
);
const lifetime = commands.signal ?? commands.session?.signal;
lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true });
if (lifetime?.aborted) pointer?.destroy();
return commands;
}
File diff suppressed because it is too large. Load diff
+2 -1
View File
@@ -124,7 +124,8 @@ test('a partial answer names the layers it could not read', async () => {
assert.match(result.say, /Partial; earthquakes not answered/);
assert.deepEqual(result.unanswered, ['earthquakes']);
assert.ok(view.chips.includes('partial'));
assert.ok(view.notes.includes('Not answered: earthquakes'));
// The card names the layer; the result keeps its key for the model.
assert.ok(view.notes.includes('Not answered: Earthquakes (24h)'));
});
test('"the second one" is the second row the card lists', async () => {
+59 -25
View File
@@ -34,9 +34,43 @@ const AIRCRAFT_FIELDS = {
aircraftClass: ['t'],
military: ['f'],
onGround: ['f'],
squawk: ['t'],
emergency: ['f'],
emergencyKind: ['t'],
emergencySource: ['t'],
emergencyBroadcast: ['t'],
lastContactAtMs: ['ms'],
};
const layers = [
{
id: 'gnss-integrity',
aliases: [
'gnss integrity',
'navigation integrity',
'low navigation accuracy',
'gnss interference',
'gps interference',
'gps jamming',
'gnss',
],
context: false,
noQuery: 'navigation-integrity cells; no countable analyst records',
},
{
id: 's2-ghostmaps',
aliases: [
's2 ghostmaps',
'ghost maps',
'ghostmaps',
's2 underground',
's2 intel',
'intel picture',
'common intelligence picture',
],
context: false,
noQuery: 'published third-party intel placemarks; not analyst records',
},
{
id: 'flights',
track: true,
@@ -230,6 +264,24 @@ const layers = [
context: false,
noQuery: 'map overlay; no countable records',
},
{
id: 'air-quality',
aliases: ['air quality', 'aqhi', 'smog', 'air pollution'],
context: false,
query: {
fields: {
value: ['n', 'AQHI'],
band: ['t'],
scale: ['t'],
provider: ['t'],
name: ['t'],
zone: ['t'],
risk: ['t'],
observedMs: ['ms'],
},
timeField: 'observedMs',
},
},
{
id: 'traffic',
aliases: ['street traffic', 'road traffic', 'congestion'],
@@ -327,30 +379,6 @@ const layers = [
context: true,
query: { fields: { name: ['t'], class: ['t'], kind: ['t'] } },
},
{
id: 'drone-airspace',
aliases: [
'drone restrictions',
'drone airspace',
'no-fly zones',
'no fly zones',
'drones',
],
context: false,
query: {
fields: {
name: ['t'],
typeCode: ['t'],
type: ['t'],
lowerDesc: ['t'],
upperDesc: ['t'],
timesOfUse: ['t'],
agency: ['t'],
city: ['t'],
state: ['t'],
},
},
},
{
id: 'local-datacenters',
aliases: ['datacenters', 'data centers', 'data centres'],
@@ -396,6 +424,10 @@ const layers = [
export const VOICE_OFF_LAYERS = Object.freeze({
'bhote-koshi-2026': 'scene-driven event layer (control_scene)',
'bhote-koshi-locator': 'scene-driven event layer (control_scene)',
// The Realtime prompt is at its 52,000-character budget; voice toggle and
// query for this layer wait until there is room.
'drone-airspace':
'informational airspace layer; no room in the prompt budget',
});
const TYPE_NAMES = Object.freeze({
@@ -497,6 +529,8 @@ const SELF_EVIDENT_IDS = new Set([
'radio',
'wind',
'directions',
// The brand is already the layer id; all spoken aliases remain accepted.
's2-ghostmaps',
]);
/**
@@ -511,7 +545,7 @@ export function voiceLayerAliasHint() {
entry.toggle && entry.aliases.length && !SELF_EVIDENT_IDS.has(entry.id),
)
.map((entry) => `${entry.aliases.slice(0, 2).join('/')}→${entry.id}`)
.join('; ');
.join(';');
}
/** Field names that already carry their unit (altitudeM, speedKts, …). */
+29 -99
View File
@@ -20,28 +20,14 @@ test('every catalog layer is in the voice manifest or explicitly voice-off', ()
const missing = catalogIds.filter(
(id) => !manifestIds.includes(id) && !Object.hasOwn(VOICE_OFF_LAYERS, id),
);
assert.deepEqual(
missing,
[],
`add a voice manifest entry for: ${missing.join(', ')}`,
);
assert.deepEqual(missing, [], `add a voice manifest entry for: ${missing.join(', ')}`);
for (const id of Object.keys(VOICE_OFF_LAYERS)) {
assert.ok(
catalogIds.includes(id),
`${id} is voice-off but not in the catalog`,
);
assert.ok(
!manifestIds.includes(id),
`${id} is both voice-off and in the manifest`,
);
assert.ok(catalogIds.includes(id), `${id} is voice-off but not in the catalog`);
assert.ok(!manifestIds.includes(id), `${id} is both voice-off and in the manifest`);
}
const unknown = manifestIds.filter((id) => !catalogIds.includes(id));
assert.deepEqual(unknown, [], 'the manifest names only catalog layers');
assert.equal(
new Set(manifestIds).size,
manifestIds.length,
'one entry per layer',
);
assert.equal(new Set(manifestIds).size, manifestIds.length, 'one entry per layer');
});
test('each layer declares a query or the reason it has none', () => {
@@ -55,8 +41,7 @@ test('each layer declares a query or the reason it has none', () => {
});
test('tool enums are generated from the manifest', () => {
const toggle = tool('set_layer_visibility').parameters.properties.layerId
.enum;
const toggle = tool('set_layer_visibility').parameters.properties.layerId.enum;
assert.deepEqual(toggle, [...VOICE_TOGGLE_LAYER_IDS]);
for (const id of [
'transit',
@@ -72,13 +57,9 @@ test('tool enums are generated from the manifest', () => {
'rocket-launches',
'bikeshare',
'local-adsb',
'drone-airspace',
])
assert.ok(toggle.includes(id), `${id} is voice-toggleable`);
assert.ok(
!toggle.includes('military-awareness'),
'Global Context is reached through set_context_mode',
);
assert.ok(!toggle.includes('military-awareness'), 'Global Context is reached through set_context_mode');
assert.deepEqual(
tool('show_data_layers_menu').parameters.properties.layerId.enum,
[...VOICE_TOGGLE_LAYER_IDS],
@@ -87,98 +68,47 @@ test('tool enums are generated from the manifest', () => {
tool('get_entity_context').parameters.properties.layerId.enum,
[...VOICE_CONTEXT_LAYER_IDS],
);
assert.ok(
!VOICE_CONTEXT_LAYER_IDS.includes('telegeography-submarine-cables'),
'cables register no context records',
);
assert.ok(!VOICE_CONTEXT_LAYER_IDS.includes('telegeography-submarine-cables'), 'cables register no context records');
assert.deepEqual(
tool('analyst_query').parameters.properties.layers.items.enum,
[...VOICE_QUERY_LAYER_IDS],
);
assert.deepEqual(Object.keys(ANALYST_LAYERS), [...VOICE_QUERY_LAYER_IDS]);
const panels = tool('set_panel_open').parameters.properties.panelId.enum;
assert.ok(
panels.includes('weather-panel') && panels.includes('recent-imagery-panel'),
);
assert.ok(panels.includes('weather-panel') && panels.includes('recent-imagery-panel'));
});
test('spoken aliases resolve to real layers, keep every legacy phrase and never collide', () => {
const aliases = voiceLayerAliasMap();
for (const id of aliases.values()) assert.ok(manifestIds.includes(id), id);
const legacy = {
planes: 'flights',
aircraft: 'flights',
'military flights': 'military',
quakes: 'earthquakes',
'space mission': 'rocket-launches',
'space missions': 'rocket-launches',
missions: 'rocket-launches',
'street traffic': 'traffic',
cameras: 'cctv',
'internet radio': 'radio',
'radio stations': 'radio',
bikes: 'bikeshare',
ais: 'ais-live-vessels',
ships: 'ais-live-vessels',
vessels: 'ais-live-vessels',
'live vessels': 'ais-live-vessels',
datacenters: 'local-datacenters',
'data centers': 'local-datacenters',
'data centres': 'local-datacenters',
dams: 'local-dams',
'submarine cables': 'telegeography-submarine-cables',
cables: 'telegeography-submarine-cables',
telegeography: 'telegeography-submarine-cables',
firms: 'local-firms',
fires: 'local-firms',
'active fires': 'local-firms',
alpr: 'alpr-cameras',
'alpr cameras': 'alpr-cameras',
'flock cameras': 'alpr-cameras',
'license plate readers': 'alpr-cameras',
'license plate cameras': 'alpr-cameras',
'plate readers': 'alpr-cameras',
'local-adsb': 'local-adsb',
'local adsb': 'local-adsb',
'local ads-b': 'local-adsb',
'my receiver': 'local-adsb',
'my antenna': 'local-adsb',
'my sdr': 'local-adsb',
planes: 'flights', aircraft: 'flights', 'military flights': 'military', quakes: 'earthquakes',
'space mission': 'rocket-launches', 'space missions': 'rocket-launches', missions: 'rocket-launches',
'street traffic': 'traffic', cameras: 'cctv', 'internet radio': 'radio', 'radio stations': 'radio',
bikes: 'bikeshare', ais: 'ais-live-vessels', ships: 'ais-live-vessels', vessels: 'ais-live-vessels',
'live vessels': 'ais-live-vessels', datacenters: 'local-datacenters', 'data centers': 'local-datacenters',
'data centres': 'local-datacenters', dams: 'local-dams', 'submarine cables': 'telegeography-submarine-cables',
cables: 'telegeography-submarine-cables', telegeography: 'telegeography-submarine-cables',
firms: 'local-firms', fires: 'local-firms', 'active fires': 'local-firms', alpr: 'alpr-cameras',
'alpr cameras': 'alpr-cameras', 'flock cameras': 'alpr-cameras', 'license plate readers': 'alpr-cameras',
'license plate cameras': 'alpr-cameras', 'plate readers': 'alpr-cameras', 'local-adsb': 'local-adsb',
'local adsb': 'local-adsb', 'local ads-b': 'local-adsb', 'my receiver': 'local-adsb',
'my antenna': 'local-adsb', 'my sdr': 'local-adsb',
};
for (const [phrase, id] of Object.entries(legacy))
assert.equal(aliases.get(phrase), id, phrase);
for (const [phrase, id] of Object.entries(legacy)) assert.equal(aliases.get(phrase), id, phrase);
const added = {
buses: 'transit',
trains: 'transit',
hurricanes: 'weather-cyclones',
cyclones: 'weather-cyclones',
'weather radar': 'weather-radar',
lightning: 'weather-lightning',
'military bases': 'military-installations',
'recent imagery': 'recent-imagery',
'bike share': 'bikeshare',
'rocket launches': 'rocket-launches',
'street level': 'street-level',
mapillary: 'street-level',
wind: 'wind',
directions: 'directions',
'fire perimeters': 'fire-perimeters',
'wildfire perimeters': 'fire-perimeters',
'burn areas': 'fire-perimeters',
'drone restrictions': 'drone-airspace',
'drone airspace': 'drone-airspace',
'no-fly zones': 'drone-airspace',
drones: 'drone-airspace',
buses: 'transit', trains: 'transit', hurricanes: 'weather-cyclones', cyclones: 'weather-cyclones',
'weather radar': 'weather-radar', lightning: 'weather-lightning', 'military bases': 'military-installations',
'recent imagery': 'recent-imagery', 'bike share': 'bikeshare', 'rocket launches': 'rocket-launches',
'street level': 'street-level', mapillary: 'street-level',
wind: 'wind', directions: 'directions', 'fire perimeters': 'fire-perimeters',
'wildfire perimeters': 'fire-perimeters', 'burn areas': 'fire-perimeters',
};
for (const [phrase, id] of Object.entries(added))
assert.equal(aliases.get(phrase), id, phrase);
for (const [phrase, id] of Object.entries(added)) assert.equal(aliases.get(phrase), id, phrase);
const claimed = new Map();
for (const entry of VOICE_LAYER_MANIFEST)
for (const alias of entry.aliases) {
assert.ok(
!claimed.has(alias),
`"${alias}" claimed by ${claimed.get(alias)} and ${entry.id}`,
);
assert.ok(!claimed.has(alias), `"${alias}" claimed by ${claimed.get(alias)} and ${entry.id}`);
claimed.set(alias, entry.id);
}
});
+1 -1
View File
@@ -4,7 +4,7 @@ import { progressLine } from './speech.js';
* Timing policy for filling silence while tools run. Tuned against the turn
* spans' `max_silence_ms`.
*/
export const NARRATION_TIMING = Object.freeze({
const NARRATION_TIMING = Object.freeze({
/** Start the earcon when nothing is audible this long after end of speech. */
earconAfterMs: 700,
/** First spoken progress line when a tool is still running after this. */
+178
View File
@@ -0,0 +1,178 @@
/**
* Private action accounting prototype. Keys describe invocations, never input
* provenance, successful effects, provider admission or physical settlement.
* The supplied runAction continues to own execution and currentness.
*/
export function createPanelActionObserver({
runAction,
onObservation,
signal,
}) {
if (typeof runAction !== 'function')
throw new TypeError('runAction is required');
const pending = new Map();
let disposed = false;
const publish = (type, record, details = {}) => {
try {
onObservation?.(
Object.freeze({
type,
key: record.key,
name: record.name,
pendingCount: pending.size,
...details,
}),
);
} catch {
// A presentation observer cannot change execution or its result.
}
};
const release = (record) => {
if (!pending.delete(record.key)) return false;
try {
record.signal?.removeEventListener('abort', record.abort);
} catch {
/* Observer cleanup only. */
}
return true;
};
const retireOne = (record, reason) => {
if (release(record)) publish('ownership-retired', record, { reason });
};
const retire = (reason = 'owner-retired') => {
for (const record of [...pending.values()])
retireOne(record, typeof reason === 'string' ? reason : 'owner-retired');
};
const dispose = () => {
if (disposed) return;
disposed = true;
try {
signal?.removeEventListener('abort', dispose);
} catch {
/* Observer cleanup only. */
}
retire('mount-retired');
};
signal?.addEventListener('abort', dispose, { once: true });
if (signal?.aborted) dispose();
function observedAction(...args) {
if (disposed) return Reflect.apply(runAction, this, args);
const record = {
key: Object.freeze({}),
name: typeof args[0] === 'string' ? args[0] : undefined,
};
const originalOptions = args[2];
let forwarded = args;
// Main supplies ordinary data options. Leave exotic/accessor options intact
// rather than evaluating getters or pretending all custom factories fit.
try {
const prototype =
originalOptions == null ? null : Object.getPrototypeOf(originalOptions);
const descriptors =
originalOptions == null
? {}
: Object.getOwnPropertyDescriptors(originalOptions);
const callerSignal = descriptors.signal?.value;
if (callerSignal instanceof AbortSignal) record.signal = callerSignal;
const dataOnly = Object.values(descriptors).every((d) => 'value' in d);
const originalProgress = descriptors.progress?.value;
if (
originalOptions !== undefined &&
originalOptions !== null &&
(prototype === Object.prototype || prototype === null) &&
dataOnly &&
(originalProgress == null || typeof originalProgress === 'function')
) {
const progressDescriptor = descriptors.progress;
delete descriptors.progress;
const options = Object.create(prototype, descriptors);
Object.defineProperty(options, 'progress', {
configurable: true,
enumerable: progressDescriptor?.enumerable ?? true,
writable: true,
value: function (...updates) {
const receiver = this === options ? originalOptions : this;
const returned =
originalProgress == null
? undefined
: Reflect.apply(originalProgress, receiver, updates);
if (pending.has(record.key)) {
// No raw mutable result/options/update object escapes to observers.
try {
const step = Object.getOwnPropertyDescriptor(
updates[0] ?? {},
'step',
)?.value;
const label = Object.getOwnPropertyDescriptor(
updates[0] ?? {},
'label',
)?.value;
if (typeof step === 'string')
publish('progress', record, {
step,
...(typeof label === 'string' ? { label } : {}),
});
} catch {
/* Private metadata inspection cannot change progress. */
}
}
return returned;
},
});
forwarded = [...args];
forwarded[2] = options;
}
} catch {
// Unsupported option introspection is not an execution failure.
}
if (!record.signal?.aborted) {
pending.set(record.key, record);
record.abort = () => retireOne(record, 'caller-aborted');
try {
record.signal?.addEventListener('abort', record.abort, { once: true });
} catch {
retireOne(record, 'unobserved-signal');
}
if (pending.has(record.key)) publish('attempt-started', record);
}
const settle = (outcome) => {
if (release(record)) publish('attempt-settled', record, { outcome });
};
let result;
try {
result = Reflect.apply(runAction, this, forwarded);
} catch (error) {
settle('synchronous-throw');
throw error;
}
// The actual common session returns a same-realm native Promise. Do not
// assimilate arbitrary thenables, read their .then getter or invent their
// settlement; custom return values still reach the caller unchanged.
try {
if (result instanceof Promise) {
Promise.prototype.then.call(
result,
() => settle('fulfilled'),
() => settle('rejected'),
);
} else if (
result === null ||
!['object', 'function'].includes(typeof result)
) {
settle('synchronous-return');
} else retireOne(record, 'unobserved-return-kind');
} catch {
retireOne(record, 'unobserved-return-kind');
}
return result;
}
return {
runAction: observedAction,
retire,
dispose,
get pendingCount() {
return pending.size;
},
};
}
+498
View File
@@ -0,0 +1,498 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { getEventListeners } from 'node:events';
import { createPanelActionObserver } from './actionObserver.js';
import { createVoiceSession } from '../session.js';
function deferred() {
let resolve, reject;
const promise = new Promise((yes, no) => {
resolve = yes;
reject = no;
});
return { promise, resolve, reject };
}
test('sync value/throw, receiver, arguments and opaque unsupported result stay exact', () => {
const receiver = {},
args = {},
options = { extra: 7 };
const error = Error('original');
let calls = 0;
const events = [];
const observer = createPanelActionObserver({
onObservation: (event) => events.push(event),
runAction(name, received, opts, tail) {
assert.equal(this, receiver);
assert.equal(received, args);
assert.equal(opts.extra, 7);
assert.equal(tail, 'tail');
calls++;
if (name === 'throw') throw error;
return 42;
},
});
assert.equal(
observer.runAction.call(receiver, 'value', args, options, 'tail'),
42,
);
assert.throws(
() => observer.runAction.call(receiver, 'throw', args, options, 'tail'),
(e) => e === error,
);
assert.equal(calls, 2);
assert.equal(observer.pendingCount, 0);
assert.deepEqual(
events.filter((e) => e.type === 'attempt-settled').map((e) => e.outcome),
['synchronous-return', 'synchronous-throw'],
);
const opaque = Object.defineProperty({}, 'then', {
get() {
throw Error('must not read then');
},
});
const another = createPanelActionObserver({ runAction: () => opaque });
assert.equal(another.runAction('opaque'), opaque);
assert.equal(another.pendingCount, 0);
});
test('native Promise identity, fulfillment object and rejection object are preserved', async () => {
for (const reject of [false, true]) {
const d = deferred(),
value = { exact: true },
events = [];
const observer = createPanelActionObserver({
runAction: () => d.promise,
onObservation: (e) => events.push(e),
});
const returned = observer.runAction('promise');
assert.equal(returned, d.promise);
assert.equal(observer.pendingCount, 1);
if (reject) {
d.reject(value);
await assert.rejects(returned, (e) => e === value);
} else {
d.resolve(value);
assert.equal(await returned, value);
}
assert.equal(observer.pendingCount, 0);
assert.equal(events.at(-1).outcome, reject ? 'rejected' : 'fulfilled');
assert.equal('result' in events.at(-1), false);
assert.equal('error' in events.at(-1), false);
}
});
test('null/reused callIds have independent keys and settlement cannot retire a sibling', async () => {
const a = deferred(),
b = deferred(),
events = [];
const observer = createPanelActionObserver({
runAction: (name) => (name === 'a' ? a.promise : b.promise),
onObservation: (e) => events.push(e),
});
const pa = observer.runAction('a', {}, { callId: null });
const pb = observer.runAction('b', {}, { callId: null });
const starts = events.filter((e) => e.type === 'attempt-started');
assert.notEqual(starts[0].key, starts[1].key);
assert.equal(observer.pendingCount, 2);
b.resolve('b');
await pb;
assert.equal(observer.pendingCount, 1);
assert.equal(events.at(-1).key, starts[1].key);
a.resolve('a');
await pa;
assert.equal(observer.pendingCount, 0);
});
test('progress preserves public-before-original-before-private order, receiver and option references', async () => {
const order = [],
signal = new AbortController().signal,
symbol = Symbol('unknown');
let currentCalls = 0;
const isCurrent = () => {
currentCalls++;
return true;
};
const options = {
signal,
isCurrent,
progress(update) {
assert.equal(this, options);
order.push('original');
return update;
},
};
Object.defineProperty(options, 'hidden', { value: 8 });
options[symbol] = 9;
const custom = {};
const observer = createPanelActionObserver({
onObservation(e) {
if (e.type === 'progress') order.push('private');
},
runAction(name, args, received) {
assert.equal(received.signal, signal);
assert.equal(received.isCurrent, isCurrent);
assert.equal(received.hidden, 8);
assert.equal(received[symbol], 9);
assert.equal(
Object.getOwnPropertyDescriptor(received, 'hidden').enumerable,
false,
);
order.push('public');
assert.equal(received.progress({ step: 'running' }).step, 'running');
return Promise.resolve();
},
});
await observer.runAction('action', {}, options);
assert.deepEqual(order, ['public', 'original', 'private']);
assert.equal(currentCalls, 0);
const explicit = createPanelActionObserver({
runAction(name, args, received) {
received.progress.call(custom, { step: 'running' });
return 0;
},
});
let originalReceiver;
explicit.runAction(
'explicit',
{},
{
progress() {
originalReceiver = this;
},
},
);
assert.equal(originalReceiver, custom);
});
test('observer faults and private metadata faults cannot replace original callback errors', async () => {
const originalError = Error('callback');
const observer = createPanelActionObserver({
onObservation() {
throw Error('observer');
},
runAction(name, args, options) {
options.progress({ step: 'work' });
return Promise.resolve('ok');
},
});
await assert.rejects(
Promise.resolve().then(() =>
observer.runAction(
'error',
{},
{
progress() {
throw originalError;
},
},
),
),
(e) => e === originalError,
);
const hostile = new Proxy(
{},
{
getOwnPropertyDescriptor() {
throw Error('private read');
},
},
);
const safe = createPanelActionObserver({
runAction(name, args, options) {
options.progress(hostile);
return Promise.resolve('unchanged');
},
onObservation() {
throw Error('observer');
},
});
assert.equal(await safe.runAction('safe', {}, {}), 'unchanged');
});
test('accessor/custom options are forwarded untouched without extra getter/currentness probes', async () => {
let reads = 0,
predicates = 0;
const options = Object.defineProperty(
{
isCurrent() {
predicates++;
return true;
},
},
'progress',
{
get() {
reads++;
return () => {};
},
},
);
const observer = createPanelActionObserver({
runAction(name, args, received) {
assert.equal(received, options);
received.progress({});
received.isCurrent();
return Promise.resolve();
},
});
await observer.runAction('accessor', {}, options);
assert.equal(reads, 1);
assert.equal(predicates, 1);
assert.equal(observer.pendingCount, 0);
});
test('real caller abort retires ownership only; late A cannot release or report over B', async () => {
const a = deferred(),
b = deferred(),
caller = new AbortController(),
events = [],
callbacks = [];
let oldOptions;
const observer = createPanelActionObserver({
onObservation: (e) => events.push(e),
runAction(name, args, options) {
if (name === 'a') oldOptions = options;
return name === 'a' ? a.promise : b.promise;
},
});
const pa = observer.runAction(
'a',
{},
{ signal: caller.signal, progress: () => callbacks.push('original') },
);
caller.abort();
assert.equal(observer.pendingCount, 0);
assert.equal(events.at(-1).type, 'ownership-retired');
assert.equal(
events.some((e) => e.type === 'attempt-settled'),
false,
);
const pb = observer.runAction('b');
const count = events.length;
oldOptions.progress({ step: 'late' });
a.resolve('a');
await pa;
assert.deepEqual(callbacks, ['original']);
assert.equal(events.length, count);
assert.equal(observer.pendingCount, 1);
b.resolve('b');
await pb;
assert.equal(observer.pendingCount, 0);
assert.equal(getEventListeners(caller.signal, 'abort').length, 0);
});
test('already-aborted caller and disposed mount still invoke original exactly once', async () => {
const caller = new AbortController(),
mount = new AbortController();
caller.abort();
let calls = 0;
const events = [];
const observer = createPanelActionObserver({
signal: mount.signal,
onObservation: (e) => events.push(e),
runAction: () => {
calls++;
return Promise.resolve('original');
},
});
assert.equal(
await observer.runAction('aborted', {}, { signal: caller.signal }),
'original',
);
assert.equal(observer.pendingCount, 0);
mount.abort();
assert.equal(await observer.runAction('disposed'), 'original');
assert.equal(calls, 2);
assert.equal(events.length, 0);
assert.equal(getEventListeners(mount.signal, 'abort').length, 0);
});
test('reentrant start retirement retains only fresh ownership and never skips execution', async () => {
const a = deferred(),
b = deferred(),
called = [];
let observer, pb;
observer = createPanelActionObserver({
runAction(name) {
called.push(name);
return name === 'a' ? a.promise : b.promise;
},
onObservation(e) {
if (e.type === 'attempt-started' && e.name === 'a') {
observer.retire('actual-interruption');
pb = observer.runAction('b');
}
},
});
const pa = observer.runAction('a');
assert.deepEqual(called, ['b', 'a']);
assert.equal(observer.pendingCount, 1);
a.resolve();
await pa;
assert.equal(observer.pendingCount, 1);
b.resolve();
await pb;
assert.equal(observer.pendingCount, 0);
});
test('actual pinned-main common session: private invocation stream is separate from public events', async (t) => {
const publicEvents = [],
privateEvents = [],
order = [];
let hooks, observer;
const result = { ok: true, unchanged: true };
const session = createVoiceSession({
runner: async (name, args, options) => {
assert.equal(options.isCurrent(), true);
options.progress({ step: 'actual', label: 'Actual' });
return result;
},
createAdapter(h) {
hooks = h;
observer = createPanelActionObserver({
runAction: h.runAction,
signal: h.signal,
onObservation(e) {
privateEvents.push(e);
if (e.type === 'progress') order.push('private');
},
});
return {
start() {
h.emit({ type: 'state', state: 'listening' });
},
stop() {},
sendText() {},
sendMapEvent() {},
};
},
});
t.after(() => session.destroy());
session.subscribe((e) => {
publicEvents.push(e);
if (e.type === 'progress') order.push('public');
});
await session.start();
assert.equal(
await observer.runAction(
'actual',
{},
{
progress() {
order.push('caller');
},
},
),
result,
);
assert.deepEqual(order, ['public', 'caller', 'private']);
const call = publicEvents.find((e) => e.type === 'action-call');
assert.deepEqual(call, {
type: 'action-call',
name: 'actual',
callId: null,
arguments: {},
});
assert.equal(
publicEvents.some((e) => e.type === 'action-settled'),
false,
);
assert.equal(privateEvents.at(-1).type, 'attempt-settled');
assert.equal(observer.pendingCount, 0);
assert.equal(hooks.signal.aborted, false);
});
test('actual common interruption aborts resources; injected real-lifecycle observation retires uncooperative A', async (t) => {
const a = deferred(),
b = deferred(),
events = [];
let hooks, observer, aSignal;
const session = createVoiceSession({
runner: (name, args, options) => {
if (name === 'a') aSignal = options.signal;
return name === 'a' ? a.promise : b.promise;
},
createAdapter(h) {
hooks = h;
observer = createPanelActionObserver({
runAction: h.runAction,
signal: h.signal,
onObservation: (e) => events.push(e),
});
return {
start() {
h.emit({ type: 'state', state: 'listening' });
},
stop() {},
sendText() {},
sendMapEvent() {},
};
},
});
t.after(() => session.destroy());
await session.start();
const pa = observer.runAction('a');
const rejection = assert.rejects(pa, { name: 'AbortError' });
hooks.emit({ type: 'interruption', reason: 'actual-common-event' });
assert.equal(aSignal.aborted, true);
// This bridge is explicitly supplied by the test, not automatically inferred.
observer.retire('observed-common-interruption');
assert.equal(observer.pendingCount, 0);
const pb = observer.runAction('b');
const count = events.length;
a.resolve({ ok: true });
await rejection;
assert.equal(events.length, count);
assert.equal(observer.pendingCount, 1);
b.resolve({ ok: true });
await pb;
assert.equal(observer.pendingCount, 0);
});
test('null-prototype option descriptors survive instrumentation; mount abort does not fabricate settlement', async () => {
const mount = new AbortController(),
d = deferred(),
events = [],
originalProgress = [];
const options = Object.assign(Object.create(null), {
progress: () => originalProgress.push('original'),
});
let received;
const observer = createPanelActionObserver({
signal: mount.signal,
onObservation: (e) => events.push(e),
runAction(name, args, opts) {
received = opts;
return d.promise;
},
});
const promise = observer.runAction('pending', {}, options);
assert.equal(Object.getPrototypeOf(received), null);
mount.abort();
assert.equal(observer.pendingCount, 0);
assert.equal(events.at(-1).type, 'ownership-retired');
const count = events.length;
received.progress({ step: 'late' });
d.resolve('done');
assert.equal(await promise, 'done');
assert.deepEqual(originalProgress, ['original']);
assert.equal(events.length, count);
assert.equal(getEventListeners(mount.signal, 'abort').length, 0);
});
test('omitted and explicit undefined options keep original arity and values', () => {
const calls = [];
const observer = createPanelActionObserver({
runAction: function (...args) {
calls.push(args);
return 1;
},
});
assert.equal(observer.runAction('omitted', {}), 1);
assert.equal(observer.runAction('explicit', {}, undefined), 1);
assert.equal(calls[0].length, 2);
assert.equal(calls[1].length, 3);
assert.equal(calls[0][2], undefined);
assert.equal(calls[1][2], undefined);
});
+304
View File
@@ -0,0 +1,304 @@
import { t } from '../../i18n/index.js';
import { createManualVoicePanel } from './manualPanel.js';
import {
createVoicePreferences,
bindPreferenceControls,
} from './preferences.js';
import { createPanelActionObserver } from './actionObserver.js';
import { observeNativeActivity } from './nativeActivity.js';
import { createVoiceInactivity } from './inactivity.js';
/** One UI/preference owner across provider mounts; no semantic voice routes. */
export function createVoicePanelHost({
storage,
clock = {},
readCockpitState = () => ({
active:
globalThis.document?.body?.classList?.contains('cockpit-mode') === true,
}),
} = {}) {
const preferences = createVoicePreferences({ storage });
let current = null;
let disposed = false;
let saved = { view: 'preview', panelOpen: false, automatic: true };
function observeActions(hooks) {
const observer = createPanelActionObserver({
runAction: hooks.runAction,
signal: hooks.signal,
onObservation: () => {
if (current?.observer === observer) current.workChanged();
},
});
return observer;
}
function mount({ ui, card, session, observer, nativeProvider }) {
if (disposed || !ui.voicePanel || !card?.setEnabled || !card?.setPresented)
return null;
current?.destroy();
let dead = false;
let revision = 0;
let operationEpoch = 0;
let operationWrite = false;
let panel;
let idle;
let refreshQueued = false;
let automatic = saved.automatic;
let inputBusy = false;
let finalTranscript = null;
const presentationLifetime = new AbortController();
const rearm = () => {
if (!owns() || !session.isActive()) return;
automatic = true;
saved.automatic = true;
card.rearm?.();
// Unknown IDs cannot distinguish a fresh identical answer after input.
if (!finalTranscript?.id) finalTranscript = null;
};
const owns = () =>
!dead && !disposed && current === binding && !session.disposed;
const remember = ({ automatic: auto = false } = {}) => {
if (!auto) {
revision++;
automatic = false;
}
if (panel) {
const value = panel.snapshot();
saved = { view: value.view, panelOpen: value.panelOpen, automatic };
}
};
const renderActivity = () => {
if (ui.voiceTimeoutStatus)
ui.voiceTimeoutStatus.textContent =
nativeProvider === 'openai' || nativeProvider === 'gemini'
? ''
: t('Automatic stop is unavailable for this voice connection.');
};
const changed = () => {
if (!owns()) return;
const busy = activity.snapshot().inputBusy;
if (busy && !inputBusy) rearm();
inputBusy = busy;
idle?.update({ activity: true });
renderActivity();
};
const activity = observeNativeActivity({
adapter: session.adapter,
nativeProvider,
signal: session.signal,
onChange: changed,
onInput: rearm,
});
const closeAfterStop = (ticket) => {
if (owns() && ticket === revision && !session.isActive()) panel.close();
};
const binding = {
observer,
get panel() {
return panel;
},
stopTicket: () => revision,
closeAfterStop,
workChanged: changed,
captureOperation() {
const stamp = revision;
const epoch = operationEpoch;
const isCurrent = () =>
owns() && stamp === revision && epoch === operationEpoch;
const cancelled = () => ({
ok: false,
applied: false,
cancelled: true,
});
return {
isCurrent,
setMinutes(value) {
if (!isCurrent()) return cancelled();
if (typeof value !== 'number' || !Number.isInteger(value))
return {
ok: false,
applied: false,
changed: false,
persisted: null,
reason: 'invalid-value',
minutes: preferences.snapshot().minutes,
};
let result;
operationWrite = true;
try {
result = preferences.setMinutes(value);
} finally {
operationWrite = false;
}
return {
ok: result.ok,
applied: result.applied,
changed: result.changed,
persisted: result.persisted,
minutes: result.minutes,
...(result.reason ? { reason: result.reason } : {}),
};
},
show(view, open = true) {
if (!isCurrent()) return cancelled();
const result = open ? panel.select(view) : panel.close(view);
return { ...result, applied: result.ok };
},
};
},
handle(event) {
if (!owns()) return;
if (event.type !== 'state' && !session.isActive()) return;
if (
event.type === 'state' &&
['connecting', 'idle', 'error'].includes(event.state)
) {
finalTranscript = null;
if (event.state !== 'connecting') saved.automatic = automatic = false;
}
if (
event.type === 'interruption' &&
!event.preserveAdvisoryCaption &&
['user-speech', 'user-text', 'push-to-talk'].includes(event.reason)
)
rearm();
if (
event.type === 'transcript' &&
event.role === 'assistant' &&
event.text
) {
const id = event.responseId || event.itemId || null;
if (
finalTranscript &&
(id ? id === finalTranscript.id : !finalTranscript.id) &&
((!event.final && id) ||
(event.final && event.text === finalTranscript.text))
)
return;
if (event.final) finalTranscript = { id, text: event.text };
if (
automatic &&
preferences.snapshot().previewEnabled &&
!readCockpitState()?.active
)
panel.select('preview', { automatic: true });
}
// A retired native boundary is not fresh input and must not revive
// dismissal or erase the advisory caption it explicitly preserves.
if (!(event.type === 'interruption' && event.preserveAdvisoryCaption))
card.handle(event);
if (
event.type === 'interruption' ||
(event.type === 'state' &&
['connecting', 'idle', 'error'].includes(event.state))
) {
operationEpoch++;
observer.retire('session-boundary');
}
activity.handle(event);
idle?.update();
// Native controllers finish processing after forwarding some events.
// One guarded microtask refresh reads the final current local state.
if (!refreshQueued) {
refreshQueued = true;
queueMicrotask(() => {
refreshQueued = false;
if (!owns()) return;
activity.handle({ type: 'refresh' });
idle?.update();
});
}
},
snapshot: () => ({
...panel.snapshot(),
preferences: preferences.snapshot(),
activity: activity.snapshot(),
inactivity: idle.snapshot(),
}),
destroy() {
if (dead) return;
dead = true;
presentationLifetime.abort();
idle?.destroy();
activity.destroy();
fields.dispose();
observer.dispose();
panel.destroy();
session.signal.removeEventListener('abort', binding.destroy);
if (current === binding) current = null;
},
};
current = binding;
panel = createManualVoicePanel({
ui,
card,
getPreviewEnabled: () => preferences.snapshot().previewEnabled,
getCockpitActive: () => readCockpitState()?.active === true,
copy: {
previewEmpty: t('Voice replies appear here.'),
previewDisabled: t('Preview is off.'),
},
onPresentationIntent: remember,
});
const fields = bindPreferenceControls({
ui,
preferences,
copy: {
previewNote: '',
timeoutNote: t('These preferences stay in this browser.'),
notSaved: t('Your choice applies now, but could not be saved.'),
},
onChange: (state) => {
if (!owns()) return;
if (!operationWrite) revision++;
card.setEnabled(state.previewEnabled);
panel.sync();
idle?.setMinutes(state.minutes);
},
});
globalThis.window?.addEventListener(
'gev:cockpit-mode-changed',
() => {
if (owns()) panel.sync();
},
{ signal: presentationLifetime.signal },
);
ui.root?.querySelector('#gev-voice-card-dismiss')?.addEventListener(
'click',
() => {
remember();
if (panel.snapshot().panelOpen) ui.previewTab?.focus();
},
{ signal: presentationLifetime.signal },
);
card.setEnabled(preferences.snapshot().previewEnabled);
panel.sync();
idle = createVoiceInactivity({
readActivity: activity.snapshot,
readPending: () => observer.pendingCount,
minutes: preferences.snapshot().minutes,
signal: session.signal,
...clock,
onExpire: () => {
if (!owns()) return;
const ticket = revision;
session.stop();
closeAfterStop(ticket);
},
});
if (saved.panelOpen) panel.select(saved.view, { automatic: true });
renderActivity();
session.signal.addEventListener('abort', binding.destroy, { once: true });
if (session.signal.aborted) binding.destroy();
return binding;
}
return {
observeActions,
mount,
snapshot: () => preferences.snapshot(),
destroy() {
if (disposed) return;
current?.destroy();
disposed = true;
},
};
}
+756
View File
@@ -0,0 +1,756 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceControl } from '../control.js';
import { createGevActionRunner } from '../gevActions.js';
import { createVoiceCommands as bind } from '../sessionCommands.js';
import { createVoiceCommands as compose } from '../commands.js';
import { createProviderCommands } from '../providerCommands.js';
import { createVoicePanelHost } from './host.js';
import { installPanelDom, controlledClock } from './testDom.mjs';
function factory(instances, { stopError = null, native = false } = {}) {
return (hooks) => {
const input = {
pushToTalkKeyHeld: false,
setMicrophoneEnabled() {},
releasePushToTalkKey() {
this.pushToTalkKeyHeld = false;
},
resetSession() {},
};
const controller = native
? {
input,
_input: input,
_turns: { observeSpeech() {} },
status: 'idle',
dc: null,
sendTextCommand(text) {
if (!text) return false;
hooks.emit({
type: 'pointer',
pointer: { kind: 'here', label: 'Typed target' },
});
},
createAudio(options) {
this.audioCallbacks = options;
return {
pending: false,
play() {},
clearPlayback() {},
close() {},
};
},
handleMessage() {},
userActivityOpen: false,
pendingTools: 0,
isActive: () => true,
}
: {};
const adapter = {
controller,
capabilities: { pushToTalk: true },
starts: 0,
stops: 0,
bindControls() {},
start() {
this.starts++;
if (native) {
hooks.emit({ type: 'state', state: 'connecting' });
controller.audio = controller.createAudio({ pushToTalk: false });
hooks.emit({ type: 'state', state: 'connecting' });
// Native open-mic startup marks presentation pending before speech.
controller.userTurnPending = true;
}
hooks.emit({ type: 'state', state: 'listening' });
},
stop() {
this.stops++;
if (stopError) throw stopError;
},
sendText: () => false,
sendMapEvent() {},
getStartOptions: () => ({ pushToTalk: false }),
hooks,
input,
};
instances.push(adapter);
return adapter;
};
}
function mounted(t, options = {}) {
const doc = installPanelDom(t);
const clock = controlledClock();
const host = createVoicePanelHost({
storage: options.storage ?? null,
clock,
readCockpitState: options.readCockpitState,
});
const instances = [];
const commands = bind({
runner: options.runner ?? (async () => ({ ok: true })),
dataManager: { layers: new Map() },
createSession: factory(instances, options),
panelHost: host,
nativePanelProvider: options.native
? (options.nativeProvider ?? 'gemini')
: null,
});
t.after(() => {
try {
commands.session.destroy();
} catch {}
host.destroy();
});
return { doc, clock, host, instances, commands, ui: commands.ui };
}
test('actual control default remains opt-out; panel markup has unique controlled panes and an outside alert', (t) => {
const doc = installPanelDom(t);
const baseline = createVoiceControl();
assert.equal(baseline.voicePanel, null);
const ui = createVoiceControl({ reset: true, panel: true });
assert.equal(doc.getElementById('gev-voice-panel'), ui.voicePanel);
assert.equal(
ui.voiceSettingsButton.getAttribute('aria-controls'),
ui.voicePanel.id,
);
assert.equal(ui.previewTab.getAttribute('aria-controls'), ui.previewPanel.id);
assert.equal(
ui.settingsTab.getAttribute('aria-controls'),
ui.voiceSettingsPanel.id,
);
assert.equal(ui.previewPanel.contains(ui.card), true);
assert.equal(
ui.voicePanel.contains(ui.root.querySelector('.gev-voice-error-tray')),
false,
);
assert.equal(ui.root.querySelectorAll('#gev-voice-card').length, 1);
});
test('actual binder/card has launcher toggle, idempotent tabs, focus and disabled Preview', (t) => {
const { ui, commands, doc } = mounted(t);
ui.voiceSettingsButton.fire('click');
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
ui.voiceSettingsButton.fire('click');
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
ui.settingsTab.fire('click');
ui.settingsTab.fire('keydown', { key: 'End' });
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
ui.settingsTab.fire('click');
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
commands.voicePanel.select('preview');
commands.voiceCard.handle({
type: 'transcript',
role: 'assistant',
final: true,
text: 'A visible reply',
});
const dismiss = ui.root.querySelector('#gev-voice-card-dismiss');
dismiss.focus();
dismiss.fire('click');
assert.equal(ui.card.hidden, true);
assert.equal(commands.voicePanel.snapshot().panelOpen, true);
assert.equal(doc.activeElement?.id, ui.previewTab.id);
ui.voicePanel.fire('keydown', { key: 'Escape', target: doc.activeElement });
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
commands.voicePanel.select('settings');
ui.voicePreviewInput.checked = false;
ui.voicePreviewInput.fire('change');
commands.voicePanel.select('preview');
commands.voiceCard.handle({
type: 'transcript',
role: 'assistant',
final: true,
text: 'A hidden reply',
});
assert.equal(ui.card.hidden, true);
assert.equal(ui.previewEmpty.textContent, 'Preview is off.');
ui.previewTab.focus();
ui.voicePanel.fire('keydown', { key: 'Escape' });
assert.equal(doc.activeElement, ui.voiceSettingsButton);
commands.voiceCard.handle({
type: 'transcript',
role: 'assistant',
final: true,
text: 'Later reply',
});
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
});
test('actual facade remount keeps failed-save live preferences/view and manual close, with no restore writes or focus theft', async (t) => {
const doc = installPanelDom(t);
const writes = [];
const storage = {
getItem: () => null,
setItem(key, value) {
writes.push([key, value]);
throw Error('blocked');
},
};
const host = createVoicePanelHost({ storage });
const instances = [];
const commands = createProviderCommands(
{
runner: async () => ({}),
dataManager: { layers: new Map() },
supportedProviders: ['openai', 'gemini'],
},
{
bind: (options) => bind({ ...options, panelHost: host }),
factories: { openai: factory(instances), gemini: factory(instances) },
storage,
},
);
t.after(() => {
commands.destroy();
host.destroy();
});
const old = commands.ui;
commands.voicePanel.select('settings');
old.voicePreviewInput.checked = false;
old.voicePreviewInput.fire('change');
old.voiceTimeoutInput.value = '7';
old.voiceTimeoutSelect.value = 'custom';
old.voiceTimeoutSelect.fire('change');
const preferenceWrites = () =>
writes.filter(([key]) => key.startsWith('godsEyeView.voice.')).length;
assert.equal(preferenceWrites(), 2);
old.providerSelect.value = 'gemini';
old.providerSelect.fire('change');
assert.notEqual(commands.ui, old);
assert.equal(old.root.isConnected, false);
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
assert.deepEqual(commands.getVoicePreferences(), {
previewEnabled: false,
minutes: 7,
previewPersisted: false,
timeoutPersisted: false,
});
assert.equal(commands.ui.voicePreviewInput.checked, false);
assert.match(commands.ui.voiceTimeoutNote.textContent, /could not be saved/);
assert.equal(preferenceWrites(), 2);
assert.equal(doc.activeElement, commands.ui.providerSelect);
commands.ui.voicePreviewInput.checked = true;
commands.ui.voicePreviewInput.fire('change');
commands.voicePanel.close();
await commands.start();
const retired = instances[1];
commands.ui.providerSelect.value = 'openai';
commands.ui.providerSelect.fire('change');
await commands.start();
assert.equal(commands.session.isActive(), true);
retired.hooks.emit({
type: 'transcript',
role: 'assistant',
text: 'Retired output',
final: true,
});
const emit = instances[2].hooks.emit;
emit({
type: 'transcript',
role: 'assistant',
text: 'Current output after remount',
final: true,
});
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
emit({ type: 'interruption', reason: 'user-speech' });
emit({
type: 'transcript',
role: 'assistant',
text: 'Reply after genuine input',
final: true,
});
assert.equal(commands.voicePanel.snapshot().previewVisible, true);
});
test('explicit successful mic Stop closes; failed Stop idle and startup error do not', async (t) => {
const good = mounted(t);
await good.commands.session.start();
good.commands.voicePanel.select('settings');
good.ui.button.fire('click');
assert.equal(good.commands.voicePanel.snapshot().panelOpen, false);
const error = Error('stop failed');
const bad = mounted(t, { stopError: error });
await bad.commands.session.start();
bad.commands.voicePanel.select('settings');
assert.throws(
() => bad.ui.button.fire('click'),
(value) => value === error,
);
assert.equal(bad.commands.session.state, 'idle');
assert.equal(bad.commands.voicePanel.snapshot().settingsOpen, true);
bad.instances[0].hooks.emit({
type: 'state',
state: 'error',
detail: 'startup failed',
});
assert.equal(bad.commands.voicePanel.snapshot().settingsOpen, true);
});
test('reentrant manual choice/new start during Stop cannot close the successor choice', async (t) => {
const f = mounted(t);
await f.commands.session.start();
f.commands.voicePanel.select('settings');
const unsubscribe = f.commands.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'idle')
f.commands.voicePanel.select('preview');
});
f.ui.button.fire('click');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
unsubscribe();
});
test('integrated inactivity rechecks held PTT at expiry, then normal stop closes on later quiet expiry', async (t) => {
const f = mounted(t, { native: true });
await f.commands.session.start();
f.commands.voicePanel.select('settings');
f.ui.voiceTimeoutInput.value = '1';
f.ui.voiceTimeoutSelect.value = 'custom';
f.ui.voiceTimeoutSelect.fire('change');
f.instances[0].input.pushToTalkKeyHeld = true;
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 0);
f.instances[0].input.releasePushToTalkKey();
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 1);
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
});
test('None cancels integrated app timer and unknown custom hosts never guess expiry', async (t) => {
const native = mounted(t, { native: true });
await native.commands.session.start();
native.ui.voiceTimeoutSelect.value = '0';
native.ui.voiceTimeoutSelect.fire('change');
native.clock.advance(600000);
assert.equal(native.instances[0].stops, 0);
assert.equal(native.commands.getVoicePreferences().minutes, 0);
const unknown = mounted(t);
await unknown.commands.session.start();
unknown.clock.advance(600000);
assert.equal(unknown.instances[0].stops, 0);
assert.match(unknown.ui.voiceTimeoutStatus.textContent, /unavailable/);
});
test('transient native OpenAI response activity does not announce connection unavailability', async (t) => {
const f = mounted(t, { native: true, nativeProvider: 'openai' });
await f.commands.session.start();
f.ui.voiceTimeoutInput.value = '1';
f.ui.voiceTimeoutSelect.value = 'custom';
f.ui.voiceTimeoutSelect.fire('change');
f.instances[0].controller._turns.observeSpeech({ type: 'response.created' });
assert.equal(f.ui.voiceTimeoutStatus.textContent, '');
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 0, 'unknown output still prevents expiry');
f.instances[0].controller._turns.observeSpeech({
type: 'output_audio_buffer.cleared',
});
f.clock.advance(60000);
assert.equal(f.instances[0].stops, 1, 'known quiet output permits expiry');
});
test('native factory with custom controller remains uninstrumented despite panel opt-in', (t) => {
installPanelDom(t);
let inspected = 0;
const commands = compose({
panel: true,
panelStorage: null,
runner: async () => ({}),
dataManager: { layers: new Map() },
createController: (options) => ({
ui: options.ui,
get _turns() {
inspected++;
throw Error('custom internal getter');
},
start() {},
stop() {},
sendTextCommand() {},
notifyMapEvent() {},
syncCostUi() {},
bindPushToTalkShortcut() {},
toggleVoiceTier() {},
}),
});
t.after(() => commands.session.destroy());
assert.equal(inspected, 0);
assert.equal(commands.getVoicePanelState().activity.known, false);
assert.equal(inspected, 0);
});
test('native connecting phases retain output observation; idle starts before first speech and waits for drain', async (t) => {
const f = mounted(t, { native: true });
await f.commands.session.start();
const adapter = f.instances[0];
const controller = adapter.controller;
assert.equal(
f.commands.getVoicePanelState().inactivity.eligible,
true,
'open microphone presentation-pending is not admitted speech',
);
controller.audio.pending = true;
controller.handleMessage({});
f.clock.advance(300000);
assert.equal(adapter.stops, 0);
assert.equal(f.commands.getVoicePanelState().activity.outputBusy, true);
controller.audio.pending = false;
controller.audioCallbacks.onDrain();
f.clock.advance(300000);
assert.equal(adapter.stops, 1);
});
async function directActions(
t,
{ storage = null, delayExit = false, exposePanel = true } = {},
) {
installPanelDom(t);
let f, releaseExit;
const state = { active: true, exits: 0, visual: [] };
const styleManager = {
getCockpitState: () => ({ active: state.active }),
controlCockpit(action) {
assert.equal(action, 'exit');
state.exits++;
state.active = false;
const result = { ok: true, state: { active: false } };
return delayExit
? new Promise((resolve) => {
releaseExit = () => resolve(result);
})
: result;
},
setPanelCollapsed(...args) {
state.visual.push(args);
},
};
const runner = createGevActionRunner({
viewer: {
clock: { onTick: { addEventListener: () => () => {} } },
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
camera: { moveEnd: { addEventListener() {} } },
},
styleManager,
dataManager: { layers: new Map(), getAll: () => [] },
readVoicePanelOperation: () =>
exposePanel ? f?.commands.captureVoicePanelOperation() : undefined,
});
f = mounted(t, { native: true, runner, storage });
await f.commands.session.start();
return {
...f,
runner,
state,
run: f.instances[0].hooks.runAction,
releaseExit: () => releaseExit(),
};
}
test('actual runner applies exact timeout directly and reports live-unsaved truth, invalid/cancelled attempts stay inert', async (t) => {
const f = await directActions(t);
const result = await f.run('set_voice_inactivity_timeout', { minutes: 7 });
assert.equal(result.applied, true);
assert.equal(result.persisted, false);
assert.equal(result.minutes, 7);
assert.match(result.say, /could not be saved/);
assert.equal(f.commands.getVoicePreferences().minutes, 7);
const invalid = await f.run('set_voice_inactivity_timeout', { minutes: 7.5 });
assert.equal(invalid.applied, false);
assert.equal(f.commands.getVoicePreferences().minutes, 7);
const abort = new AbortController();
abort.abort();
const cancelled = await f.runner(
'set_voice_inactivity_timeout',
{ minutes: 2 },
{ signal: abort.signal },
);
assert.equal(cancelled.cancelled, true);
assert.equal(f.commands.getVoicePreferences().minutes, 7);
const operation = f.commands.captureVoicePanelOperation();
f.commands.session.stop();
assert.equal(operation.setMinutes(2).cancelled, true);
});
test('named voice settings exits Cockpit directly; Preview stays disabled and visual presets stay separate from Scenes', async (t) => {
const f = await directActions(t);
const result = await f.run('set_panel_open', {
panelId: 'voice-settings',
open: true,
});
assert.equal(f.state.exits, 1);
assert.equal(result.settingsOpen, true);
f.ui.voicePreviewInput.checked = false;
f.ui.voicePreviewInput.fire('change');
const preview = await f.run('set_panel_open', {
panelId: 'voice control',
open: true,
});
assert.equal(preview.view, 'preview');
assert.equal(preview.previewEnabled, false);
assert.equal(f.commands.getVoicePreferences().previewEnabled, false);
for (const panelId of [
'visual styles',
'visual preset',
'visual presets',
'scenes',
])
await f.run('set_panel_open', { panelId, open: true });
assert.equal(f.state.exits, 1);
assert.deepEqual(
f.state.visual.map(([id]) => id),
['control-panel', 'control-panel', 'control-panel', 'scene-panel'],
);
await assert.rejects(
() => f.run('set_panel_open', { panelId: 'settings', open: true }),
/Unknown panel: settings/,
);
assert.equal(
f.state.visual.length,
4,
'an unnamed settings target never guesses a panel',
);
});
test('canonical exit reveals requested voice UI; newer manual choice or Stop defeats pending reveal', async (t) => {
const f = await directActions(t, { delayExit: true });
const pending = f.run('control_cockpit', {
action: 'exit',
voicePanel: 'settings',
});
f.commands.voicePanel.select('preview');
f.releaseExit();
const result = await pending;
assert.equal(result.cancelled, true);
assert.equal(f.state.active, false);
assert.equal(result.exit.ok, true);
assert.equal(result.cockpitExited, true);
assert.match(result.say, /Cockpit exited/);
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
f.state.active = true;
const alias = f.run('control_cockpit', {
action: 'leave cockpit',
voicePanel: 'settings',
});
f.releaseExit();
assert.equal((await alias).settingsOpen, true);
f.commands.voicePanel.select('preview');
f.state.active = true;
const stopped = f.run('control_cockpit', {
action: 'quit cockpit',
voicePanel: 'settings',
});
f.commands.session.stop();
f.releaseExit();
await assert.rejects(stopped, { name: 'AbortError' });
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
});
test('unavailable host refuses direct voice operations before Cockpit or preference effects', async (t) => {
const f = await directActions(t, { exposePanel: false });
const timeout = await f.run('set_voice_inactivity_timeout', { minutes: 1 });
assert.equal(timeout.unavailable, true);
assert.equal(timeout.applied, false);
const panel = await f.run('set_panel_open', {
panelId: 'voice-settings',
open: true,
});
assert.equal(panel.unavailable, true);
assert.equal(f.state.exits, 0);
assert.equal(f.commands.getVoicePreferences().minutes, 5);
});
test('voice timeout change preserves a pending voice reveal', async (t) => {
const f = await directActions(t, { delayExit: true });
const pending = f.run('control_cockpit', {
action: 'exit',
voicePanel: 'settings',
});
const timeout = await f.run('set_voice_inactivity_timeout', { minutes: 0 });
assert.equal(timeout.minutes, 0);
assert.equal(timeout.applied, true);
f.releaseExit();
const result = await pending;
assert.equal(result.settingsOpen, true);
assert.equal(f.commands.getVoicePreferences().minutes, 0);
});
test('active provider output reveals Preview after done; manual choice wins until genuine input, replays do not reset idle', async (t) => {
const f = mounted(t, { native: true, nativeProvider: 'openai' });
await f.commands.session.start();
const emit = f.instances[0].hooks.emit;
const reply = (text, responseId = 'r1', extra = {}) =>
emit({
type: 'transcript',
role: 'assistant',
text,
responseId,
final: true,
...extra,
});
emit({ type: 'completion', responseId: 'r1' });
reply('Late answer');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
const timer = f.commands.voiceCard.hideTimer;
reply('Late answer');
reply('Late', 'r1', { final: false });
assert.equal(f.commands.voiceCard.hideTimer, timer);
assert.equal(f.commands.voiceCard.state.assistant, 'Late answer');
f.commands.voicePanel.close();
emit({
type: 'transcript',
role: 'user',
text: 'advisory caption',
final: false,
});
f.commands.voiceCard.handle({ type: 'dismiss' });
const advisory = f.commands.voiceCard.state.user;
emit({
type: 'interruption',
reason: 'user-speech',
preserveAdvisoryCaption: true,
});
assert.equal(f.commands.voiceCard.state.dismissed, true);
assert.equal(f.commands.voiceCard.state.user, advisory);
reply('Older reply', 'r2');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
emit({ type: 'interruption', reason: 'user-speech' });
reply('A reply with no newest-request attribution', 'older-id');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
f.commands.voicePanel.select('settings');
reply('Another reply', 'r3');
assert.equal(f.commands.voicePanel.snapshot().settingsOpen, true);
f.commands.voicePanel.close();
f.commands.voiceCard.handle({ type: 'dismiss' });
f.commands.sendTextCommand('Show the typed target');
reply('Typed reply', 'typed-id');
assert.equal(f.commands.voicePanel.snapshot().previewVisible, true);
assert.equal(f.commands.voiceCard.state.dismissed, false);
assert.equal(f.commands.voiceCard.state.pointer.label, 'Typed target');
f.commands.session.stop();
reply('Stopped stale output', 'r4');
assert.equal(f.commands.voiceCard.state.assistant, null);
f.commands.session.destroy();
reply('Disposed stale output', 'r5');
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
});
test('Preview off and actual Cockpit state suppress automatic presentation without enabling preference', async (t) => {
let cockpit = false;
const f = mounted(t, {
native: true,
readCockpitState: () => ({ active: cockpit }),
});
await f.commands.session.start();
const emit = f.instances[0].hooks.emit;
const reply = (text) =>
emit({ type: 'transcript', role: 'assistant', text, final: true });
reply('Visible');
f.ui.previewTab.focus();
cockpit = true;
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
assert.equal(f.ui.voicePanel.hidden, true);
assert.equal(f.doc.activeElement, f.ui.voiceSettingsButton);
f.commands.voicePanel.select('settings', { focus: true });
assert.equal(f.doc.activeElement, f.ui.settingsTab);
f.commands.voicePanel.select('preview', { focus: true });
assert.equal(f.doc.activeElement, f.ui.voiceSettingsButton);
const outside = f.doc.createElement('button');
f.doc.body.appendChild(outside);
outside.focus();
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
assert.equal(f.doc.activeElement, outside);
assert.equal(f.commands.voicePanel.snapshot().previewVisible, false);
reply('Suppressed in Cockpit');
assert.equal(f.ui.voicePanel.hidden, true);
f.commands.voicePanel.close();
cockpit = false;
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
assert.equal(f.ui.voicePanel.hidden, true);
f.ui.voicePreviewInput.checked = false;
f.ui.voicePreviewInput.fire('change');
emit({ type: 'interruption', reason: 'user-speech' });
reply('Preference stays off');
assert.equal(f.ui.voicePanel.hidden, true);
assert.equal(f.commands.getVoicePreferences().previewEnabled, false);
});
test('Provider Settings uses its ready dialog handlers and refuses unavailable or cancelled requests', async (t) => {
const f = await directActions(t);
t.mock.timers.enable({ apis: ['setTimeout'] });
const request = (open, options) =>
f.run('set_panel_open', { panelId: 'provider-settings', open }, options);
assert.equal((await request(true)).unavailable, true);
const doc = globalThis.document;
const chip = doc.createElement('button');
chip.id = 'key-setup-chip';
const panel = doc.createElement('aside');
panel.id = 'key-setup';
panel.hidden = true;
const close = doc.createElement('button');
panel.appendChild(close);
doc.body.append(chip, panel);
const query = panel.querySelector.bind(panel);
panel.querySelector = (selector) =>
selector === '[data-key-setup-close]' ? close : query(selector);
chip.click = () => chip.fire('click');
close.click = () => close.fire('click');
let frames = 0;
const previousFrame = globalThis.requestAnimationFrame;
globalThis.requestAnimationFrame = (callback) => {
frames++;
callback();
};
t.after(() => {
if (previousFrame === undefined) delete globalThis.requestAnimationFrame;
else globalThis.requestAnimationFrame = previousFrame;
});
const { initKeySetup } = await import('../../keySetup.js');
let finishStatus;
const pending = initKeySetup({
documentRef: doc,
fetchImpl: () =>
new Promise((resolve) => {
finishStatus = resolve;
}),
});
assert.equal(
(await request(true)).unavailable,
true,
'initializing is not ready',
);
finishStatus({
ok: true,
json: async () => ({ keys: [], setCount: 0, total: 1 }),
});
const setup = await pending;
t.after(() => setup.destroy());
doc.body.classList.add('cockpit-mode');
assert.equal((await request(true)).unavailable, true);
assert.equal(
panel.hidden,
true,
'a concealed view cannot schedule a later surprise reveal',
);
doc.body.classList.remove('cockpit-mode');
const abort = new AbortController();
abort.abort();
await assert.rejects(() => request(true, { signal: abort.signal }), {
name: 'AbortError',
});
assert.equal(panel.hidden, true);
assert.equal((await request(true)).ok, true);
assert.equal(panel.hidden, false);
assert.equal(panel.classList.contains('visible'), true);
assert.equal((await request(true)).ok, true);
assert.equal(frames, 1, 'repeated open reuses the dialog');
assert.equal((await request(false)).ok, true);
assert.equal(panel.classList.contains('visible'), false);
t.mock.timers.tick(400);
assert.equal(panel.hidden, true);
assert.equal((await request(false)).ok, true);
assert.equal(
f.state.visual.length,
0,
'provider setup never opens a visual or scene panel',
);
});
+77
View File
@@ -0,0 +1,77 @@
/** Local inactivity only. Unknown activity never authorizes expiry. */
export function createVoiceInactivity({
readActivity,
readPending,
minutes = 5,
onExpire,
signal,
setTimer = setTimeout,
clearTimer = clearTimeout,
now = Date.now,
}) {
if (!Number.isInteger(minutes) || minutes < 0 || minutes > 60) minutes = 5;
let disposed = false,
timer = null,
deadline = null,
eligible = false,
epoch = 0;
function cancel() {
epoch++;
if (timer !== null) clearTimer(timer);
timer = deadline = null;
}
function quiet() {
const a = readActivity();
return (
a.known &&
a.connected &&
!a.held &&
!a.inputBusy &&
!a.outputBusy &&
!a.generationBusy &&
!readPending()
);
}
function arm() {
cancel();
if (!minutes || disposed) return;
const id = epoch;
deadline = now() + minutes * 60000;
timer = setTimer(() => {
if (disposed || id !== epoch) return;
timer = deadline = null;
eligible = quiet();
if (eligible) onExpire?.();
}, minutes * 60000);
}
function update({ activity = false } = {}) {
if (disposed) return;
const next = quiet();
if (!next) cancel();
else if (!eligible || activity) arm();
eligible = next;
}
function setMinutes(value) {
if (!Number.isInteger(value) || value < 0 || value > 60) return false;
if (disposed) return false;
minutes = value;
eligible = false;
update();
return true;
}
function destroy() {
if (disposed) return;
disposed = true;
cancel();
signal?.removeEventListener('abort', destroy);
}
signal?.addEventListener('abort', destroy, { once: true });
if (signal?.aborted) destroy();
update();
return {
update,
setMinutes,
snapshot: () => ({ minutes, deadline, eligible, disposed }),
destroy,
};
}
+108
View File
@@ -0,0 +1,108 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceInactivity } from './inactivity.js';
function fixture() {
let activity = {
known: true,
connected: true,
held: false,
inputBusy: false,
outputBusy: false,
generationBusy: false,
};
let pending = 0,
expired = 0,
time = 100,
serial = 0;
const timers = new Map();
const idle = createVoiceInactivity({
readActivity: () => activity,
readPending: () => pending,
onExpire: () => expired++,
now: () => time,
setTimer: (fn, delay) => {
timers.set(++serial, { fn, delay });
return serial;
},
clearTimer: (id) => timers.delete(id),
});
return {
idle,
activity,
timers,
setPending: (n) => (pending = n),
expired: () => expired,
fire: () => {
const entry = [...timers][0];
timers.delete(entry[0]);
entry[1].fn();
},
tick: (n) => (time += n),
};
}
test('one clock; quiet arms default five minutes; busy blocks and quiescence rearms', () => {
const f = fixture();
assert.equal([...f.timers.values()][0].delay, 300000);
assert.equal(f.idle.snapshot().deadline, 300100);
f.activity.outputBusy = true;
f.idle.update();
assert.equal(f.timers.size, 0);
f.activity.outputBusy = false;
f.tick(200);
f.idle.update();
assert.equal(f.idle.snapshot().deadline, 300300);
f.fire();
assert.equal(f.expired(), 1);
});
test('actual activity resets but silent unchanged snapshots do not', () => {
const f = fixture();
const deadline = f.idle.snapshot().deadline;
f.tick(1000);
f.idle.update();
assert.equal(f.idle.snapshot().deadline, deadline);
f.idle.update({ activity: true });
assert.equal(f.idle.snapshot().deadline, deadline + 1000);
});
test('None cancels; invalid values cannot replace setting; explicit retry rearms', () => {
const f = fixture();
assert.equal(f.idle.setMinutes(0), true);
assert.equal(f.timers.size, 0);
for (const bad of [1.5, -1, 61, '5', NaN])
assert.equal(f.idle.setMinutes(bad), false);
assert.equal(f.idle.setMinutes(1), true);
assert.equal([...f.timers.values()][0].delay, 60000);
});
test('expiry rereads held, unknown and pending counts despite missing update', () => {
for (const kind of ['held', 'unknown', 'pending']) {
const f = fixture();
if (kind === 'held') f.activity.held = true;
if (kind === 'unknown') f.activity.known = false;
if (kind === 'pending') f.setPending(2);
f.fire();
assert.equal(f.expired(), 0);
}
});
test('pending siblings and held-before-connect forbid timer; stale callback cannot expire', () => {
const f = fixture();
const stale = [...f.timers.values()][0].fn;
f.setPending(2);
f.idle.update();
stale();
assert.equal(f.expired(), 0);
f.setPending(1);
f.idle.update();
assert.equal(f.timers.size, 0);
f.setPending(0);
f.activity.connected = false;
f.activity.held = true;
f.idle.update();
assert.equal(f.timers.size, 0);
f.activity.connected = true;
f.idle.update();
assert.equal(f.timers.size, 0);
f.activity.held = false;
f.idle.update();
assert.equal(f.timers.size, 1);
f.idle.destroy();
assert.equal(f.timers.size, 0);
});
+182
View File
@@ -0,0 +1,182 @@
const ROW_GRACE_MS = 700;
/** Manual presentation only. Session and action events belong to the host. */
export function createManualVoicePanel({
ui,
card,
getPreviewEnabled,
getCockpitActive = () => false,
copy,
onPresentationIntent,
}) {
if (!ui?.voicePanel || typeof getPreviewEnabled !== 'function')
throw new TypeError(
'A voice panel and Preview preference reader are required.',
);
if (
typeof copy?.previewEmpty !== 'string' ||
typeof copy?.previewDisabled !== 'string'
)
throw new TypeError('Localized Preview messages are required.');
const lifetime = new AbortController();
const panel = ui.voicePanel;
const tabs = [ui.previewTab, ui.settingsTab];
const doc = panel.ownerDocument;
let disposed = false;
let open = false;
let view = 'preview';
const enabled = () => getPreviewEnabled() === true;
const suppressed = () => view === 'preview' && getCockpitActive() === true;
const visible = (element) =>
Boolean(
element &&
!element.hidden &&
element.isConnected !== false &&
element.checkVisibility?.({
opacityProperty: true,
visibilityProperty: true,
}) !== false,
);
const snapshot = () => ({
view,
panelOpen: !disposed && open,
settingsOpen: !disposed && open && view === 'settings',
previewEnabled: enabled(),
previewVisible:
!disposed &&
open &&
view === 'preview' &&
!suppressed() &&
enabled() &&
visible(ui.card),
});
const sync = () => {
if (disposed) return;
const hidePanel = !open || suppressed();
const restoreFocus = hidePanel && panel.contains?.(doc?.activeElement);
panel.hidden = hidePanel;
panel.dataset.view = view;
if (ui.previewPanel)
ui.previewPanel.hidden = !open || view !== 'preview' || suppressed();
if (ui.voiceSettingsPanel)
ui.voiceSettingsPanel.hidden = !open || view !== 'settings';
ui.voiceSettingsButton?.setAttribute(
'aria-expanded',
String(open && view === 'settings'),
);
tabs.forEach((tab, index) => {
if (!tab) return;
const selected = view === (index === 0 ? 'preview' : 'settings');
tab.setAttribute('aria-selected', String(selected));
tab.tabIndex = selected ? 0 : -1;
});
card?.setPresented?.(open && view === 'preview' && !suppressed());
if (ui.previewEmpty) {
ui.previewEmpty.hidden = Boolean(ui.card && !ui.card.hidden);
ui.previewEmpty.textContent = enabled()
? copy.previewEmpty
: copy.previewDisabled;
}
if (restoreFocus)
[ui.voiceSettingsButton, ui.button].find(visible)?.focus?.();
};
const refuse = (reason) => ({
ok: false,
changed: false,
reason,
...snapshot(),
});
const select = (next, { focus = false, automatic = false } = {}) => {
if (disposed) return refuse('disposed');
if (!['preview', 'settings'].includes(next)) return refuse('unknown-view');
const changed = !open || view !== next;
const outgoing =
view === 'preview' ? ui.previewPanel : ui.voiceSettingsPanel;
const transferFocus = changed && outgoing?.contains?.(doc?.activeElement);
open = true;
view = next;
onPresentationIntent?.({ automatic });
sync();
const tab = tabs[next === 'preview' ? 0 : 1];
if ((focus || transferFocus) && visible(panel) && visible(tab))
tab.focus?.();
return { ok: true, changed, ...snapshot() };
};
const close = (onlyView) => {
if (disposed) return refuse('disposed');
if (onlyView && !['preview', 'settings'].includes(onlyView))
return refuse('unknown-view');
if (onlyView && onlyView !== view)
return { ok: true, changed: false, ...snapshot() };
const changed = open;
open = false;
onPresentationIntent?.();
sync();
return { ok: true, changed, ...snapshot() };
};
const listen = (element, type, handler) =>
element?.addEventListener(type, handler, { signal: lifetime.signal });
const toggleSettings = () =>
open && view === 'settings'
? close('settings')
: select('settings', { focus: true });
try {
listen(ui.voiceSettingsButton, 'click', toggleSettings);
tabs.forEach((tab, index) => {
listen(tab, 'click', () =>
select(index === 0 ? 'preview' : 'settings', { focus: true }),
);
listen(tab, 'keydown', (event) => {
const next =
event.key === 'Home'
? 0
: event.key === 'End'
? 1
: ['ArrowLeft', 'ArrowRight'].includes(event.key)
? 1 - index
: null;
if (next === null) return;
event.preventDefault();
select(next === 0 ? 'preview' : 'settings', { focus: true });
});
});
listen(ui.voiceSettingsClose, 'click', () => close());
listen(panel, 'keydown', (event) => {
if (event.key !== 'Escape' || !open) return;
event.preventDefault();
close();
});
// Keep the raised row reachable across pinned trays without intercepting
// their controls. The same lifetime owns the listeners and pending timer.
let graceTimer;
const endGrace = () => {
clearTimeout(graceTimer);
if (ui.root) delete ui.root.dataset.rowGrace;
};
listen(ui.root, 'pointerleave', (event) => {
if (event.pointerType === 'touch') return;
endGrace();
ui.root.dataset.rowGrace = 'true';
graceTimer = setTimeout(endGrace, ROW_GRACE_MS);
});
listen(ui.root, 'pointerenter', endGrace);
lifetime.signal.addEventListener('abort', endGrace, { once: true });
sync();
} catch (error) {
lifetime.abort();
throw error;
}
return {
select,
close,
sync,
snapshot,
destroy() {
if (disposed) return;
open = false;
sync();
disposed = true;
lifetime.abort();
},
};
}
+198
View File
@@ -0,0 +1,198 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createManualVoicePanel } from './manualPanel.js';
function fixture(t) {
const doc = { activeElement: null };
const node = () =>
Object.assign(new EventTarget(), {
hidden: false,
isConnected: true,
dataset: {},
attributes: {},
ownerDocument: doc,
setAttribute(name, value) {
this.attributes[name] = value;
},
focus() {
doc.activeElement = this;
},
contains(other) {
return other === this;
},
checkVisibility() {
return !this.hidden;
},
});
const ui = Object.fromEntries(
[
'root',
'voicePanel',
'previewPanel',
'voiceSettingsPanel',
'previewTab',
'settingsTab',
'voiceSettingsButton',
'voiceSettingsClose',
'previewEmpty',
'card',
'button',
].map((key) => [key, node()]),
);
ui.voicePanel.contains = (element) =>
Object.values(ui).includes(element) &&
![ui.root, ui.voiceSettingsButton, ui.button].includes(element);
let enabled = true,
content = false,
intents = 0;
const card = {
setPresented(value) {
this.presented = value;
ui.card.hidden = !(value && enabled && content);
},
};
const copy = {
previewEmpty: 'Habla para ver resultados.',
previewDisabled: 'Vista previa desactivada.',
};
const panel = createManualVoicePanel({
ui,
card,
getPreviewEnabled: () => enabled,
copy,
onPresentationIntent: () => intents++,
});
t.after(() => panel.destroy());
return {
ui,
panel,
doc,
card,
copy,
intents: () => intents,
enabled(value) {
enabled = value;
panel.sync();
},
content(value) {
content = value;
panel.sync();
},
};
}
const click = (element) => element.dispatchEvent(new Event('click'));
function key(element, value) {
const event = new Event('keydown', { cancelable: true });
Object.defineProperty(event, 'key', { value });
element.dispatchEvent(event);
return event;
}
test('Settings launcher toggles while tab and programmatic selection remain idempotent', (t) => {
const f = fixture(t);
assert.equal(f.panel.snapshot().panelOpen, false);
click(f.ui.voiceSettingsButton);
assert.equal(f.panel.snapshot().settingsOpen, true);
assert.equal(f.doc.activeElement, f.ui.settingsTab);
assert.equal(f.ui.voiceSettingsButton.attributes['aria-expanded'], 'true');
click(f.ui.voiceSettingsButton);
assert.equal(f.panel.snapshot().panelOpen, false);
click(f.ui.settingsTab);
assert.equal(f.panel.snapshot().settingsOpen, true);
assert.equal(f.panel.select('settings').changed, false);
assert.equal(f.panel.snapshot().settingsOpen, true);
click(f.ui.settingsTab);
assert.equal(f.panel.snapshot().settingsOpen, true);
assert.equal(f.panel.select('unknown').reason, 'unknown-view');
});
test('tab keyboard navigation and Escape preserve focus and accessibility state', (t) => {
const f = fixture(t);
click(f.ui.voiceSettingsButton);
assert.equal(key(f.ui.settingsTab, 'ArrowLeft').defaultPrevented, true);
assert.equal(f.panel.snapshot().view, 'preview');
assert.equal(f.doc.activeElement, f.ui.previewTab);
assert.equal(f.ui.previewTab.tabIndex, 0);
assert.equal(f.ui.settingsTab.tabIndex, -1);
assert.equal(f.ui.previewTab.attributes['aria-selected'], 'true');
key(f.ui.previewTab, 'End');
assert.equal(f.panel.snapshot().view, 'settings');
key(f.ui.settingsTab, 'Home');
assert.equal(f.panel.snapshot().view, 'preview');
assert.equal(key(f.ui.voicePanel, 'Escape').defaultPrevented, true);
assert.equal(f.panel.snapshot().panelOpen, false);
assert.equal(f.doc.activeElement, f.ui.voiceSettingsButton);
assert.equal(key(f.ui.voicePanel, 'Escape').defaultPrevented, false);
});
test('manual switching transfers hidden-panel focus without stealing unrelated focus', (t) => {
const f = fixture(t);
f.panel.select('settings');
f.ui.voiceSettingsPanel.focus();
f.panel.select('preview');
assert.equal(f.doc.activeElement, f.ui.previewTab);
const unrelated = (f.doc.activeElement = {});
f.panel.select('settings');
assert.equal(f.doc.activeElement, unrelated);
assert.equal(f.panel.close('preview').changed, false);
assert.equal(f.panel.snapshot().settingsOpen, true);
});
test('manual Preview selection never enables a disabled preference and uses injected copy', (t) => {
const f = fixture(t);
f.enabled(false);
f.content(true);
f.panel.select('preview');
assert.equal(f.panel.snapshot().previewEnabled, false);
assert.equal(f.panel.snapshot().previewVisible, false);
assert.equal(f.ui.previewEmpty.textContent, f.copy.previewDisabled);
assert.equal(f.ui.previewEmpty.hidden, false);
f.enabled(true);
assert.equal(f.panel.snapshot().previewVisible, true);
assert.equal(f.ui.previewEmpty.hidden, true);
f.content(false);
assert.equal(f.ui.previewEmpty.textContent, f.copy.previewEmpty);
});
test('disposal hides owned presentation and removes listeners without destroying the card', (t) => {
const f = fixture(t);
f.panel.select('settings');
const count = f.intents();
f.panel.destroy();
f.panel.destroy();
click(f.ui.voiceSettingsButton);
click(f.ui.settingsTab);
assert.equal(f.intents(), count);
assert.equal(f.ui.voicePanel.hidden, true);
assert.equal(f.card.presented, false);
assert.equal(f.panel.select('preview').reason, 'disposed');
assert.equal(f.panel.close().reason, 'disposed');
});
test('pointer grace expires, resets on reentry, ignores touch and clears on disposal', (t) => {
t.mock.timers.enable({ apis: ['setTimeout'] });
const f = fixture(t);
const pointer = (type, pointerType = 'mouse') => {
const event = new Event(type);
Object.defineProperty(event, 'pointerType', { value: pointerType });
f.ui.root.dispatchEvent(event);
};
pointer('pointerleave');
t.mock.timers.tick(699);
assert.equal(f.ui.root.dataset.rowGrace, 'true');
pointer('pointerenter');
assert.equal(f.ui.root.dataset.rowGrace, undefined);
pointer('pointerleave');
t.mock.timers.tick(699);
assert.equal(f.ui.root.dataset.rowGrace, 'true');
t.mock.timers.tick(1);
assert.equal(f.ui.root.dataset.rowGrace, undefined);
pointer('pointerleave', 'touch');
assert.equal(f.ui.root.dataset.rowGrace, undefined);
pointer('pointerleave');
f.panel.destroy();
assert.equal(f.ui.root.dataset.rowGrace, undefined);
pointer('pointerleave');
t.mock.timers.tick(700);
assert.equal(f.ui.root.dataset.rowGrace, undefined);
});
+292
View File
@@ -0,0 +1,292 @@
/** Inspected native capabilities only; scheduled/reported output is not acoustics. */
export function observeNativeActivity({
adapter,
nativeProvider,
signal,
onChange,
onInput,
}) {
const native = nativeProvider === 'openai' || nativeProvider === 'gemini';
// Do not even inspect custom controller properties.
const controller = native ? adapter.controller : null;
const input = native
? nativeProvider === 'gemini'
? controller.input
: controller._input
: null;
let dead = false,
connected = false,
inputBusy = false,
recoverableError = false;
function cleanNativeOutput() {
if (nativeProvider === 'gemini') return true;
if (nativeProvider !== 'openai') return false;
const turns = controller._turns;
return (
['idle', 'connecting', 'error'].includes(controller.status) &&
controller.dc == null &&
!turns.assistantAudioPlaying &&
!turns.responseActive &&
!turns.responseCreatePending &&
!turns.userTurnPending &&
!turns.activeResponseId &&
!turns.progressResponse
);
}
let outputKnown = cleanNativeOutput();
let audio = null,
version = 0;
const restores = [];
const audioRestores = [];
const pendingOutput = new Map();
let untrackedOutput = !outputKnown;
function retireAudio() {
audio = null;
for (const restore of audioRestores.splice(0).reverse()) restore();
}
function refreshOutputKnowledge() {
outputKnown = !untrackedOutput && pendingOutput.size === 0;
}
function hasNoAudioOutput(response) {
return (
['completed', 'cancelled', 'failed', 'incomplete'].includes(
response?.status,
) &&
Array.isArray(response.output) &&
response.output.every(
(item) =>
item?.type === 'function_call' ||
(item?.type === 'message' &&
Array.isArray(item.content) &&
item.content.every((part) =>
['text', 'output_text', 'refusal'].includes(part?.type),
)),
)
);
}
function snapshot() {
const turns = nativeProvider === 'openai' ? controller?._turns : controller;
return {
known: native && outputKnown && !dead,
connected: connected && !dead,
held: Boolean(input?.pushToTalkKeyHeld),
inputBusy: Boolean(
inputBusy ||
(nativeProvider === 'gemini' && controller?.userActivityOpen),
),
outputBusy: Boolean(
nativeProvider === 'gemini'
? audio && audio === controller.audio && audio.pending
: turns?.assistantAudioPlaying,
),
generationBusy: Boolean(
turns?.responseActive ||
turns?.responseCreatePending ||
(nativeProvider === 'openai' && turns?.userTurnPending) ||
(nativeProvider === 'gemini' &&
(turns?.replyPending ||
turns?.retiredReplyPending ||
turns?.deferredInput ||
turns?.automaticTurnOpen ||
turns?.pendingTools ||
turns?.controlQueue?.length)),
),
};
}
function notify() {
if (!dead) {
try {
onChange?.(snapshot());
} catch {}
}
}
function wrap(object, key, after, ownedRestores = restores) {
const original = object?.[key];
if (typeof original !== 'function') return;
function wrapped(...args) {
const stamp = version;
const result = Reflect.apply(original, this, args);
if (!dead && stamp === version) {
try {
after?.(args, result);
} catch {}
}
return result;
}
object[key] = wrapped;
ownedRestores.push(() => {
if (object[key] === wrapped) object[key] = original;
});
}
function handle(event) {
if (dead || !native) return;
if (event?.type === 'state') {
const state = event.state;
const wasConnected = connected;
const liveError =
state === 'error' &&
nativeProvider === 'openai' &&
controller.dc?.readyState === 'open';
const restartingAfterError = state === 'connecting' && recoverableError;
recoverableError = liveError;
connected = !['idle', 'connecting', 'error'].includes(state);
// Connecting has multiple native phases around audio allocation. Retire
// a previous connected generation once, not each phase of this start.
// A failed reply can briefly report error on a live OpenAI transport.
// Preserve its output evidence through recovery to listening; a later
// reconnect still retires the failed connection's generation.
if (
state === 'idle' ||
(state === 'error' && !liveError) ||
(state === 'connecting' && (wasConnected || restartingAfterError))
) {
version++;
retireAudio();
inputBusy = false;
outputKnown = cleanNativeOutput();
untrackedOutput = !outputKnown;
pendingOutput.clear();
}
}
notify();
}
if (native) {
wrap(controller, 'sendTextCommand', ([text], result) => {
if (String(text || '').trim() && result !== false) onInput?.();
});
for (const key of [
'setMicrophoneEnabled',
'releasePushToTalkKey',
'resetSession',
])
wrap(input, key, notify);
if (nativeProvider === 'openai') {
wrap(controller._turns, 'observeSpeech', ([event]) => {
if (event?.type === 'input_audio_buffer.speech_started')
inputBusy = true;
if (event?.type === 'input_audio_buffer.speech_stopped')
inputBusy = false;
const type = event?.type;
const id = event?.response_id || event?.response?.id;
// Keep each unresolved response separate: a later silent reply cannot
// establish that an earlier audio reply has drained.
if (type === 'response.created') {
if (id && pendingOutput.size < 64)
pendingOutput.set(id, { audioSeen: false });
else untrackedOutput = true;
}
const pending = pendingOutput.get(id);
if (
pending &&
(type?.startsWith('response.output_audio') ||
type?.startsWith('response.audio') ||
['audio', 'output_audio'].includes(event?.part?.type))
)
pending.audioSeen = true;
if (
type === 'response.done' &&
pending &&
!pending.audioSeen &&
hasNoAudioOutput(event.response)
)
pendingOutput.delete(id);
if (
type === 'output_audio_buffer.started' ||
type === 'output_audio_buffer.cleared'
) {
// Native playback events establish the transport's output lifecycle.
// Generation completion alone never establishes audio drainage.
untrackedOutput = false;
if (id) pendingOutput.delete(id);
else pendingOutput.clear();
}
refreshOutputKnowledge();
notify();
});
} else {
wrap(controller, 'handleMessage', notify);
const original = controller.createAudio;
function createAudio(...args) {
const stamp = version;
const options = args[0];
let current;
const decorated = { ...options };
for (const key of [
'onAudioStart',
'onAudioEnd',
'onAudioDiscard',
'onDrain',
]) {
const callback = options?.[key];
decorated[key] = function (...values) {
const result =
typeof callback === 'function'
? Reflect.apply(
callback,
this === decorated ? options : this,
values,
)
: undefined;
if (
!dead &&
stamp === version &&
audio === current &&
controller.audio === current
) {
if (
key === 'onAudioStart' &&
options.pushToTalk &&
controller.captureOwner?.admitted
)
inputBusy = true;
if (key === 'onAudioEnd' || key === 'onAudioDiscard')
inputBusy = false;
notify();
}
return result;
};
}
const forwarded = args.slice();
forwarded[0] = decorated;
current = Reflect.apply(original, this, forwarded);
if (!dead && stamp === version) {
retireAudio();
audio = current;
inputBusy = false;
for (const key of ['play', 'clearPlayback', 'close'])
wrap(
current,
key,
() => {
if (audio !== current) return;
if (key === 'close') {
retireAudio();
inputBusy = false;
}
if (controller.audio === current) notify();
},
audioRestores,
);
}
return current;
}
controller.createAudio = createAudio;
restores.push(() => {
if (controller.createAudio === createAudio)
controller.createAudio = original;
});
}
}
function destroy() {
if (dead) return;
dead = true;
version++;
retireAudio();
pendingOutput.clear();
for (const restore of restores.splice(0).reverse()) restore();
signal?.removeEventListener('abort', destroy);
}
signal?.addEventListener('abort', destroy, { once: true });
if (signal?.aborted) destroy();
return { snapshot, handle, destroy };
}
+770
View File
@@ -0,0 +1,770 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceInactivity } from './inactivity.js';
import { RealtimeTurns } from '../realtimeTurns.js';
import { observeNativeActivity } from './nativeActivity.js';
// Deliberately small native-shaped fixtures exercise wrapper contracts, not devices.
function gemini(onChange = () => {}) {
const callbacks = [];
const input = {
pushToTalkKeyHeld: false,
setMicrophoneEnabled() {},
releasePushToTalkKey() {
this.pushToTalkKeyHeld = false;
},
resetSession() {
this.pushToTalkKeyHeld = false;
},
};
const c = {
input,
captureOwner: { admitted: false },
createAudio(opts) {
callbacks.push(opts);
return {
pending: 0,
play() {
this.pending++;
return 42;
},
clearPlayback() {
this.pending = 0;
},
close() {
this.pending = 0;
},
};
},
handleMessage() {},
};
const observer = observeNativeActivity({
adapter: { controller: c },
nativeProvider: 'gemini',
onChange,
});
const start = () => {
observer.handle({ type: 'state', state: 'connecting' });
c.audio = c.createAudio({ pushToTalk: true });
observer.handle({ type: 'state', state: 'listening' });
return c.audio;
};
return { c, input, observer, callbacks, start };
}
test('custom host never inspected, including hostile controller getter', () => {
const adapter = {
get controller() {
throw Error('must not inspect');
},
};
const o = observeNativeActivity({ adapter, nativeProvider: null });
assert.equal(o.snapshot().known, false);
o.handle({ type: 'state', state: 'listening' });
o.destroy();
});
test('PTT hold is direct before connect; silent automatic stream is not admitted speech', () => {
const f = gemini();
f.input.pushToTalkKeyHeld = true;
assert.equal(f.observer.snapshot().held, true);
assert.equal(f.observer.snapshot().connected, false);
f.start();
f.callbacks[0].onAudioStart({});
assert.equal(f.observer.snapshot().inputBusy, false);
f.c.captureOwner.admitted = true;
f.callbacks[0].onAudioStart({});
assert.equal(f.observer.snapshot().inputBusy, true);
f.callbacks[0].onAudioDiscard();
f.input.releasePushToTalkKey();
assert.equal(f.observer.snapshot().inputBusy, false);
assert.equal(f.observer.snapshot().held, false);
});
test('queue-to-drain, clear and close use current instance, without an acoustic claim', () => {
const f = gemini();
const a = f.start();
assert.equal(a.play(), 42);
assert.equal(f.observer.snapshot().outputBusy, true);
a.clearPlayback();
assert.equal(f.observer.snapshot().outputBusy, false);
a.play();
a.close();
assert.equal(f.observer.snapshot().outputBusy, false);
a.play();
a.pending = 0;
f.callbacks[0].onDrain();
assert.equal(f.observer.snapshot().outputBusy, false);
});
test('same-mount restart old callbacks cannot clear new admitted capture/output', () => {
const f = gemini();
f.start();
const old = f.callbacks[0];
const b = f.start();
f.c.captureOwner.admitted = true;
f.callbacks[1].onAudioStart();
b.play();
old.onAudioDiscard();
old.onDrain();
assert.equal(f.observer.snapshot().inputBusy, true);
assert.equal(f.observer.snapshot().outputBusy, true);
});
test('native callbacks original first with receiver, arguments, return and thrown identity; observation failure isolated', () => {
const f = gemini(() => {
throw Error('observer');
});
const receiver = {};
const argument = {};
let seen;
f.c.audio = f.c.createAudio({
pushToTalk: true,
onAudioEnd: function (...args) {
seen = [this, args];
return argument;
},
});
const cb = f.callbacks[0];
assert.equal(cb.onAudioEnd.call(receiver, argument), argument);
assert.equal(seen[0], receiver);
assert.equal(seen[1][0], argument);
const error = Error('same');
f.c.audio = f.c.createAudio({
onDrain() {
throw error;
},
});
assert.throws(
() => f.callbacks[1].onDrain(),
(e) => e === error,
);
});
test('reentrant Stop in original callback suppresses stale post-callback observation; teardown preserves newer replacement', () => {
let count = 0;
const f = gemini(() => count++);
f.c.audio = f.c.createAudio({
onDrain() {
f.observer.handle({ type: 'state', state: 'idle' });
},
});
const before = count;
f.callbacks[0].onDrain();
assert.equal(count, before + 1);
assert.equal(f.observer.snapshot().connected, false);
const newer = () => {};
f.input.releasePushToTalkKey = newer;
f.observer.destroy();
assert.equal(f.input.releasePushToTalkKey, newer);
});
test('OpenAI response.done does not drain output; absent lifecycle remains unknown', () => {
const turns = {
assistantAudioPlaying: false,
responseActive: false,
observeSpeech(e) {
if (e.type === 'output_audio_buffer.started')
this.assistantAudioPlaying = true;
if (
e.type === 'output_audio_buffer.stopped' ||
e.type === 'output_audio_buffer.cleared'
)
this.assistantAudioPlaying = false;
},
};
const c = { _turns: turns, _input: { pushToTalkKeyHeld: false } };
const original = turns.observeSpeech;
const o = observeNativeActivity({
adapter: { controller: c },
nativeProvider: 'openai',
});
o.handle({ type: 'state', state: 'listening' });
assert.equal(o.snapshot().known, false);
turns.observeSpeech({ type: 'output_audio_buffer.started' });
assert.equal(o.snapshot().known, true);
turns.responseActive = false;
o.handle({ type: 'completion' });
assert.equal(o.snapshot().outputBusy, true);
turns.observeSpeech({ type: 'output_audio_buffer.cleared' });
assert.equal(o.snapshot().outputBusy, false);
o.destroy();
assert.equal(turns.observeSpeech, original);
});
test('method receiver/args/result and throw are exact; abort restores wrappers', () => {
const abort = new AbortController();
const result = Promise.resolve();
let seen;
const error = Error('original');
const input = {
setMicrophoneEnabled(...args) {
seen = [this, args];
if (args[0] === 'throw') throw error;
return result;
},
};
const original = input.setMicrophoneEnabled;
const c = {
input,
createAudio() {
return {};
},
handleMessage() {},
};
const o = observeNativeActivity({
adapter: { controller: c },
nativeProvider: 'gemini',
signal: abort.signal,
onChange() {
throw Error('observe');
},
});
assert.equal(input.setMicrophoneEnabled(1, 2), result);
assert.equal(seen[0], input);
assert.deepEqual(seen[1], [1, 2]);
assert.throws(
() => input.setMicrophoneEnabled('throw'),
(e) => e === error,
);
abort.abort();
assert.equal(input.setMicrophoneEnabled, original);
assert.equal(o.snapshot().known, false);
});
test('options-method callback receiver maps to original options, explicit receiver is preserved', () => {
const f = gemini();
let receiver;
const options = {
onDrain() {
receiver = this;
return 17;
},
};
f.c.audio = f.c.createAudio(options);
assert.equal(f.callbacks[0].onDrain(), 17);
assert.equal(receiver, options);
const explicit = {};
f.callbacks[0].onDrain.call(explicit);
assert.equal(receiver, explicit);
});
test('actual native RealtimeTurns output lifecycle remains busy beyond generation', () => {
const turns = new RealtimeTurns({});
const observer = observeNativeActivity({
adapter: { controller: { _turns: turns, _input: {} } },
nativeProvider: 'openai',
});
observer.handle({ type: 'state', state: 'listening' });
turns.observeSpeech({ type: 'output_audio_buffer.started' });
turns.responseActive = false;
observer.handle({ type: 'completion' });
assert.equal(observer.snapshot().outputBusy, true);
turns.observeSpeech({ type: 'output_audio_buffer.stopped' });
assert.equal(observer.snapshot().outputBusy, false);
turns.observeSpeech({ type: 'response.created' });
turns.observeSpeech({ type: 'output_audio_buffer.stopped' });
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('verified clean native mount can arm initial idle, eventless response invalidates it', () => {
const turns = new RealtimeTurns({});
const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'openai',
});
let armed = 0,
cleared = 0;
const idle = createVoiceInactivity({
readActivity: observer.snapshot,
readPending: () => 0,
setTimer() {
armed++;
return 1;
},
clearTimer() {
cleared++;
},
});
assert.equal(observer.snapshot().known, true);
assert.equal(armed, 0);
controller.status = 'listening';
controller.dc = { readyState: 'open' };
observer.handle({ type: 'state', state: 'listening' });
idle.update();
assert.equal(armed, 1);
turns.observeSpeech({ type: 'response.created' });
idle.update();
assert.equal(observer.snapshot().known, false);
assert.equal(cleared, 1);
turns.responseActive = false;
observer.handle({ type: 'completion' });
idle.update();
assert.equal(armed, 1);
assert.equal(observer.snapshot().known, false);
controller.status = 'idle';
controller.dc = null;
turns.reset();
observer.handle({ type: 'state', state: 'idle' });
assert.equal(observer.snapshot().known, true);
idle.destroy();
observer.destroy();
});
test('attaching to already active native transport never assumes empty playback', () => {
const turns = new RealtimeTurns({});
for (const controller of [
{ status: 'listening', dc: {}, _turns: turns, _input: {} },
{ status: 'idle', dc: {}, _turns: turns, _input: {} },
]) {
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'openai',
});
assert.equal(observer.snapshot().known, false);
observer.destroy();
}
turns.responseActive = true;
const observer = observeNativeActivity({
adapter: {
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
},
nativeProvider: 'openai',
});
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('Gemini queue-only work blocks idle, queue drain rearms, expiry rereads delayed queue', () => {
const f = gemini();
f.start();
f.c.controlQueue = [];
let callback,
arms = 0,
clears = 0,
expired = 0;
const idle = createVoiceInactivity({
readActivity: f.observer.snapshot,
readPending: () => 0,
onExpire() {
expired++;
},
setTimer(fn) {
callback = fn;
return ++arms;
},
clearTimer() {
clears++;
},
});
assert.equal(arms, 1);
f.c.controlQueue.push({});
assert.equal(f.observer.snapshot().generationBusy, true);
idle.update();
assert.equal(clears, 1);
f.c.controlQueue.shift();
assert.equal(f.observer.snapshot().generationBusy, false);
idle.update();
assert.equal(arms, 2);
// A queue may change after the last event-driven snapshot and before the deadline.
f.c.controlQueue.push({});
callback();
assert.equal(expired, 0);
f.c.controlQueue = [];
idle.update();
assert.equal(arms, 3);
callback();
assert.equal(expired, 1);
idle.destroy();
f.observer.destroy();
});
test('Gemini presentation-only pending does not block idle; retired/deferred work still does', () => {
const f = gemini();
f.start();
f.c.userTurnPending = true;
assert.equal(f.observer.snapshot().generationBusy, false);
f.c.retiredReplyPending = true;
assert.equal(f.observer.snapshot().generationBusy, true);
f.c.retiredReplyPending = false;
f.c.deferredInput = {};
assert.equal(f.observer.snapshot().generationBusy, true);
f.c.deferredInput = null;
assert.equal(f.observer.snapshot().generationBusy, false);
f.observer.destroy();
});
test('actual Gemini startup phases keep playback observation and permit quiet open-mic expiry', async () => {
const { harness } = await import('../../testSupport/geminiNativeHarness.mjs');
let observer,
idle,
deadline,
expiries = 0;
const h = harness({ emit: (event) => observer?.handle(event) });
observer = observeNativeActivity({
adapter: { controller: h.c },
nativeProvider: 'gemini',
onChange: () => idle?.update(),
});
idle = createVoiceInactivity({
readActivity: observer.snapshot,
readPending: () => 0,
setTimer(fn) {
deadline = fn;
return fn;
},
clearTimer() {},
onExpire() {
expiries++;
},
});
try {
await h.c.start();
assert.equal(h.c.userTurnPending, true);
assert.equal(idle.snapshot().eligible, true);
const beforeOutput = deadline;
h.sockets[0].message({
serverContent: {
modelTurn: {
parts: [
{
inlineData: { data: 'output', mimeType: 'audio/pcm;rate=24000' },
},
],
},
turnComplete: true,
},
});
assert.equal(observer.snapshot().outputBusy, true);
assert.equal(idle.snapshot().eligible, false);
beforeOutput();
assert.equal(expiries, 0);
h.captures[0].drain();
assert.equal(observer.snapshot().outputBusy, false);
assert.equal(idle.snapshot().eligible, true);
deadline();
assert.equal(expiries, 1);
} finally {
idle.destroy();
observer.destroy();
h.c.stop({ removeUi: true });
}
});
test('native typed input observation preserves receiver/result, refuses failed submissions and retires old wrapper', () => {
let inputs = 0;
const controller = {
input: {},
handleMessage() {},
createAudio() {},
sendTextCommand(text) {
assert.equal(this, controller);
if (text === 'throw') throw new Error('rejected');
return text === 'reject' ? false : undefined;
},
};
const original = controller.sendTextCommand;
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'gemini',
onInput: () => inputs++,
});
const old = controller.sendTextCommand;
assert.equal(controller.sendTextCommand('typed request'), undefined);
controller.sendTextCommand('reject');
controller.sendTextCommand('');
assert.throws(() => controller.sendTextCommand('throw'), /rejected/);
assert.equal(inputs, 1);
observer.destroy();
assert.equal(controller.sendTextCommand, original);
old.call(controller, 'late');
assert.equal(inputs, 1);
});
test('explicit silent OpenAI completion rearms expiry, including cancelled and failed responses', () => {
for (const status of ['completed', 'cancelled', 'failed', 'incomplete']) {
const turns = new RealtimeTurns({});
const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
let idle;
let timer;
let expired = 0;
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'openai',
onChange: () => idle?.update(),
});
idle = createVoiceInactivity({
readActivity: observer.snapshot,
readPending: () => false,
setTimer(callback) {
timer = callback;
return 1;
},
clearTimer() {
timer = null;
},
onExpire: () => expired++,
});
controller.status = 'listening';
controller.dc = {};
observer.handle({ type: 'state', state: 'listening' });
assert.equal(typeof timer, 'function');
turns.responseActive = true;
turns.observeSpeech({
type: 'response.created',
response: { id: 'silent' },
});
assert.equal(timer, null);
turns.observeSpeech({
type: 'response.done',
response: {
id: 'silent',
status,
output:
status === 'completed'
? [{ type: 'function_call', name: 'set_panel_open' }]
: [],
},
});
// The provider updates generation state after forwarding the raw event.
assert.equal(timer, null);
turns.responseActive = false;
observer.handle({ type: 'completion' });
assert.equal(observer.snapshot().known, true);
assert.equal(typeof timer, 'function');
timer();
assert.equal(expired, 1);
idle.destroy();
observer.destroy();
}
});
test('silent OpenAI replies cannot clear unresolved older audio or incomplete output metadata', () => {
const turns = new RealtimeTurns({});
const observer = observeNativeActivity({
adapter: {
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
},
nativeProvider: 'openai',
});
const created = (id) =>
turns.observeSpeech({ type: 'response.created', response: { id } });
const done = (id, output) =>
turns.observeSpeech({
type: 'response.done',
response: { id, status: 'completed', output },
});
created('audio');
turns.observeSpeech({
type: 'response.output_audio.delta',
response_id: 'audio',
});
created('silent');
done('silent', []);
assert.equal(observer.snapshot().known, false);
done('audio', []); // Even contradictory completion metadata cannot erase audio evidence.
assert.equal(observer.snapshot().known, false);
turns.observeSpeech({
type: 'output_audio_buffer.started',
response_id: 'audio',
});
assert.equal(observer.snapshot().outputBusy, true);
done('silent', []);
assert.equal(observer.snapshot().outputBusy, true);
turns.observeSpeech({
type: 'output_audio_buffer.stopped',
response_id: 'audio',
});
assert.equal(observer.snapshot().outputBusy, false);
created('unknown');
done('unknown', undefined);
assert.equal(observer.snapshot().known, false);
done('unobserved', []);
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('overlapping silent replies resolve independently and audio output items remain unknown until playback', () => {
const turns = new RealtimeTurns({});
const observer = observeNativeActivity({
adapter: {
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
},
nativeProvider: 'openai',
});
for (const id of ['a', 'b'])
turns.observeSpeech({ type: 'response.created', response: { id } });
for (const id of ['a', 'b']) {
turns.observeSpeech({
type: 'response.done',
response: {
id,
status: 'completed',
output: [
{ type: 'message', content: [{ type: 'text', text: 'Done' }] },
],
},
});
assert.equal(observer.snapshot().known, id === 'b');
}
turns.observeSpeech({ type: 'response.created', response: { id: 'audio' } });
turns.observeSpeech({
type: 'response.done',
response: {
id: 'audio',
status: 'completed',
output: [{ type: 'message', content: [{ type: 'audio' }] }],
},
});
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('silent completion cannot establish output knowledge when observer attached mid-session', () => {
const turns = new RealtimeTurns({});
const observer = observeNativeActivity({
adapter: {
controller: { status: 'listening', dc: {}, _turns: turns, _input: {} },
},
nativeProvider: 'openai',
});
turns.observeSpeech({ type: 'response.created', response: { id: 'new' } });
turns.observeSpeech({
type: 'response.done',
response: { id: 'new', status: 'completed', output: [] },
});
assert.equal(observer.snapshot().known, false);
observer.destroy();
});
test('Gemini restores each retired audio immediately across repeated same-mount restarts', () => {
const originals = new Map();
const controller = {
input: {},
createAudio() {
const audio = { pending: 0, play() {}, clearPlayback() {}, close() {} };
originals.set(audio, { ...audio });
return audio;
},
handleMessage() {},
};
const observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'gemini',
});
const assertRestored = (audio) => {
for (const key of ['play', 'clearPlayback', 'close'])
assert.equal(audio[key], originals.get(audio)[key]);
};
for (let i = 0; i < 5; i++) {
observer.handle({ type: 'state', state: 'connecting' });
controller.audio = controller.createAudio({});
const audio = controller.audio;
observer.handle({ type: 'state', state: 'listening' });
assert.notEqual(audio.close, originals.get(audio).close);
audio.close();
assertRestored(audio);
controller.audio = null;
observer.handle({ type: 'state', state: 'idle' });
}
controller.audio = controller.createAudio({});
const replaced = controller.audio;
controller.audio = controller.createAudio({});
assertRestored(replaced);
const current = controller.audio;
const replacement = () => {};
current.play = replacement;
observer.handle({ type: 'state', state: 'error' });
assert.equal(current.play, replacement);
assert.equal(current.close, originals.get(current).close);
observer.destroy();
});
test('actual OpenAI failed-response error/listening recovery preserves only proven no-audio knowledge', async (t) => {
const { GevRealtimeController } = await import('../realtimeController.js');
t.mock.method(console, 'error', () => {});
for (const [output, known] of [
[[], true],
[[{ type: 'message', content: [{ type: 'audio' }] }], false],
[undefined, false],
]) {
let observer, idle, timer;
const states = [];
let expired = 0;
const controller = new GevRealtimeController({
runner: async () => ({ ok: true }),
backend: {},
debugSink: null,
ui: {
root: {
dataset: {},
classList: { remove() {} },
querySelectorAll: () => [],
},
status: {},
detail: {},
},
onSessionEvent(event) {
if (event.type === 'state') states.push(event.state);
observer?.handle(event);
},
});
observer = observeNativeActivity({
adapter: { controller },
nativeProvider: 'openai',
onChange: () => idle?.update(),
});
idle = createVoiceInactivity({
readActivity: observer.snapshot,
readPending: () => false,
setTimer(callback) {
timer = callback;
return 1;
},
clearTimer() {
timer = null;
},
onExpire() {
expired++;
},
});
try {
controller.dc = {
readyState: 'open',
send() {},
close() {
this.readyState = 'closed';
},
};
controller.setStatus('listening', 'Ready');
const event = (payload) =>
controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
await event({
type: 'response.created',
response: { id: 'failed-reply' },
});
await event({
type: 'response.done',
response: {
id: 'failed-reply',
status: 'failed',
output,
status_details: { error: { message: 'Synthetic failed reply' } },
},
});
assert.deepEqual(states.slice(-2), ['error', 'listening']);
assert.equal(controller.dc.readyState, 'open');
assert.equal(observer.snapshot().known, known);
assert.equal(idle.snapshot().eligible, known);
if (known) {
timer();
assert.equal(expired, 1);
} else assert.equal(timer, null);
// Fatal teardown can preserve the error status; the next connection must
// still retire the old generation rather than inherit unresolved output.
controller.setStatus('error', 'Transport failed');
controller.stop({ preserveStatus: true });
controller.setStatus('connecting', 'Reconnecting');
assert.equal(observer.snapshot().connected, false);
assert.equal(observer.snapshot().known, true);
controller.stop();
assert.equal(observer.snapshot().connected, false);
assert.equal(observer.snapshot().known, true);
} finally {
idle.destroy();
observer.destroy();
controller.stop();
}
}
});
+194
View File
@@ -0,0 +1,194 @@
export const VOICE_PREVIEW_STORAGE_KEY = 'godsEyeView.voice.previewEnabled';
export const VOICE_TIMEOUT_STORAGE_KEY = 'godsEyeView.voice.inactivityMinutes';
const PRESETS = ['3', '5', '10', '15'];
function parseMinutes(value) {
if (typeof value !== 'number' && typeof value !== 'string') return undefined;
if (typeof value === 'string' && !/^\d+$/.test(value)) return undefined;
const number = Number(value);
return Number.isInteger(number) && number >= 0 && number <= 60
? number
: undefined;
}
/** Outer, browser-local live state. Construction never writes storage. */
export function createVoicePreferences({ storage } = {}) {
const listeners = new Set();
const target = () =>
storage === undefined ? globalThis.localStorage : storage;
const read = (key) => {
try {
return target()?.getItem?.(key);
} catch {
return undefined;
}
};
const rawMinutes = read(VOICE_TIMEOUT_STORAGE_KEY);
let state = {
previewEnabled: read(VOICE_PREVIEW_STORAGE_KEY) !== 'false',
minutes: rawMinutes === 'none' ? 0 : (parseMinutes(rawMinutes) ?? 5),
previewPersisted: null,
timeoutPersisted: null,
};
const snapshot = () => ({ ...state });
function apply(field, status, key, value, serialized) {
const changed = state[field] !== value;
let persisted = false;
state = { ...state, [field]: value };
try {
const destination = target();
if (
typeof destination?.setItem === 'function' &&
typeof destination?.getItem === 'function'
) {
destination.setItem(key, serialized);
persisted = destination.getItem(key) === serialized;
}
} catch {
/* The applied live value remains authoritative across mounts. */
}
state = { ...state, [status]: persisted };
const result = {
ok: true,
applied: true,
changed,
persisted,
...snapshot(),
};
for (const listener of [...listeners]) listener(snapshot());
return result;
}
const invalid = () => ({
ok: false,
applied: false,
changed: false,
persisted: null,
reason: 'invalid-value',
...snapshot(),
});
return {
snapshot,
setPreviewEnabled(value) {
return typeof value === 'boolean'
? apply(
'previewEnabled',
'previewPersisted',
VOICE_PREVIEW_STORAGE_KEY,
value,
String(value),
)
: invalid();
},
setMinutes(value) {
const parsed = parseMinutes(value);
return parsed === undefined
? invalid()
: apply(
'minutes',
'timeoutPersisted',
VOICE_TIMEOUT_STORAGE_KEY,
parsed,
parsed === 0 ? 'none' : String(parsed),
);
},
subscribe(listener) {
if (typeof listener !== 'function')
throw new TypeError('Preference listener required');
listeners.add(listener);
return () => listeners.delete(listener);
},
};
}
/** Mount-local controls, backed by an independently owned preference model. */
export function bindPreferenceControls({ ui, preferences, copy, onChange }) {
const {
voicePreviewInput: preview,
voicePreviewNote: previewNote,
voiceTimeoutSelect: select,
voiceTimeoutInput: input,
voiceTimeoutNote: timeoutNote,
} = ui;
let disposed = false;
const initial = preferences.snapshot();
const displayed = parseMinutes(input?.value);
let custom =
!PRESETS.includes(String(initial.minutes)) && initial.minutes > 0
? initial.minutes
: displayed > 0
? displayed
: 5;
function render(state) {
if (disposed) return;
if (preview) preview.checked = state.previewEnabled;
if (previewNote)
previewNote.textContent = `${copy.previewNote}${state.previewPersisted === false ? ` ${copy.notSaved}` : ''}`;
if (timeoutNote)
timeoutNote.textContent = `${copy.timeoutNote}${state.timeoutPersisted === false ? ` ${copy.notSaved}` : ''}`;
const value = String(state.minutes);
const customSelected = state.minutes > 0 && !PRESETS.includes(value);
if (customSelected) custom = state.minutes;
if (select)
select.value =
state.minutes === 0 ? '0' : customSelected ? 'custom' : value;
if (input) {
input.value = String(custom);
input.hidden = !customSelected;
}
}
const unsubscribe = preferences.subscribe((state) => {
render(state);
onChange?.(state);
});
const previewChange = () => {
if (!disposed && !preferences.setPreviewEnabled(preview.checked).ok)
render(preferences.snapshot());
};
const selectChange = () => {
if (disposed) return;
const value = select.value;
if (value === 'custom') {
const parsed = parseMinutes(input?.value);
if (parsed === undefined || parsed === 0) {
render(preferences.snapshot());
return;
}
custom = parsed;
preferences.setMinutes(parsed);
// A custom value may equal a preset: retain the user's explicit editor choice.
select.value = 'custom';
if (input) input.hidden = false;
} else if (value === '0' || PRESETS.includes(value))
preferences.setMinutes(value);
else render(preferences.snapshot());
};
const inputChange = () => {
if (disposed) return;
const parsed = parseMinutes(input.value);
if (parsed === undefined || parsed === 0) {
render(preferences.snapshot());
return;
}
custom = parsed;
preferences.setMinutes(parsed);
if (select) select.value = 'custom';
input.hidden = false;
};
const bindings = [
[preview, previewChange],
[select, selectChange],
[input, inputChange],
];
for (const [element, listener] of bindings)
element?.addEventListener('change', listener);
render(initial);
return {
dispose() {
if (disposed) return;
disposed = true;
unsubscribe();
for (const [element, listener] of bindings)
element?.removeEventListener('change', listener);
},
};
}
+272
View File
@@ -0,0 +1,272 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
createVoicePreferences,
bindPreferenceControls,
VOICE_PREVIEW_STORAGE_KEY as P,
VOICE_TIMEOUT_STORAGE_KEY as T,
} from './preferences.js';
function storage(entries = {}, mode = 'normal') {
const values = new Map(Object.entries(entries));
const writes = [];
return {
writes,
values,
getItem(key) {
if (mode === 'readthrow') throw Error('blocked');
return values.get(key) ?? null;
},
setItem(key, value) {
writes.push([key, value]);
if (mode === 'throw') throw Error('blocked');
if (mode !== 'noop' && mode !== 'readthrow')
values.set(key, mode === 'mismatch' ? `${value}!` : value);
},
};
}
class Element {
value = '';
checked = false;
textContent = '';
hidden = false;
listeners = new Set();
addEventListener(type, listener) {
assert.equal(type, 'change');
this.listeners.add(listener);
}
removeEventListener(type, listener) {
assert.equal(type, 'change');
this.listeners.delete(listener);
}
change(value) {
if (typeof value === 'boolean') this.checked = value;
else this.value = value;
for (const listener of [...this.listeners]) listener();
}
}
function controls(custom = '7') {
const ui = Object.fromEntries(
[
'voicePreviewInput',
'voicePreviewNote',
'voiceTimeoutSelect',
'voiceTimeoutInput',
'voiceTimeoutNote',
].map((key) => [key, new Element()]),
);
ui.voiceTimeoutInput.value = custom;
return ui;
}
const copy = {
previewNote: 'Preview local',
timeoutNote: 'Provider limits still apply',
notSaved: 'Not saved',
};
function bind(ui, preferences, onChange) {
return bindPreferenceControls({ ui, preferences, copy, onChange });
}
test('defaults and restore are passive; legacy None and stored custom restore', () => {
for (const [entries, minutes, previewEnabled] of [
[{}, 5, true],
[{ [P]: 'false', [T]: 'none' }, 0, false],
[{ [T]: '17' }, 17, true],
[{ [T]: 'broken' }, 5, true],
]) {
const disk = storage(entries);
const prefs = createVoicePreferences({ storage: disk });
assert.deepEqual(prefs.snapshot(), {
previewEnabled,
minutes,
previewPersisted: null,
timeoutPersisted: null,
});
const ui = controls();
bind(ui, prefs).dispose();
assert.deepEqual(disk.writes, []);
}
});
test('exact valid values, presets, None and serialized readback', () => {
const disk = storage();
const prefs = createVoicePreferences({ storage: disk });
for (const value of [3, 5, 10, 15, '1', '60', 0]) {
const result = prefs.setMinutes(value);
assert.equal(result.ok, true);
assert.equal(result.applied, true);
assert.equal(result.persisted, true);
assert.equal(result.minutes, Number(value));
assert.deepEqual(disk.writes.at(-1), [
T,
Number(value) === 0 ? 'none' : String(Number(value)),
]);
}
assert.equal(prefs.setPreviewEnabled(false).previewEnabled, false);
assert.deepEqual(disk.writes.at(-1), [P, 'false']);
});
test('fractional, malformed, coercible and out-of-range inputs never write or notify', () => {
const disk = storage();
const prefs = createVoicePreferences({ storage: disk });
let notices = 0;
prefs.subscribe(() => notices++);
const before = prefs.snapshot();
for (const value of [
null,
undefined,
true,
{},
[],
'',
' ',
' 3',
'3 ',
'3.0',
'1e1',
'0x10',
'none',
-1,
61,
3.2,
NaN,
Infinity,
]) {
const result = prefs.setMinutes(value);
assert.equal(result.ok, false, String(value));
assert.equal(result.applied, false);
assert.equal(result.changed, false);
assert.deepEqual(prefs.snapshot(), before);
}
for (const value of [0, 'true', null])
assert.equal(prefs.setPreviewEnabled(value).ok, false);
assert.equal(notices, 0);
assert.deepEqual(disk.writes, []);
});
for (const mode of ['throw', 'noop', 'mismatch', 'readthrow', 'missing']) {
test(`storage ${mode} preserves live value and exposes unsaved state across A → B`, () => {
const disk = mode === 'missing' ? null : storage({}, mode);
const prefs = createVoicePreferences({ storage: disk });
const a = controls();
const bindingA = bind(a, prefs);
a.voicePreviewInput.change(false);
a.voiceTimeoutSelect.change('custom');
a.voiceTimeoutInput.change('19');
assert.deepEqual(prefs.snapshot(), {
previewEnabled: false,
minutes: 19,
previewPersisted: false,
timeoutPersisted: false,
});
bindingA.dispose();
const writeCount = disk?.writes.length;
const b = controls();
const bindingB = bind(b, prefs);
assert.equal(b.voicePreviewInput.checked, false);
assert.equal(b.voiceTimeoutInput.value, '19');
assert.match(b.voicePreviewNote.textContent, /Not saved/);
assert.match(b.voiceTimeoutNote.textContent, /Not saved/);
assert.equal(disk?.writes.length, writeCount);
a.voicePreviewInput.change(true);
a.voiceTimeoutSelect.change('3');
assert.equal(prefs.snapshot().minutes, 19);
assert.equal(prefs.snapshot().previewEnabled, false);
bindingB.dispose();
});
}
test('same-value retry really writes and clears warnings after storage recovers', () => {
const disk = storage();
const original = disk.setItem;
disk.setItem = () => {
throw Error('blocked');
};
const prefs = createVoicePreferences({ storage: disk });
const ui = controls();
bind(ui, prefs);
prefs.setPreviewEnabled(false);
prefs.setMinutes(12);
assert.match(ui.voiceTimeoutNote.textContent, /Not saved/);
disk.setItem = original;
const timeout = prefs.setMinutes(12);
const preview = prefs.setPreviewEnabled(false);
assert.equal(timeout.changed, false);
assert.equal(preview.changed, false);
assert.equal(timeout.persisted, true);
assert.equal(preview.persisted, true);
assert.deepEqual(disk.writes, [
[T, '12'],
[P, 'false'],
]);
assert.equal(ui.voiceTimeoutNote.textContent, copy.timeoutNote);
assert.equal(ui.voicePreviewNote.textContent, copy.previewNote);
});
test('Custom commits displayed valid value immediately, even when equal to preset', () => {
const disk = storage();
const prefs = createVoicePreferences({ storage: disk });
const ui = controls('10');
bind(ui, prefs);
assert.equal(disk.writes.length, 0);
ui.voiceTimeoutSelect.change('custom');
assert.equal(prefs.snapshot().minutes, 10);
assert.equal(ui.voiceTimeoutSelect.value, 'custom');
assert.equal(ui.voiceTimeoutInput.hidden, false);
assert.deepEqual(disk.writes, [[T, '10']]);
ui.voiceTimeoutInput.change('23');
ui.voiceTimeoutSelect.change('5');
assert.equal(ui.voiceTimeoutInput.value, '23');
assert.equal(ui.voiceTimeoutInput.hidden, true);
ui.voiceTimeoutSelect.change('custom');
assert.equal(prefs.snapshot().minutes, 23);
assert.equal(ui.voiceTimeoutInput.value, '23');
});
test('invalid control edits restore valid state without writes; None remains distinct', () => {
const disk = storage({ [T]: '17' });
const prefs = createVoicePreferences({ storage: disk });
const ui = controls();
bind(ui, prefs);
assert.equal(ui.voiceTimeoutInput.value, '17');
for (const value of ['1.5', '', '61', '0']) {
ui.voiceTimeoutInput.change(value);
assert.equal(ui.voiceTimeoutInput.value, '17');
assert.equal(prefs.snapshot().minutes, 17);
}
ui.voiceTimeoutSelect.change('malformed');
assert.equal(ui.voiceTimeoutSelect.value, 'custom');
assert.equal(disk.writes.length, 0);
ui.voiceTimeoutSelect.change('0');
assert.equal(prefs.snapshot().minutes, 0);
assert.equal(ui.voiceTimeoutInput.hidden, true);
ui.voiceTimeoutInput.value = 'bad';
ui.voiceTimeoutSelect.change('custom');
assert.equal(ui.voiceTimeoutSelect.value, '0');
assert.equal(prefs.snapshot().minutes, 0);
assert.deepEqual(disk.writes, [[T, 'none']]);
});
test('model snapshots cannot mutate state; unsubscribe and binder disposal release ownership', () => {
const prefs = createVoicePreferences({ storage: storage() });
const ui = controls();
let calls = 0;
const binding = bind(ui, prefs, () => calls++);
const snap = prefs.snapshot();
snap.minutes = 41;
assert.equal(prefs.snapshot().minutes, 5);
const unsub = prefs.subscribe((state) => {
state.minutes = 22;
});
prefs.setMinutes(11);
assert.equal(calls, 1);
assert.equal(prefs.snapshot().minutes, 11);
unsub();
binding.dispose();
binding.dispose();
prefs.setMinutes(12);
assert.equal(calls, 1);
assert.equal(ui.voiceTimeoutInput.value, '11');
for (const element of Object.values(ui))
assert.equal(element.listeners.size, 0);
});
+204
View File
@@ -0,0 +1,204 @@
// Minimal DOM fixture for actual control/card/binder composition, not layout proof.
class Element {
constructor(doc, tag = 'div') {
Object.assign(this, {
ownerDocument: doc,
tagName: tag.toUpperCase(),
children: [],
attributes: new Map(),
dataset: {},
hidden: false,
value: '',
checked: false,
listeners: new Map(),
style: { setProperty() {}, removeProperty() {} },
});
this.classList = {
add: (...names) => names.forEach((n) => this.classes.add(n)),
remove: (...names) => names.forEach((n) => this.classes.delete(n)),
contains: (name) => this.classes.has(name),
};
this.classes = new Set();
}
set id(value) {
this.attributes.set('id', value);
}
get id() {
return this.attributes.get('id') || '';
}
set className(value) {
this.classes = new Set(value.split(/\s+/).filter(Boolean));
}
get className() {
return [...this.classes].join(' ');
}
setAttribute(name, value) {
this.attributes.set(name, String(value));
if (name === 'class') this.className = value;
if (name === 'hidden') this.hidden = true;
if (name === 'value') this.value = value;
if (name.startsWith('data-'))
this.dataset[
name.slice(5).replace(/-([a-z])/g, (_, c) => c.toUpperCase())
] = value;
}
getAttribute(name) {
return this.attributes.get(name) ?? null;
}
get isConnected() {
return (
this === this.ownerDocument.body || Boolean(this.parentNode?.isConnected)
);
}
get parentElement() {
return this.parentNode;
}
set innerHTML(html) {
this.replaceChildren();
const stack = [this];
const voids = new Set(['INPUT', 'IMG', 'BR', 'HR']);
for (const match of html.matchAll(/<(\/?)([a-z][\w-]*)\b([^>]*)>/gi)) {
if (match[1]) {
if (stack.length > 1) stack.pop();
continue;
}
const node = new Element(this.ownerDocument, match[2]);
for (const attr of match[3].matchAll(/([^\s=\/]+)(?:="([^"]*)")?/g))
node.setAttribute(attr[1], attr[2] ?? '');
stack.at(-1).appendChild(node);
if (!voids.has(node.tagName)) stack.push(node);
}
}
set textContent(value) {
this.text = String(value);
}
get textContent() {
return this.text ?? '';
}
appendChild(node) {
node.remove();
this.children.push(node);
node.parentNode = this;
return node;
}
append(...nodes) {
nodes.forEach((node) => this.appendChild(node));
}
insertBefore(node, before) {
node.remove();
const at = this.children.indexOf(before);
this.children.splice(at < 0 ? this.children.length : at, 0, node);
node.parentNode = this;
}
replaceChildren(...nodes) {
for (const child of this.children) child.parentNode = null;
this.children = [];
this.append(...nodes);
}
remove() {
if (this.parentNode)
this.parentNode.children = this.parentNode.children.filter(
(n) => n !== this,
);
this.parentNode = null;
}
contains(node) {
return node === this || this.children.some((child) => child.contains(node));
}
matches(selector) {
if (selector.startsWith('#')) return this.id === selector.slice(1);
if (selector.startsWith('.')) return this.classes.has(selector.slice(1));
return this.tagName === selector.toUpperCase();
}
querySelectorAll(selector) {
return this.children.flatMap((n) => [
...(n.matches(selector) ? [n] : []),
...n.querySelectorAll(selector),
]);
}
querySelector(selector) {
return this.querySelectorAll(selector)[0] || null;
}
checkVisibility() {
return (
this.isConnected &&
!this.hidden &&
(!this.parentNode || this.parentNode.checkVisibility())
);
}
focus() {
this.ownerDocument.activeElement = this;
}
addEventListener(type, listener, options = {}) {
if (options.signal?.aborted) return;
if (!this.listeners.has(type)) this.listeners.set(type, new Set());
this.listeners.get(type).add(listener);
if (options.signal)
options.signal.addEventListener(
'abort',
() => this.removeEventListener(type, listener),
{ once: true },
);
}
removeEventListener(type, listener) {
this.listeners.get(type)?.delete(listener);
}
fire(type, fields = {}) {
const event = {
type,
target: this,
preventDefault() {
this.defaultPrevented = true;
},
...fields,
};
for (const listener of [...(this.listeners.get(type) || [])])
listener.call(this, event);
return event;
}
}
const installed = new WeakMap();
export function installPanelDom(t) {
if (installed.has(t)) return installed.get(t);
const previous = {
window: globalThis.window,
document: globalThis.document,
Element: globalThis.Element,
};
const doc = new EventTarget();
doc.body = new Element(doc, 'body');
doc.documentElement = new Element(doc, 'html');
doc.documentElement.lang = 'en';
doc.activeElement = doc.body;
doc.createElement = (tag) => new Element(doc, tag);
doc.getElementById = (id) => doc.body.querySelector(`#${id}`);
doc.querySelector = (selector) => doc.body.querySelector(selector);
const win = new EventTarget();
globalThis.window = win;
globalThis.document = doc;
globalThis.Element = Element;
installed.set(t, doc);
t.after(() => Object.assign(globalThis, previous));
return doc;
}
export function controlledClock() {
let now = 0,
next = 0;
const tasks = new Map();
return {
now: () => now,
setTimer(fn, ms) {
const id = ++next;
tasks.set(id, { fn, at: now + ms });
return id;
},
clearTimer: (id) => tasks.delete(id),
advance(ms) {
now += ms;
for (const [id, task] of [...tasks])
if (task.at <= now && tasks.delete(id)) task.fn();
},
};
}
+69 -5
View File
@@ -20,15 +20,15 @@
*/
/** Movement below this many CSS pixels is still the same dwell. */
export const POINTER_DWELL_PX = 12;
const POINTER_DWELL_PX = 12;
/** Stillness that makes a dwell point. */
export const POINTER_DWELL_MS = 250;
/** A resting pointer stays a referent this long after it last moved. */
export const POINTER_FRESH_MS = 30000;
/** Pixels from the canvas edge that count as "parked", never a referent. */
export const POINTER_EDGE_PX = 8;
const POINTER_EDGE_PX = 8;
/** A dwell pick is reused by a snapshot only while this recent (contacts move). */
export const POINTER_DWELL_REUSE_MS = 1000;
const POINTER_DWELL_REUSE_MS = 1000;
/**
* Open mic has no gesture, so the pointer only counts when it moved or
* settled on the map this recently: a cursor resting there for minutes is
@@ -76,7 +76,7 @@ export function cameraPoseKey(viewer) {
}
/** Bound and de-control a label before it goes to the model or the card. */
export function pointerLabel(value, max = LABEL_MAX) {
function pointerLabel(value, max = LABEL_MAX) {
const clean = String(value ?? '')
.replace(/[\p{Cc}\p{Cf}]+/gu, ' ')
.replace(/\s+/g, ' ')
@@ -182,6 +182,63 @@ export function pointerChip(snapshot) {
};
}
/** Dedupe key of the clearing pointer_context item. */
export const POINTER_NONE_KEY = 'none';
/**
* The voice-card event for a turn's pointer: a gesture reveals its target at
* once; open-mic and typed turns reveal it only through announcePointer.
* @param {object|null} snapshot
* @param {boolean} gesture
* @returns {{announced: object|null, pointer: object|null}}
*/
export function pointerTurnEvent(snapshot, gesture) {
const chip = gesture ? pointerChip(snapshot) : null;
return chip
? { announced: snapshot, pointer: { ...chip, screenPx: snapshot.screenPx } }
: { announced: null, pointer: null };
}
/**
* The pointer_context item a turn should send, or null when there is nothing
* new. Without a gesture the model learns only what kind of thing is pointed
* at, never which. Losing a sent pointer produces one clearing item.
* @param {object|null} snapshot
* @param {{at: string, gesture: boolean, sentKey: string|null}} turn
* @returns {{item: object, key: string}|null}
*/
export function pointerTurnContext(snapshot, { at, gesture, sentKey }) {
const chip = pointerChip(snapshot);
const item = !chip
? sentKey && sentKey !== POINTER_NONE_KEY
? { type: 'pointer_context', at, target: 'none' }
: null
: gesture
? pointerContextItem(snapshot)
: {
type: 'pointer_context',
at,
pointing: snapshot.entity?.kind || 'ground',
};
if (!item) return null;
const { at: _at, ...content } = item;
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
return key === sentKey ? null : { item, key };
}
/**
* The voice-card event revealing a pointer a tool resolved, once per snapshot.
* @param {object|null} snapshot
* @param {object|null} announced Snapshot already shown.
* @returns {object|null}
*/
export function pointerAnnouncement(snapshot, announced) {
const chip = pointerChip(snapshot);
return chip && snapshot !== announced
? { ...chip, screenPx: snapshot.screenPx }
: null;
}
/**
* Track the pointer over one element (the globe canvas).
* @param {object} options
@@ -330,7 +387,14 @@ export function createPointerTracker({
* (open mic only when it moved recently: `recentMs`). The result becomes
* the turn's referent (null when stale).
*/
beginTurn(at, { recentMs = POINTER_FRESH_MS } = {}) {
beginTurn(at, { recentMs = POINTER_FRESH_MS, capturedSnapshot } = {}) {
// Delayed admitted audio keeps its capture's pointer without replacing a
// newer physical hold. Undefined preserves existing caller behavior.
if (capturedSnapshot !== undefined) {
const snapshot = destroyed ? { fresh: false, at } : capturedSnapshot;
active = snapshot?.fresh ? snapshot : null;
return snapshot;
}
const reuse =
at !== 'text' && held && now() - held.heldAt <= POINTER_HOLD_MS;
const snapshot = reuse ? held.snapshot : capture(at, { recentMs });
+112
View File
@@ -6,8 +6,13 @@ import {
POINTER_HOLD_MS,
buildPointerSnapshot,
createPointerTracker,
POINTER_NONE_KEY,
formatPointerContext,
pointerAnnouncement,
pointerChip,
pointerContextItem,
pointerTurnContext,
pointerTurnEvent,
} from './pointerContext.js';
function harness({ pick } = {}) {
@@ -151,6 +156,51 @@ test('destroy removes listeners and ignores later moves', () => {
assert.equal(h.tracker.capture('keydown').fresh, false);
});
test('delayed speech uses its released capture without replacing a newer physical hold', () => {
const h = harness();
h.move(700, 250);
const capturedA = h.tracker.hold();
h.tracker.releaseHold();
h.move(200, 100);
h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA });
assert.equal(h.tracker.activeSnapshot(), capturedA);
const heldB = h.tracker.hold();
h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA });
assert.equal(h.tracker.activeSnapshot(), capturedA);
assert.equal(h.tracker.isHeld(), true);
assert.equal(h.tracker.beginTurn('speech_start'), heldB);
h.tracker.destroy();
});
test('explicit stale captures clear the referent while a silent hold leaves it alone', () => {
const h = harness();
h.move(700, 250);
const current = h.tracker.beginTurn('text');
h.move(200, 100);
h.tracker.hold();
h.tracker.releaseHold();
assert.equal(h.tracker.activeSnapshot(), current);
for (const capturedSnapshot of [null, { fresh: false, at: 'keydown' }]) {
h.tracker.beginTurn('text');
h.tracker.beginTurn('speech_start', { capturedSnapshot });
assert.equal(h.tracker.activeSnapshot(), null);
}
h.tracker.destroy();
});
test('a capture arriving after destruction cannot restore an active referent', () => {
const h = harness();
h.move(700, 250);
const capturedSnapshot = h.tracker.hold();
assert.equal(capturedSnapshot.fresh, true);
h.tracker.destroy();
const result = h.tracker.beginTurn('speech_start', { capturedSnapshot });
assert.equal(result.fresh, false);
assert.equal(h.tracker.activeSnapshot(), null);
assert.equal(h.tracker.isHeld(), false);
assert.equal(h.tracker.beginTurn('speech_start').fresh, false);
});
test('format: the model item is compact inert JSON; labels are bounded data', () => {
const snapshot = buildPointerSnapshot({
at: 'speech_start',
@@ -182,3 +232,65 @@ test('chip: "this" names an entity, "here" gives the ground coordinate', () => {
assert.deepEqual(pointerChip(ground), { kind: 'here', label: '30.27, -97.74' });
assert.equal(pointerChip(null), null);
});
test('pointer turn helpers reveal gesture targets, describe implicit ones by kind and dedupe repeats', () => {
const snapshot = {
fresh: true,
at: 'keydown',
entity: { kind: 'aircraft', layerId: 'flights', label: 'UAL1' },
lat: 10,
lon: 20,
screen: { x: 5, y: 6 },
screenPx: { x: 5, y: 6 },
};
const shown = pointerTurnEvent(snapshot, true);
assert.equal(shown.announced, snapshot);
assert.deepEqual(shown.pointer, {
kind: 'this',
label: 'UAL1',
screenPx: { x: 5, y: 6 },
});
assert.deepEqual(pointerTurnEvent(snapshot, false), {
announced: null,
pointer: null,
});
const gesture = pointerTurnContext(snapshot, {
at: 'keydown',
gesture: true,
sentKey: null,
});
assert.deepEqual(gesture.item, pointerContextItem(snapshot));
assert.equal(
pointerTurnContext(snapshot, {
at: 'keydown',
gesture: true,
sentKey: gesture.key,
}),
null,
);
assert.deepEqual(
pointerTurnContext(snapshot, { at: 'text', gesture: false, sentKey: null })
.item,
{ type: 'pointer_context', at: 'text', pointing: 'aircraft' },
);
const cleared = pointerTurnContext(null, {
at: 'text',
gesture: false,
sentKey: gesture.key,
});
assert.deepEqual(cleared, {
item: { type: 'pointer_context', at: 'text', target: 'none' },
key: POINTER_NONE_KEY,
});
for (const sentKey of [null, POINTER_NONE_KEY])
assert.equal(
pointerTurnContext(null, { at: 'text', gesture: false, sentKey }),
null,
);
assert.deepEqual(pointerAnnouncement(snapshot, null), shown.pointer);
assert.equal(pointerAnnouncement(snapshot, snapshot), null);
assert.equal(pointerAnnouncement(null, null), null);
});
+1 -1
View File
@@ -4,7 +4,7 @@ import { cameraPoseKey } from './pointerContext.js';
import * as defaultAnnotationResolver from '../annotations/annotationResolver.js';
/** "Around here" radius as a share of camera height, clamped to sane bounds. */
export function pointerRadiusKm(cameraHeightM) {
function pointerRadiusKm(cameraHeightM) {
const heightKm = Number(cameraHeightM) / 1000;
if (!Number.isFinite(heightKm) || heightKm <= 0) return 50;
return Math.round(Math.min(400, Math.max(5, heightKm * 0.75)));
+1 -1
View File
@@ -14,7 +14,7 @@ export const RETICLE_MAX_MS = 10000;
export const RETICLE_LINGER_MS = 1500;
/** Tools that move the camera away from the pointed spot. */
export const CAMERA_TOOLS = new Set([
const CAMERA_TOOLS = new Set([
'fly_to_location',
'track_entity',
'zoom_to_globe',
+21
View File
@@ -0,0 +1,21 @@
export const VOICE_PROVIDERS = Object.freeze({
openai: Object.freeze({ label: 'OpenAI' }), // i18n-ignore: provider product name
gemini: Object.freeze({ label: 'Gemini' }), // i18n-ignore: provider product name
});
/** Optional providers require an explicit host opt-in and an available factory. */
export function supportedVoiceProviders(
requested = ['openai'],
factories = {},
) {
if (!Array.isArray(requested))
throw new TypeError('supportedProviders must be an array');
const selected = [...new Set(requested)].filter(
(name) =>
Object.hasOwn(VOICE_PROVIDERS, name) &&
typeof factories[name] === 'function',
);
if (!selected.length)
throw new Error('No supported voice provider is configured');
return Object.freeze(selected);
}
+161
View File
@@ -0,0 +1,161 @@
import { supportedVoiceProviders } from './providerCapabilities.js';
const STORAGE_KEY = 'gev.voice.provider';
export function readVoiceProvider(storage, supported = ['openai', 'gemini']) {
let saved;
try {
saved = (storage ?? globalThis.localStorage)?.getItem(STORAGE_KEY);
} catch {
/* Storage is optional. */
}
return supported.includes(saved)
? saved
: supported.includes('openai')
? 'openai'
: supported[0];
}
/** Keep one public controller identity while replacing each provider's lifetime. */
export function createProviderCommands(options, { bind, factories, storage }) {
const supportedProviders = supportedVoiceProviders(
options.supportedProviders ?? Object.keys(factories),
factories,
);
window.__gevVoiceCommands?.stop?.({ removeUi: true });
let provider = readVoiceProvider(storage, supportedProviders);
let current;
let disposed = false;
const lifetime = new AbortController();
let mountGeneration = 0;
let voiceEnabled = false;
const stored = () => {
try {
return storage ?? globalThis.localStorage;
} catch {
return null;
}
};
const stop = (settings = {}) => {
if (disposed) return;
voiceEnabled = false;
if (settings.removeUi) {
disposed = true;
options.signal?.removeEventListener('abort', destroy);
}
try {
current?.session.stop(settings);
} finally {
if (settings.removeUi) {
lifetime.abort();
options.runner.dispose?.();
}
}
};
const destroy = () => stop({ removeUi: true });
const selectProvider = (next) => {
if (disposed || !supportedProviders.includes(next) || next === provider)
return false;
const resume = voiceEnabled && current.session.isActive();
// Carry the input mode, never a held key, microphone or conversation.
const pushToTalk = Boolean(
current.session.adapter.getStartOptions?.().pushToTalk,
);
const generation = ++mountGeneration;
voiceEnabled = false;
current.session.destroy();
// Disposal suppresses the retired adapter's idle event. Clear the shared
// conversation explicitly before the new mount can use these owners.
if (disposed || generation !== mountGeneration) return false;
options.pointer?.clear();
options.referents?.clear();
options.runner.resetConversation?.();
if (disposed || generation !== mountGeneration) return false;
provider = next;
try {
stored()?.setItem(STORAGE_KEY, provider);
} catch {
/* Storage is optional. */
}
mount();
if (resume && !disposed && provider === next)
void current.session.start({ pushToTalk });
globalThis.document?.getElementById('gev-voice-provider')?.focus();
return true;
};
const handleSessionEvent = (event) => {
if (event?.type === 'state' && event.state === 'connecting')
voiceEnabled = true;
if (event?.type === 'state' && ['idle', 'error'].includes(event.state)) {
voiceEnabled = false;
}
// The host still observes the mounted provider's neutral session events.
options.onSessionEvent?.(event);
};
const sendTextCommand = (text) => {
if (disposed) return false;
return current.session.sendText(text);
};
const api = {
// The shared action surface is independent of an active voice connection.
runner: options.runner,
supportedProviders,
signal: lifetime.signal,
get provider() {
return provider;
},
get session() {
return current?.session;
},
get status() {
return current?.status ?? current?.session.state ?? 'idle';
},
get state() {
return current?.session.state ?? 'idle';
},
get disposed() {
return disposed;
},
start: (settings) => !disposed && current.session.start(settings),
stop,
destroy,
isActive: () => !disposed && current.session.isActive(),
sendTextCommand,
sendText: sendTextCommand,
notifyMapEvent: (event) => !disposed && current.session.sendMapEvent(event),
};
// Existing diagnostic and controller accessors continue to reflect the active provider.
const facade = new Proxy(api, {
get(target, name, receiver) {
if (Reflect.has(target, name)) return Reflect.get(target, name, receiver);
const value = current?.[name];
return typeof value === 'function' ? value.bind(current) : value;
},
});
function mount() {
voiceEnabled = false;
const generation = ++mountGeneration;
current = bind({
...options,
signal: lifetime.signal,
resetExisting: false,
disposeRunner: false,
provider,
supportedProviders,
onProviderChange: selectProvider,
onSessionEvent: (event) => {
if (!disposed && generation === mountGeneration)
handleSessionEvent(event);
},
createSession: factories[provider],
});
window.__gevVoiceCommands = facade;
}
mount();
options.signal?.addEventListener('abort', destroy, { once: true });
if (options.signal?.aborted) destroy();
return facade;
}
+888
View File
@@ -0,0 +1,888 @@
// Imports intentionally target the candidate, never the immutable reference.
import test from 'node:test';
import assert from 'node:assert/strict';
import {
createProviderCommands,
readVoiceProvider,
} from './providerCommands.js';
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
import { createVoiceCommands as createCommands } from './commands.js';
import { createGevActionRunner } from './gevActions.js';
import { createPointerTracker } from './pointerContext.js';
import { createReferentRegistry } from './referents.js';
import { describeUnsavedWork, t as translate } from '../i18n/index.js';
const PROVIDER_KEY = 'gev.voice.provider';
const settle = () => new Promise((resolve) => setImmediate(resolve));
function deferred() {
let resolve;
let reject;
const promise = new Promise((yes, no) => {
resolve = yes;
reject = no;
});
return { promise, resolve, reject };
}
// Listener counts observe the actual binder's registrations/removals.
class Element extends EventTarget {
listeners = new Map();
attributes = new Map();
addEventListener(type, handler, options) {
if (!this.listeners.has(type)) this.listeners.set(type, new Set());
this.listeners.get(type).add(handler);
super.addEventListener(type, handler, options);
}
removeEventListener(type, handler, options) {
this.listeners.get(type)?.delete(handler);
super.removeEventListener(type, handler, options);
}
listenerCount(type) {
return this.listeners.get(type)?.size || 0;
}
setAttribute(name, value) {
this.attributes.set(name, String(value));
}
focus() {
this.focusCount = (this.focusCount || 0) + 1;
}
}
for (const custom of [false, true]) {
test(`public composition preserves the ${custom ? 'custom adapter' : 'default OpenAI adapter'} path`, async (t) => {
const prior = globalThis.window;
globalThis.window = new EventTarget();
let controls;
t.after(() => {
controls?.session.destroy();
globalThis.window = prior;
});
const ui = {
root: { dataset: {}, remove() {} },
button: new Element(),
status: {},
detail: {},
providerField: {},
};
let defaultCreated = 0;
let customCreated = 0;
const options = {
runner: async () => ({ ok: true }),
createControl: () => ui,
createController: ({ onSessionEvent }) => {
defaultCreated++;
return {
ui,
start({ pushToTalk }) {
this.pushToTalkMode = pushToTalk;
onSessionEvent({ type: 'state', state: 'listening' });
},
stop() {},
syncCostUi() {},
bindPushToTalkShortcut() {},
sendTextCommand: (text) => `default:${text}`,
};
},
};
if (custom) {
options.supportedProviders = ['gemini'];
options.createSession = ({ emit }) => {
customCreated++;
return {
capabilities: { pushToTalk: false },
start() {
emit({ type: 'state', state: 'listening' });
},
stop() {},
sendText: (text) => `custom:${text}`,
sendMapEvent: () => true,
};
};
}
controls = createCommands(options);
await controls.session.start({ pushToTalk: true });
assert.equal(defaultCreated, custom ? 0 : 1);
assert.equal(customCreated, custom ? 1 : 0);
assert.equal(
controls.session.sendText('test'),
`${custom ? 'custom' : 'default'}:test`,
);
assert.equal(ui.providerField.hidden, true);
assert.equal(ui.root.dataset.providerChoice, 'false');
if (!custom)
assert.deepEqual(controls.session.adapter.getStartOptions(), {
pushToTalk: true,
});
});
}
/** Real facade -> real binder -> real common session and real action runner.
* Adapters and passive UI/viewer boundaries are fixtures. Wrappers only count
* calls to real pointer/referent/conversation cleanup; they do not replace it.
*/
function compositionFixture(
t,
{
supportedProviders = ['openai', 'gemini'],
factoryProviders = ['openai', 'gemini'],
savedProvider = null,
storageDenied = false,
gateStarts = () => false,
onSessionEvent = () => {},
} = {},
) {
const original = new Map(
['window', 'document'].map((key) => [
key,
Object.getOwnPropertyDescriptor(globalThis, key),
]),
);
const lifetime = new AbortController();
const views = [];
const instances = [];
const mounts = [];
const events = [];
const order = [];
const annotations = new Set();
const storageWrites = [];
const counters = { pointer: 0, referents: 0, conversation: 0, dispose: 0 };
const state = { height: 1000, zooms: 0, renders: 0 };
let facade;
let pointer;
globalThis.window = new EventTarget();
globalThis.document = {
getElementById: (id) =>
id === 'gev-voice-provider' ? views.at(-1)?.providerSelect : null,
};
t.after(async () => {
try {
facade?.destroy();
pointer?.destroy();
for (const instance of instances)
for (const gate of instance.gates) gate.resolve();
await Promise.allSettled(instances.flatMap((instance) => instance.jobs));
} finally {
for (const [key, descriptor] of original) {
if (descriptor) Object.defineProperty(globalThis, key, descriptor);
else delete globalThis[key];
}
}
});
const canvas = new EventTarget();
Object.assign(canvas, { clientWidth: 800, clientHeight: 600 });
const position = { x: 6379137, y: 0, z: 0 };
const viewer = {
clock: { onTick: { addEventListener: () => () => {} } },
camera: {
positionWC: position,
positionCartographic: { height: state.height },
moveEnd: { addEventListener() {} },
cancelFlight() {},
zoomIn(metres) {
position.x -= metres;
state.height -= metres;
this.positionCartographic.height = state.height;
state.zooms++;
},
zoomOut(metres) {
position.x += metres;
state.height += metres;
this.positionCartographic.height = state.height;
state.zooms++;
},
},
scene: {
canvas,
requestRender() {
state.renders++;
},
},
};
const dataManager = { layers: new Map() };
pointer = createPointerTracker({ element: null, now: () => 1000 });
const referents = createReferentRegistry();
const runner = createGevActionRunner({
viewer,
dataManager,
styleManager: {},
deixis: { pointer, referents },
});
for (const [object, method, counter] of [
[pointer, 'clear', 'pointer'],
[referents, 'clear', 'referents'],
[runner, 'resetConversation', 'conversation'],
[runner, 'dispose', 'dispose'],
]) {
const originalMethod = object[method];
object[method] = (...args) => {
counters[counter]++;
order.push(counter);
return originalMethod.apply(object, args);
};
}
const factories = Object.fromEntries(
factoryProviders.map((provider) => [
provider,
(hooks) => {
const id = instances.length;
let generation = 0;
const instance = {
id,
provider,
hooks,
starts: [],
stops: [],
jobs: [],
gates: [],
acquired: 0,
microphoneEnabled: false,
bound: 0,
input: { pushToTalk: false, held: false },
texts: [],
mapEvents: [],
};
instances.push(instance);
order.push(`factory:${id}`);
const adapter = {
identity: `${provider}:${id}`,
capabilities: {
pushToTalk: true,
costControls: provider === 'openai',
},
controller: {
providerDiagnostic() {
return this.session.adapter.identity;
},
},
start(settings = {}) {
const attempt = ++generation;
instance.starts.push({ ...settings });
instance.input.pushToTalk = Boolean(settings.pushToTalk);
hooks.emit({
type: 'state',
state: 'connecting',
detail: `adapter:${provider}`,
});
const job = (async () => {
if (attempt !== generation || hooks.signal.aborted) return;
if (gateStarts(instance)) {
const gate = deferred();
instance.gates.push(gate);
await gate.promise;
}
// Adapters own cancellation of their asynchronous acquisition.
if (attempt !== generation || hooks.signal.aborted) return;
instance.acquired++;
instance.microphoneEnabled =
!instance.input.pushToTalk || instance.input.held;
hooks.emit({
type: 'state',
state: 'listening',
detail: provider,
});
})();
instance.jobs.push(job);
return job;
},
stop(settings = {}) {
generation++;
instance.stops.push({ ...settings });
order.push(`stop:${id}`);
instance.input.pushToTalk = instance.input.held = false;
instance.microphoneEnabled = false;
},
getStartOptions: () => ({ pushToTalk: instance.input.pushToTalk }),
bindControls() {
instance.bound++;
},
sendText(text) {
instance.texts.push(text);
return `${provider}:${text}`;
},
sendMapEvent(event) {
instance.mapEvents.push(event);
return true;
},
};
instance.adapter = adapter;
return adapter;
},
]),
);
function createControl() {
const id = views.length;
order.push(`control:${id}`);
const providerSelect = new Element();
providerSelect.ownerDocument = { createElement: () => ({}) };
providerSelect.replaceChildren = (...options) => {
providerSelect.options = options;
};
const view = {
root: {
dataset: {},
classList: { remove() {} },
removeCount: 0,
remove() {
this.removeCount++;
order.push(`remove:${id}`);
},
},
button: new Element(),
providerSelect,
providerField: {},
status: {},
detail: {},
errorDetail: {},
tierButton: {},
costValue: {},
helpDetail: {},
card: {},
};
views.push(view);
return view;
}
function passiveView() {
return {
events: [],
outlines: [],
destroyed: 0,
handle(event) {
this.events.push(event);
},
outline(event) {
this.outlines.push(event);
},
destroy() {
this.destroyed++;
},
};
}
facade = createProviderCommands(
{
runner,
pointer,
referents,
dataManager,
signal: lifetime.signal,
supportedProviders,
createControl,
createCard() {
const card = passiveView();
views.at(-1).cardView = card;
return card;
},
createReticle() {
const reticle = passiveView();
views.at(-1).reticle = reticle;
return reticle;
},
annotations: {
onOutlineEvent(callback) {
annotations.add(callback);
return () => annotations.delete(callback);
},
},
onSessionEvent(event) {
events.push(event);
onSessionEvent(event, facade);
},
},
{
factories,
storage: {
getItem() {
if (storageDenied) throw Error('Storage denied');
return savedProvider;
},
setItem(key, value) {
if (storageDenied) throw Error('Storage denied');
storageWrites.push([key, value]);
savedProvider = value;
},
},
bind(options) {
// Keep the real callback so stale-event suppression tests exercise the
// facade's mount generation as well as the common session's disposal.
const mount = { options, callback: options.onSessionEvent };
mounts.push(mount);
mount.controls = bindVoiceCommands(options);
return mount.controls;
},
},
);
return {
facade,
runner,
pointer,
referents,
lifetime,
instances,
views,
mounts,
events,
order,
annotations,
counters,
storageWrites,
state,
select(provider) {
const select = views.at(-1).providerSelect;
select.value = provider;
select.dispatchEvent(new Event('change'));
},
seedConversation() {
pointer.beginTurn('text', {
capturedSnapshot: { fresh: true, at: 'text' },
});
referents.recordResult('frame_overhead', {
ok: true,
display: { title: 'One aircraft' },
referents: [
{ n: 1, id: 'aaa111', label: 'Fixture', layerId: 'flights' },
],
});
assert.ok(pointer.activeSnapshot());
assert.equal(referents.list().length, 1);
},
};
}
test('provider preference uses only the supplied host capability list', () => {
const stored = (value) => ({ getItem: () => value });
assert.equal(readVoiceProvider(stored('gemini'), ['openai']), 'openai');
assert.equal(readVoiceProvider(stored('openai'), ['gemini']), 'gemini');
assert.equal(
readVoiceProvider(stored('unknown'), ['gemini', 'openai']),
'openai',
);
assert.equal(
readVoiceProvider(stored('gemini'), ['openai', 'gemini']),
'gemini',
);
assert.equal(
readVoiceProvider(
{
getItem() {
throw Error('denied');
},
},
['gemini'],
),
'gemini',
);
});
for (const scenario of [
{
title: 'OpenAI-only host ignores saved Gemini',
requested: ['openai'],
saved: 'gemini',
expected: ['openai'],
provider: 'openai',
},
{
title: 'Gemini-only host falls back from saved OpenAI',
requested: ['gemini'],
saved: 'openai',
expected: ['gemini'],
provider: 'gemini',
},
{
title: 'dual host restores Gemini',
requested: ['openai', 'gemini'],
saved: 'gemini',
expected: ['openai', 'gemini'],
provider: 'gemini',
},
{
title: 'unknown, duplicate and unavailable factories are excluded',
requested: ['unknown', 'gemini', 'openai', 'gemini'],
factories: ['gemini'],
saved: 'openai',
expected: ['gemini'],
provider: 'gemini',
},
]) {
test(scenario.title, (t) => {
const f = compositionFixture(t, {
supportedProviders: scenario.requested,
factoryProviders: scenario.factories,
savedProvider: scenario.saved,
});
assert.deepEqual(f.facade.supportedProviders, scenario.expected);
assert.equal(Object.isFrozen(f.facade.supportedProviders), true);
assert.equal(f.facade.provider, scenario.provider);
assert.deepEqual(
f.views[0].providerSelect.options.map((option) => option.value),
scenario.expected,
);
assert.equal(
f.views[0].providerField.hidden,
scenario.expected.length === 1,
);
assert.equal(
f.views[0].root.dataset.providerChoice,
String(scenario.expected.length > 1),
);
assert.equal(f.instances.length, 1);
assert.equal(
f.instances[0].starts.length,
0,
'discovery must not start a provider',
);
assert.equal(f.facade.state, 'idle');
});
}
test('invalid or empty host support fails before mounting a control', () => {
let mounts = 0;
const dependencies = {
bind() {
mounts++;
},
factories: { openai() {} },
};
assert.throws(
() =>
createProviderCommands({ supportedProviders: 'gemini' }, dependencies),
/must be an array/,
);
assert.throws(
() =>
createProviderCommands(
{ supportedProviders: ['gemini', 'unknown'] },
dependencies,
),
/No supported voice provider/,
);
assert.equal(mounts, 0);
});
test('active switch keeps PTT mode, retires the held key, and disposes before mounting', async (t) => {
const f = compositionFixture(t);
const firstSession = f.facade.session;
const outerSignal = f.facade.signal;
const first = f.instances[0];
// PTT controls can activate the adapter directly; the facade must learn
// activity from its common events, not only from facade.start().
first.input.held = true;
await first.adapter.start({ pushToTalk: true });
assert.equal(first.microphoneEnabled, true);
assert.equal(f.facade.isActive(), true);
f.seedConversation();
const before = { ...f.counters };
f.select('gemini');
await settle();
const second = f.instances[1];
assert.equal(window.__gevVoiceCommands, f.facade);
assert.equal(f.facade.runner, f.runner);
assert.equal(f.facade.signal, outerSignal);
assert.equal(outerSignal.aborted, false);
assert.equal(firstSession.disposed, true);
assert.equal(first.hooks.signal.aborted, true);
assert.deepEqual(first.stops, [{ removeUi: true }]);
assert.equal(first.input.held, false);
assert.equal(first.microphoneEnabled, false);
assert.deepEqual(second.starts, [{ pushToTalk: true }]);
assert.equal(second.input.held, false);
assert.equal(
second.microphoneEnabled,
false,
'a held key must never migrate to the new adapter',
);
assert.equal(f.facade.state, 'listening');
assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('control:1'));
assert.ok(f.order.indexOf('remove:0') < f.order.indexOf('control:1'));
assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('factory:1'));
assert.equal(f.views[0].button.listenerCount('click'), 0);
assert.equal(f.views[0].providerSelect.listenerCount('change'), 0);
assert.equal(f.views[1].providerSelect.listenerCount('change'), 1);
assert.equal(f.views[1].providerSelect.focusCount, 1);
assert.equal(f.views[1].tierButton.hidden, true);
assert.equal(f.views[1].costValue.hidden, true);
assert.equal(f.facade.providerDiagnostic(), 'gemini:1');
assert.equal(f.pointer.activeSnapshot(), null);
assert.deepEqual(f.referents.list(), []);
for (const key of ['pointer', 'referents', 'conversation'])
assert.equal(
f.counters[key],
before[key] + 1,
`one explicit replacement reset: ${key}`,
);
assert.equal(
f.counters.dispose,
0,
'provider replacement keeps the page runner',
);
assert.deepEqual(f.storageWrites, [[PROVIDER_KEY, 'gemini']]);
assert.equal(f.facade.sendTextCommand('hello'), 'gemini:hello');
assert.deepEqual(first.texts, []);
assert.deepEqual(second.texts, ['hello']);
const result = await f.facade.runner('adjust_camera_zoom', {
direction: 'in',
amount: 'little',
});
assert.equal(
result.ok,
true,
'the actual shared action runner remains usable after switching',
);
assert.equal(f.state.zooms, 1);
});
for (const initialState of ['idle', 'error']) {
test(`${initialState} selection never autostarts; manual start still works`, async (t) => {
const f = compositionFixture(t, { storageDenied: true });
if (initialState === 'error') {
await f.facade.start({ pushToTalk: true });
f.instances[0].adapter.stop({ preserveStatus: true });
f.instances[0].hooks.emit({
type: 'state',
state: 'error',
detail: 'Fixture failure',
});
}
f.select('gemini');
await settle();
assert.equal(
f.facade.provider,
'gemini',
'denied storage cannot block a supported selection',
);
assert.equal(f.facade.state, 'idle');
assert.deepEqual(f.instances[1].starts, []);
f.select('gemini');
f.select('unknown');
assert.equal(
f.instances.length,
2,
'same and unsupported selections cannot remount',
);
assert.equal(f.facade.provider, 'gemini');
await f.facade.start({ pushToTalk: false });
assert.deepEqual(f.instances[1].starts, [{ pushToTalk: false }]);
assert.equal(f.instances[1].microphoneEnabled, true);
});
}
for (const outcome of ['resolve', 'reject']) {
test(`retired startup ${outcome} and stale events cannot affect the current mount`, async (t) => {
const f = compositionFixture(t, {
gateStarts: (instance) => instance.id === 0,
});
const oldStart = f.facade.start({ pushToTalk: false });
const old = f.instances[0];
assert.equal(f.facade.state, 'connecting');
f.select('gemini');
await settle();
assert.equal(f.facade.state, 'listening');
const current = f.instances[1];
const eventCount = f.events.length;
const currentStops = current.stops.length;
const currentCardCount = f.views[1].cardView.events.length;
old.gates[0][outcome](new Error('Late retired startup failure'));
await oldStart;
for (const state of ['idle', 'error', 'connecting']) {
old.hooks.emit({ type: 'state', state, detail: 'retired adapter' });
f.mounts[0].callback({
type: 'state',
state,
detail: 'retired forward callback',
});
}
assert.equal(f.events.length, eventCount);
assert.equal(f.views[1].cardView.events.length, currentCardCount);
assert.equal(f.facade.state, 'listening');
assert.equal(old.acquired, 0);
assert.equal(current.stops.length, currentStops);
assert.equal(current.starts.length, 1);
f.facade.stop();
f.mounts[0].callback({ type: 'state', state: 'connecting' });
f.select('openai');
assert.deepEqual(
f.instances[2].starts,
[],
'a retired connecting callback cannot revive voice intent',
);
assert.equal(f.facade.state, 'idle');
});
}
test('Stop during replacement startup defeats late settlement and further automatic resume', async (t) => {
const f = compositionFixture(t, {
gateStarts: (instance) => instance.id === 1,
});
await f.facade.start();
f.select('gemini');
const replacement = f.instances[1];
assert.equal(f.facade.state, 'connecting');
f.facade.stop();
replacement.gates[0].resolve();
await Promise.all(replacement.jobs);
assert.equal(replacement.acquired, 0);
assert.equal(f.facade.state, 'idle');
f.select('openai');
assert.deepEqual(f.instances[2].starts, []);
await f.facade.start();
assert.equal(
f.facade.state,
'listening',
'a later explicit start remains healthy',
);
});
for (const emitsIdle of [false, true]) {
test(`Stop and error clear pointer/referents/conversation once: adapter emitsIdle=${emitsIdle}`, async (t) => {
const f = compositionFixture(t);
if (emitsIdle) {
const instance = f.instances[0];
const originalStop = instance.adapter.stop;
instance.adapter.stop = function (settings = {}) {
originalStop.call(this, settings);
if (!settings.preserveStatus && !settings.removeUi)
instance.hooks.emit({ type: 'state', state: 'idle' });
};
}
for (const end of ['stop', 'error']) {
await f.facade.start();
f.seedConversation();
const before = { ...f.counters };
const generation = f.referents.generation();
if (end === 'stop') f.facade.stop();
else {
f.instances[0].adapter.stop({ preserveStatus: true });
f.instances[0].hooks.emit({
type: 'state',
state: 'error',
detail: 'Fixture failure',
});
}
assert.equal(f.pointer.activeSnapshot(), null);
assert.deepEqual(f.referents.list(), []);
assert.equal(f.referents.generation(), generation + 1);
for (const key of ['pointer', 'referents', 'conversation'])
assert.equal(f.counters[key], before[key] + 1, `${end}: ${key}`);
}
});
}
test('outer destruction disposes the runner once and removes each provider binding', async (t) => {
const priorUnsaved = describeUnsavedWork();
const f = compositionFixture(t);
await f.facade.start();
f.select('gemini');
await settle();
assert.equal(f.annotations.size, 1);
assert.ok(
describeUnsavedWork().includes(translate('the active voice session')),
);
const oldCardEvents = f.views[0].cardView.events.length;
for (const callback of f.annotations) callback({ id: 'fixture-outline' });
assert.equal(f.views[0].cardView.outlines.length, 0);
assert.equal(f.views[1].cardView.outlines.length, 1);
assert.equal(f.instances[1].mapEvents.length, 1);
f.facade.destroy();
f.facade.destroy();
f.facade.stop({ removeUi: true });
f.lifetime.abort();
assert.equal(f.counters.dispose, 1);
assert.equal(f.facade.signal.aborted, true);
assert.equal(f.facade.disposed, true);
assert.equal(f.annotations.size, 0);
assert.deepEqual(describeUnsavedWork(), priorUnsaved);
const finalEvents = f.events.length;
const starts = f.instances.map((instance) => instance.starts.length);
for (const [index, view] of f.views.entries()) {
assert.equal(view.root.removeCount, 1);
assert.equal(view.cardView.destroyed, 1);
assert.equal(view.reticle.destroyed, 1);
assert.equal(view.button.listenerCount('click'), 0);
assert.equal(view.providerSelect.listenerCount('change'), 0);
assert.equal(f.instances[index].hooks.signal.aborted, true);
view.button.dispatchEvent(new Event('click'));
view.providerSelect.value = 'openai';
view.providerSelect.dispatchEvent(new Event('change'));
f.instances[index].hooks.emit({ type: 'state', state: 'connecting' });
f.mounts[index].callback({ type: 'state', state: 'connecting' });
}
assert.equal(f.events.length, finalEvents);
assert.equal(f.views[0].cardView.events.length, oldCardEvents);
assert.deepEqual(
f.instances.map((instance) => instance.starts.length),
starts,
);
assert.equal(f.instances.length, 2);
assert.equal(f.facade.start(), false);
assert.equal(f.facade.sendText('after destroy'), false);
assert.equal(f.facade.notifyMapEvent({}), false);
const result = await f.facade.runner('adjust_camera_zoom', {
direction: 'in',
amount: 'little',
});
assert.equal(result.cancelled, true);
assert.equal(
f.state.zooms,
0,
'the actual runner cannot move the camera after outer disposal',
);
});
for (const action of ['stop', 'destroy']) {
for (const phase of ['common', 'adapter']) {
test(`${action} from replacement ${phase} connecting observer prevents resume`, async (t) => {
let intercepted = 0;
const f = compositionFixture(t, {
onSessionEvent(event, facade) {
if (
facade?.provider !== 'gemini' ||
event.type !== 'state' ||
event.state !== 'connecting'
)
return;
const matching =
phase === 'common'
? event.detail === undefined
: event.detail === 'adapter:gemini';
if (matching && intercepted++ === 0) facade[action]();
},
});
await f.facade.start({ pushToTalk: true });
f.select('gemini');
await settle();
const replacement = f.instances[1];
assert.equal(intercepted, 1);
assert.equal(replacement.starts.length, phase === 'common' ? 0 : 1);
assert.equal(replacement.acquired, 0);
assert.equal(replacement.microphoneEnabled, false);
assert.equal(f.facade.isActive(), false);
assert.equal(f.facade.state, 'idle');
assert.equal(
f.events.filter(
(event) => event.state === 'listening' && event.detail === 'gemini',
).length,
0,
);
assert.equal(f.facade.disposed, action === 'destroy');
assert.equal(f.counters.dispose, action === 'destroy' ? 1 : 0);
if (action === 'stop') {
f.select('openai');
assert.deepEqual(
f.instances[2].starts,
[],
'Stop cleared the intent to resume',
);
await f.facade.start({ pushToTalk: false });
assert.equal(f.facade.state, 'listening');
} else {
f.select('openai');
assert.equal(f.instances.length, 2);
assert.equal(f.facade.start(), false);
}
});
}
}
+7 -5
View File
@@ -1,3 +1,5 @@
import { t } from '../i18n/index.js';
const DISCONNECT_GRACE_MS = 6000;
function releaseStartResources({ localStream = null, localPc = null } = {}) {
@@ -59,7 +61,7 @@ export class RealtimeConnection {
this.input.pushToTalkKeyHeld = pushToTalkKeyHeld;
this.input.spaceKeyHeld = spaceKeyHeld;
if (!window.RTCPeerConnection || !navigator.mediaDevices?.getUserMedia) {
this.setStatus('error', 'WebRTC microphone support unavailable');
this.setStatus('error', t('WebRTC microphone support unavailable'));
return;
}
@@ -78,7 +80,7 @@ export class RealtimeConnection {
// — this is what "applies next session" means.
this.cost.prepareSession();
this.syncCostUi();
this.setStatus('connecting', 'Requesting microphone');
this.setStatus('connecting', t('Requesting microphone'));
this.debugLog('session.starting', {
epoch,
tier: this.cost.voiceTier,
@@ -179,9 +181,9 @@ export class RealtimeConnection {
if (!ownsChannel()) return;
const detail = this.input.pushToTalkMode
? this.input.pushToTalkKeyHeld
? 'Release Space to send'
: 'Hold Space to talk'
: 'Ask or command';
? t('Release Space to send')
: t('Hold Space to talk')
: t('Ask or command');
this.setStatus('listening', detail);
this.debugLog('data_channel.open', {
connection: this.connectionDiagnostics(dataChannel),
+37 -49
View File
@@ -10,14 +10,13 @@ import { TurnMetrics } from './turnMetrics.js';
import { createNarrationScheduler } from './narration.js';
import { createEarcon } from './earcon.js';
import {
POINTER_NONE_KEY,
POINTER_OPEN_MIC_RECENT_MS,
pointerChip,
pointerContextItem,
pointerAnnouncement,
pointerTurnContext,
pointerTurnEvent,
} from './pointerContext.js';
/** Dedupe key of the clearing pointer_context item. */
const POINTER_NONE_KEY = 'none';
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
import { postDebugLog } from './realtimeDiagnostics.js';
@@ -55,13 +54,17 @@ export {
import { createRealtimeBackend } from './realtimeBackend.js';
const STATUS = {
idle: 'OFF',
connecting: 'CONNECTING',
listening: 'LISTENING',
executing: 'EXECUTING',
error: 'ERROR',
};
import { t } from '../i18n/index.js';
import { voiceStatusLabel } from './control.js';
// Machine states (dataset.status, CSS); voiceStatusLabel() translates labels.
const STATUS_STATES = new Set([
'idle',
'connecting',
'listening',
'executing',
'error',
]);
/** Compose voice state owners and coordinate ordered session startup/teardown. */
export class GevRealtimeController extends RealtimeFacade {
@@ -283,31 +286,18 @@ export class GevRealtimeController extends RealtimeFacade {
} catch {
snapshot = null;
}
const chip = pointerChip(snapshot);
this.pointerAnnounced = gesture && chip ? snapshot : null;
this.emitSessionEvent({
type: 'pointer',
pointer:
gesture && chip ? { ...chip, screenPx: snapshot.screenPx } : null,
const event = pointerTurnEvent(snapshot, gesture);
this.pointerAnnounced = event.announced;
this.emitSessionEvent({ type: 'pointer', pointer: event.pointer });
const context = pointerTurnContext(snapshot, {
at,
gesture,
sentKey: this.pointerSentKey,
});
const item = !chip
? this.pointerSentKey && this.pointerSentKey !== POINTER_NONE_KEY
? { type: 'pointer_context', at, target: 'none' }
: null
: gesture
? pointerContextItem(snapshot)
: // What kind of thing, never which: no label or coordinates.
{
type: 'pointer_context',
at,
pointing: snapshot.entity?.kind || 'ground',
};
if (!item) return snapshot;
const { at: _at, ...content } = item;
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
if (key === this.pointerSentKey) return snapshot;
if (this._turns.notifyMapEvent(item)) {
this.pointerSentKey = key;
if (!context) return snapshot;
if (this._turns.notifyMapEvent(context.item)) {
this.pointerSentKey = context.key;
const chip = context.key !== POINTER_NONE_KEY;
this.debugLog('pointer.context', {
at,
target: chip ? snapshot.target : 'none',
@@ -323,13 +313,10 @@ export class GevRealtimeController extends RealtimeFacade {
*/
announcePointer() {
const snapshot = this.pointer?.activeSnapshot?.() || null;
const chip = pointerChip(snapshot);
if (!chip || snapshot === this.pointerAnnounced) return false;
const pointer = pointerAnnouncement(snapshot, this.pointerAnnounced);
if (!pointer) return false;
this.pointerAnnounced = snapshot;
this.emitSessionEvent({
type: 'pointer',
pointer: { ...chip, screenPx: snapshot.screenPx },
});
this.emitSessionEvent({ type: 'pointer', pointer });
return true;
}
@@ -412,7 +399,7 @@ export class GevRealtimeController extends RealtimeFacade {
this.ui.root.remove();
}
if (!preserveStatus && !removeUi) {
this.setStatus('idle', 'Voice off');
this.setStatus('idle', t('Voice off'));
}
this.setRadioVoiceDucking(false);
if (removeUi) this.emitSessionEvent({ type: 'disposed' });
@@ -432,24 +419,25 @@ export class GevRealtimeController extends RealtimeFacade {
this.ui.root.dataset.status = status;
if (status === 'error') this.ui.root.classList.remove('error-dismissed');
this.updateVoiceButtonLabel();
this.ui.status.textContent = STATUS[status] || STATUS.idle;
const shownStatus = STATUS_STATES.has(status) ? status : 'idle';
this.ui.status.textContent = voiceStatusLabel(shownStatus);
const resolvedDetail =
status === 'listening' && this.pushToTalkMode
? this.pushToTalkKeyHeld
? 'Release Space to send'
: 'Hold Space to talk'
? t('Release Space to send')
: t('Hold Space to talk')
: detail;
const primaryDetail =
status === 'error'
? 'VOICE UNAVAILABLE'
? t('VOICE UNAVAILABLE')
: resolvedDetail ||
(status === 'idle' ? 'VOICE STANDBY' : 'VOICE ACTIVE');
(status === 'idle' ? t('VOICE STANDBY') : t('VOICE ACTIVE'));
this.ui.detail.textContent = primaryDetail;
this.ui.detail.title = primaryDetail;
if (this.ui.errorDetail) {
this.ui.errorDetail.textContent =
status === 'error'
? resolvedDetail || 'Voice session could not be started.'
? resolvedDetail || t('Voice session could not be started.')
: '';
}
if (status === 'idle' || status === 'connecting' || status === 'error') {
+66 -17
View File
@@ -1,3 +1,4 @@
import { t } from '../i18n/index.js';
import {
readStoredVoiceTier,
readStoredVoiceLimits,
@@ -45,10 +46,21 @@ export class RealtimeCost {
*/
syncCostUi() {
const state = this.costTracker.state();
const syncWarning = () => {
const warning = this.ui?.costWarning;
const value = this.ui?.costValue;
if (!warning || !value) return;
warning.hidden = !['warn', 'cap', 'unknown'].includes(
value.dataset.level,
);
warning.textContent = warning.hidden ? '' : value.textContent;
warning.title = value.title;
warning.dataset.level = value.dataset.level;
};
const pendingTier = resolveVoiceModel(this.voiceTier).tier;
const isMini = pendingTier === 'mini';
if (this.ui?.tierButton) {
this.ui.tierButton.textContent = isMini ? 'MINI' : 'STD';
this.ui.tierButton.textContent = isMini ? t('MINI') : t('STD');
this.ui.tierButton.setAttribute(
'aria-pressed',
isMini ? 'true' : 'false',
@@ -56,33 +68,65 @@ export class RealtimeCost {
const pendingId = resolveVoiceModel(pendingTier).id;
this.ui.tierButton.title =
this.isActive() && state.modelId !== pendingId
? `Next session: ${pendingId} — this session stays on ${state.modelId}`
: `Voice model: ${pendingId} — click to switch to ${
isMini ? 'standard' : 'mini'
}; applies next session`;
? t('Next session: {pendingId} — this session stays on {modelId}', {
pendingId,
modelId: state.modelId,
})
: t(
'Voice model: {pendingId} — click to switch to {target}; applies next session',
{
pendingId,
target: isMini ? t('standard') : t('mini'),
},
);
}
if (this.ui?.costValue) {
const liveAuth = this.sessionCloudVoiceAuth || this.cloudVoiceAuth;
if (liveAuth === 'oauth') {
const { responses, input, output, captions } = this.oauthUsage;
this.ui.costValue.hidden = false;
this.ui.costValue.textContent = 'COST UNKNOWN';
this.ui.costValue.textContent = t('COST UNKNOWN');
this.ui.costValue.dataset.level = 'unknown';
this.ui.costValue.title =
`ChatGPT OAuth session: ${responses} response(s), ${input} input and ${output} output tokens, ${captions} caption transcription(s) reported. ` +
'USD cost and the API spend cap are unavailable for this auth mode.' +
t(
'ChatGPT OAuth session: {responses} response(s), {input} input and {output} output tokens, {captions} caption transcription(s) reported.',
{ responses, input, output, captions },
) +
' ' +
t(
'USD cost and the API spend cap are unavailable for this auth mode.',
) +
(state.incomplete
? ' Usage is incomplete because a response was still in flight when the session ended.'
? ' ' +
t(
'Usage is incomplete because a response was still in flight when the session ended.',
)
: '');
syncWarning();
return;
}
this.ui.costValue.hidden = false;
this.ui.costValue.textContent = state.display;
this.ui.costValue.dataset.level = state.level;
this.ui.costValue.title =
`Estimated session cost on ${state.modelId} — ${state.responses} response(s). ` +
`Warns at ${formatCostUsd(state.warnUsd)}, ends the session at ${formatCostUsd(state.capUsd)}.` +
(state.note ? ` ${state.note}` : '');
t(
'Estimated session cost on {modelId} — {responses} response(s). Warns at {warn}, ends the session at {cap}.',
{
modelId: state.modelId,
responses: state.responses,
warn: formatCostUsd(state.warnUsd),
cap: formatCostUsd(state.capUsd),
},
) +
// `state.note` is the tracker's English diagnostic; the tooltip says
// the same in the interface language.
(state.incomplete
? ' ' +
t(
'Estimate is incomplete — a response was still in flight when the session ended, so its usage was never reported.',
)
: '');
syncWarning();
}
}
@@ -123,10 +167,12 @@ export class RealtimeCost {
}
this.syncCostUi();
if (this.isActive() && this.ui?.detail) {
this.setStatus(
this.status,
`${this.voiceTier.toUpperCase()} applies next session`,
);
// The tier as the interface names it; the stored value stays English.
const tier =
resolveVoiceModel(this.voiceTier).tier === 'mini'
? t('MINI')
: t('STANDARD');
this.setStatus(this.status, t('{tier} applies next session', { tier }));
}
return this.voiceTier;
}
@@ -240,7 +286,10 @@ export class RealtimeCost {
try {
this.stop({ preserveStatus: true });
} finally {
this.setStatus('idle', `Session ended — cost cap ${state.display}`);
this.setStatus(
'idle',
t('Session ended — cost cap {cost}', { cost: state.display }),
);
this.syncCostUi();
}
}
@@ -0,0 +1,54 @@
// The API-key cost tooltip states an incomplete estimate in the
// interface language; the tracker keeps its English note for diagnostics.
import test from 'node:test';
import assert from 'node:assert/strict';
import { setLocale } from '../i18n/index.js';
import { RealtimeCost } from './realtimeCost.js';
import { createVoiceCostTracker } from './voiceCost.js';
for (const locale of ['es', 'ru']) {
test(`API-key cost tooltip for an incomplete session has no English (${locale})`, () => {
setLocale(locale);
try {
const costTracker = createVoiceCostTracker({ tier: 'standard' });
costTracker.markIncomplete();
const host = {
costTracker,
voiceTier: 'standard',
cloudVoiceAuth: 'api-key',
sessionCloudVoiceAuth: 'api-key',
isActive: () => false,
ui: {
costValue: { hidden: true, dataset: {}, title: '', textContent: '' },
},
};
RealtimeCost.prototype.syncCostUi.call(host);
const title = host.ui.costValue.title;
assert.ok(title.length > 0);
assert.doesNotMatch(
title,
/Estimate is incomplete|still in flight/,
title,
);
} finally {
setLocale('en');
}
});
}
test('API-key cost tooltip in English still ends with the tracker note', () => {
const costTracker = createVoiceCostTracker({ tier: 'standard' });
costTracker.markIncomplete();
const host = {
costTracker,
voiceTier: 'standard',
cloudVoiceAuth: 'api-key',
sessionCloudVoiceAuth: 'api-key',
isActive: () => false,
ui: {
costValue: { hidden: true, dataset: {}, title: '', textContent: '' },
},
};
RealtimeCost.prototype.syncCostUi.call(host);
assert.ok(host.ui.costValue.title.endsWith(` ${costTracker.state().note}`));
});
+100
View File
@@ -0,0 +1,100 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceControl } from './control.js';
import { RealtimeCost } from './realtimeCost.js';
import { createVoiceCostTracker, resolveVoiceModel } from './voiceCost.js';
import { installPanelDom } from './panel/testDom.mjs';
function harness(t) {
installPanelDom(t);
const ui = createVoiceControl({ panel: true });
const stops = [];
const statuses = [];
const cost = new RealtimeCost({
readUi: () => ui,
readStatus: () => 'listening',
operations: {
isActive: () => true,
stop: (options) => stops.push(options),
setStatus: (...args) => statuses.push(args),
debugLog() {},
},
});
cost.cloudVoiceAuth = 'api-key';
cost.costTracker = createVoiceCostTracker({
tier: 'standard',
limits: { warnUsd: 2, capUsd: 5 },
});
const recordThreeDollars = () => {
const tokens = Math.ceil(
(3 * 1_000_000) / resolveVoiceModel('standard').rates.textOutput,
);
const state = cost.costTracker.record({
output_tokens: tokens,
output_token_details: { text_tokens: tokens },
});
cost.applyCost(state);
};
t.mock.method(console, 'warn', () => {});
return { cost, ui, stops, statuses, recordThreeDollars };
}
test('soft cost warning is visible outside closed Settings, while ordinary spend stays inside', (t) => {
const { cost, ui, recordThreeDollars } = harness(t);
cost.syncCostUi();
assert.equal(ui.voicePanel.hidden, true);
assert.equal(ui.costValue.checkVisibility(), false);
assert.equal(ui.costWarning.hidden, true);
assert.equal(ui.voicePanel.contains(ui.costWarning), false);
assert.equal(ui.root.querySelectorAll('#gev-voice-cost-value').length, 1);
recordThreeDollars();
assert.equal(ui.costWarning.checkVisibility(), true);
assert.equal(ui.costWarning.dataset.level, 'warn');
assert.equal(ui.costWarning.textContent, ui.costValue.textContent);
assert.equal(ui.costWarning.title, ui.costValue.title);
assert.match(ui.costWarning.title, /Warns at/);
assert.equal(
ui.voicePanel.hidden,
true,
'warning never forces Settings open',
);
cost.prepareSession();
cost.syncCostUi();
assert.equal(
ui.costWarning.hidden,
true,
'new session clears the old warning',
);
assert.equal(ui.costWarning.textContent, '');
});
test('closed Settings cannot hide the hard cap or suppress its ordinary stop and status', (t) => {
const { ui, stops, statuses, recordThreeDollars } = harness(t);
recordThreeDollars();
assert.equal(stops.length, 0);
recordThreeDollars();
assert.equal(ui.costWarning.checkVisibility(), true);
assert.equal(ui.costWarning.dataset.level, 'cap');
assert.deepEqual(stops, [{ preserveStatus: true }]);
assert.equal(statuses.at(-1)[0], 'idle');
assert.match(statuses.at(-1)[1], /Session ended — cost cap/);
});
test('unknown OAuth cost remains visible outside Settings and clears on a fresh API meter', (t) => {
const { cost, ui } = harness(t);
cost.sessionCloudVoiceAuth = 'oauth';
cost.syncCostUi();
assert.equal(ui.costWarning.checkVisibility(), true);
assert.equal(ui.costWarning.dataset.level, 'unknown');
assert.equal(ui.costWarning.textContent, 'COST UNKNOWN');
assert.match(
ui.costWarning.title,
/USD cost and the API spend cap are unavailable/,
);
assert.equal(ui.voicePanel.hidden, true);
cost.sessionCloudVoiceAuth = 'api-key';
cost.syncCostUi();
assert.equal(ui.costWarning.hidden, true);
});
+50
View File
@@ -0,0 +1,50 @@
// The status after a tier switch names the tier in the interface
// language, as the tier button does; the stored tier value stays English.
import test from 'node:test';
import assert from 'node:assert/strict';
import { setLocale, t } from '../i18n/index.js';
import { RealtimeCost } from './realtimeCost.js';
function harness() {
const button = { textContent: '', title: '', setAttribute() {} };
const ui = { tierButton: button, detail: {} };
const statusCalls = [];
const cost = new RealtimeCost({
readUi: () => ui,
readStatus: () => 'listening',
operations: {
isActive: () => true,
isVoiceSessionSettled: () => false,
setStatus: (status, detail) => statusCalls.push(detail),
},
});
return { cost, button, statusCalls };
}
const EXPECTED = {
en: {
mini: 'MINI applies next session',
standard: 'STANDARD applies next session',
},
ru: {
mini: 'МИНИ — со следующей сессии',
standard: 'СТАНДАРТ — со следующей сессии',
},
es: {
mini: 'MINI desde la próxima sesión',
standard: 'ESTÁNDAR desde la próxima sesión',
},
};
for (const locale of Object.keys(EXPECTED)) {
test(`${locale}: tier status names the tier in the interface language`, (ctx) => {
setLocale(locale);
ctx.after(() => setLocale('en'));
for (const tier of ['mini', 'standard']) {
const { cost, button, statusCalls } = harness();
assert.equal(cost.setVoiceTier(tier), tier, 'stored tier stays English');
assert.equal(statusCalls.at(-1), EXPECTED[locale][tier]);
assert.equal(button.textContent, tier === 'mini' ? t('MINI') : t('STD'));
}
});
}
+11 -18
View File
@@ -1,10 +1,10 @@
export const ERROR_LOG_LIMIT = 30;
const ERROR_LOG_LIMIT = 30;
export const ERROR_STORAGE_KEY = 'gev-realtime-errors';
const ERROR_STORAGE_KEY = 'gev-realtime-errors';
export const DEBUG_LOG_URL = '/api/realtime/debug-log';
const DEBUG_LOG_URL = '/api/realtime/debug-log';
export function createDebugSessionId() {
function createDebugSessionId() {
const randomPart = Math.random().toString(36).slice(2, 10);
return `gev-${Date.now().toString(36)}-${randomPart}`;
}
@@ -47,7 +47,7 @@ export function sanitizeDebugValue(value, depth = 0) {
return output;
}
export function sanitizeDebugString(value) {
function sanitizeDebugString(value) {
if (value.startsWith('data:image/')) {
return `[Redacted image data URL, ${value.length} chars]`;
}
@@ -65,13 +65,13 @@ export function sanitizeDebugString(value) {
: redacted;
}
export function isSecretLikeKey(key) {
function isSecretLikeKey(key) {
return /(?:api[_-]?key|authorization|bearer|client[_-]?secret|token|secret|password)/i.test(
key,
);
}
export function createErrorRecord(source, error, extra = {}) {
function createErrorRecord(source, error, extra = {}) {
const rtcError = error?.error || error;
return {
timestamp: new Date().toISOString(),
@@ -90,7 +90,7 @@ export function createErrorRecord(source, error, extra = {}) {
};
}
export function formatErrorForDisplay(record) {
function formatErrorForDisplay(record) {
const primary = [record.source, record.message].filter(Boolean).join(': ');
const state = [
record.errorDetail && `detail=${record.errorDetail}`,
@@ -105,7 +105,7 @@ export function formatErrorForDisplay(record) {
return state ? `${primary}\n${state}` : primary;
}
export function removeEmptyValues(value) {
function removeEmptyValues(value) {
return Object.fromEntries(
Object.entries(value || {}).filter(
([, item]) => item !== null && item !== undefined && item !== '',
@@ -113,14 +113,7 @@ export function removeEmptyValues(value) {
);
}
export function compactText(value, maxLength) {
const text = String(value || '')
.replace(/\s+/g, ' ')
.trim();
return text.length <= maxLength ? text : `${text.slice(0, maxLength - 1)}…`;
}
export function loadStoredErrors() {
function loadStoredErrors() {
try {
const value = JSON.parse(localStorage.getItem(ERROR_STORAGE_KEY) || '[]');
return Array.isArray(value) ? value.slice(0, ERROR_LOG_LIMIT) : [];
@@ -129,7 +122,7 @@ export function loadStoredErrors() {
}
}
export function storeErrors(errors) {
function storeErrors(errors) {
try {
localStorage.setItem(
ERROR_STORAGE_KEY,
+5 -3
View File
@@ -1,3 +1,4 @@
import { t } from '../i18n/index.js';
import {
PUSH_TO_TALK_HOLD_DELAY_MS,
MICROPHONE_VISUALIZER_GATE,
@@ -118,7 +119,7 @@ export class RealtimeInput {
this.ui.root.dataset.pushToTalk = 'held';
this.setMicrophoneEnabled(true);
if (this.status === 'listening')
this.setStatus('listening', 'Release Space to send');
this.setStatus('listening', t('Release Space to send'));
} else {
this.start({ pushToTalk: true });
// start() performs a controlled stop() before connecting. Restore the
@@ -194,7 +195,7 @@ export class RealtimeInput {
if (!this.pushToTalkMode) return;
this.setMicrophoneEnabled(false);
if (this.status === 'listening')
this.setStatus('listening', 'Hold Space to talk');
this.setStatus('listening', t('Hold Space to talk'));
else this.updateVoiceButtonLabel();
}
@@ -363,7 +364,8 @@ export class RealtimeInput {
*/
updateVoiceButtonLabel() {
if (!this.ui.buttonLabel) return;
this.ui.buttonLabel.textContent = 'MIC';
// The mic button's caption; the first-run tip names it the same way.
this.ui.buttonLabel.textContent = t('MIC');
if (this.ui.helpDetail) {
this.ui.helpDetail.textContent = resolveVoiceControlHint(
this.pushToTalkMode,
+4 -2
View File
@@ -1,3 +1,5 @@
import { t } from '../i18n/index.js';
export const PUSH_TO_TALK_HOLD_DELAY_MS = 500;
// The input meter is intentionally stricter than the assistant-output meter:
@@ -150,8 +152,8 @@ export function resolveVoiceVisualizerSpeaker(
*/
export function resolveVoiceControlHint(pushToTalkMode, pushToTalkKeyHeld) {
return pushToTalkMode && pushToTalkKeyHeld
? 'Release Space to send'
: 'Hold Space to speak · tap Space to activate focused controls';
? t('Release Space to send')
: t('Hold Space to speak · tap Space to activate focused controls');
}
/**
+1 -1
View File
@@ -135,7 +135,7 @@ test('an old progress completion leaves the active in-band response alone', asyn
});
test('user speech stops an audible progress line: cancel and clear its audio', async () => {
const { controller, sent, event, progressLine } = controllerFixture();
const { sent, event, progressLine } = controllerFixture();
await progressLine('resp_p');
await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' });
sent.length = 0;
+3 -3
View File
@@ -8,12 +8,12 @@ import {
// Voice cost control (repo-wide `godsEyeView.<feature>.<field>` convention;
// the neighbouring ERROR_STORAGE_KEY predates it).
export const VOICE_TIER_STORAGE_KEY = 'godsEyeView.voiceCost.tier';
const VOICE_TIER_STORAGE_KEY = 'godsEyeView.voiceCost.tier';
export const VOICE_LIMITS_STORAGE_KEY = 'godsEyeView.voiceCost.limits';
const VOICE_LIMITS_STORAGE_KEY = 'godsEyeView.voiceCost.limits';
/** Best-effort localStorage handle; absent in tests and locked-down browsers. */
export function voiceStorage(storage) {
function voiceStorage(storage) {
if (storage) return storage;
try {
return typeof localStorage === 'undefined' ? null : localStorage;
+26
View File
@@ -99,6 +99,18 @@ export function hasStructuredViewIdentity(result) {
);
}
export function isGlobeViewRefusal(result) {
return (
result?.action === 'move_camera' &&
result.ok === false &&
[
'globe-view-required',
'globe-reset-unavailable',
'globe-reset-failed',
].includes(result.reason)
);
}
export function responseInstructionForToolResult(result) {
if (result?.action === 'control_radio' && result.radioPlaybackSuppressed) {
if (result.audioState === 'paused') {
@@ -123,6 +135,20 @@ export function responseInstructionForToolResult(result) {
: 'Be concise: useful names, place, layer and one or two notable properties.',
].join(' ');
}
if (isGlobeViewRefusal(result)) {
// A refusal is not a completed action. Keep tool text out of instructions.
if (result.reason === 'globe-reset-failed')
return 'The globe did not rotate after the Globe View reset attempt. Say so briefly. Do not retry, offer or perform another reset for this request.';
if (result.reason === 'globe-reset-unavailable')
return 'The globe did not rotate. Ask the user to zoom out further; the Globe View preset does not fit this viewport. Do not offer or perform a reset.';
return 'The globe did not rotate. Say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" The suggested command is not a user request. Do not ask a question, perform a reset, or call tools in this reply.';
}
if (
(result?.action ?? result?.tool) === 'zoom_to_globe' &&
(result.ok !== true || result.cancelled)
) {
return 'The Globe View reset did not finish. Say so briefly; do not claim Globe View was reached. Do not retry the reset or rotate the globe for this request.';
}
if (result?.action === 'adjust_camera_zoom') {
return result.ok
? `Confirm once that the camera zoomed ${result.direction}. Do not claim any other change.`
+24 -2
View File
@@ -1,3 +1,4 @@
import { t } from '../i18n/index.js';
import {
silenceRadioForVoice,
startPreparedRadioAfterPlaybackReady,
@@ -246,7 +247,9 @@ export class RealtimeRadio {
handoffChannel?.readyState === 'open',
},
);
const stillCurrent = handoffEpoch === this.radioHandoffEpoch;
const stillCurrent =
handoffEpoch === this.radioHandoffEpoch &&
this.radioHandoffAttemptId === handoffAttemptId;
if (this.radioHandoffAttemptId === handoffAttemptId) {
this.radioHandoffInFlight = false;
this.radioHandoffAttemptId = null;
@@ -258,13 +261,32 @@ export class RealtimeRadio {
if (radioHandoff.result?.ok || radioHandoff.cancelled || !stillCurrent)
return;
if (this.dc?.readyState === 'open' && !this.userTurnPending) {
this.setStatus('listening', 'Radio did not start');
this.setStatus('listening', t('Radio did not start'));
this.queueResponseCreate(
'Say exactly one short correction: “The Radio station could not start. Voice is still on.”',
);
}
}
/** Revoke one call's prepared/preflight result, preserving other owners. */
cancelPlaybackResult(result) {
if (!result) return;
if (this.pendingRadioPlaybackResult === result) {
this.pendingRadioPlaybackResult = null;
this.radioHandoffDeferredByReservation = false;
}
if (this.radioHandoffInFlightResult !== result) return;
const attemptId = this.radioHandoffAttemptId;
const shouldStopPlayback = this.radioHandoffInFlight;
this.radioHandoffInFlightResult = null;
this.radioHandoffInFlight = false;
this.radioHandoffAttemptId = null;
// Keep the group epoch and sibling reservations; release ownership before
// Stop can synchronously notify playback observers.
if (shouldStopPlayback)
this.radioLayer?.stopPlayback?.({ origin: 'voice-cleanup', attemptId });
}
/** Invalidate delayed Radio work inside the requested authority scope. */
abortRadioSiblingTools({ responseId = null, scope = 'all' } = {}) {
for (const [controller, metadata] of this.activeRadioToolControllers) {
+120
View File
@@ -0,0 +1,120 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { GevRealtimeController } from './realtimeController.js';
const tick = () => new Promise((resolve) => setImmediate(resolve));
const playback = (id) => ({
ok: true,
action: 'control_radio',
radioAction: 'play',
radioPlaybackRequested: true,
id,
});
function fixture(t) {
const attempts = [],
stopped = [],
corrections = [],
states = [];
const controller = new GevRealtimeController({
ui: {
root: {
dataset: {},
classList: { remove() {} },
querySelectorAll: () => [],
remove() {},
},
status: {},
detail: {},
errorDetail: {},
},
runner: async () => ({ ok: false }),
debugSink: null,
onSessionEvent: (event) => states.push(event),
radioLayer: {
setVoiceDucked() {},
playForVoice: ({ attemptId }) =>
new Promise((resolve, reject) =>
attempts.push({ attemptId, resolve, reject }),
),
stopPlayback: (options) => stopped.push(options),
},
});
controller.queueResponseCreate = (message) => corrections.push(message);
controller.dc = { readyState: 'open', send() {}, close() {} };
t.after(() => controller.stop({ removeUi: true }));
return { controller, attempts, stopped, corrections, states };
}
test('default OpenAI reservation silences rejected frozen preflight and resumes retained result after failure', async (t) => {
const h = fixture(t),
c = h.controller;
const a = playback('A');
c.pendingRadioPlaybackResult = a;
const pending = c.startPendingRadioHandoff();
const reservation = c.reserveRadioToolHandoff();
assert.equal(c.pendingRadioPlaybackResult, a);
assert.equal(c.radioHandoffInFlight, false);
h.attempts[0].reject(new Error('obsolete frozen preflight'));
await pending;
assert.deepEqual(h.corrections, []);
assert.equal(
h.states.some((event) => event.detail === 'Radio did not start'),
false,
);
c.settleRadioToolHandoffReservation(reservation, { commit: false });
await tick();
assert.equal(h.attempts.length, 2);
assert.equal(c.radioHandoffInFlightResult, a);
assert.notEqual(h.attempts[0].attemptId, h.attempts[1].attemptId);
h.attempts[1].reject(new Error('current preflight failed'));
await tick();
assert.equal(h.corrections.length, 1);
assert.equal(c.status, 'listening');
});
test('default OpenAI group cancellation silences rejected preflight without blocking the next failure correction', async (t) => {
const h = fixture(t),
c = h.controller;
c.pendingRadioPlaybackResult = playback('A');
const old = c.startPendingRadioHandoff();
c.cancelRadioHandoff();
h.attempts[0].reject(new Error('cancelled preflight'));
await old;
assert.deepEqual(h.corrections, []);
c.pendingRadioPlaybackResult = playback('B');
const current = c.startPendingRadioHandoff();
h.attempts[1].resolve(false);
await current;
assert.equal(h.corrections.length, 1);
});
test('per-result cancellation retains sibling result and epoch and ignores late failure or duplicate cancellation', async (t) => {
const h = fixture(t),
c = h.controller;
const a = playback('A'),
b = playback('B');
c.pendingRadioPlaybackResult = a;
const old = c.startPendingRadioHandoff();
const siblingEpoch = c.radioHandoffEpoch;
c.pendingRadioPlaybackResult = b;
c._radio.cancelPlaybackResult(a);
c._radio.cancelPlaybackResult(a);
c._radio.cancelPlaybackResult(playback('unknown'));
assert.equal(c.radioHandoffEpoch, siblingEpoch);
assert.equal(c.pendingRadioPlaybackResult, b);
assert.equal(h.stopped.length, 1);
h.attempts[0].reject(new Error('late cancelled preflight'));
await old;
assert.deepEqual(h.corrections, []);
const current = c.startPendingRadioHandoff();
const reservation = c.reserveRadioToolHandoff();
c._radio.cancelPlaybackResult(b);
assert.equal(c._radio.radioToolHandoffReservations.has(reservation), true);
h.attempts[1].resolve(false);
await current;
assert.deepEqual(h.corrections, []);
c.settleRadioToolHandoffReservation(reservation, { commit: false });
await tick();
assert.equal(h.attempts.length, 2, 'a cancelled result cannot be resumed');
});
+1
View File
@@ -16,6 +16,7 @@ export function createRealtimeSession({
controller,
capabilities: { costControls: true, pushToTalk: true },
start: (settings) => controller.start(settings),
getStartOptions: () => ({ pushToTalk: Boolean(controller.pushToTalkMode) }),
stop: (settings) => controller.stop(settings),
sendText: (text) => controller.sendTextCommand(text),
sendMapEvent: (event) => controller.notifyMapEvent(event),
+236 -16
View File
@@ -1,10 +1,12 @@
import { realtimeSessionEvent } from './realtimeEvents.js';
import { t } from '../i18n/index.js';
import { readLayerLifecycleSummary } from './layerSummary.js';
import {
CALL_DEDUPE_MS,
SUPERSEDED_RESPONSE_MEMORY,
shouldStopVoiceAfterRadioTool,
responseInstructionForToolResult,
isGlobeViewRefusal,
extractFunctionCalls,
callDedupeKeys,
parseArguments,
@@ -12,7 +14,7 @@ import {
import { narrationLabel } from './speech.js';
/** Out-of-band progress responses carry this metadata tag. */
export const PROGRESS_RESPONSE_TAG = 'progress';
const PROGRESS_RESPONSE_TAG = 'progress';
/**
* Speaks one code-authored progress line out of band. The line travels as
@@ -65,6 +67,11 @@ export class RealtimeTurns {
this.pendingUserTextResponse = false;
this.activeResponseId = null;
this.supersededResponseIds = new Set();
this.globeRefusals = new Map();
this.globeAdmissionGates = new Map();
this.globeResets = new Map();
this.globeResponseRequests = new Map();
this.globeRequest = { failed: false, pending: new Set() };
this.activeToolAbortControllers = new Set();
this.narration = null;
this.narrationSequence = 0;
@@ -136,7 +143,8 @@ export class RealtimeTurns {
content: [{ type: 'input_text', text: cleanText }],
},
};
this.sendRealtimeEvent(itemEvent, 'client.user_text');
if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return;
this.globeRequest = { failed: false, pending: new Set() };
this.requestUserTextResponse();
}
@@ -277,7 +285,7 @@ export class RealtimeTurns {
eventId: payload.event_id,
activeResponseMessage: payload.error?.message || null,
});
this.setStatus('listening', 'Ask or command');
this.setStatus('listening', t('Ask or command'));
return;
}
// A conversation.item.delete for a stale viewport screenshot can land
@@ -329,6 +337,7 @@ export class RealtimeTurns {
this.userTurnPending = true;
this.pendingResponseInstructions = null;
this.cancelRadioHandoff({ abortTools: true });
this.globeRequest = { failed: false, pending: new Set() };
this.setVoiceSpeaker('user');
// Point-and-ask: read the pointer as speech starts (open mic), or reuse
// the push-to-talk keydown snapshot, before the user audio commits.
@@ -383,6 +392,27 @@ export class RealtimeTurns {
}
const toolResponseId = payload.response_id || payload.response?.id || null;
// Follow-up responses retain reset ownership until fresh admitted input.
const arrivalRequest = this.globeRequest;
const globeRequest =
this.globeResponseRequests.get(toolResponseId) ?? arrivalRequest;
// Later siblings must wait for globe admission before they can mutate state.
// Other tool responses keep their existing concurrency.
let heldForAdmission = false;
while (toolResponseId && this.globeAdmissionGates.has(toolResponseId)) {
// Drop claimed duplicates before every wait, before their keys can expire.
if (
calls.every((call) =>
callDedupeKeys(call).some((key) => this.processedCalls.has(key)),
)
)
return;
heldForAdmission = true;
await this.globeAdmissionGates.get(toolResponseId).promise;
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
return;
}
// A newer typed command superseded the response these calls belong to.
// They are stale intent — dispatching one would let the old turn mutate
// the map after the operator asked for something else.
@@ -393,7 +423,10 @@ export class RealtimeTurns {
// hazard `callDedupeKeys` is written to avoid). So each refused call gets
// a terminal output saying plainly that the turn moved on. No
// `response.create` follows — the deferred typed turn is the single answer.
if (this.isSupersededResponse(toolResponseId)) {
if (
this.isSupersededResponse(toolResponseId) ||
(heldForAdmission && arrivalRequest !== this.globeRequest)
) {
this.pruneProcessedCalls();
for (const call of calls) {
const keys = callDedupeKeys(call);
@@ -414,7 +447,7 @@ export class RealtimeTurns {
return;
}
this.setStatus('executing', 'Running command');
this.setStatus('executing', t('Running command'));
this.pruneProcessedCalls();
let sentOutput = false;
let lastResult = null;
@@ -428,7 +461,19 @@ export class RealtimeTurns {
if (keys.some((key) => this.processedCalls.has(key))) continue;
keys.forEach((key) => this.processedCalls.set(key, performance.now()));
if (this.globeRefusals.get(toolResponseId)?.blocking) {
this.sendToolOutput(call.call_id || call.id, {
ok: false,
action: call.name,
skipped: true,
error: 'Not run: this response must wait for the Globe View reply.',
});
continue;
}
let result;
let globeAdmissionGate = null;
let globeResetGate = null;
const resultChannel = this.dc;
let radioHandoffEpochAtStart = this.radio.radioHandoffEpoch;
let toolController = null;
@@ -442,6 +487,80 @@ export class RealtimeTurns {
this.narration?.preamble();
try {
const parsedArguments = parseArguments(call.arguments);
if (
toolResponseId &&
call.name === 'move_camera' &&
String(parsedArguments.motion || '').toLowerCase() === 'rotate_globe'
) {
let resolve;
const promise = new Promise((settle) => {
resolve = settle;
});
globeAdmissionGate = { promise, resolve };
this.globeAdmissionGates.set(toolResponseId, globeAdmissionGate);
// OpenAI delivers sibling calls concurrently. A reset dispatched
// first must finish before rotation evaluates the camera's fit.
// Hold later siblings at admission while this rotation waits.
const resets = this.globeResets.get(toolResponseId);
if (
resets ||
globeRequest.failed ||
globeRequest.pending.size ||
arrivalRequest !== this.globeRequest
) {
while (resets?.pending.size || globeRequest.pending.size)
await Promise.all(
[
...new Set([
...(resets?.pending || []),
...globeRequest.pending,
]),
].map((gate) => gate.promise),
);
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
return;
keys.forEach((key) =>
this.processedCalls.set(key, performance.now()),
);
const superseded =
this.isSupersededResponse(toolResponseId) ||
arrivalRequest !== this.globeRequest;
if (superseded || resets?.failed || globeRequest.failed) {
result = {
ok: false,
action: call.name,
reason: 'globe-reset-failed',
skipped: true,
...(superseded ? { superseded: true } : {}),
error: superseded
? 'Superseded by a newer command from the operator — this call was not run.'
: 'Not run: the Globe View reset/rotation sequence failed or was cancelled.',
};
}
}
}
if (toolResponseId && call.name === 'zoom_to_globe') {
let resolve;
const promise = new Promise((settle) => {
resolve = settle;
});
let resets = this.globeResets.get(toolResponseId);
if (!resets) {
resets = { pending: new Set(), failed: false };
this.globeResets.set(toolResponseId, resets);
// Retain settled outcomes for later siblings, without evicting an
// active reset that another response still needs to await.
for (const [id, record] of this.globeResets) {
if (this.globeResets.size <= SUPERSEDED_RESPONSE_MEMORY) break;
if (!record.pending.size && id !== toolResponseId)
this.globeResets.delete(id);
}
}
globeResetGate = { promise, resolve, resets, request: globeRequest };
resets.pending.add(globeResetGate);
globeRequest.pending.add(globeResetGate);
}
const isRadioControlCall = call.name === 'control_radio';
const isRadioVisibilityCall =
call.name === 'set_layer_visibility' &&
@@ -498,7 +617,7 @@ export class RealtimeTurns {
authorityDomain: radioAuthorityDomain,
});
}
result = await (this.actionExecutor || this.runner)(
result ??= await (this.actionExecutor || this.runner)(
call.name,
parsedArguments,
{
@@ -514,7 +633,28 @@ export class RealtimeTurns {
radioHandoffEpochAtStart === this.radio.radioHandoffEpoch),
},
);
if (
globeAdmissionGate &&
this.globeResets.has(toolResponseId) &&
result?.reason === 'globe-view-required'
)
result = { ...result, reason: 'globe-reset-failed' };
if (!this.ownsConversation(eventChannel)) return;
if (
isGlobeViewRefusal(result) &&
toolResponseId &&
arrivalRequest === this.globeRequest &&
!this.isSupersededResponse(toolResponseId)
) {
this.globeRefusals.set(toolResponseId, {
result,
followupQueued: false,
blocking: !result.skipped,
});
while (this.globeRefusals.size > SUPERSEDED_RESPONSE_MEMORY) {
this.globeRefusals.delete(this.globeRefusals.keys().next().value);
}
}
if (result?.ok && result.radioPlaybackRequested) {
const sessionIsCurrent =
this.activeToolAbortControllers.has(toolController) &&
@@ -594,6 +734,23 @@ export class RealtimeTurns {
: {}),
};
} finally {
if (globeResetGate) {
const { resets, request } = globeResetGate;
if (result?.ok !== true || result.cancelled) {
resets.failed = true;
request.failed = true;
}
resets.pending.delete(globeResetGate);
request.pending.delete(globeResetGate);
globeResetGate.resolve();
}
if (globeAdmissionGate) {
if (
this.globeAdmissionGates.get(toolResponseId) === globeAdmissionGate
)
this.globeAdmissionGates.delete(toolResponseId);
globeAdmissionGate.resolve();
}
if (toolController) {
this.activeToolAbortControllers.delete(toolController);
this.radio.releaseTool(toolController);
@@ -670,15 +827,40 @@ export class RealtimeTurns {
});
}
if (!this.ownsConversation(eventChannel)) return;
// Keep the Radio handoff wording authoritative even when another tool
// result follows Radio in the same multi-intent response.
this.queueResponseCreate(
responseInstructionForToolResult(
this.radio.pendingRadioPlaybackResult || lastResult,
),
);
const refusal = this.globeRefusals.get(toolResponseId);
// A newer turn may arrive during admission or the viewport-image await.
// Preserve its reply; neither the refusal map nor lastResult may revive
// an old refusal follow-up after supersession.
if (
(this.isSupersededResponse(toolResponseId) ||
arrivalRequest !== this.globeRequest) &&
(refusal || isGlobeViewRefusal(lastResult) || lastResult?.superseded)
) {
// Complete an idle turn below, but leave a newer running tool's status
// authoritative while suppressing this obsolete refusal follow-up.
if (this.activeToolAbortControllers.size) return;
} else if (refusal) {
// Even an earlier sibling finishing late cannot replace the reset instruction.
if (!refusal.followupQueued) {
refusal.followupQueued = true;
this.queueResponseCreate({
instructions: responseInstructionForToolResult(refusal.result),
tool_choice: 'none',
});
}
} else {
// Keep Radio handoff wording authoritative for ordinary multi-tool replies.
const followupResult =
this.radio.pendingRadioPlaybackResult || lastResult;
const instructions = responseInstructionForToolResult(followupResult);
this.queueResponseCreate(
isGlobeViewRefusal(followupResult)
? { instructions, tool_choice: 'none' }
: instructions,
);
}
}
this.setStatus('listening', 'Ask or command');
this.setStatus('listening', t('Ask or command'));
}
/**
@@ -965,6 +1147,22 @@ export class RealtimeTurns {
this.userTurnPending = false;
this.activeResponseId =
payload.response?.id || payload.response_id || null;
// Late calls retain only their globe reset/request ownership. Ordinary
// tool admission and concurrency continue to use their existing policy.
if (
this.activeResponseId &&
!this.globeResponseRequests.has(this.activeResponseId)
) {
this.globeResponseRequests.set(
this.activeResponseId,
this.globeRequest,
);
while (this.globeResponseRequests.size > SUPERSEDED_RESPONSE_MEMORY)
this.globeResponseRequests.delete(
this.globeResponseRequests.keys().next().value,
);
}
this.setVoiceSpeaker('ai');
return;
}
@@ -997,7 +1195,7 @@ export class RealtimeTurns {
// connection is still live. Recover to listening so the user can retry
// (mirrors the transient-blip philosophy, H8).
if (this.dc?.readyState === 'open') {
this.setStatus('listening', 'Ask or command');
this.setStatus('listening', t('Ask or command'));
}
}
if (!this.radio.pendingRadioPlaybackResult) {
@@ -1044,7 +1242,8 @@ export class RealtimeTurns {
const sent = this.sendRealtimeEvent(
{
type: 'response.create',
response: { instructions },
response:
typeof instructions === 'string' ? { instructions } : instructions,
},
'client.response_create.tool_followup',
);
@@ -1056,6 +1255,20 @@ export class RealtimeTurns {
controller.abort();
this.activeToolAbortControllers.clear();
this.radio.clearTools();
this.releaseGlobeResets();
}
releaseGlobeResets() {
for (const resets of this.globeResets.values()) {
if (!resets.pending.size) continue;
resets.failed = true;
for (const gate of resets.pending) {
gate.request.failed = true;
gate.request.pending.delete(gate);
gate.resolve();
}
resets.pending.clear();
}
}
reset() {
@@ -1067,6 +1280,13 @@ export class RealtimeTurns {
this.pendingUserTextResponse = false;
this.activeResponseId = null;
this.supersededResponseIds.clear();
this.globeRefusals.clear();
for (const gate of this.globeAdmissionGates.values()) gate.resolve();
this.globeAdmissionGates.clear();
this.releaseGlobeResets();
this.globeResets.clear();
this.globeResponseRequests.clear();
this.globeRequest = { failed: false, pending: new Set() };
this.narration?.cancel();
this.assistantAudioPlaying = false;
this.speechResponseIds.clear();
+268
View File
@@ -0,0 +1,268 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { GevRealtimeController } from './realtimeController.js';
function fixture({ pointer = null, onUserSend = () => {} } = {}) {
const sent = [];
const executed = [];
const rejectedTexts = new Set();
let rejectResponse = false;
const controller = new GevRealtimeController({
runner: async (name) => {
executed.push(name);
return { ok: true };
},
backend: {},
debugSink: null,
pointer,
ui: {
root: {
dataset: {},
classList: { remove() {} },
querySelectorAll: () => [],
},
status: {},
detail: {},
},
});
controller.dc = {
readyState: 'open',
send(body) {
const value = JSON.parse(body);
const text = value.item?.role === 'user' && value.item.content?.[0]?.text;
if (value.item?.role === 'user') onUserSend();
if (
rejectedTexts.has(text) ||
(rejectResponse && value.type === 'response.create')
)
throw new Error('Synthetic channel rejection');
sent.push(value);
},
close() {
this.readyState = 'closed';
},
};
controller.status = 'listening';
controller._turns.narration = null;
const event = (payload) =>
controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
const created = (id) => event({ type: 'response.created', response: { id } });
const done = (id) =>
event({
type: 'response.done',
response: { id, status: 'completed', output: [] },
});
const creates = () =>
sent.filter(
(x) =>
x.type === 'response.create' &&
x.response?.metadata?.gev !== 'progress',
);
const texts = () =>
sent
.filter((x) => x.item?.role === 'user')
.map((x) => x.item.content[0].text);
return {
controller,
sent,
executed,
rejectedTexts,
event,
created,
done,
creates,
texts,
rejectResponses(value) {
rejectResponse = value;
},
};
}
test('rejected idle text schedules no answer, and a successful successor answers once', async () => {
const f = fixture();
try {
f.rejectedTexts.add('rejected');
assert.equal(f.controller.sendTextCommand('rejected'), undefined);
assert.equal(f.creates().length, 0);
assert.equal(f.controller.pendingUserTextResponse, false);
assert.equal(f.controller.responseCreatePending, false);
assert.equal(f.controller.sendTextCommand('successor'), undefined);
assert.deepEqual(f.texts(), ['successor']);
assert.equal(f.creates().length, 1);
await f.created('successor');
await f.done('successor');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
test('rejected text behind an active response cannot queue an answer', async () => {
const f = fixture();
try {
await f.created('active');
f.controller.queueResponseCreate('Old tool follow-up');
assert.equal(
f.controller.pendingResponseInstructions,
'Old tool follow-up',
);
const runningTool = new AbortController();
f.controller._turns.activeToolAbortControllers.add(runningTool);
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.equal(f.controller.pendingUserTextResponse, false);
assert.equal(f.controller.pendingResponseInstructions, null);
assert.equal(f.controller._turns.isSupersededResponse('active'), true);
assert.equal(runningTool.signal.aborted, true);
const staleCall = {
type: 'response.function_call_arguments.done',
response_id: 'active',
call_id: 'stale-call',
name: 'fly_to_location',
arguments: '{"query":"Paris"}',
};
await f.event(staleCall);
await f.event(staleCall);
assert.deepEqual(f.executed, []);
const outputs = f.sent.filter(
(x) => x.item?.type === 'function_call_output',
);
assert.equal(outputs.length, 1);
assert.equal(outputs[0].item.call_id, 'stale-call');
assert.equal(JSON.parse(outputs[0].item.output).superseded, true);
await f.done('active');
assert.equal(f.creates().length, 0);
} finally {
f.controller.stop();
}
});
test('rejected text still prepares narration and pointer context before sending', () => {
const steps = [];
const f = fixture({
pointer: {
beginTurn() {
steps.push('pointer');
return null;
},
clear() {},
},
onUserSend() {
steps.push('send');
},
});
f.controller._turns.narration = {
cancel() {
steps.push('cancel');
},
userTurnEnded() {
steps.push('user-turn-ended');
},
};
try {
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.deepEqual(steps, ['cancel', 'user-turn-ended', 'pointer', 'send']);
assert.equal(f.creates().length, 0);
} finally {
f.controller.stop();
}
});
test('earlier accepted deferred input survives a newer failed send and answers once', async () => {
const f = fixture();
try {
await f.created('active');
f.controller.sendTextCommand('accepted');
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.deepEqual(f.texts(), ['accepted']);
assert.equal(f.controller.pendingUserTextResponse, true);
assert.equal(f.creates().length, 0);
await f.done('active');
assert.equal(f.creates().length, 1);
await f.done('active');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
test('rejected text preserves an outstanding create without adding deferred input', async () => {
const f = fixture();
try {
f.controller.sendTextCommand('accepted');
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.equal(f.controller.responseCreatePending, true);
assert.equal(f.controller.pendingUserTextResponse, false);
await f.created('accepted');
await f.done('accepted');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
for (const acceptedFirst of [false, true]) {
test(`progress completion answers only accepted deferred input (accepted first=${acceptedFirst})`, async () => {
const f = fixture();
try {
assert.equal(
f.controller._turns.speakProgressLine('Checking the map.'),
true,
);
await f.event({
type: 'response.created',
response: { id: 'progress', metadata: { gev: 'progress' } },
});
if (acceptedFirst) f.controller.sendTextCommand('accepted');
f.rejectedTexts.add('rejected');
f.controller.sendTextCommand('rejected');
assert.equal(f.controller.pendingUserTextResponse, acceptedFirst);
await f.event({
type: 'response.done',
response: {
id: 'progress',
status: 'cancelled',
metadata: { gev: 'progress' },
},
});
assert.equal(f.creates().length, acceptedFirst ? 1 : 0);
} finally {
f.controller.stop();
}
});
}
test('accepted text with a rejected response.create releases the pending latch for a successor', async () => {
const f = fixture();
try {
f.rejectResponses(true);
f.controller.sendTextCommand('accepted without response');
assert.equal(f.controller.responseCreatePending, false);
assert.equal(f.creates().length, 0);
f.rejectResponses(false);
f.controller.sendTextCommand('successor');
assert.deepEqual(f.texts(), ['accepted without response', 'successor']);
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
test('successful deferred typed burst still retains both items and coalesces one answer', async () => {
const f = fixture();
try {
await f.created('active');
f.controller.sendTextCommand('first');
f.controller.sendTextCommand('second');
assert.deepEqual(f.texts(), ['first', 'second']);
assert.equal(f.creates().length, 0);
await f.done('active');
await f.done('active');
assert.equal(f.creates().length, 1);
} finally {
f.controller.stop();
}
});
+8 -8
View File
@@ -5,17 +5,17 @@ import {
import { cameraPoseKey } from './pointerContext.js';
/** The live viewer's camera pose identity (Cesium-free read). */
export function readViewerCameraKey() {
function readViewerCameraKey() {
return cameraPoseKey(globalThis.window?.__godsEyeView?.viewer);
}
// Viewport-screenshot size guards (M13). The old code clamped WIDTH only, so a
// tall portrait window produced an oversized capture whose dc.send could throw.
// Cap total pixels (clamps both dimensions) and drop the image entirely if the
// encoded data URL is still too big for the data channel.
export const VIEWPORT_MAX_PIXELS = 1200 * 900;
const VIEWPORT_MAX_PIXELS = 1200 * 900;
// ~1.08 MP, matches the old 1200px-wide landscape budget
export const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
export async function captureViewportImage() {
const viewer = window.__godsEyeView?.viewer;
@@ -66,11 +66,11 @@ export async function captureViewportImage() {
}
/** Side of the point-and-ask crop, in CSS pixels, and its output size. */
export const POINTER_CROP_SIZE = 512;
const POINTER_CROP_SIZE = 512;
/** A pointer snapshot older than this no longer matches the frame. */
export const POINTER_CROP_MAX_AGE_MS = 20000;
const POINTER_CROP_MAX_AGE_MS = 20000;
/** View scales where a crop shows buildings and terrain, not a smear. */
export const POINTER_CROP_SCALES = new Set(['local', 'city']);
const POINTER_CROP_SCALES = new Set(['local', 'city']);
/**
* Square source rectangle around a canvas point, clamped inside the canvas,
@@ -128,7 +128,7 @@ export function pointerCropRect({
* @returns {Promise<{dataUrl: string, rect: object}|null>} JPEG data URL and
* the crop's normalized bounds in the full canvas, or null.
*/
export async function capturePointerCrop(screenPx) {
async function capturePointerCrop(screenPx) {
const viewer = window.__godsEyeView?.viewer;
const source = viewer?.scene?.canvas;
if (!source?.width || !source?.height || !screenPx) return null;
@@ -259,7 +259,7 @@ export async function renderFreshCesiumFrame(viewer) {
}
}
export function isNearlyBlackFrame(ctx, width, height) {
function isNearlyBlackFrame(ctx, width, height) {
const sampleWidth = Math.min(48, width);
const sampleHeight = Math.min(32, height);
if (!sampleWidth || !sampleHeight) return true;
+337 -12
View File
@@ -9,11 +9,27 @@
* referents the card lists (the referent registry reads the same answer, so
* "the second one" always means the second row on screen).
*
* What the model receives stays English (`analystHeadline` feeds the spoken
* answer); the card shows the same answer in the active language.
*
* Pure: no DOM, no Cesium.
* @module voice/resultDisplay
*/
import { voiceLayer } from './layerManifest.js';
import {
formatDate,
formatNumber,
getLocale,
msg,
t,
tn,
translateKnown,
} from '../i18n/index.js';
import { layerNameForId, sourceLabel } from '../ui/layerNames.js';
import { localizePlaceName } from '../ui/placeNames.js';
import { translateStatusLine } from '../ui/statusText.js';
import { formatQuantity } from '../ui/unitLabels.js';
const LAYER_NOUNS = Object.freeze({
flights: 'aircraft',
@@ -43,7 +59,7 @@ function notesOf(...values) {
];
}
export function analystNounFor(layerKeys) {
function analystNounFor(layerKeys) {
const keys = [...new Set(layerKeys.filter(Boolean))];
if (keys.length !== 1) return 'results';
const key = keys[0];
@@ -52,13 +68,17 @@ export function analystNounFor(layerKeys) {
/** The count/scope headline shared by the analyst card and spoken answer. */
const ANALYST_HEADLINE_MAX = 64;
// The card's headline wraps, and a translation runs longer than the English:
// "Al menos 12 aeronaves militares en un radio de 40 km de Los Ángeles" is 67.
// English keeps the spoken bound, so its card title still equals `say`.
const SHOWN_HEADLINE_MAX = 120;
function boundedHeadline(value) {
function boundedHeadline(value, max = ANALYST_HEADLINE_MAX) {
const text = String(value || '')
.replace(/\s+/g, ' ')
.trim();
if (text.length <= ANALYST_HEADLINE_MAX) return text;
return `${text.slice(0, ANALYST_HEADLINE_MAX - 1).trimEnd()}…`;
if (text.length <= max) return text;
return `${text.slice(0, max - 1).trimEnd()}…`;
}
export function analystHeadline(result) {
@@ -72,6 +92,307 @@ export function analystHeadline(result) {
);
}
/* ---------------- the analyst card in the active language ---------------- */
/**
* The count and its noun as the card shows them. The English matches
* `analystNounFor`, one form for any count as the spoken headline has it.
*/
function shownCount(count, noun) {
const values = { n: formatNumber(count) };
switch (noun) {
case 'aircraft':
return tn(count, '{n} aircraft', '{n} aircraft', values);
case 'military aircraft':
return tn(
count,
'{n} military aircraft',
'{n} military aircraft',
values,
);
case 'ships':
return tn(count, '{n} ships', '{n} ships', values);
case 'satellites':
return tn(count, '{n} satellites', '{n} satellites', values);
case 'launches':
return tn(count, '{n} launches', '{n} launches', values);
case 'earthquakes':
return tn(count, '{n} earthquakes', '{n} earthquakes', values);
case 'fires':
return tn(count, '{n} fires', '{n} fires', values);
case 'fire perimeters':
return tn(count, '{n} fire perimeters', '{n} fire perimeters', values);
case 'cyclones':
return tn(count, '{n} cyclones', '{n} cyclones', values);
case 'buses':
return tn(count, '{n} buses', '{n} buses', values);
case 'bike share':
return tn(count, '{n} bike share', '{n} bike share', values);
case 'flock cameras':
return tn(count, '{n} flock cameras', '{n} flock cameras', values);
case 'military bases':
return tn(count, '{n} military bases', '{n} military bases', values);
case 'datacenters':
return tn(count, '{n} datacenters', '{n} datacenters', values);
case 'dams':
return tn(count, '{n} dams', '{n} dams', values);
case 'air quality':
return tn(count, '{n} air quality', '{n} air quality', values);
case 'results':
return tn(count, '{n} results', '{n} results', values);
default:
return `${values.n} ${noun}`;
}
}
const shownRadius = (km) => formatQuantity(Number(km), 'km');
const FIXED_SCOPES = new Map(
[msg('anywhere in the loaded data'), msg('in view')].map((text) => [
text,
text,
]),
);
/**
* The engine's scope phrases (analystEngine, the Contacts window in
* gevActions) as the card shows them. The English is exactly the engine's;
* a phrase not listed shows as given.
*/
const SCOPE_PATTERNS = [
[
/^within (\d+(?:\.\d+)?) km of the view centre$/,
([, km]) =>
t('within {radius} of the view centre', { radius: shownRadius(km) }),
],
[
/^within (\d+(?:\.\d+)?) km of (.+) \(Contacts window\)$/,
([, km, place]) =>
t('within {radius} of {place} (Contacts window)', {
radius: shownRadius(km),
place: localizePlaceName(place),
}),
],
[
/^within (\d+(?:\.\d+)?) km of (.+)$/,
([, km, place]) =>
t('within {radius} of {place}', {
radius: shownRadius(km),
place: localizePlaceName(place),
}),
],
[
/^within (\d+(?:\.\d+)?) km$/,
([, km]) => t('within {radius}', { radius: shownRadius(km) }),
],
[/^over (.+)$/, ([, region]) => t('over {region}', { region })],
];
function shownScope(text) {
const scope = String(text || '').trim();
if (!scope) return '';
if (FIXED_SCOPES.has(scope)) return t(FIXED_SCOPES.get(scope));
for (const [pattern, show] of SCOPE_PATTERNS) {
const match = pattern.exec(scope);
if (match) return show(match);
}
return translateKnown(scope);
}
/** The analyst headline in the active language, for the card. */
function shownAnalystHeadline(result) {
const keys = (result?.coverage?.layersQueried || []).map(
(layer) => layer.layerKey,
);
const count = Number(result?.count) || 0;
const counted = shownCount(count, analystNounFor(keys));
const items =
result?.complete === false
? t('At least {items}', { items: counted })
: counted;
const scope = shownScope(result?.scopeLabel);
return boundedHeadline(
scope ? t('{items} {scope}', { items, scope }) : items,
getLocale() === 'en' ? ANALYST_HEADLINE_MAX : SHOWN_HEADLINE_MAX,
);
}
/** A layer key as the card names it. */
function shownLayerKey(key) {
const text = String(key || '').trim();
return layerNameForId(text) || text;
}
const shownLayerKeys = (list) =>
String(list || '')
.split(/,\s*/)
.map(shownLayerKey)
.filter(Boolean)
.join(', ');
/** The feed windows layers declare in the voice manifest. */
const FIXED_WINDOWS = new Map(
[msg('last 24 h'), msg('past 30 d and upcoming')].map((text) => [text, text]),
);
function shownWindow(text) {
return String(text || '')
.split(/;\s*/)
.map((part) => part.trim())
.filter(Boolean)
.map((part) =>
FIXED_WINDOWS.has(part)
? t(FIXED_WINDOWS.get(part))
: translateKnown(part),
)
.join('; ');
}
const UTC_CLOCK = Object.freeze({
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
hourCycle: 'h23',
timeZone: 'UTC',
});
const UTC_DATE = Object.freeze({
day: '2-digit',
month: '2-digit',
year: 'numeric',
timeZone: 'UTC',
});
/**
* The note's ISO snapshot time as the card shows it: "14:32:05 UTC", dated
* ("15.10.2026, 14:32:05 UTC") when not from today (UTC). English and
* non-time values show as given.
*/
export function shownSnapshotTime(iso, now = Date.now()) {
const time = Date.parse(iso);
if (getLocale() === 'en' || !Number.isFinite(time)) return iso;
const clock = `${formatDate(time, UTC_CLOCK)} UTC`;
const today = new Date(now).toISOString().slice(0, 10);
return new Date(time).toISOString().slice(0, 10) === today
? clock
: `${formatDate(time, UTC_DATE)}, ${clock}`;
}
/** Coverage notes and caveats the engine and the action runner write. */
const FIXED_NOTES = new Map(
[
msg('the flights layer loads by viewport'),
msg('distance is ground distance, not overhead'),
msg(
'the count is a lower bound because at least one layer reached its retained limit.',
),
].map((text) => [text, text]),
);
const LAYER_STATUS_NOTES = Object.freeze({
off: (label) => t('{label} off', { label }),
loading: (label) => t('{label} loading', { label }),
error: (label) => t('{label} error', { label }),
});
const NOTE_PATTERNS = [
[
/^counted the first ([\d,]+) loaded records$/,
([, returned]) =>
t('counted the first {returned} loaded records', {
returned: formatNumber(Number(returned.replace(/,/g, ''))),
}),
],
[
/^counted ([\d,]+) of ([\d,]+) loaded records$/,
([, returned, total]) =>
t('counted {returned} of {total} loaded records', {
returned: formatNumber(Number(returned.replace(/,/g, ''))),
total: formatNumber(Number(total.replace(/,/g, ''))),
}),
],
[
/^(.+) just turned on — still loading, counts will rise$/,
([, layers]) =>
t('{layers} just turned on — still loading, counts will rise', {
layers: shownLayerKeys(layers),
}),
],
[
// The note ends in a period unless a lower-bound clause follows it.
/^Contacts panel snapshot(?: at (\S+))? — the same retained cohort supplies the card, speech, and follow-up memory(\.?)$/,
([, time, period]) =>
(time
? t(
'Contacts panel snapshot at {time} — the same retained cohort supplies the card, speech, and follow-up memory',
{ time: shownSnapshotTime(time) },
)
: t(
'Contacts panel snapshot — the same retained cohort supplies the card, speech, and follow-up memory',
)) + period,
],
// Transit's empty note when no feed covers the view (transit/queries.js);
// the model gets the English.
[
/^Fly below ([\d,]+) km to a covered region$/,
([, km]) =>
t('Fly below {km} km to a covered region', {
km: formatNumber(Number(km.replace(/,/g, ''))),
}),
],
[
/^No feed here yet · ([\d,]+) regions available$/,
([, regions]) => {
const count = Number(regions.replace(/,/g, ''));
return tn(
count,
'No feed here yet · {n} region available',
'No feed here yet · {n} regions available',
{ n: formatNumber(count) },
);
},
],
[
/^([a-z0-9-]+) (off|loading|error)$/,
([, key, status]) =>
layerNameForId(key)
? LAYER_STATUS_NOTES[status](shownLayerKey(key))
: null,
],
[
/^([a-z0-9-]+): (.+)$/,
([, key, note]) =>
layerNameForId(key) ? `${shownLayerKey(key)}: ${shownNote(note)}` : null,
],
];
function shownNote(text) {
if (FIXED_NOTES.has(text)) return t(FIXED_NOTES.get(text));
for (const [pattern, show] of NOTE_PATTERNS) {
const match = pattern.exec(text);
const shown = match ? show(match) : null;
if (shown) return shown;
}
// A layer's empty note can be its status line, English pieces joined by
// " · " ("feeds 1090, 978 stale · 0 heard", "TTC 0 (95s old)").
const known = translateKnown(text);
return known !== text ? known : translateStatusLine(text);
}
/** Feed states as the card's chips show them; the values stay English. */
const FEED_STATE_LABELS = Object.freeze({
loading: msg('loading'),
degraded: msg('degraded'),
stale: msg('stale'),
partial: msg('partial'),
fallback: msg('fallback'),
unavailable: msg('unavailable'),
});
function shownFeedState(state) {
const key = FEED_STATE_LABELS[state];
return key ? t(key) : state;
}
function analystCard(result) {
if (!result?.ok) return null;
const display = result.display || {};
@@ -91,28 +412,32 @@ function analystCard(result) {
].map((label) => ({ label }));
return {
display: {
title: analystHeadline(result),
title: shownAnalystHeadline(result),
// The precise scope, unless the title already says exactly that.
lines: [
display.scope !== result.scopeLabel ? display.scope : null,
display.window,
display.scope !== result.scopeLabel ? shownScope(display.scope) : null,
shownWindow(display.window),
].filter(Boolean),
chips: [
...(result.feedState && result.feedState !== 'nominal'
? [{ label: result.feedState }]
? [{ label: shownFeedState(result.feedState) }]
: []),
...(result.partial ? [{ label: 'partial' }] : []),
...(result.partial ? [{ label: t('partial') }] : []),
],
// Coverage qualifications lead the feed detail. The bounded card groups
// overflow in its existing Notes disclosure rather than dropping them.
preserveNotes: true,
notes: [
...notesOf(result.coverage?.note, display.caveat),
...notesOf(result.coverage?.note, display.caveat).map(shownNote),
...(unanswered.length
? [`Not answered: ${unanswered.join(', ')}`]
? [
t('Not answered: {layers}', {
layers: unanswered.map(shownLayerKey).join(', '),
}),
]
: []),
],
sources,
sources: sources.map(({ label }) => ({ label: sourceLabel(label) })),
},
referents: Array.isArray(result.items) ? result.items : [],
resultSet: true,
+9 -3
View File
@@ -147,8 +147,9 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
},
async start(options) {
if (disposed || !['idle', 'error'].includes(state)) return;
emit({ type: 'state', state: 'connecting' });
const attempt = ++startAttempt;
emit({ type: 'state', state: 'connecting' });
if (disposed || attempt !== startAttempt) return;
try {
await adapter.start(options);
} catch (error) {
@@ -164,12 +165,17 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
stop(options = {}) {
if (options.removeUi) return dispose();
if (disposed) return;
startAttempt++;
const attempt = ++startAttempt;
cancelActions();
try {
adapter.stop(options);
} finally {
if (!options.preserveStatus)
if (
!options.preserveStatus &&
!disposed &&
attempt === startAttempt &&
state !== 'idle'
)
emit({ type: 'state', state: 'idle', detail: 'Voice off' });
}
},
+196 -1
View File
@@ -6,8 +6,10 @@ import {
voiceCardView,
} from './voiceCardPresentation.js';
import { createVoiceSession } from './session.js';
import { createRealtimeSession } from './realtimeSession.js';
import { realtimeSessionEvent } from './realtimeEvents.js';
import { createVoiceCommands } from './sessionCommands.js';
import { describeUnsavedWork } from '../i18n/index.js';
function deferred() {
let resolve;
@@ -300,17 +302,24 @@ test('common button and annotation bindings work with an alternate adapter and c
assert.equal(ui.helpDetail.textContent, 'Activate to toggle voice');
assert.equal(ui.status.textContent, 'LISTENING');
assert.equal(ui.detail.textContent, 'Ready');
// Switching language (a reload) asks first during a session.
assert.deepEqual(describeUnsavedWork(), ['the active voice session']);
subscribed({ status: 'resolved' });
assert.deepEqual(events, [
{ type: 'map_annotation_outline', status: 'resolved' },
]);
button.dispatchEvent(new Event('click'));
assert.equal(controls.state, 'idle');
assert.deepEqual(describeUnsavedWork(), []);
button.dispatchEvent(new Event('click'));
await new Promise((done) => setTimeout(done, 0));
assert.deepEqual(describeUnsavedWork(), ['the active voice session']);
lifetime.abort();
assert.deepEqual(describeUnsavedWork(), [], 'disposal unregisters');
assert.equal(subscribed, null);
assert.equal(removed, 1);
button.dispatchEvent(new Event('click'));
assert.equal(calls, 1);
assert.equal(calls, 2);
} finally {
globalThis.window = previous;
}
@@ -433,3 +442,189 @@ test('a late rejection cannot settle a newer turn card', async () => {
);
f.session.destroy();
});
function startReentrancyFixture() {
let starts = 0;
let stops = 0;
const settings = [];
const session = createVoiceSession({
runner: async () => ({ ok: true }),
createAdapter: ({ emit }) => ({
async start(options) {
starts++;
settings.push(options);
emit({ type: 'state', state: 'listening' });
},
stop() {
stops++;
},
sendText() {},
sendMapEvent() {},
}),
});
return { session, settings, starts: () => starts, stops: () => stops };
}
for (const action of ['stop', 'destroy']) {
test(`initial connecting observer ${action} prevents adapter acquisition`, async () => {
const f = startReentrancyFixture();
const unsubscribe = f.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'connecting') {
unsubscribe();
f.session[action]();
}
});
await f.session.start();
assert.equal(
f.starts(),
0,
'cancelled attempt must not call adapter.start',
);
assert.equal(f.stops(), 1);
assert.equal(f.session.state, 'idle');
if (action === 'stop') {
await f.session.start();
assert.equal(f.starts(), 1, 'later explicit Start remains usable');
assert.equal(f.session.state, 'listening');
}
f.session.destroy();
});
}
test('preserveStatus Stop still prevents adapter acquisition', async () => {
const f = startReentrancyFixture();
const unsubscribe = f.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'connecting') {
unsubscribe();
f.session.stop({ preserveStatus: true });
}
});
await f.session.start();
assert.equal(f.starts(), 0);
assert.equal(
f.session.state,
'connecting',
'preserved state is not ownership',
);
f.session.stop();
await f.session.start();
assert.equal(f.starts(), 1);
f.session.destroy();
});
test('Stop and explicit reentrant successor start acquire only for the successor', async () => {
const f = startReentrancyFixture();
let successor;
const unsubscribe = f.session.subscribe((event) => {
if (event.type === 'state' && event.state === 'connecting') {
unsubscribe();
f.session.stop();
successor = f.session.start();
}
});
await f.session.start();
await successor;
assert.equal(f.starts(), 1);
assert.equal(f.session.state, 'listening');
f.session.destroy();
});
test('ordinary start forwards caller settings once and remains stoppable', async () => {
const f = startReentrancyFixture();
const settings = { pushToTalk: true };
await f.session.start(settings);
assert.equal(f.settings.length, 1);
assert.equal(f.settings[0], settings);
assert.equal(f.starts(), 1);
assert.equal(f.session.state, 'listening');
f.session.stop();
assert.equal(f.session.state, 'idle');
f.session.destroy();
});
for (const emitsIdle of [false, true]) {
for (const outcome of ['normal', 'preserveStatus', 'throw']) {
test(`stop preserves custom adapter contract: emitsIdle=${emitsIdle}, ${outcome}`, async (t) => {
let adapter;
const calls = [];
const events = [];
const failure = new Error('Controlled stop failure');
const session = createVoiceSession({
runner: async () => ({}),
createAdapter: ({ emit }) =>
(adapter = {
async start() {
emit({ type: 'state', state: 'listening' });
},
stop(settings = {}) {
calls.push({ receiver: this, settings });
if (settings.removeUi) return;
if (emitsIdle && !settings.preserveStatus)
emit({ type: 'state', state: 'idle' });
if (outcome === 'throw') throw failure;
return 'adapter-only return';
},
sendText() {},
sendMapEvent() {},
}),
});
t.after(() => session.destroy());
session.subscribe((event) => events.push(event));
await session.start();
events.length = 0;
const settings =
outcome === 'preserveStatus' ? { preserveStatus: true } : {};
if (outcome === 'throw')
assert.throws(
() => session.stop(settings),
(error) => error === failure,
);
else assert.equal(session.stop(settings), undefined);
assert.equal(calls[0].receiver, adapter);
assert.equal(calls[0].settings, settings);
assert.equal(
session.state,
outcome === 'preserveStatus' ? 'listening' : 'idle',
);
assert.equal(
events.filter(
(event) => event.type === 'state' && event.state === 'idle',
).length,
outcome === 'preserveStatus' ? 0 : 1,
);
});
}
}
test('actual default OpenAI adapter stop publishes one common idle', (t) => {
const events = [];
const ui = {
root: {
dataset: {},
querySelectorAll: () => [],
classList: { remove() {} },
remove() {},
},
status: {},
detail: {},
};
const runner = async () => ({});
const session = createVoiceSession({
runner,
createAdapter: (hooks) =>
createRealtimeSession({ ...hooks, runner, ui, debugSink() {} }),
});
t.after(() => session.destroy());
session.subscribe((event) => events.push(event));
// Exercise the real stop path without acquiring microphone/WebRTC resources.
session.adapter.controller.setStatus('listening', 'Controlled active state');
events.length = 0;
session.stop();
assert.equal(
events.filter((event) => event.type === 'state' && event.state === 'idle')
.length,
1,
);
assert.equal(session.state, 'idle');
assert.equal(session.adapter.controller.status, 'idle');
});
+115 -35
View File
@@ -1,7 +1,9 @@
import { createVoiceControl } from './control.js';
import { createVoiceControl, voiceStatusLabel } from './control.js';
import { createVoiceSession } from './session.js';
import { VoiceCardControls } from './voiceCard.js';
import { PointerReticle } from './pointerReticle.js';
import { VOICE_PROVIDERS } from './providerCapabilities.js';
import { msg, registerUnsavedWork, t } from '../i18n/index.js';
function createDefaultReticle(pointer) {
const element = pointer?.element;
@@ -24,19 +26,34 @@ export function createVoiceCommands({
signal,
debugSink,
createControl = createVoiceControl,
resetExisting = true,
disposeRunner = true,
provider,
supportedProviders,
onProviderChange,
onSessionEvent,
createCard = (options) => new VoiceCardControls(options),
pointer = null,
referents = null,
createReticle = createDefaultReticle,
panelHost = null,
nativePanelProvider = null,
}) {
window.__gevVoiceCommands?.stop?.({ removeUi: true });
const ui = createControl({ reset: true });
if (resetExisting) window.__gevVoiceCommands?.stop?.({ removeUi: true });
const ui = createControl({
reset: true,
...(panelHost ? { panel: true } : {}),
});
let actionObserver;
let panelMount;
const session = createVoiceSession({
runner,
signal,
createAdapter: (hooks) =>
createSession({
createAdapter: (hooks) => {
if (panelHost) actionObserver = panelHost.observeActions(hooks);
return createSession({
...hooks,
...(actionObserver ? { runAction: actionObserver.runAction } : {}),
runner,
ui,
dataManager,
@@ -45,41 +62,96 @@ export function createVoiceCommands({
createController,
pointer,
radioLayer: dataManager?.layers?.get('radio')?.module || null,
}),
});
},
});
const adapter = session.adapter;
const capabilities = adapter.capabilities || {};
const providerHandler = () => onProviderChange?.(ui.providerSelect.value);
if (ui.providerSelect && onProviderChange) {
if (supportedProviders && ui.providerSelect.replaceChildren) {
const doc = ui.providerSelect.ownerDocument;
ui.providerSelect.replaceChildren(
...supportedProviders.map((name) => {
const option = doc.createElement('option');
option.value = name;
option.textContent = VOICE_PROVIDERS[name].label;
return option;
}),
);
}
ui.providerSelect.value = provider;
ui.providerSelect.title = [
t('Changing provider reconnects voice if it is on.'),
capabilities.advisoryLimit,
]
.filter(Boolean)
.join(' ');
ui.providerSelect.addEventListener('change', providerHandler);
ui.root.dataset.provider = provider;
}
const hasProviderChoice = Boolean(
onProviderChange && supportedProviders?.length > 1,
);
if (ui.providerField) ui.providerField.hidden = !hasProviderChoice;
ui.root.dataset.providerChoice = String(hasProviderChoice);
if (ui.tierButton) ui.tierButton.hidden = !capabilities.costControls;
if (ui.costValue) ui.costValue.hidden = !capabilities.costControls;
if (!capabilities.pushToTalk) {
ui.button.setAttribute('aria-label', 'Toggle voice control');
if (ui.helpDetail) ui.helpDetail.textContent = 'Activate to toggle voice';
ui.button.setAttribute('aria-label', t('Toggle voice control'));
if (ui.helpDetail)
ui.helpDetail.textContent = t('Activate to toggle voice');
}
// Retain the existing controller's inspection surface for browser tools.
const controls = adapter.controller || session;
controls.session = session;
controls.ui = ui;
const updateStatus = session.subscribe((event) => {
if (event.type !== 'state') return;
ui.root.dataset.status = event.state;
ui.status.textContent =
event.state === 'idle' ? 'OFF' : event.state.toUpperCase();
// States are machine values; only the label is translated.
ui.status.textContent = voiceStatusLabel(event.state);
// The portable session reports 'Voice off' in English: show the idle
// label instead. Other details are provider messages, shown as they come.
const idle = event.state === 'idle';
const detail = event.detail === 'Voice off' ? '' : event.detail;
ui.detail.textContent =
event.detail || (event.state === 'idle' ? 'Voice off' : 'Voice active');
detail || (idle ? t('Voice off') : t('Voice active'));
ui.button.setAttribute('aria-pressed', String(session.isActive()));
if (ui.errorDetail)
ui.errorDetail.textContent =
event.state === 'error'
? event.detail || 'Voice could not be started.'
? event.detail || t('Voice could not be started.')
: '';
if (event.state === 'error') ui.root.classList?.remove('error-dismissed');
});
// Captions, plan and result card nested in the control; it only reads
// provider-neutral session events.
const card = ui.card ? createCard({ root: ui.root }) : null;
const updateCard = card
? session.subscribe((event) => card.handle(event))
const card = ui.card
? createCard({
root: ui.root,
...(panelHost ? { onRender: () => panelMount?.panel.sync() } : {}),
})
: null;
panelMount = panelHost?.mount({
ui,
card,
session,
observer: actionObserver,
nativeProvider: nativePanelProvider,
});
const updatePanel = panelMount ? session.subscribe(panelMount.handle) : null;
const updateCard =
card && !panelMount
? session.subscribe((event) => card.handle(event))
: null;
controls.voiceCard = card;
if (panelMount) {
controls.voicePanel = panelMount.panel;
controls.captureVoicePanelOperation = panelMount.captureOperation;
controls.getVoicePanelState = panelMount.snapshot;
controls.getVoicePreferences = panelHost.snapshot;
}
// Point-and-ask: the reticle follows turn events; pointer and referents are
// session-scoped, so a session that ends forgets both.
const reticle = pointer ? createReticle(pointer) : null;
@@ -95,42 +167,50 @@ export function createVoiceCommands({
});
controls.pointer = pointer;
controls.referents = referents;
// Forward last: a host observer may synchronously replace or stop this mount.
const forwardEvents = onSessionEvent
? session.subscribe(onSessionEvent)
: null;
const annotationUnsubscribe = annotations?.onOutlineEvent?.((event) => {
card?.outline(event);
session.sendMapEvent({ type: 'map_annotation_outline', ...event });
});
// A reload (the language switch) ends the conversation: ask first.
const unregisterUnsavedWork = registerUnsavedWork(() =>
session.isActive() ? msg('the active voice session') : null,
);
const buttonHandler = () => {
if (adapter.ignoreButtonClick?.()) return;
if (session.isActive()) session.stop();
else void session.start({ pushToTalk: false });
if (session.isActive()) {
const ticket = panelMount?.stopTicket();
session.stop();
panelMount?.closeAfterStop(ticket);
} else void session.start({ pushToTalk: false });
};
ui.button.addEventListener('click', buttonHandler);
session.signal.addEventListener(
'abort',
() => {
ui.button.removeEventListener('click', buttonHandler);
annotationUnsubscribe?.();
updateStatus();
updateCard?.();
updatePointer();
reticle?.destroy();
card?.destroy();
runner.dispose?.();
ui.root.remove();
},
{ once: true },
);
if (session.disposed) {
let cleaned = false;
const cleanup = () => {
if (cleaned) return;
cleaned = true;
ui.button.removeEventListener('click', buttonHandler);
ui.providerSelect?.removeEventListener('change', providerHandler);
unregisterUnsavedWork();
annotationUnsubscribe?.();
updateStatus();
updateCard?.();
updatePanel?.();
panelMount?.destroy();
actionObserver?.dispose();
updatePointer();
forwardEvents?.();
reticle?.destroy();
card?.destroy();
runner.dispose?.();
if (disposeRunner) runner.dispose?.();
ui.root.remove();
} else adapter.bindControls?.();
};
session.signal.addEventListener('abort', cleanup, { once: true });
if (session.disposed) cleanup();
else adapter.bindControls?.();
window.__gevVoiceCommands = controls;
return controls;
}
+482 -63
View File
@@ -16,9 +16,29 @@
*
* Builders are small pure functions keyed by tool name. Tools without one are
* returned unchanged.
*
* Only `display` and plan/progress labels are UI text (t()). `say`,
* `identityLine`, spoken progress lines and the tool result stay English for
* the model; the card shows them through the shown* helpers.
*/
import { analystHeadline } from './resultDisplay.js';
import {
formatNumber,
getLocale,
msg,
t,
tn,
translateKnown,
} from '../i18n/index.js';
import {
layerNameForId,
shownLayerName,
sourceLabel,
} from '../ui/layerNames.js';
import { localizePlaceName } from '../ui/placeNames.js';
import { CITY_POIS } from '../cityPois.js';
import { formatQuantity } from '../ui/unitLabels.js';
const LAYER_NOUNS = Object.freeze({
flights: 'aircraft',
@@ -53,17 +73,180 @@ function listPhrase(items) {
return `${list.slice(0, -1).join(', ')} and ${list.at(-1)}`;
}
function feet(meters) {
function feetRounded(meters) {
const value = Number(meters);
if (!Number.isFinite(value)) return null;
const rounded = Math.round((value * 3.28084) / 100) * 100;
return Number.isFinite(value)
? Math.round((value * 3.28084) / 100) * 100
: null;
}
function feet(meters) {
const rounded = feetRounded(meters);
if (rounded === null) return null;
return `${rounded.toLocaleString('en-US')} feet`;
}
function kilometers(km) {
/** The card's altitude, in the active language. */
function shownFeet(meters) {
const rounded = feetRounded(meters);
if (rounded === null) return null;
return t('{value} feet', { value: formatNumber(rounded) });
}
function roundedKm(km) {
const value = Number(km);
if (!Number.isFinite(value)) return null;
return `${value < 10 ? Number(value.toFixed(1)) : Math.round(value)} km`;
return value < 10 ? Number(value.toFixed(1)) : Math.round(value);
}
function kilometers(km) {
const value = roundedKm(km);
return value === null ? null : `${value} km`;
}
/** The card's distance, in the active language: "2,5 км". */
function shownKm(km) {
const value = roundedKm(km);
return value === null ? null : formatQuantity(value, 'km');
}
/** Sources as the card lists them. */
function shownSources(source) {
return source ? [{ label: sourceLabel(source) }] : [];
}
// Entity labels that layers register as English data (firms/, alpr/); keep in
// step with the producers. Other names (callsigns, places) show as they came.
const ENTITY_LABELS = new Set([msg('ALPR camera')]);
const FIRE_LABEL = /^Fire · FRP (\S+) MW$/;
/** An entity label as the card shows it, in the active language. */
export function shownEntityLabel(label) {
if (ENTITY_LABELS.has(label)) return t(label);
const fire = FIRE_LABEL.exec(label ?? '');
return fire ? t('Fire · FRP {frp} MW', { frp: fire[1] }) : label;
}
/*
* Tool errors stay English in the result (the model reads them). The card
* translates the known ones; the English must match gevActions exactly.
*/
const KNOWN_ERRORS = new Map(
[
msg('The flights layer is not enabled'),
msg('The military layer is not enabled'),
msg('The satellites layer is not enabled'),
msg('The ais-live-vessels layer is not enabled'),
msg('Could not enable the requested layer'),
msg('Could not disable the requested layer'),
msg('Radio layer unavailable'),
msg('Camera navigation is unavailable in the current view'),
msg('Camera navigation policy unavailable'),
msg('Nearest-aircraft selection supports Flights or Military Flights only'),
msg(
'Nearest-aircraft selection needs a preset, place name, or latitude and longitude',
),
msg('Could not arrive at the requested place'),
msg('The nearest airborne aircraft could not be selected'),
].map((text) => [text, text]),
);
function noneWithinView(target, radius) {
switch (target) {
case 'flights':
return t('No flights within {radius} of the current view', { radius });
case 'military':
return t('No military within {radius} of the current view', { radius });
case 'satellites':
return t('No satellites within {radius} of the current view', {
radius,
});
case 'vessels':
return t('No vessels within {radius} of the current view', { radius });
default:
return null;
}
}
const ERROR_PATTERNS = [
[
/^(.+) is enabled on the (.+) feed, but no airborne aircraft is loaded in the (.+) view yet$/,
([, layer, source, place]) =>
t(
'{layer} is enabled on the {source} feed, but no airborne aircraft is loaded in the {place} view yet',
{
layer: shownLayerName(layer),
source: sourceLabel(source),
place: localizePlaceName(place),
},
),
],
[
/^(.+) is enabled, but no airborne aircraft is loaded in the (.+) view yet$/,
([, layer, place]) =>
t(
'{layer} is enabled, but no airborne aircraft is loaded in the {place} view yet',
{ layer: shownLayerName(layer), place: localizePlaceName(place) },
),
],
[
/^(.+) is enabled, but its aircraft feed is unavailable$/,
([, layer]) =>
t('{layer} is enabled, but its aircraft feed is unavailable', {
layer: shownLayerName(layer),
}),
],
[
/^(.+) is enabled, but its destination refresh did not complete$/,
([, layer]) =>
t('{layer} is enabled, but its destination refresh did not complete', {
layer: shownLayerName(layer),
}),
],
[
/^(.+) is enabled, but (.+) is unavailable$/,
([, layer, source]) =>
t('{layer} is enabled, but {source} is unavailable', {
layer: shownLayerName(layer),
source: sourceLabel(source),
}),
],
[
/^Could not arrive at (.+)$/,
([, place]) =>
t('Could not arrive at {place}', { place: localizePlaceName(place) }),
],
[
/^(.+) is no longer loaded$/,
([, name]) => t('{name} is no longer loaded', { name }),
],
[
/^Unknown target layer: (.*)$/,
([, target]) => t('Unknown target layer: {target}', { target }),
],
[
/^No (\S+) within (\d+(?:\.\d+)?) km of the current view$/,
([, target, km]) => noneWithinView(target, shownKm(km)),
],
];
/** A tool error as the card shows it. */
function shownError(error) {
if (typeof error !== 'string' || !error) return null;
const known = KNOWN_ERRORS.get(error);
if (known) return t(known);
for (const [pattern, show] of ERROR_PATTERNS) {
const match = pattern.exec(error);
const shown = match ? show(match) : null;
if (shown) return shown;
}
return translateKnown(error);
}
// The card wraps a failed result's lines rather than cutting them: a
// translated error runs longer than the English (voiceCardPresentation).
function errorLines(result) {
return [shownError(result?.error)].filter(Boolean);
}
function sentence(text) {
@@ -76,9 +259,110 @@ function feedTag(state) {
return MATERIAL_FEED_TAGS[state] || null;
}
/** Feed states as the card shows them; the state values stay English. */
const FEED_STATE_LABELS = Object.freeze({
nominal: msg('nominal'),
loading: msg('loading'),
degraded: msg('degraded'),
stale: msg('stale'),
partial: msg('partial'),
fallback: msg('fallback'),
unavailable: msg('unavailable'),
});
function feedStateLabel(state) {
const key = FEED_STATE_LABELS[state];
return key ? t(key) : state;
}
/** A feed chip only when the state is worth seeing; nominal is the default. */
function feedChips(state) {
return state && state !== 'nominal' ? [{ label: state }] : [];
return state && state !== 'nominal' ? [{ label: feedStateLabel(state) }] : [];
}
/** '{label} on' / '{label} off' as the card shows it. */
function switchedTitle(label, on) {
return on ? t('{label} on', { label }) : t('{label} off', { label });
}
// Card titles name the layer's things in whole phrases, so each language can
// inflect the noun; the English matches LAYER_NOUNS.
function frameTitle(layerId) {
switch (layerId) {
case 'flights':
return t('Frame aircraft');
case 'military':
return t('Frame military aircraft');
case 'ais-live-vessels':
return t('Frame ships');
case 'satellites':
return t('Frame satellites');
default:
return t('Frame contacts');
}
}
function inFrameTitle(layerId, count) {
switch (layerId) {
case 'flights':
return tn(
count,
'{count} aircraft in frame',
'{count} aircraft in frame',
);
case 'military':
return tn(
count,
'{count} military aircraft in frame',
'{count} military aircraft in frame',
);
case 'ais-live-vessels':
return tn(count, '{count} ships in frame', '{count} ships in frame');
case 'satellites':
return tn(
count,
'{count} satellites in frame',
'{count} satellites in frame',
);
default:
return tn(
count,
'{count} contacts in frame',
'{count} contacts in frame',
);
}
}
function nearestTitle(layerId) {
switch (layerId) {
case 'military':
return t('Nearest military aircraft');
case 'ais-live-vessels':
return t('Nearest ships');
case 'satellites':
return t('Nearest satellites');
default:
return t('Nearest aircraft');
}
}
/** Two shown items joined, or the one item. */
function shownPair(items) {
return items.length === 2
? t('{first} and {second}', { first: items[0], second: items[1] })
: items[0] || '';
}
/** A preset city id (`sf`, `nyc`) as its English name; any other value unchanged. */
function presetPlaceName(value) {
const id = typeof value === 'string' ? value.trim() : '';
return Object.hasOwn(CITY_POIS, id) ? CITY_POIS[id].name : value;
}
// A place as the card shows it: a preset city in the interface language,
// whether the call named it ("Tokyo") or gave its id ("tokyo", "nyc").
function shownPlace(value, max) {
return localizePlaceName(displayName(presetPlaceName(value), max));
}
/** Ids such as `austin` or `local-firms` read as names on screen. */
@@ -96,6 +380,12 @@ function layerLabel(result) {
return spokenLabel(result?.label || result?.layerId || 'Layer');
}
/** The layer as the card names it, in the active language. */
function shownLayerLabel(result) {
const name = shownLayerName(result?.label, result?.layerId);
return spokenLabel(name || t('Layer'));
}
/* ---------------- per-tool builders ---------------- */
function setLayerVisibility(result, args = {}) {
@@ -105,7 +395,7 @@ function setLayerVisibility(result, args = {}) {
const wanted = args.enabled === false ? 'off' : 'on';
return {
say: `Couldn't turn ${wanted} ${label}.`,
display: { title: label, lines: [result?.error].filter(Boolean) },
display: { title: shownLayerLabel(result), lines: errorLines(result) },
};
}
const on = result.enabled !== false;
@@ -117,28 +407,35 @@ function setLayerVisibility(result, args = {}) {
return {
say,
display: {
title: `${label} ${on ? 'on' : 'off'}`,
title: switchedTitle(shownLayerLabel(result), on),
chips: on ? feedChips(result.feedState) : [],
sources: result.source ? [{ label: result.source }] : [],
sources: shownSources(result.source),
},
};
}
function flyToLocation(result) {
const label = spokenLabel(result?.label || result?.query || 'there');
const title =
spokenLabel(localizePlaceName(result?.label || result?.query)) ||
t('there');
if (result?.cancelled) return null;
if (!result?.ok) {
return {
say: sentence(`Couldn't find ${label}`),
display: { title: label, lines: [result?.error].filter(Boolean) },
display: { title, lines: errorLines(result) },
};
}
return {
say: sentence(result.arrived ? `Over ${label}` : `Flying to ${label}`),
display: {
title: label,
title,
lines: result.rangeM
? [`Camera range ${kilometers(result.rangeM / 1000)}`]
? [
t('Camera range {range}', {
range: shownKm(result.rangeM / 1000),
}),
]
: [],
},
};
@@ -162,8 +459,8 @@ function frameOverhead(result) {
return {
say,
display: {
title: `Frame ${noun}`,
lines: [result?.error].filter(Boolean),
title: frameTitle(result?.layerId),
lines: errorLines(result),
},
};
}
@@ -181,12 +478,15 @@ function frameOverhead(result) {
return {
say: sentence(`Framed ${countText} ${noun} within ${radius}`),
display: {
title: `${count} ${noun} in frame`,
title: inFrameTitle(result.layerId, count),
lines: [
`Within ${radius} of the view`,
result.detectionEnabled ? 'Labels on' : null,
t('Within {radius} of the view', {
radius: shownKm(result?.radiusKm),
}),
result.detectionEnabled ? t('Labels on') : null,
].filter(Boolean),
notes: count >= 80 ? ['Framing uses the nearest 80 loaded contacts'] : [],
notes:
count >= 80 ? [t('Framing uses the nearest 80 loaded contacts')] : [],
},
referents,
};
@@ -197,8 +497,12 @@ function selectNearestAircraft(result) {
result?.location?.label || result?.location || 'there',
);
const noun = LAYER_NOUNS[result?.layerId] || 'aircraft';
const shownLocation =
spokenLabel(
localizePlaceName(result?.location?.label || result?.location),
) || t('there');
const feed = result?.feed || {};
const sources = feed.source ? [{ label: feed.source }] : [];
const sources = shownSources(feed.source);
if (result?.cancelled) return null;
if (!result?.ok) {
let say;
@@ -212,8 +516,8 @@ function selectNearestAircraft(result) {
return {
say,
display: {
title: `Nearest ${noun}`,
lines: [result?.error].filter(Boolean),
title: nearestTitle(result?.layerId),
lines: errorLines(result),
chips: feedChips(feed.state),
sources,
},
@@ -239,8 +543,17 @@ function selectNearestAircraft(result) {
display: {
title: name,
lines: [
distance ? `${distance} from ${location}` : null,
altitude ? `Altitude ${altitude}` : null,
distance
? t('{distance} from {location}', {
distance: shownKm(aircraft.distanceKm),
location: shownLocation,
})
: null,
altitude
? t('Altitude {altitude}', {
altitude: shownFeet(aircraft.altitudeM),
})
: null,
].filter(Boolean),
chips: feedChips(feed.state),
sources,
@@ -251,14 +564,30 @@ function selectNearestAircraft(result) {
};
}
function routeText(properties) {
const SPOKEN_ROUTE = Object.freeze({
between: (origin, destination) => `${origin} to ${destination}`,
from: (origin) => `from ${origin}`,
to: (destination) => `to ${destination}`,
});
const SHOWN_ROUTE = Object.freeze({
between: (origin, destination) =>
t('{origin} to {destination}', { origin, destination }),
from: (origin) => t('from {origin}', { origin }),
to: (destination) => t('to {destination}', { destination }),
});
function routeText(properties, words = SPOKEN_ROUTE) {
const route = spokenLabel(properties.route || '', 40);
if (route) return route.replace(/\s*(?:-|→|>)\s*/g, ' to ');
if (route)
return route
.split(/\s*(?:-|→|>)\s*/)
.reduce((origin, destination) => words.between(origin, destination));
const origin = spokenLabel(properties.routeOrigin || '', 12);
const destination = spokenLabel(properties.routeDestination || '', 12);
if (origin && destination) return `${origin} to ${destination}`;
if (origin) return `from ${origin}`;
if (destination) return `to ${destination}`;
if (origin && destination) return words.between(origin, destination);
if (origin) return words.from(origin);
if (destination) return words.to(destination);
return null;
}
@@ -275,9 +604,13 @@ function getEntityContext(result) {
return {
say: null,
display: {
title,
lines: [selected.layerName].filter(Boolean),
sources: selected.source ? [{ label: selected.source }] : [],
title: spokenLabel(shownEntityLabel(title)),
lines: [
selected.layerName
? shownLayerName(selected.layerName, selected.layerId)
: null,
].filter(Boolean),
sources: shownSources(selected.source),
},
};
}
@@ -301,11 +634,15 @@ function getEntityContext(result) {
title,
lines: [
properties.registration
? `Registration ${spokenLabel(properties.registration, 16)}`
? t('Registration {registration}', {
registration: spokenLabel(properties.registration, 16),
})
: null,
route
? t('Route {route}', { route: routeText(properties, SHOWN_ROUTE) })
: null,
route ? `Route ${route}` : null,
].filter(Boolean),
sources: selected.source ? [{ label: selected.source }] : [],
sources: shownSources(selected.source),
},
referents: [
{
@@ -350,10 +687,10 @@ function getCurrentViewState(result) {
.map((part) => sentence(part[0].toUpperCase() + part.slice(1)))
.join(' '),
display: {
title: 'View state',
title: t('View state'),
lines: enabled.map(
(layer) =>
`${spokenLabel(layer.name || layer.id, 32)} · ${layer.count ?? 0} · ${layer.feedState || 'nominal'}`,
`${spokenLabel(shownLayerName(layer.name, layer.id), 32)} · ${layer.count ?? 0} · ${feedStateLabel(layer.feedState || 'nominal')}`,
),
},
schedule: 'silent',
@@ -387,13 +724,15 @@ function annotateMap(result) {
say: parts.length ? parts.join(' ') : null,
display: {
title: result?.ok
? `Marked ${drawn.length} ${drawn.length === 1 ? 'place' : 'places'}`
: 'Nothing marked',
? tn(drawn.length, 'Marked {count} place', 'Marked {count} places')
: t('Nothing marked'),
// The numbered referents list the marks; lines stay for extra facts.
lines: [],
notes: [
...(failed.length ? [`Not found: ${failed.join(', ')}`] : []),
...(result?.outlinePending ? ['Tracing outlines'] : []),
...(failed.length
? [t('Not found: {places}', { places: failed.join(', ') })]
: []),
...(result?.outlinePending ? [t('Tracing outlines')] : []),
],
},
referents,
@@ -487,29 +826,99 @@ function annotationTargets(args) {
.filter(Boolean);
}
// frame_overhead targets as the schema names them.
function frameStepLabel(target) {
switch (target) {
case 'flights':
return t('Frame flights');
case 'military':
return t('Frame military');
case 'satellites':
return t('Frame satellites');
case 'vessels':
return t('Frame vessels');
default:
return t('Frame {target}', { target });
}
}
// Plan-step labels for the other tools in actionSchemas. A tool missing here
// shows its raw name in English only.
const STEP_TITLES = Object.freeze({
zoom_to_globe: msg('Zoom out to globe'),
show_data_layers_menu: msg('Open data layers'),
set_panel_open: msg('Open or close a panel'),
set_voice_inactivity_timeout: msg('Voice inactivity timeout'),
set_context_mode: msg('Change context mode'),
control_cockpit: msg('Control cockpit'),
set_visual_style: msg('Change visual style'),
set_hud: msg('Adjust HUD'),
set_cyber_sonar: msg('Adjust cyber sonar'),
set_detection: msg('Adjust detection overlay'),
set_map_stack: msg('Change map stack'),
set_post_processing: msg('Adjust post-processing'),
control_scene: msg('Control scene'),
control_cctv: msg('Control CCTV'),
control_radio: msg('Control radio'),
track_entity: msg('Track contact'),
stop_tracking: msg('Stop tracking'),
clear_annotations: msg('Clear marks'),
move_camera: msg('Move camera'),
fly_route: msg('Fly route'),
analyst_query: msg('Analyst query'),
next_iss_pass: msg('Next ISS pass'),
next_satellite_pass: msg('Next satellite pass'),
});
/** A short on-screen label for one tool call in the voice card's plan. */
export function planStepLabel(name, args = {}) {
if (Object.hasOwn(STEP_TITLES, name)) return t(STEP_TITLES[name]);
switch (name) {
case 'annotate_map': {
const targets = annotationTargets(args);
return targets.length
? `Mark ${listPhrase(targets.slice(0, 2))}${targets.length > 2 ? ` +${targets.length - 2}` : ''}`
: 'Mark the map';
if (!targets.length) return t('Mark the map');
const shown = shownPair(targets.slice(0, 2));
return targets.length > 2
? t('Mark {targets} +{more}', {
targets: shown,
more: targets.length - 2,
})
: t('Mark {targets}', { targets: shown });
}
case 'fly_to_location':
return `Fly to ${displayName(args.query || args.locationId || 'location', 40)}`;
case 'select_nearest_aircraft':
return `Nearest aircraft to ${displayName(args.locationQuery || args.locationId || 'here', 32)}`;
case 'set_layer_visibility':
return `${displayName(args.layerId || 'Layer', 32)} ${args.enabled === false ? 'off' : 'on'}`;
case 'fly_to_location': {
const place = args.query || args.locationId;
return place
? t('Fly to {place}', { place: shownPlace(place, 40) })
: t('Fly to Location');
}
case 'select_nearest_aircraft': {
const place = args.locationQuery || args.locationId;
return place
? t('Nearest aircraft to {place}', { place: shownPlace(place, 32) })
: t('Nearest aircraft to Here');
}
case 'set_layer_visibility': {
const on = args.enabled !== false;
const label =
layerNameForId(args.layerId) ||
(args.layerId ? displayName(args.layerId, 32) : '');
// "Layer" alone is a noun the languages inflect the state for.
if (!label) return on ? t('Layer on') : t('Layer off');
return switchedTitle(label, on);
}
case 'adjust_camera_zoom':
return args.direction === 'out' ? t('Zoom out') : t('Zoom in');
case 'get_entity_context':
return 'Read the view';
return t('Read the view');
case 'get_current_view_state':
return 'Check view state';
return t('Check view state');
case 'frame_overhead':
return `Frame ${spokenLabel(args.target || 'flights', 24)}`;
return frameStepLabel(spokenLabel(args.target || 'flights', 24));
default:
return spokenLabel(String(name || 'Action').replace(/_/g, ' '), 40);
// A tool with no label: its name in English, "Action" elsewhere.
return getLocale() === 'en'
? spokenLabel(String(name || 'Action').replace(/_/g, ' '), 40)
: t('Action');
}
}
@@ -522,9 +931,15 @@ export function narrationLabel(name, args = {}) {
case 'annotate_map':
return annotationTargets(args)[0] || '';
case 'fly_to_location':
return displayName(args.query || args.locationId || '', 32);
return displayName(
presetPlaceName(args.query || args.locationId || ''),
32,
);
case 'select_nearest_aircraft':
return displayName(args.locationQuery || args.locationId || '', 32);
return displayName(
presetPlaceName(args.locationQuery || args.locationId || ''),
32,
);
default:
return '';
}
@@ -532,18 +947,22 @@ export function narrationLabel(name, args = {}) {
/** On-screen labels for progress steps. */
const STEP_LABELS = Object.freeze({
resolve: 'Finding places',
outline: 'Tracing outline',
search: 'Looking up place',
fly: 'Flying',
layer: 'Turning on layer',
refresh: 'Loading aircraft',
nearest: 'Picking nearest',
resolve: msg('Finding places'),
outline: msg('Tracing outline'),
search: msg('Looking up place'),
fly: msg('Flying'),
layer: msg('Turning on layer'),
refresh: msg('Loading aircraft'),
nearest: msg('Picking nearest'),
});
/** On-screen label for a progress step. */
export function progressStepLabel(step, label) {
const base = STEP_LABELS[step] || spokenLabel(step || 'Working', 24);
const base = STEP_LABELS[step]
? t(STEP_LABELS[step])
: step
? spokenLabel(step, 24)
: t('Working');
return label ? `${base}: ${spokenLabel(label, 32)}` : base;
}
+1 -1
View File
@@ -7,7 +7,7 @@
* Phrases that add length without adding an answer. Tool numbers are spoken
* verbatim, so approximations are hedges too.
*/
export const HEDGE_PHRASES = Object.freeze([
const HEDGE_PHRASES = Object.freeze([
'keep in mind',
"it's worth noting",
'it is worth noting',
+1 -1
View File
@@ -1,7 +1,7 @@
import { countWords, findHedges } from './speechLint.js';
/** Quiet period after the last audio or response before a turn is closed. */
export const TURN_SETTLE_MS = 2500;
const TURN_SETTLE_MS = 2500;
/**
* Per-turn latency and speech spans for one voice session.
+57 -4
View File
@@ -5,7 +5,7 @@ import {
} from './voiceCardPresentation.js';
/** Hide an idle card this long after its last update. */
export const VOICE_CARD_IDLE_MS = 14000;
const VOICE_CARD_IDLE_MS = 14000;
const ELEMENT_IDS = Object.freeze({
card: 'gev-voice-card',
@@ -13,6 +13,7 @@ const ELEMENT_IDS = Object.freeze({
dismiss: 'gev-voice-card-dismiss',
user: 'gev-voice-card-user',
userText: 'gev-voice-card-user-text',
userWho: 'gev-voice-card-user-who',
pointer: 'gev-voice-card-pointer',
pointerKind: 'gev-voice-card-pointer-kind',
pointerText: 'gev-voice-card-pointer-text',
@@ -70,8 +71,9 @@ export class VoiceCardControls {
idleMs = VOICE_CARD_IDLE_MS,
setTimer = (fn, ms) => setTimeout(fn, ms),
clearTimer = (id) => clearTimeout(id),
onRender,
}) {
Object.assign(this, { idleMs, setTimer, clearTimer });
Object.assign(this, { idleMs, setTimer, clearTimer, onRender });
this.elements = Object.fromEntries(
Object.entries(ELEMENT_IDS).map(([key, id]) => [
key,
@@ -79,6 +81,10 @@ export class VoiceCardControls {
]),
);
this.state = initialVoiceCardState();
this.enabled = true;
this.presented = true;
this.announcementRevision = 0;
this.consumedPresentationAnnouncement = -1;
this.hideTimer = null;
this.destroyed = false;
this.listeners = new AbortController();
@@ -96,17 +102,53 @@ export class VoiceCardControls {
if (this.destroyed || !this.elements.card) return;
const next = reduceVoiceCard(this.state, event);
if (next === this.state) return;
if (
next.announce !== this.state.announce ||
(next.announce &&
((event.type === 'transcript' &&
event.role === 'assistant' &&
event.final) ||
(event.type === 'action-result' &&
next.result !== this.state.result) ||
(event.type === 'pointer' &&
next.pointer &&
next.pointer !== this.state.pointer)))
)
this.announcementRevision++;
this.state = next;
this.render();
if (this.state.visible) this.scheduleIdle();
else this.cancelIdle();
}
/** A local submission allows future output after dismissal; keep its content. */
rearm() {
if (this.destroyed || !this.elements.card) return;
this.state = { ...this.state, dismissed: false };
}
/** An annotation outline resolved or failed after its tool result. */
outline(event) {
this.handle({ type: 'outline', id: event?.id, status: event?.status });
}
/** Change presentation only; session events and the idle deadline continue. */
setEnabled(enabled) {
if (this.destroyed || !this.elements.card) return;
this.enabled = Boolean(enabled);
this.render();
}
/** View selection changes presentation without changing the saved preference. */
setPresented(presented) {
if (this.destroyed || !this.elements.card || this.presented === presented)
return;
if (!presented)
this.consumedPresentationAnnouncement = this.announcementRevision;
this.presented = presented;
this.render();
}
scheduleIdle() {
this.cancelIdle();
this.hideTimer = this.setTimer(() => {
@@ -129,12 +171,13 @@ export class VoiceCardControls {
render() {
const el = this.elements;
const view = voiceCardView(this.state);
const view = this.snapshot();
setHidden(el.card, !view.visible);
el.card.dataset.busy = String(view.busy);
setText(el.phase, view.phase);
setHidden(el.user, !view.user.visible);
setText(el.userText, view.user.text);
setText(el.userWho, view.user.label);
setHidden(el.pointer, !view.pointer.visible);
setText(el.pointerKind, view.pointer.kind);
setText(el.pointerText, view.pointer.text);
@@ -186,11 +229,21 @@ export class VoiceCardControls {
node.textContent = note;
});
setText(el.live, view.announce);
this.onRender?.();
}
/** Test/diagnostic view of what the card shows. */
snapshot() {
return voiceCardView(this.state);
const view = voiceCardView(this.state);
return this.enabled && this.presented
? {
...view,
announce:
this.consumedPresentationAnnouncement === this.announcementRevision
? ''
: view.announce,
}
: { ...view, visible: false, announce: '' };
}
/** Idempotent: removes listeners and the idle timer; markup stays owned by the control. */
+465 -47
View File
@@ -3,13 +3,17 @@ import { readFileSync } from 'node:fs';
import test from 'node:test';
import { readStylesheet } from '../testSupport/readStylesheet.mjs';
import { VoiceCardControls } from './voiceCard.js';
import { getLocale, setLocale } from '../i18n/index.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
const controlSource = readFileSync(new URL('./control.js', import.meta.url), 'utf8');
const controlSource = readFileSync(
new URL('./control.js', import.meta.url),
'utf8',
);
const css = readStylesheet(new URL('../../style.css', import.meta.url));
function fold(events) {
@@ -19,11 +23,37 @@ function fold(events) {
test('presentation: a slow annotate turn shows captions, plan steps and the result card', () => {
const state = fold([
{ type: 'interruption' },
{ type: 'transcript', role: 'user', text: 'Annotate the Texas State Capitol', final: true },
{ type: 'transcript', role: 'assistant', text: 'Finding the Capitol, ', responseId: 'r1' },
{ type: 'transcript', role: 'assistant', text: 'then outlining it.', responseId: 'r1' },
{ type: 'action-call', name: 'annotate_map', callId: 'call_a', arguments: { annotations: [{ target: 'Texas State Capitol' }] } },
{ type: 'progress', name: 'annotate_map', callId: 'call_a', step: 'resolve', label: 'Texas State Capitol' },
{
type: 'transcript',
role: 'user',
text: 'Annotate the Texas State Capitol',
final: true,
},
{
type: 'transcript',
role: 'assistant',
text: 'Finding the Capitol, ',
responseId: 'r1',
},
{
type: 'transcript',
role: 'assistant',
text: 'then outlining it.',
responseId: 'r1',
},
{
type: 'action-call',
name: 'annotate_map',
callId: 'call_a',
arguments: { annotations: [{ target: 'Texas State Capitol' }] },
},
{
type: 'progress',
name: 'annotate_map',
callId: 'call_a',
step: 'resolve',
label: 'Texas State Capitol',
},
]);
let view = voiceCardView(state);
assert.equal(view.visible, true);
@@ -31,9 +61,16 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
assert.equal(view.phase, 'WORKING');
assert.equal(view.user.text, 'Annotate the Texas State Capitol');
assert.equal(view.assistant.text, 'Finding the Capitol, then outlining it.');
assert.deepEqual(view.steps.map(({ label, status, detail }) => ({ label, status, detail })), [
{ label: 'Mark Texas State Capitol', status: 'running', detail: 'Finding places: Texas State Capitol' },
]);
assert.deepEqual(
view.steps.map(({ label, status, detail }) => ({ label, status, detail })),
[
{
label: 'Mark Texas State Capitol',
status: 'running',
detail: 'Finding places: Texas State Capitol',
},
],
);
const done = [
{
type: 'action-result',
@@ -43,7 +80,12 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
ok: true,
outlinePending: true,
items: [{ ok: true, id: 'anno-1', outlinePending: true }],
display: { title: 'Marked 1 place', lines: ['Texas State Capitol'], notes: ['Tracing outlines'], sources: [{ label: 'OpenStreetMap' }] },
display: {
title: 'Marked 1 place',
lines: ['Texas State Capitol'],
notes: ['Tracing outlines'],
sources: [{ label: 'OpenStreetMap' }],
},
referents: [{ n: 1, id: 'anno-1', label: 'Texas State Capitol' }],
},
},
@@ -53,35 +95,80 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
assert.equal(view.phase, 'DONE');
assert.equal(view.steps[0].detail, 'Outline traced');
assert.equal(view.result.title, 'Marked 1 place');
assert.deepEqual(view.result.referents, [{ n: 1, label: 'Texas State Capitol' }]);
assert.deepEqual(view.result.notes, ['Tracing outlines', 'Source: OpenStreetMap']);
assert.equal(view.announce, 'Marked 1 place', 'the live region carries the result title');
assert.deepEqual(view.result.referents, [
{ n: 1, label: 'Texas State Capitol' },
]);
assert.deepEqual(view.result.notes, [
'Tracing outlines',
'Source: OpenStreetMap',
]);
assert.equal(
view.announce,
'Marked 1 place',
'the live region carries the result title',
);
});
test('presentation: silent lookups, dismissal and session end', () => {
const lookup = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'get_current_view_state', arguments: {} },
{ type: 'action-result', name: 'get_current_view_state', result: { ok: true, schedule: 'silent', display: { title: 'View state' } } },
{
type: 'action-result',
name: 'get_current_view_state',
result: {
ok: true,
schedule: 'silent',
display: { title: 'View state' },
},
},
]);
assert.equal(voiceCardView(lookup).result.visible, false, 'a silent lookup does not take the result card');
assert.equal(
voiceCardView(lookup).result.visible,
false,
'a silent lookup does not take the result card',
);
const dismissed = reduceVoiceCard(lookup, { type: 'dismiss' });
assert.equal(voiceCardView(dismissed).visible, false);
const stillDismissed = reduceVoiceCard(dismissed, { type: 'transcript', role: 'assistant', text: 'Hi', final: true });
assert.equal(voiceCardView(stillDismissed).visible, false, 'dismissed until the next turn');
assert.equal(voiceCardView(reduceVoiceCard(stillDismissed, { type: 'interruption' })).visible, true);
assert.deepEqual(reduceVoiceCard(lookup, { type: 'state', state: 'idle' }), initialVoiceCardState());
assert.equal(reduceVoiceCard(lookup, { type: 'turn-metrics' }), lookup, 'unrelated events are no-ops');
const stillDismissed = reduceVoiceCard(dismissed, {
type: 'transcript',
role: 'assistant',
text: 'Hi',
final: true,
});
assert.equal(
voiceCardView(stillDismissed).visible,
false,
'dismissed until the next turn',
);
assert.equal(
voiceCardView(reduceVoiceCard(stillDismissed, { type: 'interruption' }))
.visible,
true,
);
assert.deepEqual(
reduceVoiceCard(lookup, { type: 'state', state: 'idle' }),
initialVoiceCardState(),
);
assert.equal(
reduceVoiceCard(lookup, { type: 'turn-metrics' }),
lookup,
'unrelated events are no-ops',
);
});
test('markup: the card nests inside the voice control with every element and a polite live region', () => {
const root = controlSource.indexOf("root.id = 'gev-voice-control'");
const card = controlSource.indexOf('id="gev-voice-card"');
assert.ok(root >= 0 && card > root, 'the card is part of the voice control markup');
assert.ok(
root >= 0 && card > root,
'the card is part of the voice control markup',
);
for (const id of [
'gev-voice-card-phase',
'gev-voice-card-dismiss',
'gev-voice-card-user-text',
'gev-voice-card-user-who',
'gev-voice-card-assistant-text',
'gev-voice-card-plan',
'gev-voice-card-result-title',
@@ -92,11 +179,22 @@ test('markup: the card nests inside the voice control with every element and a p
]) {
assert.match(controlSource, new RegExp(`id="${id}"`), `${id} is missing`);
}
assert.match(controlSource, /id="gev-voice-card-live"[^>]*aria-live="polite"/);
assert.match(controlSource, /aria-label="Dismiss voice card"/);
assert.match(
controlSource,
/id="gev-voice-card-live"[^>]*aria-live="polite"/,
);
assert.match(controlSource, /aria-label="\$\{t\('Dismiss voice card'\)\}"/);
assert.match(css, /#gev-voice-control \.gev-voice-card \{/);
assert.match(css, /body\.recording-mode #gev-voice-control/, 'recording mode hides the control, card included');
assert.match(css, /body\.ui-clean-view #command-dock/, 'Clean-UI hides the dock, card included');
assert.match(
css,
/body\.recording-mode #gev-voice-control/,
'recording mode hides the control, card included',
);
assert.match(
css,
/body\.ui-clean-view #command-dock/,
'Clean-UI hides the dock, card included',
);
});
function fakeRoot() {
@@ -111,7 +209,12 @@ function fakeRoot() {
ownerDocument: null,
addEventListener(type, handler, options) {
listeners.set(type, handler);
options?.signal?.addEventListener('abort', () => listeners.delete(type));
options?.signal?.addEventListener('abort', () =>
listeners.delete(type),
);
},
removeEventListener(type, handler) {
if (listeners.get(type) === handler) listeners.delete(type);
},
fire: (type) => listeners.get(type)?.({}),
listenerCount: () => listeners.size,
@@ -128,6 +231,7 @@ function fakeRoot() {
const nodes = new Map();
const doc = { createElement: (tag) => ({ ...make(tag), tag }) };
const root = {
...make('gev-voice-control'),
querySelector(selector) {
const id = selector.slice(1);
if (!nodes.has(id)) {
@@ -160,12 +264,26 @@ test('controls: render, idle hide, dismiss and idempotent teardown', () => {
});
assert.equal(nodes.get('gev-voice-card').hidden, true, 'starts hidden');
card.handle({ type: 'interruption' });
card.handle({ type: 'action-call', name: 'fly_to_location', arguments: { query: 'Tokyo' } });
card.handle({ type: 'action-result', name: 'fly_to_location', result: { ok: true, display: { title: 'Tokyo', lines: [], chips: [{ label: 'stale' }] } } });
card.handle({
type: 'action-call',
name: 'fly_to_location',
arguments: { query: 'Tokyo' },
});
card.handle({
type: 'action-result',
name: 'fly_to_location',
result: {
ok: true,
display: { title: 'Tokyo', lines: [], chips: [{ label: 'stale' }] },
},
});
assert.equal(nodes.get('gev-voice-card').hidden, false);
assert.equal(nodes.get('gev-voice-card-result-title').textContent, 'Tokyo');
assert.equal(nodes.get('gev-voice-card-plan').children.length, 1);
assert.equal(nodes.get('gev-voice-card-chips').children[0].textContent, 'stale');
assert.equal(
nodes.get('gev-voice-card-chips').children[0].textContent,
'stale',
);
assert.equal(timers.size, 1, 'one idle timer at a time');
[...timers.values()][0]();
assert.equal(nodes.get('gev-voice-card').hidden, true, 'hides when idle');
@@ -177,7 +295,11 @@ test('controls: render, idle hide, dismiss and idempotent teardown', () => {
assert.equal(timers.size, 0);
assert.equal(nodes.get('gev-voice-card-dismiss').listenerCount(), 0);
card.handle({ type: 'interruption' });
assert.equal(nodes.get('gev-voice-card').hidden, true, 'a destroyed card ignores late events');
assert.equal(
nodes.get('gev-voice-card').hidden,
true,
'a destroyed card ignores late events',
);
});
test('controls: a control without card markup is inert', () => {
@@ -189,17 +311,41 @@ test('controls: a control without card markup is inert', () => {
test('presentation: concurrent calls of one tool update their own steps', () => {
const state = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'fly_to_location', callId: 'c1', arguments: { query: 'Tokyo' } },
{ type: 'action-call', name: 'fly_to_location', callId: 'c2', arguments: { query: 'Paris' } },
{
type: 'action-call',
name: 'fly_to_location',
callId: 'c1',
arguments: { query: 'Tokyo' },
},
{
type: 'action-call',
name: 'fly_to_location',
callId: 'c2',
arguments: { query: 'Paris' },
},
// Out of order: the first call finishes last.
{ type: 'action-result', name: 'fly_to_location', callId: 'c2', result: { ok: false, error: 'x' } },
{ type: 'progress', name: 'fly_to_location', callId: 'c1', step: 'search', label: 'Tokyo' },
{
type: 'action-result',
name: 'fly_to_location',
callId: 'c2',
result: { ok: false, error: 'x' },
},
{
type: 'progress',
name: 'fly_to_location',
callId: 'c1',
step: 'search',
label: 'Tokyo',
},
]);
const steps = voiceCardView(state).steps;
assert.deepEqual(steps.map(({ label, status }) => [label, status]), [
['Fly to Tokyo', 'running'],
['Fly to Paris', 'failed'],
]);
assert.deepEqual(
steps.map(({ label, status }) => [label, status]),
[
['Fly to Tokyo', 'running'],
['Fly to Paris', 'failed'],
],
);
assert.equal(steps[0].detail, 'Looking up place: Tokyo');
});
@@ -207,27 +353,299 @@ test('presentation: stale and unmatched events never reopen the card', () => {
const ended = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'annotate_map', callId: 'c1', arguments: {} },
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, outlinePending: true, items: [{ ok: true, id: 'anno-9', outlinePending: true }] } },
{
type: 'action-result',
name: 'annotate_map',
callId: 'c1',
result: {
ok: true,
outlinePending: true,
items: [{ ok: true, id: 'anno-9', outlinePending: true }],
},
},
{ type: 'state', state: 'idle' },
]);
for (const late of [
{ type: 'outline', id: 'anno-9', status: 'resolved' },
{ type: 'outline', status: 'resolved' },
{ type: 'progress', name: 'annotate_map', callId: 'c1', step: 'resolve' },
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, display: { title: 'Marked 1 place' } } },
{
type: 'action-result',
name: 'annotate_map',
callId: 'c1',
result: { ok: true, display: { title: 'Marked 1 place' } },
},
]) {
assert.equal(reduceVoiceCard(ended, late), ended, `${late.type} after stop is a no-op`);
assert.equal(
reduceVoiceCard(ended, late),
ended,
`${late.type} after stop is a no-op`,
);
}
const live = fold([
{ type: 'interruption' },
{ type: 'action-call', name: 'annotate_map', callId: 'c1', arguments: {} },
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, outlinePending: true, items: [{ ok: true, id: 'anno-1', outlinePending: true }, { ok: true, id: 'anno-2', outlinePending: true }] } },
{
type: 'action-result',
name: 'annotate_map',
callId: 'c1',
result: {
ok: true,
outlinePending: true,
items: [
{ ok: true, id: 'anno-1', outlinePending: true },
{ ok: true, id: 'anno-2', outlinePending: true },
],
},
},
{ type: 'dismiss' },
]);
assert.equal(reduceVoiceCard(live, { type: 'outline', id: 'anno-7', status: 'resolved' }), live, 'another mark is ignored');
const first = reduceVoiceCard(live, { type: 'outline', id: 'anno-1', status: 'resolved' });
assert.equal(voiceCardView(first).steps[0].detail, 'Tracing outline', 'one of two outlines still pending');
assert.equal(voiceCardView(first).visible, false, 'a dismissed card stays hidden');
const both = reduceVoiceCard(first, { type: 'outline', id: 'anno-2', status: 'failed' });
assert.equal(
reduceVoiceCard(live, {
type: 'outline',
id: 'anno-7',
status: 'resolved',
}),
live,
'another mark is ignored',
);
const first = reduceVoiceCard(live, {
type: 'outline',
id: 'anno-1',
status: 'resolved',
});
assert.equal(
voiceCardView(first).steps[0].detail,
'Tracing outline',
'one of two outlines still pending',
);
assert.equal(
voiceCardView(first).visible,
false,
'a dismissed card stays hidden',
);
const both = reduceVoiceCard(first, {
type: 'outline',
id: 'anno-2',
status: 'failed',
});
assert.equal(voiceCardView(both).steps[0].detail, 'Outline unavailable');
});
test('presentation controls preserve real reducer content and the existing 14-second idle deadline', () => {
const { root, nodes } = fakeRoot();
const timers = new Map();
let next = 0,
renders = 0;
const card = new VoiceCardControls({
root,
setTimer(fn, ms) {
const id = ++next;
timers.set(id, { fn, ms });
return id;
},
clearTimer(id) {
timers.delete(id);
},
onRender() {
renders++;
},
});
card.handle({ type: 'interruption' });
card.handle({
type: 'transcript',
role: 'assistant',
text: 'A current answer',
final: true,
responseId: 'r1',
});
const content = card.state;
const timer = [...timers.keys()][0];
assert.equal(timers.get(timer).ms, 14000);
card.setPresented(false);
assert.equal(card.snapshot().visible, false);
assert.equal(nodes.get('gev-voice-card-live').textContent, '');
card.setPresented(true);
assert.equal(card.snapshot().visible, true);
assert.equal(card.snapshot().assistant.text, 'A current answer');
assert.equal(
card.snapshot().announce,
'',
'returning to a tab does not repeat an already presented announcement',
);
assert.equal(card.state, content);
assert.equal(
[...timers.keys()][0],
timer,
'presentation switches do not reset idle',
);
card.setEnabled(false);
card.handle({
type: 'transcript',
role: 'assistant',
text: 'A later answer',
final: true,
responseId: 'r2',
});
assert.equal(card.snapshot().visible, false);
card.setEnabled(true);
assert.equal(card.snapshot().assistant.text, 'A later answer');
assert.equal(card.snapshot().announce, 'A later answer');
assert.ok(renders > 4);
const idle = [...timers.values()][0];
idle.fn();
assert.equal(
card.snapshot().visible,
false,
'original idle still ends content presentation',
);
card.destroy();
const lastRenders = renders;
card.setPresented(false);
card.setEnabled(false);
assert.equal(renders, lastRenders);
});
test('a new hidden reply with identical words is still a fresh announcement', () => {
const { root } = fakeRoot();
const card = new VoiceCardControls({
root,
setTimer: () => 1,
clearTimer() {},
});
card.handle({ type: 'interruption' });
card.handle({
type: 'transcript',
role: 'assistant',
text: 'Same words',
final: true,
responseId: 'r1',
});
card.setPresented(false);
card.handle({
type: 'transcript',
role: 'assistant',
text: 'Same words',
final: true,
responseId: 'r2',
});
card.setPresented(true);
assert.equal(card.snapshot().announce, 'Same words');
card.destroy();
});
test('additive presentation gates retain the current localized reducer labels', () => {
const previous = getLocale();
try {
setLocale('es');
const { root } = fakeRoot();
const card = new VoiceCardControls({
root,
setTimer: () => 1,
clearTimer() {},
});
card.handle({ type: 'interruption' });
card.handle({
type: 'action-call',
name: 'annotate_map',
callId: 'a',
arguments: {},
});
assert.equal(card.snapshot().phase, 'EN CURSO');
const steps = card.state.steps;
const result = card.state.result;
const who = root.querySelector('#gev-voice-card-user-who');
card.handle({ type: 'transcript', role: 'user', text: 'Same words', attribution: 'unverified' });
assert.equal(card.state.userAttribution, 'unverified');
assert.equal(card.snapshot().user.label, 'Habla reciente');
assert.equal(who.textContent, 'Habla reciente');
card.handle({ type: 'transcript', role: 'user', text: 'Same words' });
assert.equal(card.state.userAttribution, null);
assert.equal(card.snapshot().user.label, 'TÚ');
assert.equal(who.textContent, 'TÚ');
assert.equal(card.state.steps, steps);
assert.equal(card.state.result, result);
card.setPresented(false);
card.setPresented(true);
assert.equal(card.snapshot().phase, 'EN CURSO');
card.setEnabled(false);
card.setEnabled(true);
assert.equal(card.snapshot().phase, 'EN CURSO');
card.destroy();
} finally {
setLocale(previous);
}
});
test('manual panel and preference compose with the actual card without changing voice content or enabling Preview', async () => {
const { createManualVoicePanel } = await import('./panel/manualPanel.js');
const { createVoicePreferences, bindPreferenceControls } =
await import('./panel/preferences.js');
const { root } = fakeRoot();
let panel;
const card = new VoiceCardControls({
root,
setTimer: () => 1,
clearTimer() {},
onRender: () => panel?.sync(),
});
const ui = {
root,
card: root.querySelector('#gev-voice-card'),
voicePanel: root.querySelector('#panel'),
previewPanel: root.querySelector('#preview-panel'),
voiceSettingsPanel: root.querySelector('#settings-panel'),
previewEmpty: root.querySelector('#empty'),
voicePreviewInput: root.querySelector('#preview-input'),
voicePreviewNote: root.querySelector('#preview-note'),
};
const preference = createVoicePreferences({ storage: null });
const fields = bindPreferenceControls({
ui,
preferences: preference,
copy: {
previewNote: 'Mostrar respuestas.',
timeoutNote: '',
notSaved: 'No guardado.',
},
onChange: (state) => {
card.setEnabled(state.previewEnabled);
panel?.sync();
},
});
card.setEnabled(preference.snapshot().previewEnabled);
panel = createManualVoicePanel({
ui,
card,
getPreviewEnabled: () => preference.snapshot().previewEnabled,
copy: { previewEmpty: 'Sin respuesta.', previewDisabled: 'Desactivada.' },
});
card.handle({ type: 'interruption' });
card.handle({
type: 'transcript',
role: 'assistant',
text: 'Actual current answer',
final: true,
});
const content = card.state;
assert.equal(
card.snapshot().visible,
false,
'manual panel stays closed despite content',
);
panel.select('preview');
assert.equal(card.snapshot().assistant.text, 'Actual current answer');
assert.equal(card.snapshot().visible, true);
preference.setPreviewEnabled(false);
assert.equal(card.snapshot().visible, false);
assert.equal(ui.previewEmpty.textContent, 'Desactivada.');
panel.select('settings');
panel.select('preview');
assert.equal(preference.snapshot().previewEnabled, false);
assert.equal(card.snapshot().visible, false);
assert.equal(card.state, content);
fields.dispose();
panel.destroy();
card.destroy();
});
+109
View File
@@ -0,0 +1,109 @@
// A failed tool step: the card shows the tool error in the UI language; the
// result keeps gevActions' English for the model.
import assert from 'node:assert/strict';
import test from 'node:test';
import { setLocale } from '../i18n/index.js';
import { attachVoiceResult, spokenLabel } from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
// Error texts as gevActions.js builds them (frameOverhead, select_nearest_aircraft).
const CASES = [
{
name: 'frame_overhead',
args: { target: 'military' },
result: {
ok: false,
action: 'frame_overhead',
layerId: 'military',
error: 'The military layer is not enabled',
},
},
{
name: 'select_nearest_aircraft',
args: { locationQuery: 'Madrid' },
result: {
ok: false,
action: 'select_nearest_aircraft',
stage: 'nearest',
cancelled: false,
error:
'Live Flights is enabled, but no airborne aircraft is loaded in the Madrid view yet',
location: { label: 'Madrid' },
layer: { label: 'Live Flights' },
feed: { state: 'nominal', source: 'OpenSky' },
count: 0,
},
},
{
name: 'frame_overhead',
args: { target: 'flights' },
result: {
ok: false,
action: 'frame_overhead',
layerId: 'flights',
radiusKm: 150,
count: 0,
error: 'No flights within 150 km of the current view',
},
},
];
function cardFor(locale, { name, args, result }) {
setLocale(locale);
let state = reduceVoiceCard(initialVoiceCardState(), {
type: 'action-call',
id: 'c1',
name,
arguments: args,
});
state = reduceVoiceCard(state, {
type: 'action-result',
id: 'c1',
name,
result: attachVoiceResult(name, result, args),
});
return voiceCardView(state);
}
for (const locale of ['es', 'ru']) {
for (const c of CASES) {
test(`${locale}: ${c.name} failure shows the card lines in the UI language`, () => {
const view = cardFor(locale, c);
try {
assert.ok(view.result.visible, 'card shows a result');
assert.notEqual(view.result.title, '', 'card has a title');
// Title and step status are translated.
assert.notEqual(
view.steps[0].statusText,
'failed',
'step status translated',
);
// The error line is translated, not the raw English.
for (const line of view.result.lines) {
assert.ok(
!/\b(is not enabled|no airborne aircraft|of the current view)\b/.test(
line,
),
`English tool error shown under ${locale} title "${view.result.title}" / step "${view.steps[0].label}: ${view.steps[0].statusText}": "${line}"`,
);
}
} finally {
setLocale('en');
}
});
}
}
test('en: the card shows the tool error as the model got it', () => {
for (const c of CASES) {
const view = cardFor('en', c);
// Whole, not cut at 80 characters: the card wraps an error line.
assert.deepEqual(view.result.lines, [
spokenLabel(c.result.error, Infinity),
]);
}
});
+142
View File
@@ -0,0 +1,142 @@
// The voice card names layers, sources and distances in the interface
// language; what the model receives stays English.
import test from 'node:test';
import assert from 'node:assert/strict';
import { setLocale } from '../i18n/index.js';
import {
attachVoiceResult,
planStepLabel,
shownEntityLabel,
} from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
for (const locale of ['ru', 'es']) {
test(`voice card is in ${locale} for a layer switch`, (t) => {
t.after(() => setLocale('en'));
setLocale(locale);
const out = attachVoiceResult(
'set_layer_visibility',
{
ok: true,
layerId: 'flights',
label: 'Live Flights',
enabled: true,
source: 'Flights',
},
{ layerId: 'flights', enabled: true },
);
const view = voiceCardView({ steps: [], result: out });
assert.doesNotMatch(out.display.title, /Live Flights/);
assert.doesNotMatch(JSON.stringify(view), /Flights/);
});
test(`voice card distance is in ${locale}`, (t) => {
t.after(() => setLocale('en'));
setLocale(locale);
const out = attachVoiceResult('fly_to_location', {
ok: true,
label: 'Paris',
rangeM: 2500,
});
assert.doesNotMatch(out.display.lines[0], /2\.5 km/);
});
test(`plan label is in ${locale}`, (t) => {
t.after(() => setLocale('en'));
setLocale(locale);
const a = planStepLabel('select_nearest_aircraft', {});
const b = planStepLabel('set_layer_visibility', {
layerId: 'earthquakes',
enabled: true,
});
assert.doesNotMatch(a, /Here/);
assert.doesNotMatch(b, /Earthquakes/);
});
}
test('en: layer switch, distance and plan labels in English', () => {
const out = attachVoiceResult(
'set_layer_visibility',
{
ok: true,
layerId: 'flights',
label: 'Live Flights',
enabled: true,
source: 'Flights',
},
{ layerId: 'flights', enabled: true },
);
assert.equal(out.say, 'Live Flights on.');
assert.equal(out.display.title, 'Live Flights on');
assert.deepEqual(out.display.sources, [{ label: 'Flights' }]);
const fly = attachVoiceResult('fly_to_location', {
ok: true,
label: 'Paris',
rangeM: 2500,
});
assert.equal(fly.display.lines[0].replace(/\s/g, ' '), 'Camera range 2.5 km');
assert.equal(
planStepLabel('select_nearest_aircraft', {}),
'Nearest aircraft to Here',
);
assert.equal(
planStepLabel('frame_overhead', { target: 'vessels' }),
'Frame vessels',
);
// The plan names the layer as its result card does, not by its id.
assert.equal(
planStepLabel('set_layer_visibility', {
layerId: 'earthquakes',
enabled: true,
}),
'Earthquakes (24h) on',
);
});
// FIRMS and ALPR register English entity labels: the model reads them as data
// (get_entity_context, pointer_context), the card shows them translated.
const ENTITY_LABELS = {
ru: { fire: 'Пожар · FRP 12 МВт', alpr: 'Камера ALPR' },
es: { fire: 'Incendio · FRP 12 MW', alpr: 'Cámara ALPR' },
};
for (const [locale, shown] of Object.entries(ENTITY_LABELS)) {
test(`entity labels stay English in the result and show in ${locale}`, (t) => {
t.after(() => setLocale('en'));
setLocale(locale);
const out = attachVoiceResult('get_entity_context', {
ok: true,
selected: {
id: 'fire-1',
name: 'Fire · FRP 12 MW',
layerId: 'local-firms',
layerName: 'NASA FIRMS',
},
});
assert.equal(out.selected.name, 'Fire · FRP 12 MW');
assert.equal(out.display.title, shown.fire);
const state = reduceVoiceCard(initialVoiceCardState(), {
type: 'pointer',
pointer: { kind: 'this', label: 'ALPR camera' },
});
assert.equal(state.pointer.label, 'ALPR camera');
assert.equal(voiceCardView(state).pointer.text, shown.alpr);
assert.equal(state.announce, `This: ${shown.alpr}`);
});
}
test('only the registered entity labels are translated', () => {
assert.equal(shownEntityLabel('Fire · FRP 4.9 MW'), 'Fire · FRP 4.9 MW');
assert.equal(shownEntityLabel('ALPR camera'), 'ALPR camera');
setLocale('ru');
try {
assert.equal(shownEntityLabel('Fire · FRP 4.9 MW'), 'Пожар · FRP 4.9 МВт');
assert.equal(shownEntityLabel('UPS793'), 'UPS793');
assert.equal(shownEntityLabel('Fire Station 5'), 'Fire Station 5');
} finally {
setLocale('en');
}
});
@@ -0,0 +1,48 @@
// The voice card names a preset city in the interface language in the
// fly-to headline and the nearest-aircraft distance line; what the model
// says stays English.
import assert from 'node:assert/strict';
import test from 'node:test';
import { setLocale } from '../i18n/index.js';
import { SPEECH_BUILDERS } from './speech.js';
function inLocale(locale, run) {
setLocale(locale);
try {
return run();
} finally {
setLocale('en');
}
}
const FLY = { ok: true, arrived: true, label: 'Tokyo' };
const NEAREST = {
ok: true,
layerId: 'flights',
location: { label: 'Tokyo' },
aircraft: { callsign: 'JAL123', distanceKm: 12, altitudeM: 3000 },
feed: {},
};
test('the fly-to headline shows the city in the interface language', () => {
assert.equal(SPEECH_BUILDERS.fly_to_location(FLY).display.title, 'Tokyo');
const ru = inLocale('ru', () => SPEECH_BUILDERS.fly_to_location(FLY));
assert.equal(ru.display.title, 'Токио');
assert.match(ru.say, /Tokyo/);
const es = inLocale('es', () => SPEECH_BUILDERS.fly_to_location(FLY));
assert.equal(es.display.title, 'Tokio');
});
test('the nearest-aircraft distance line shows the city in the interface language', () => {
const en = SPEECH_BUILDERS.select_nearest_aircraft(NEAREST);
assert.match(en.display.lines[0], /from Tokyo$/);
const ru = inLocale('ru', () =>
SPEECH_BUILDERS.select_nearest_aircraft(NEAREST),
);
assert.match(ru.display.lines[0], /Токио/);
assert.match(ru.say, /Tokyo/);
const es = inLocale('es', () =>
SPEECH_BUILDERS.select_nearest_aircraft(NEAREST),
);
assert.match(es.display.lines[0], /Tokio/);
});
+282
View File
@@ -0,0 +1,282 @@
// Voice card in the UI language: every tool's plan step, the bare "Layer"
// step, whole error lines and long headlines, the Contacts snapshot clock.
// What the model receives stays English.
import assert from 'node:assert/strict';
import test from 'node:test';
import { setLocale } from '../i18n/index.js';
import { GEV_ACTION_SCHEMAS } from './actionSchemas.js';
import { presentResult, shownSnapshotTime } from './resultDisplay.js';
import { CITY_POIS } from '../cityPois.js';
import { attachVoiceResult, narrationLabel, planStepLabel } from './speech.js';
import {
initialVoiceCardState,
reduceVoiceCard,
voiceCardView,
} from './voiceCardPresentation.js';
const TOOLS = GEV_ACTION_SCHEMAS.map((schema) => schema.name);
function inLocale(locale, run) {
setLocale(locale);
try {
return run();
} finally {
setLocale('en');
}
}
test('every tool has a plan label, never its raw name', () => {
for (const name of TOOLS) {
const label = planStepLabel(name, {});
assert.notEqual(label, name.replace(/_/g, ' '), name);
assert.doesNotMatch(label, /_/, name);
}
assert.equal(planStepLabel('analyst_query', {}), 'Analyst query');
assert.equal(
planStepLabel('adjust_camera_zoom', { direction: 'in' }),
'Zoom in',
);
assert.equal(
planStepLabel('adjust_camera_zoom', { direction: 'out' }),
'Zoom out',
);
// A tool the card does not know keeps its name in English.
assert.equal(planStepLabel('brand_new_tool', {}), 'brand new tool');
});
for (const locale of ['ru', 'es']) {
test(`${locale} plan labels are all translated`, () => {
const english = Object.fromEntries(
TOOLS.map((name) => [name, planStepLabel(name, { direction: 'out' })]),
);
inLocale(locale, () => {
for (const name of TOOLS) {
// A layer switch without a layer is checked below.
const label = planStepLabel(name, { direction: 'out' });
assert.notEqual(label, english[name], `${name}: "${label}"`);
}
});
});
}
test('analyst query and an unknown tool in ru and es', () => {
inLocale('ru', () => {
assert.equal(planStepLabel('analyst_query', {}), 'Аналитический запрос');
assert.equal(planStepLabel('brand_new_tool', {}), 'Действие');
});
inLocale('es', () => {
assert.equal(planStepLabel('analyst_query', {}), 'Consulta analítica');
assert.equal(planStepLabel('brand_new_tool', {}), 'Acción');
});
});
test('a layer step with no layer agrees with "Capa"', () => {
const step = (enabled) => planStepLabel('set_layer_visibility', { enabled });
assert.equal(step(true), 'Layer on');
assert.equal(step(false), 'Layer off');
inLocale('es', () => {
assert.equal(step(true), 'Capa: activada');
assert.equal(step(false), 'Capa: desactivada');
// A named layer keeps the shared "{label} on" wording.
assert.match(
planStepLabel('set_layer_visibility', {
layerId: 'earthquakes',
enabled: true,
}),
/: activado$/,
);
});
inLocale('ru', () => {
assert.equal(step(true), 'Слой: вкл.');
assert.equal(step(false), 'Слой: выкл.');
});
});
function cardAfter(name, result, args = {}) {
let state = reduceVoiceCard(initialVoiceCardState(), {
type: 'action-call',
id: 'c1',
name,
arguments: args,
});
state = reduceVoiceCard(state, {
type: 'action-result',
id: 'c1',
name,
result: attachVoiceResult(name, result, args),
});
return voiceCardView(state);
}
const NO_PLACE = {
ok: false,
action: 'select_nearest_aircraft',
stage: 'location',
error:
'Nearest-aircraft selection needs a preset, place name, or latitude and longitude',
feed: {},
};
test('an error line shows whole, in every language', () => {
for (const [locale, expected] of [
['en', NO_PLACE.error],
[
'ru',
'Для выбора ближайшего самолёта нужен пресет, название места или широта и долгота',
],
[
'es',
'Para seleccionar la aeronave más cercana se necesita un preajuste, un nombre de lugar o la latitud y la longitud',
],
]) {
const view = inLocale(locale, () =>
cardAfter('select_nearest_aircraft', NO_PLACE),
);
assert.deepEqual(view.result.lines, [expected], locale);
}
});
test('a successful line keeps its 80-character cap', () => {
const long = 'x'.repeat(120);
const view = voiceCardView({
...initialVoiceCardState(),
result: { display: { title: 'T', lines: [long] } },
});
assert.equal(view.result.lines[0].length, 80);
});
test('a translated analyst headline fits the card title whole', () => {
const title =
'Al menos 12 aeronaves militares en un radio de 40 km de Los Angeles';
const card = () =>
voiceCardView({
...initialVoiceCardState(),
result: { display: { title, lines: [] } },
}).result.title;
assert.equal(inLocale('es', card), title);
// English keeps its 64-character title, equal to the spoken headline.
assert.equal(card().length, 64);
});
test('the snapshot time reads as a UTC clock, dated when not today', () => {
const iso = '2026-10-15T14:32:05.000Z';
const sameDay = Date.parse('2026-10-15T23:00:00Z');
const nextDay = Date.parse('2026-10-16T00:10:00Z');
assert.equal(shownSnapshotTime(iso, sameDay), iso, 'English keeps ISO');
inLocale('ru', () => {
assert.equal(shownSnapshotTime(iso, sameDay), '14:32:05 UTC');
assert.equal(shownSnapshotTime(iso, nextDay), '15.10.2026, 14:32:05 UTC');
});
inLocale('es', () => {
assert.equal(shownSnapshotTime(iso, sameDay), '14:32:05 UTC');
assert.equal(shownSnapshotTime(iso, nextDay), '15/10/2026, 14:32:05 UTC');
assert.equal(shownSnapshotTime('not a time'), 'not a time');
});
});
test('the Contacts note shows the clock; the model note stays ISO', () => {
const evaluatedAt = new Date();
evaluatedAt.setUTCHours(14, 32, 5, 0);
const iso = evaluatedAt.toISOString();
const note = `Contacts panel snapshot at ${iso} — the same retained cohort supplies the card, speech, and follow-up memory.`;
const result = {
ok: true,
count: 12,
complete: true,
scopeLabel: 'in view',
display: { scope: 'in view' },
coverage: { note },
items: [],
};
const notes = inLocale('es', () =>
presentResult('analyst_query', result).display.notes.join(' | '),
);
assert.match(notes, /\(14:32:05 UTC\)/);
assert.doesNotMatch(notes, /T14:32/);
assert.equal(result.coverage.note, note, 'the result is not changed');
});
test('plan steps and errors name a preset place in the UI language', () => {
const failed = {
ok: false,
action: 'select_nearest_aircraft',
stage: 'nearest',
error:
'Live Flights is enabled, but no airborne aircraft is loaded in the Tokyo view yet',
location: { label: 'Tokyo' },
feed: { state: 'nominal' },
};
assert.equal(
planStepLabel('fly_to_location', { query: 'Tokyo' }),
'Fly to Tokyo',
);
inLocale('es', () => {
assert.match(planStepLabel('fly_to_location', { query: 'Tokyo' }), /Tokio/);
assert.match(
planStepLabel('select_nearest_aircraft', { locationId: 'tokyo' }),
/Tokio/,
);
const view = cardAfter('select_nearest_aircraft', failed);
assert.match(view.result.lines[0], /Tokio/);
});
inLocale('ru', () => {
assert.match(planStepLabel('fly_to_location', { query: 'Tokyo' }), /Токио/);
});
assert.match(failed.error, /Tokyo view/, 'the result stays English');
});
test('the analyst scope names Los Angeles in the UI language', () => {
const result = {
ok: true,
count: 12,
complete: false,
scopeLabel: 'within 40 km of Los Angeles',
display: { scope: 'within 40 km of Los Angeles' },
coverage: { layersQueried: [{ layerKey: 'military' }] },
items: [],
};
const title = inLocale(
'es',
() => presentResult('analyst_query', result).display.title,
);
assert.match(title, /Los Ángeles$/, 'the headline is not cut at 64');
assert.equal(result.scopeLabel, 'within 40 km of Los Angeles');
});
// A preset city id from the tool schema (sf, nyc, dc) reads as its city name.
test('a preset city id names the city in plan steps and narration', () => {
const step = (id, action = 'fly_to_location') =>
planStepLabel(action, { locationId: id });
assert.equal(step('sf'), 'Fly to San Francisco');
assert.equal(
narrationLabel('fly_to_location', { locationId: 'dc' }),
'Washington DC',
);
assert.equal(
planStepLabel('fly_to_location', { query: 'Ulaanbaatar' }),
'Fly to Ulaanbaatar',
'other places are shown as given',
);
inLocale('es', () => {
assert.equal(step('tokyo'), 'Volar a Tokio');
assert.equal(step('nyc'), 'Volar a Nueva York');
for (const id of ['sf', 'nyc', 'dc'])
assert.doesNotMatch(step(id), /\b(Sf|Nyc|Dc)$/, id);
});
inLocale('ru', () => {
assert.match(step('dc', 'select_nearest_aircraft'), /Вашингтон/);
});
});
test('every locationId the voice tools accept is a preset city', () => {
const ids = new Set();
const visit = (node) => {
if (!node || typeof node !== 'object') return;
if (Array.isArray(node.locationId?.enum))
node.locationId.enum.forEach((id) => ids.add(id));
Object.values(node).forEach(visit);
};
visit(GEV_ACTION_SCHEMAS);
assert.ok(ids.size > 0, 'found the locationId enums');
for (const id of ids) assert.ok(Object.hasOwn(CITY_POIS, id), id);
});
+50 -19
View File
@@ -1,7 +1,13 @@
import { planStepLabel, progressStepLabel, spokenLabel } from './speech.js';
import {
planStepLabel,
progressStepLabel,
shownEntityLabel,
spokenLabel,
} from './speech.js';
import { presentResult } from './resultDisplay.js';
import { MAX_DISPLAYED_REFERENTS, normalizeReferents } from './referents.js';
import { getLocale, msg, t } from '../i18n/index.js';
/**
* Pure presentation of the voice card nested in the voice control: captions,
* the current plan, the latest result and its numbered referents.
@@ -16,12 +22,20 @@ const MAX_STEPS = 6;
const MAX_LINES = 6;
const MAX_NOTES = 6;
const MAX_CAPTION = 220;
// Titles and lines wrap; the caps only stop runaway values. Translations run
// longer (resultDisplay.js), so they cap at 120. Error lines are never cut.
const MAX_TITLE = 64;
const MAX_TRANSLATED_TITLE = 120;
const MAX_LINE = 80;
const titleMax = () =>
getLocale() === 'en' ? MAX_TITLE : MAX_TRANSLATED_TITLE;
const STATUS_TEXT = Object.freeze({
running: 'in progress',
done: 'done',
failed: 'failed',
cancelled: 'cancelled',
running: msg('in progress'),
done: msg('done'),
failed: msg('failed'),
cancelled: msg('cancelled'),
});
/** An empty, hidden card. */
@@ -30,6 +44,7 @@ export function initialVoiceCardState() {
visible: false,
dismissed: false,
user: null,
userAttribution: null,
pointer: null,
assistant: null,
assistantResponse: null,
@@ -55,6 +70,7 @@ function newTurn(state) {
visible: true,
dismissed: false,
user: '',
userAttribution: null,
pointer: null,
assistant: null,
assistantResponse: null,
@@ -123,13 +139,20 @@ export function reduceVoiceCard(state, event) {
return reveal({
...state,
pointer,
announce: `${kind}: ${spokenLabel(pointer.label, 32)}`,
announce: `${kind}: ${spokenLabel(shownEntityLabel(pointer.label), 32)}`,
});
}
case 'transcript': {
const text = event.text || '';
if (event.role === 'user') {
return text ? reveal({ ...state, user: caption(text) }) : state;
return text
? reveal({
...state,
user: caption(text),
userAttribution:
event.attribution === 'unverified' ? 'unverified' : null,
})
: state;
}
if (event.role !== 'assistant' || !text) return state;
const sameResponse =
@@ -190,7 +213,7 @@ export function reduceVoiceCard(state, event) {
: result.ok
? 'done'
: 'failed',
detail: result.outlinePending ? 'Tracing outline' : '',
detail: result.outlinePending ? t('Tracing outline') : '',
annotationIds: result.outlinePending ? annotationIds(result) : [],
}),
);
@@ -204,12 +227,14 @@ export function reduceVoiceCard(state, event) {
...next,
result: {
display: shown.display,
// A failed result's lines are its error: shown whole, wrapped.
wholeLines: result.ok === false,
referents: normalizeReferents(
shown.referents,
MAX_DISPLAYED_REFERENTS,
),
},
announce: spokenLabel(shown.display.title, 64),
announce: spokenLabel(shown.display.title, titleMax()),
});
}
case 'outline': {
@@ -224,10 +249,10 @@ export function reduceVoiceCard(state, event) {
...step,
annotationIds: remaining,
detail: remaining.length
? 'Tracing outline'
? t('Tracing outline')
: event.status === 'resolved'
? 'Outline traced'
: 'Outline unavailable',
? t('Outline traced')
: t('Outline unavailable'),
};
return reveal({ ...state, steps });
}
@@ -244,7 +269,9 @@ export function voiceCardView(state) {
const display = state.result?.display || null;
const allNotes = [
...(display?.notes || []),
...(display?.sources || []).map((source) => `Source: ${source.label}`),
...(display?.sources || []).map((source) =>
t('Source: {label}', { label: source.label }),
),
];
// Analyst notes carry count/scope/provenance qualifications. Preserve their
// full sanitized text in at most six existing rows, joining overflow into
@@ -267,15 +294,17 @@ export function voiceCardView(state) {
return {
visible: Boolean(state.visible),
busy: running,
phase: running ? 'WORKING' : state.steps.length ? 'DONE' : '',
phase: running ? t('WORKING') : state.steps.length ? t('DONE') : '',
user: {
label:
state.userAttribution === 'unverified' ? t('Recent speech') : t('YOU'),
visible: state.user !== null,
text: state.user || '…',
},
pointer: {
visible: Boolean(state.pointer),
kind: state.pointer?.kind === 'here' ? 'HERE' : 'THIS',
text: spokenLabel(state.pointer?.label || '', 32),
kind: state.pointer?.kind === 'here' ? t('HERE') : t('THIS'),
text: spokenLabel(shownEntityLabel(state.pointer?.label || ''), 32),
},
assistant: {
visible: Boolean(state.assistant),
@@ -285,15 +314,17 @@ export function voiceCardView(state) {
key: String(step.id),
label: step.label,
status: step.status,
statusText: STATUS_TEXT[step.status] || '',
statusText: STATUS_TEXT[step.status] ? t(STATUS_TEXT[step.status]) : '',
detail: step.detail || '',
})),
result: {
visible: Boolean(display?.title),
title: spokenLabel(display?.title || '', 64),
title: spokenLabel(display?.title || '', titleMax()),
chips: (display?.chips || []).map((chip) => spokenLabel(chip.label, 24)),
lines: (display?.lines || [])
.map((line) => spokenLabel(line, 80))
.map((line) =>
spokenLabel(line, state.result?.wholeLines ? Infinity : MAX_LINE),
)
.slice(0, MAX_LINES),
referents: (state.result?.referents || [])
.slice(0, MAX_DISPLAYED_REFERENTS)
+2 -2
View File
@@ -50,7 +50,7 @@ export const VOICE_MODELS = Object.freeze({
standard: Object.freeze({
tier: 'standard',
id: 'gpt-realtime-2',
label: 'STANDARD',
label: 'STANDARD', // i18n-ignore: model data; the tier button has its own text
/** USD per 1M tokens — gpt-realtime-2. */
rates: Object.freeze({
textInput: 4,
@@ -66,7 +66,7 @@ export const VOICE_MODELS = Object.freeze({
mini: Object.freeze({
tier: 'mini',
id: 'gpt-realtime-2.1-mini',
label: 'MINI',
label: 'MINI', // i18n-ignore: model data; the tier button has its own text
/** USD per 1M tokens — gpt-realtime-2.1-mini (~3.2× cheaper on audio). */
rates: Object.freeze({
textInput: 0.6,