mirror of
https://github.com/bilawalsidhu/gods-eye-view.git
synced 2026-11-11 22:02:21 +00:00
Merge upstream/main into fix/drone-airspace
Move the share-link token to 6 (3 is gnss-integrity), build altitude ranges from LOWER/UPPER VAL, UOM and CODE, mark card and legend strings for translation, rename the layer to Prohibited & Restricted Airspace, move the changelog entry to changelog.d, and keep the layer voice-off while the Realtime prompt is at its budget.
This commit is contained in:
1387 files changed
+160157
-23954
No files matched your search
@@ -178,6 +178,9 @@ const schemas = [
|
||||
'global-context-panel',
|
||||
'weather-panel',
|
||||
'recent-imagery-panel',
|
||||
'voice-settings',
|
||||
'voice-preview',
|
||||
'provider-settings',
|
||||
],
|
||||
},
|
||||
open: {
|
||||
@@ -187,6 +190,15 @@ const schemas = [
|
||||
required: ['panelId', 'open'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'set_voice_inactivity_timeout',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: { minutes: { type: 'integer', minimum: 0, maximum: 60 } },
|
||||
required: ['minutes'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'set_context_mode',
|
||||
parameters: {
|
||||
@@ -223,6 +235,7 @@ const schemas = [
|
||||
aircraftClass: {
|
||||
type: 'string',
|
||||
},
|
||||
voicePanel: { type: 'string', enum: ['preview', 'settings'] },
|
||||
},
|
||||
required: ['action'],
|
||||
},
|
||||
@@ -729,7 +742,7 @@ const schemas = [
|
||||
properties: {
|
||||
motion: {
|
||||
type: 'string',
|
||||
enum: ['orbit', 'pan', 'tilt', 'rotate', 'stop'],
|
||||
enum: ['orbit', 'pan', 'tilt', 'rotate', 'rotate_globe', 'stop'],
|
||||
},
|
||||
direction: {
|
||||
type: 'string',
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { realtimeInstructions } from '../../server/providers/openai/instructions.js';
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createHash } from 'node:crypto';
|
||||
@@ -31,14 +32,20 @@ test('the complete Realtime tool payload pins the manifest-generated layer relea
|
||||
.digest('hex');
|
||||
assert.equal(
|
||||
digest,
|
||||
// Re-derived for the voice layer manifest (generated layer enums and
|
||||
// Re-derived for Provider Settings and explicit Visual Presets/Scenes routing,
|
||||
// plus the deliberate direct Panel tool/target additions and voice layer manifest (generated layer enums and
|
||||
// aliases), point-and-ask (pointer sentinels, referent args) and the
|
||||
// consolidated tool wording (each policy stated once); model-facing tools
|
||||
// omit top-level anyOf (OpenAI rejects it). Contacts list routing preserves
|
||||
// requested radius and center authority in descriptions only.
|
||||
// Street Level adds its toggle enum values and generated alias hint.
|
||||
// Drone Restrictions adds toggle/query enum values and spoken aliases.
|
||||
'3ceef7975c1437eacb2c602bf9728829b299532b7dd95534cf7d9eb47115fa54',
|
||||
// Street Level and GNSS extend the manifest; tracked and analyst aircraft describe broadcasts.
|
||||
// Air quality (ECCC AQHI) extends the manifest.
|
||||
// Globe rotation adds one motion; visual keyhole fit and explicit
|
||||
// reset guidance/defaults update only move_camera and mode descriptions.
|
||||
// S2 GhostMaps extends the generated layer enums.
|
||||
// Panel adds named voice targets and timeout; shared Panel/Cockpit guidance
|
||||
// is consolidated without changing either feature's arguments.
|
||||
'9ee82c4214499280c208c93751b19d7b724c81b7727a2d0c6dab0184749fbe15',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -96,19 +103,43 @@ test('metadata cannot add tools, fields, types or enum values', () => {
|
||||
|
||||
test('all legacy action arguments are byte-identical after removing the deliberate additions', () => {
|
||||
const legacy = structuredClone(GEV_ACTION_SCHEMAS).filter(
|
||||
(tool) => !['next_satellite_pass', 'set_cyber_sonar'].includes(tool.name),
|
||||
(tool) =>
|
||||
![
|
||||
'next_satellite_pass',
|
||||
'set_cyber_sonar',
|
||||
'set_voice_inactivity_timeout',
|
||||
].includes(tool.name),
|
||||
);
|
||||
// Layer enums are generated from the voice layer manifest and pinned by
|
||||
// layerManifest.test.mjs; the two shipped right-rail panels are additive.
|
||||
const property = (name) =>
|
||||
legacy.find((tool) => tool.name === name).parameters.properties;
|
||||
delete property('control_cockpit').voicePanel;
|
||||
delete property('set_layer_visibility').layerId.enum;
|
||||
delete property('show_data_layers_menu').layerId.enum;
|
||||
delete property('get_entity_context').layerId.enum;
|
||||
delete property('analyst_query').layers.items.enum;
|
||||
// Earth-centered rotation is additive; local camera yaw keeps its contract.
|
||||
const motion = property('move_camera').motion;
|
||||
assert.deepEqual(motion.enum, [
|
||||
'orbit',
|
||||
'pan',
|
||||
'tilt',
|
||||
'rotate',
|
||||
'rotate_globe',
|
||||
'stop',
|
||||
]);
|
||||
motion.enum = motion.enum.filter((value) => value !== 'rotate_globe');
|
||||
const panels = property('set_panel_open').panelId;
|
||||
panels.enum = panels.enum.filter(
|
||||
(id) => !['weather-panel', 'recent-imagery-panel'].includes(id),
|
||||
(id) =>
|
||||
![
|
||||
'weather-panel',
|
||||
'recent-imagery-panel',
|
||||
'voice-settings',
|
||||
'voice-preview',
|
||||
'provider-settings',
|
||||
].includes(id),
|
||||
);
|
||||
for (const tool of legacy) {
|
||||
// Point-and-ask adds `referent` arguments and 'pointer' enum values.
|
||||
@@ -145,6 +176,24 @@ test('all legacy action arguments are byte-identical after removing the delibera
|
||||
);
|
||||
});
|
||||
|
||||
test('globe rotation is an additive camera motion with the existing control arguments', () => {
|
||||
const schema = GEV_ACTION_SCHEMAS.find(
|
||||
(tool) => tool.name === 'move_camera',
|
||||
).parameters;
|
||||
for (const motion of ['rotate_globe', 'rotate', 'orbit', 'pan', 'tilt']) {
|
||||
assert.deepEqual(
|
||||
validateArgs(schema, {
|
||||
motion,
|
||||
direction: 'right',
|
||||
speed: 'slow',
|
||||
mode: 'continuous',
|
||||
}),
|
||||
[],
|
||||
);
|
||||
}
|
||||
assert.ok(validateArgs(schema, { motion: 'spin_camera' }).length > 0);
|
||||
});
|
||||
|
||||
test('track_entity accepts a referent alone but rejects an empty target', () => {
|
||||
const schema = GEV_ACTION_SCHEMAS.find(
|
||||
(tool) => tool.name === 'track_entity',
|
||||
@@ -183,3 +232,25 @@ test('the benchmark still rejects an empty track_entity although the model-facin
|
||||
[],
|
||||
);
|
||||
});
|
||||
|
||||
test('globe reset guidance requires a separate explicit command after refusal', () => {
|
||||
const paragraph = realtimeInstructions()
|
||||
.split('\n')
|
||||
.find((line) => line.includes('globe-view-required'));
|
||||
assert.ok(paragraph);
|
||||
assert.match(paragraph, /calls only move_camera motion rotate_globe/);
|
||||
assert.match(paragraph, /explicit user Reset\/Show Globe View/);
|
||||
assert.match(paragraph, /never from your own plan/);
|
||||
assert.match(paragraph, /bare yes\/ok/);
|
||||
assert.match(paragraph, /Do not ask a question/);
|
||||
});
|
||||
|
||||
test('direct voice timeout schema accepts None and exact integers, rejects malformed duration', () => {
|
||||
const tool = GEV_REALTIME_TOOLS.find(
|
||||
(tool) => tool.name === 'set_voice_inactivity_timeout',
|
||||
);
|
||||
for (const minutes of [0, 1, 5, 60])
|
||||
assert.equal(validateArgs(tool.parameters, { minutes }).length, 0);
|
||||
for (const minutes of [-1, 61, 1.5, '5'])
|
||||
assert.ok(validateArgs(tool.parameters, { minutes }).length > 0);
|
||||
});
|
||||
@@ -0,0 +1,102 @@
|
||||
// The analyst card shows its headline, scope, chips and caveats in the UI
|
||||
// language; the spoken headline stays English for the model.
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { setLocale } from '../i18n/index.js';
|
||||
import { attachVoiceResult as withSpeech } from './speech.js';
|
||||
import {
|
||||
initialVoiceCardState,
|
||||
reduceVoiceCard,
|
||||
voiceCardView,
|
||||
} from './voiceCardPresentation.js';
|
||||
|
||||
function cardFor(name, result, args = {}) {
|
||||
const shaped = withSpeech(name, result, args);
|
||||
return {
|
||||
shaped,
|
||||
view: voiceCardView(
|
||||
[
|
||||
{ type: 'action-call', callId: 'c1', name, arguments: args },
|
||||
{ type: 'action-result', callId: 'c1', name, result: shaped },
|
||||
].reduce(reduceVoiceCard, initialVoiceCardState()),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const analyst = {
|
||||
ok: true,
|
||||
count: 3,
|
||||
complete: true,
|
||||
scopeLabel: 'anywhere in the loaded data',
|
||||
partial: true,
|
||||
feedState: 'degraded',
|
||||
unanswered: ['earthquakes'],
|
||||
coverage: { layersQueried: [{ layerKey: 'flights', source: 'OpenSky' }] },
|
||||
display: { scope: 'anywhere in the loaded data' },
|
||||
items: [],
|
||||
};
|
||||
|
||||
const radius = {
|
||||
...analyst,
|
||||
count: 1234,
|
||||
complete: false,
|
||||
partial: false,
|
||||
feedState: 'nominal',
|
||||
unanswered: [],
|
||||
scopeLabel: 'in view',
|
||||
display: {
|
||||
scope: 'within 42 km of the view centre',
|
||||
window: 'last 24 h',
|
||||
caveat:
|
||||
'counted 1,000 of 2,345 loaded records; earthquakes loading; flights: distance is ground distance, not overhead',
|
||||
},
|
||||
coverage: {
|
||||
layersQueried: [{ layerKey: 'flights', source: 'OpenSky' }],
|
||||
note: 'the flights layer loads by viewport',
|
||||
},
|
||||
};
|
||||
|
||||
test('analyst card in es has no English text', () => {
|
||||
setLocale('es');
|
||||
try {
|
||||
const { view } = cardFor('analyst_query', analyst);
|
||||
assert.doesNotMatch(view.result.title, /anywhere in the loaded data/);
|
||||
assert.ok(!view.result.chips.includes('partial'));
|
||||
assert.ok(!view.result.chips.includes('degraded'));
|
||||
assert.ok(!view.result.notes.some((n) => n.startsWith('Not answered')));
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
|
||||
test('analyst scope, window and caveats in ru', () => {
|
||||
setLocale('ru');
|
||||
try {
|
||||
const { shaped, view } = cardFor('analyst_query', radius);
|
||||
assert.match(shaped.say, /^At least 1,234 aircraft in view/);
|
||||
assert.doesNotMatch(
|
||||
JSON.stringify(view.result),
|
||||
/aircraft|within|view centre|last 24 h|counted|loading|ground distance|loads by viewport/,
|
||||
);
|
||||
assert.match(view.result.title, /^1\s234 /);
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
|
||||
test('en: the card keeps its English text and names layers', () => {
|
||||
const { shaped, view } = cardFor('analyst_query', analyst);
|
||||
assert.equal(view.result.title, '3 aircraft anywhere in the loaded data');
|
||||
assert.deepEqual(view.result.chips, ['degraded', 'partial']);
|
||||
assert.ok(view.result.notes.includes('Not answered: Earthquakes (24h)'));
|
||||
// The model still gets the layer key.
|
||||
assert.match(shaped.say, /earthquakes not answered/);
|
||||
const capped = cardFor('analyst_query', radius).view.result;
|
||||
assert.equal(capped.title, 'At least 1,234 aircraft in view');
|
||||
assert.deepEqual(capped.lines, [
|
||||
'within 42 km of the view centre',
|
||||
'last 24 h',
|
||||
]);
|
||||
assert.ok(capped.notes.includes('counted 1,000 of 2,345 loaded records'));
|
||||
assert.ok(capped.notes.includes('the flights layer loads by viewport'));
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
// The analyst card's empty notes can be a layer's status line: Local ADS-B
|
||||
// with every decoder feed stale, Transit with a quiet feed ("(95s old)") or
|
||||
// its getStats().coverage text. The card shows them in the UI language; the
|
||||
// model gets the English.
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { setLocale } from '../i18n/index.js';
|
||||
import { attachVoiceResult } from './speech.js';
|
||||
import {
|
||||
initialVoiceCardState,
|
||||
reduceVoiceCard,
|
||||
voiceCardView,
|
||||
} from './voiceCardPresentation.js';
|
||||
|
||||
function viewFor(layer, caveat) {
|
||||
const shaped = attachVoiceResult(
|
||||
'analyst_query',
|
||||
{
|
||||
ok: true,
|
||||
count: 0,
|
||||
complete: true,
|
||||
scopeLabel: 'in view',
|
||||
feedState: 'nominal',
|
||||
unanswered: [],
|
||||
coverage: { layersQueried: [{ ...layer, status: 'empty' }] },
|
||||
display: { scope: 'in view', caveat },
|
||||
items: [],
|
||||
},
|
||||
{},
|
||||
);
|
||||
return voiceCardView(
|
||||
[
|
||||
{
|
||||
type: 'action-call',
|
||||
callId: 'c1',
|
||||
name: 'analyst_query',
|
||||
arguments: {},
|
||||
},
|
||||
{
|
||||
type: 'action-result',
|
||||
callId: 'c1',
|
||||
name: 'analyst_query',
|
||||
result: shaped,
|
||||
},
|
||||
].reduce(reduceVoiceCard, initialVoiceCardState()),
|
||||
);
|
||||
}
|
||||
|
||||
function notesFor(layerKey, caveat) {
|
||||
return viewFor({ layerKey }, caveat).result.notes;
|
||||
}
|
||||
|
||||
function inLocale(locale, run) {
|
||||
setLocale(locale);
|
||||
try {
|
||||
run();
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
}
|
||||
|
||||
// Caveat = `${layerKey}: ${getEmptyNote(layerKey)}` (data/analystEngine.js;
|
||||
// gevActions.js getEmptyNote → stats.coverage || stats.statusMessage).
|
||||
const cases = [
|
||||
['local-adsb', 'local-adsb: feeds 1090, 978 stale · 0 heard'],
|
||||
['local-adsb', 'local-adsb: feed 1090 stale · 0 heard'],
|
||||
['transit', 'transit: TTC 0 (95s old)'],
|
||||
['transit', 'transit: TTC 0 · GO Transit 0 (95s old)'],
|
||||
];
|
||||
|
||||
// Transit's coverage text when nothing is in view (transit/queries.js).
|
||||
const transitCoverage = [
|
||||
'transit: Fly below 3,000 km to a covered region',
|
||||
'transit: No feed here yet · 12 regions available',
|
||||
];
|
||||
|
||||
for (const locale of ['es', 'ru']) {
|
||||
for (const [layerKey, caveat] of cases) {
|
||||
test(`${locale}: analyst note "${caveat}" is translated`, () => {
|
||||
inLocale(locale, () => {
|
||||
const notes = notesFor(layerKey, caveat);
|
||||
assert.ok(notes.length > 0, 'the note is shown');
|
||||
assert.doesNotMatch(notes.join(' | '), /\b(stale|heard|old)\b/);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test(`${locale}: Transit's empty note is translated on the analyst card`, () => {
|
||||
inLocale(locale, () => {
|
||||
for (const caveat of transitCoverage) {
|
||||
const view = viewFor(
|
||||
{ layerKey: 'transit', source: 'GTFS-RT' },
|
||||
caveat,
|
||||
);
|
||||
const text = JSON.stringify(view.result);
|
||||
assert.doesNotMatch(
|
||||
text,
|
||||
/Fly below|No feed here yet|regions available/,
|
||||
text,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
for (const [layerKey, caveat] of cases) {
|
||||
test(`en: analyst note "${caveat}" keeps its status line`, () => {
|
||||
setLocale('en');
|
||||
const line = caveat.slice(caveat.indexOf(': ') + 2);
|
||||
assert.ok(
|
||||
notesFor(layerKey, caveat).some((note) => note.endsWith(`: ${line}`)),
|
||||
);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
import { after, before, test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { DataLayerManager } from '../data/manager.js';
|
||||
import { CCTV_FOCUS_RESULT } from '../layers/cctv/index.js';
|
||||
import { controlCctv, createGevActionRunner } from './gevActions.js';
|
||||
|
||||
const originalWindow = globalThis.window;
|
||||
before(() => {
|
||||
globalThis.window ??= { clearTimeout, setTimeout, requestIdleCallback: null };
|
||||
});
|
||||
after(() => {
|
||||
if (originalWindow === undefined) delete globalThis.window;
|
||||
else globalThis.window = originalWindow;
|
||||
});
|
||||
|
||||
function viewerFixture() {
|
||||
return {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } },
|
||||
};
|
||||
}
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
const promise = new Promise((done) => {
|
||||
resolve = done;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function runnerFixture(t, dataManager, options = {}) {
|
||||
const runner = createGevActionRunner({
|
||||
viewer: viewerFixture(),
|
||||
styleManager: {},
|
||||
dataManager,
|
||||
...options,
|
||||
});
|
||||
t.after(() => runner.dispose());
|
||||
return runner;
|
||||
}
|
||||
|
||||
// Only the inert module's lifecycle completion is controlled. The manager's
|
||||
// intent queue, cancellation, compensation, events, and runner are production.
|
||||
async function lifecycleFixture(t, targetEnabled) {
|
||||
const viewer = viewerFixture();
|
||||
const manager = new DataLayerManager(viewer);
|
||||
const calls = [];
|
||||
const gates = [];
|
||||
const pending = [];
|
||||
let nextGate = null;
|
||||
let moduleEnabled = false;
|
||||
const phase = targetEnabled ? 'enable' : 'disable';
|
||||
const lifecycle = async (name, options = {}) => {
|
||||
calls.push(name);
|
||||
if (name === phase && nextGate) {
|
||||
const gate = nextGate;
|
||||
nextGate = null;
|
||||
gate.entered.resolve(options.signal);
|
||||
await gate.release.promise;
|
||||
if (gate.outcome instanceof Error) throw gate.outcome;
|
||||
if (gate.outcome === false) return false;
|
||||
}
|
||||
moduleEnabled = name === 'enable';
|
||||
return true;
|
||||
};
|
||||
manager.register({
|
||||
id: 'cctv',
|
||||
name: 'CCTV',
|
||||
source: 'test',
|
||||
updateInterval: -1,
|
||||
init: async () => true,
|
||||
enable: (_viewer, options) => lifecycle('enable', options),
|
||||
disable: (_viewer, options) => lifecycle('disable', options),
|
||||
update: async () => true,
|
||||
getStats: () => ({ count: 0, lastUpdate: 0 }),
|
||||
});
|
||||
if (!targetEnabled)
|
||||
assert.equal(await manager.setEnabled('cctv', true), true);
|
||||
calls.length = 0;
|
||||
const events = [];
|
||||
const unsubscribe = manager.subscribe((event) => events.push(event));
|
||||
const runner = runnerFixture(t, manager, { viewer });
|
||||
t.after(async () => {
|
||||
for (const gate of gates) gate.release.resolve();
|
||||
await Promise.allSettled(pending);
|
||||
unsubscribe();
|
||||
await manager.destroyAll();
|
||||
});
|
||||
return {
|
||||
manager,
|
||||
events,
|
||||
calls,
|
||||
get moduleEnabled() {
|
||||
return moduleEnabled;
|
||||
},
|
||||
gate(outcome = true) {
|
||||
const gate = { entered: deferred(), release: deferred(), outcome };
|
||||
gates.push(gate);
|
||||
nextGate = gate;
|
||||
return gate;
|
||||
},
|
||||
run(action, options = {}) {
|
||||
const work = runner('control_cctv', { action }, options);
|
||||
pending.push(work);
|
||||
return work;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function assertLifecycle(result, manager, enabled, uncertain = false) {
|
||||
assert.equal(result.action, 'control_cctv');
|
||||
assert.equal(result.layerId, 'cctv');
|
||||
assert.equal(result.enabled, enabled);
|
||||
assert.equal(result.lifecycleState, enabled ? 'enabled' : 'disabled');
|
||||
assert.equal(result.lifecycleUncertain, uncertain);
|
||||
assert.deepEqual(manager.getLayerLifecycleState('cctv'), {
|
||||
enabled,
|
||||
lifecycleState: result.lifecycleState,
|
||||
uncertain,
|
||||
});
|
||||
}
|
||||
|
||||
const terminalTypes = new Set([
|
||||
'visibility',
|
||||
'visibility-cancelled',
|
||||
'visibility-failed',
|
||||
'visibility-blocked',
|
||||
]);
|
||||
|
||||
for (const action of ['enable', 'disable']) {
|
||||
const targetEnabled = action === 'enable';
|
||||
for (const outcome of [
|
||||
'success',
|
||||
'false',
|
||||
'throw',
|
||||
'caller-abort',
|
||||
'resource-abort',
|
||||
'late-abort',
|
||||
]) {
|
||||
test(
|
||||
`CCTV ${action}: ${outcome} preserves the manager outcome`,
|
||||
{ timeout: 2000 },
|
||||
async (t) => {
|
||||
const h = await lifecycleFixture(t, targetEnabled);
|
||||
const controller = new AbortController();
|
||||
const failure =
|
||||
outcome === 'resource-abort'
|
||||
? Object.assign(new Error('fixture resource ended'), {
|
||||
name: 'AbortError',
|
||||
})
|
||||
: outcome === 'throw'
|
||||
? new Error('fixture lifecycle failed')
|
||||
: null;
|
||||
const gate = h.gate(failure || (outcome === 'false' ? false : true));
|
||||
let lateAbortObserved = false;
|
||||
if (outcome === 'late-abort') {
|
||||
t.after(
|
||||
h.manager.subscribe((event) => {
|
||||
if (event.type === 'visibility' && event.origin === 'voice') {
|
||||
lateAbortObserved = true;
|
||||
controller.abort();
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
let settled = false;
|
||||
// Uppercase exercises the existing case-insensitive action contract.
|
||||
const work = h.run(
|
||||
outcome === 'success' ? action.toUpperCase() : action,
|
||||
{
|
||||
signal: controller.signal,
|
||||
isCurrent: () => !controller.signal.aborted,
|
||||
},
|
||||
);
|
||||
void work.then(
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
);
|
||||
const moduleSignal = await gate.entered.promise;
|
||||
assert.equal(
|
||||
settled,
|
||||
false,
|
||||
'the tool must wait for lifecycle completion',
|
||||
);
|
||||
if (outcome === 'caller-abort') {
|
||||
controller.abort();
|
||||
assert.ok(
|
||||
moduleSignal,
|
||||
'the manager must receive a lifecycle signal',
|
||||
);
|
||||
assert.equal(
|
||||
moduleSignal.aborted,
|
||||
true,
|
||||
'caller abort must reach the manager lifecycle',
|
||||
);
|
||||
}
|
||||
gate.release.resolve();
|
||||
const result = await work;
|
||||
const committed = outcome === 'success' || outcome === 'late-abort';
|
||||
const cancelled =
|
||||
outcome === 'caller-abort' || outcome === 'resource-abort';
|
||||
assert.equal(result.ok, committed);
|
||||
assert.equal(result.cancelled, cancelled ? true : undefined);
|
||||
assertLifecycle(
|
||||
result,
|
||||
h.manager,
|
||||
committed ? targetEnabled : !targetEnabled,
|
||||
!targetEnabled && (outcome === 'false' || outcome === 'throw'),
|
||||
);
|
||||
assert.equal(
|
||||
h.moduleEnabled,
|
||||
committed ? targetEnabled : !targetEnabled,
|
||||
'the module must match the committed or compensated manager state',
|
||||
);
|
||||
if (cancelled) {
|
||||
assert.equal(result.cancellationReason, outcome);
|
||||
assert.equal(result.phase, action);
|
||||
assert.equal(result.successorIntentEpoch, null);
|
||||
assert.equal(result.successorEnabled, null);
|
||||
assert.equal(result.successorOrigin, null);
|
||||
} else if (!committed) {
|
||||
assert.equal(result.error, `Could not ${action} the requested layer`);
|
||||
}
|
||||
const terminal = h.events.filter(
|
||||
(event) => event.origin === 'voice' && terminalTypes.has(event.type),
|
||||
);
|
||||
assert.equal(
|
||||
terminal.length,
|
||||
1,
|
||||
'each voice intent has one terminal manager event',
|
||||
);
|
||||
assert.equal(
|
||||
terminal[0].type,
|
||||
committed
|
||||
? 'visibility'
|
||||
: cancelled
|
||||
? 'visibility-cancelled'
|
||||
: 'visibility-failed',
|
||||
);
|
||||
if (outcome === 'resource-abort')
|
||||
assert.equal(controller.signal.aborted, false);
|
||||
if (outcome === 'late-abort') assert.equal(lateAbortObserved, true);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test(
|
||||
`CCTV ${action}: already-aborted caller cannot enter lifecycle work`,
|
||||
{ timeout: 2000 },
|
||||
async (t) => {
|
||||
const h = await lifecycleFixture(t, targetEnabled);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const result = await h.run(action, { signal: controller.signal });
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(result.cancelled, true);
|
||||
assert.equal(result.cancellationReason, 'caller-abort');
|
||||
assert.equal(result.phase, 'queued');
|
||||
assertLifecycle(result, h.manager, !targetEnabled);
|
||||
assert.equal(h.moduleEnabled, !targetEnabled);
|
||||
assert.deepEqual(h.calls, []);
|
||||
assert.equal(
|
||||
h.events.filter((event) => terminalTypes.has(event.type)).length,
|
||||
1,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test(
|
||||
'CCTV toggles retain missing-module errors without calling visibility',
|
||||
{ timeout: 2000 },
|
||||
async (t) => {
|
||||
for (const layers of [new Map(), new Map([['cctv', {}]])]) {
|
||||
const runner = runnerFixture(t, {
|
||||
layers,
|
||||
getAll: () => [],
|
||||
setEnabled: () =>
|
||||
assert.fail('missing CCTV must not dispatch visibility'),
|
||||
});
|
||||
for (const action of ['enable', 'disable']) {
|
||||
assert.deepEqual(await runner('control_cctv', { action }), {
|
||||
ok: false,
|
||||
action: 'control_cctv',
|
||||
error: 'CCTV layer unavailable',
|
||||
});
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test('exported controlCctv refuses toggles without calling the manager', async () => {
|
||||
const manager = {
|
||||
layers: new Map([['cctv', { module: {} }]]),
|
||||
setEnabled: () => assert.fail('direct helper must not change visibility'),
|
||||
isEnabled: () => assert.fail('direct toggle must not read enabled state'),
|
||||
};
|
||||
for (const action of ['ENABLE', 'disable']) {
|
||||
const result = await controlCctv(manager, { action });
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(result.action, 'control_cctv');
|
||||
assert.match(result.error, /set_layer_visibility/);
|
||||
}
|
||||
});
|
||||
|
||||
test(
|
||||
'CCTV fallback managers report rejection, transitional state, and uncertainty',
|
||||
{ timeout: 2000 },
|
||||
async (t) => {
|
||||
for (const enabled of [true, false]) {
|
||||
const action = enabled ? 'enable' : 'disable';
|
||||
for (const mode of [
|
||||
'rejection',
|
||||
'mismatch',
|
||||
'transitional',
|
||||
'uncertain',
|
||||
]) {
|
||||
const actualEnabled =
|
||||
mode === 'rejection' || mode === 'mismatch' ? !enabled : enabled;
|
||||
const lifecycleState =
|
||||
mode === 'transitional'
|
||||
? enabled
|
||||
? 'enabling'
|
||||
: 'disabling'
|
||||
: actualEnabled
|
||||
? 'enabled'
|
||||
: 'disabled';
|
||||
const manager = {
|
||||
layers: new Map([['cctv', { module: {} }]]),
|
||||
getAll: () => [{ id: 'cctv', name: 'CCTV' }],
|
||||
isEnabled: () => actualEnabled,
|
||||
getLayerLifecycleState: () => ({
|
||||
enabled: actualEnabled,
|
||||
lifecycleState,
|
||||
uncertain: mode === 'uncertain',
|
||||
}),
|
||||
async setEnabled() {
|
||||
if (mode === 'rejection') throw new Error('adapter rejected');
|
||||
return true;
|
||||
},
|
||||
};
|
||||
const runner = runnerFixture(t, manager);
|
||||
const result = await runner('control_cctv', { action });
|
||||
assert.equal(result.ok, false, `${action}: ${mode}`);
|
||||
assert.equal(result.action, 'control_cctv');
|
||||
assert.equal(result.enabled, actualEnabled);
|
||||
assert.equal(result.lifecycleState, lifecycleState);
|
||||
assert.equal(result.lifecycleUncertain, mode === 'uncertain');
|
||||
assert.equal(
|
||||
result.error,
|
||||
mode === 'rejection'
|
||||
? 'adapter rejected'
|
||||
: `Could not ${action} the requested layer`,
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'CCTV visibility composes only its outer speech envelope',
|
||||
{ timeout: 2000 },
|
||||
async (t) => {
|
||||
let enabled = false;
|
||||
const calls = [];
|
||||
const runner = runnerFixture(
|
||||
t,
|
||||
{
|
||||
layers: new Map([['cctv', { module: {} }]]),
|
||||
getAll: () => [{ id: 'cctv', name: 'CCTV' }],
|
||||
isEnabled: () => enabled,
|
||||
async setEnabled(_id, value) {
|
||||
enabled = value;
|
||||
return true;
|
||||
},
|
||||
},
|
||||
{
|
||||
speechBuilders: {
|
||||
control_cctv(result, args) {
|
||||
calls.push(['cctv', result.action, args.action]);
|
||||
return {
|
||||
say: 'CCTV response',
|
||||
display: { title: 'CCTV', lines: [] },
|
||||
};
|
||||
},
|
||||
set_layer_visibility() {
|
||||
calls.push(['generic']);
|
||||
return { say: 'Layer response' };
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
const result = await runner('control_cctv', { action: 'enable' });
|
||||
assert.equal(result.ok, true);
|
||||
assert.equal(result.action, 'control_cctv');
|
||||
assert.equal(result.say, 'CCTV response');
|
||||
assert.deepEqual(result.display, { title: 'CCTV', lines: [] });
|
||||
assert.deepEqual(calls, [['cctv', 'control_cctv', 'enable']]);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'CCTV non-toggle actions retain camera navigation and durable coverage',
|
||||
{ timeout: 2000 },
|
||||
async (t) => {
|
||||
const calls = [];
|
||||
const cameras = [
|
||||
{ id: 'a', name: 'Camera A' },
|
||||
{ id: 'b', name: 'Camera B' },
|
||||
];
|
||||
let activeCameraId = 'a';
|
||||
let coverageMode = 'viewshed';
|
||||
const module = {
|
||||
getUIState: () => ({
|
||||
cameras,
|
||||
activeCameraId,
|
||||
activeCamera: cameras.find((camera) => camera.id === activeCameraId),
|
||||
coverageMode,
|
||||
showCoverage: coverageMode !== 'off',
|
||||
}),
|
||||
selectCamera(id) {
|
||||
calls.push(['select', id]);
|
||||
activeCameraId = id;
|
||||
return true;
|
||||
},
|
||||
cycleCamera(step) {
|
||||
calls.push(['cycle', step]);
|
||||
activeCameraId = step > 0 ? 'b' : 'a';
|
||||
return activeCameraId;
|
||||
},
|
||||
focusNearest(options) {
|
||||
calls.push(['nearest', options]);
|
||||
activeCameraId = 'a';
|
||||
return 'a';
|
||||
},
|
||||
focusCamera(id, duration) {
|
||||
calls.push(['focus', id, duration]);
|
||||
return CCTV_FOCUS_RESULT.TRACKING_HOLDS_VIEW;
|
||||
},
|
||||
};
|
||||
let superseded = 0;
|
||||
const runner = runnerFixture(
|
||||
t,
|
||||
{
|
||||
layers: new Map([['cctv', { module }]]),
|
||||
isEnabled: () => true,
|
||||
setEnabled: () =>
|
||||
assert.fail('non-toggle action must not dispatch visibility'),
|
||||
setLayerParams(id, params, options) {
|
||||
calls.push(['params', id, params, options]);
|
||||
coverageMode = params.coverageMode;
|
||||
},
|
||||
},
|
||||
{
|
||||
styleManager: {
|
||||
supersedeDeferredNavigation() {
|
||||
superseded += 1;
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
for (const args of [
|
||||
{ action: 'select', cameraQuery: 'Camera B' },
|
||||
{ action: 'next' },
|
||||
{ action: 'prev' },
|
||||
{ action: 'nearest' },
|
||||
{ action: 'focus' },
|
||||
]) {
|
||||
const result = await runner('control_cctv', args);
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(result.action, 'control_cctv');
|
||||
assert.match(result.error, /tracking holds the view/);
|
||||
}
|
||||
const coverage = await runner('control_cctv', { action: 'coverage' });
|
||||
assert.equal(coverage.ok, true);
|
||||
assert.equal(coverage.coverageMode, 'off');
|
||||
assert.equal(superseded, 5, 'coverage must not acquire camera navigation');
|
||||
assert.deepEqual(calls, [
|
||||
['select', 'b'],
|
||||
['focus', 'b', 1.8],
|
||||
['cycle', 1],
|
||||
['focus', 'b', 1.8],
|
||||
['cycle', -1],
|
||||
['focus', 'a', 1.8],
|
||||
['nearest', { focus: false }],
|
||||
['focus', 'a', 1.8],
|
||||
['focus', 'a', undefined],
|
||||
['params', 'cctv', { coverageMode: 'off' }, { origin: 'voice' }],
|
||||
]);
|
||||
},
|
||||
);
|
||||
@@ -1,5 +1,5 @@
|
||||
export const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']);
|
||||
export const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key';
|
||||
const CLOUD_VOICE_AUTH_MODES = Object.freeze(['api-key', 'oauth']);
|
||||
const DEFAULT_CLOUD_VOICE_AUTH_MODE = 'api-key';
|
||||
export const CLOUD_VOICE_AUTH_STORAGE_KEY = 'godsEyeView.voice.cloudAuth';
|
||||
|
||||
function storageRef(storage) {
|
||||
@@ -11,7 +11,7 @@ function storageRef(storage) {
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeCloudVoiceAuthMode(mode) {
|
||||
function normalizeCloudVoiceAuthMode(mode) {
|
||||
const normalized = String(mode || '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
||||
+46
-5
@@ -1,10 +1,51 @@
|
||||
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
|
||||
import { createRealtimeSession } from './realtimeSession.js';
|
||||
import { createGeminiSession } from './geminiSession.js';
|
||||
import { createProviderCommands } from './providerCommands.js';
|
||||
import { createVoicePanelHost } from './panel/host.js';
|
||||
|
||||
/** Default composition; callers may supply another session adapter factory. */
|
||||
export function createVoiceCommands(options) {
|
||||
return bindVoiceCommands({
|
||||
createSession: createRealtimeSession,
|
||||
...options,
|
||||
});
|
||||
export function createVoiceCommands(options = {}) {
|
||||
const panelHost =
|
||||
options.panel === true
|
||||
? createVoicePanelHost({
|
||||
storage: options.panelStorage,
|
||||
clock: options.panelClock,
|
||||
readCockpitState: options.styleManager?.getCockpitState
|
||||
? () => options.styleManager.getCockpitState()
|
||||
: undefined,
|
||||
})
|
||||
: null;
|
||||
const bind = (settings) =>
|
||||
bindVoiceCommands({
|
||||
...settings,
|
||||
...(panelHost
|
||||
? {
|
||||
panelHost,
|
||||
nativePanelProvider: settings.createController
|
||||
? null
|
||||
: settings.createSession === createRealtimeSession
|
||||
? 'openai'
|
||||
: settings.createSession === createGeminiSession
|
||||
? 'gemini'
|
||||
: null,
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
let commands;
|
||||
// Generic hosts retain the existing default; standalone opts in explicitly.
|
||||
if (!options.createSession && options.supportedProviders !== undefined)
|
||||
commands = createProviderCommands(options, {
|
||||
bind,
|
||||
factories: { openai: createRealtimeSession, gemini: createGeminiSession },
|
||||
});
|
||||
else commands = bind({ createSession: createRealtimeSession, ...options });
|
||||
if (panelHost) {
|
||||
const lifetime = commands.signal ?? commands.session?.signal;
|
||||
lifetime?.addEventListener('abort', () => panelHost.destroy(), {
|
||||
once: true,
|
||||
});
|
||||
if (lifetime?.aborted) panelHost.destroy();
|
||||
}
|
||||
return commands;
|
||||
}
|
||||
+130
-26
@@ -1,5 +1,33 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
|
||||
// The assistant's name on its card. i18n-ignore: product name
|
||||
const ASSISTANT_NAME = 'GEV';
|
||||
|
||||
/**
|
||||
* Short state label for the narrow voice heading. The `voice` context lets
|
||||
* languages use shorter words; unknown states show upper-cased.
|
||||
* @param {string} status machine state (dataset.status)
|
||||
* @returns {string}
|
||||
*/
|
||||
export function voiceStatusLabel(status) {
|
||||
switch (status) {
|
||||
case 'idle':
|
||||
return t('OFF', { context: 'voice' });
|
||||
case 'connecting':
|
||||
return t('CONNECTING', { context: 'voice' });
|
||||
case 'listening':
|
||||
return t('LISTENING', { context: 'voice' });
|
||||
case 'executing':
|
||||
return t('EXECUTING', { context: 'voice' });
|
||||
case 'error':
|
||||
return t('ERROR', { context: 'voice' });
|
||||
default:
|
||||
return String(status || '').toUpperCase();
|
||||
}
|
||||
}
|
||||
|
||||
/** Build the voice control independently of its connection backend. */
|
||||
export function createVoiceControl({ reset = false } = {}) {
|
||||
export function createVoiceControl({ reset = false, panel = false } = {}) {
|
||||
let root = document.getElementById('gev-voice-control');
|
||||
if (root && reset) {
|
||||
root.remove();
|
||||
@@ -11,57 +39,116 @@ export function createVoiceControl({ reset = false } = {}) {
|
||||
root.dataset.status = 'idle';
|
||||
root.dataset.speaker = 'idle';
|
||||
root.innerHTML = `
|
||||
<label id="gev-voice-provider-field" class="gev-voice-provider-field" hidden>
|
||||
<span>${t('Voice provider')}</span>
|
||||
<select id="gev-voice-provider" aria-label="${t('Voice provider')}" title="${t('Changing provider reconnects voice if it is on.')}"></select>
|
||||
</label>
|
||||
${panel ? `<button id="gev-voice-settings-button" class="gev-voice-settings-button" type="button" aria-label="${t('Voice Settings')}" aria-controls="gev-voice-panel" aria-expanded="false"><span class="material-symbols-outlined gev-voice-settings-icon" aria-hidden="true">settings</span></button>` : ''}
|
||||
<div class="gev-voice-heading">
|
||||
<div class="gev-voice-kicker">AI AGENT</div>
|
||||
<div id="gev-voice-status">OFF</div>
|
||||
<div class="gev-voice-cost">
|
||||
<button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="Voice model tier — applies next session">STD</button>
|
||||
<span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="Estimated session cost">~$0.00</span>
|
||||
</div>
|
||||
<div class="gev-voice-kicker">${t('AI AGENT')}</div>
|
||||
<div id="gev-voice-status">${voiceStatusLabel('idle')}</div>
|
||||
${
|
||||
panel
|
||||
? `<span id="gev-voice-cost-warning" class="gev-voice-cost-value gev-voice-cost-warning" hidden role="status" aria-live="polite"></span>`
|
||||
: `<div class="gev-voice-cost">
|
||||
<button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="${t('Voice model tier — applies next session')}">${t('STD')}</button>
|
||||
<span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="${t('Estimated session cost')}">~$0.00</span>
|
||||
</div>`
|
||||
}
|
||||
</div>
|
||||
<button id="gev-voice-button" type="button" aria-label="Voice control — activate to toggle voice; hold Space to speak" aria-describedby="gev-voice-help">
|
||||
<button id="gev-voice-button" type="button" aria-label="${t('Voice control — activate to toggle voice; hold Space to speak')}" aria-describedby="gev-voice-help">
|
||||
<span class="gev-mic-orbit"><img src="/mic.svg" alt="" /></span>
|
||||
<span class="gev-mic-label">ON/OFF</span>
|
||||
<span class="gev-mic-label">${t('ON/OFF')}</span>
|
||||
</button>
|
||||
<div class="gev-voice-visualizer" aria-hidden="true">
|
||||
${Array.from({ length: 15 }, (_, index) => `<span style="--bar:${index}"></span>`).join('')}
|
||||
</div>
|
||||
<div class="gev-voice-readout">
|
||||
<div id="gev-voice-detail">VOICE STANDBY</div>
|
||||
<div id="gev-voice-detail">${t('VOICE STANDBY')}</div>
|
||||
</div>
|
||||
<div id="gev-voice-help" class="gev-voice-help-tray" role="tooltip">
|
||||
<span class="gev-voice-help-kicker">VOICE CONTROL</span>
|
||||
<span class="gev-voice-help-detail">Hold Space to speak · tap Space to activate focused controls</span>
|
||||
<span class="gev-voice-help-kicker">${t('VOICE CONTROL')}</span>
|
||||
<span class="gev-voice-help-detail">${t('Hold Space to speak · tap Space to activate focused controls')}</span>
|
||||
</div>
|
||||
<section id="gev-voice-card" class="gev-voice-card" hidden aria-label="Voice assistant">
|
||||
<div class="gev-voice-card-header">
|
||||
<span class="gev-voice-card-kicker">VOICE</span>
|
||||
<span id="gev-voice-card-phase" class="gev-voice-card-phase"></span>
|
||||
<button id="gev-voice-card-dismiss" class="gev-voice-card-dismiss" type="button" aria-label="Dismiss voice card">×</button>
|
||||
${
|
||||
panel
|
||||
? `<section id="gev-voice-panel" class="gev-voice-panel" hidden aria-label="${t('Voice Settings')}">
|
||||
<div class="gev-voice-panel-header">
|
||||
<span class="gev-voice-panel-title">${t('VOICE')}</span>
|
||||
<div class="gev-voice-panel-tabs" role="tablist" aria-label="${t('Voice Settings')}">
|
||||
<button id="gev-voice-preview-tab" class="gev-voice-panel-tab" type="button" role="tab" aria-label="${t('Preview')}" title="${t('Preview')}" aria-selected="true" aria-controls="gev-voice-preview-panel"><span class="material-symbols-outlined gev-voice-panel-tab-icon" aria-hidden="true">visibility</span></button>
|
||||
<button id="gev-voice-settings-tab" class="gev-voice-panel-tab" type="button" role="tab" aria-label="${t('Settings')}" title="${t('Settings')}" aria-selected="false" tabindex="-1" aria-controls="gev-voice-settings-panel"><span class="material-symbols-outlined gev-voice-panel-tab-icon" aria-hidden="true">settings</span></button>
|
||||
</div>
|
||||
<button id="gev-voice-settings-close" type="button" aria-label="${t('Close voice panel')}">×</button>
|
||||
</div>
|
||||
<p id="gev-voice-card-user" class="gev-voice-card-caption" data-role="user" hidden><span class="gev-voice-card-who">YOU</span><span id="gev-voice-card-user-text" class="gev-voice-card-text"></span></p>
|
||||
<p id="gev-voice-card-pointer" class="gev-voice-card-pointer" hidden><span id="gev-voice-card-pointer-kind" class="gev-voice-card-who">THIS</span><span class="gev-voice-card-sr">: </span><span id="gev-voice-card-pointer-text" class="gev-voice-card-pointer-label"></span></p>
|
||||
<p id="gev-voice-card-assistant" class="gev-voice-card-caption" data-role="assistant" hidden><span class="gev-voice-card-who">GEV</span><span id="gev-voice-card-assistant-text" class="gev-voice-card-text"></span></p>
|
||||
<ol id="gev-voice-card-plan" class="gev-voice-card-plan" aria-label="Plan" hidden></ol>
|
||||
<div class="gev-voice-panel-body">
|
||||
<section id="gev-voice-preview-panel" role="tabpanel" aria-labelledby="gev-voice-preview-tab">
|
||||
<p id="gev-voice-preview-empty">${t('Voice replies appear here.')}</p>`
|
||||
: ''
|
||||
}
|
||||
<section id="gev-voice-card" class="gev-voice-card" hidden aria-label="${t('Voice assistant')}">
|
||||
<div class="gev-voice-card-header">
|
||||
<span class="gev-voice-card-kicker">${t('VOICE')}</span>
|
||||
<span id="gev-voice-card-phase" class="gev-voice-card-phase"></span>
|
||||
<button id="gev-voice-card-dismiss" class="gev-voice-card-dismiss" type="button" aria-label="${t('Dismiss voice card')}">×</button>
|
||||
</div>
|
||||
<p id="gev-voice-card-user" class="gev-voice-card-caption" data-role="user" hidden><span id="gev-voice-card-user-who" class="gev-voice-card-who">${t('YOU')}</span><span id="gev-voice-card-user-text" class="gev-voice-card-text"></span></p>
|
||||
<p id="gev-voice-card-pointer" class="gev-voice-card-pointer" hidden><span id="gev-voice-card-pointer-kind" class="gev-voice-card-who">${t('THIS')}</span><span class="gev-voice-card-sr">: </span><span id="gev-voice-card-pointer-text" class="gev-voice-card-pointer-label"></span></p>
|
||||
<p id="gev-voice-card-assistant" class="gev-voice-card-caption" data-role="assistant" hidden><span class="gev-voice-card-who">${ASSISTANT_NAME}</span><span id="gev-voice-card-assistant-text" class="gev-voice-card-text"></span></p>
|
||||
<ol id="gev-voice-card-plan" class="gev-voice-card-plan" aria-label="${t('Plan')}" hidden></ol>
|
||||
<div id="gev-voice-card-result" class="gev-voice-card-result" hidden>
|
||||
<div id="gev-voice-card-result-title" class="gev-voice-card-result-title"></div>
|
||||
<div id="gev-voice-card-chips" class="gev-voice-card-chips" hidden></div>
|
||||
<ul id="gev-voice-card-lines" class="gev-voice-card-lines"></ul>
|
||||
<ol id="gev-voice-card-referents" class="gev-voice-card-referents" aria-label="Referenced items" hidden></ol>
|
||||
<ol id="gev-voice-card-referents" class="gev-voice-card-referents" aria-label="${t('Referenced items')}" hidden></ol>
|
||||
<details id="gev-voice-card-notes" class="gev-voice-card-notes" hidden>
|
||||
<summary>NOTES & SOURCES</summary>
|
||||
<summary>${t('NOTES & SOURCES')}</summary>
|
||||
<ul id="gev-voice-card-note-list" class="gev-voice-card-lines"></ul>
|
||||
</details>
|
||||
</div>
|
||||
<div id="gev-voice-card-live" class="gev-voice-card-sr" aria-live="polite" aria-atomic="true"></div>
|
||||
</section>
|
||||
${
|
||||
panel
|
||||
? `</section>
|
||||
<section id="gev-voice-settings-panel" role="tabpanel" aria-labelledby="gev-voice-settings-tab" hidden>
|
||||
<label class="gev-voice-preference gev-voice-preference-toggle"><input id="gev-voice-preview-enabled" type="checkbox" checked aria-describedby="gev-voice-preview-note" /><span>${t('Show Voice Preview')}</span></label>
|
||||
<p id="gev-voice-preview-note" class="gev-voice-settings-note"></p>
|
||||
<div class="gev-voice-timeout-group">
|
||||
<label class="gev-voice-preference gev-voice-setting-title" for="gev-voice-timeout-select">${t('Voice inactivity timeout')}</label>
|
||||
<div class="gev-voice-timeout-controls">
|
||||
<select id="gev-voice-timeout-select" aria-describedby="gev-voice-timeout-note gev-voice-timeout-status">
|
||||
<option value="3">3 ${t('Minutes')}</option><option value="5" selected>5 ${t('Minutes')}</option><option value="10">10 ${t('Minutes')}</option><option value="15">15 ${t('Minutes')}</option>
|
||||
<option value="custom">${t('Custom')}</option><option value="0">${t('None')}</option>
|
||||
</select>
|
||||
<label class="gev-voice-timeout-unit" for="gev-voice-timeout-input">${t('Minutes')}</label>
|
||||
<input id="gev-voice-timeout-input" type="number" min="1" max="60" step="1" value="5" aria-describedby="gev-voice-timeout-note" />
|
||||
</div>
|
||||
<p id="gev-voice-timeout-note" class="gev-voice-settings-note">${t('These preferences stay in this browser.')}</p>
|
||||
<p id="gev-voice-timeout-status" class="gev-voice-settings-note" role="status" aria-live="polite"></p>
|
||||
</div>
|
||||
<div class="gev-voice-cost-settings">
|
||||
<div class="gev-voice-cost-setting-row">
|
||||
<span>${t('Voice model tier — applies next session')}</span>
|
||||
<button id="gev-voice-tier" class="gev-voice-tier-btn" type="button" aria-pressed="false" title="${t('Voice model tier — applies next session')}">${t('STD')}</button>
|
||||
</div>
|
||||
<div class="gev-voice-cost-setting-row">
|
||||
<span>${t('Estimated session cost')}</span>
|
||||
<span id="gev-voice-cost-value" class="gev-voice-cost-value" data-level="ok" title="${t('Estimated session cost')}">~$0.00</span>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</section>`
|
||||
: ''
|
||||
}
|
||||
<div class="gev-voice-error-tray" role="alert" aria-live="assertive">
|
||||
<div class="gev-voice-error-header">
|
||||
<span>VOICE SYSTEM ERROR</span>
|
||||
<button class="gev-voice-error-dismiss" type="button">DISMISS</button>
|
||||
<span>${t('VOICE SYSTEM ERROR')}</span>
|
||||
<button class="gev-voice-error-dismiss" type="button">${t('DISMISS')}</button>
|
||||
</div>
|
||||
<div id="gev-voice-error-detail"></div>
|
||||
<div class="gev-voice-error-hint">Check microphone permission and network access, then try again.</div>
|
||||
<div class="gev-voice-error-hint">${t('Check microphone permission and network access, then try again.')}</div>
|
||||
</div>
|
||||
`;
|
||||
const commandDock = document.getElementById('command-dock');
|
||||
@@ -82,6 +169,22 @@ export function createVoiceControl({ reset = false } = {}) {
|
||||
}
|
||||
return {
|
||||
root,
|
||||
voicePanel: root.querySelector('#gev-voice-panel'),
|
||||
previewPanel: root.querySelector('#gev-voice-preview-panel'),
|
||||
voiceSettingsPanel: root.querySelector('#gev-voice-settings-panel'),
|
||||
previewTab: root.querySelector('#gev-voice-preview-tab'),
|
||||
settingsTab: root.querySelector('#gev-voice-settings-tab'),
|
||||
voiceSettingsButton: root.querySelector('#gev-voice-settings-button'),
|
||||
voiceSettingsClose: root.querySelector('#gev-voice-settings-close'),
|
||||
previewEmpty: root.querySelector('#gev-voice-preview-empty'),
|
||||
voicePreviewInput: root.querySelector('#gev-voice-preview-enabled'),
|
||||
voicePreviewNote: root.querySelector('#gev-voice-preview-note'),
|
||||
voiceTimeoutSelect: root.querySelector('#gev-voice-timeout-select'),
|
||||
voiceTimeoutInput: root.querySelector('#gev-voice-timeout-input'),
|
||||
voiceTimeoutNote: root.querySelector('#gev-voice-timeout-note'),
|
||||
voiceTimeoutStatus: root.querySelector('#gev-voice-timeout-status'),
|
||||
providerField: root.querySelector('#gev-voice-provider-field'),
|
||||
providerSelect: root.querySelector('#gev-voice-provider'),
|
||||
button: root.querySelector('#gev-voice-button'),
|
||||
buttonLabel: root.querySelector('.gev-mic-label'),
|
||||
status: root.querySelector('#gev-voice-status'),
|
||||
@@ -90,6 +193,7 @@ export function createVoiceControl({ reset = false } = {}) {
|
||||
errorDetail: root.querySelector('#gev-voice-error-detail'),
|
||||
tierButton: root.querySelector('#gev-voice-tier'),
|
||||
costValue: root.querySelector('#gev-voice-cost-value'),
|
||||
costWarning: root.querySelector('#gev-voice-cost-warning'),
|
||||
card: root.querySelector('#gev-voice-card'),
|
||||
};
|
||||
}
|
||||
+1
-1
@@ -18,7 +18,7 @@
|
||||
|
||||
import { finiteOrNull } from './pointerContext.js';
|
||||
|
||||
export const POINTER_SENTINEL = 'pointer';
|
||||
const POINTER_SENTINEL = 'pointer';
|
||||
|
||||
const TRACKABLE_LAYERS = new Set([
|
||||
'flights',
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
harness,
|
||||
deferred,
|
||||
tick,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
|
||||
function unreadResponse(bytes = null) {
|
||||
const state = { reads: 0, cancelled: 0 };
|
||||
const response = new Response(
|
||||
new ReadableStream(
|
||||
{
|
||||
pull(controller) {
|
||||
state.reads++;
|
||||
if (bytes) controller.enqueue(bytes);
|
||||
},
|
||||
cancel() {
|
||||
state.cancelled++;
|
||||
},
|
||||
},
|
||||
{ highWaterMark: 0 },
|
||||
),
|
||||
);
|
||||
return { response, state };
|
||||
}
|
||||
|
||||
test('already aborted token request does not acquire or read a response', async (t) => {
|
||||
let fetches = 0;
|
||||
const h = harness({
|
||||
fetchImpl: () => {
|
||||
fetches++;
|
||||
throw Error('must not fetch');
|
||||
},
|
||||
});
|
||||
t.after(() => h.c.stop());
|
||||
const abort = new AbortController();
|
||||
abort.abort();
|
||||
await assert.rejects(h.c.requestToken(abort.signal), { name: 'AbortError' });
|
||||
assert.equal(fetches, 0);
|
||||
});
|
||||
|
||||
for (const phase of [
|
||||
'before response',
|
||||
'between resolution and continuation',
|
||||
]) {
|
||||
test(`aborted token acquisition closes the unread body ${phase}`, async (t) => {
|
||||
const gate = deferred();
|
||||
const h = harness({ fetchImpl: () => gate.promise });
|
||||
t.after(() => h.c.stop());
|
||||
const abort = new AbortController();
|
||||
const { response, state } = unreadResponse();
|
||||
const result = assert.rejects(h.c.requestToken(abort.signal), {
|
||||
name: 'AbortError',
|
||||
});
|
||||
if (phase === 'before response') abort.abort();
|
||||
gate.resolve(response);
|
||||
if (phase !== 'before response') queueMicrotask(() => abort.abort());
|
||||
await result;
|
||||
await tick();
|
||||
assert.deepEqual(state, { reads: 0, cancelled: 1 });
|
||||
});
|
||||
}
|
||||
|
||||
test('token body cap cancels a native response stream', async (t) => {
|
||||
const { response, state } = unreadResponse(new Uint8Array(256 * 1024 + 1));
|
||||
const h = harness({ fetchImpl: async () => response });
|
||||
t.after(() => h.c.stop());
|
||||
await assert.rejects(
|
||||
h.c.requestToken(new AbortController().signal),
|
||||
/too large|exceed|limit/i,
|
||||
);
|
||||
assert.equal(state.cancelled, 1);
|
||||
assert.equal(state.reads, 1);
|
||||
});
|
||||
|
||||
test('token body read ends promptly on caller abort and releases the reader', async (t) => {
|
||||
const { response, state } = unreadResponse();
|
||||
const h = harness({ fetchImpl: async () => response });
|
||||
t.after(() => h.c.stop());
|
||||
const abort = new AbortController();
|
||||
const result = assert.rejects(h.c.requestToken(abort.signal), {
|
||||
name: 'AbortError',
|
||||
});
|
||||
await tick();
|
||||
assert.equal(state.reads, 1);
|
||||
abort.abort();
|
||||
await result;
|
||||
assert.equal(state.cancelled, 1);
|
||||
assert.equal(response.body.locked, false);
|
||||
});
|
||||
|
||||
test('permission grant resolved just before Stop releases its tracks and preserves a fresh start', async (t) => {
|
||||
const gate = deferred();
|
||||
const h = harness({ getUserMedia: () => gate.promise });
|
||||
t.after(() => h.c.stop());
|
||||
const stream = h.makeStream();
|
||||
const start = h.c.start();
|
||||
gate.resolve(stream);
|
||||
queueMicrotask(() => h.c.stop());
|
||||
await start;
|
||||
await tick();
|
||||
assert.equal(h.tracks[0].readyState, 'ended');
|
||||
assert.equal(h.requests.length, 0);
|
||||
assert.equal(h.sockets.length, 0);
|
||||
assert.equal(h.captures[0].closed, true);
|
||||
h.c.getUserMedia = async () => h.makeStream();
|
||||
await h.c.start();
|
||||
assert.equal(h.c.status, 'listening');
|
||||
assert.equal(h.tracks[1].readyState, 'live');
|
||||
assert.equal(h.captures[1].closed, false);
|
||||
});
|
||||
|
||||
test('late audio initialization cannot connect a retired microphone or close successor audio', async (t) => {
|
||||
const gate = deferred();
|
||||
const h = harness();
|
||||
t.after(() => h.c.stop());
|
||||
const createAudio = h.c.createAudio;
|
||||
let first = true;
|
||||
h.c.createAudio = (options) => {
|
||||
const audio = createAudio(options);
|
||||
if (first) {
|
||||
first = false;
|
||||
audio.initialize = () => gate.promise;
|
||||
audio.connectMicrophone = () =>
|
||||
assert.fail('retired initialization must not connect');
|
||||
}
|
||||
return audio;
|
||||
};
|
||||
const oldStart = h.c.start();
|
||||
await tick();
|
||||
h.c.stop();
|
||||
await h.c.start();
|
||||
gate.resolve();
|
||||
await oldStart;
|
||||
await tick();
|
||||
assert.equal(h.c.status, 'listening');
|
||||
assert.equal(h.captures[0].closed, true);
|
||||
assert.equal(h.captures[1].closed, false);
|
||||
assert.equal(h.tracks[0].readyState, 'ended');
|
||||
assert.equal(h.tracks[1].readyState, 'live');
|
||||
});
|
||||
@@ -0,0 +1,380 @@
|
||||
import { msg } from '../i18n/marker.js';
|
||||
const MAX_AUDIO_BYTES = 2 * 1024 * 1024;
|
||||
const MAX_QUEUED_SECONDS = 60;
|
||||
const MAX_QUEUED_SOURCES = 256;
|
||||
const PTT_WINDOW_SAMPLES = 160; // 10 ms at the worklet's fixed 16 kHz.
|
||||
const PTT_ENERGY_FLOOR = 0.002;
|
||||
const PTT_PRE_ROLL_BYTES = 16000 * 2 * 0.4;
|
||||
|
||||
/** Admit manual input only after sustained energy; not an utterance detector. */
|
||||
function pttCapture(captureId) {
|
||||
return {
|
||||
captureId,
|
||||
admitted: false,
|
||||
buffers: [],
|
||||
bytes: 0,
|
||||
samples: 0,
|
||||
energy: 0,
|
||||
aboveFloor: 0,
|
||||
voicedWindows: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function hasPttEnergy(state, pcm) {
|
||||
for (let index = 0; index + 1 < pcm.byteLength; index += 2) {
|
||||
const sample = pcm.getInt16(index, true) / 32768;
|
||||
state.energy += sample * sample;
|
||||
if (Math.abs(sample) >= PTT_ENERGY_FLOOR) state.aboveFloor++;
|
||||
if (++state.samples !== PTT_WINDOW_SAMPLES) continue;
|
||||
// Requiring occupancy as well as RMS rejects isolated high-peak key clicks.
|
||||
const voiced =
|
||||
state.energy / state.samples >= PTT_ENERGY_FLOOR ** 2 &&
|
||||
state.aboveFloor >= PTT_WINDOW_SAMPLES / 8;
|
||||
state.voicedWindows = voiced ? state.voicedWindows + 1 : 0;
|
||||
state.samples = state.energy = state.aboveFloor = 0;
|
||||
if (state.voicedWindows >= 2) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function pcmBytesToBase64(buffer) {
|
||||
const bytes = new Uint8Array(buffer);
|
||||
let binary = '';
|
||||
for (let offset = 0; offset < bytes.length; offset += 8192) {
|
||||
binary += String.fromCharCode(...bytes.subarray(offset, offset + 8192));
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
export function decodeGeminiPcm(data, mimeType = 'audio/pcm;rate=24000') {
|
||||
if (
|
||||
typeof data !== 'string' ||
|
||||
data.length > 4 * Math.ceil(MAX_AUDIO_BYTES / 3)
|
||||
)
|
||||
throw new Error(msg('Gemini returned an oversized audio chunk'));
|
||||
const match = /^audio\/pcm(?:;rate=(\d+))?$/.exec(mimeType);
|
||||
if (!match)
|
||||
throw new Error(msg('Gemini returned an unsupported audio format'));
|
||||
const sampleRate = Number(match[1] || 24000);
|
||||
if (!Number.isInteger(sampleRate) || sampleRate < 8000 || sampleRate > 96000)
|
||||
throw new Error(msg('Gemini returned an invalid audio sample rate'));
|
||||
const binary = atob(data);
|
||||
if (binary.length > MAX_AUDIO_BYTES)
|
||||
throw new Error(msg('Gemini returned an oversized audio chunk'));
|
||||
if (!binary.length || binary.length % 2)
|
||||
throw new Error(msg('Gemini returned incomplete PCM audio'));
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let index = 0; index < binary.length; index++)
|
||||
bytes[index] = binary.charCodeAt(index);
|
||||
const view = new DataView(bytes.buffer);
|
||||
const samples = new Float32Array(bytes.length / 2);
|
||||
for (let index = 0; index < samples.length; index++)
|
||||
samples[index] = view.getInt16(index * 2, true) / 32768;
|
||||
return { samples, sampleRate };
|
||||
}
|
||||
|
||||
/** Own capture and playback independently from the Live protocol. */
|
||||
export function createGeminiAudio({
|
||||
pushToTalk = false,
|
||||
onAudioStart,
|
||||
onAudio,
|
||||
onAudioEnd,
|
||||
onAudioDiscard,
|
||||
onDrain,
|
||||
onError,
|
||||
AudioContextClass = globalThis.AudioContext || globalThis.webkitAudioContext,
|
||||
AudioWorkletNodeClass = globalThis.AudioWorkletNode,
|
||||
} = {}) {
|
||||
if (!AudioContextClass || !AudioWorkletNodeClass)
|
||||
throw new Error(
|
||||
msg('This browser does not support Gemini microphone audio'),
|
||||
);
|
||||
let captureContext;
|
||||
let playbackContext;
|
||||
let output;
|
||||
let meter;
|
||||
let resumed;
|
||||
const closeContext = (context) => {
|
||||
try {
|
||||
return Promise.resolve(context?.close());
|
||||
} catch (error) {
|
||||
return Promise.reject(error);
|
||||
}
|
||||
};
|
||||
try {
|
||||
// Capture at the device rate; the worklet explicitly resamples to 16 kHz.
|
||||
captureContext = new AudioContextClass();
|
||||
playbackContext = new AudioContextClass({ sampleRate: 24000 });
|
||||
// Resume both synchronously within the mic gesture, before any awaits.
|
||||
const resume = (context) => {
|
||||
try {
|
||||
return Promise.resolve(context.resume());
|
||||
} catch (error) {
|
||||
return Promise.reject(error);
|
||||
}
|
||||
};
|
||||
resumed = Promise.all([resume(captureContext), resume(playbackContext)]);
|
||||
// Permission may remain pending before initialize observes this failure.
|
||||
void resumed.catch(() => {});
|
||||
output = playbackContext.createGain();
|
||||
meter = playbackContext.createMediaStreamDestination();
|
||||
output.connect(playbackContext.destination);
|
||||
output.connect(meter);
|
||||
} catch (error) {
|
||||
output?.disconnect();
|
||||
meter?.disconnect();
|
||||
meter?.stream.getTracks?.().forEach((track) => track.stop());
|
||||
void Promise.allSettled([
|
||||
closeContext(captureContext),
|
||||
closeContext(playbackContext),
|
||||
]);
|
||||
throw error;
|
||||
}
|
||||
let closed = false;
|
||||
let closeResult;
|
||||
let capture = null;
|
||||
let microphone = null;
|
||||
let mute = null;
|
||||
let nextTime = 0;
|
||||
let inputChunks = 0;
|
||||
let inputPeak = 0;
|
||||
let nonSilentInputChunks = 0;
|
||||
let outputChunks = 0;
|
||||
let outputSamples = 0;
|
||||
let completedOutputSamples = 0;
|
||||
let discardedOutputSamples = 0;
|
||||
let pendingPtt = null;
|
||||
let latestPttCaptureId = 0;
|
||||
let suppressedPttCaptures = 0;
|
||||
let admittedPttCaptures = 0;
|
||||
const sources = new Map();
|
||||
|
||||
const emitAudio = (buffer, metadata) =>
|
||||
onAudio?.(
|
||||
{ data: pcmBytesToBase64(buffer), mimeType: 'audio/pcm;rate=16000' },
|
||||
metadata,
|
||||
);
|
||||
|
||||
function clearPlayback() {
|
||||
const oldSources = [...sources];
|
||||
sources.clear();
|
||||
nextTime = playbackContext.currentTime;
|
||||
for (const [source, sampleCount] of oldSources) {
|
||||
discardedOutputSamples += sampleCount;
|
||||
source.onended = null;
|
||||
try {
|
||||
source.stop();
|
||||
} catch {
|
||||
/* Already ended. */
|
||||
}
|
||||
source.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
outputStream: meter.stream,
|
||||
get pending() {
|
||||
return sources.size > 0;
|
||||
},
|
||||
async initialize(signal) {
|
||||
try {
|
||||
await Promise.all([
|
||||
resumed,
|
||||
captureContext.audioWorklet.addModule(
|
||||
new URL('./geminiCaptureWorklet.js', import.meta.url),
|
||||
),
|
||||
]);
|
||||
signal?.throwIfAborted();
|
||||
if (closed) throw new DOMException('Voice stopped', 'AbortError');
|
||||
} catch (error) {
|
||||
await this.close();
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
connectMicrophone(stream, enabled, metadata) {
|
||||
if (closed) throw new DOMException('Voice stopped', 'AbortError');
|
||||
capture = new AudioWorkletNodeClass(
|
||||
captureContext,
|
||||
'gev-gemini-capture',
|
||||
{
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
outputChannelCount: [1],
|
||||
},
|
||||
);
|
||||
capture.port.onmessage = ({ data }) => {
|
||||
if (closed) return;
|
||||
// Capture ownership is transport metadata, never part of Live audio.
|
||||
if (!Number.isSafeInteger(data?.captureId) || data.captureId <= 0)
|
||||
return;
|
||||
const metadata = { captureId: data.captureId };
|
||||
try {
|
||||
if (data?.type === 'audio' && data.buffer instanceof ArrayBuffer) {
|
||||
inputChunks++;
|
||||
const pcm = new DataView(data.buffer);
|
||||
let peak = 0;
|
||||
for (let index = 0; index + 1 < pcm.byteLength; index += 2)
|
||||
peak = Math.max(
|
||||
peak,
|
||||
Math.abs(pcm.getInt16(index, true)) / 32768,
|
||||
);
|
||||
inputPeak = Math.max(inputPeak, peak);
|
||||
if (peak >= 0.01) nonSilentInputChunks++;
|
||||
if (!pushToTalk) emitAudio(data.buffer, metadata);
|
||||
else {
|
||||
const state = pendingPtt;
|
||||
if (state?.captureId !== data.captureId) return;
|
||||
if (state.admitted) emitAudio(data.buffer, metadata);
|
||||
else {
|
||||
state.buffers.push(data.buffer);
|
||||
state.bytes += data.buffer.byteLength;
|
||||
if (hasPttEnergy(state, pcm)) {
|
||||
state.admitted = true;
|
||||
admittedPttCaptures++;
|
||||
// Context and activityStart are gated together with PCM.
|
||||
onAudioStart?.(metadata);
|
||||
for (const buffer of state.buffers) {
|
||||
if (closed || pendingPtt !== state) break;
|
||||
emitAudio(buffer, metadata);
|
||||
}
|
||||
state.buffers = [];
|
||||
state.bytes = 0;
|
||||
} else {
|
||||
while (state.bytes > PTT_PRE_ROLL_BYTES) {
|
||||
state.bytes -= state.buffers.shift().byteLength;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (data?.type === 'start') {
|
||||
if (!pushToTalk) onAudioStart?.(metadata);
|
||||
else if (data.captureId > latestPttCaptureId) {
|
||||
latestPttCaptureId = data.captureId;
|
||||
pendingPtt = pttCapture(data.captureId);
|
||||
}
|
||||
} else if (data?.type === 'end') {
|
||||
if (!pushToTalk) onAudioEnd?.(metadata);
|
||||
else if (pendingPtt?.captureId === data.captureId) {
|
||||
const admitted = pendingPtt.admitted;
|
||||
pendingPtt = null;
|
||||
if (admitted) onAudioEnd?.(metadata);
|
||||
else {
|
||||
suppressedPttCaptures++;
|
||||
onAudioDiscard?.(metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
onError?.(error);
|
||||
}
|
||||
};
|
||||
capture.onprocessorerror = () => {
|
||||
if (!closed)
|
||||
onError?.(new Error(msg('Microphone audio processing failed')));
|
||||
};
|
||||
microphone = captureContext.createMediaStreamSource(stream);
|
||||
mute = captureContext.createGain();
|
||||
mute.gain.value = 0;
|
||||
capture.port.postMessage({
|
||||
enabled: Boolean(enabled),
|
||||
captureId: metadata?.captureId,
|
||||
});
|
||||
microphone.connect(capture);
|
||||
capture.connect(mute);
|
||||
mute.connect(captureContext.destination);
|
||||
},
|
||||
setMicrophoneEnabled(enabled, metadata) {
|
||||
if (closed) return;
|
||||
capture?.port.postMessage({
|
||||
enabled: Boolean(enabled),
|
||||
captureId: metadata?.captureId,
|
||||
});
|
||||
},
|
||||
play(data, mimeType) {
|
||||
if (closed) return;
|
||||
// Tiny chunks can exhaust source nodes long before the duration limit.
|
||||
if (sources.size >= MAX_QUEUED_SOURCES)
|
||||
throw new Error(
|
||||
msg('Gemini audio playback fell behind. Reconnect to continue.'),
|
||||
);
|
||||
const { samples, sampleRate } = decodeGeminiPcm(data, mimeType);
|
||||
const start = Math.max(playbackContext.currentTime + 0.02, nextTime);
|
||||
if (
|
||||
start + samples.length / sampleRate - playbackContext.currentTime >
|
||||
MAX_QUEUED_SECONDS
|
||||
)
|
||||
throw new Error(
|
||||
msg('Gemini audio playback fell behind. Reconnect to continue.'),
|
||||
);
|
||||
const buffer = playbackContext.createBuffer(
|
||||
1,
|
||||
samples.length,
|
||||
sampleRate,
|
||||
);
|
||||
buffer.copyToChannel(samples, 0);
|
||||
const source = playbackContext.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
source.connect(output);
|
||||
sources.set(source, samples.length);
|
||||
source.onended = () => {
|
||||
if (!sources.delete(source)) return;
|
||||
completedOutputSamples += samples.length;
|
||||
source.disconnect();
|
||||
if (!closed && !sources.size) onDrain?.();
|
||||
};
|
||||
nextTime = start + buffer.duration;
|
||||
outputChunks++;
|
||||
outputSamples += samples.length;
|
||||
source.start(start);
|
||||
},
|
||||
clearPlayback,
|
||||
diagnostics() {
|
||||
return {
|
||||
inputChunks,
|
||||
inputPeak,
|
||||
nonSilentInputChunks,
|
||||
outputChunks,
|
||||
outputSamples,
|
||||
completedOutputSamples,
|
||||
discardedOutputSamples,
|
||||
queuedSeconds: Math.max(0, nextTime - playbackContext.currentTime),
|
||||
contextState:
|
||||
captureContext.state === playbackContext.state
|
||||
? captureContext.state
|
||||
: 'suspended',
|
||||
captureContextState: captureContext.state,
|
||||
playbackContextState: playbackContext.state,
|
||||
captureSampleRate: captureContext.sampleRate,
|
||||
playbackSampleRate: playbackContext.sampleRate,
|
||||
inputSampleRate: 16000,
|
||||
suppressedPttCaptures,
|
||||
admittedPttCaptures,
|
||||
pttBufferedBytes: pendingPtt?.bytes || 0,
|
||||
};
|
||||
},
|
||||
close() {
|
||||
if (closed) return closeResult;
|
||||
closed = true;
|
||||
pendingPtt = null;
|
||||
clearPlayback();
|
||||
if (capture) {
|
||||
capture.port.onmessage = null;
|
||||
capture.onprocessorerror = null;
|
||||
capture.port.close();
|
||||
capture.disconnect();
|
||||
}
|
||||
microphone?.disconnect();
|
||||
mute?.disconnect();
|
||||
output.disconnect();
|
||||
meter.disconnect();
|
||||
meter.stream.getTracks?.().forEach((track) => track.stop());
|
||||
closeResult = Promise.allSettled([
|
||||
closeContext(captureContext),
|
||||
closeContext(playbackContext),
|
||||
]).then((results) =>
|
||||
results.every((result) => result.status === 'fulfilled'),
|
||||
);
|
||||
return closeResult;
|
||||
},
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,791 @@
|
||||
// Native automatic activity, reply ownership and expiry lifecycle regressions.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
harness,
|
||||
tick,
|
||||
deferred,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
const vad = (ws, type) => ws.message({ voiceActivity: { type } });
|
||||
const onset = (ws) => {
|
||||
vad(ws, 'ACTIVITY_START');
|
||||
vad(ws, 'ACTIVITY_END');
|
||||
};
|
||||
const split = (ws) => {
|
||||
onset(ws);
|
||||
onset(ws);
|
||||
};
|
||||
const audio = {
|
||||
serverContent: {
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{ inlineData: { data: 'AAAA', mimeType: 'audio/pcm;rate=24000' } },
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
const done = (ws) => ws.message({ serverContent: { turnComplete: true } });
|
||||
const tools = (ws, id) =>
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id,
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'out', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const responses = (ws) =>
|
||||
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
|
||||
function close(h) {
|
||||
h.c.stop();
|
||||
assert.equal(h.timers.size, 0, 'owned timers released');
|
||||
}
|
||||
test('N1 VAD-only split admits the first real answer and completes after playback', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
split(ws);
|
||||
ws.message(audio);
|
||||
ws.message({ serverContent: { outputTranscription: { text: 'Answer' } } });
|
||||
done(ws);
|
||||
assert.equal(h.captures[0].played.length, 1);
|
||||
h.captures[0].drain();
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion')?.status,
|
||||
'completed',
|
||||
);
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N2 VAD-only split does not arm a nonexistent reply timeout', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
split(h.sockets[0]);
|
||||
h.advance(5001);
|
||||
assert.equal(h.c.status, 'listening');
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
assert.equal(h.c.retiredBoundaryWait, null);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N3 server interruption before output still fences until ordered turnComplete', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
split(ws);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
ws.message(audio);
|
||||
tools(ws, 'retired-before-boundary');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.captures[0].played.length, 0);
|
||||
assert.equal(h.calls.length, 0);
|
||||
assert.equal(responses(ws).length, 0);
|
||||
ws.message({
|
||||
serverContent: { ...audio.serverContent, turnComplete: true },
|
||||
});
|
||||
assert.equal(
|
||||
h.captures[0].played.length,
|
||||
0,
|
||||
'coalesced retired content is discarded',
|
||||
);
|
||||
ws.message(audio);
|
||||
assert.equal(h.captures[0].played.length, 1);
|
||||
done(ws);
|
||||
h.captures[0].drain();
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion')?.status,
|
||||
'completed',
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N4 generationComplete is server evidence and next onset retires it', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message({ serverContent: { generationComplete: true } });
|
||||
onset(ws);
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
assert.ok(h.c.retiredBoundaryWait);
|
||||
ws.message(audio);
|
||||
tools(ws, 'old-generation');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.captures[0].played.length, 0);
|
||||
assert.equal(h.calls.length, 0);
|
||||
done(ws);
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N5 goAway during VAD-only activity waits for actual reply and playback', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
split(ws);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
assert.ok(h.c.goAway);
|
||||
assert.equal(ws.closed, undefined);
|
||||
h.advance(5001);
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message(audio);
|
||||
done(ws);
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
true,
|
||||
'pending actual PCM drains before expiry',
|
||||
);
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.c.status, 'error');
|
||||
assert.equal(ws.closed, true);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N6 first server tool after VAD-only split executes exactly once', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
split(ws);
|
||||
tools(ws, 'new-generation');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(
|
||||
responses(ws).filter((x) => x.id === 'new-generation').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(responses(ws).at(-1).response.ok, true);
|
||||
tools(ws, 'new-generation');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(
|
||||
responses(ws).filter((x) => x.id === 'new-generation').length,
|
||||
1,
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('C1 actual old audio followed by onset fences late audio and tool until boundary', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(audio);
|
||||
onset(ws);
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
ws.message(audio);
|
||||
tools(ws, 'late-old');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.captures[0].played.length, 1);
|
||||
assert.equal(h.calls.length, 0);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
ws.message(audio);
|
||||
assert.equal(h.captures[0].played.length, 1);
|
||||
done(ws);
|
||||
ws.message(audio);
|
||||
assert.equal(h.captures[0].played.length, 2);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('C2 first server tool in flight is aborted and never answered after new onset', async () => {
|
||||
const gate = deferred(),
|
||||
entered = [];
|
||||
const h = harness({
|
||||
runAction: async (_n, _a, o) => {
|
||||
entered.push(o);
|
||||
await gate.promise;
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
tools(ws, 'old-running');
|
||||
await tick();
|
||||
assert.equal(entered.length, 1);
|
||||
onset(ws);
|
||||
assert.equal(entered[0].signal.aborted, true);
|
||||
gate.resolve();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(responses(ws).filter((x) => x.id === 'old-running').length, 0);
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
} finally {
|
||||
gate.resolve();
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('C5 typed request before first server output still retires on next onset', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
assert.equal(h.c.sendTextCommand('Zoom in'), true);
|
||||
onset(ws);
|
||||
assert.equal(h.c.retiredReplyPending, true);
|
||||
ws.message(audio);
|
||||
tools(ws, 'late-typed');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.captures[0].played.length, 0);
|
||||
assert.equal(h.calls.length, 0);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N7 old retired boundary preserves the newer automatic owner for goAway', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
const oldOwner = h.c.automaticActivityOwner;
|
||||
ws.message(audio);
|
||||
onset(ws);
|
||||
const successor = h.c.automaticActivityOwner;
|
||||
assert.notEqual(successor, oldOwner);
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, oldOwner);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.automaticActivityOwner, successor);
|
||||
assert.equal(h.c.automaticTurnOpen, true);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message(audio);
|
||||
done(ws);
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.c.status, 'error');
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N8 old automatic owner with no successor is cleared by its retired boundary', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
const owner = h.c.automaticActivityOwner;
|
||||
ws.message(audio);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, owner);
|
||||
done(ws);
|
||||
assert.equal(h.c.automaticTurnOpen, false);
|
||||
assert.equal(h.c.automaticActivityOwner, null);
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, null);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), false, 'no invented successor wait');
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N9 coalesced old completion and new VAD preserves the new owner', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
vad(ws, 'ACTIVITY_END');
|
||||
ws.message({
|
||||
voiceActivity: { type: 'ACTIVITY_START' },
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
assert.equal(h.c.automaticTurnOpen, true);
|
||||
assert.ok(h.c.automaticActivityOwner);
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, null);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N10 stop, restart and destroy release automatic owner identities and timers', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
onset(h.sockets[0]);
|
||||
const first = h.c.automaticActivityOwner;
|
||||
h.c.stop();
|
||||
assert.equal(h.c.automaticTurnOpen, false);
|
||||
assert.equal(h.c.automaticActivityOwner, null);
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, null);
|
||||
assert.equal(h.c.automaticActivitySequence, 0);
|
||||
await h.c.start();
|
||||
onset(h.sockets[1]);
|
||||
assert.notEqual(h.c.automaticActivityOwner, first);
|
||||
assert.notEqual(h.c.automaticActivityOwner.epoch, first.epoch);
|
||||
h.c.stop({ removeUi: true });
|
||||
assert.equal(h.c.disposed, true);
|
||||
assert.equal(h.c.automaticTurnOpen, false);
|
||||
assert.equal(h.c.automaticActivityOwner, null);
|
||||
assert.equal(h.c.retiredAutomaticActivityOwner, null);
|
||||
assert.equal(h.timers.size, 0);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N11 deferred typed ownership counts discarded open-mic PCM without queuing or replay', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(audio);
|
||||
assert.equal(h.c.sendTextCommand('Zoom in'), true);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out'), true);
|
||||
assert.equal(h.c.deferredInput.kind, 'user-text');
|
||||
const before = ws.sent.filter((x) => x.realtimeInput?.audio).length;
|
||||
const metadata = { captureId: h.c.captureOwner.captureId };
|
||||
h.captures[0].callbacks.onAudio(
|
||||
{ data: 'AAAA', mimeType: 'audio/pcm;rate=16000' },
|
||||
metadata,
|
||||
);
|
||||
assert.equal(ws.sent.filter((x) => x.realtimeInput?.audio).length, before);
|
||||
assert.equal(
|
||||
h.c.deferredInput.messages.some((x) => x.realtimeInput?.audio),
|
||||
false,
|
||||
);
|
||||
assert.equal(h.c.getDiagnostics().droppedAudioChunks, 1);
|
||||
assert.deepEqual(h.c.getDiagnostics().deferredAudioDrops, {
|
||||
nonPttOwner: 1,
|
||||
closedActivity: 0,
|
||||
});
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
done(ws);
|
||||
assert.equal(
|
||||
ws.sent.filter((x) => x.realtimeInput?.audio).length,
|
||||
before,
|
||||
'no wrong-owner PCM replay after boundaries',
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N12 goAway before old completion still waits for the newer automatic owner', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
onset(ws);
|
||||
const successor = h.c.automaticActivityOwner;
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.automaticActivityOwner, successor);
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
true,
|
||||
'old boundary during goAway cannot end successor wait',
|
||||
);
|
||||
ws.message(audio);
|
||||
done(ws);
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.c.status, 'error');
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
for (const interruptedBeforeExpiry of [false, true])
|
||||
test(`N13 no successor clears retired ownership with goAway ${interruptedBeforeExpiry ? 'after interruption' : 'before interruption'}`, async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
if (interruptedBeforeExpiry)
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.automaticTurnOpen, false);
|
||||
assert.equal(h.c.automaticActivityOwner, null);
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
false,
|
||||
'no successor finishes at its actual boundary',
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N14 native interruption during goAway aborts pending old tool and suppresses late response', async () => {
|
||||
const gate = deferred(),
|
||||
entered = [];
|
||||
const h = harness({
|
||||
runAction: async (_n, _a, o) => {
|
||||
entered.push(o);
|
||||
await gate.promise;
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
tools(ws, 'expiring-old-tool');
|
||||
await tick();
|
||||
assert.equal(entered.length, 1);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(entered[0].signal.aborted, true);
|
||||
gate.resolve();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(
|
||||
responses(ws).filter((x) => x.id === 'expiring-old-tool').length,
|
||||
0,
|
||||
);
|
||||
assert.equal(h.c.isActive(), false);
|
||||
} finally {
|
||||
gate.resolve();
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('N15 goAway still refuses local PTT and typed replacement without reopening microphone', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
const turn = h.c.turn,
|
||||
sent = ws.sent.length;
|
||||
assert.equal(h.c.bargeIn(), false);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out'), false);
|
||||
h.c.interrupt('push-to-talk');
|
||||
assert.equal(h.c.turn, turn);
|
||||
assert.equal(ws.sent.length, sent);
|
||||
assert.equal(
|
||||
h.tracks.every((t) => !t.enabled),
|
||||
true,
|
||||
);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.isActive(), false);
|
||||
assert.equal(
|
||||
h.tracks.every((t) => t.readyState === 'ended'),
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
|
||||
test('T-F1a PTT successor PCM survives retired boundary and a second input fences its late output', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start({ pushToTalk: true });
|
||||
const ws = h.sockets[0],
|
||||
a = h.captures[0];
|
||||
ws.message(audio);
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
assert.equal(h.c.input.bargeIn(), true);
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
a.callbacks.onAudio(
|
||||
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
|
||||
a.metadata,
|
||||
);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(
|
||||
h.c.replyPending,
|
||||
true,
|
||||
'old boundary cannot clear submitted successor',
|
||||
);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out second request'), true);
|
||||
assert.ok(h.c.currentTypedInput());
|
||||
const played = a.played.length;
|
||||
ws.message(audio);
|
||||
tools(ws, 'late-ptt-predecessor');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(a.played.length, played);
|
||||
assert.equal(
|
||||
h.calls.length,
|
||||
0,
|
||||
'no old tool under newer local typed input',
|
||||
);
|
||||
done(ws);
|
||||
tools(ws, 'current-typed');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(h.calls[0].options.isCurrent(), true);
|
||||
assert.equal(h.c.currentTypedInput().text, 'Zoom out second request');
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
for (const expiryBeforeBoundary of [false, true])
|
||||
test(`T-F1b VAD then submitted text survives goAway ${expiryBeforeBoundary ? 'before' : 'after'} retired boundary`, async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(audio);
|
||||
onset(ws);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out successor'), true);
|
||||
const typed = h.c.currentTypedInput();
|
||||
assert.ok(typed);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
if (expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } });
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
assert.equal(h.c.replyPending, true);
|
||||
if (!expiryBeforeBoundary) ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true, 'submitted text must finish');
|
||||
tools(ws, 'valid-successor');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(h.calls[0].options.isCurrent(), true);
|
||||
assert.equal(h.c.currentTypedInput(), typed);
|
||||
ws.message(audio);
|
||||
done(ws);
|
||||
assert.equal(h.c.isActive(), true);
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.c.isActive(), false);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
for (const repeat of [false, true])
|
||||
test(`T-F2 forwarded A superseded by deferred B cannot run after goAway${repeat ? ' shortened twice' : ''}`, async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(audio);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out A'), true);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out B'), true);
|
||||
assert.equal(h.c.deferredInput.kind, 'user-text');
|
||||
assert.ok(h.c.currentTypedInput());
|
||||
onset(ws);
|
||||
assert.equal(h.c.automaticTurnOpen, true);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.deferredInput, null);
|
||||
if (repeat) {
|
||||
ws.message({ goAway: { timeLeft: '20s' } });
|
||||
ws.message({ goAway: { timeLeft: '10s' } });
|
||||
}
|
||||
assert.equal(h.c.goAway.supersededSuccessor, true);
|
||||
assert.equal(h.c.isActive(), true);
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
tools(ws, 'superseded-A');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 0, 'A cannot run with dropped B authority');
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
false,
|
||||
'ordered boundary closes superseded forwarded predecessor',
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-F3 VAD-only no-generation expiry waits to bounded native deadline and releases resources', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
onset(h.sockets[0]);
|
||||
h.sockets[0].message({ goAway: { timeLeft: '30s' } });
|
||||
h.advance(29999);
|
||||
assert.equal(h.c.isActive(), true);
|
||||
h.advance(1);
|
||||
assert.equal(h.c.isActive(), false);
|
||||
assert.equal(h.timers.size, 0);
|
||||
assert.equal(
|
||||
h.tracks.every((t) => t.readyState === 'ended'),
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C1 coalesced new START and natural old completion preserve newer automatic activity', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
ws.message({
|
||||
voiceActivity: { type: 'ACTIVITY_START' },
|
||||
serverContent: { turnComplete: true },
|
||||
});
|
||||
assert.equal(h.c.retiredReplyPending, false);
|
||||
assert.equal(h.c.automaticTurnOpen, true);
|
||||
ws.message({ goAway: { timeLeft: '30s' } });
|
||||
assert.equal(h.c.isActive(), true);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C2 generationComplete alone followed by natural completion emits completion with no retirement timer', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message({ serverContent: { generationComplete: true } });
|
||||
assert.equal(h.c.replyPending, true);
|
||||
done(ws);
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion')?.status,
|
||||
'completed',
|
||||
);
|
||||
assert.equal(h.c.replyPending, false);
|
||||
assert.equal(h.c.automaticTurnOpen, false);
|
||||
assert.equal(h.c.retiredBoundaryWait, null);
|
||||
assert.equal(h.timers.size, 0);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
|
||||
test('T-C3 silent PTT terminal completion emits spoken false without fabricated audio', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start({ pushToTalk: true });
|
||||
const ws = h.sockets[0],
|
||||
a = h.captures[0];
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
a.callbacks.onAudio(
|
||||
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
|
||||
a.metadata,
|
||||
);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
ws.message({ serverContent: { generationComplete: true } });
|
||||
done(ws);
|
||||
const event = h.events.findLast((e) => e.type === 'completion');
|
||||
assert.equal(event.status, 'completed');
|
||||
assert.equal(event.spoken, false);
|
||||
assert.equal(a.played.length, 0);
|
||||
assert.equal(h.c.userTurnPending, false);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C4 silent terminal boundary waits for pending tool and native follow-up completion', async () => {
|
||||
const gate = deferred(),
|
||||
entered = [];
|
||||
const h = harness({
|
||||
runAction: async (_n, _a, o) => {
|
||||
entered.push(o);
|
||||
await gate.promise;
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
tools(ws, 'pending-terminal-tool');
|
||||
await tick();
|
||||
assert.equal(entered.length, 1);
|
||||
done(ws);
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
assert.equal(h.c.userTurnPending, false);
|
||||
gate.resolve();
|
||||
await h.c.toolQueue;
|
||||
h.c.finishPlayback();
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
assert.equal(h.c.replyPending, true, 'tool result awaits native follow-up');
|
||||
done(ws);
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion')?.status,
|
||||
'completed',
|
||||
);
|
||||
} finally {
|
||||
gate.resolve();
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C5 terminal boundary with queued PCM waits for drain and preserves real spoken state', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
ws.message(audio);
|
||||
ws.message({ serverContent: { generationComplete: true } });
|
||||
done(ws);
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
h.captures[0].drain();
|
||||
const event = h.events.findLast((e) => e.type === 'completion');
|
||||
assert.equal(event.status, 'completed');
|
||||
assert.equal(event.spoken, true);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C6 old retired terminal boundary cannot clear successor userTurnPending', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(audio);
|
||||
assert.equal(h.c.sendTextCommand('Zoom out successor'), true);
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
done(ws);
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
test('T-C7 reentrant silent completion observer keeps new input pending', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
const emit = h.c.emit.bind(h.c);
|
||||
h.c.emit = (event) => {
|
||||
emit(event);
|
||||
if (event.type === 'completion')
|
||||
h.c.sendTextCommand('Zoom out reentrant');
|
||||
};
|
||||
onset(ws);
|
||||
ws.message({ serverContent: { generationComplete: true } });
|
||||
done(ws);
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
assert.ok(h.c.currentTypedInput());
|
||||
assert.equal(h.c.modelComplete, false);
|
||||
} finally {
|
||||
close(h);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
const OUTPUT_RATE = 16000;
|
||||
const FILTER_TAPS = 64;
|
||||
const FILTER_PHASES = 128;
|
||||
|
||||
// A causal, windowed-sinc low-pass filter avoids folding device frequencies
|
||||
// above 8 kHz into speech. Precompute phases; process allocates no filter data.
|
||||
function resamplingFilters(inputRate) {
|
||||
const cutoff = 0.45 * Math.min(1, OUTPUT_RATE / inputRate);
|
||||
const filters = new Float32Array(FILTER_TAPS * FILTER_PHASES);
|
||||
for (let phase = 0; phase < FILTER_PHASES; phase++) {
|
||||
let total = 0;
|
||||
for (let tap = 0; tap < FILTER_TAPS; tap++) {
|
||||
const distance = tap - (FILTER_TAPS - 1) / 2 - phase / FILTER_PHASES;
|
||||
const sinc =
|
||||
distance === 0
|
||||
? 2 * cutoff
|
||||
: Math.sin(2 * Math.PI * cutoff * distance) / (Math.PI * distance);
|
||||
const window =
|
||||
0.5 - 0.5 * Math.cos((2 * Math.PI * tap) / (FILTER_TAPS - 1));
|
||||
const weight = sinc * window;
|
||||
filters[phase * FILTER_TAPS + tap] = weight;
|
||||
total += weight;
|
||||
}
|
||||
for (let tap = 0; tap < FILTER_TAPS; tap++)
|
||||
filters[phase * FILTER_TAPS + tap] /= total;
|
||||
}
|
||||
return filters;
|
||||
}
|
||||
|
||||
/** PCM microphone capture. Only explicitly enabled frames leave this processor. */
|
||||
class GeminiCaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.enabled = false;
|
||||
this.captureId = null;
|
||||
this.samples = new Int16Array(1024);
|
||||
this.length = 0;
|
||||
this.inputRate = sampleRate;
|
||||
this.filters =
|
||||
this.inputRate === OUTPUT_RATE ? null : resamplingFilters(this.inputRate);
|
||||
this.history = new Float32Array(FILTER_TAPS);
|
||||
this.resetResampler();
|
||||
this.port.onmessage = ({ data }) => {
|
||||
if (data?.enabled === true) {
|
||||
if (
|
||||
this.enabled ||
|
||||
!Number.isSafeInteger(data.captureId) ||
|
||||
data.captureId <= 0
|
||||
)
|
||||
return;
|
||||
this.captureId = data.captureId;
|
||||
this.resetResampler();
|
||||
this.enabled = true;
|
||||
this.port.postMessage({ type: 'start', captureId: this.captureId });
|
||||
} else if (
|
||||
data?.enabled === false &&
|
||||
this.enabled &&
|
||||
data.captureId === this.captureId
|
||||
) {
|
||||
this.flush();
|
||||
this.port.postMessage({ type: 'end', captureId: this.captureId });
|
||||
this.enabled = false;
|
||||
this.captureId = null;
|
||||
this.length = 0;
|
||||
// Do not drain filter tails into a later physical microphone capture.
|
||||
this.resetResampler();
|
||||
}
|
||||
};
|
||||
}
|
||||
resetResampler() {
|
||||
this.phase = 0;
|
||||
this.historyIndex = 0;
|
||||
this.history.fill(0);
|
||||
}
|
||||
appendSample(sample) {
|
||||
const value = Math.max(-1, Math.min(1, sample));
|
||||
this.samples[this.length++] = Math.round(
|
||||
value * (value < 0 ? 32768 : 32767),
|
||||
);
|
||||
if (this.length === this.samples.length) this.flush();
|
||||
}
|
||||
flush() {
|
||||
if (!this.length) return;
|
||||
const buffer = new ArrayBuffer(this.length * 2);
|
||||
const view = new DataView(buffer);
|
||||
for (let i = 0; i < this.length; i++)
|
||||
view.setInt16(i * 2, this.samples[i], true);
|
||||
this.port.postMessage(
|
||||
{ type: 'audio', buffer, captureId: this.captureId },
|
||||
[buffer],
|
||||
);
|
||||
this.length = 0;
|
||||
}
|
||||
process(inputs) {
|
||||
if (!this.enabled) return true;
|
||||
const channel = inputs[0]?.[0];
|
||||
if (!channel) return true;
|
||||
for (const sample of channel) {
|
||||
if (!this.filters) {
|
||||
this.appendSample(sample);
|
||||
continue;
|
||||
}
|
||||
this.history[this.historyIndex] = sample;
|
||||
this.historyIndex = (this.historyIndex + 1) % FILTER_TAPS;
|
||||
this.phase += OUTPUT_RATE;
|
||||
while (this.phase >= this.inputRate) {
|
||||
this.phase -= this.inputRate;
|
||||
const phase = Math.min(
|
||||
FILTER_PHASES - 1,
|
||||
Math.floor((this.phase * FILTER_PHASES) / OUTPUT_RATE),
|
||||
);
|
||||
let value = 0;
|
||||
for (let tap = 0; tap < FILTER_TAPS; tap++) {
|
||||
const index =
|
||||
(this.historyIndex - 1 - tap + FILTER_TAPS) % FILTER_TAPS;
|
||||
value +=
|
||||
this.history[index] * this.filters[phase * FILTER_TAPS + tap];
|
||||
}
|
||||
this.appendSample(value);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('gev-gemini-capture', GeminiCaptureProcessor);
|
||||
@@ -0,0 +1,285 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
nativeIntegrationFixture,
|
||||
assertSuccessfulZoom,
|
||||
} from '../testSupport/geminiIntegrationFixture.mjs';
|
||||
|
||||
const options = { concurrency: false, timeout: 5000 };
|
||||
const OLD_PCM = 'BAAFAA==';
|
||||
const TEXT = 'Zoom in for the owned typed successor';
|
||||
|
||||
function exposeCapture(h) {
|
||||
const createAudio = h.controller.createAudio;
|
||||
h.controller.createAudio = (callbacks) => {
|
||||
const audio = createAudio(callbacks);
|
||||
audio.captureCallbacks = callbacks;
|
||||
return audio;
|
||||
};
|
||||
}
|
||||
|
||||
async function startManual(h) {
|
||||
await h.session.start({ pushToTalk: true });
|
||||
assert.equal(h.session.state, 'listening');
|
||||
assert.equal(h.controller.input.pushToTalkMode, true);
|
||||
assert.equal(Boolean(h.controller.input.pushToTalkKeyHeld), false);
|
||||
return h.sockets.at(-1);
|
||||
}
|
||||
|
||||
// Explicit fake worklet callbacks retain the native physical-capture path;
|
||||
// this helper does not populate controller queues or ownership fields itself.
|
||||
function speakAndRelease(h, data) {
|
||||
const c = h.controller;
|
||||
const audio = h.audios.at(-1);
|
||||
c.input.pushToTalkKeyHeld = true;
|
||||
c.input.setMicrophoneEnabled(true);
|
||||
const metadata = audio.metadata;
|
||||
assert.ok(Number.isSafeInteger(metadata?.captureId));
|
||||
audio.captureCallbacks.onAudioStart(metadata);
|
||||
audio.captureCallbacks.onAudio(
|
||||
{ data, mimeType: 'audio/pcm;rate=16000' },
|
||||
metadata,
|
||||
);
|
||||
c.input.pushToTalkKeyHeld = false;
|
||||
c.input.setMicrophoneEnabled(false);
|
||||
audio.captureCallbacks.onAudioEnd(metadata);
|
||||
return metadata;
|
||||
}
|
||||
|
||||
async function prepareDeferredPtt(h) {
|
||||
exposeCapture(h);
|
||||
const socket = await startManual(h);
|
||||
speakAndRelease(h, 'AAABAA=='); // A is submitted and awaits its reply.
|
||||
speakAndRelease(h, 'AgADAA=='); // B is submitted, retiring A.
|
||||
const wireBeforeC = socket.sent.length;
|
||||
const metadata = speakAndRelease(h, OLD_PCM); // C waits behind A's boundary.
|
||||
const owner = h.controller.deferredInput;
|
||||
assert.equal(owner?.kind, 'push-to-talk');
|
||||
assert.equal(owner.captureId, metadata.captureId);
|
||||
assert.equal(
|
||||
socket.sent.length,
|
||||
wireBeforeC,
|
||||
'C must still be wholly deferred',
|
||||
);
|
||||
assert.deepEqual(
|
||||
owner.messages.map((message) =>
|
||||
message.realtimeInput?.activityStart
|
||||
? 'start'
|
||||
: message.realtimeInput?.audio
|
||||
? 'audio'
|
||||
: message.realtimeInput?.activityEnd
|
||||
? 'end'
|
||||
: 'other',
|
||||
),
|
||||
['start', 'audio', 'end'],
|
||||
);
|
||||
return { socket, owner, metadata };
|
||||
}
|
||||
|
||||
async function assertHealthySuccessor(h, socket, callId) {
|
||||
const beforeHeight = h.state.height;
|
||||
await h.tool(callId, socket);
|
||||
assertSuccessfulZoom(h, socket, callId, beforeHeight);
|
||||
assert.equal(h.controller.pendingTools, 0);
|
||||
h.complete(socket); // Native post-tool response boundary, with no output PCM.
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.session.state, 'listening');
|
||||
}
|
||||
|
||||
for (const mode of ['typed replacement', 'Stop', 'Stop+restart']) {
|
||||
test(
|
||||
`F04-5 deferred PTT flush loses its owner on ${mode}`,
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const { socket: oldSocket, owner } = await prepareDeferredPtt(h);
|
||||
const oldAudio = h.audios.at(-1);
|
||||
let observed = 0;
|
||||
let sentAtObserverReturn;
|
||||
let restart;
|
||||
let stateAtObserverReturn;
|
||||
const unsubscribe = h.session.subscribe((event) => {
|
||||
if (
|
||||
event.type !== 'transcript' ||
|
||||
event.role !== 'user' ||
|
||||
event.text !== '' ||
|
||||
observed
|
||||
)
|
||||
return;
|
||||
observed++;
|
||||
if (mode === 'typed replacement') {
|
||||
assert.equal(h.session.sendText(TEXT), true);
|
||||
} else {
|
||||
h.session.stop();
|
||||
if (mode === 'Stop+restart')
|
||||
restart = h.session.start({ pushToTalk: true });
|
||||
}
|
||||
sentAtObserverReturn = oldSocket.sent.length;
|
||||
stateAtObserverReturn = {
|
||||
epoch: h.controller.epoch,
|
||||
turn: h.controller.turn,
|
||||
replyPending: h.controller.replyPending,
|
||||
forwardedActivityOpen: h.controller.forwardedActivityOpen,
|
||||
};
|
||||
});
|
||||
t.after(unsubscribe);
|
||||
|
||||
h.complete(oldSocket); // A's boundary starts flushing C. C's start emits.
|
||||
assert.equal(
|
||||
observed,
|
||||
1,
|
||||
'must enter the actual flush/onset observer boundary',
|
||||
);
|
||||
assert.ok(
|
||||
stateAtObserverReturn,
|
||||
'observer must complete its replacement action',
|
||||
);
|
||||
assert.deepEqual(
|
||||
oldSocket.sent.slice(sentAtObserverReturn),
|
||||
[],
|
||||
'retired C must send neither queued PCM nor an extra end after the observer returns',
|
||||
);
|
||||
assert.deepEqual(
|
||||
{
|
||||
epoch: h.controller.epoch,
|
||||
turn: h.controller.turn,
|
||||
replyPending: h.controller.replyPending,
|
||||
forwardedActivityOpen: h.controller.forwardedActivityOpen,
|
||||
},
|
||||
stateAtObserverReturn,
|
||||
'old flush must not write successor state',
|
||||
);
|
||||
assert.equal(h.controller.deferredInput, null);
|
||||
assert.equal(
|
||||
h.controller.typedIntent?.text ?? null,
|
||||
mode === 'typed replacement' ? TEXT : null,
|
||||
);
|
||||
|
||||
let successorSocket;
|
||||
if (mode === 'typed replacement') {
|
||||
assert.equal(h.controller.epoch, owner.epoch);
|
||||
assert.notEqual(h.controller.turn, owner.turn);
|
||||
assert.equal(
|
||||
oldSocket.sent.some(
|
||||
(message) => message.realtimeInput?.audio?.data === OLD_PCM,
|
||||
),
|
||||
false,
|
||||
);
|
||||
const typedIndex = oldSocket.sent.findIndex((message) =>
|
||||
message.clientContent?.turns?.some((turn) =>
|
||||
turn.parts?.some((part) => part.text === TEXT),
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
typedIndex >= 0,
|
||||
'owned replacement must actually reach transport',
|
||||
);
|
||||
// The replacement legitimately closes the already-forwarded C start once.
|
||||
assert.ok(oldSocket.sent[typedIndex - 1]?.realtimeInput?.activityEnd);
|
||||
h.complete(oldSocket); // B's remaining retired boundary, deliberately separate.
|
||||
assert.equal(h.controller.retiredReplyPending, false);
|
||||
assert.equal(h.controller.replyPending, true);
|
||||
successorSocket = oldSocket;
|
||||
} else {
|
||||
assert.equal(oldAudio.closed, true);
|
||||
assert.equal(oldSocket.closed, true);
|
||||
if (!restart) restart = h.session.start({ pushToTalk: true });
|
||||
await restart;
|
||||
successorSocket = h.sockets.at(-1);
|
||||
assert.notEqual(successorSocket, oldSocket);
|
||||
assert.equal(
|
||||
successorSocket.sent.some((message) => message.realtimeInput?.audio),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
successorSocket.sent.some(
|
||||
(message) => message.realtimeInput?.activityEnd,
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(h.session.sendText(TEXT), true);
|
||||
}
|
||||
await assertHealthySuccessor(
|
||||
h,
|
||||
successorSocket,
|
||||
`healthy-after-deferred-${mode}`,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test(
|
||||
'F04-5 deferred typed onSent cannot resurrect replyPending after Stop+restart',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const oldSocket = await h.start();
|
||||
assert.equal(h.session.sendText('A'), true);
|
||||
assert.equal(h.session.sendText('B'), true);
|
||||
assert.equal(h.session.sendText('C deferred text'), true);
|
||||
assert.equal(h.controller.deferredInput?.kind, 'user-text');
|
||||
let restart;
|
||||
let observed = 0;
|
||||
const unsubscribe = h.session.subscribe((event) => {
|
||||
if (
|
||||
event.type !== 'transcript' ||
|
||||
event.role !== 'user' ||
|
||||
event.text !== 'C deferred text'
|
||||
)
|
||||
return;
|
||||
observed++;
|
||||
h.session.stop();
|
||||
restart = h.session.start();
|
||||
});
|
||||
t.after(unsubscribe);
|
||||
h.complete(oldSocket);
|
||||
assert.equal(observed, 1);
|
||||
assert.equal(
|
||||
h.controller.replyPending,
|
||||
false,
|
||||
'old onSent wrapper must not resurrect a stopped reply',
|
||||
);
|
||||
await restart;
|
||||
assert.equal(h.controller.replyPending, false);
|
||||
const socket = h.sockets.at(-1);
|
||||
assert.equal(
|
||||
socket.sent.length,
|
||||
1,
|
||||
'replacement connection should contain setup only',
|
||||
);
|
||||
assert.equal(h.session.sendText(TEXT), true);
|
||||
await assertHealthySuccessor(h, socket, 'healthy-after-deferred-text');
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'F04-5 ordinary deferred PTT flush preserves start, PCM and end exactly once',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const { socket } = await prepareDeferredPtt(h);
|
||||
const before = socket.sent.length;
|
||||
h.complete(socket);
|
||||
assert.deepEqual(
|
||||
socket.sent
|
||||
.slice(before)
|
||||
.map((message) =>
|
||||
message.realtimeInput?.activityStart
|
||||
? 'start'
|
||||
: message.realtimeInput?.audio?.data === OLD_PCM
|
||||
? 'audio'
|
||||
: message.realtimeInput?.activityEnd
|
||||
? 'end'
|
||||
: 'other',
|
||||
),
|
||||
['start', 'audio', 'end'],
|
||||
);
|
||||
assert.equal(h.controller.deferredInput, null);
|
||||
assert.equal(h.controller.replyPending, true);
|
||||
assert.equal(h.controller.retiredReplyPending, true);
|
||||
h.complete(socket); // B's boundary; C remains the owned submitted input.
|
||||
assert.equal(h.controller.retiredReplyPending, false);
|
||||
assert.equal(h.controller.replyPending, true);
|
||||
await assertHealthySuccessor(h, socket, 'healthy-ordinary-deferred');
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,287 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
nativeIntegrationFixture,
|
||||
assertSuccessfulZoom,
|
||||
turnOfEventLoop,
|
||||
} from '../testSupport/geminiIntegrationFixture.mjs';
|
||||
|
||||
const options = { concurrency: false, timeout: 5000 };
|
||||
const resultFor = (h, id) =>
|
||||
h.events.filter(
|
||||
(event) => event.type === 'action-result' && event.callId === id,
|
||||
);
|
||||
const callFor = (h, id) =>
|
||||
h.events.filter(
|
||||
(event) => event.type === 'action-call' && event.callId === id,
|
||||
);
|
||||
|
||||
test(
|
||||
'native tool reaches actual common session and camera runner without a caption',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const ws = await h.start();
|
||||
h.activity(ws); // Native activity is input framing, not transcript admission.
|
||||
await h.tool('ordinary', ws);
|
||||
assertSuccessfulZoom(h, ws, 'ordinary', 1000);
|
||||
assert.equal(h.runnerCalls.length, 1);
|
||||
assert.equal(h.runnerCalls[0].options.callId, 'ordinary');
|
||||
assert.equal(h.runnerCalls[0].options.signal.aborted, false);
|
||||
assert.equal(h.runnerCalls[0].options.isCurrent(), true);
|
||||
h.complete(ws);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
},
|
||||
);
|
||||
|
||||
for (const cancellation of ['stop', 'native interruption']) {
|
||||
test(
|
||||
`${cancellation} discards a stale completed action result; successor changes camera once`,
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const oldSocket = await h.start();
|
||||
h.activity(oldSocket);
|
||||
const gate = h.holdResult('old');
|
||||
const oldDispatch = h.tool('old', oldSocket);
|
||||
const committedResult = await gate.ready;
|
||||
assert.equal(committedResult.ok, true);
|
||||
assert.equal(
|
||||
h.state.height,
|
||||
750,
|
||||
'the legitimate first action committed before cancellation',
|
||||
);
|
||||
assert.equal(callFor(h, 'old').length, 1);
|
||||
assert.deepEqual(resultFor(h, 'old'), []);
|
||||
assert.deepEqual(h.outputs(oldSocket), []);
|
||||
|
||||
if (cancellation === 'stop') h.session.stop();
|
||||
else
|
||||
oldSocket.message({
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
assert.equal(h.runnerCalls[0].options.signal.aborted, true);
|
||||
assert.equal(h.runnerCalls[0].options.isCurrent(), false);
|
||||
await oldDispatch; // Cancellation must settle before the held result returns.
|
||||
gate.release();
|
||||
await turnOfEventLoop();
|
||||
assert.equal(
|
||||
h.state.height,
|
||||
750,
|
||||
'settling a cancelled result adds no camera mutation',
|
||||
);
|
||||
assert.equal(h.state.zooms.length, 1);
|
||||
assert.deepEqual(resultFor(h, 'old'), []);
|
||||
assert.deepEqual(h.outputs(oldSocket), []);
|
||||
if (cancellation === 'native interruption')
|
||||
assert.ok(h.events.some((event) => event.type === 'interruption'));
|
||||
|
||||
const successorSocket =
|
||||
cancellation === 'stop' ? await h.start() : oldSocket;
|
||||
h.activity(successorSocket);
|
||||
await h.tool('successor', successorSocket);
|
||||
assertSuccessfulZoom(h, successorSocket, 'successor', 750);
|
||||
assert.equal(h.state.zooms.length, 2);
|
||||
assert.equal(h.runnerCalls.length, 2);
|
||||
assert.equal(
|
||||
h.outputs(successorSocket).some((output) => output.id === 'old'),
|
||||
false,
|
||||
);
|
||||
h.complete(successorSocket);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
for (const phase of ['common connecting', 'adapter connecting']) {
|
||||
test(
|
||||
`Stop from ${phase} observer prevents acquisition and permits fresh Start`,
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
let stopped = false;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
const matchingDetail =
|
||||
phase === 'common connecting'
|
||||
? event.detail === undefined
|
||||
: typeof event.detail === 'string' && event.detail.length > 0;
|
||||
if (
|
||||
!stopped &&
|
||||
event.type === 'state' &&
|
||||
event.state === 'connecting' &&
|
||||
matchingDetail
|
||||
) {
|
||||
stopped = true;
|
||||
h.session.stop();
|
||||
}
|
||||
});
|
||||
await h.session.start();
|
||||
remove();
|
||||
assert.equal(
|
||||
stopped,
|
||||
true,
|
||||
'the chosen production observer boundary was exercised',
|
||||
);
|
||||
assert.deepEqual(h.acquisitions, {
|
||||
microphone: 0,
|
||||
audio: 0,
|
||||
token: 0,
|
||||
socket: 0,
|
||||
});
|
||||
assert.equal(h.session.state, 'idle');
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
assert.equal(h.clock.timers.size, 0);
|
||||
const ws = await h.start();
|
||||
h.activity(ws);
|
||||
await h.tool('fresh-after-start-cancel', ws);
|
||||
assertSuccessfulZoom(h, ws, 'fresh-after-start-cancel', 1000);
|
||||
h.complete(ws);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test(
|
||||
'Stop from common action-call observer prevents runner and leaves no unhandled rejection',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const ws = await h.start();
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (event.type === 'action-call' && event.callId === 'cancel-at-call')
|
||||
h.session.stop();
|
||||
});
|
||||
h.activity(ws);
|
||||
await h.tool('cancel-at-call', ws);
|
||||
// Do not install a swallowing process rejection listener. node:test records
|
||||
// an unhandled rejection as a failure; these checkpoints let it surface here.
|
||||
await turnOfEventLoop();
|
||||
await turnOfEventLoop();
|
||||
remove();
|
||||
assert.equal(callFor(h, 'cancel-at-call').length, 1);
|
||||
assert.deepEqual(resultFor(h, 'cancel-at-call'), []);
|
||||
assert.equal(h.runnerCalls.length, 0);
|
||||
assert.equal(h.state.height, 1000);
|
||||
assert.deepEqual(h.outputs(ws), []);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
assert.equal(h.clock.timers.size, 0);
|
||||
const successor = await h.start();
|
||||
h.activity(successor);
|
||||
await h.tool('after-call-cancel', successor);
|
||||
assertSuccessfulZoom(h, successor, 'after-call-cancel', 1000);
|
||||
h.complete(successor);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'Stop from executing observer cannot recreate narration or dispatch the runner',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const ws = await h.start();
|
||||
let stopped = false;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (!stopped && event.type === 'state' && event.state === 'executing') {
|
||||
stopped = true;
|
||||
h.session.stop();
|
||||
}
|
||||
});
|
||||
h.activity(ws);
|
||||
await h.tool('cancel-at-executing', ws);
|
||||
remove();
|
||||
assert.equal(stopped, true);
|
||||
assert.equal(h.runnerCalls.length, 0);
|
||||
assert.deepEqual(callFor(h, 'cancel-at-executing'), []);
|
||||
assert.deepEqual(resultFor(h, 'cancel-at-executing'), []);
|
||||
assert.equal(h.state.height, 1000);
|
||||
assert.deepEqual(h.outputs(ws), []);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
assert.equal(h.clock.timers.size, 0);
|
||||
assert.equal(ws.closed, true);
|
||||
assert.ok(h.audios.every((audio) => audio.closed));
|
||||
assert.ok(h.tracks.every((track) => track.readyState === 'ended'));
|
||||
const before = {
|
||||
events: h.events.length,
|
||||
wire: ws.sent.length,
|
||||
earcon: h.state.earconStarts,
|
||||
};
|
||||
h.clock.advance(60_000);
|
||||
assert.deepEqual(
|
||||
{
|
||||
events: h.events.length,
|
||||
wire: ws.sent.length,
|
||||
earcon: h.state.earconStarts,
|
||||
},
|
||||
before,
|
||||
);
|
||||
const successor = await h.start();
|
||||
h.activity(successor);
|
||||
await h.tool('after-executing-cancel', successor);
|
||||
assertSuccessfulZoom(h, successor, 'after-executing-cancel', 1000);
|
||||
h.complete(successor);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'native idle observer restart remains connecting and public Stop cancels its pending permission',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
await h.start();
|
||||
let grant;
|
||||
const permission = new Promise((resolve) => {
|
||||
grant = resolve;
|
||||
});
|
||||
h.controller.getUserMedia = () => permission;
|
||||
const states = [];
|
||||
let restarted = false;
|
||||
let startup;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (event.type === 'state') states.push(event.state);
|
||||
if (!restarted && event.type === 'state' && event.state === 'idle') {
|
||||
restarted = true;
|
||||
startup = h.session.start();
|
||||
}
|
||||
});
|
||||
h.session.stop();
|
||||
remove();
|
||||
assert.equal(restarted, true);
|
||||
assert.equal(states.filter((state) => state === 'idle').length, 1);
|
||||
assert.equal(h.session.state, 'connecting');
|
||||
assert.equal(h.controller.status, 'connecting');
|
||||
assert.equal(
|
||||
h.session.isActive(),
|
||||
true,
|
||||
'the common button/facade must offer Stop',
|
||||
);
|
||||
h.session.stop();
|
||||
assert.equal(h.session.state, 'idle');
|
||||
assert.equal(h.controller.status, 'idle');
|
||||
const lateTrack = {
|
||||
readyState: 'live',
|
||||
stopCalls: 0,
|
||||
stop() {
|
||||
this.stopCalls++;
|
||||
this.readyState = 'ended';
|
||||
},
|
||||
};
|
||||
h.tracks.push(lateTrack);
|
||||
grant({ getTracks: () => [lateTrack], getAudioTracks: () => [lateTrack] });
|
||||
await startup;
|
||||
await turnOfEventLoop();
|
||||
assert.equal(lateTrack.stopCalls, 1);
|
||||
assert.equal(
|
||||
h.requests.length,
|
||||
1,
|
||||
'cancelled restart must not mint another token',
|
||||
);
|
||||
assert.equal(
|
||||
h.sockets.length,
|
||||
1,
|
||||
'cancelled restart must not open another socket',
|
||||
);
|
||||
assert.equal(h.session.state, 'idle');
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,85 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { harness } from '../testSupport/geminiNativeHarness.mjs';
|
||||
|
||||
const caption = (socket, text) =>
|
||||
socket.message({ serverContent: { inputTranscription: { text } } });
|
||||
const userEvents = (events) =>
|
||||
events.filter(
|
||||
(event) => event.type === 'transcript' && event.role === 'user',
|
||||
);
|
||||
|
||||
test('native caption events preserve received advisory segments without granting tool authority', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
const start = h.events.length;
|
||||
for (const text of ['little. ', 'Zoom in a ']) caption(ws, text);
|
||||
const received = userEvents(h.events.slice(start));
|
||||
assert.deepEqual(
|
||||
received.map((event) => event.text),
|
||||
['little. ', 'Zoom in a '],
|
||||
);
|
||||
assert.equal(
|
||||
received.every(
|
||||
(event) =>
|
||||
event.delta === true &&
|
||||
event.final === false &&
|
||||
event.attribution === 'unverified' &&
|
||||
event.responseId === null,
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(h.c.currentTypedInput(), null);
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: 'ordinary-native',
|
||||
name: 'get_current_view_state',
|
||||
args: {},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls.length, 1);
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(h.calls[0].options.isCurrent(), true);
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(
|
||||
userEvents(h.events.slice(start)).some((event) => event.final),
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('native old socket captions cannot enter a restarted controller event stream', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const old = h.sockets[0];
|
||||
caption(old, 'old partial');
|
||||
h.c.stop();
|
||||
await h.c.start();
|
||||
const start = h.events.length;
|
||||
const current = h.sockets[1];
|
||||
caption(current, 'Zoom ');
|
||||
caption(old, 'stale old socket');
|
||||
caption(current, 'in a little.');
|
||||
current.message({ serverContent: { turnComplete: true } });
|
||||
assert.deepEqual(
|
||||
userEvents(h.events.slice(start)).map((event) => event.text),
|
||||
['Zoom ', 'in a little.'],
|
||||
);
|
||||
assert.equal(h.tracks[0].readyState, 'ended');
|
||||
assert.equal(h.captures[0].closed, true);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
@@ -0,0 +1,667 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
harness,
|
||||
tick,
|
||||
deferred,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
import { withToolCatalog } from './gevRealtime.js';
|
||||
|
||||
const audioContent = (data) => ({
|
||||
modelTurn: {
|
||||
parts: [{ inlineData: { data, mimeType: 'audio/pcm;rate=24000' } }],
|
||||
},
|
||||
});
|
||||
const request = (ws, id, name = 'get_current_view_state', args = {}) =>
|
||||
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
|
||||
const responses = (ws) =>
|
||||
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
|
||||
const onset = (ws) => {
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
};
|
||||
const caption = (ws, text) =>
|
||||
ws.message({ serverContent: { inputTranscription: { text } } });
|
||||
|
||||
for (const [id, name, args] of [
|
||||
['H01', 'get_entity_context', {}],
|
||||
['H02', 'control_radio', { action: 'status' }],
|
||||
]) {
|
||||
test(`${id} native: cancelling a current ${name} query preserves unrelated Radio preflight`, async () => {
|
||||
const ready = deferred(),
|
||||
query = deferred(),
|
||||
executions = [];
|
||||
const h = harness({
|
||||
runAction: (name, args, options) => {
|
||||
executions.push({ name, args, options });
|
||||
return args.action === 'play'
|
||||
? Promise.resolve({
|
||||
ok: true,
|
||||
action: 'control_radio',
|
||||
radioAction: 'play',
|
||||
radioPlaybackRequested: true,
|
||||
})
|
||||
: query.promise;
|
||||
},
|
||||
});
|
||||
h.c.radio.radioLayer.playForVoice = () => ready.promise;
|
||||
try {
|
||||
await h.c.start();
|
||||
h.c.sendTextCommand('Play radio');
|
||||
const ws = h.sockets[0];
|
||||
request(ws, 'owner', 'control_radio', { action: 'play' });
|
||||
await h.c.toolQueue;
|
||||
ws.message({
|
||||
serverContent: { ...audioContent('confirmation'), turnComplete: true },
|
||||
});
|
||||
h.captures[0].drain();
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, true);
|
||||
request(ws, 'query', name, args);
|
||||
await tick();
|
||||
assert.equal(
|
||||
executions.length,
|
||||
2,
|
||||
'current native query may start after provider completion',
|
||||
);
|
||||
const epoch = h.c.radio.radioHandoffEpoch;
|
||||
const pending = h.c.radio.pendingRadioPlaybackResult;
|
||||
const attempt = h.c.radio.radioHandoffAttemptId;
|
||||
const ownerCurrent = executions[0].options.isCurrent();
|
||||
ws.message({ toolCallCancellation: { ids: ['unknown', 'query'] } });
|
||||
await h.c.toolQueue;
|
||||
assert.equal(executions[1].options.signal.aborted, true);
|
||||
assert.equal(executions[1].options.isCurrent(), false);
|
||||
assert.equal(h.c.radio.radioHandoffEpoch, epoch);
|
||||
assert.equal(executions[0].options.signal.aborted, false);
|
||||
assert.equal(executions[0].options.isCurrent(), ownerCurrent);
|
||||
assert.equal(h.c.radio.pendingRadioPlaybackResult, pending);
|
||||
assert.equal(h.c.radio.radioHandoffAttemptId, attempt);
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, true);
|
||||
assert.equal(h.state.stopped, 0);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.deepEqual(
|
||||
responses(ws).map((r) => r.id),
|
||||
['owner'],
|
||||
);
|
||||
assert.equal(responses(ws)[0].response.ok, true);
|
||||
query.resolve({ ok: true });
|
||||
ready.resolve(true);
|
||||
await tick();
|
||||
assert.equal(h.c.status, 'idle');
|
||||
assert.equal(h.state.ducked, false);
|
||||
assert.deepEqual(
|
||||
responses(ws).map((r) => r.id),
|
||||
['owner'],
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const [id, prior] of [
|
||||
['H03', 'playing'],
|
||||
['H04', 'before-first-output'],
|
||||
]) {
|
||||
test(`${id} native: PTT fences ${prior} reply and recovers the new call after its boundary`, async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start({ pushToTalk: true });
|
||||
const ws = h.sockets[0],
|
||||
audio = h.captures[0];
|
||||
if (prior === 'playing')
|
||||
ws.message({ serverContent: audioContent('old') });
|
||||
else {
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
audio.callbacks.onAudio(
|
||||
{ data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
|
||||
audio.metadata,
|
||||
);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
}
|
||||
h.c.input.pushToTalkKeyHeld = true;
|
||||
assert.equal(h.c.input.bargeIn(), true);
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
const turn = h.c.turn,
|
||||
played = audio.played.length,
|
||||
firstEvent = h.events.length;
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...audioContent('late-old'),
|
||||
outputTranscription: { text: 'retired output' },
|
||||
inputTranscription: { text: 'new request' },
|
||||
},
|
||||
});
|
||||
request(ws, 'retired-call', 'fly_to_location', { query: 'old target' });
|
||||
await h.c.toolQueue;
|
||||
assert.equal(audio.played.length, played);
|
||||
assert.equal(audio.pending, false);
|
||||
assert.equal(h.calls.length, 0);
|
||||
assert.equal(
|
||||
h.events.slice(firstEvent).some((e) => e.role === 'assistant'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
h.events
|
||||
.slice(firstEvent)
|
||||
.some((e) => e.role === 'user' && e.text === 'new request'),
|
||||
true,
|
||||
);
|
||||
ws.message({ serverContent: { interrupted: true } });
|
||||
assert.equal(h.c.turn, turn);
|
||||
ws.message({ serverContent: audioContent('after-interrupted') });
|
||||
assert.equal(
|
||||
audio.played.length,
|
||||
played,
|
||||
'interrupted alone cannot release the fence',
|
||||
);
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...audioContent('boundary-old'),
|
||||
outputTranscription: { text: 'still retired' },
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
audio.played.length,
|
||||
played,
|
||||
'coalesced old boundary output is discarded',
|
||||
);
|
||||
assert.equal(
|
||||
h.events.slice(firstEvent).some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
caption(ws, 'unrelated advisory words');
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: 'retired-call',
|
||||
name: 'fly_to_location',
|
||||
args: { query: 'replayed old target' },
|
||||
},
|
||||
{
|
||||
id: 'new-call',
|
||||
name: 'fly_to_location',
|
||||
args: { query: 'new target' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(
|
||||
h.calls.map((c) => c.args.query),
|
||||
['new target'],
|
||||
);
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...audioContent('new'),
|
||||
outputTranscription: { text: 'new reply' },
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
assert.equal(audio.played.at(-1).data, 'new');
|
||||
audio.drain();
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion').status,
|
||||
'completed',
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
test('H05 native: local typed presentation survives audio drain and new input revokes old call ownership', async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0],
|
||||
audio = h.captures[0];
|
||||
h.c.sendTextCommand('Show the current view');
|
||||
const response = h.c.currentTypedInput();
|
||||
assert.equal(response.text, 'Show the current view');
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
|
||||
.text,
|
||||
response.text,
|
||||
);
|
||||
request(ws, 'first');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls[0].options.userTurn, undefined);
|
||||
assert.equal(h.calls[0].options.isCurrent(), true);
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...audioContent('confirmation'),
|
||||
outputTranscription: { text: 'Here is the current view' },
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
h.events.some((e) => e.type === 'completion'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.role === 'assistant').responseId,
|
||||
response.responseId,
|
||||
);
|
||||
audio.drain();
|
||||
const completion = h.events.findLast((e) => e.type === 'completion');
|
||||
assert.equal(completion.responseId, response.responseId);
|
||||
assert.equal(completion.spoken, true);
|
||||
assert.equal(completion.audioDrained, true);
|
||||
assert.equal(h.c.currentTypedInput(), response);
|
||||
assert.equal(response.text, 'Show the current view');
|
||||
request(ws, 'same-turn');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.calls[1].options.userTurn, undefined);
|
||||
assert.equal(h.c.currentTypedInput(), response);
|
||||
h.c.sendTextCommand('New request');
|
||||
assert.notEqual(h.c.currentTypedInput(), response);
|
||||
assert.equal(h.c.currentTypedInput().text, 'New request');
|
||||
assert.equal(h.calls[0].options.isCurrent(), false);
|
||||
assert.equal(h.calls[1].options.isCurrent(), false);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('H08 native: replacing pending runner work cannot underflow tool counts or block successor', async () => {
|
||||
const gate = deferred(),
|
||||
executions = [];
|
||||
const h = harness({
|
||||
runAction: (name, args, options) => {
|
||||
executions.push({ name, args, options });
|
||||
return options.callId === 'old'
|
||||
? gate.promise
|
||||
: Promise.resolve({ ok: true });
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
request(ws, 'old');
|
||||
await tick();
|
||||
const oldWork = h.c.toolQueue;
|
||||
assert.equal(h.c.pendingTools, 1);
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
await oldWork;
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(executions[0].options.signal.aborted, true);
|
||||
assert.equal(executions[0].options.isCurrent(), false);
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
request(ws, 'fresh');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(executions.at(-1).options.callId, 'fresh');
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
gate.resolve({ ok: true });
|
||||
await tick();
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.deepEqual(
|
||||
responses(ws).map((r) => r.id),
|
||||
['fresh'],
|
||||
);
|
||||
ws.message({
|
||||
serverContent: { ...audioContent('fresh reply'), turnComplete: true },
|
||||
});
|
||||
h.captures[0].drain();
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'completion').status,
|
||||
'completed',
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('H10 native: held speech dispatches without caption authority and new text revokes its call', async () => {
|
||||
const gate = deferred(),
|
||||
executions = [];
|
||||
const h = harness({
|
||||
runAction: (name, args, options) => {
|
||||
executions.push({ name, args, options });
|
||||
return gate.promise;
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start({ pushToTalk: true });
|
||||
const ws = h.sockets[0],
|
||||
audio = h.captures[0];
|
||||
h.c.input.bargeIn();
|
||||
h.c.input.setMicrophoneEnabled(true);
|
||||
audio.callbacks.onAudio(
|
||||
{ data: 'AA==', mimeType: 'audio/pcm;rate=16000' },
|
||||
audio.metadata,
|
||||
);
|
||||
h.c.input.setMicrophoneEnabled(false);
|
||||
caption(ws, 'Show the current view');
|
||||
assert.equal(h.c.currentTypedInput(), null);
|
||||
assert.equal(
|
||||
h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
|
||||
.text,
|
||||
'Show the current view',
|
||||
);
|
||||
request(ws, 'held');
|
||||
await tick();
|
||||
const oldWork = h.c.toolQueue;
|
||||
assert.equal(executions.length, 1);
|
||||
assert.equal(executions[0].options.userTurn, undefined);
|
||||
assert.equal(executions[0].options.isCurrent(), true);
|
||||
h.c.sendTextCommand('new intent');
|
||||
await oldWork;
|
||||
assert.equal(executions[0].options.signal.aborted, true);
|
||||
assert.equal(executions[0].options.isCurrent(), false);
|
||||
gate.resolve({ ok: true });
|
||||
await tick();
|
||||
assert.equal(
|
||||
responses(ws).some((r) => r.id === 'held'),
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
for (const [id, restart] of [
|
||||
['H11', false],
|
||||
['H12', true],
|
||||
]) {
|
||||
test(`${id} native: pending tool cannot revive voice after Stop${restart ? '/restart' : ''}`, async () => {
|
||||
const gate = deferred(),
|
||||
mutations = [],
|
||||
executions = [];
|
||||
const h = harness({
|
||||
runAction: async (name, args, options) => {
|
||||
executions.push(options);
|
||||
await gate.promise;
|
||||
if (options.isCurrent()) mutations.push(name);
|
||||
return { ok: true };
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
request(ws, 'old');
|
||||
await tick();
|
||||
const pending = h.c.toolQueue;
|
||||
assert.equal(h.c.pendingTools, 1);
|
||||
h.c.stop();
|
||||
const boundary = h.events.length;
|
||||
const successor = restart ? h.c.start() : null;
|
||||
await pending;
|
||||
assert.equal(
|
||||
h.events
|
||||
.slice(boundary)
|
||||
.some((e) => e.type === 'state' && e.state === 'listening'),
|
||||
false,
|
||||
);
|
||||
assert.equal(executions[0].signal.aborted, true);
|
||||
assert.equal(executions[0].isCurrent(), false);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
if (successor) await successor;
|
||||
const laterBoundary = h.events.length;
|
||||
gate.resolve();
|
||||
caption(ws, 'late old speech');
|
||||
await tick();
|
||||
assert.deepEqual(mutations, []);
|
||||
assert.equal(
|
||||
h.events.slice(laterBoundary).some((e) => e.type === 'state'),
|
||||
false,
|
||||
);
|
||||
assert.equal(responses(ws).length, 0);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const [id, boundary] of [
|
||||
['H14', 'interrupted'],
|
||||
['H15', 'natural'],
|
||||
]) {
|
||||
test(`${id} native: shared runner zoom executes promptly after ${boundary} retired boundary`, async () => {
|
||||
const mutations = [],
|
||||
observed = [];
|
||||
const guarded = withToolCatalog(
|
||||
async (name, args) => {
|
||||
mutations.push({ name, args });
|
||||
return { ok: true };
|
||||
},
|
||||
async () => null,
|
||||
);
|
||||
const h = harness({
|
||||
runAction: (name, args, options) => {
|
||||
observed.push(options);
|
||||
return guarded(name, args, options);
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message({ serverContent: audioContent('old reply') });
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...(boundary === 'interrupted' ? { interrupted: true } : {}),
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
caption(ws, 'zoom in');
|
||||
request(ws, 'zoom', 'adjust_camera_zoom', { direction: 'in' });
|
||||
await tick();
|
||||
assert.equal(
|
||||
mutations.length,
|
||||
1,
|
||||
'native action executes without a transcript wait',
|
||||
);
|
||||
assert.equal(observed[0].userTurn, undefined);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(
|
||||
responses(ws).some((r) => r.response.voiceIntentUnverified),
|
||||
false,
|
||||
);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const [id, exit] of [
|
||||
['H18', 'stop'],
|
||||
['H19', 'replacement'],
|
||||
]) {
|
||||
test(`${id} native: ${exit} prevents queued camera dispatch and late captions cannot revive it`, async () => {
|
||||
const mutations = [],
|
||||
observed = [];
|
||||
const guarded = withToolCatalog(
|
||||
async (name) => {
|
||||
mutations.push(name);
|
||||
return { ok: true };
|
||||
},
|
||||
async () => null,
|
||||
);
|
||||
const h = harness({
|
||||
runAction: (name, args, options) => {
|
||||
observed.push(name);
|
||||
return guarded(name, args, options);
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
request(ws, 'queued-old', 'adjust_camera_zoom', { direction: 'in' });
|
||||
const oldWork = h.c.toolQueue;
|
||||
assert.equal(h.c.pendingTools, 1);
|
||||
if (exit === 'stop') h.c.stop();
|
||||
else h.c.sendTextCommand('Show the whole globe');
|
||||
caption(ws, 'zoom in');
|
||||
await oldWork;
|
||||
assert.deepEqual(mutations, []);
|
||||
assert.deepEqual(observed, []);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
if (exit === 'replacement') {
|
||||
ws.message({
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
request(ws, 'fresh-camera', 'zoom_to_globe');
|
||||
await h.c.toolQueue;
|
||||
assert.deepEqual(mutations, ['zoom_to_globe']);
|
||||
assert.deepEqual(observed, ['zoom_to_globe']);
|
||||
assert.equal(responses(ws).at(-1).response.ok, true);
|
||||
}
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const settlement of ['queued', 'running']) {
|
||||
test(`native: cancelled ${settlement} tool settles a completed provider turn`, async () => {
|
||||
const result = deferred();
|
||||
let executions = 0;
|
||||
const h = harness({
|
||||
runAction: () => {
|
||||
executions++;
|
||||
return result.promise;
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
h.c.sendTextCommand('Read current view');
|
||||
const ws = h.sockets[0];
|
||||
request(ws, 'cancelled-query');
|
||||
if (settlement === 'running') await tick();
|
||||
ws.message({ toolCallCancellation: { ids: ['cancelled-query'] } });
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(executions, settlement === 'queued' ? 0 : 1);
|
||||
assert.deepEqual(responses(ws), []);
|
||||
assert.equal(
|
||||
h.events.filter((event) => event.type === 'completion').length,
|
||||
1,
|
||||
);
|
||||
assert.equal(h.c.pendingCompletionStatus, null);
|
||||
result.resolve({ ok: true });
|
||||
await tick();
|
||||
h.captures[0].drain();
|
||||
assert.equal(
|
||||
h.events.filter((event) => event.type === 'completion').length,
|
||||
1,
|
||||
);
|
||||
assert.deepEqual(responses(ws), []);
|
||||
} finally {
|
||||
result.resolve({ ok: true });
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const order of ['before', 'after']) {
|
||||
test(`native: cancelling queued output ${order} turnComplete emits one completion`, async () => {
|
||||
const h = harness();
|
||||
try {
|
||||
await h.c.start();
|
||||
h.c.sendTextCommand('Read current view');
|
||||
const ws = h.sockets[0];
|
||||
ws.bufferedAmount = 1e9;
|
||||
request(ws, 'queued-output');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.controlQueue.length, 1);
|
||||
assert.equal(responses(ws).length, 0);
|
||||
if (order === 'after')
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
ws.message({ toolCallCancellation: { ids: ['queued-output'] } });
|
||||
if (order === 'before')
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(h.c.controlQueue.length, 0);
|
||||
assert.equal(h.c.controlTimer, null);
|
||||
assert.equal(h.c.pendingCompletionStatus, null);
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
ws.bufferedAmount = 0;
|
||||
h.advance(1000);
|
||||
h.captures[0].drain();
|
||||
assert.deepEqual(responses(ws), []);
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const successor of ['input', 'stop']) {
|
||||
test(`native: queued-output cancellation completion fences reentrant ${successor}`, async () => {
|
||||
let h;
|
||||
let successorTurn;
|
||||
h = harness({
|
||||
emit: (event) => {
|
||||
h.events.push(event);
|
||||
if (event.type !== 'completion') return;
|
||||
h.sockets[0].bufferedAmount = 0;
|
||||
if (successor === 'stop') h.c.stop();
|
||||
else {
|
||||
h.c.sendTextCommand('Show the whole globe');
|
||||
successorTurn = h.c.turn;
|
||||
}
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
h.c.sendTextCommand('Read current view');
|
||||
const ws = h.sockets[0];
|
||||
ws.bufferedAmount = 1e9;
|
||||
request(ws, 'queued-output');
|
||||
await h.c.toolQueue;
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
const callCount = h.calls.length;
|
||||
ws.message({
|
||||
toolCallCancellation: { ids: ['queued-output'] },
|
||||
serverContent: {
|
||||
outputTranscription: { text: 'Retired packet caption' },
|
||||
turnComplete: true,
|
||||
},
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{ id: 'retired-tool', name: 'get_current_view_state', args: {} },
|
||||
],
|
||||
},
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
|
||||
assert.equal(h.calls.length, callCount);
|
||||
assert.deepEqual(responses(ws), []);
|
||||
assert.ok(
|
||||
!h.events.some((e) =>
|
||||
JSON.stringify(e).includes('Retired packet caption'),
|
||||
),
|
||||
);
|
||||
if (successor === 'input') {
|
||||
assert.equal(h.c.turn, successorTurn);
|
||||
assert.equal(h.c.modelComplete, false);
|
||||
assert.equal(h.c.replyPending, true);
|
||||
} else assert.equal(h.c.isActive(), false);
|
||||
} finally {
|
||||
h.c.stop();
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,734 @@
|
||||
// Native ownership replacements for historical Radio regression cases.
|
||||
// Real Gemini controller, action runner, DataLayerManager and RealtimeRadio consumers.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
harness,
|
||||
tick,
|
||||
deferred,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
import { createGevActionRunner } from './gevActions.js';
|
||||
import { createVoiceSession } from './session.js';
|
||||
import { DataLayerManager } from '../data/manager.js';
|
||||
|
||||
const viewerFixture = () => ({
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
});
|
||||
const request = (ws, id, name, args = {}) =>
|
||||
ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
|
||||
const responses = (ws) =>
|
||||
ws.sent.flatMap((frame) => frame.toolResponse?.functionResponses || []);
|
||||
const responseFor = (ws, id) =>
|
||||
responses(ws).findLast((response) => response.id === id)?.response;
|
||||
const speechTurn = (ws) => {
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
};
|
||||
function drainRadioReply(h) {
|
||||
h.sockets.at(-1).message({
|
||||
serverContent: {
|
||||
modelTurn: { parts: [{ inlineData: { data: 'AAAA' } }] },
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
h.captures.at(-1).drain();
|
||||
}
|
||||
|
||||
function radioSiblingFixture() {
|
||||
const viewer = viewerFixture();
|
||||
const manager = new DataLayerManager(viewer);
|
||||
const lookupEntered = deferred();
|
||||
const lookupReleased = deferred();
|
||||
const state = {
|
||||
stationCount: 2,
|
||||
selected: { id: 'existing-station', name: 'Existing Station' },
|
||||
audioState: 'stopped',
|
||||
};
|
||||
const trace = { stops: 0, pauses: 0, selections: 0, options: new Map() };
|
||||
const radio = {
|
||||
async init() {},
|
||||
async enable() {},
|
||||
async disable() {},
|
||||
async update() {},
|
||||
getStats: () => ({ count: 2 }),
|
||||
getUIState: () => ({ ...state }),
|
||||
setVoiceDucked() {},
|
||||
pause({ origin } = {}) {
|
||||
if (origin === 'voice') trace.pauses++;
|
||||
state.audioState = 'paused';
|
||||
return true;
|
||||
},
|
||||
stopPlayback() {
|
||||
trace.stops++;
|
||||
state.audioState = 'stopped';
|
||||
return true;
|
||||
},
|
||||
selectRequestedStation() {
|
||||
trace.selections++;
|
||||
state.selected = { id: 'late-station', name: 'Late Station' };
|
||||
return state.selected;
|
||||
},
|
||||
subscribePlaybackControls: () => () => {},
|
||||
};
|
||||
manager.register(
|
||||
Object.assign(radio, {
|
||||
id: 'radio',
|
||||
name: 'Radio',
|
||||
updateInterval: -1,
|
||||
}),
|
||||
);
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
dataManager: manager,
|
||||
placeSearch: {
|
||||
async geocode(_query, { signal }) {
|
||||
trace.lookupSignal = signal;
|
||||
lookupEntered.resolve();
|
||||
// Deliberately ignore abort: production currentness must still prevent
|
||||
// this late service completion from committing a station selection.
|
||||
await lookupReleased.promise;
|
||||
return { place: { lat: 10, lng: 20, label: 'Deferred Place' } };
|
||||
},
|
||||
},
|
||||
});
|
||||
const h = harness({
|
||||
dataManager: manager,
|
||||
radioLayer: radio,
|
||||
runAction(name, args, options) {
|
||||
trace.options.set(options.callId, options);
|
||||
return runner(name, args, options);
|
||||
},
|
||||
});
|
||||
return {
|
||||
h,
|
||||
manager,
|
||||
runner,
|
||||
radio,
|
||||
trace,
|
||||
state,
|
||||
lookupEntered,
|
||||
lookupReleased,
|
||||
};
|
||||
}
|
||||
|
||||
for (const action of ['select', 'play']) {
|
||||
for (const stronger of ['stop', 'pause', 'disable', 'off']) {
|
||||
test(`native same-turn Radio ${stronger} preempts deferred ${action} without cancelling a query sibling`, async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h, trace, state } = f;
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
const turn = h.c.turn;
|
||||
request(ws, 'slow-selection', 'control_radio', {
|
||||
action,
|
||||
locationQuery: 'Deferred Place',
|
||||
});
|
||||
await f.lookupEntered.promise;
|
||||
request(ws, 'query-sibling', 'control_radio', { action: 'status' });
|
||||
request(
|
||||
ws,
|
||||
'stronger-radio',
|
||||
stronger === 'off' ? 'set_layer_visibility' : 'control_radio',
|
||||
stronger === 'off'
|
||||
? { layerId: 'radio', enabled: false }
|
||||
: { action: stronger },
|
||||
);
|
||||
await tick();
|
||||
assert.equal(
|
||||
responseFor(ws, 'stronger-radio')?.ok,
|
||||
true,
|
||||
'stronger control must finish before lookup release',
|
||||
);
|
||||
assert.equal(trace.lookupSignal.aborted, true);
|
||||
await h.c.toolQueue;
|
||||
assert.equal(
|
||||
h.c.turn,
|
||||
turn,
|
||||
'sibling control cannot invent a new turn',
|
||||
);
|
||||
assert.equal(trace.selections, 0);
|
||||
assert.equal(state.selected.id, 'existing-station');
|
||||
assert.equal(responseFor(ws, 'slow-selection').ok, false);
|
||||
assert.match(responseFor(ws, 'slow-selection').error, /cancelled/);
|
||||
assert.equal(
|
||||
responseFor(ws, 'slow-selection').enabled,
|
||||
!['disable', 'off'].includes(stronger),
|
||||
);
|
||||
assert.equal(responseFor(ws, 'query-sibling').ok, true);
|
||||
assert.equal(trace.options.get('query-sibling').signal.aborted, false);
|
||||
assert.equal(
|
||||
responseFor(ws, 'stronger-radio').enabled,
|
||||
!['disable', 'off'].includes(stronger),
|
||||
);
|
||||
assert.equal(trace.stops, stronger === 'stop' ? 1 : 0);
|
||||
assert.equal(trace.pauses, stronger === 'pause' ? 1 : 0);
|
||||
f.lookupReleased.resolve();
|
||||
await tick();
|
||||
assert.equal(trace.selections, 0, 'late lookup cannot commit');
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.tools.size, 0);
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
||||
for (const id of ['slow-selection', 'query-sibling', 'stronger-radio'])
|
||||
assert.equal(responses(ws).filter((r) => r.id === id).length, 1);
|
||||
} finally {
|
||||
f.lookupReleased.resolve();
|
||||
h.c.stop();
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const guard of ['malformed', 'replay', 'invalid-arguments']) {
|
||||
test(`native stronger Radio ${guard} cannot revoke a same-turn lookup`, async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h, trace } = f;
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
if (guard === 'replay') {
|
||||
request(ws, 'stronger-radio', 'control_radio', { action: 'stop' });
|
||||
await h.c.toolQueue;
|
||||
}
|
||||
speechTurn(ws);
|
||||
if (guard === 'replay')
|
||||
ws.message({
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
request(ws, 'slow-selection', 'control_radio', {
|
||||
action: 'select',
|
||||
locationQuery: 'Deferred Place',
|
||||
});
|
||||
await f.lookupEntered.promise;
|
||||
const stops = trace.stops;
|
||||
const handoffEpoch = h.c.radio.radioHandoffEpoch;
|
||||
request(
|
||||
ws,
|
||||
guard === 'malformed' ? '\n' : 'stronger-radio',
|
||||
'control_radio',
|
||||
{
|
||||
action: 'stop',
|
||||
...(guard === 'invalid-arguments'
|
||||
? { latitude: 91, longitude: 0 }
|
||||
: {}),
|
||||
},
|
||||
);
|
||||
await tick();
|
||||
assert.equal(trace.lookupSignal.aborted, false);
|
||||
assert.equal(trace.stops, stops);
|
||||
assert.equal(h.c.radio.radioHandoffEpoch, handoffEpoch);
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
||||
if (guard === 'invalid-arguments') {
|
||||
assert.equal(responseFor(ws, 'stronger-radio').ok, false);
|
||||
assert.match(responseFor(ws, 'stronger-radio').error, /coordinates/);
|
||||
}
|
||||
f.lookupReleased.resolve();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(
|
||||
trace.selections,
|
||||
1,
|
||||
'older eligible work retains authority',
|
||||
);
|
||||
assert.equal(responseFor(ws, 'slow-selection').ok, true);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.tools.size, 0);
|
||||
} finally {
|
||||
f.lookupReleased.resolve();
|
||||
h.c.stop();
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
test('native failed Stop releases its reservation and lets the older selection finish', async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h, trace } = f;
|
||||
const stopReleased = deferred();
|
||||
f.radio.stopPlayback = () => stopReleased.promise;
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
request(ws, 'slow-selection', 'control_radio', {
|
||||
action: 'select',
|
||||
locationQuery: 'Deferred Place',
|
||||
});
|
||||
await f.lookupEntered.promise;
|
||||
request(ws, 'failed-stop', 'control_radio', { action: 'stop' });
|
||||
await tick();
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 1);
|
||||
assert.equal(
|
||||
trace.lookupSignal.aborted,
|
||||
false,
|
||||
'provisional Stop cannot revoke work',
|
||||
);
|
||||
assert.equal(h.c.pendingTools, 2);
|
||||
stopReleased.resolve(false);
|
||||
await tick();
|
||||
assert.equal(responseFor(ws, 'failed-stop').ok, false);
|
||||
assert.match(responseFor(ws, 'failed-stop').error, /could not be stopped/);
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
||||
assert.equal(trace.lookupSignal.aborted, false);
|
||||
f.lookupReleased.resolve();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(responseFor(ws, 'slow-selection').ok, true);
|
||||
assert.equal(trace.selections, 1);
|
||||
assert.equal(f.state.selected.id, 'late-station');
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
} finally {
|
||||
stopReleased.resolve(false);
|
||||
f.lookupReleased.resolve();
|
||||
h.c.stop();
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test('native pending Stop preserves selection completed before its own successful settlement', async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h } = f;
|
||||
const stopReleased = deferred();
|
||||
f.radio.stopPlayback = () => stopReleased.promise;
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
request(ws, 'slow-selection', 'control_radio', {
|
||||
action: 'select',
|
||||
locationQuery: 'Deferred Place',
|
||||
});
|
||||
await f.lookupEntered.promise;
|
||||
request(ws, 'pending-stop', 'control_radio', { action: 'stop' });
|
||||
await tick();
|
||||
f.lookupReleased.resolve();
|
||||
await tick();
|
||||
assert.equal(responseFor(ws, 'slow-selection').ok, true);
|
||||
assert.equal(f.state.selected.id, 'late-station');
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 1);
|
||||
assert.ok(h.c.radio.pendingRadioPlaybackResult);
|
||||
stopReleased.resolve(true);
|
||||
await h.c.toolQueue;
|
||||
assert.equal(responseFor(ws, 'pending-stop').ok, true);
|
||||
assert.equal(responseFor(ws, 'pending-stop').stationId, 'late-station');
|
||||
assert.equal(f.state.selected.id, 'late-station');
|
||||
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
} finally {
|
||||
stopReleased.resolve(false);
|
||||
f.lookupReleased.resolve();
|
||||
h.c.stop();
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
for (const restart of [false, true]) {
|
||||
test(`native Stop${restart ? '/restart' : ''} retires concurrent stronger Radio work without late output or count underflow`, async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h } = f;
|
||||
const stopReleased = deferred();
|
||||
f.radio.stopPlayback = () => stopReleased.promise;
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
request(ws, 'slow-selection', 'control_radio', {
|
||||
action: 'select',
|
||||
locationQuery: 'Deferred Place',
|
||||
});
|
||||
await f.lookupEntered.promise;
|
||||
request(ws, 'pending-stop', 'control_radio', { action: 'stop' });
|
||||
await tick();
|
||||
const oldDrain = h.c.toolQueue;
|
||||
const stopOptions = f.trace.options.get('pending-stop');
|
||||
assert.equal(h.c.pendingTools, 2);
|
||||
h.c.stop();
|
||||
assert.equal(stopOptions.signal.aborted, true);
|
||||
assert.equal(f.trace.lookupSignal.aborted, true);
|
||||
if (restart) {
|
||||
await h.c.start();
|
||||
request(h.sockets[1], 'new-query', 'control_radio', {
|
||||
action: 'status',
|
||||
});
|
||||
await h.c.toolQueue;
|
||||
assert.equal(responseFor(h.sockets[1], 'new-query').ok, true);
|
||||
}
|
||||
stopReleased.resolve(true);
|
||||
f.lookupReleased.resolve();
|
||||
await oldDrain;
|
||||
await tick();
|
||||
assert.equal(responseFor(ws, 'pending-stop'), undefined);
|
||||
assert.equal(responseFor(ws, 'slow-selection'), undefined);
|
||||
assert.equal(f.trace.selections, 0);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.tools.size, 0);
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
||||
assert.equal(h.c.isActive(), restart);
|
||||
} finally {
|
||||
stopReleased.resolve(false);
|
||||
f.lookupReleased.resolve();
|
||||
h.c.stop();
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const phase of ['prepared', 'preflight']) {
|
||||
test(`native same-turn Stop revokes only voice-owned ${phase} playback and preserves completed station`, async () => {
|
||||
const f = radioSiblingFixture();
|
||||
const { h } = f;
|
||||
const ready = deferred();
|
||||
const stopped = [];
|
||||
let preflight;
|
||||
f.radio.playForVoice = () => ready.promise;
|
||||
f.radio.stopPlayback = (options) => {
|
||||
stopped.push(options);
|
||||
return true;
|
||||
};
|
||||
try {
|
||||
await f.manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
request(ws, 'prepared-play', 'control_radio', { action: 'play' });
|
||||
await h.c.toolQueue;
|
||||
assert.equal(
|
||||
responseFor(ws, 'prepared-play').radioPlaybackRequested,
|
||||
true,
|
||||
);
|
||||
if (phase === 'preflight')
|
||||
preflight = h.c.radio.startPendingRadioHandoff();
|
||||
const attemptId = h.c.radio.radioHandoffAttemptId;
|
||||
request(ws, 'stronger-stop', 'control_radio', { action: 'stop' });
|
||||
await h.c.toolQueue;
|
||||
assert.equal(responseFor(ws, 'stronger-stop').ok, true);
|
||||
assert.equal(f.state.selected.id, 'existing-station');
|
||||
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, false);
|
||||
const cleanups = stopped.filter((o) => o.origin === 'voice-cleanup');
|
||||
assert.deepEqual(
|
||||
cleanups,
|
||||
phase === 'preflight' ? [{ origin: 'voice-cleanup', attemptId }] : [],
|
||||
);
|
||||
ready.resolve(true);
|
||||
await preflight;
|
||||
await tick();
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
true,
|
||||
'late old readiness cannot stop voice',
|
||||
);
|
||||
assert.equal(h.c.radio.radioToolHandoffReservations.size, 0);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
} finally {
|
||||
ready.resolve(false);
|
||||
h.c.stop();
|
||||
await preflight;
|
||||
f.runner.dispose();
|
||||
await f.manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
for (const phase of ['prepared', 'preflight', 'settling']) {
|
||||
test(`native Radio cancellation: ${phase} preserves selection truth and prevents stale speaker transfer`, async () => {
|
||||
const viewer = viewerFixture();
|
||||
const manager = new DataLayerManager(viewer);
|
||||
const prepared = deferred();
|
||||
const settleResult = deferred();
|
||||
const playbackReady = deferred();
|
||||
let playCalls = 0;
|
||||
let stopCalls = 0;
|
||||
let selections = 0;
|
||||
let activeOptions;
|
||||
const state = {
|
||||
stationCount: 2,
|
||||
selected: null,
|
||||
audioState: 'stopped',
|
||||
voiceDucked: false,
|
||||
};
|
||||
const radio = {
|
||||
async init() {},
|
||||
async enable() {},
|
||||
async disable() {},
|
||||
async update() {},
|
||||
getStats: () => ({ count: 2 }),
|
||||
getUIState: () => ({ ...state }),
|
||||
setVoiceDucked(value) {
|
||||
state.voiceDucked = value;
|
||||
},
|
||||
pause() {
|
||||
state.audioState = 'paused';
|
||||
},
|
||||
subscribePlaybackControls() {
|
||||
return () => {};
|
||||
},
|
||||
selectRequestedStation(_criteria, options) {
|
||||
assert.equal(options.autoplay, false);
|
||||
selections++;
|
||||
state.selected = { id: 'station-1', name: 'Station One' };
|
||||
return state.selected;
|
||||
},
|
||||
playForVoice() {
|
||||
playCalls++;
|
||||
return playbackReady.promise;
|
||||
},
|
||||
stopPlayback() {
|
||||
stopCalls++;
|
||||
state.audioState = 'stopped';
|
||||
},
|
||||
};
|
||||
manager.register({
|
||||
id: 'radio',
|
||||
name: 'Radio',
|
||||
updateInterval: -1,
|
||||
...radio,
|
||||
});
|
||||
const runner = createGevActionRunner({ viewer, dataManager: manager });
|
||||
let h;
|
||||
let preflight;
|
||||
const session = createVoiceSession({
|
||||
runner: async (name, args, options) => {
|
||||
activeOptions = options;
|
||||
const result = await runner(name, args, options);
|
||||
prepared.resolve(result);
|
||||
if (phase === 'settling') await settleResult.promise;
|
||||
return result;
|
||||
},
|
||||
createAdapter: (shared) => {
|
||||
h = harness({ ...shared, dataManager: manager, radioLayer: radio });
|
||||
return {
|
||||
start: (settings) => h.c.start(settings),
|
||||
stop: (settings) => h.c.stop(settings),
|
||||
sendText: (text) => h.c.sendTextCommand(text),
|
||||
sendMapEvent: (event) => h.c.notifyMapEvent(event),
|
||||
};
|
||||
},
|
||||
});
|
||||
try {
|
||||
await session.start();
|
||||
const ws = h.sockets[0];
|
||||
assert.equal(session.sendText('Select Station One on Radio'), true);
|
||||
request(ws, 'prepared-radio', 'control_radio', {
|
||||
action: 'select',
|
||||
stationQuery: 'Station One',
|
||||
});
|
||||
const actual = await prepared.promise;
|
||||
assert.equal(actual.ok, true);
|
||||
assert.equal(actual.radioPlaybackRequested, true);
|
||||
assert.equal(actual.stationId, 'station-1');
|
||||
assert.equal(manager.isEnabled('radio'), true);
|
||||
if (phase !== 'settling') {
|
||||
await h.c.toolQueue;
|
||||
assert.ok(h.c.radio.pendingRadioPlaybackResult);
|
||||
}
|
||||
if (phase === 'preflight') {
|
||||
preflight = h.c.radio.startPendingRadioHandoff();
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, true);
|
||||
assert.equal(playCalls, 1);
|
||||
}
|
||||
// Native call ownership revokes only this deferred playback. No caption
|
||||
// parsing or caption-driven provider policy participates in cancellation.
|
||||
ws.message({ toolCallCancellation: { ids: ['prepared-radio'] } });
|
||||
if (phase === 'settling')
|
||||
assert.equal(activeOptions.signal.aborted, true);
|
||||
settleResult.resolve();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.c.radio.pendingRadioPlaybackResult, null);
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, false);
|
||||
assert.equal(manager.isEnabled('radio'), true);
|
||||
assert.equal(state.selected.id, 'station-1');
|
||||
assert.equal(selections, 1);
|
||||
const modelResult = responseFor(ws, 'prepared-radio');
|
||||
if (phase === 'settling')
|
||||
assert.equal(
|
||||
modelResult,
|
||||
undefined,
|
||||
'cancelled active call never sends a late result',
|
||||
);
|
||||
else {
|
||||
assert.equal(modelResult.ok, true);
|
||||
assert.equal(modelResult.stationId, 'station-1');
|
||||
}
|
||||
const stopsAfterCancellation = stopCalls;
|
||||
assert.equal(stopsAfterCancellation, phase === 'preflight' ? 1 : 0);
|
||||
drainRadioReply(h);
|
||||
playbackReady.resolve(true);
|
||||
await preflight;
|
||||
await tick();
|
||||
assert.equal(
|
||||
playCalls,
|
||||
phase === 'preflight' ? 1 : 0,
|
||||
'drain cannot revive cancelled playback',
|
||||
);
|
||||
assert.equal(h.c.isActive(), true, 'late readiness cannot stop voice');
|
||||
assert.equal(h.c.radio.radioHandoffInFlight, false);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.tools.size, 0);
|
||||
assert.equal(session.sendText('Play radio'), true);
|
||||
request(ws, 'fresh-radio', 'control_radio', { action: 'play' });
|
||||
await h.c.toolQueue;
|
||||
assert.ok(h.c.radio.pendingRadioPlaybackResult);
|
||||
drainRadioReply(h);
|
||||
await tick();
|
||||
assert.equal(playCalls, phase === 'preflight' ? 2 : 1);
|
||||
assert.equal(
|
||||
h.c.isActive(),
|
||||
false,
|
||||
'fresh typed Radio handoff alone ends voice',
|
||||
);
|
||||
} finally {
|
||||
settleResult.resolve();
|
||||
playbackReady.resolve(false);
|
||||
session.destroy();
|
||||
await preflight;
|
||||
runner.dispose();
|
||||
await manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
|
||||
// A stronger Radio control skips the ordered queue. An older Radio call still
|
||||
// queued behind a slow sibling must not start after that control succeeded,
|
||||
// while a failed control or a later call in the same turn keeps its order.
|
||||
for (const [label, ids, stopOk, expectPending] of [
|
||||
[
|
||||
'older queued select is revoked by a later successful stop',
|
||||
['slow', 'sel', 'stop'],
|
||||
true,
|
||||
false,
|
||||
],
|
||||
[
|
||||
'a failed stop leaves the older queued select current',
|
||||
['slow', 'sel', 'stop'],
|
||||
false,
|
||||
true,
|
||||
],
|
||||
[
|
||||
'a select after the stop in the same turn still runs',
|
||||
['slow', 'stop', 'sel'],
|
||||
true,
|
||||
true,
|
||||
],
|
||||
]) {
|
||||
test(`stronger Radio ordering: ${label}`, async () => {
|
||||
const viewer = viewerFixture();
|
||||
const manager = new DataLayerManager(viewer);
|
||||
const state = {
|
||||
stationCount: 2,
|
||||
selected: { id: 'existing', name: 'Existing' },
|
||||
audioState: 'stopped',
|
||||
};
|
||||
const radio = {
|
||||
id: 'radio',
|
||||
name: 'Radio',
|
||||
updateInterval: -1,
|
||||
async init() {},
|
||||
async enable() {},
|
||||
async disable() {},
|
||||
async update() {},
|
||||
getStats: () => ({ count: 2 }),
|
||||
getUIState: () => ({ ...state }),
|
||||
setVoiceDucked() {},
|
||||
pause() {
|
||||
state.audioState = 'paused';
|
||||
return true;
|
||||
},
|
||||
stopPlayback() {
|
||||
state.audioState = 'stopped';
|
||||
return stopOk;
|
||||
},
|
||||
selectRequestedStation() {
|
||||
state.selected = { id: 'late', name: 'Late' };
|
||||
return state.selected;
|
||||
},
|
||||
async playForVoice() {
|
||||
return true;
|
||||
},
|
||||
subscribePlaybackControls: () => () => {},
|
||||
};
|
||||
manager.register(radio);
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
dataManager: manager,
|
||||
placeSearch: {
|
||||
async geocode() {
|
||||
return { place: { lat: 10, lng: 20, label: 'Place' } };
|
||||
},
|
||||
},
|
||||
});
|
||||
const slow = deferred();
|
||||
const h = harness({
|
||||
dataManager: manager,
|
||||
radioLayer: radio,
|
||||
async runAction(name, args, options) {
|
||||
if (name === 'slow_sibling') {
|
||||
await slow.promise;
|
||||
return { ok: true, action: 'slow_sibling' };
|
||||
}
|
||||
if (name === 'control_radio' && args.action === 'stop' && !stopOk)
|
||||
return {
|
||||
ok: false,
|
||||
action: 'control_radio',
|
||||
error: 'Radio stop failed',
|
||||
};
|
||||
return runner(name, args, options);
|
||||
},
|
||||
});
|
||||
const calls = {
|
||||
slow: { id: 'slow', name: 'slow_sibling', args: {} },
|
||||
sel: {
|
||||
id: 'sel',
|
||||
name: 'control_radio',
|
||||
args: { action: 'select', locationQuery: 'Place' },
|
||||
},
|
||||
stop: { id: 'stop', name: 'control_radio', args: { action: 'stop' } },
|
||||
};
|
||||
try {
|
||||
await manager.setEnabled('radio', true);
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
speechTurn(ws);
|
||||
ws.message({ toolCall: { functionCalls: ids.map((id) => calls[id]) } });
|
||||
await tick();
|
||||
assert.equal(
|
||||
responseFor(ws, 'stop')?.ok,
|
||||
stopOk,
|
||||
'stop settles without waiting',
|
||||
);
|
||||
slow.resolve();
|
||||
await h.c.toolQueue;
|
||||
await tick();
|
||||
assert.equal(
|
||||
Boolean(h.c.radio.pendingRadioPlaybackResult),
|
||||
expectPending,
|
||||
);
|
||||
assert.equal(responseFor(ws, 'sel')?.ok, expectPending);
|
||||
} finally {
|
||||
slow.resolve();
|
||||
h.c.stop();
|
||||
runner.dispose();
|
||||
await manager.setEnabled('radio', false);
|
||||
}
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
harness,
|
||||
tick,
|
||||
deferred,
|
||||
} from '../testSupport/geminiNativeHarness.mjs';
|
||||
import { createGevActionRunner } from './gevActions.js';
|
||||
|
||||
const tool = (ws, id) =>
|
||||
ws.message({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id,
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'in', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const caption = (ws, text) =>
|
||||
ws.message({ serverContent: { inputTranscription: { text } } });
|
||||
const onset = (ws) => {
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
|
||||
ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
|
||||
};
|
||||
const complete = (ws) => ws.message({ serverContent: { turnComplete: true } });
|
||||
const oldAudio = {
|
||||
serverContent: {
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{
|
||||
inlineData: {
|
||||
data: 'AAAA',
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
const outputs = (ws) =>
|
||||
ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
|
||||
|
||||
// Socket/media are deterministic. Mutations use the production action runner.
|
||||
function fixture({ holdFirst = false } = {}) {
|
||||
const state = { height: 1000, renders: 0, providerWrites: [], panels: [] };
|
||||
const position = { x: 6379137, y: 0, z: 0 };
|
||||
const viewer = {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
camera: {
|
||||
positionWC: position,
|
||||
positionCartographic: { height: 1000 },
|
||||
moveEnd: { addEventListener() {} },
|
||||
cancelFlight() {},
|
||||
zoomIn(n) {
|
||||
position.x -= n;
|
||||
state.height -= n;
|
||||
this.positionCartographic.height = state.height;
|
||||
},
|
||||
zoomOut(n) {
|
||||
position.x += n;
|
||||
state.height += n;
|
||||
this.positionCartographic.height = state.height;
|
||||
},
|
||||
},
|
||||
scene: {
|
||||
canvas: {
|
||||
clientWidth: 800,
|
||||
clientHeight: 600,
|
||||
addEventListener() {},
|
||||
removeEventListener() {},
|
||||
},
|
||||
requestRender() {
|
||||
state.renders++;
|
||||
},
|
||||
},
|
||||
};
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
dataManager: { layers: new Map() },
|
||||
styleManager: {
|
||||
setVoiceProvider: (p) => state.providerWrites.push(p),
|
||||
setPanelCollapsed: (...args) => state.panels.push(args),
|
||||
},
|
||||
});
|
||||
const entered = [];
|
||||
const gate = deferred();
|
||||
const h = harness({
|
||||
runAction: async (name, args, options) => {
|
||||
entered.push({ name, args, options });
|
||||
if (holdFirst && entered.length === 1) await gate.promise;
|
||||
if (!options.isCurrent())
|
||||
return { ok: false, action: name, changed: false };
|
||||
return runner(name, args, options);
|
||||
},
|
||||
});
|
||||
return {
|
||||
...h,
|
||||
state,
|
||||
entered,
|
||||
gate,
|
||||
close() {
|
||||
gate.resolve();
|
||||
h.c.stop();
|
||||
runner.dispose();
|
||||
assert.equal(h.timers.size, 0);
|
||||
assert.deepEqual(state.providerWrites, []);
|
||||
assert.deepEqual(state.panels, []);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
for (const [id, order] of [
|
||||
['H20', 'before-boundary'],
|
||||
['H21', 'after-boundary'],
|
||||
]) {
|
||||
test(`native replacement ${id}: ${order} tool respects retired output and next onset recovers`, async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
ws.message(oldAudio);
|
||||
onset(ws);
|
||||
caption(ws, 'zoom in');
|
||||
if (order === 'before-boundary') {
|
||||
tool(ws, 'ambiguous-before');
|
||||
await tick();
|
||||
assert.equal(h.state.height, 1000);
|
||||
assert.equal(h.entered.length, 0);
|
||||
assert.equal(outputs(ws).length, 0);
|
||||
}
|
||||
ws.message({ serverContent: { interrupted: true, turnComplete: true } });
|
||||
if (order === 'after-boundary') {
|
||||
tool(ws, 'native-after');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.state.height, 750);
|
||||
assert.equal(h.entered.length, 1);
|
||||
assert.equal(outputs(ws).at(-1).response.ok, true);
|
||||
complete(ws);
|
||||
}
|
||||
onset(ws);
|
||||
caption(ws, 'zoom in');
|
||||
tool(ws, 'fresh-clean-onset');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.state.height, order === 'before-boundary' ? 750 : 562.5);
|
||||
assert.equal(h.entered.length, order === 'before-boundary' ? 1 : 2);
|
||||
assert.equal(h.entered.at(-1).options.userTurn, undefined);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(
|
||||
outputs(ws).some((o) => o.id === 'ambiguous-before'),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
outputs(ws).find((o) => o.id === 'fresh-clean-onset').response.ok,
|
||||
true,
|
||||
);
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('native replacement H30: ordinary tools run promptly and late caption fragments cannot strand siblings', async () => {
|
||||
const h = fixture();
|
||||
try {
|
||||
await h.c.start();
|
||||
const ws = h.sockets[0];
|
||||
onset(ws);
|
||||
caption(ws, 'switch to');
|
||||
tool(ws, 'ordinary-before-caption-resolution');
|
||||
await tick();
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.state.height, 750);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
h.advance(1499);
|
||||
caption(ws, 'Austin');
|
||||
h.advance(5000);
|
||||
tool(ws, 'released-sibling');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.state.height, 562.5);
|
||||
assert.equal(h.entered.length, 2);
|
||||
assert.equal(
|
||||
h.entered.every((c) => c.options.userTurn === undefined),
|
||||
true,
|
||||
);
|
||||
assert.equal(outputs(ws).length, 2);
|
||||
assert.equal(
|
||||
outputs(ws).every((o) => o.response.ok),
|
||||
true,
|
||||
);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.status, 'listening');
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
|
||||
for (const [id, exit] of [
|
||||
['H32', 'stop'],
|
||||
['H33', 'restart'],
|
||||
['H34', 'new-input'],
|
||||
]) {
|
||||
test(`native replacement ${id}: ${exit} retires owned in-flight work and late callbacks`, async () => {
|
||||
const h = fixture({ holdFirst: true });
|
||||
try {
|
||||
await h.c.start();
|
||||
let ws = h.sockets[0];
|
||||
onset(ws);
|
||||
caption(ws, 'switch to');
|
||||
tool(ws, 'old-owned-action');
|
||||
await tick();
|
||||
assert.equal(h.entered.length, 1);
|
||||
assert.equal(h.c.pendingTools, 1);
|
||||
assert.equal(h.state.height, 1000);
|
||||
const oldWork = h.c.toolQueue;
|
||||
const oldMessage = ws.onmessage;
|
||||
const oldOwner = h.entered[0].options;
|
||||
if (exit === 'new-input') h.c.sendTextCommand('zoom out');
|
||||
else h.c.stop();
|
||||
assert.equal(oldOwner.signal.aborted, true);
|
||||
assert.equal(oldOwner.isCurrent(), false);
|
||||
if (exit === 'restart') {
|
||||
await h.c.start();
|
||||
ws = h.sockets.at(-1);
|
||||
}
|
||||
if (exit !== 'new-input')
|
||||
oldMessage({
|
||||
data: JSON.stringify({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: 'late-old-call',
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'in' },
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
});
|
||||
caption(ws, 'Gemini');
|
||||
h.advance(1500);
|
||||
h.gate.resolve();
|
||||
await oldWork;
|
||||
await tick();
|
||||
assert.equal(h.state.height, 1000);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(
|
||||
outputs(ws).some(
|
||||
(o) => o.id === 'old-owned-action' || o.id === 'late-old-call',
|
||||
),
|
||||
false,
|
||||
);
|
||||
if (exit === 'stop') {
|
||||
assert.equal(h.c.status, 'idle');
|
||||
assert.equal(h.c.isActive(), false);
|
||||
} else {
|
||||
if (exit === 'new-input') {
|
||||
ws.message({
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
} else h.c.sendTextCommand('zoom out');
|
||||
tool(ws, 'fresh-after-retirement');
|
||||
await h.c.toolQueue;
|
||||
assert.equal(h.state.height, 750);
|
||||
assert.equal(h.entered.length, 2);
|
||||
assert.equal(
|
||||
outputs(ws).find((o) => o.id === 'fresh-after-retirement').response
|
||||
.ok,
|
||||
true,
|
||||
);
|
||||
assert.equal(h.c.pendingTools, 0);
|
||||
assert.equal(h.c.isActive(), true);
|
||||
}
|
||||
} finally {
|
||||
h.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,529 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
nativeIntegrationFixture,
|
||||
assertSuccessfulZoom,
|
||||
turnOfEventLoop,
|
||||
} from '../testSupport/geminiIntegrationFixture.mjs';
|
||||
|
||||
const options = { concurrency: false, timeout: 5000 };
|
||||
const OLD_TEXT = 'Old packet caption.';
|
||||
const OLD_PCM = 'AAABAA==';
|
||||
const POST_TOOL_PCM = 'AgADAA==';
|
||||
const OLD_CALL = 'old-coalesced-call';
|
||||
|
||||
function coalescedPacket(callId = OLD_CALL) {
|
||||
return {
|
||||
serverContent: {
|
||||
outputTranscription: { text: OLD_TEXT },
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{ inlineData: { data: OLD_PCM, mimeType: 'audio/pcm;rate=24000' } },
|
||||
],
|
||||
},
|
||||
turnComplete: true,
|
||||
},
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id: callId,
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'in', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Local test observations wrap only injected device fakes and the actual
|
||||
// scheduler methods, delegating unchanged. No frozen fixture file is modified.
|
||||
function observePacketEffects(h) {
|
||||
const played = [];
|
||||
const narration = { audible: 0, toolStarted: 0, toolFinished: 0 };
|
||||
const createAudio = h.controller.createAudio;
|
||||
h.controller.createAudio = (callbacks) => {
|
||||
const audio = createAudio(callbacks);
|
||||
const play = audio.play.bind(audio);
|
||||
audio.play = (data, mimeType) => {
|
||||
played.push({ audio, data, mimeType });
|
||||
return play(data, mimeType);
|
||||
};
|
||||
return audio;
|
||||
};
|
||||
for (const method of Object.keys(narration)) {
|
||||
const invoke = h.controller.narration[method].bind(h.controller.narration);
|
||||
h.controller.narration[method] = (...args) => {
|
||||
narration[method]++;
|
||||
return invoke(...args);
|
||||
};
|
||||
}
|
||||
return { played, narration };
|
||||
}
|
||||
|
||||
function packetState(h) {
|
||||
const c = h.controller;
|
||||
return {
|
||||
epoch: c.epoch,
|
||||
inputGeneration: c.inputGeneration,
|
||||
turn: c.turn,
|
||||
completedTurns: c.completedTurns,
|
||||
pendingTools: c.pendingTools,
|
||||
modelComplete: c.modelComplete,
|
||||
pendingCompletionStatus: c.pendingCompletionStatus,
|
||||
postToolOutput: c.postToolOutput,
|
||||
modelSpoke: c.modelSpoke,
|
||||
assistant: c.transcripts.assistant,
|
||||
replyPending: c.replyPending,
|
||||
retiredReplyPending: c.retiredReplyPending,
|
||||
typedText: c.typedIntent?.text ?? null,
|
||||
typedResponseId: c.typedIntent?.responseId ?? null,
|
||||
narrationTools: c.narration.state().tools,
|
||||
narrationTimers: c.narration.state().pendingTimers,
|
||||
ownedTimers: h.clock.timers.size,
|
||||
};
|
||||
}
|
||||
|
||||
const completions = (h) =>
|
||||
h.events.filter((event) => event.type === 'completion');
|
||||
const toolEvents = (h, callId) =>
|
||||
h.events.filter(
|
||||
(event) =>
|
||||
['action-call', 'action-result', 'progress'].includes(event.type) &&
|
||||
event.callId === callId,
|
||||
);
|
||||
|
||||
function assertOldRemainderAbsent(h, observed) {
|
||||
assert.equal(h.state.height, 1000);
|
||||
assert.deepEqual(h.state.zooms, []);
|
||||
assert.deepEqual(h.runnerCalls, []);
|
||||
assert.deepEqual(toolEvents(h, OLD_CALL), []);
|
||||
assert.deepEqual(observed.played, []);
|
||||
assert.deepEqual(observed.narration, {
|
||||
audible: 0,
|
||||
toolStarted: 0,
|
||||
toolFinished: 0,
|
||||
});
|
||||
assert.deepEqual(completions(h), []);
|
||||
assert.equal(h.controller.completedTurns, 0);
|
||||
assert.equal(h.controller.pendingTools, 0);
|
||||
assert.equal(
|
||||
h.events.filter(
|
||||
(event) =>
|
||||
event.type === 'transcript' &&
|
||||
event.role === 'assistant' &&
|
||||
event.final &&
|
||||
event.text === OLD_TEXT,
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
for (const socket of h.sockets)
|
||||
assert.equal(
|
||||
h.outputs(socket).some((output) => output.id === OLD_CALL),
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
for (const mode of ['Stop', 'Stop then Start', 'typed successor']) {
|
||||
test(
|
||||
`assistant transcript observer ${mode} fences the rest of a coalesced native packet`,
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const observed = observePacketEffects(h);
|
||||
const oldSocket = await h.start();
|
||||
const oldAudio = h.audios[0];
|
||||
h.activity(oldSocket);
|
||||
const before = packetState(h);
|
||||
let reached = false;
|
||||
let restart;
|
||||
let typedAccepted;
|
||||
let afterObserver;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (
|
||||
reached ||
|
||||
event.type !== 'transcript' ||
|
||||
event.role !== 'assistant' ||
|
||||
event.final ||
|
||||
event.text !== OLD_TEXT
|
||||
)
|
||||
return;
|
||||
reached = true;
|
||||
if (mode === 'typed successor') {
|
||||
typedAccepted = h.session.sendText('Zoom in for the typed successor');
|
||||
} else {
|
||||
h.session.stop();
|
||||
if (mode === 'Stop then Start') restart = h.session.start();
|
||||
}
|
||||
afterObserver = packetState(h);
|
||||
});
|
||||
oldSocket.message(coalescedPacket());
|
||||
remove();
|
||||
assert.equal(
|
||||
reached,
|
||||
true,
|
||||
'the actual common transcript observer ran synchronously',
|
||||
);
|
||||
// A retired reply may consume its own received terminal, but cannot
|
||||
// complete or overwrite the successor. Stop/restart permits no old writes.
|
||||
const afterPacketExpected =
|
||||
mode === 'typed successor'
|
||||
? {
|
||||
...afterObserver,
|
||||
retiredReplyPending: false,
|
||||
ownedTimers: afterObserver.ownedTimers - 1,
|
||||
}
|
||||
: afterObserver;
|
||||
assert.deepEqual(packetState(h), afterPacketExpected);
|
||||
await h.controller.toolQueue;
|
||||
await turnOfEventLoop();
|
||||
if (restart) await restart;
|
||||
assertOldRemainderAbsent(h, observed);
|
||||
|
||||
let successorSocket;
|
||||
if (mode === 'typed successor') {
|
||||
assert.equal(typedAccepted, true);
|
||||
assert.equal(
|
||||
h.controller.epoch,
|
||||
before.epoch,
|
||||
'typed replacement shares the socket epoch',
|
||||
);
|
||||
assert.ok(h.controller.inputGeneration > before.inputGeneration);
|
||||
assert.ok(h.controller.turn > before.turn);
|
||||
assert.deepEqual(packetState(h), afterPacketExpected);
|
||||
assert.equal(
|
||||
h.controller.typedIntent.text,
|
||||
'Zoom in for the typed successor',
|
||||
);
|
||||
assert.equal(h.controller.replyPending, true);
|
||||
assert.equal(h.controller.retiredReplyPending, false);
|
||||
assert.equal(oldAudio.closed, false);
|
||||
const typedMessages = oldSocket.sent.filter((message) =>
|
||||
message.clientContent?.turns?.some((turn) =>
|
||||
turn.parts?.some(
|
||||
(part) => part.text === 'Zoom in for the typed successor',
|
||||
),
|
||||
),
|
||||
);
|
||||
assert.equal(typedMessages.length, 1);
|
||||
// The received packet already carried the old terminal. No extra
|
||||
// provider boundary may be invented before testing the actual successor.
|
||||
assert.equal(h.controller.retiredBoundaryWait, null);
|
||||
assert.equal(h.controller.cancelledCalls.has(OLD_CALL), true);
|
||||
assert.equal(h.controller.replyPending, true);
|
||||
assert.equal(h.controller.completedTurns, 0);
|
||||
assert.deepEqual(completions(h), []);
|
||||
successorSocket = oldSocket;
|
||||
} else {
|
||||
assert.equal(oldSocket.closed, true);
|
||||
assert.equal(oldAudio.closed, true);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
assert.equal(h.clock.timers.size, 0);
|
||||
if (mode === 'Stop') {
|
||||
assert.equal(h.session.state, 'idle');
|
||||
successorSocket = await h.start();
|
||||
} else {
|
||||
assert.equal(h.session.state, 'listening');
|
||||
assert.equal(h.sockets.length, 2);
|
||||
assert.equal(h.audios.length, 2);
|
||||
assert.notEqual(h.audios[1], oldAudio);
|
||||
assert.equal(h.audios[1].closed, false);
|
||||
assert.equal(h.audios[1].pending, false);
|
||||
successorSocket = h.sockets[1];
|
||||
}
|
||||
h.activity(successorSocket);
|
||||
}
|
||||
await h.tool('owned-after-packet-cancel', successorSocket);
|
||||
assertSuccessfulZoom(
|
||||
h,
|
||||
successorSocket,
|
||||
'owned-after-packet-cancel',
|
||||
1000,
|
||||
);
|
||||
assert.equal(h.state.zooms.length, 1);
|
||||
assert.equal(h.runnerCalls.length, 1);
|
||||
assert.equal(
|
||||
h.outputs(successorSocket).some((output) => output.id === OLD_CALL),
|
||||
false,
|
||||
);
|
||||
h.complete(successorSocket);
|
||||
assert.equal(completions(h).length, 1);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
test(
|
||||
'ordinary coalesced packet plays and executes once, then completes only after its post-tool reply',
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const observed = observePacketEffects(h);
|
||||
const ws = await h.start();
|
||||
h.activity(ws);
|
||||
ws.message(coalescedPacket('ordinary-coalesced'));
|
||||
assert.equal(
|
||||
h.controller.completedTurns,
|
||||
1,
|
||||
'first generation terminal is counted once',
|
||||
);
|
||||
assert.equal(h.controller.pendingTools, 1);
|
||||
assert.deepEqual(
|
||||
observed.played.map(({ data }) => data),
|
||||
[OLD_PCM],
|
||||
);
|
||||
assert.equal(observed.played[0].audio, h.audios[0]);
|
||||
assert.deepEqual(completions(h), [], 'tool and playback are still pending');
|
||||
await h.controller.toolQueue;
|
||||
assertSuccessfulZoom(h, ws, 'ordinary-coalesced', 1000);
|
||||
assert.equal(h.controller.pendingTools, 0);
|
||||
assert.equal(observed.narration.toolStarted, 1);
|
||||
assert.equal(observed.narration.toolFinished, 1);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
h.audios[0].drain();
|
||||
assert.equal(
|
||||
h.controller.modelComplete,
|
||||
false,
|
||||
'function response waits for the post-tool generation',
|
||||
);
|
||||
assert.equal(h.controller.completedTurns, 1);
|
||||
assert.deepEqual(
|
||||
completions(h),
|
||||
[],
|
||||
'draining first-generation PCM cannot finish the tool reply',
|
||||
);
|
||||
|
||||
// The accepted contract has two generations: the function response prompts
|
||||
// a fresh model reply. This boundary is intentional, not a stale-packet fix.
|
||||
ws.message({
|
||||
serverContent: {
|
||||
outputTranscription: { text: 'The requested zoom is complete.' },
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{
|
||||
inlineData: {
|
||||
data: POST_TOOL_PCM,
|
||||
mimeType: 'audio/pcm;rate=24000',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
assert.equal(h.controller.completedTurns, 2);
|
||||
assert.deepEqual(
|
||||
observed.played.map(({ data }) => data),
|
||||
[OLD_PCM, POST_TOOL_PCM],
|
||||
);
|
||||
assert.deepEqual(
|
||||
completions(h),
|
||||
[],
|
||||
'post-tool playback must drain before completion',
|
||||
);
|
||||
h.audios[0].drain();
|
||||
assert.equal(completions(h).length, 1);
|
||||
assert.equal(completions(h)[0].status, 'completed');
|
||||
assert.equal(completions(h)[0].spoken, true);
|
||||
assert.equal(completions(h)[0].audioDrained, true);
|
||||
assert.equal(h.controller.pendingCompletionStatus, null);
|
||||
assert.equal(h.controller.narration.state().tools, 0);
|
||||
assert.equal(h.controller.narration.state().pendingTimers, 0);
|
||||
assert.equal(h.clock.timers.size, 0);
|
||||
assert.equal(observed.narration.audible, 2);
|
||||
assert.equal(h.runnerCalls.length, 1);
|
||||
assert.equal(h.state.zooms.length, 1);
|
||||
assert.equal(h.outputs(ws).length, 1);
|
||||
h.audios[0].drain();
|
||||
assert.equal(
|
||||
completions(h).length,
|
||||
1,
|
||||
'a duplicate device drain cannot complete twice',
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// Real native message-type fields are separate packets here; no extra old
|
||||
// completion is inserted before the accepted successor.
|
||||
const BOUNDARY_OLD = 'Prior reply caption.';
|
||||
const BOUNDARY_NEXT = 'Zoom in for the typed successor';
|
||||
const boundaryZoomCall = (id) => ({
|
||||
toolCall: {
|
||||
functionCalls: [
|
||||
{
|
||||
id,
|
||||
name: 'adjust_camera_zoom',
|
||||
args: { direction: 'in', amount: 'little' },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
function boundarySnapshot(h) {
|
||||
const c = h.controller;
|
||||
return {
|
||||
retiredReplyPending: c.retiredReplyPending,
|
||||
retiredBoundaryWait: Boolean(c.retiredBoundaryWait),
|
||||
replyPending: c.replyPending,
|
||||
typedText: c.typedIntent?.text ?? null,
|
||||
completions: h.events.filter((event) => event.type === 'completion').length,
|
||||
};
|
||||
}
|
||||
|
||||
for (const mode of [
|
||||
'partial assistant transcript',
|
||||
'input transcript',
|
||||
'interruption event',
|
||||
'listening event',
|
||||
'final assistant transcript control',
|
||||
]) {
|
||||
test(
|
||||
`received native terminal preserves typed successor after ${mode}`,
|
||||
{ concurrency: false, timeout: 5000 },
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const ws = await h.start();
|
||||
h.activity(ws);
|
||||
let oldDispatch, gate;
|
||||
const interruptionMode =
|
||||
mode === 'interruption event' || mode === 'listening event';
|
||||
if (interruptionMode) {
|
||||
h.session.sendText('Prior zoom request');
|
||||
gate = h.holdResult('already-committed-old-tool');
|
||||
oldDispatch = h.tool('already-committed-old-tool', ws);
|
||||
await gate.ready;
|
||||
} else if (mode === 'input transcript') {
|
||||
ws.message({
|
||||
serverContent: {
|
||||
outputTranscription: { text: 'Previously emitted output' },
|
||||
},
|
||||
});
|
||||
}
|
||||
let entered = 0,
|
||||
accepted;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (entered) return;
|
||||
const matches =
|
||||
mode === 'interruption event'
|
||||
? event.type === 'interruption' && event.reason === 'user-speech'
|
||||
: mode === 'listening event'
|
||||
? event.type === 'state' && event.state === 'listening'
|
||||
: mode === 'input transcript'
|
||||
? event.type === 'transcript' &&
|
||||
event.role === 'user' &&
|
||||
event.text === 'Old input caption'
|
||||
: event.type === 'transcript' &&
|
||||
event.role === 'assistant' &&
|
||||
event.text === BOUNDARY_OLD &&
|
||||
event.final ===
|
||||
(mode === 'final assistant transcript control');
|
||||
if (!matches) return;
|
||||
entered++;
|
||||
accepted = h.session.sendText(BOUNDARY_NEXT);
|
||||
});
|
||||
if (interruptionMode)
|
||||
ws.message({
|
||||
serverContent: { interrupted: true, turnComplete: true },
|
||||
});
|
||||
else
|
||||
ws.message({
|
||||
serverContent: {
|
||||
...(mode === 'input transcript'
|
||||
? { inputTranscription: { text: 'Old input caption' } }
|
||||
: {}),
|
||||
outputTranscription: { text: BOUNDARY_OLD },
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
remove();
|
||||
if (oldDispatch) {
|
||||
await oldDispatch;
|
||||
gate.release();
|
||||
}
|
||||
await turnOfEventLoop();
|
||||
const afterPacket = boundarySnapshot(h);
|
||||
const beforeSuccessorHeight = h.state.height;
|
||||
// No extra synthetic old boundary. This is the next reply to the accepted text.
|
||||
ws.message({
|
||||
serverContent: {
|
||||
outputTranscription: { text: 'Handling your new zoom.' },
|
||||
},
|
||||
});
|
||||
ws.message(boundaryZoomCall('typed-successor-call'));
|
||||
await h.controller.toolQueue;
|
||||
await turnOfEventLoop();
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
const afterSuccessor = boundarySnapshot(h);
|
||||
assert.equal(entered, 1);
|
||||
assert.equal(accepted, true);
|
||||
assert.equal(
|
||||
afterPacket.retiredReplyPending,
|
||||
false,
|
||||
'received old terminal must settle its retirement',
|
||||
);
|
||||
assert.equal(afterPacket.retiredBoundaryWait, false);
|
||||
assert.equal(afterPacket.typedText, BOUNDARY_NEXT);
|
||||
assert.equal(
|
||||
h.runnerCalls.filter((x) => x.options.callId === 'typed-successor-call')
|
||||
.length,
|
||||
1,
|
||||
'accepted typed successor must reach the actual runner',
|
||||
);
|
||||
assert.equal(h.state.height, beforeSuccessorHeight * 0.75);
|
||||
assert.equal(afterSuccessor.completions, 1);
|
||||
assert.equal(afterSuccessor.replyPending, false);
|
||||
assert.equal(
|
||||
h.controller.cancelledCalls.has('typed-successor-call'),
|
||||
false,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
for (const terminalAlreadyApplied of [false, true]) {
|
||||
test(
|
||||
`typed reentry retains a genuinely pending boundary when terminal already applied is ${terminalAlreadyApplied}`,
|
||||
options,
|
||||
async (t) => {
|
||||
const h = nativeIntegrationFixture(t);
|
||||
const ws = await h.start();
|
||||
h.activity(ws);
|
||||
let entered = false;
|
||||
const remove = h.session.subscribe((event) => {
|
||||
if (
|
||||
entered ||
|
||||
event.type !== 'transcript' ||
|
||||
event.role !== 'assistant' ||
|
||||
event.text !== 'Boundary control' ||
|
||||
event.final !== terminalAlreadyApplied
|
||||
)
|
||||
return;
|
||||
entered = true;
|
||||
assert.equal(h.session.sendText('First typed replacement'), true);
|
||||
if (terminalAlreadyApplied)
|
||||
assert.equal(h.session.sendText('Second typed replacement'), true);
|
||||
});
|
||||
ws.message({
|
||||
serverContent: {
|
||||
outputTranscription: { text: 'Boundary control' },
|
||||
...(terminalAlreadyApplied ? { turnComplete: true } : {}),
|
||||
},
|
||||
});
|
||||
remove();
|
||||
assert.equal(entered, true);
|
||||
// With no old terminal received, or when that terminal was consumed
|
||||
// before the observer, a real predecessor boundary is still outstanding.
|
||||
assert.equal(h.controller.retiredReplyPending, true);
|
||||
assert.ok(h.controller.retiredBoundaryWait);
|
||||
assert.equal(completions(h).length, 0);
|
||||
assert.equal(h.runnerCalls.length, 0);
|
||||
ws.message({ serverContent: { turnComplete: true } });
|
||||
assert.equal(h.controller.retiredReplyPending, false);
|
||||
assert.equal(h.controller.retiredBoundaryWait, null);
|
||||
await h.tool('after-real-pending-boundary', ws);
|
||||
assertSuccessfulZoom(h, ws, 'after-real-pending-boundary', 1000);
|
||||
h.complete(ws);
|
||||
assert.equal(completions(h).length, 1);
|
||||
},
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
+244
-251
@@ -1,6 +1,6 @@
|
||||
import { interruptGlobeRotation } from '../cameraVerbs.js';
|
||||
import {
|
||||
layerSnapshot,
|
||||
layerSnapshots,
|
||||
feedProvenanceEnvelope,
|
||||
} from '../data/layerSnapshot.js';
|
||||
import { readLayerLifecycleSummary } from './layerSummary.js';
|
||||
@@ -8,9 +8,11 @@ import { SPEECH_BUILDERS, attachVoiceResult } from './speech.js';
|
||||
import { resolveDeicticArgs } from './deixis.js';
|
||||
export { readLayerLifecycleSummary } from './layerSummary.js';
|
||||
import { defaultGeospatial } from '../search/defaults.js';
|
||||
import { msg } from '../i18n/index.js';
|
||||
import * as Cesium from 'cesium';
|
||||
import { CITY_POIS } from '../cityPois.js';
|
||||
import { greatCircleKm } from '../geoDistance.js';
|
||||
import {
|
||||
CITY_POIS,
|
||||
findPoiByName,
|
||||
flyToGlobeView,
|
||||
flyToLandmark,
|
||||
@@ -25,7 +27,6 @@ import {
|
||||
isContextRecordActive,
|
||||
} from '../data/contextStore.js';
|
||||
import { CCTV_FOCUS_RESULT } from '../layers/cctv/index.js';
|
||||
import { contextModeWord } from '../contextModePolicy.js';
|
||||
import {
|
||||
createAnalystEngine,
|
||||
normalizeAnalystSpec,
|
||||
@@ -46,6 +47,12 @@ import { unavailablePlaceSearch } from '../search/placeSearch.js';
|
||||
import * as defaultAnnotationResolver from '../annotations/annotationResolver.js';
|
||||
import { normalizeRadioCountryInput } from '../data/radioCountry.js';
|
||||
import { TR3B_CLASS } from '../data/tr3bRegistry.js';
|
||||
import {
|
||||
getWorldPacket,
|
||||
withContextModeVocabulary,
|
||||
activeContactsWindow,
|
||||
TRACKABLE_FAMILIES,
|
||||
} from '../agent/worldPacket.js';
|
||||
|
||||
const ALLOWED_STYLES = new Set([
|
||||
'normal',
|
||||
@@ -67,8 +74,17 @@ const PANEL_ALIASES = new Map([
|
||||
['styles', 'control-panel'],
|
||||
['filters', 'control-panel'],
|
||||
['visual styles', 'control-panel'],
|
||||
['visual preset', 'control-panel'],
|
||||
['visual presets', 'control-panel'],
|
||||
['provider settings', 'provider-settings'],
|
||||
['cctv', 'cctv-panel'],
|
||||
['cameras', 'cctv-panel'],
|
||||
['voice settings', 'voice-settings'],
|
||||
['voice-settings', 'voice-settings'],
|
||||
['voice preview', 'voice-preview'],
|
||||
['voice-preview', 'voice-preview'],
|
||||
['voice control', 'voice-preview'],
|
||||
['voice controls', 'voice-preview'],
|
||||
['radio', 'radio-panel'],
|
||||
['internet radio', 'radio-panel'],
|
||||
['radio stations', 'radio-panel'],
|
||||
@@ -103,6 +119,7 @@ const PANEL_IDS = new Set([
|
||||
'pp-toggles',
|
||||
'weather-panel',
|
||||
'recent-imagery-panel',
|
||||
'provider-settings',
|
||||
]);
|
||||
const CONTEXT_MODE_ALIASES = new Map([
|
||||
['off', 'off'],
|
||||
@@ -117,64 +134,6 @@ const CONTEXT_MODE_ALIASES = new Map([
|
||||
['space-missions', 'space-missions'],
|
||||
['missions', 'space-missions'],
|
||||
]);
|
||||
/**
|
||||
* Every model-readable field that carries a context-mode id, and what an
|
||||
* absent value means for each.
|
||||
*
|
||||
* `mode` always names a mode, so nothing is 'off'. `entering` and `priorMode`
|
||||
* are absent when there is no such mode at all — calling those 'off' would
|
||||
* assert a state that does not exist.
|
||||
*/
|
||||
const CONTEXT_MODE_RESULT_FIELDS = Object.freeze([
|
||||
{ field: 'mode', emptyAs: 'off' },
|
||||
{ field: 'entering', emptyAs: null },
|
||||
{ field: 'priorMode', emptyAs: null },
|
||||
]);
|
||||
|
||||
/** Nested results that are themselves context-mode payloads the model reads. */
|
||||
const NESTED_CONTEXT_RESULT_FIELDS = Object.freeze([
|
||||
'context',
|
||||
'contextRollback',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Report a context-mode payload in the tools' own vocabulary.
|
||||
*
|
||||
* `set_context_mode` accepts 'contacts' while the mode's internal id is
|
||||
* 'flights'. Reporting the internal id back made the model read
|
||||
* `mode:'flights'` as "Contacts is off" and refuse to answer from the Contacts
|
||||
* window counts sitting in the very same payload (owner field session
|
||||
* 2026-08-21). Secondary fields and nested transition/rollback results are
|
||||
* translated too — one leaked internal id is enough to recreate the confusion,
|
||||
* and a rollback result is exactly what the model reads when something went
|
||||
* wrong. Each internal id is kept alongside as `<field>Internal` for anything
|
||||
* reasoning about layers.
|
||||
* @param {object|null|undefined} state Any payload carrying context-mode fields.
|
||||
* @returns {object|null|undefined} The same payload, modes translated.
|
||||
*/
|
||||
function withContextModeVocabulary(state) {
|
||||
if (!state || typeof state !== 'object') return state;
|
||||
let out = state;
|
||||
const mutable = () => {
|
||||
if (out === state) out = { ...state };
|
||||
return out;
|
||||
};
|
||||
for (const { field, emptyAs } of CONTEXT_MODE_RESULT_FIELDS) {
|
||||
if (!(field in state)) continue;
|
||||
const internal = state[field] ?? null;
|
||||
const target = mutable();
|
||||
target[field] = contextModeWord(internal, { emptyAs });
|
||||
target[`${field}Internal`] = internal;
|
||||
}
|
||||
for (const field of NESTED_CONTEXT_RESULT_FIELDS) {
|
||||
const nested = state[field];
|
||||
if (!nested || typeof nested !== 'object') continue;
|
||||
const translated = withContextModeVocabulary(nested);
|
||||
if (translated !== nested) mutable()[field] = translated;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const COCKPIT_ACTION_ALIASES = new Map([
|
||||
['next', 'next'],
|
||||
['previous', 'previous'],
|
||||
@@ -244,14 +203,6 @@ const STACK_ALIASES = new Map([
|
||||
['road map', 'osm'],
|
||||
]);
|
||||
|
||||
/** Search order for track_entity across entity layer families. */
|
||||
const TRACKABLE_FAMILIES = [
|
||||
{ layerId: 'flights', kind: 'aircraft' },
|
||||
{ layerId: 'military', kind: 'aircraft' },
|
||||
{ layerId: 'ais-live-vessels', kind: 'vessel' },
|
||||
{ layerId: 'satellites', kind: 'satellite' },
|
||||
];
|
||||
|
||||
const FRAME_TARGETS = new Map([
|
||||
['flights', 'flights'],
|
||||
['planes', 'flights'],
|
||||
@@ -302,6 +253,7 @@ export function createGevActionRunner({
|
||||
searchNavigation = searchAndFlyTo,
|
||||
speechBuilders = null,
|
||||
deixis = null,
|
||||
readVoicePanelOperation = null,
|
||||
}) {
|
||||
// Voice enable times and analyst follow-up memory belong to this runner.
|
||||
// Speech builders turn results into say/display envelopes; callers extend
|
||||
@@ -322,9 +274,86 @@ export function createGevActionRunner({
|
||||
}),
|
||||
));
|
||||
installViewTargetPrewarm(viewer);
|
||||
initCameraVerbs(viewer, getViewTargetCartesian);
|
||||
initCameraVerbs(viewer, getViewTargetCartesian, {
|
||||
isCockpitActive: () => Boolean(styleManager?.getCockpitState?.()?.active),
|
||||
});
|
||||
// Nested calls (select_nearest_aircraft composing layer + flight) use the
|
||||
// raw runner; only the outer call gets a voice envelope.
|
||||
const voiceUnavailable = (action) => ({
|
||||
ok: false,
|
||||
action,
|
||||
applied: false,
|
||||
unavailable: true,
|
||||
error: 'Voice panel controls are unavailable in this host.',
|
||||
say: 'Voice panel controls are unavailable in this host.',
|
||||
});
|
||||
async function showVoicePanel(
|
||||
view,
|
||||
open,
|
||||
options,
|
||||
operation = readVoicePanelOperation?.(),
|
||||
) {
|
||||
const current = () =>
|
||||
!options.signal?.aborted &&
|
||||
(!options.isCurrent || options.isCurrent()) &&
|
||||
operation?.isCurrent();
|
||||
if (!operation) return voiceUnavailable('set_panel_open');
|
||||
if (!current())
|
||||
return {
|
||||
ok: false,
|
||||
action: 'set_panel_open',
|
||||
applied: false,
|
||||
cancelled: true,
|
||||
};
|
||||
let exit = null;
|
||||
if (open) {
|
||||
if (typeof styleManager?.getCockpitState !== 'function')
|
||||
return voiceUnavailable('set_panel_open');
|
||||
if (styleManager.getCockpitState()?.active) {
|
||||
if (typeof styleManager.controlCockpit !== 'function')
|
||||
return voiceUnavailable('set_panel_open');
|
||||
exit = await styleManager.controlCockpit('exit');
|
||||
if (styleManager.getCockpitState()?.active)
|
||||
return {
|
||||
ok: false,
|
||||
action: 'set_panel_open',
|
||||
applied: false,
|
||||
exit,
|
||||
error: exit?.error || 'Cockpit did not exit.',
|
||||
};
|
||||
}
|
||||
}
|
||||
if (!current())
|
||||
return {
|
||||
ok: false,
|
||||
action: 'set_panel_open',
|
||||
applied: false,
|
||||
cancelled: true,
|
||||
exit,
|
||||
cockpitExited: Boolean(exit && !styleManager.getCockpitState()?.active),
|
||||
say:
|
||||
exit && !styleManager.getCockpitState()?.active
|
||||
? 'Cockpit exited; the voice panel was left as you last set it.'
|
||||
: 'The voice panel was left as you last set it.',
|
||||
};
|
||||
const result = operation.show(view, open);
|
||||
return {
|
||||
...result,
|
||||
action: 'set_panel_open',
|
||||
panelId: view === 'settings' ? 'voice-settings' : 'voice-preview',
|
||||
exit,
|
||||
state: styleManager?.getCockpitState?.() ?? null,
|
||||
say: !result.ok
|
||||
? 'The voice panel was not changed.'
|
||||
: result.settingsOpen
|
||||
? 'Voice Settings is open.'
|
||||
: result.panelOpen
|
||||
? result.previewEnabled
|
||||
? 'Voice Preview is open.'
|
||||
: 'Voice control is open; Preview remains off.'
|
||||
: 'The voice panel is closed.',
|
||||
};
|
||||
}
|
||||
const runGevAction = async function runGevAction(
|
||||
name,
|
||||
rawArgs = {},
|
||||
@@ -358,6 +387,7 @@ export function createGevActionRunner({
|
||||
// Zoom during an active orbit adjusts the orbit RADIUS (spiral in/out) —
|
||||
// a straight camera move would be snapped back by the per-frame lookAt.
|
||||
if (name === 'adjust_camera_zoom') {
|
||||
interruptGlobeRotation('voice-zoom-intent');
|
||||
const zoomOut = String(args.direction || '').toLowerCase() === 'out';
|
||||
const amt = String(args.amount || 'medium').toLowerCase();
|
||||
const factor = { little: 1.25, medium: 1.6, lot: 2.4 }[amt] || 1.6;
|
||||
@@ -695,6 +725,25 @@ export function createGevActionRunner({
|
||||
return { ok: true, action: 'set_visual_style', style };
|
||||
}
|
||||
|
||||
if (name === 'set_voice_inactivity_timeout') {
|
||||
const operation = readVoicePanelOperation?.();
|
||||
if (!operation) return voiceUnavailable(name);
|
||||
if (!current() || !operation.isCurrent())
|
||||
return { ok: false, action: name, applied: false, cancelled: true };
|
||||
const result = operation.setMinutes(args.minutes);
|
||||
const duration =
|
||||
result.minutes === 0 ? 'disabled' : `${result.minutes} minutes`;
|
||||
return {
|
||||
...result,
|
||||
action: name,
|
||||
say: !result.applied
|
||||
? 'Voice inactivity was not changed.'
|
||||
: result.persisted
|
||||
? `Voice inactivity is ${duration}, saved in this browser.`
|
||||
: `Voice inactivity is ${duration} for now, but could not be saved in this browser.`,
|
||||
};
|
||||
}
|
||||
|
||||
if (name === 'set_panel_open') {
|
||||
const panelId = normalizePanelId(args.panelId || args.panel);
|
||||
if (!panelId)
|
||||
@@ -702,6 +751,70 @@ export function createGevActionRunner({
|
||||
`Unknown panel: ${args.panelId || args.panel || 'missing'}`,
|
||||
);
|
||||
const open = args.open !== false;
|
||||
if (panelId === 'voice-settings' || panelId === 'voice-preview')
|
||||
return showVoicePanel(
|
||||
panelId === 'voice-settings' ? 'settings' : 'preview',
|
||||
open,
|
||||
runOptions,
|
||||
);
|
||||
if (panelId === 'provider-settings') {
|
||||
if (!current())
|
||||
return {
|
||||
ok: false,
|
||||
action: name,
|
||||
panelId,
|
||||
applied: false,
|
||||
cancelled: true,
|
||||
};
|
||||
const doc = globalThis.document;
|
||||
const panel = doc?.getElementById?.('key-setup');
|
||||
const button = open
|
||||
? doc?.getElementById?.('key-setup-chip')
|
||||
: panel?.querySelector?.('[data-key-setup-close]');
|
||||
const concealed =
|
||||
open &&
|
||||
[
|
||||
'ui-clean-view',
|
||||
'recording-mode',
|
||||
'cockpit-mode',
|
||||
'scene-playback-mode',
|
||||
].some((mode) => doc?.body?.classList?.contains(mode));
|
||||
if (
|
||||
concealed ||
|
||||
panel?.dataset?.ready !== 'true' ||
|
||||
typeof button?.click !== 'function'
|
||||
) {
|
||||
const error = concealed
|
||||
? 'Provider Settings is hidden in the current view. Leave that view before opening it.'
|
||||
: 'Provider Settings is unavailable in this runtime.';
|
||||
return {
|
||||
ok: false,
|
||||
action: name,
|
||||
panelId,
|
||||
applied: false,
|
||||
unavailable: true,
|
||||
error,
|
||||
say: error,
|
||||
};
|
||||
}
|
||||
button.click();
|
||||
// Opening unhides immediately; closing removes visible before its fade ends.
|
||||
const applied = open
|
||||
? !panel.hidden
|
||||
: !panel.classList.contains('visible');
|
||||
const say = applied
|
||||
? `Provider Settings is ${open ? 'open' : 'closed'}.`
|
||||
: 'Provider Settings did not change.';
|
||||
return {
|
||||
ok: applied,
|
||||
action: name,
|
||||
panelId,
|
||||
applied,
|
||||
open: applied ? open : !panel.hidden,
|
||||
say,
|
||||
...(!applied ? { error: say } : {}),
|
||||
};
|
||||
}
|
||||
setPanelOpen(styleManager, panelId, open);
|
||||
return { ok: true, action: 'set_panel_open', panelId, open };
|
||||
}
|
||||
@@ -790,6 +903,25 @@ export function createGevActionRunner({
|
||||
error: `Unknown cockpit action: ${args.action || args.command || 'missing'}`,
|
||||
};
|
||||
}
|
||||
if (args.voicePanel !== undefined) {
|
||||
if (
|
||||
action !== 'exit' ||
|
||||
!['preview', 'settings'].includes(args.voicePanel)
|
||||
)
|
||||
return {
|
||||
ok: false,
|
||||
action: 'control_cockpit',
|
||||
applied: false,
|
||||
error: 'Voice panel reveal requires exit and a valid view.',
|
||||
};
|
||||
const result = await showVoicePanel(args.voicePanel, true, runOptions);
|
||||
return {
|
||||
...result,
|
||||
action: 'control_cockpit',
|
||||
cockpitAction: 'exit',
|
||||
voicePanel: args.voicePanel,
|
||||
};
|
||||
}
|
||||
const inferred = normalizeCockpitNavigationHints(rawAction);
|
||||
const targetLayer = normalizeCockpitTargetLayer(
|
||||
args.targetLayer || inferred.targetLayer || args.layer || args.layerId,
|
||||
@@ -1152,6 +1284,18 @@ export function createGevActionRunner({
|
||||
}
|
||||
|
||||
if (name === 'control_cctv') {
|
||||
const action = String(args.action || '').toLowerCase();
|
||||
if (
|
||||
(action === 'enable' || action === 'disable') &&
|
||||
dataManager.layers.get('cctv')?.module
|
||||
) {
|
||||
const result = await runGevAction(
|
||||
'set_layer_visibility',
|
||||
{ layerId: 'cctv', enabled: action === 'enable' },
|
||||
runOptions,
|
||||
);
|
||||
return { ...result, action: 'control_cctv' };
|
||||
}
|
||||
return controlCctv(dataManager, args, styleManager);
|
||||
}
|
||||
|
||||
@@ -1442,7 +1586,7 @@ function controlScene(sceneDirector, args = {}) {
|
||||
};
|
||||
}
|
||||
if (sceneAction === 'stop') {
|
||||
sceneDirector.stopScene('Stopped by voice');
|
||||
sceneDirector.stopScene(msg('Stopped by voice'));
|
||||
return { ok: true, action: 'control_scene', running: false };
|
||||
}
|
||||
if (sceneAction === 'next') {
|
||||
@@ -1492,13 +1636,10 @@ export async function controlCctv(dataManager, args = {}, styleManager = null) {
|
||||
}
|
||||
|
||||
if (action === 'enable' || action === 'disable') {
|
||||
await dataManager.setEnabled('cctv', action === 'enable', {
|
||||
origin: 'voice',
|
||||
});
|
||||
return {
|
||||
ok: true,
|
||||
ok: false,
|
||||
action: 'control_cctv',
|
||||
enabled: dataManager.isEnabled('cctv'),
|
||||
error: 'Enable or disable CCTV through set_layer_visibility',
|
||||
};
|
||||
}
|
||||
if (!dataManager.isEnabled('cctv')) {
|
||||
@@ -1643,14 +1784,18 @@ export async function controlCctv(dataManager, args = {}, styleManager = null) {
|
||||
}
|
||||
|
||||
const RADIO_COUNTRY_CENTERS = new Map([
|
||||
// i18n-ignore: place name in tool data
|
||||
['us', { lat: 39.8, lon: -98.6, country: 'US', label: 'United States' }],
|
||||
// i18n-ignore: place name in tool data
|
||||
['usa', { lat: 39.8, lon: -98.6, country: 'US', label: 'United States' }],
|
||||
[
|
||||
'united states',
|
||||
// i18n-ignore: place name in tool data
|
||||
{ lat: 39.8, lon: -98.6, country: 'US', label: 'United States' },
|
||||
],
|
||||
[
|
||||
'united states of america',
|
||||
// i18n-ignore: place name in tool data
|
||||
{ lat: 39.8, lon: -98.6, country: 'US', label: 'United States' },
|
||||
],
|
||||
]);
|
||||
@@ -2251,6 +2396,7 @@ async function trackEntity(
|
||||
action: 'track_entity',
|
||||
kind: 'fire',
|
||||
layerId: 'local-firms',
|
||||
// i18n-ignore: entity name in a tool result for the voice model
|
||||
label: strongest.label || 'Strongest fire',
|
||||
latitude: strongest.latitude,
|
||||
longitude: strongest.longitude,
|
||||
@@ -2546,26 +2692,6 @@ function runManagedVoiceNavigation(
|
||||
};
|
||||
}
|
||||
|
||||
/** Gathers tracked/selected entities across layer families for read-back. */
|
||||
function collectTrackedEntities(dataManager) {
|
||||
const tracked = [];
|
||||
for (const family of TRACKABLE_FAMILIES) {
|
||||
const module = dataManager.layers.get(family.layerId)?.module;
|
||||
if (!module) continue;
|
||||
try {
|
||||
const info =
|
||||
family.kind === 'vessel'
|
||||
? module.getSelectedInfo?.()
|
||||
: module.getTrackedInfo?.();
|
||||
if (info)
|
||||
tracked.push({ kind: family.kind, layerId: family.layerId, ...info });
|
||||
} catch {
|
||||
// layer not ready
|
||||
}
|
||||
}
|
||||
return tracked;
|
||||
}
|
||||
|
||||
export async function getBasemapLabelContext(
|
||||
viewer,
|
||||
service = defaultGeospatial,
|
||||
@@ -2997,7 +3123,11 @@ function focusDataLayerRow(layerId) {
|
||||
void row.offsetWidth;
|
||||
row.classList.add('gev-voice-focus');
|
||||
window.setTimeout(() => row.classList.remove('gev-voice-focus'), 3000);
|
||||
const name = row.querySelector('.data-name')?.textContent?.trim() || layerId;
|
||||
// The English name: tool results stay English whatever the UI language.
|
||||
const name =
|
||||
row.dataset.layerName ||
|
||||
row.querySelector('.data-name')?.textContent?.trim() ||
|
||||
layerId;
|
||||
return { id: layerId, name };
|
||||
}
|
||||
|
||||
@@ -3199,60 +3329,12 @@ function getCurrentViewState(
|
||||
dataManager,
|
||||
sceneDirector = null,
|
||||
) {
|
||||
const cartographic = Cesium.Cartographic.fromCartesian(
|
||||
viewer.camera.positionWC,
|
||||
);
|
||||
return {
|
||||
ok: true,
|
||||
action: 'get_current_view_state',
|
||||
camera: {
|
||||
latitude: Cesium.Math.toDegrees(cartographic.latitude),
|
||||
longitude: Cesium.Math.toDegrees(cartographic.longitude),
|
||||
heightM: cartographic.height,
|
||||
},
|
||||
style: styleManager.activeStyle || 'normal',
|
||||
context:
|
||||
typeof styleManager.getContextModeState === 'function'
|
||||
? {
|
||||
...withContextModeVocabulary(styleManager.getContextModeState()),
|
||||
// The numbers on the operator's Contacts panel, so a window/count
|
||||
// question can be answered from what they are looking at.
|
||||
...(activeContactsWindow(dataManager)
|
||||
? { contactsWindow: activeContactsWindow(dataManager) }
|
||||
: {}),
|
||||
}
|
||||
: null,
|
||||
cockpit:
|
||||
typeof styleManager.getCockpitState === 'function'
|
||||
? styleManager.getCockpitState()
|
||||
: null,
|
||||
controls:
|
||||
typeof styleManager.getControlState === 'function'
|
||||
? styleManager.getControlState()
|
||||
: null,
|
||||
scenePlayback: sceneDirector?.getPlaybackStatus?.() || null,
|
||||
tracked: collectTrackedEntities(dataManager),
|
||||
// Enabled layers only: the full registry cost ~1k tokens per call and
|
||||
// the model only reasons about what is on.
|
||||
disabledLayerCount: dataManager.getAll().filter((layer) => !layer.enabled)
|
||||
.length,
|
||||
layers: dataManager
|
||||
.getAll()
|
||||
.filter((layer) => layer.enabled)
|
||||
.map((layer) => ({
|
||||
id: layer.id,
|
||||
name: layer.name,
|
||||
enabled: layer.enabled,
|
||||
count: layer.stats?.count || 0,
|
||||
error: layer.stats?.error || null,
|
||||
feedState: layerSnapshot(layer).feedState,
|
||||
source: layerSnapshot(layer).source,
|
||||
lastUpdate: layerSnapshot(layer).lastUpdate,
|
||||
})),
|
||||
feedProvenance: feedProvenanceEnvelope(
|
||||
layerSnapshots(dataManager.getAll()).filter((layer) => layer.enabled),
|
||||
),
|
||||
};
|
||||
return getWorldPacket({
|
||||
viewer,
|
||||
styleManager,
|
||||
dataManager,
|
||||
sceneDirector,
|
||||
});
|
||||
}
|
||||
|
||||
async function getEntityContext(
|
||||
@@ -3732,8 +3814,10 @@ function centerMatchesSubject(center, subjectPosition) {
|
||||
const subjectLat = Cesium.Math.toDegrees(carto.latitude);
|
||||
const subjectLon = Cesium.Math.toDegrees(carto.longitude);
|
||||
return (
|
||||
haversineKm(subjectLat, subjectLon, center.lat, center.lon) <=
|
||||
SUBJECT_CENTER_TOLERANCE_KM
|
||||
greatCircleKm(
|
||||
{ lat: subjectLat, lon: subjectLon },
|
||||
{ lat: center.lat, lon: center.lon },
|
||||
) <= SUBJECT_CENTER_TOLERANCE_KM
|
||||
);
|
||||
}
|
||||
|
||||
@@ -4023,7 +4107,7 @@ function nearbyKnownLandmarks(latitude, longitude, cameraHeightM) {
|
||||
for (const [cityId, city] of Object.entries(CITY_POIS)) {
|
||||
for (let poiIndex = 0; poiIndex < city.pois.length; poiIndex++) {
|
||||
const poi = city.pois[poiIndex];
|
||||
const distanceKm = haversineKm(latitude, longitude, poi.lat, poi.lon);
|
||||
const distanceKm = greatCircleKm({ lat: latitude, lon: longitude }, poi);
|
||||
if (distanceKm > maxDistanceKm) continue;
|
||||
matches.push({
|
||||
name: poi.name,
|
||||
@@ -4038,20 +4122,6 @@ function nearbyKnownLandmarks(latitude, longitude, cameraHeightM) {
|
||||
return matches.sort((a, b) => a.distanceKm - b.distanceKm).slice(0, 5);
|
||||
}
|
||||
|
||||
function haversineKm(lat1, lon1, lat2, lon2) {
|
||||
const toRad = (value) => Cesium.Math.toRadians(value);
|
||||
const radiusKm = 6371.0088;
|
||||
const dLat = toRad(lat2 - lat1);
|
||||
const dLon = toRad(lon2 - lon1);
|
||||
const a =
|
||||
Math.sin(dLat / 2) * Math.sin(dLat / 2) +
|
||||
Math.cos(toRad(lat1)) *
|
||||
Math.cos(toRad(lat2)) *
|
||||
Math.sin(dLon / 2) *
|
||||
Math.sin(dLon / 2);
|
||||
return 2 * radiusKm * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
|
||||
}
|
||||
|
||||
function coarseBasemapPlace(
|
||||
viewScale,
|
||||
latitude,
|
||||
@@ -4491,46 +4561,6 @@ function dominantValue(values) {
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeEntity(viewer, entity, { includeProperties = false } = {}) {
|
||||
const now = Cesium.JulianDate.now();
|
||||
if (entity.__gevContextId) {
|
||||
const store = window.__gevContextStore;
|
||||
const record = store?.entities?.get(entity.__gevContextId);
|
||||
if (record) return summarizeContextRecord(record, { includeProperties });
|
||||
}
|
||||
const props = propertyObject(entity);
|
||||
const layerId = entity.__localLayerId || props.layerId || null;
|
||||
const tags = props.tags || {};
|
||||
const label = cleanText(
|
||||
props.name ||
|
||||
tags.name ||
|
||||
tags['name:en'] ||
|
||||
tags.official_name ||
|
||||
tags.operator ||
|
||||
props.operator ||
|
||||
entity.name ||
|
||||
layerTitle(layerId),
|
||||
);
|
||||
const position =
|
||||
entity.__localBaseCartesian ||
|
||||
entity.position?.getValue?.(now) ||
|
||||
polygonCenter(entity, now);
|
||||
const carto = position ? Cesium.Cartographic.fromCartesian(position) : null;
|
||||
return {
|
||||
id: String(entity.id || ''),
|
||||
name: label || layerTitle(layerId),
|
||||
layerId,
|
||||
layerName: layerTitle(layerId),
|
||||
latitude: carto
|
||||
? Number(Cesium.Math.toDegrees(carto.latitude).toFixed(6))
|
||||
: null,
|
||||
longitude: carto
|
||||
? Number(Cesium.Math.toDegrees(carto.longitude).toFixed(6))
|
||||
: null,
|
||||
properties: includeProperties ? compactProperties(props) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeContextRecord(record, { includeProperties = false } = {}) {
|
||||
return {
|
||||
id: String(record.id || ''),
|
||||
@@ -4549,31 +4579,6 @@ function summarizeContextRecord(record, { includeProperties = false } = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function polygonCenter(entity, now) {
|
||||
const hierarchy = entity.polygon?.hierarchy?.getValue?.(now);
|
||||
const positions = hierarchy?.positions;
|
||||
if (!positions?.length) return null;
|
||||
return Cesium.BoundingSphere.fromPoints(positions).center;
|
||||
}
|
||||
|
||||
function propertyObject(entity) {
|
||||
const raw = entity?.properties?.getValue?.(Cesium.JulianDate.now()) || {};
|
||||
return unwrapProperties(raw);
|
||||
}
|
||||
|
||||
function unwrapProperties(value) {
|
||||
if (!value || typeof value !== 'object') return value;
|
||||
if (Array.isArray(value)) return value.map(unwrapProperties);
|
||||
const out = {};
|
||||
for (const [key, entry] of Object.entries(value)) {
|
||||
out[key] =
|
||||
entry && typeof entry.getValue === 'function'
|
||||
? unwrapProperties(entry.getValue(Cesium.JulianDate.now()))
|
||||
: unwrapProperties(entry);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function compactProperties(props) {
|
||||
const preferredKeys = [
|
||||
'name',
|
||||
@@ -4636,23 +4641,6 @@ function clampNumber(value, min, max, fallback) {
|
||||
* ("which of those is closest?") scoped to the session.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The Contacts panel's own counts, or null when Contacts has no subject.
|
||||
* Read through the awareness snapshot the panel renders, so the two cannot
|
||||
* diverge no matter which surface asks.
|
||||
* @returns {object|null} Panel-equivalent window counts.
|
||||
*/
|
||||
function activeContactsWindow(dataManager) {
|
||||
try {
|
||||
const layer = dataManager?.layers?.get('military-awareness')?.module;
|
||||
return (
|
||||
layer?.contactsWindowFromSnapshot?.(layer.getContextSnapshot?.()) ?? null
|
||||
);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Most records one layer hands a single query. Above every loaded feed seen
|
||||
* so far (flights ~8k, vessels ~12k, a 24 h global FIRMS day ~160k), so
|
||||
@@ -4809,7 +4797,12 @@ function compactAnalystItem(r) {
|
||||
if (!present(r[key]) || key in compact) continue;
|
||||
if (spec.type === 'time') {
|
||||
const ms = Number(r[key]);
|
||||
if (Number.isFinite(ms)) compact.time = new Date(ms).toISOString();
|
||||
// A finite value can still be past Date's range, where toISOString throws.
|
||||
const time = new Date(ms);
|
||||
if (Number.isFinite(time.getTime())) {
|
||||
if (key === 'lastContactAtMs') compact.lastContactAtMs = ms;
|
||||
else compact.time = time.toISOString();
|
||||
}
|
||||
} else compact[key] = r[key];
|
||||
}
|
||||
if (present(r.ageHours)) compact.ageHours = r.ageHours;
|
||||
|
||||
@@ -1079,7 +1079,9 @@ test('generic layer visibility exposes lifecycle truth for every manager phase a
|
||||
});
|
||||
// The voice envelope is additive and states the failure plainly.
|
||||
assert.equal(say, "Couldn't turn on radio.");
|
||||
assert.equal(display.title, 'radio');
|
||||
// The card names the layer, not its id; the result keeps the id.
|
||||
assert.equal(display.title, 'Radio');
|
||||
assert.deepEqual(display.lines, ['Radio layer unavailable']);
|
||||
});
|
||||
|
||||
test('generic voice visibility maps Space Missions to the explicit mission layer', async () => {
|
||||
@@ -3180,7 +3182,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl
|
||||
const requests = [];
|
||||
installKeyedFetch(t, async (url) => {
|
||||
requests.push(String(url));
|
||||
if (String(url).startsWith('https://maps.googleapis.com/')) {
|
||||
if (String(url).startsWith('/api/google/geocode')) {
|
||||
return { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) };
|
||||
}
|
||||
assert.match(String(url), /^https:\/\/photon\.komoot\.io\/api\/\?/);
|
||||
@@ -3205,7 +3207,7 @@ test('voice Radio: a key that geocodes to nothing still places the station, keyl
|
||||
assert.ok(Math.abs(calls[0].criteria.anchor.lat - 20.9101) < 1e-9);
|
||||
assert.ok(Math.abs(calls[0].criteria.anchor.lon - 107.1839) < 1e-9);
|
||||
// Google is asked first and exactly once; Photon answers unbiased, in one call.
|
||||
assert.equal(requests.filter((url) => url.includes('maps.googleapis.com')).length, 1);
|
||||
assert.equal(requests.filter((url) => url.includes('/api/google/geocode')).length, 1);
|
||||
assert.equal(requests.filter((url) => url.includes('photon.komoot.io')).length, 1);
|
||||
assert.match(requests.at(-1), /[?&]q=H%E1%BA%A1\+Long\+Bay/);
|
||||
assert.doesNotMatch(requests.at(-1), /[?&](lat|lon|bbox)=/, 'a named radio location is not viewport-biased');
|
||||
@@ -3218,7 +3220,7 @@ test('voice Radio: the keyless path applies no country filter the keyed path wou
|
||||
// "No Radio station matched" for exactly the places it just resolved, while a
|
||||
// keyed install placed a station. The label may carry it; the filter may not.
|
||||
const { calls, dataManager } = radioSelectionHarness();
|
||||
installKeyedFetch(t, async (url) => (String(url).startsWith('https://maps.googleapis.com/')
|
||||
installKeyedFetch(t, async (url) => (String(url).startsWith('/api/google/geocode')
|
||||
? { ok: true, json: async () => ({ status: 'ZERO_RESULTS', results: [] }) }
|
||||
: {
|
||||
ok: true,
|
||||
@@ -3263,6 +3265,30 @@ test('ALPR common names toggle only the registered camera layer through the norm
|
||||
}
|
||||
});
|
||||
|
||||
test('GNSS integrity and interference common names toggle only the GNSS layer through the normal voice action', async () => {
|
||||
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
|
||||
const viewer = { clock: { onTick: { addEventListener: () => () => {} } },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } } };
|
||||
const calls = [];
|
||||
let enabled = false;
|
||||
const dataManager = {
|
||||
layers: new Map([['gnss-integrity', { module: {} }]]),
|
||||
getAll: () => [{ id: 'gnss-integrity', name: 'GNSS Integrity' }],
|
||||
isEnabled: () => enabled,
|
||||
setEnabled: async (id, value) => { calls.push([id, value]); enabled = value; return true; },
|
||||
};
|
||||
const runner = createGevActionRunner({ viewer, styleManager: {}, dataManager });
|
||||
for (const alias of ['gnss-integrity', 'gnss integrity', 'navigation integrity', 'low navigation accuracy', 'gnss interference', 'gps interference', 'gps jamming', 'gnss']) {
|
||||
for (const value of [true, false]) {
|
||||
const result = await runner('set_layer_visibility', { layerId: alias, enabled: value });
|
||||
assert.equal(result.ok, true);
|
||||
assert.equal(result.layerId, 'gnss-integrity');
|
||||
assert.deepEqual(calls.at(-1), ['gnss-integrity', value]);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('Local ADS-B common names toggle only the receiver layer through the normal voice action', async () => {
|
||||
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
|
||||
const viewer = { clock: { onTick: { addEventListener: () => () => {} } },
|
||||
@@ -3459,6 +3485,26 @@ test('analyst_query refusals carry their code and the allowed values', async ()
|
||||
assert.match(ok.items[0].time, /^\d{4}-\d\d-\d\dT/);
|
||||
});
|
||||
|
||||
test('analyst_query keeps its answer when one record has an out-of-range time', async () => {
|
||||
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
|
||||
const now = Date.now();
|
||||
const earthquakes = {
|
||||
getStats: () => ({ count: 2, lastUpdate: now }),
|
||||
getAnalystRecords: () => [
|
||||
{ id: 'q', lat: 0, lon: 0, magnitude: 5, timeMs: now },
|
||||
{ id: 'r', lat: 1, lon: 1, magnitude: 4, timeMs: 8.64e15 + 1 },
|
||||
],
|
||||
};
|
||||
const { runner } = manifestRunner({ earthquakes }, { enabled: ['earthquakes'] });
|
||||
const result = await runner('analyst_query', { layers: ['earthquakes'], scope: { kind: 'anywhere' } });
|
||||
assert.equal(result.ok, true);
|
||||
const q = result.items.find((item) => item.id === 'q');
|
||||
assert.match(q.time, /^\d{4}-\d\d-\d\dT/);
|
||||
const r = result.items.find((item) => item.id === 'r');
|
||||
assert.ok(r, 'the record is still listed');
|
||||
assert.equal(r.time, undefined, 'an unrepresentable time is omitted');
|
||||
});
|
||||
|
||||
test('spoken layer names from the manifest reach the newer layers', async () => {
|
||||
globalThis.window = globalThis.window || { clearTimeout, setTimeout, requestIdleCallback: null };
|
||||
const stub = () => ({ getStats: () => ({ count: 0 }) });
|
||||
|
||||
@@ -50,21 +50,24 @@ export function initGevVoiceCommands(options) {
|
||||
const referents = createReferentRegistry();
|
||||
// The controller exists only after the runner, so its hooks bind late.
|
||||
const deixis = { pointer, referents, cameraKey };
|
||||
const commands = createVoiceCommands({
|
||||
let commands;
|
||||
commands = createVoiceCommands({
|
||||
...options,
|
||||
pointer,
|
||||
referents,
|
||||
runner: withToolCatalog(
|
||||
createGevActionRunner({ ...options, deixis }),
|
||||
createGevActionRunner({
|
||||
...options,
|
||||
deixis,
|
||||
readVoicePanelOperation: () => commands?.captureVoicePanelOperation?.(),
|
||||
}),
|
||||
options.toolCatalog,
|
||||
),
|
||||
});
|
||||
deixis.imageFrame = () => commands.retainedImageFrame?.() || null;
|
||||
deixis.onPointerUsed = () => commands.announcePointer?.();
|
||||
commands.session?.signal?.addEventListener(
|
||||
'abort',
|
||||
() => pointer?.destroy(),
|
||||
{ once: true },
|
||||
);
|
||||
const lifetime = commands.signal ?? commands.session?.signal;
|
||||
lifetime?.addEventListener('abort', () => pointer?.destroy(), { once: true });
|
||||
if (lifetime?.aborted) pointer?.destroy();
|
||||
return commands;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -124,7 +124,8 @@ test('a partial answer names the layers it could not read', async () => {
|
||||
assert.match(result.say, /Partial; earthquakes not answered/);
|
||||
assert.deepEqual(result.unanswered, ['earthquakes']);
|
||||
assert.ok(view.chips.includes('partial'));
|
||||
assert.ok(view.notes.includes('Not answered: earthquakes'));
|
||||
// The card names the layer; the result keeps its key for the model.
|
||||
assert.ok(view.notes.includes('Not answered: Earthquakes (24h)'));
|
||||
});
|
||||
|
||||
test('"the second one" is the second row the card lists', async () => {
|
||||
|
||||
+59
-25
@@ -34,9 +34,43 @@ const AIRCRAFT_FIELDS = {
|
||||
aircraftClass: ['t'],
|
||||
military: ['f'],
|
||||
onGround: ['f'],
|
||||
squawk: ['t'],
|
||||
emergency: ['f'],
|
||||
emergencyKind: ['t'],
|
||||
emergencySource: ['t'],
|
||||
emergencyBroadcast: ['t'],
|
||||
lastContactAtMs: ['ms'],
|
||||
};
|
||||
|
||||
const layers = [
|
||||
{
|
||||
id: 'gnss-integrity',
|
||||
aliases: [
|
||||
'gnss integrity',
|
||||
'navigation integrity',
|
||||
'low navigation accuracy',
|
||||
'gnss interference',
|
||||
'gps interference',
|
||||
'gps jamming',
|
||||
'gnss',
|
||||
],
|
||||
context: false,
|
||||
noQuery: 'navigation-integrity cells; no countable analyst records',
|
||||
},
|
||||
{
|
||||
id: 's2-ghostmaps',
|
||||
aliases: [
|
||||
's2 ghostmaps',
|
||||
'ghost maps',
|
||||
'ghostmaps',
|
||||
's2 underground',
|
||||
's2 intel',
|
||||
'intel picture',
|
||||
'common intelligence picture',
|
||||
],
|
||||
context: false,
|
||||
noQuery: 'published third-party intel placemarks; not analyst records',
|
||||
},
|
||||
{
|
||||
id: 'flights',
|
||||
track: true,
|
||||
@@ -230,6 +264,24 @@ const layers = [
|
||||
context: false,
|
||||
noQuery: 'map overlay; no countable records',
|
||||
},
|
||||
{
|
||||
id: 'air-quality',
|
||||
aliases: ['air quality', 'aqhi', 'smog', 'air pollution'],
|
||||
context: false,
|
||||
query: {
|
||||
fields: {
|
||||
value: ['n', 'AQHI'],
|
||||
band: ['t'],
|
||||
scale: ['t'],
|
||||
provider: ['t'],
|
||||
name: ['t'],
|
||||
zone: ['t'],
|
||||
risk: ['t'],
|
||||
observedMs: ['ms'],
|
||||
},
|
||||
timeField: 'observedMs',
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'traffic',
|
||||
aliases: ['street traffic', 'road traffic', 'congestion'],
|
||||
@@ -327,30 +379,6 @@ const layers = [
|
||||
context: true,
|
||||
query: { fields: { name: ['t'], class: ['t'], kind: ['t'] } },
|
||||
},
|
||||
{
|
||||
id: 'drone-airspace',
|
||||
aliases: [
|
||||
'drone restrictions',
|
||||
'drone airspace',
|
||||
'no-fly zones',
|
||||
'no fly zones',
|
||||
'drones',
|
||||
],
|
||||
context: false,
|
||||
query: {
|
||||
fields: {
|
||||
name: ['t'],
|
||||
typeCode: ['t'],
|
||||
type: ['t'],
|
||||
lowerDesc: ['t'],
|
||||
upperDesc: ['t'],
|
||||
timesOfUse: ['t'],
|
||||
agency: ['t'],
|
||||
city: ['t'],
|
||||
state: ['t'],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'local-datacenters',
|
||||
aliases: ['datacenters', 'data centers', 'data centres'],
|
||||
@@ -396,6 +424,10 @@ const layers = [
|
||||
export const VOICE_OFF_LAYERS = Object.freeze({
|
||||
'bhote-koshi-2026': 'scene-driven event layer (control_scene)',
|
||||
'bhote-koshi-locator': 'scene-driven event layer (control_scene)',
|
||||
// The Realtime prompt is at its 52,000-character budget; voice toggle and
|
||||
// query for this layer wait until there is room.
|
||||
'drone-airspace':
|
||||
'informational airspace layer; no room in the prompt budget',
|
||||
});
|
||||
|
||||
const TYPE_NAMES = Object.freeze({
|
||||
@@ -497,6 +529,8 @@ const SELF_EVIDENT_IDS = new Set([
|
||||
'radio',
|
||||
'wind',
|
||||
'directions',
|
||||
// The brand is already the layer id; all spoken aliases remain accepted.
|
||||
's2-ghostmaps',
|
||||
]);
|
||||
|
||||
/**
|
||||
@@ -511,7 +545,7 @@ export function voiceLayerAliasHint() {
|
||||
entry.toggle && entry.aliases.length && !SELF_EVIDENT_IDS.has(entry.id),
|
||||
)
|
||||
.map((entry) => `${entry.aliases.slice(0, 2).join('/')}→${entry.id}`)
|
||||
.join('; ');
|
||||
.join(';');
|
||||
}
|
||||
|
||||
/** Field names that already carry their unit (altitudeM, speedKts, …). */
|
||||
|
||||
@@ -20,28 +20,14 @@ test('every catalog layer is in the voice manifest or explicitly voice-off', ()
|
||||
const missing = catalogIds.filter(
|
||||
(id) => !manifestIds.includes(id) && !Object.hasOwn(VOICE_OFF_LAYERS, id),
|
||||
);
|
||||
assert.deepEqual(
|
||||
missing,
|
||||
[],
|
||||
`add a voice manifest entry for: ${missing.join(', ')}`,
|
||||
);
|
||||
assert.deepEqual(missing, [], `add a voice manifest entry for: ${missing.join(', ')}`);
|
||||
for (const id of Object.keys(VOICE_OFF_LAYERS)) {
|
||||
assert.ok(
|
||||
catalogIds.includes(id),
|
||||
`${id} is voice-off but not in the catalog`,
|
||||
);
|
||||
assert.ok(
|
||||
!manifestIds.includes(id),
|
||||
`${id} is both voice-off and in the manifest`,
|
||||
);
|
||||
assert.ok(catalogIds.includes(id), `${id} is voice-off but not in the catalog`);
|
||||
assert.ok(!manifestIds.includes(id), `${id} is both voice-off and in the manifest`);
|
||||
}
|
||||
const unknown = manifestIds.filter((id) => !catalogIds.includes(id));
|
||||
assert.deepEqual(unknown, [], 'the manifest names only catalog layers');
|
||||
assert.equal(
|
||||
new Set(manifestIds).size,
|
||||
manifestIds.length,
|
||||
'one entry per layer',
|
||||
);
|
||||
assert.equal(new Set(manifestIds).size, manifestIds.length, 'one entry per layer');
|
||||
});
|
||||
|
||||
test('each layer declares a query or the reason it has none', () => {
|
||||
@@ -55,8 +41,7 @@ test('each layer declares a query or the reason it has none', () => {
|
||||
});
|
||||
|
||||
test('tool enums are generated from the manifest', () => {
|
||||
const toggle = tool('set_layer_visibility').parameters.properties.layerId
|
||||
.enum;
|
||||
const toggle = tool('set_layer_visibility').parameters.properties.layerId.enum;
|
||||
assert.deepEqual(toggle, [...VOICE_TOGGLE_LAYER_IDS]);
|
||||
for (const id of [
|
||||
'transit',
|
||||
@@ -72,13 +57,9 @@ test('tool enums are generated from the manifest', () => {
|
||||
'rocket-launches',
|
||||
'bikeshare',
|
||||
'local-adsb',
|
||||
'drone-airspace',
|
||||
])
|
||||
assert.ok(toggle.includes(id), `${id} is voice-toggleable`);
|
||||
assert.ok(
|
||||
!toggle.includes('military-awareness'),
|
||||
'Global Context is reached through set_context_mode',
|
||||
);
|
||||
assert.ok(!toggle.includes('military-awareness'), 'Global Context is reached through set_context_mode');
|
||||
assert.deepEqual(
|
||||
tool('show_data_layers_menu').parameters.properties.layerId.enum,
|
||||
[...VOICE_TOGGLE_LAYER_IDS],
|
||||
@@ -87,98 +68,47 @@ test('tool enums are generated from the manifest', () => {
|
||||
tool('get_entity_context').parameters.properties.layerId.enum,
|
||||
[...VOICE_CONTEXT_LAYER_IDS],
|
||||
);
|
||||
assert.ok(
|
||||
!VOICE_CONTEXT_LAYER_IDS.includes('telegeography-submarine-cables'),
|
||||
'cables register no context records',
|
||||
);
|
||||
assert.ok(!VOICE_CONTEXT_LAYER_IDS.includes('telegeography-submarine-cables'), 'cables register no context records');
|
||||
assert.deepEqual(
|
||||
tool('analyst_query').parameters.properties.layers.items.enum,
|
||||
[...VOICE_QUERY_LAYER_IDS],
|
||||
);
|
||||
assert.deepEqual(Object.keys(ANALYST_LAYERS), [...VOICE_QUERY_LAYER_IDS]);
|
||||
const panels = tool('set_panel_open').parameters.properties.panelId.enum;
|
||||
assert.ok(
|
||||
panels.includes('weather-panel') && panels.includes('recent-imagery-panel'),
|
||||
);
|
||||
assert.ok(panels.includes('weather-panel') && panels.includes('recent-imagery-panel'));
|
||||
});
|
||||
|
||||
test('spoken aliases resolve to real layers, keep every legacy phrase and never collide', () => {
|
||||
const aliases = voiceLayerAliasMap();
|
||||
for (const id of aliases.values()) assert.ok(manifestIds.includes(id), id);
|
||||
const legacy = {
|
||||
planes: 'flights',
|
||||
aircraft: 'flights',
|
||||
'military flights': 'military',
|
||||
quakes: 'earthquakes',
|
||||
'space mission': 'rocket-launches',
|
||||
'space missions': 'rocket-launches',
|
||||
missions: 'rocket-launches',
|
||||
'street traffic': 'traffic',
|
||||
cameras: 'cctv',
|
||||
'internet radio': 'radio',
|
||||
'radio stations': 'radio',
|
||||
bikes: 'bikeshare',
|
||||
ais: 'ais-live-vessels',
|
||||
ships: 'ais-live-vessels',
|
||||
vessels: 'ais-live-vessels',
|
||||
'live vessels': 'ais-live-vessels',
|
||||
datacenters: 'local-datacenters',
|
||||
'data centers': 'local-datacenters',
|
||||
'data centres': 'local-datacenters',
|
||||
dams: 'local-dams',
|
||||
'submarine cables': 'telegeography-submarine-cables',
|
||||
cables: 'telegeography-submarine-cables',
|
||||
telegeography: 'telegeography-submarine-cables',
|
||||
firms: 'local-firms',
|
||||
fires: 'local-firms',
|
||||
'active fires': 'local-firms',
|
||||
alpr: 'alpr-cameras',
|
||||
'alpr cameras': 'alpr-cameras',
|
||||
'flock cameras': 'alpr-cameras',
|
||||
'license plate readers': 'alpr-cameras',
|
||||
'license plate cameras': 'alpr-cameras',
|
||||
'plate readers': 'alpr-cameras',
|
||||
'local-adsb': 'local-adsb',
|
||||
'local adsb': 'local-adsb',
|
||||
'local ads-b': 'local-adsb',
|
||||
'my receiver': 'local-adsb',
|
||||
'my antenna': 'local-adsb',
|
||||
'my sdr': 'local-adsb',
|
||||
planes: 'flights', aircraft: 'flights', 'military flights': 'military', quakes: 'earthquakes',
|
||||
'space mission': 'rocket-launches', 'space missions': 'rocket-launches', missions: 'rocket-launches',
|
||||
'street traffic': 'traffic', cameras: 'cctv', 'internet radio': 'radio', 'radio stations': 'radio',
|
||||
bikes: 'bikeshare', ais: 'ais-live-vessels', ships: 'ais-live-vessels', vessels: 'ais-live-vessels',
|
||||
'live vessels': 'ais-live-vessels', datacenters: 'local-datacenters', 'data centers': 'local-datacenters',
|
||||
'data centres': 'local-datacenters', dams: 'local-dams', 'submarine cables': 'telegeography-submarine-cables',
|
||||
cables: 'telegeography-submarine-cables', telegeography: 'telegeography-submarine-cables',
|
||||
firms: 'local-firms', fires: 'local-firms', 'active fires': 'local-firms', alpr: 'alpr-cameras',
|
||||
'alpr cameras': 'alpr-cameras', 'flock cameras': 'alpr-cameras', 'license plate readers': 'alpr-cameras',
|
||||
'license plate cameras': 'alpr-cameras', 'plate readers': 'alpr-cameras', 'local-adsb': 'local-adsb',
|
||||
'local adsb': 'local-adsb', 'local ads-b': 'local-adsb', 'my receiver': 'local-adsb',
|
||||
'my antenna': 'local-adsb', 'my sdr': 'local-adsb',
|
||||
};
|
||||
for (const [phrase, id] of Object.entries(legacy))
|
||||
assert.equal(aliases.get(phrase), id, phrase);
|
||||
for (const [phrase, id] of Object.entries(legacy)) assert.equal(aliases.get(phrase), id, phrase);
|
||||
const added = {
|
||||
buses: 'transit',
|
||||
trains: 'transit',
|
||||
hurricanes: 'weather-cyclones',
|
||||
cyclones: 'weather-cyclones',
|
||||
'weather radar': 'weather-radar',
|
||||
lightning: 'weather-lightning',
|
||||
'military bases': 'military-installations',
|
||||
'recent imagery': 'recent-imagery',
|
||||
'bike share': 'bikeshare',
|
||||
'rocket launches': 'rocket-launches',
|
||||
'street level': 'street-level',
|
||||
mapillary: 'street-level',
|
||||
wind: 'wind',
|
||||
directions: 'directions',
|
||||
'fire perimeters': 'fire-perimeters',
|
||||
'wildfire perimeters': 'fire-perimeters',
|
||||
'burn areas': 'fire-perimeters',
|
||||
'drone restrictions': 'drone-airspace',
|
||||
'drone airspace': 'drone-airspace',
|
||||
'no-fly zones': 'drone-airspace',
|
||||
drones: 'drone-airspace',
|
||||
buses: 'transit', trains: 'transit', hurricanes: 'weather-cyclones', cyclones: 'weather-cyclones',
|
||||
'weather radar': 'weather-radar', lightning: 'weather-lightning', 'military bases': 'military-installations',
|
||||
'recent imagery': 'recent-imagery', 'bike share': 'bikeshare', 'rocket launches': 'rocket-launches',
|
||||
'street level': 'street-level', mapillary: 'street-level',
|
||||
wind: 'wind', directions: 'directions', 'fire perimeters': 'fire-perimeters',
|
||||
'wildfire perimeters': 'fire-perimeters', 'burn areas': 'fire-perimeters',
|
||||
};
|
||||
for (const [phrase, id] of Object.entries(added))
|
||||
assert.equal(aliases.get(phrase), id, phrase);
|
||||
for (const [phrase, id] of Object.entries(added)) assert.equal(aliases.get(phrase), id, phrase);
|
||||
const claimed = new Map();
|
||||
for (const entry of VOICE_LAYER_MANIFEST)
|
||||
for (const alias of entry.aliases) {
|
||||
assert.ok(
|
||||
!claimed.has(alias),
|
||||
`"${alias}" claimed by ${claimed.get(alias)} and ${entry.id}`,
|
||||
);
|
||||
assert.ok(!claimed.has(alias), `"${alias}" claimed by ${claimed.get(alias)} and ${entry.id}`);
|
||||
claimed.set(alias, entry.id);
|
||||
}
|
||||
});
|
||||
@@ -4,7 +4,7 @@ import { progressLine } from './speech.js';
|
||||
* Timing policy for filling silence while tools run. Tuned against the turn
|
||||
* spans' `max_silence_ms`.
|
||||
*/
|
||||
export const NARRATION_TIMING = Object.freeze({
|
||||
const NARRATION_TIMING = Object.freeze({
|
||||
/** Start the earcon when nothing is audible this long after end of speech. */
|
||||
earconAfterMs: 700,
|
||||
/** First spoken progress line when a tool is still running after this. */
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* Private action accounting prototype. Keys describe invocations, never input
|
||||
* provenance, successful effects, provider admission or physical settlement.
|
||||
* The supplied runAction continues to own execution and currentness.
|
||||
*/
|
||||
export function createPanelActionObserver({
|
||||
runAction,
|
||||
onObservation,
|
||||
signal,
|
||||
}) {
|
||||
if (typeof runAction !== 'function')
|
||||
throw new TypeError('runAction is required');
|
||||
const pending = new Map();
|
||||
let disposed = false;
|
||||
const publish = (type, record, details = {}) => {
|
||||
try {
|
||||
onObservation?.(
|
||||
Object.freeze({
|
||||
type,
|
||||
key: record.key,
|
||||
name: record.name,
|
||||
pendingCount: pending.size,
|
||||
...details,
|
||||
}),
|
||||
);
|
||||
} catch {
|
||||
// A presentation observer cannot change execution or its result.
|
||||
}
|
||||
};
|
||||
const release = (record) => {
|
||||
if (!pending.delete(record.key)) return false;
|
||||
try {
|
||||
record.signal?.removeEventListener('abort', record.abort);
|
||||
} catch {
|
||||
/* Observer cleanup only. */
|
||||
}
|
||||
return true;
|
||||
};
|
||||
const retireOne = (record, reason) => {
|
||||
if (release(record)) publish('ownership-retired', record, { reason });
|
||||
};
|
||||
const retire = (reason = 'owner-retired') => {
|
||||
for (const record of [...pending.values()])
|
||||
retireOne(record, typeof reason === 'string' ? reason : 'owner-retired');
|
||||
};
|
||||
const dispose = () => {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
try {
|
||||
signal?.removeEventListener('abort', dispose);
|
||||
} catch {
|
||||
/* Observer cleanup only. */
|
||||
}
|
||||
retire('mount-retired');
|
||||
};
|
||||
signal?.addEventListener('abort', dispose, { once: true });
|
||||
if (signal?.aborted) dispose();
|
||||
|
||||
function observedAction(...args) {
|
||||
if (disposed) return Reflect.apply(runAction, this, args);
|
||||
const record = {
|
||||
key: Object.freeze({}),
|
||||
name: typeof args[0] === 'string' ? args[0] : undefined,
|
||||
};
|
||||
const originalOptions = args[2];
|
||||
let forwarded = args;
|
||||
// Main supplies ordinary data options. Leave exotic/accessor options intact
|
||||
// rather than evaluating getters or pretending all custom factories fit.
|
||||
try {
|
||||
const prototype =
|
||||
originalOptions == null ? null : Object.getPrototypeOf(originalOptions);
|
||||
const descriptors =
|
||||
originalOptions == null
|
||||
? {}
|
||||
: Object.getOwnPropertyDescriptors(originalOptions);
|
||||
const callerSignal = descriptors.signal?.value;
|
||||
if (callerSignal instanceof AbortSignal) record.signal = callerSignal;
|
||||
const dataOnly = Object.values(descriptors).every((d) => 'value' in d);
|
||||
const originalProgress = descriptors.progress?.value;
|
||||
if (
|
||||
originalOptions !== undefined &&
|
||||
originalOptions !== null &&
|
||||
(prototype === Object.prototype || prototype === null) &&
|
||||
dataOnly &&
|
||||
(originalProgress == null || typeof originalProgress === 'function')
|
||||
) {
|
||||
const progressDescriptor = descriptors.progress;
|
||||
delete descriptors.progress;
|
||||
const options = Object.create(prototype, descriptors);
|
||||
Object.defineProperty(options, 'progress', {
|
||||
configurable: true,
|
||||
enumerable: progressDescriptor?.enumerable ?? true,
|
||||
writable: true,
|
||||
value: function (...updates) {
|
||||
const receiver = this === options ? originalOptions : this;
|
||||
const returned =
|
||||
originalProgress == null
|
||||
? undefined
|
||||
: Reflect.apply(originalProgress, receiver, updates);
|
||||
if (pending.has(record.key)) {
|
||||
// No raw mutable result/options/update object escapes to observers.
|
||||
try {
|
||||
const step = Object.getOwnPropertyDescriptor(
|
||||
updates[0] ?? {},
|
||||
'step',
|
||||
)?.value;
|
||||
const label = Object.getOwnPropertyDescriptor(
|
||||
updates[0] ?? {},
|
||||
'label',
|
||||
)?.value;
|
||||
if (typeof step === 'string')
|
||||
publish('progress', record, {
|
||||
step,
|
||||
...(typeof label === 'string' ? { label } : {}),
|
||||
});
|
||||
} catch {
|
||||
/* Private metadata inspection cannot change progress. */
|
||||
}
|
||||
}
|
||||
return returned;
|
||||
},
|
||||
});
|
||||
forwarded = [...args];
|
||||
forwarded[2] = options;
|
||||
}
|
||||
} catch {
|
||||
// Unsupported option introspection is not an execution failure.
|
||||
}
|
||||
if (!record.signal?.aborted) {
|
||||
pending.set(record.key, record);
|
||||
record.abort = () => retireOne(record, 'caller-aborted');
|
||||
try {
|
||||
record.signal?.addEventListener('abort', record.abort, { once: true });
|
||||
} catch {
|
||||
retireOne(record, 'unobserved-signal');
|
||||
}
|
||||
if (pending.has(record.key)) publish('attempt-started', record);
|
||||
}
|
||||
const settle = (outcome) => {
|
||||
if (release(record)) publish('attempt-settled', record, { outcome });
|
||||
};
|
||||
let result;
|
||||
try {
|
||||
result = Reflect.apply(runAction, this, forwarded);
|
||||
} catch (error) {
|
||||
settle('synchronous-throw');
|
||||
throw error;
|
||||
}
|
||||
// The actual common session returns a same-realm native Promise. Do not
|
||||
// assimilate arbitrary thenables, read their .then getter or invent their
|
||||
// settlement; custom return values still reach the caller unchanged.
|
||||
try {
|
||||
if (result instanceof Promise) {
|
||||
Promise.prototype.then.call(
|
||||
result,
|
||||
() => settle('fulfilled'),
|
||||
() => settle('rejected'),
|
||||
);
|
||||
} else if (
|
||||
result === null ||
|
||||
!['object', 'function'].includes(typeof result)
|
||||
) {
|
||||
settle('synchronous-return');
|
||||
} else retireOne(record, 'unobserved-return-kind');
|
||||
} catch {
|
||||
retireOne(record, 'unobserved-return-kind');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
return {
|
||||
runAction: observedAction,
|
||||
retire,
|
||||
dispose,
|
||||
get pendingCount() {
|
||||
return pending.size;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,498 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { getEventListeners } from 'node:events';
|
||||
import { createPanelActionObserver } from './actionObserver.js';
|
||||
import { createVoiceSession } from '../session.js';
|
||||
|
||||
function deferred() {
|
||||
let resolve, reject;
|
||||
const promise = new Promise((yes, no) => {
|
||||
resolve = yes;
|
||||
reject = no;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
test('sync value/throw, receiver, arguments and opaque unsupported result stay exact', () => {
|
||||
const receiver = {},
|
||||
args = {},
|
||||
options = { extra: 7 };
|
||||
const error = Error('original');
|
||||
let calls = 0;
|
||||
const events = [];
|
||||
const observer = createPanelActionObserver({
|
||||
onObservation: (event) => events.push(event),
|
||||
runAction(name, received, opts, tail) {
|
||||
assert.equal(this, receiver);
|
||||
assert.equal(received, args);
|
||||
assert.equal(opts.extra, 7);
|
||||
assert.equal(tail, 'tail');
|
||||
calls++;
|
||||
if (name === 'throw') throw error;
|
||||
return 42;
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
observer.runAction.call(receiver, 'value', args, options, 'tail'),
|
||||
42,
|
||||
);
|
||||
assert.throws(
|
||||
() => observer.runAction.call(receiver, 'throw', args, options, 'tail'),
|
||||
(e) => e === error,
|
||||
);
|
||||
assert.equal(calls, 2);
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
assert.deepEqual(
|
||||
events.filter((e) => e.type === 'attempt-settled').map((e) => e.outcome),
|
||||
['synchronous-return', 'synchronous-throw'],
|
||||
);
|
||||
const opaque = Object.defineProperty({}, 'then', {
|
||||
get() {
|
||||
throw Error('must not read then');
|
||||
},
|
||||
});
|
||||
const another = createPanelActionObserver({ runAction: () => opaque });
|
||||
assert.equal(another.runAction('opaque'), opaque);
|
||||
assert.equal(another.pendingCount, 0);
|
||||
});
|
||||
|
||||
test('native Promise identity, fulfillment object and rejection object are preserved', async () => {
|
||||
for (const reject of [false, true]) {
|
||||
const d = deferred(),
|
||||
value = { exact: true },
|
||||
events = [];
|
||||
const observer = createPanelActionObserver({
|
||||
runAction: () => d.promise,
|
||||
onObservation: (e) => events.push(e),
|
||||
});
|
||||
const returned = observer.runAction('promise');
|
||||
assert.equal(returned, d.promise);
|
||||
assert.equal(observer.pendingCount, 1);
|
||||
if (reject) {
|
||||
d.reject(value);
|
||||
await assert.rejects(returned, (e) => e === value);
|
||||
} else {
|
||||
d.resolve(value);
|
||||
assert.equal(await returned, value);
|
||||
}
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
assert.equal(events.at(-1).outcome, reject ? 'rejected' : 'fulfilled');
|
||||
assert.equal('result' in events.at(-1), false);
|
||||
assert.equal('error' in events.at(-1), false);
|
||||
}
|
||||
});
|
||||
|
||||
test('null/reused callIds have independent keys and settlement cannot retire a sibling', async () => {
|
||||
const a = deferred(),
|
||||
b = deferred(),
|
||||
events = [];
|
||||
const observer = createPanelActionObserver({
|
||||
runAction: (name) => (name === 'a' ? a.promise : b.promise),
|
||||
onObservation: (e) => events.push(e),
|
||||
});
|
||||
const pa = observer.runAction('a', {}, { callId: null });
|
||||
const pb = observer.runAction('b', {}, { callId: null });
|
||||
const starts = events.filter((e) => e.type === 'attempt-started');
|
||||
assert.notEqual(starts[0].key, starts[1].key);
|
||||
assert.equal(observer.pendingCount, 2);
|
||||
b.resolve('b');
|
||||
await pb;
|
||||
assert.equal(observer.pendingCount, 1);
|
||||
assert.equal(events.at(-1).key, starts[1].key);
|
||||
a.resolve('a');
|
||||
await pa;
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
});
|
||||
|
||||
test('progress preserves public-before-original-before-private order, receiver and option references', async () => {
|
||||
const order = [],
|
||||
signal = new AbortController().signal,
|
||||
symbol = Symbol('unknown');
|
||||
let currentCalls = 0;
|
||||
const isCurrent = () => {
|
||||
currentCalls++;
|
||||
return true;
|
||||
};
|
||||
const options = {
|
||||
signal,
|
||||
isCurrent,
|
||||
progress(update) {
|
||||
assert.equal(this, options);
|
||||
order.push('original');
|
||||
return update;
|
||||
},
|
||||
};
|
||||
Object.defineProperty(options, 'hidden', { value: 8 });
|
||||
options[symbol] = 9;
|
||||
const custom = {};
|
||||
const observer = createPanelActionObserver({
|
||||
onObservation(e) {
|
||||
if (e.type === 'progress') order.push('private');
|
||||
},
|
||||
runAction(name, args, received) {
|
||||
assert.equal(received.signal, signal);
|
||||
assert.equal(received.isCurrent, isCurrent);
|
||||
assert.equal(received.hidden, 8);
|
||||
assert.equal(received[symbol], 9);
|
||||
assert.equal(
|
||||
Object.getOwnPropertyDescriptor(received, 'hidden').enumerable,
|
||||
false,
|
||||
);
|
||||
order.push('public');
|
||||
assert.equal(received.progress({ step: 'running' }).step, 'running');
|
||||
return Promise.resolve();
|
||||
},
|
||||
});
|
||||
await observer.runAction('action', {}, options);
|
||||
assert.deepEqual(order, ['public', 'original', 'private']);
|
||||
assert.equal(currentCalls, 0);
|
||||
const explicit = createPanelActionObserver({
|
||||
runAction(name, args, received) {
|
||||
received.progress.call(custom, { step: 'running' });
|
||||
return 0;
|
||||
},
|
||||
});
|
||||
let originalReceiver;
|
||||
explicit.runAction(
|
||||
'explicit',
|
||||
{},
|
||||
{
|
||||
progress() {
|
||||
originalReceiver = this;
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(originalReceiver, custom);
|
||||
});
|
||||
|
||||
test('observer faults and private metadata faults cannot replace original callback errors', async () => {
|
||||
const originalError = Error('callback');
|
||||
const observer = createPanelActionObserver({
|
||||
onObservation() {
|
||||
throw Error('observer');
|
||||
},
|
||||
runAction(name, args, options) {
|
||||
options.progress({ step: 'work' });
|
||||
return Promise.resolve('ok');
|
||||
},
|
||||
});
|
||||
await assert.rejects(
|
||||
Promise.resolve().then(() =>
|
||||
observer.runAction(
|
||||
'error',
|
||||
{},
|
||||
{
|
||||
progress() {
|
||||
throw originalError;
|
||||
},
|
||||
},
|
||||
),
|
||||
),
|
||||
(e) => e === originalError,
|
||||
);
|
||||
const hostile = new Proxy(
|
||||
{},
|
||||
{
|
||||
getOwnPropertyDescriptor() {
|
||||
throw Error('private read');
|
||||
},
|
||||
},
|
||||
);
|
||||
const safe = createPanelActionObserver({
|
||||
runAction(name, args, options) {
|
||||
options.progress(hostile);
|
||||
return Promise.resolve('unchanged');
|
||||
},
|
||||
onObservation() {
|
||||
throw Error('observer');
|
||||
},
|
||||
});
|
||||
assert.equal(await safe.runAction('safe', {}, {}), 'unchanged');
|
||||
});
|
||||
|
||||
test('accessor/custom options are forwarded untouched without extra getter/currentness probes', async () => {
|
||||
let reads = 0,
|
||||
predicates = 0;
|
||||
const options = Object.defineProperty(
|
||||
{
|
||||
isCurrent() {
|
||||
predicates++;
|
||||
return true;
|
||||
},
|
||||
},
|
||||
'progress',
|
||||
{
|
||||
get() {
|
||||
reads++;
|
||||
return () => {};
|
||||
},
|
||||
},
|
||||
);
|
||||
const observer = createPanelActionObserver({
|
||||
runAction(name, args, received) {
|
||||
assert.equal(received, options);
|
||||
received.progress({});
|
||||
received.isCurrent();
|
||||
return Promise.resolve();
|
||||
},
|
||||
});
|
||||
await observer.runAction('accessor', {}, options);
|
||||
assert.equal(reads, 1);
|
||||
assert.equal(predicates, 1);
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
});
|
||||
|
||||
test('real caller abort retires ownership only; late A cannot release or report over B', async () => {
|
||||
const a = deferred(),
|
||||
b = deferred(),
|
||||
caller = new AbortController(),
|
||||
events = [],
|
||||
callbacks = [];
|
||||
let oldOptions;
|
||||
const observer = createPanelActionObserver({
|
||||
onObservation: (e) => events.push(e),
|
||||
runAction(name, args, options) {
|
||||
if (name === 'a') oldOptions = options;
|
||||
return name === 'a' ? a.promise : b.promise;
|
||||
},
|
||||
});
|
||||
const pa = observer.runAction(
|
||||
'a',
|
||||
{},
|
||||
{ signal: caller.signal, progress: () => callbacks.push('original') },
|
||||
);
|
||||
caller.abort();
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
assert.equal(events.at(-1).type, 'ownership-retired');
|
||||
assert.equal(
|
||||
events.some((e) => e.type === 'attempt-settled'),
|
||||
false,
|
||||
);
|
||||
const pb = observer.runAction('b');
|
||||
const count = events.length;
|
||||
oldOptions.progress({ step: 'late' });
|
||||
a.resolve('a');
|
||||
await pa;
|
||||
assert.deepEqual(callbacks, ['original']);
|
||||
assert.equal(events.length, count);
|
||||
assert.equal(observer.pendingCount, 1);
|
||||
b.resolve('b');
|
||||
await pb;
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
assert.equal(getEventListeners(caller.signal, 'abort').length, 0);
|
||||
});
|
||||
|
||||
test('already-aborted caller and disposed mount still invoke original exactly once', async () => {
|
||||
const caller = new AbortController(),
|
||||
mount = new AbortController();
|
||||
caller.abort();
|
||||
let calls = 0;
|
||||
const events = [];
|
||||
const observer = createPanelActionObserver({
|
||||
signal: mount.signal,
|
||||
onObservation: (e) => events.push(e),
|
||||
runAction: () => {
|
||||
calls++;
|
||||
return Promise.resolve('original');
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
await observer.runAction('aborted', {}, { signal: caller.signal }),
|
||||
'original',
|
||||
);
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
mount.abort();
|
||||
assert.equal(await observer.runAction('disposed'), 'original');
|
||||
assert.equal(calls, 2);
|
||||
assert.equal(events.length, 0);
|
||||
assert.equal(getEventListeners(mount.signal, 'abort').length, 0);
|
||||
});
|
||||
|
||||
test('reentrant start retirement retains only fresh ownership and never skips execution', async () => {
|
||||
const a = deferred(),
|
||||
b = deferred(),
|
||||
called = [];
|
||||
let observer, pb;
|
||||
observer = createPanelActionObserver({
|
||||
runAction(name) {
|
||||
called.push(name);
|
||||
return name === 'a' ? a.promise : b.promise;
|
||||
},
|
||||
onObservation(e) {
|
||||
if (e.type === 'attempt-started' && e.name === 'a') {
|
||||
observer.retire('actual-interruption');
|
||||
pb = observer.runAction('b');
|
||||
}
|
||||
},
|
||||
});
|
||||
const pa = observer.runAction('a');
|
||||
assert.deepEqual(called, ['b', 'a']);
|
||||
assert.equal(observer.pendingCount, 1);
|
||||
a.resolve();
|
||||
await pa;
|
||||
assert.equal(observer.pendingCount, 1);
|
||||
b.resolve();
|
||||
await pb;
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
});
|
||||
|
||||
test('actual pinned-main common session: private invocation stream is separate from public events', async (t) => {
|
||||
const publicEvents = [],
|
||||
privateEvents = [],
|
||||
order = [];
|
||||
let hooks, observer;
|
||||
const result = { ok: true, unchanged: true };
|
||||
const session = createVoiceSession({
|
||||
runner: async (name, args, options) => {
|
||||
assert.equal(options.isCurrent(), true);
|
||||
options.progress({ step: 'actual', label: 'Actual' });
|
||||
return result;
|
||||
},
|
||||
createAdapter(h) {
|
||||
hooks = h;
|
||||
observer = createPanelActionObserver({
|
||||
runAction: h.runAction,
|
||||
signal: h.signal,
|
||||
onObservation(e) {
|
||||
privateEvents.push(e);
|
||||
if (e.type === 'progress') order.push('private');
|
||||
},
|
||||
});
|
||||
return {
|
||||
start() {
|
||||
h.emit({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop() {},
|
||||
sendText() {},
|
||||
sendMapEvent() {},
|
||||
};
|
||||
},
|
||||
});
|
||||
t.after(() => session.destroy());
|
||||
session.subscribe((e) => {
|
||||
publicEvents.push(e);
|
||||
if (e.type === 'progress') order.push('public');
|
||||
});
|
||||
await session.start();
|
||||
assert.equal(
|
||||
await observer.runAction(
|
||||
'actual',
|
||||
{},
|
||||
{
|
||||
progress() {
|
||||
order.push('caller');
|
||||
},
|
||||
},
|
||||
),
|
||||
result,
|
||||
);
|
||||
assert.deepEqual(order, ['public', 'caller', 'private']);
|
||||
const call = publicEvents.find((e) => e.type === 'action-call');
|
||||
assert.deepEqual(call, {
|
||||
type: 'action-call',
|
||||
name: 'actual',
|
||||
callId: null,
|
||||
arguments: {},
|
||||
});
|
||||
assert.equal(
|
||||
publicEvents.some((e) => e.type === 'action-settled'),
|
||||
false,
|
||||
);
|
||||
assert.equal(privateEvents.at(-1).type, 'attempt-settled');
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
assert.equal(hooks.signal.aborted, false);
|
||||
});
|
||||
|
||||
test('actual common interruption aborts resources; injected real-lifecycle observation retires uncooperative A', async (t) => {
|
||||
const a = deferred(),
|
||||
b = deferred(),
|
||||
events = [];
|
||||
let hooks, observer, aSignal;
|
||||
const session = createVoiceSession({
|
||||
runner: (name, args, options) => {
|
||||
if (name === 'a') aSignal = options.signal;
|
||||
return name === 'a' ? a.promise : b.promise;
|
||||
},
|
||||
createAdapter(h) {
|
||||
hooks = h;
|
||||
observer = createPanelActionObserver({
|
||||
runAction: h.runAction,
|
||||
signal: h.signal,
|
||||
onObservation: (e) => events.push(e),
|
||||
});
|
||||
return {
|
||||
start() {
|
||||
h.emit({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop() {},
|
||||
sendText() {},
|
||||
sendMapEvent() {},
|
||||
};
|
||||
},
|
||||
});
|
||||
t.after(() => session.destroy());
|
||||
await session.start();
|
||||
const pa = observer.runAction('a');
|
||||
const rejection = assert.rejects(pa, { name: 'AbortError' });
|
||||
hooks.emit({ type: 'interruption', reason: 'actual-common-event' });
|
||||
assert.equal(aSignal.aborted, true);
|
||||
// This bridge is explicitly supplied by the test, not automatically inferred.
|
||||
observer.retire('observed-common-interruption');
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
const pb = observer.runAction('b');
|
||||
const count = events.length;
|
||||
a.resolve({ ok: true });
|
||||
await rejection;
|
||||
assert.equal(events.length, count);
|
||||
assert.equal(observer.pendingCount, 1);
|
||||
b.resolve({ ok: true });
|
||||
await pb;
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
});
|
||||
|
||||
test('null-prototype option descriptors survive instrumentation; mount abort does not fabricate settlement', async () => {
|
||||
const mount = new AbortController(),
|
||||
d = deferred(),
|
||||
events = [],
|
||||
originalProgress = [];
|
||||
const options = Object.assign(Object.create(null), {
|
||||
progress: () => originalProgress.push('original'),
|
||||
});
|
||||
let received;
|
||||
const observer = createPanelActionObserver({
|
||||
signal: mount.signal,
|
||||
onObservation: (e) => events.push(e),
|
||||
runAction(name, args, opts) {
|
||||
received = opts;
|
||||
return d.promise;
|
||||
},
|
||||
});
|
||||
const promise = observer.runAction('pending', {}, options);
|
||||
assert.equal(Object.getPrototypeOf(received), null);
|
||||
mount.abort();
|
||||
assert.equal(observer.pendingCount, 0);
|
||||
assert.equal(events.at(-1).type, 'ownership-retired');
|
||||
const count = events.length;
|
||||
received.progress({ step: 'late' });
|
||||
d.resolve('done');
|
||||
assert.equal(await promise, 'done');
|
||||
assert.deepEqual(originalProgress, ['original']);
|
||||
assert.equal(events.length, count);
|
||||
assert.equal(getEventListeners(mount.signal, 'abort').length, 0);
|
||||
});
|
||||
|
||||
test('omitted and explicit undefined options keep original arity and values', () => {
|
||||
const calls = [];
|
||||
const observer = createPanelActionObserver({
|
||||
runAction: function (...args) {
|
||||
calls.push(args);
|
||||
return 1;
|
||||
},
|
||||
});
|
||||
assert.equal(observer.runAction('omitted', {}), 1);
|
||||
assert.equal(observer.runAction('explicit', {}, undefined), 1);
|
||||
assert.equal(calls[0].length, 2);
|
||||
assert.equal(calls[1].length, 3);
|
||||
assert.equal(calls[0][2], undefined);
|
||||
assert.equal(calls[1][2], undefined);
|
||||
});
|
||||
@@ -0,0 +1,304 @@
|
||||
import { t } from '../../i18n/index.js';
|
||||
import { createManualVoicePanel } from './manualPanel.js';
|
||||
import {
|
||||
createVoicePreferences,
|
||||
bindPreferenceControls,
|
||||
} from './preferences.js';
|
||||
import { createPanelActionObserver } from './actionObserver.js';
|
||||
import { observeNativeActivity } from './nativeActivity.js';
|
||||
import { createVoiceInactivity } from './inactivity.js';
|
||||
|
||||
/** One UI/preference owner across provider mounts; no semantic voice routes. */
|
||||
export function createVoicePanelHost({
|
||||
storage,
|
||||
clock = {},
|
||||
readCockpitState = () => ({
|
||||
active:
|
||||
globalThis.document?.body?.classList?.contains('cockpit-mode') === true,
|
||||
}),
|
||||
} = {}) {
|
||||
const preferences = createVoicePreferences({ storage });
|
||||
let current = null;
|
||||
let disposed = false;
|
||||
let saved = { view: 'preview', panelOpen: false, automatic: true };
|
||||
function observeActions(hooks) {
|
||||
const observer = createPanelActionObserver({
|
||||
runAction: hooks.runAction,
|
||||
signal: hooks.signal,
|
||||
onObservation: () => {
|
||||
if (current?.observer === observer) current.workChanged();
|
||||
},
|
||||
});
|
||||
return observer;
|
||||
}
|
||||
function mount({ ui, card, session, observer, nativeProvider }) {
|
||||
if (disposed || !ui.voicePanel || !card?.setEnabled || !card?.setPresented)
|
||||
return null;
|
||||
current?.destroy();
|
||||
let dead = false;
|
||||
let revision = 0;
|
||||
let operationEpoch = 0;
|
||||
let operationWrite = false;
|
||||
let panel;
|
||||
let idle;
|
||||
let refreshQueued = false;
|
||||
let automatic = saved.automatic;
|
||||
let inputBusy = false;
|
||||
let finalTranscript = null;
|
||||
const presentationLifetime = new AbortController();
|
||||
const rearm = () => {
|
||||
if (!owns() || !session.isActive()) return;
|
||||
automatic = true;
|
||||
saved.automatic = true;
|
||||
card.rearm?.();
|
||||
// Unknown IDs cannot distinguish a fresh identical answer after input.
|
||||
if (!finalTranscript?.id) finalTranscript = null;
|
||||
};
|
||||
const owns = () =>
|
||||
!dead && !disposed && current === binding && !session.disposed;
|
||||
const remember = ({ automatic: auto = false } = {}) => {
|
||||
if (!auto) {
|
||||
revision++;
|
||||
automatic = false;
|
||||
}
|
||||
if (panel) {
|
||||
const value = panel.snapshot();
|
||||
saved = { view: value.view, panelOpen: value.panelOpen, automatic };
|
||||
}
|
||||
};
|
||||
const renderActivity = () => {
|
||||
if (ui.voiceTimeoutStatus)
|
||||
ui.voiceTimeoutStatus.textContent =
|
||||
nativeProvider === 'openai' || nativeProvider === 'gemini'
|
||||
? ''
|
||||
: t('Automatic stop is unavailable for this voice connection.');
|
||||
};
|
||||
const changed = () => {
|
||||
if (!owns()) return;
|
||||
const busy = activity.snapshot().inputBusy;
|
||||
if (busy && !inputBusy) rearm();
|
||||
inputBusy = busy;
|
||||
idle?.update({ activity: true });
|
||||
renderActivity();
|
||||
};
|
||||
const activity = observeNativeActivity({
|
||||
adapter: session.adapter,
|
||||
nativeProvider,
|
||||
signal: session.signal,
|
||||
onChange: changed,
|
||||
onInput: rearm,
|
||||
});
|
||||
const closeAfterStop = (ticket) => {
|
||||
if (owns() && ticket === revision && !session.isActive()) panel.close();
|
||||
};
|
||||
const binding = {
|
||||
observer,
|
||||
get panel() {
|
||||
return panel;
|
||||
},
|
||||
stopTicket: () => revision,
|
||||
closeAfterStop,
|
||||
workChanged: changed,
|
||||
captureOperation() {
|
||||
const stamp = revision;
|
||||
const epoch = operationEpoch;
|
||||
const isCurrent = () =>
|
||||
owns() && stamp === revision && epoch === operationEpoch;
|
||||
const cancelled = () => ({
|
||||
ok: false,
|
||||
applied: false,
|
||||
cancelled: true,
|
||||
});
|
||||
return {
|
||||
isCurrent,
|
||||
setMinutes(value) {
|
||||
if (!isCurrent()) return cancelled();
|
||||
if (typeof value !== 'number' || !Number.isInteger(value))
|
||||
return {
|
||||
ok: false,
|
||||
applied: false,
|
||||
changed: false,
|
||||
persisted: null,
|
||||
reason: 'invalid-value',
|
||||
minutes: preferences.snapshot().minutes,
|
||||
};
|
||||
let result;
|
||||
operationWrite = true;
|
||||
try {
|
||||
result = preferences.setMinutes(value);
|
||||
} finally {
|
||||
operationWrite = false;
|
||||
}
|
||||
return {
|
||||
ok: result.ok,
|
||||
applied: result.applied,
|
||||
changed: result.changed,
|
||||
persisted: result.persisted,
|
||||
minutes: result.minutes,
|
||||
...(result.reason ? { reason: result.reason } : {}),
|
||||
};
|
||||
},
|
||||
show(view, open = true) {
|
||||
if (!isCurrent()) return cancelled();
|
||||
const result = open ? panel.select(view) : panel.close(view);
|
||||
return { ...result, applied: result.ok };
|
||||
},
|
||||
};
|
||||
},
|
||||
handle(event) {
|
||||
if (!owns()) return;
|
||||
if (event.type !== 'state' && !session.isActive()) return;
|
||||
if (
|
||||
event.type === 'state' &&
|
||||
['connecting', 'idle', 'error'].includes(event.state)
|
||||
) {
|
||||
finalTranscript = null;
|
||||
if (event.state !== 'connecting') saved.automatic = automatic = false;
|
||||
}
|
||||
if (
|
||||
event.type === 'interruption' &&
|
||||
!event.preserveAdvisoryCaption &&
|
||||
['user-speech', 'user-text', 'push-to-talk'].includes(event.reason)
|
||||
)
|
||||
rearm();
|
||||
if (
|
||||
event.type === 'transcript' &&
|
||||
event.role === 'assistant' &&
|
||||
event.text
|
||||
) {
|
||||
const id = event.responseId || event.itemId || null;
|
||||
if (
|
||||
finalTranscript &&
|
||||
(id ? id === finalTranscript.id : !finalTranscript.id) &&
|
||||
((!event.final && id) ||
|
||||
(event.final && event.text === finalTranscript.text))
|
||||
)
|
||||
return;
|
||||
if (event.final) finalTranscript = { id, text: event.text };
|
||||
if (
|
||||
automatic &&
|
||||
preferences.snapshot().previewEnabled &&
|
||||
!readCockpitState()?.active
|
||||
)
|
||||
panel.select('preview', { automatic: true });
|
||||
}
|
||||
// A retired native boundary is not fresh input and must not revive
|
||||
// dismissal or erase the advisory caption it explicitly preserves.
|
||||
if (!(event.type === 'interruption' && event.preserveAdvisoryCaption))
|
||||
card.handle(event);
|
||||
if (
|
||||
event.type === 'interruption' ||
|
||||
(event.type === 'state' &&
|
||||
['connecting', 'idle', 'error'].includes(event.state))
|
||||
) {
|
||||
operationEpoch++;
|
||||
observer.retire('session-boundary');
|
||||
}
|
||||
activity.handle(event);
|
||||
idle?.update();
|
||||
// Native controllers finish processing after forwarding some events.
|
||||
// One guarded microtask refresh reads the final current local state.
|
||||
if (!refreshQueued) {
|
||||
refreshQueued = true;
|
||||
queueMicrotask(() => {
|
||||
refreshQueued = false;
|
||||
if (!owns()) return;
|
||||
activity.handle({ type: 'refresh' });
|
||||
idle?.update();
|
||||
});
|
||||
}
|
||||
},
|
||||
snapshot: () => ({
|
||||
...panel.snapshot(),
|
||||
preferences: preferences.snapshot(),
|
||||
activity: activity.snapshot(),
|
||||
inactivity: idle.snapshot(),
|
||||
}),
|
||||
destroy() {
|
||||
if (dead) return;
|
||||
dead = true;
|
||||
presentationLifetime.abort();
|
||||
idle?.destroy();
|
||||
activity.destroy();
|
||||
fields.dispose();
|
||||
observer.dispose();
|
||||
panel.destroy();
|
||||
session.signal.removeEventListener('abort', binding.destroy);
|
||||
if (current === binding) current = null;
|
||||
},
|
||||
};
|
||||
current = binding;
|
||||
panel = createManualVoicePanel({
|
||||
ui,
|
||||
card,
|
||||
getPreviewEnabled: () => preferences.snapshot().previewEnabled,
|
||||
getCockpitActive: () => readCockpitState()?.active === true,
|
||||
copy: {
|
||||
previewEmpty: t('Voice replies appear here.'),
|
||||
previewDisabled: t('Preview is off.'),
|
||||
},
|
||||
onPresentationIntent: remember,
|
||||
});
|
||||
const fields = bindPreferenceControls({
|
||||
ui,
|
||||
preferences,
|
||||
copy: {
|
||||
previewNote: '',
|
||||
timeoutNote: t('These preferences stay in this browser.'),
|
||||
notSaved: t('Your choice applies now, but could not be saved.'),
|
||||
},
|
||||
onChange: (state) => {
|
||||
if (!owns()) return;
|
||||
if (!operationWrite) revision++;
|
||||
card.setEnabled(state.previewEnabled);
|
||||
panel.sync();
|
||||
idle?.setMinutes(state.minutes);
|
||||
},
|
||||
});
|
||||
globalThis.window?.addEventListener(
|
||||
'gev:cockpit-mode-changed',
|
||||
() => {
|
||||
if (owns()) panel.sync();
|
||||
},
|
||||
{ signal: presentationLifetime.signal },
|
||||
);
|
||||
ui.root?.querySelector('#gev-voice-card-dismiss')?.addEventListener(
|
||||
'click',
|
||||
() => {
|
||||
remember();
|
||||
if (panel.snapshot().panelOpen) ui.previewTab?.focus();
|
||||
},
|
||||
{ signal: presentationLifetime.signal },
|
||||
);
|
||||
card.setEnabled(preferences.snapshot().previewEnabled);
|
||||
panel.sync();
|
||||
idle = createVoiceInactivity({
|
||||
readActivity: activity.snapshot,
|
||||
readPending: () => observer.pendingCount,
|
||||
minutes: preferences.snapshot().minutes,
|
||||
signal: session.signal,
|
||||
...clock,
|
||||
onExpire: () => {
|
||||
if (!owns()) return;
|
||||
const ticket = revision;
|
||||
session.stop();
|
||||
closeAfterStop(ticket);
|
||||
},
|
||||
});
|
||||
if (saved.panelOpen) panel.select(saved.view, { automatic: true });
|
||||
renderActivity();
|
||||
session.signal.addEventListener('abort', binding.destroy, { once: true });
|
||||
if (session.signal.aborted) binding.destroy();
|
||||
return binding;
|
||||
}
|
||||
return {
|
||||
observeActions,
|
||||
mount,
|
||||
snapshot: () => preferences.snapshot(),
|
||||
destroy() {
|
||||
if (disposed) return;
|
||||
current?.destroy();
|
||||
disposed = true;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,756 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createVoiceControl } from '../control.js';
|
||||
import { createGevActionRunner } from '../gevActions.js';
|
||||
import { createVoiceCommands as bind } from '../sessionCommands.js';
|
||||
import { createVoiceCommands as compose } from '../commands.js';
|
||||
import { createProviderCommands } from '../providerCommands.js';
|
||||
import { createVoicePanelHost } from './host.js';
|
||||
import { installPanelDom, controlledClock } from './testDom.mjs';
|
||||
|
||||
function factory(instances, { stopError = null, native = false } = {}) {
|
||||
return (hooks) => {
|
||||
const input = {
|
||||
pushToTalkKeyHeld: false,
|
||||
setMicrophoneEnabled() {},
|
||||
releasePushToTalkKey() {
|
||||
this.pushToTalkKeyHeld = false;
|
||||
},
|
||||
resetSession() {},
|
||||
};
|
||||
const controller = native
|
||||
? {
|
||||
input,
|
||||
_input: input,
|
||||
_turns: { observeSpeech() {} },
|
||||
status: 'idle',
|
||||
dc: null,
|
||||
sendTextCommand(text) {
|
||||
if (!text) return false;
|
||||
hooks.emit({
|
||||
type: 'pointer',
|
||||
pointer: { kind: 'here', label: 'Typed target' },
|
||||
});
|
||||
},
|
||||
createAudio(options) {
|
||||
this.audioCallbacks = options;
|
||||
return {
|
||||
pending: false,
|
||||
play() {},
|
||||
clearPlayback() {},
|
||||
close() {},
|
||||
};
|
||||
},
|
||||
handleMessage() {},
|
||||
userActivityOpen: false,
|
||||
pendingTools: 0,
|
||||
isActive: () => true,
|
||||
}
|
||||
: {};
|
||||
const adapter = {
|
||||
controller,
|
||||
capabilities: { pushToTalk: true },
|
||||
starts: 0,
|
||||
stops: 0,
|
||||
bindControls() {},
|
||||
start() {
|
||||
this.starts++;
|
||||
if (native) {
|
||||
hooks.emit({ type: 'state', state: 'connecting' });
|
||||
controller.audio = controller.createAudio({ pushToTalk: false });
|
||||
hooks.emit({ type: 'state', state: 'connecting' });
|
||||
// Native open-mic startup marks presentation pending before speech.
|
||||
controller.userTurnPending = true;
|
||||
}
|
||||
hooks.emit({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop() {
|
||||
this.stops++;
|
||||
if (stopError) throw stopError;
|
||||
},
|
||||
sendText: () => false,
|
||||
sendMapEvent() {},
|
||||
getStartOptions: () => ({ pushToTalk: false }),
|
||||
hooks,
|
||||
input,
|
||||
};
|
||||
instances.push(adapter);
|
||||
return adapter;
|
||||
};
|
||||
}
|
||||
function mounted(t, options = {}) {
|
||||
const doc = installPanelDom(t);
|
||||
const clock = controlledClock();
|
||||
const host = createVoicePanelHost({
|
||||
storage: options.storage ?? null,
|
||||
clock,
|
||||
readCockpitState: options.readCockpitState,
|
||||
});
|
||||
const instances = [];
|
||||
const commands = bind({
|
||||
runner: options.runner ?? (async () => ({ ok: true })),
|
||||
dataManager: { layers: new Map() },
|
||||
createSession: factory(instances, options),
|
||||
panelHost: host,
|
||||
nativePanelProvider: options.native
|
||||
? (options.nativeProvider ?? 'gemini')
|
||||
: null,
|
||||
});
|
||||
t.after(() => {
|
||||
try {
|
||||
commands.session.destroy();
|
||||
} catch {}
|
||||
host.destroy();
|
||||
});
|
||||
return { doc, clock, host, instances, commands, ui: commands.ui };
|
||||
}
|
||||
|
||||
test('actual control default remains opt-out; panel markup has unique controlled panes and an outside alert', (t) => {
|
||||
const doc = installPanelDom(t);
|
||||
const baseline = createVoiceControl();
|
||||
assert.equal(baseline.voicePanel, null);
|
||||
const ui = createVoiceControl({ reset: true, panel: true });
|
||||
assert.equal(doc.getElementById('gev-voice-panel'), ui.voicePanel);
|
||||
assert.equal(
|
||||
ui.voiceSettingsButton.getAttribute('aria-controls'),
|
||||
ui.voicePanel.id,
|
||||
);
|
||||
assert.equal(ui.previewTab.getAttribute('aria-controls'), ui.previewPanel.id);
|
||||
assert.equal(
|
||||
ui.settingsTab.getAttribute('aria-controls'),
|
||||
ui.voiceSettingsPanel.id,
|
||||
);
|
||||
assert.equal(ui.previewPanel.contains(ui.card), true);
|
||||
assert.equal(
|
||||
ui.voicePanel.contains(ui.root.querySelector('.gev-voice-error-tray')),
|
||||
false,
|
||||
);
|
||||
assert.equal(ui.root.querySelectorAll('#gev-voice-card').length, 1);
|
||||
});
|
||||
|
||||
test('actual binder/card has launcher toggle, idempotent tabs, focus and disabled Preview', (t) => {
|
||||
const { ui, commands, doc } = mounted(t);
|
||||
ui.voiceSettingsButton.fire('click');
|
||||
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
|
||||
ui.voiceSettingsButton.fire('click');
|
||||
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
|
||||
ui.settingsTab.fire('click');
|
||||
ui.settingsTab.fire('keydown', { key: 'End' });
|
||||
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
|
||||
ui.settingsTab.fire('click');
|
||||
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
|
||||
commands.voicePanel.select('preview');
|
||||
commands.voiceCard.handle({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
final: true,
|
||||
text: 'A visible reply',
|
||||
});
|
||||
const dismiss = ui.root.querySelector('#gev-voice-card-dismiss');
|
||||
dismiss.focus();
|
||||
dismiss.fire('click');
|
||||
assert.equal(ui.card.hidden, true);
|
||||
assert.equal(commands.voicePanel.snapshot().panelOpen, true);
|
||||
assert.equal(doc.activeElement?.id, ui.previewTab.id);
|
||||
ui.voicePanel.fire('keydown', { key: 'Escape', target: doc.activeElement });
|
||||
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
|
||||
commands.voicePanel.select('settings');
|
||||
ui.voicePreviewInput.checked = false;
|
||||
ui.voicePreviewInput.fire('change');
|
||||
commands.voicePanel.select('preview');
|
||||
commands.voiceCard.handle({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
final: true,
|
||||
text: 'A hidden reply',
|
||||
});
|
||||
assert.equal(ui.card.hidden, true);
|
||||
assert.equal(ui.previewEmpty.textContent, 'Preview is off.');
|
||||
ui.previewTab.focus();
|
||||
ui.voicePanel.fire('keydown', { key: 'Escape' });
|
||||
assert.equal(doc.activeElement, ui.voiceSettingsButton);
|
||||
commands.voiceCard.handle({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
final: true,
|
||||
text: 'Later reply',
|
||||
});
|
||||
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
|
||||
});
|
||||
|
||||
test('actual facade remount keeps failed-save live preferences/view and manual close, with no restore writes or focus theft', async (t) => {
|
||||
const doc = installPanelDom(t);
|
||||
const writes = [];
|
||||
const storage = {
|
||||
getItem: () => null,
|
||||
setItem(key, value) {
|
||||
writes.push([key, value]);
|
||||
throw Error('blocked');
|
||||
},
|
||||
};
|
||||
const host = createVoicePanelHost({ storage });
|
||||
const instances = [];
|
||||
const commands = createProviderCommands(
|
||||
{
|
||||
runner: async () => ({}),
|
||||
dataManager: { layers: new Map() },
|
||||
supportedProviders: ['openai', 'gemini'],
|
||||
},
|
||||
{
|
||||
bind: (options) => bind({ ...options, panelHost: host }),
|
||||
factories: { openai: factory(instances), gemini: factory(instances) },
|
||||
storage,
|
||||
},
|
||||
);
|
||||
t.after(() => {
|
||||
commands.destroy();
|
||||
host.destroy();
|
||||
});
|
||||
const old = commands.ui;
|
||||
commands.voicePanel.select('settings');
|
||||
old.voicePreviewInput.checked = false;
|
||||
old.voicePreviewInput.fire('change');
|
||||
old.voiceTimeoutInput.value = '7';
|
||||
old.voiceTimeoutSelect.value = 'custom';
|
||||
old.voiceTimeoutSelect.fire('change');
|
||||
const preferenceWrites = () =>
|
||||
writes.filter(([key]) => key.startsWith('godsEyeView.voice.')).length;
|
||||
assert.equal(preferenceWrites(), 2);
|
||||
old.providerSelect.value = 'gemini';
|
||||
old.providerSelect.fire('change');
|
||||
assert.notEqual(commands.ui, old);
|
||||
assert.equal(old.root.isConnected, false);
|
||||
assert.equal(commands.voicePanel.snapshot().settingsOpen, true);
|
||||
assert.deepEqual(commands.getVoicePreferences(), {
|
||||
previewEnabled: false,
|
||||
minutes: 7,
|
||||
previewPersisted: false,
|
||||
timeoutPersisted: false,
|
||||
});
|
||||
assert.equal(commands.ui.voicePreviewInput.checked, false);
|
||||
assert.match(commands.ui.voiceTimeoutNote.textContent, /could not be saved/);
|
||||
assert.equal(preferenceWrites(), 2);
|
||||
assert.equal(doc.activeElement, commands.ui.providerSelect);
|
||||
commands.ui.voicePreviewInput.checked = true;
|
||||
commands.ui.voicePreviewInput.fire('change');
|
||||
commands.voicePanel.close();
|
||||
await commands.start();
|
||||
const retired = instances[1];
|
||||
commands.ui.providerSelect.value = 'openai';
|
||||
commands.ui.providerSelect.fire('change');
|
||||
await commands.start();
|
||||
assert.equal(commands.session.isActive(), true);
|
||||
retired.hooks.emit({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'Retired output',
|
||||
final: true,
|
||||
});
|
||||
const emit = instances[2].hooks.emit;
|
||||
emit({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'Current output after remount',
|
||||
final: true,
|
||||
});
|
||||
assert.equal(commands.voicePanel.snapshot().panelOpen, false);
|
||||
emit({ type: 'interruption', reason: 'user-speech' });
|
||||
emit({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'Reply after genuine input',
|
||||
final: true,
|
||||
});
|
||||
assert.equal(commands.voicePanel.snapshot().previewVisible, true);
|
||||
});
|
||||
|
||||
test('explicit successful mic Stop closes; failed Stop idle and startup error do not', async (t) => {
|
||||
const good = mounted(t);
|
||||
await good.commands.session.start();
|
||||
good.commands.voicePanel.select('settings');
|
||||
good.ui.button.fire('click');
|
||||
assert.equal(good.commands.voicePanel.snapshot().panelOpen, false);
|
||||
const error = Error('stop failed');
|
||||
const bad = mounted(t, { stopError: error });
|
||||
await bad.commands.session.start();
|
||||
bad.commands.voicePanel.select('settings');
|
||||
assert.throws(
|
||||
() => bad.ui.button.fire('click'),
|
||||
(value) => value === error,
|
||||
);
|
||||
assert.equal(bad.commands.session.state, 'idle');
|
||||
assert.equal(bad.commands.voicePanel.snapshot().settingsOpen, true);
|
||||
bad.instances[0].hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: 'startup failed',
|
||||
});
|
||||
assert.equal(bad.commands.voicePanel.snapshot().settingsOpen, true);
|
||||
});
|
||||
|
||||
test('reentrant manual choice/new start during Stop cannot close the successor choice', async (t) => {
|
||||
const f = mounted(t);
|
||||
await f.commands.session.start();
|
||||
f.commands.voicePanel.select('settings');
|
||||
const unsubscribe = f.commands.session.subscribe((event) => {
|
||||
if (event.type === 'state' && event.state === 'idle')
|
||||
f.commands.voicePanel.select('preview');
|
||||
});
|
||||
f.ui.button.fire('click');
|
||||
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
|
||||
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
|
||||
unsubscribe();
|
||||
});
|
||||
|
||||
test('integrated inactivity rechecks held PTT at expiry, then normal stop closes on later quiet expiry', async (t) => {
|
||||
const f = mounted(t, { native: true });
|
||||
await f.commands.session.start();
|
||||
f.commands.voicePanel.select('settings');
|
||||
f.ui.voiceTimeoutInput.value = '1';
|
||||
f.ui.voiceTimeoutSelect.value = 'custom';
|
||||
f.ui.voiceTimeoutSelect.fire('change');
|
||||
f.instances[0].input.pushToTalkKeyHeld = true;
|
||||
f.clock.advance(60000);
|
||||
assert.equal(f.instances[0].stops, 0);
|
||||
f.instances[0].input.releasePushToTalkKey();
|
||||
f.clock.advance(60000);
|
||||
assert.equal(f.instances[0].stops, 1);
|
||||
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
|
||||
});
|
||||
|
||||
test('None cancels integrated app timer and unknown custom hosts never guess expiry', async (t) => {
|
||||
const native = mounted(t, { native: true });
|
||||
await native.commands.session.start();
|
||||
native.ui.voiceTimeoutSelect.value = '0';
|
||||
native.ui.voiceTimeoutSelect.fire('change');
|
||||
native.clock.advance(600000);
|
||||
assert.equal(native.instances[0].stops, 0);
|
||||
assert.equal(native.commands.getVoicePreferences().minutes, 0);
|
||||
const unknown = mounted(t);
|
||||
await unknown.commands.session.start();
|
||||
unknown.clock.advance(600000);
|
||||
assert.equal(unknown.instances[0].stops, 0);
|
||||
assert.match(unknown.ui.voiceTimeoutStatus.textContent, /unavailable/);
|
||||
});
|
||||
|
||||
test('transient native OpenAI response activity does not announce connection unavailability', async (t) => {
|
||||
const f = mounted(t, { native: true, nativeProvider: 'openai' });
|
||||
await f.commands.session.start();
|
||||
f.ui.voiceTimeoutInput.value = '1';
|
||||
f.ui.voiceTimeoutSelect.value = 'custom';
|
||||
f.ui.voiceTimeoutSelect.fire('change');
|
||||
f.instances[0].controller._turns.observeSpeech({ type: 'response.created' });
|
||||
assert.equal(f.ui.voiceTimeoutStatus.textContent, '');
|
||||
f.clock.advance(60000);
|
||||
assert.equal(f.instances[0].stops, 0, 'unknown output still prevents expiry');
|
||||
f.instances[0].controller._turns.observeSpeech({
|
||||
type: 'output_audio_buffer.cleared',
|
||||
});
|
||||
f.clock.advance(60000);
|
||||
assert.equal(f.instances[0].stops, 1, 'known quiet output permits expiry');
|
||||
});
|
||||
|
||||
test('native factory with custom controller remains uninstrumented despite panel opt-in', (t) => {
|
||||
installPanelDom(t);
|
||||
let inspected = 0;
|
||||
const commands = compose({
|
||||
panel: true,
|
||||
panelStorage: null,
|
||||
runner: async () => ({}),
|
||||
dataManager: { layers: new Map() },
|
||||
createController: (options) => ({
|
||||
ui: options.ui,
|
||||
get _turns() {
|
||||
inspected++;
|
||||
throw Error('custom internal getter');
|
||||
},
|
||||
start() {},
|
||||
stop() {},
|
||||
sendTextCommand() {},
|
||||
notifyMapEvent() {},
|
||||
syncCostUi() {},
|
||||
bindPushToTalkShortcut() {},
|
||||
toggleVoiceTier() {},
|
||||
}),
|
||||
});
|
||||
t.after(() => commands.session.destroy());
|
||||
assert.equal(inspected, 0);
|
||||
assert.equal(commands.getVoicePanelState().activity.known, false);
|
||||
assert.equal(inspected, 0);
|
||||
});
|
||||
|
||||
test('native connecting phases retain output observation; idle starts before first speech and waits for drain', async (t) => {
|
||||
const f = mounted(t, { native: true });
|
||||
await f.commands.session.start();
|
||||
const adapter = f.instances[0];
|
||||
const controller = adapter.controller;
|
||||
assert.equal(
|
||||
f.commands.getVoicePanelState().inactivity.eligible,
|
||||
true,
|
||||
'open microphone presentation-pending is not admitted speech',
|
||||
);
|
||||
controller.audio.pending = true;
|
||||
controller.handleMessage({});
|
||||
f.clock.advance(300000);
|
||||
assert.equal(adapter.stops, 0);
|
||||
assert.equal(f.commands.getVoicePanelState().activity.outputBusy, true);
|
||||
controller.audio.pending = false;
|
||||
controller.audioCallbacks.onDrain();
|
||||
f.clock.advance(300000);
|
||||
assert.equal(adapter.stops, 1);
|
||||
});
|
||||
|
||||
async function directActions(
|
||||
t,
|
||||
{ storage = null, delayExit = false, exposePanel = true } = {},
|
||||
) {
|
||||
installPanelDom(t);
|
||||
let f, releaseExit;
|
||||
const state = { active: true, exits: 0, visual: [] };
|
||||
const styleManager = {
|
||||
getCockpitState: () => ({ active: state.active }),
|
||||
controlCockpit(action) {
|
||||
assert.equal(action, 'exit');
|
||||
state.exits++;
|
||||
state.active = false;
|
||||
const result = { ok: true, state: { active: false } };
|
||||
return delayExit
|
||||
? new Promise((resolve) => {
|
||||
releaseExit = () => resolve(result);
|
||||
})
|
||||
: result;
|
||||
},
|
||||
setPanelCollapsed(...args) {
|
||||
state.visual.push(args);
|
||||
},
|
||||
};
|
||||
const runner = createGevActionRunner({
|
||||
viewer: {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
scene: { canvas: { addEventListener() {}, removeEventListener() {} } },
|
||||
camera: { moveEnd: { addEventListener() {} } },
|
||||
},
|
||||
styleManager,
|
||||
dataManager: { layers: new Map(), getAll: () => [] },
|
||||
readVoicePanelOperation: () =>
|
||||
exposePanel ? f?.commands.captureVoicePanelOperation() : undefined,
|
||||
});
|
||||
f = mounted(t, { native: true, runner, storage });
|
||||
await f.commands.session.start();
|
||||
return {
|
||||
...f,
|
||||
runner,
|
||||
state,
|
||||
run: f.instances[0].hooks.runAction,
|
||||
releaseExit: () => releaseExit(),
|
||||
};
|
||||
}
|
||||
|
||||
test('actual runner applies exact timeout directly and reports live-unsaved truth, invalid/cancelled attempts stay inert', async (t) => {
|
||||
const f = await directActions(t);
|
||||
const result = await f.run('set_voice_inactivity_timeout', { minutes: 7 });
|
||||
assert.equal(result.applied, true);
|
||||
assert.equal(result.persisted, false);
|
||||
assert.equal(result.minutes, 7);
|
||||
assert.match(result.say, /could not be saved/);
|
||||
assert.equal(f.commands.getVoicePreferences().minutes, 7);
|
||||
const invalid = await f.run('set_voice_inactivity_timeout', { minutes: 7.5 });
|
||||
assert.equal(invalid.applied, false);
|
||||
assert.equal(f.commands.getVoicePreferences().minutes, 7);
|
||||
const abort = new AbortController();
|
||||
abort.abort();
|
||||
const cancelled = await f.runner(
|
||||
'set_voice_inactivity_timeout',
|
||||
{ minutes: 2 },
|
||||
{ signal: abort.signal },
|
||||
);
|
||||
assert.equal(cancelled.cancelled, true);
|
||||
assert.equal(f.commands.getVoicePreferences().minutes, 7);
|
||||
const operation = f.commands.captureVoicePanelOperation();
|
||||
f.commands.session.stop();
|
||||
assert.equal(operation.setMinutes(2).cancelled, true);
|
||||
});
|
||||
|
||||
test('named voice settings exits Cockpit directly; Preview stays disabled and visual presets stay separate from Scenes', async (t) => {
|
||||
const f = await directActions(t);
|
||||
const result = await f.run('set_panel_open', {
|
||||
panelId: 'voice-settings',
|
||||
open: true,
|
||||
});
|
||||
assert.equal(f.state.exits, 1);
|
||||
assert.equal(result.settingsOpen, true);
|
||||
f.ui.voicePreviewInput.checked = false;
|
||||
f.ui.voicePreviewInput.fire('change');
|
||||
const preview = await f.run('set_panel_open', {
|
||||
panelId: 'voice control',
|
||||
open: true,
|
||||
});
|
||||
assert.equal(preview.view, 'preview');
|
||||
assert.equal(preview.previewEnabled, false);
|
||||
assert.equal(f.commands.getVoicePreferences().previewEnabled, false);
|
||||
for (const panelId of [
|
||||
'visual styles',
|
||||
'visual preset',
|
||||
'visual presets',
|
||||
'scenes',
|
||||
])
|
||||
await f.run('set_panel_open', { panelId, open: true });
|
||||
assert.equal(f.state.exits, 1);
|
||||
assert.deepEqual(
|
||||
f.state.visual.map(([id]) => id),
|
||||
['control-panel', 'control-panel', 'control-panel', 'scene-panel'],
|
||||
);
|
||||
await assert.rejects(
|
||||
() => f.run('set_panel_open', { panelId: 'settings', open: true }),
|
||||
/Unknown panel: settings/,
|
||||
);
|
||||
assert.equal(
|
||||
f.state.visual.length,
|
||||
4,
|
||||
'an unnamed settings target never guesses a panel',
|
||||
);
|
||||
});
|
||||
|
||||
test('canonical exit reveals requested voice UI; newer manual choice or Stop defeats pending reveal', async (t) => {
|
||||
const f = await directActions(t, { delayExit: true });
|
||||
const pending = f.run('control_cockpit', {
|
||||
action: 'exit',
|
||||
voicePanel: 'settings',
|
||||
});
|
||||
f.commands.voicePanel.select('preview');
|
||||
f.releaseExit();
|
||||
const result = await pending;
|
||||
assert.equal(result.cancelled, true);
|
||||
assert.equal(f.state.active, false);
|
||||
assert.equal(result.exit.ok, true);
|
||||
assert.equal(result.cockpitExited, true);
|
||||
assert.match(result.say, /Cockpit exited/);
|
||||
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
|
||||
f.state.active = true;
|
||||
const alias = f.run('control_cockpit', {
|
||||
action: 'leave cockpit',
|
||||
voicePanel: 'settings',
|
||||
});
|
||||
f.releaseExit();
|
||||
assert.equal((await alias).settingsOpen, true);
|
||||
f.commands.voicePanel.select('preview');
|
||||
f.state.active = true;
|
||||
const stopped = f.run('control_cockpit', {
|
||||
action: 'quit cockpit',
|
||||
voicePanel: 'settings',
|
||||
});
|
||||
f.commands.session.stop();
|
||||
f.releaseExit();
|
||||
await assert.rejects(stopped, { name: 'AbortError' });
|
||||
assert.equal(f.commands.voicePanel.snapshot().view, 'preview');
|
||||
});
|
||||
|
||||
test('unavailable host refuses direct voice operations before Cockpit or preference effects', async (t) => {
|
||||
const f = await directActions(t, { exposePanel: false });
|
||||
const timeout = await f.run('set_voice_inactivity_timeout', { minutes: 1 });
|
||||
assert.equal(timeout.unavailable, true);
|
||||
assert.equal(timeout.applied, false);
|
||||
const panel = await f.run('set_panel_open', {
|
||||
panelId: 'voice-settings',
|
||||
open: true,
|
||||
});
|
||||
assert.equal(panel.unavailable, true);
|
||||
assert.equal(f.state.exits, 0);
|
||||
assert.equal(f.commands.getVoicePreferences().minutes, 5);
|
||||
});
|
||||
|
||||
test('voice timeout change preserves a pending voice reveal', async (t) => {
|
||||
const f = await directActions(t, { delayExit: true });
|
||||
const pending = f.run('control_cockpit', {
|
||||
action: 'exit',
|
||||
voicePanel: 'settings',
|
||||
});
|
||||
const timeout = await f.run('set_voice_inactivity_timeout', { minutes: 0 });
|
||||
assert.equal(timeout.minutes, 0);
|
||||
assert.equal(timeout.applied, true);
|
||||
f.releaseExit();
|
||||
const result = await pending;
|
||||
assert.equal(result.settingsOpen, true);
|
||||
assert.equal(f.commands.getVoicePreferences().minutes, 0);
|
||||
});
|
||||
|
||||
test('active provider output reveals Preview after done; manual choice wins until genuine input, replays do not reset idle', async (t) => {
|
||||
const f = mounted(t, { native: true, nativeProvider: 'openai' });
|
||||
await f.commands.session.start();
|
||||
const emit = f.instances[0].hooks.emit;
|
||||
const reply = (text, responseId = 'r1', extra = {}) =>
|
||||
emit({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text,
|
||||
responseId,
|
||||
final: true,
|
||||
...extra,
|
||||
});
|
||||
emit({ type: 'completion', responseId: 'r1' });
|
||||
reply('Late answer');
|
||||
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
|
||||
const timer = f.commands.voiceCard.hideTimer;
|
||||
reply('Late answer');
|
||||
reply('Late', 'r1', { final: false });
|
||||
assert.equal(f.commands.voiceCard.hideTimer, timer);
|
||||
assert.equal(f.commands.voiceCard.state.assistant, 'Late answer');
|
||||
f.commands.voicePanel.close();
|
||||
emit({
|
||||
type: 'transcript',
|
||||
role: 'user',
|
||||
text: 'advisory caption',
|
||||
final: false,
|
||||
});
|
||||
f.commands.voiceCard.handle({ type: 'dismiss' });
|
||||
const advisory = f.commands.voiceCard.state.user;
|
||||
emit({
|
||||
type: 'interruption',
|
||||
reason: 'user-speech',
|
||||
preserveAdvisoryCaption: true,
|
||||
});
|
||||
assert.equal(f.commands.voiceCard.state.dismissed, true);
|
||||
assert.equal(f.commands.voiceCard.state.user, advisory);
|
||||
reply('Older reply', 'r2');
|
||||
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
|
||||
emit({ type: 'interruption', reason: 'user-speech' });
|
||||
reply('A reply with no newest-request attribution', 'older-id');
|
||||
assert.equal(f.commands.voicePanel.snapshot().panelOpen, true);
|
||||
f.commands.voicePanel.select('settings');
|
||||
reply('Another reply', 'r3');
|
||||
assert.equal(f.commands.voicePanel.snapshot().settingsOpen, true);
|
||||
f.commands.voicePanel.close();
|
||||
f.commands.voiceCard.handle({ type: 'dismiss' });
|
||||
f.commands.sendTextCommand('Show the typed target');
|
||||
reply('Typed reply', 'typed-id');
|
||||
assert.equal(f.commands.voicePanel.snapshot().previewVisible, true);
|
||||
assert.equal(f.commands.voiceCard.state.dismissed, false);
|
||||
assert.equal(f.commands.voiceCard.state.pointer.label, 'Typed target');
|
||||
f.commands.session.stop();
|
||||
reply('Stopped stale output', 'r4');
|
||||
assert.equal(f.commands.voiceCard.state.assistant, null);
|
||||
f.commands.session.destroy();
|
||||
reply('Disposed stale output', 'r5');
|
||||
assert.equal(f.commands.voicePanel.snapshot().panelOpen, false);
|
||||
});
|
||||
|
||||
test('Preview off and actual Cockpit state suppress automatic presentation without enabling preference', async (t) => {
|
||||
let cockpit = false;
|
||||
const f = mounted(t, {
|
||||
native: true,
|
||||
readCockpitState: () => ({ active: cockpit }),
|
||||
});
|
||||
await f.commands.session.start();
|
||||
const emit = f.instances[0].hooks.emit;
|
||||
const reply = (text) =>
|
||||
emit({ type: 'transcript', role: 'assistant', text, final: true });
|
||||
reply('Visible');
|
||||
f.ui.previewTab.focus();
|
||||
cockpit = true;
|
||||
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
|
||||
assert.equal(f.ui.voicePanel.hidden, true);
|
||||
assert.equal(f.doc.activeElement, f.ui.voiceSettingsButton);
|
||||
f.commands.voicePanel.select('settings', { focus: true });
|
||||
assert.equal(f.doc.activeElement, f.ui.settingsTab);
|
||||
f.commands.voicePanel.select('preview', { focus: true });
|
||||
assert.equal(f.doc.activeElement, f.ui.voiceSettingsButton);
|
||||
const outside = f.doc.createElement('button');
|
||||
f.doc.body.appendChild(outside);
|
||||
outside.focus();
|
||||
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
|
||||
assert.equal(f.doc.activeElement, outside);
|
||||
assert.equal(f.commands.voicePanel.snapshot().previewVisible, false);
|
||||
reply('Suppressed in Cockpit');
|
||||
assert.equal(f.ui.voicePanel.hidden, true);
|
||||
f.commands.voicePanel.close();
|
||||
cockpit = false;
|
||||
window.dispatchEvent(new Event('gev:cockpit-mode-changed'));
|
||||
assert.equal(f.ui.voicePanel.hidden, true);
|
||||
f.ui.voicePreviewInput.checked = false;
|
||||
f.ui.voicePreviewInput.fire('change');
|
||||
emit({ type: 'interruption', reason: 'user-speech' });
|
||||
reply('Preference stays off');
|
||||
assert.equal(f.ui.voicePanel.hidden, true);
|
||||
assert.equal(f.commands.getVoicePreferences().previewEnabled, false);
|
||||
});
|
||||
|
||||
test('Provider Settings uses its ready dialog handlers and refuses unavailable or cancelled requests', async (t) => {
|
||||
const f = await directActions(t);
|
||||
t.mock.timers.enable({ apis: ['setTimeout'] });
|
||||
const request = (open, options) =>
|
||||
f.run('set_panel_open', { panelId: 'provider-settings', open }, options);
|
||||
assert.equal((await request(true)).unavailable, true);
|
||||
const doc = globalThis.document;
|
||||
const chip = doc.createElement('button');
|
||||
chip.id = 'key-setup-chip';
|
||||
const panel = doc.createElement('aside');
|
||||
panel.id = 'key-setup';
|
||||
panel.hidden = true;
|
||||
const close = doc.createElement('button');
|
||||
panel.appendChild(close);
|
||||
doc.body.append(chip, panel);
|
||||
const query = panel.querySelector.bind(panel);
|
||||
panel.querySelector = (selector) =>
|
||||
selector === '[data-key-setup-close]' ? close : query(selector);
|
||||
chip.click = () => chip.fire('click');
|
||||
close.click = () => close.fire('click');
|
||||
let frames = 0;
|
||||
const previousFrame = globalThis.requestAnimationFrame;
|
||||
globalThis.requestAnimationFrame = (callback) => {
|
||||
frames++;
|
||||
callback();
|
||||
};
|
||||
t.after(() => {
|
||||
if (previousFrame === undefined) delete globalThis.requestAnimationFrame;
|
||||
else globalThis.requestAnimationFrame = previousFrame;
|
||||
});
|
||||
const { initKeySetup } = await import('../../keySetup.js');
|
||||
let finishStatus;
|
||||
const pending = initKeySetup({
|
||||
documentRef: doc,
|
||||
fetchImpl: () =>
|
||||
new Promise((resolve) => {
|
||||
finishStatus = resolve;
|
||||
}),
|
||||
});
|
||||
assert.equal(
|
||||
(await request(true)).unavailable,
|
||||
true,
|
||||
'initializing is not ready',
|
||||
);
|
||||
finishStatus({
|
||||
ok: true,
|
||||
json: async () => ({ keys: [], setCount: 0, total: 1 }),
|
||||
});
|
||||
const setup = await pending;
|
||||
t.after(() => setup.destroy());
|
||||
doc.body.classList.add('cockpit-mode');
|
||||
assert.equal((await request(true)).unavailable, true);
|
||||
assert.equal(
|
||||
panel.hidden,
|
||||
true,
|
||||
'a concealed view cannot schedule a later surprise reveal',
|
||||
);
|
||||
doc.body.classList.remove('cockpit-mode');
|
||||
const abort = new AbortController();
|
||||
abort.abort();
|
||||
await assert.rejects(() => request(true, { signal: abort.signal }), {
|
||||
name: 'AbortError',
|
||||
});
|
||||
assert.equal(panel.hidden, true);
|
||||
assert.equal((await request(true)).ok, true);
|
||||
assert.equal(panel.hidden, false);
|
||||
assert.equal(panel.classList.contains('visible'), true);
|
||||
assert.equal((await request(true)).ok, true);
|
||||
assert.equal(frames, 1, 'repeated open reuses the dialog');
|
||||
assert.equal((await request(false)).ok, true);
|
||||
assert.equal(panel.classList.contains('visible'), false);
|
||||
t.mock.timers.tick(400);
|
||||
assert.equal(panel.hidden, true);
|
||||
assert.equal((await request(false)).ok, true);
|
||||
assert.equal(
|
||||
f.state.visual.length,
|
||||
0,
|
||||
'provider setup never opens a visual or scene panel',
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
/** Local inactivity only. Unknown activity never authorizes expiry. */
|
||||
export function createVoiceInactivity({
|
||||
readActivity,
|
||||
readPending,
|
||||
minutes = 5,
|
||||
onExpire,
|
||||
signal,
|
||||
setTimer = setTimeout,
|
||||
clearTimer = clearTimeout,
|
||||
now = Date.now,
|
||||
}) {
|
||||
if (!Number.isInteger(minutes) || minutes < 0 || minutes > 60) minutes = 5;
|
||||
let disposed = false,
|
||||
timer = null,
|
||||
deadline = null,
|
||||
eligible = false,
|
||||
epoch = 0;
|
||||
function cancel() {
|
||||
epoch++;
|
||||
if (timer !== null) clearTimer(timer);
|
||||
timer = deadline = null;
|
||||
}
|
||||
function quiet() {
|
||||
const a = readActivity();
|
||||
return (
|
||||
a.known &&
|
||||
a.connected &&
|
||||
!a.held &&
|
||||
!a.inputBusy &&
|
||||
!a.outputBusy &&
|
||||
!a.generationBusy &&
|
||||
!readPending()
|
||||
);
|
||||
}
|
||||
function arm() {
|
||||
cancel();
|
||||
if (!minutes || disposed) return;
|
||||
const id = epoch;
|
||||
deadline = now() + minutes * 60000;
|
||||
timer = setTimer(() => {
|
||||
if (disposed || id !== epoch) return;
|
||||
timer = deadline = null;
|
||||
eligible = quiet();
|
||||
if (eligible) onExpire?.();
|
||||
}, minutes * 60000);
|
||||
}
|
||||
function update({ activity = false } = {}) {
|
||||
if (disposed) return;
|
||||
const next = quiet();
|
||||
if (!next) cancel();
|
||||
else if (!eligible || activity) arm();
|
||||
eligible = next;
|
||||
}
|
||||
function setMinutes(value) {
|
||||
if (!Number.isInteger(value) || value < 0 || value > 60) return false;
|
||||
if (disposed) return false;
|
||||
minutes = value;
|
||||
eligible = false;
|
||||
update();
|
||||
return true;
|
||||
}
|
||||
function destroy() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
cancel();
|
||||
signal?.removeEventListener('abort', destroy);
|
||||
}
|
||||
signal?.addEventListener('abort', destroy, { once: true });
|
||||
if (signal?.aborted) destroy();
|
||||
update();
|
||||
return {
|
||||
update,
|
||||
setMinutes,
|
||||
snapshot: () => ({ minutes, deadline, eligible, disposed }),
|
||||
destroy,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createVoiceInactivity } from './inactivity.js';
|
||||
function fixture() {
|
||||
let activity = {
|
||||
known: true,
|
||||
connected: true,
|
||||
held: false,
|
||||
inputBusy: false,
|
||||
outputBusy: false,
|
||||
generationBusy: false,
|
||||
};
|
||||
let pending = 0,
|
||||
expired = 0,
|
||||
time = 100,
|
||||
serial = 0;
|
||||
const timers = new Map();
|
||||
const idle = createVoiceInactivity({
|
||||
readActivity: () => activity,
|
||||
readPending: () => pending,
|
||||
onExpire: () => expired++,
|
||||
now: () => time,
|
||||
setTimer: (fn, delay) => {
|
||||
timers.set(++serial, { fn, delay });
|
||||
return serial;
|
||||
},
|
||||
clearTimer: (id) => timers.delete(id),
|
||||
});
|
||||
return {
|
||||
idle,
|
||||
activity,
|
||||
timers,
|
||||
setPending: (n) => (pending = n),
|
||||
expired: () => expired,
|
||||
fire: () => {
|
||||
const entry = [...timers][0];
|
||||
timers.delete(entry[0]);
|
||||
entry[1].fn();
|
||||
},
|
||||
tick: (n) => (time += n),
|
||||
};
|
||||
}
|
||||
test('one clock; quiet arms default five minutes; busy blocks and quiescence rearms', () => {
|
||||
const f = fixture();
|
||||
assert.equal([...f.timers.values()][0].delay, 300000);
|
||||
assert.equal(f.idle.snapshot().deadline, 300100);
|
||||
f.activity.outputBusy = true;
|
||||
f.idle.update();
|
||||
assert.equal(f.timers.size, 0);
|
||||
f.activity.outputBusy = false;
|
||||
f.tick(200);
|
||||
f.idle.update();
|
||||
assert.equal(f.idle.snapshot().deadline, 300300);
|
||||
f.fire();
|
||||
assert.equal(f.expired(), 1);
|
||||
});
|
||||
test('actual activity resets but silent unchanged snapshots do not', () => {
|
||||
const f = fixture();
|
||||
const deadline = f.idle.snapshot().deadline;
|
||||
f.tick(1000);
|
||||
f.idle.update();
|
||||
assert.equal(f.idle.snapshot().deadline, deadline);
|
||||
f.idle.update({ activity: true });
|
||||
assert.equal(f.idle.snapshot().deadline, deadline + 1000);
|
||||
});
|
||||
test('None cancels; invalid values cannot replace setting; explicit retry rearms', () => {
|
||||
const f = fixture();
|
||||
assert.equal(f.idle.setMinutes(0), true);
|
||||
assert.equal(f.timers.size, 0);
|
||||
for (const bad of [1.5, -1, 61, '5', NaN])
|
||||
assert.equal(f.idle.setMinutes(bad), false);
|
||||
assert.equal(f.idle.setMinutes(1), true);
|
||||
assert.equal([...f.timers.values()][0].delay, 60000);
|
||||
});
|
||||
test('expiry rereads held, unknown and pending counts despite missing update', () => {
|
||||
for (const kind of ['held', 'unknown', 'pending']) {
|
||||
const f = fixture();
|
||||
if (kind === 'held') f.activity.held = true;
|
||||
if (kind === 'unknown') f.activity.known = false;
|
||||
if (kind === 'pending') f.setPending(2);
|
||||
f.fire();
|
||||
assert.equal(f.expired(), 0);
|
||||
}
|
||||
});
|
||||
test('pending siblings and held-before-connect forbid timer; stale callback cannot expire', () => {
|
||||
const f = fixture();
|
||||
const stale = [...f.timers.values()][0].fn;
|
||||
f.setPending(2);
|
||||
f.idle.update();
|
||||
stale();
|
||||
assert.equal(f.expired(), 0);
|
||||
f.setPending(1);
|
||||
f.idle.update();
|
||||
assert.equal(f.timers.size, 0);
|
||||
f.setPending(0);
|
||||
f.activity.connected = false;
|
||||
f.activity.held = true;
|
||||
f.idle.update();
|
||||
assert.equal(f.timers.size, 0);
|
||||
f.activity.connected = true;
|
||||
f.idle.update();
|
||||
assert.equal(f.timers.size, 0);
|
||||
f.activity.held = false;
|
||||
f.idle.update();
|
||||
assert.equal(f.timers.size, 1);
|
||||
f.idle.destroy();
|
||||
assert.equal(f.timers.size, 0);
|
||||
});
|
||||
@@ -0,0 +1,182 @@
|
||||
const ROW_GRACE_MS = 700;
|
||||
|
||||
/** Manual presentation only. Session and action events belong to the host. */
|
||||
export function createManualVoicePanel({
|
||||
ui,
|
||||
card,
|
||||
getPreviewEnabled,
|
||||
getCockpitActive = () => false,
|
||||
copy,
|
||||
onPresentationIntent,
|
||||
}) {
|
||||
if (!ui?.voicePanel || typeof getPreviewEnabled !== 'function')
|
||||
throw new TypeError(
|
||||
'A voice panel and Preview preference reader are required.',
|
||||
);
|
||||
if (
|
||||
typeof copy?.previewEmpty !== 'string' ||
|
||||
typeof copy?.previewDisabled !== 'string'
|
||||
)
|
||||
throw new TypeError('Localized Preview messages are required.');
|
||||
const lifetime = new AbortController();
|
||||
const panel = ui.voicePanel;
|
||||
const tabs = [ui.previewTab, ui.settingsTab];
|
||||
const doc = panel.ownerDocument;
|
||||
let disposed = false;
|
||||
let open = false;
|
||||
let view = 'preview';
|
||||
const enabled = () => getPreviewEnabled() === true;
|
||||
const suppressed = () => view === 'preview' && getCockpitActive() === true;
|
||||
const visible = (element) =>
|
||||
Boolean(
|
||||
element &&
|
||||
!element.hidden &&
|
||||
element.isConnected !== false &&
|
||||
element.checkVisibility?.({
|
||||
opacityProperty: true,
|
||||
visibilityProperty: true,
|
||||
}) !== false,
|
||||
);
|
||||
const snapshot = () => ({
|
||||
view,
|
||||
panelOpen: !disposed && open,
|
||||
settingsOpen: !disposed && open && view === 'settings',
|
||||
previewEnabled: enabled(),
|
||||
previewVisible:
|
||||
!disposed &&
|
||||
open &&
|
||||
view === 'preview' &&
|
||||
!suppressed() &&
|
||||
enabled() &&
|
||||
visible(ui.card),
|
||||
});
|
||||
const sync = () => {
|
||||
if (disposed) return;
|
||||
const hidePanel = !open || suppressed();
|
||||
const restoreFocus = hidePanel && panel.contains?.(doc?.activeElement);
|
||||
panel.hidden = hidePanel;
|
||||
panel.dataset.view = view;
|
||||
if (ui.previewPanel)
|
||||
ui.previewPanel.hidden = !open || view !== 'preview' || suppressed();
|
||||
if (ui.voiceSettingsPanel)
|
||||
ui.voiceSettingsPanel.hidden = !open || view !== 'settings';
|
||||
ui.voiceSettingsButton?.setAttribute(
|
||||
'aria-expanded',
|
||||
String(open && view === 'settings'),
|
||||
);
|
||||
tabs.forEach((tab, index) => {
|
||||
if (!tab) return;
|
||||
const selected = view === (index === 0 ? 'preview' : 'settings');
|
||||
tab.setAttribute('aria-selected', String(selected));
|
||||
tab.tabIndex = selected ? 0 : -1;
|
||||
});
|
||||
card?.setPresented?.(open && view === 'preview' && !suppressed());
|
||||
if (ui.previewEmpty) {
|
||||
ui.previewEmpty.hidden = Boolean(ui.card && !ui.card.hidden);
|
||||
ui.previewEmpty.textContent = enabled()
|
||||
? copy.previewEmpty
|
||||
: copy.previewDisabled;
|
||||
}
|
||||
if (restoreFocus)
|
||||
[ui.voiceSettingsButton, ui.button].find(visible)?.focus?.();
|
||||
};
|
||||
const refuse = (reason) => ({
|
||||
ok: false,
|
||||
changed: false,
|
||||
reason,
|
||||
...snapshot(),
|
||||
});
|
||||
const select = (next, { focus = false, automatic = false } = {}) => {
|
||||
if (disposed) return refuse('disposed');
|
||||
if (!['preview', 'settings'].includes(next)) return refuse('unknown-view');
|
||||
const changed = !open || view !== next;
|
||||
const outgoing =
|
||||
view === 'preview' ? ui.previewPanel : ui.voiceSettingsPanel;
|
||||
const transferFocus = changed && outgoing?.contains?.(doc?.activeElement);
|
||||
open = true;
|
||||
view = next;
|
||||
onPresentationIntent?.({ automatic });
|
||||
sync();
|
||||
const tab = tabs[next === 'preview' ? 0 : 1];
|
||||
if ((focus || transferFocus) && visible(panel) && visible(tab))
|
||||
tab.focus?.();
|
||||
return { ok: true, changed, ...snapshot() };
|
||||
};
|
||||
const close = (onlyView) => {
|
||||
if (disposed) return refuse('disposed');
|
||||
if (onlyView && !['preview', 'settings'].includes(onlyView))
|
||||
return refuse('unknown-view');
|
||||
if (onlyView && onlyView !== view)
|
||||
return { ok: true, changed: false, ...snapshot() };
|
||||
const changed = open;
|
||||
open = false;
|
||||
onPresentationIntent?.();
|
||||
sync();
|
||||
return { ok: true, changed, ...snapshot() };
|
||||
};
|
||||
const listen = (element, type, handler) =>
|
||||
element?.addEventListener(type, handler, { signal: lifetime.signal });
|
||||
const toggleSettings = () =>
|
||||
open && view === 'settings'
|
||||
? close('settings')
|
||||
: select('settings', { focus: true });
|
||||
try {
|
||||
listen(ui.voiceSettingsButton, 'click', toggleSettings);
|
||||
tabs.forEach((tab, index) => {
|
||||
listen(tab, 'click', () =>
|
||||
select(index === 0 ? 'preview' : 'settings', { focus: true }),
|
||||
);
|
||||
listen(tab, 'keydown', (event) => {
|
||||
const next =
|
||||
event.key === 'Home'
|
||||
? 0
|
||||
: event.key === 'End'
|
||||
? 1
|
||||
: ['ArrowLeft', 'ArrowRight'].includes(event.key)
|
||||
? 1 - index
|
||||
: null;
|
||||
if (next === null) return;
|
||||
event.preventDefault();
|
||||
select(next === 0 ? 'preview' : 'settings', { focus: true });
|
||||
});
|
||||
});
|
||||
listen(ui.voiceSettingsClose, 'click', () => close());
|
||||
listen(panel, 'keydown', (event) => {
|
||||
if (event.key !== 'Escape' || !open) return;
|
||||
event.preventDefault();
|
||||
close();
|
||||
});
|
||||
// Keep the raised row reachable across pinned trays without intercepting
|
||||
// their controls. The same lifetime owns the listeners and pending timer.
|
||||
let graceTimer;
|
||||
const endGrace = () => {
|
||||
clearTimeout(graceTimer);
|
||||
if (ui.root) delete ui.root.dataset.rowGrace;
|
||||
};
|
||||
listen(ui.root, 'pointerleave', (event) => {
|
||||
if (event.pointerType === 'touch') return;
|
||||
endGrace();
|
||||
ui.root.dataset.rowGrace = 'true';
|
||||
graceTimer = setTimeout(endGrace, ROW_GRACE_MS);
|
||||
});
|
||||
listen(ui.root, 'pointerenter', endGrace);
|
||||
lifetime.signal.addEventListener('abort', endGrace, { once: true });
|
||||
sync();
|
||||
} catch (error) {
|
||||
lifetime.abort();
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
select,
|
||||
close,
|
||||
sync,
|
||||
snapshot,
|
||||
destroy() {
|
||||
if (disposed) return;
|
||||
open = false;
|
||||
sync();
|
||||
disposed = true;
|
||||
lifetime.abort();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createManualVoicePanel } from './manualPanel.js';
|
||||
|
||||
function fixture(t) {
|
||||
const doc = { activeElement: null };
|
||||
const node = () =>
|
||||
Object.assign(new EventTarget(), {
|
||||
hidden: false,
|
||||
isConnected: true,
|
||||
dataset: {},
|
||||
attributes: {},
|
||||
ownerDocument: doc,
|
||||
setAttribute(name, value) {
|
||||
this.attributes[name] = value;
|
||||
},
|
||||
focus() {
|
||||
doc.activeElement = this;
|
||||
},
|
||||
contains(other) {
|
||||
return other === this;
|
||||
},
|
||||
checkVisibility() {
|
||||
return !this.hidden;
|
||||
},
|
||||
});
|
||||
const ui = Object.fromEntries(
|
||||
[
|
||||
'root',
|
||||
'voicePanel',
|
||||
'previewPanel',
|
||||
'voiceSettingsPanel',
|
||||
'previewTab',
|
||||
'settingsTab',
|
||||
'voiceSettingsButton',
|
||||
'voiceSettingsClose',
|
||||
'previewEmpty',
|
||||
'card',
|
||||
'button',
|
||||
].map((key) => [key, node()]),
|
||||
);
|
||||
ui.voicePanel.contains = (element) =>
|
||||
Object.values(ui).includes(element) &&
|
||||
![ui.root, ui.voiceSettingsButton, ui.button].includes(element);
|
||||
let enabled = true,
|
||||
content = false,
|
||||
intents = 0;
|
||||
const card = {
|
||||
setPresented(value) {
|
||||
this.presented = value;
|
||||
ui.card.hidden = !(value && enabled && content);
|
||||
},
|
||||
};
|
||||
const copy = {
|
||||
previewEmpty: 'Habla para ver resultados.',
|
||||
previewDisabled: 'Vista previa desactivada.',
|
||||
};
|
||||
const panel = createManualVoicePanel({
|
||||
ui,
|
||||
card,
|
||||
getPreviewEnabled: () => enabled,
|
||||
copy,
|
||||
onPresentationIntent: () => intents++,
|
||||
});
|
||||
t.after(() => panel.destroy());
|
||||
return {
|
||||
ui,
|
||||
panel,
|
||||
doc,
|
||||
card,
|
||||
copy,
|
||||
intents: () => intents,
|
||||
enabled(value) {
|
||||
enabled = value;
|
||||
panel.sync();
|
||||
},
|
||||
content(value) {
|
||||
content = value;
|
||||
panel.sync();
|
||||
},
|
||||
};
|
||||
}
|
||||
const click = (element) => element.dispatchEvent(new Event('click'));
|
||||
function key(element, value) {
|
||||
const event = new Event('keydown', { cancelable: true });
|
||||
Object.defineProperty(event, 'key', { value });
|
||||
element.dispatchEvent(event);
|
||||
return event;
|
||||
}
|
||||
|
||||
test('Settings launcher toggles while tab and programmatic selection remain idempotent', (t) => {
|
||||
const f = fixture(t);
|
||||
assert.equal(f.panel.snapshot().panelOpen, false);
|
||||
click(f.ui.voiceSettingsButton);
|
||||
assert.equal(f.panel.snapshot().settingsOpen, true);
|
||||
assert.equal(f.doc.activeElement, f.ui.settingsTab);
|
||||
assert.equal(f.ui.voiceSettingsButton.attributes['aria-expanded'], 'true');
|
||||
click(f.ui.voiceSettingsButton);
|
||||
assert.equal(f.panel.snapshot().panelOpen, false);
|
||||
click(f.ui.settingsTab);
|
||||
assert.equal(f.panel.snapshot().settingsOpen, true);
|
||||
assert.equal(f.panel.select('settings').changed, false);
|
||||
assert.equal(f.panel.snapshot().settingsOpen, true);
|
||||
click(f.ui.settingsTab);
|
||||
assert.equal(f.panel.snapshot().settingsOpen, true);
|
||||
assert.equal(f.panel.select('unknown').reason, 'unknown-view');
|
||||
});
|
||||
|
||||
test('tab keyboard navigation and Escape preserve focus and accessibility state', (t) => {
|
||||
const f = fixture(t);
|
||||
click(f.ui.voiceSettingsButton);
|
||||
assert.equal(key(f.ui.settingsTab, 'ArrowLeft').defaultPrevented, true);
|
||||
assert.equal(f.panel.snapshot().view, 'preview');
|
||||
assert.equal(f.doc.activeElement, f.ui.previewTab);
|
||||
assert.equal(f.ui.previewTab.tabIndex, 0);
|
||||
assert.equal(f.ui.settingsTab.tabIndex, -1);
|
||||
assert.equal(f.ui.previewTab.attributes['aria-selected'], 'true');
|
||||
key(f.ui.previewTab, 'End');
|
||||
assert.equal(f.panel.snapshot().view, 'settings');
|
||||
key(f.ui.settingsTab, 'Home');
|
||||
assert.equal(f.panel.snapshot().view, 'preview');
|
||||
assert.equal(key(f.ui.voicePanel, 'Escape').defaultPrevented, true);
|
||||
assert.equal(f.panel.snapshot().panelOpen, false);
|
||||
assert.equal(f.doc.activeElement, f.ui.voiceSettingsButton);
|
||||
assert.equal(key(f.ui.voicePanel, 'Escape').defaultPrevented, false);
|
||||
});
|
||||
|
||||
test('manual switching transfers hidden-panel focus without stealing unrelated focus', (t) => {
|
||||
const f = fixture(t);
|
||||
f.panel.select('settings');
|
||||
f.ui.voiceSettingsPanel.focus();
|
||||
f.panel.select('preview');
|
||||
assert.equal(f.doc.activeElement, f.ui.previewTab);
|
||||
const unrelated = (f.doc.activeElement = {});
|
||||
f.panel.select('settings');
|
||||
assert.equal(f.doc.activeElement, unrelated);
|
||||
assert.equal(f.panel.close('preview').changed, false);
|
||||
assert.equal(f.panel.snapshot().settingsOpen, true);
|
||||
});
|
||||
|
||||
test('manual Preview selection never enables a disabled preference and uses injected copy', (t) => {
|
||||
const f = fixture(t);
|
||||
f.enabled(false);
|
||||
f.content(true);
|
||||
f.panel.select('preview');
|
||||
assert.equal(f.panel.snapshot().previewEnabled, false);
|
||||
assert.equal(f.panel.snapshot().previewVisible, false);
|
||||
assert.equal(f.ui.previewEmpty.textContent, f.copy.previewDisabled);
|
||||
assert.equal(f.ui.previewEmpty.hidden, false);
|
||||
f.enabled(true);
|
||||
assert.equal(f.panel.snapshot().previewVisible, true);
|
||||
assert.equal(f.ui.previewEmpty.hidden, true);
|
||||
f.content(false);
|
||||
assert.equal(f.ui.previewEmpty.textContent, f.copy.previewEmpty);
|
||||
});
|
||||
|
||||
test('disposal hides owned presentation and removes listeners without destroying the card', (t) => {
|
||||
const f = fixture(t);
|
||||
f.panel.select('settings');
|
||||
const count = f.intents();
|
||||
f.panel.destroy();
|
||||
f.panel.destroy();
|
||||
click(f.ui.voiceSettingsButton);
|
||||
click(f.ui.settingsTab);
|
||||
assert.equal(f.intents(), count);
|
||||
assert.equal(f.ui.voicePanel.hidden, true);
|
||||
assert.equal(f.card.presented, false);
|
||||
assert.equal(f.panel.select('preview').reason, 'disposed');
|
||||
assert.equal(f.panel.close().reason, 'disposed');
|
||||
});
|
||||
|
||||
test('pointer grace expires, resets on reentry, ignores touch and clears on disposal', (t) => {
|
||||
t.mock.timers.enable({ apis: ['setTimeout'] });
|
||||
const f = fixture(t);
|
||||
const pointer = (type, pointerType = 'mouse') => {
|
||||
const event = new Event(type);
|
||||
Object.defineProperty(event, 'pointerType', { value: pointerType });
|
||||
f.ui.root.dispatchEvent(event);
|
||||
};
|
||||
pointer('pointerleave');
|
||||
t.mock.timers.tick(699);
|
||||
assert.equal(f.ui.root.dataset.rowGrace, 'true');
|
||||
pointer('pointerenter');
|
||||
assert.equal(f.ui.root.dataset.rowGrace, undefined);
|
||||
pointer('pointerleave');
|
||||
t.mock.timers.tick(699);
|
||||
assert.equal(f.ui.root.dataset.rowGrace, 'true');
|
||||
t.mock.timers.tick(1);
|
||||
assert.equal(f.ui.root.dataset.rowGrace, undefined);
|
||||
pointer('pointerleave', 'touch');
|
||||
assert.equal(f.ui.root.dataset.rowGrace, undefined);
|
||||
pointer('pointerleave');
|
||||
f.panel.destroy();
|
||||
assert.equal(f.ui.root.dataset.rowGrace, undefined);
|
||||
pointer('pointerleave');
|
||||
t.mock.timers.tick(700);
|
||||
assert.equal(f.ui.root.dataset.rowGrace, undefined);
|
||||
});
|
||||
@@ -0,0 +1,292 @@
|
||||
/** Inspected native capabilities only; scheduled/reported output is not acoustics. */
|
||||
export function observeNativeActivity({
|
||||
adapter,
|
||||
nativeProvider,
|
||||
signal,
|
||||
onChange,
|
||||
onInput,
|
||||
}) {
|
||||
const native = nativeProvider === 'openai' || nativeProvider === 'gemini';
|
||||
// Do not even inspect custom controller properties.
|
||||
const controller = native ? adapter.controller : null;
|
||||
const input = native
|
||||
? nativeProvider === 'gemini'
|
||||
? controller.input
|
||||
: controller._input
|
||||
: null;
|
||||
let dead = false,
|
||||
connected = false,
|
||||
inputBusy = false,
|
||||
recoverableError = false;
|
||||
function cleanNativeOutput() {
|
||||
if (nativeProvider === 'gemini') return true;
|
||||
if (nativeProvider !== 'openai') return false;
|
||||
const turns = controller._turns;
|
||||
return (
|
||||
['idle', 'connecting', 'error'].includes(controller.status) &&
|
||||
controller.dc == null &&
|
||||
!turns.assistantAudioPlaying &&
|
||||
!turns.responseActive &&
|
||||
!turns.responseCreatePending &&
|
||||
!turns.userTurnPending &&
|
||||
!turns.activeResponseId &&
|
||||
!turns.progressResponse
|
||||
);
|
||||
}
|
||||
let outputKnown = cleanNativeOutput();
|
||||
let audio = null,
|
||||
version = 0;
|
||||
const restores = [];
|
||||
const audioRestores = [];
|
||||
const pendingOutput = new Map();
|
||||
let untrackedOutput = !outputKnown;
|
||||
function retireAudio() {
|
||||
audio = null;
|
||||
for (const restore of audioRestores.splice(0).reverse()) restore();
|
||||
}
|
||||
function refreshOutputKnowledge() {
|
||||
outputKnown = !untrackedOutput && pendingOutput.size === 0;
|
||||
}
|
||||
function hasNoAudioOutput(response) {
|
||||
return (
|
||||
['completed', 'cancelled', 'failed', 'incomplete'].includes(
|
||||
response?.status,
|
||||
) &&
|
||||
Array.isArray(response.output) &&
|
||||
response.output.every(
|
||||
(item) =>
|
||||
item?.type === 'function_call' ||
|
||||
(item?.type === 'message' &&
|
||||
Array.isArray(item.content) &&
|
||||
item.content.every((part) =>
|
||||
['text', 'output_text', 'refusal'].includes(part?.type),
|
||||
)),
|
||||
)
|
||||
);
|
||||
}
|
||||
function snapshot() {
|
||||
const turns = nativeProvider === 'openai' ? controller?._turns : controller;
|
||||
return {
|
||||
known: native && outputKnown && !dead,
|
||||
connected: connected && !dead,
|
||||
held: Boolean(input?.pushToTalkKeyHeld),
|
||||
inputBusy: Boolean(
|
||||
inputBusy ||
|
||||
(nativeProvider === 'gemini' && controller?.userActivityOpen),
|
||||
),
|
||||
outputBusy: Boolean(
|
||||
nativeProvider === 'gemini'
|
||||
? audio && audio === controller.audio && audio.pending
|
||||
: turns?.assistantAudioPlaying,
|
||||
),
|
||||
generationBusy: Boolean(
|
||||
turns?.responseActive ||
|
||||
turns?.responseCreatePending ||
|
||||
(nativeProvider === 'openai' && turns?.userTurnPending) ||
|
||||
(nativeProvider === 'gemini' &&
|
||||
(turns?.replyPending ||
|
||||
turns?.retiredReplyPending ||
|
||||
turns?.deferredInput ||
|
||||
turns?.automaticTurnOpen ||
|
||||
turns?.pendingTools ||
|
||||
turns?.controlQueue?.length)),
|
||||
),
|
||||
};
|
||||
}
|
||||
function notify() {
|
||||
if (!dead) {
|
||||
try {
|
||||
onChange?.(snapshot());
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
function wrap(object, key, after, ownedRestores = restores) {
|
||||
const original = object?.[key];
|
||||
if (typeof original !== 'function') return;
|
||||
function wrapped(...args) {
|
||||
const stamp = version;
|
||||
const result = Reflect.apply(original, this, args);
|
||||
if (!dead && stamp === version) {
|
||||
try {
|
||||
after?.(args, result);
|
||||
} catch {}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
object[key] = wrapped;
|
||||
ownedRestores.push(() => {
|
||||
if (object[key] === wrapped) object[key] = original;
|
||||
});
|
||||
}
|
||||
function handle(event) {
|
||||
if (dead || !native) return;
|
||||
if (event?.type === 'state') {
|
||||
const state = event.state;
|
||||
const wasConnected = connected;
|
||||
const liveError =
|
||||
state === 'error' &&
|
||||
nativeProvider === 'openai' &&
|
||||
controller.dc?.readyState === 'open';
|
||||
const restartingAfterError = state === 'connecting' && recoverableError;
|
||||
recoverableError = liveError;
|
||||
connected = !['idle', 'connecting', 'error'].includes(state);
|
||||
// Connecting has multiple native phases around audio allocation. Retire
|
||||
// a previous connected generation once, not each phase of this start.
|
||||
// A failed reply can briefly report error on a live OpenAI transport.
|
||||
// Preserve its output evidence through recovery to listening; a later
|
||||
// reconnect still retires the failed connection's generation.
|
||||
if (
|
||||
state === 'idle' ||
|
||||
(state === 'error' && !liveError) ||
|
||||
(state === 'connecting' && (wasConnected || restartingAfterError))
|
||||
) {
|
||||
version++;
|
||||
retireAudio();
|
||||
inputBusy = false;
|
||||
outputKnown = cleanNativeOutput();
|
||||
untrackedOutput = !outputKnown;
|
||||
pendingOutput.clear();
|
||||
}
|
||||
}
|
||||
notify();
|
||||
}
|
||||
if (native) {
|
||||
wrap(controller, 'sendTextCommand', ([text], result) => {
|
||||
if (String(text || '').trim() && result !== false) onInput?.();
|
||||
});
|
||||
for (const key of [
|
||||
'setMicrophoneEnabled',
|
||||
'releasePushToTalkKey',
|
||||
'resetSession',
|
||||
])
|
||||
wrap(input, key, notify);
|
||||
if (nativeProvider === 'openai') {
|
||||
wrap(controller._turns, 'observeSpeech', ([event]) => {
|
||||
if (event?.type === 'input_audio_buffer.speech_started')
|
||||
inputBusy = true;
|
||||
if (event?.type === 'input_audio_buffer.speech_stopped')
|
||||
inputBusy = false;
|
||||
const type = event?.type;
|
||||
const id = event?.response_id || event?.response?.id;
|
||||
// Keep each unresolved response separate: a later silent reply cannot
|
||||
// establish that an earlier audio reply has drained.
|
||||
if (type === 'response.created') {
|
||||
if (id && pendingOutput.size < 64)
|
||||
pendingOutput.set(id, { audioSeen: false });
|
||||
else untrackedOutput = true;
|
||||
}
|
||||
const pending = pendingOutput.get(id);
|
||||
if (
|
||||
pending &&
|
||||
(type?.startsWith('response.output_audio') ||
|
||||
type?.startsWith('response.audio') ||
|
||||
['audio', 'output_audio'].includes(event?.part?.type))
|
||||
)
|
||||
pending.audioSeen = true;
|
||||
if (
|
||||
type === 'response.done' &&
|
||||
pending &&
|
||||
!pending.audioSeen &&
|
||||
hasNoAudioOutput(event.response)
|
||||
)
|
||||
pendingOutput.delete(id);
|
||||
if (
|
||||
type === 'output_audio_buffer.started' ||
|
||||
type === 'output_audio_buffer.cleared'
|
||||
) {
|
||||
// Native playback events establish the transport's output lifecycle.
|
||||
// Generation completion alone never establishes audio drainage.
|
||||
untrackedOutput = false;
|
||||
if (id) pendingOutput.delete(id);
|
||||
else pendingOutput.clear();
|
||||
}
|
||||
refreshOutputKnowledge();
|
||||
notify();
|
||||
});
|
||||
} else {
|
||||
wrap(controller, 'handleMessage', notify);
|
||||
const original = controller.createAudio;
|
||||
function createAudio(...args) {
|
||||
const stamp = version;
|
||||
const options = args[0];
|
||||
let current;
|
||||
const decorated = { ...options };
|
||||
for (const key of [
|
||||
'onAudioStart',
|
||||
'onAudioEnd',
|
||||
'onAudioDiscard',
|
||||
'onDrain',
|
||||
]) {
|
||||
const callback = options?.[key];
|
||||
decorated[key] = function (...values) {
|
||||
const result =
|
||||
typeof callback === 'function'
|
||||
? Reflect.apply(
|
||||
callback,
|
||||
this === decorated ? options : this,
|
||||
values,
|
||||
)
|
||||
: undefined;
|
||||
if (
|
||||
!dead &&
|
||||
stamp === version &&
|
||||
audio === current &&
|
||||
controller.audio === current
|
||||
) {
|
||||
if (
|
||||
key === 'onAudioStart' &&
|
||||
options.pushToTalk &&
|
||||
controller.captureOwner?.admitted
|
||||
)
|
||||
inputBusy = true;
|
||||
if (key === 'onAudioEnd' || key === 'onAudioDiscard')
|
||||
inputBusy = false;
|
||||
notify();
|
||||
}
|
||||
return result;
|
||||
};
|
||||
}
|
||||
const forwarded = args.slice();
|
||||
forwarded[0] = decorated;
|
||||
current = Reflect.apply(original, this, forwarded);
|
||||
if (!dead && stamp === version) {
|
||||
retireAudio();
|
||||
audio = current;
|
||||
inputBusy = false;
|
||||
for (const key of ['play', 'clearPlayback', 'close'])
|
||||
wrap(
|
||||
current,
|
||||
key,
|
||||
() => {
|
||||
if (audio !== current) return;
|
||||
if (key === 'close') {
|
||||
retireAudio();
|
||||
inputBusy = false;
|
||||
}
|
||||
if (controller.audio === current) notify();
|
||||
},
|
||||
audioRestores,
|
||||
);
|
||||
}
|
||||
return current;
|
||||
}
|
||||
controller.createAudio = createAudio;
|
||||
restores.push(() => {
|
||||
if (controller.createAudio === createAudio)
|
||||
controller.createAudio = original;
|
||||
});
|
||||
}
|
||||
}
|
||||
function destroy() {
|
||||
if (dead) return;
|
||||
dead = true;
|
||||
version++;
|
||||
retireAudio();
|
||||
pendingOutput.clear();
|
||||
for (const restore of restores.splice(0).reverse()) restore();
|
||||
signal?.removeEventListener('abort', destroy);
|
||||
}
|
||||
signal?.addEventListener('abort', destroy, { once: true });
|
||||
if (signal?.aborted) destroy();
|
||||
return { snapshot, handle, destroy };
|
||||
}
|
||||
@@ -0,0 +1,770 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createVoiceInactivity } from './inactivity.js';
|
||||
import { RealtimeTurns } from '../realtimeTurns.js';
|
||||
import { observeNativeActivity } from './nativeActivity.js';
|
||||
// Deliberately small native-shaped fixtures exercise wrapper contracts, not devices.
|
||||
function gemini(onChange = () => {}) {
|
||||
const callbacks = [];
|
||||
const input = {
|
||||
pushToTalkKeyHeld: false,
|
||||
setMicrophoneEnabled() {},
|
||||
releasePushToTalkKey() {
|
||||
this.pushToTalkKeyHeld = false;
|
||||
},
|
||||
resetSession() {
|
||||
this.pushToTalkKeyHeld = false;
|
||||
},
|
||||
};
|
||||
const c = {
|
||||
input,
|
||||
captureOwner: { admitted: false },
|
||||
createAudio(opts) {
|
||||
callbacks.push(opts);
|
||||
return {
|
||||
pending: 0,
|
||||
play() {
|
||||
this.pending++;
|
||||
return 42;
|
||||
},
|
||||
clearPlayback() {
|
||||
this.pending = 0;
|
||||
},
|
||||
close() {
|
||||
this.pending = 0;
|
||||
},
|
||||
};
|
||||
},
|
||||
handleMessage() {},
|
||||
};
|
||||
const observer = observeNativeActivity({
|
||||
adapter: { controller: c },
|
||||
nativeProvider: 'gemini',
|
||||
onChange,
|
||||
});
|
||||
const start = () => {
|
||||
observer.handle({ type: 'state', state: 'connecting' });
|
||||
c.audio = c.createAudio({ pushToTalk: true });
|
||||
observer.handle({ type: 'state', state: 'listening' });
|
||||
return c.audio;
|
||||
};
|
||||
return { c, input, observer, callbacks, start };
|
||||
}
|
||||
test('custom host never inspected, including hostile controller getter', () => {
|
||||
const adapter = {
|
||||
get controller() {
|
||||
throw Error('must not inspect');
|
||||
},
|
||||
};
|
||||
const o = observeNativeActivity({ adapter, nativeProvider: null });
|
||||
assert.equal(o.snapshot().known, false);
|
||||
o.handle({ type: 'state', state: 'listening' });
|
||||
o.destroy();
|
||||
});
|
||||
test('PTT hold is direct before connect; silent automatic stream is not admitted speech', () => {
|
||||
const f = gemini();
|
||||
f.input.pushToTalkKeyHeld = true;
|
||||
assert.equal(f.observer.snapshot().held, true);
|
||||
assert.equal(f.observer.snapshot().connected, false);
|
||||
f.start();
|
||||
f.callbacks[0].onAudioStart({});
|
||||
assert.equal(f.observer.snapshot().inputBusy, false);
|
||||
f.c.captureOwner.admitted = true;
|
||||
f.callbacks[0].onAudioStart({});
|
||||
assert.equal(f.observer.snapshot().inputBusy, true);
|
||||
f.callbacks[0].onAudioDiscard();
|
||||
f.input.releasePushToTalkKey();
|
||||
assert.equal(f.observer.snapshot().inputBusy, false);
|
||||
assert.equal(f.observer.snapshot().held, false);
|
||||
});
|
||||
test('queue-to-drain, clear and close use current instance, without an acoustic claim', () => {
|
||||
const f = gemini();
|
||||
const a = f.start();
|
||||
assert.equal(a.play(), 42);
|
||||
assert.equal(f.observer.snapshot().outputBusy, true);
|
||||
a.clearPlayback();
|
||||
assert.equal(f.observer.snapshot().outputBusy, false);
|
||||
a.play();
|
||||
a.close();
|
||||
assert.equal(f.observer.snapshot().outputBusy, false);
|
||||
a.play();
|
||||
a.pending = 0;
|
||||
f.callbacks[0].onDrain();
|
||||
assert.equal(f.observer.snapshot().outputBusy, false);
|
||||
});
|
||||
test('same-mount restart old callbacks cannot clear new admitted capture/output', () => {
|
||||
const f = gemini();
|
||||
f.start();
|
||||
const old = f.callbacks[0];
|
||||
const b = f.start();
|
||||
f.c.captureOwner.admitted = true;
|
||||
f.callbacks[1].onAudioStart();
|
||||
b.play();
|
||||
old.onAudioDiscard();
|
||||
old.onDrain();
|
||||
assert.equal(f.observer.snapshot().inputBusy, true);
|
||||
assert.equal(f.observer.snapshot().outputBusy, true);
|
||||
});
|
||||
test('native callbacks original first with receiver, arguments, return and thrown identity; observation failure isolated', () => {
|
||||
const f = gemini(() => {
|
||||
throw Error('observer');
|
||||
});
|
||||
const receiver = {};
|
||||
const argument = {};
|
||||
let seen;
|
||||
f.c.audio = f.c.createAudio({
|
||||
pushToTalk: true,
|
||||
onAudioEnd: function (...args) {
|
||||
seen = [this, args];
|
||||
return argument;
|
||||
},
|
||||
});
|
||||
const cb = f.callbacks[0];
|
||||
assert.equal(cb.onAudioEnd.call(receiver, argument), argument);
|
||||
assert.equal(seen[0], receiver);
|
||||
assert.equal(seen[1][0], argument);
|
||||
const error = Error('same');
|
||||
f.c.audio = f.c.createAudio({
|
||||
onDrain() {
|
||||
throw error;
|
||||
},
|
||||
});
|
||||
assert.throws(
|
||||
() => f.callbacks[1].onDrain(),
|
||||
(e) => e === error,
|
||||
);
|
||||
});
|
||||
test('reentrant Stop in original callback suppresses stale post-callback observation; teardown preserves newer replacement', () => {
|
||||
let count = 0;
|
||||
const f = gemini(() => count++);
|
||||
f.c.audio = f.c.createAudio({
|
||||
onDrain() {
|
||||
f.observer.handle({ type: 'state', state: 'idle' });
|
||||
},
|
||||
});
|
||||
const before = count;
|
||||
f.callbacks[0].onDrain();
|
||||
assert.equal(count, before + 1);
|
||||
assert.equal(f.observer.snapshot().connected, false);
|
||||
const newer = () => {};
|
||||
f.input.releasePushToTalkKey = newer;
|
||||
f.observer.destroy();
|
||||
assert.equal(f.input.releasePushToTalkKey, newer);
|
||||
});
|
||||
test('OpenAI response.done does not drain output; absent lifecycle remains unknown', () => {
|
||||
const turns = {
|
||||
assistantAudioPlaying: false,
|
||||
responseActive: false,
|
||||
observeSpeech(e) {
|
||||
if (e.type === 'output_audio_buffer.started')
|
||||
this.assistantAudioPlaying = true;
|
||||
if (
|
||||
e.type === 'output_audio_buffer.stopped' ||
|
||||
e.type === 'output_audio_buffer.cleared'
|
||||
)
|
||||
this.assistantAudioPlaying = false;
|
||||
},
|
||||
};
|
||||
const c = { _turns: turns, _input: { pushToTalkKeyHeld: false } };
|
||||
const original = turns.observeSpeech;
|
||||
const o = observeNativeActivity({
|
||||
adapter: { controller: c },
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
o.handle({ type: 'state', state: 'listening' });
|
||||
assert.equal(o.snapshot().known, false);
|
||||
turns.observeSpeech({ type: 'output_audio_buffer.started' });
|
||||
assert.equal(o.snapshot().known, true);
|
||||
turns.responseActive = false;
|
||||
o.handle({ type: 'completion' });
|
||||
assert.equal(o.snapshot().outputBusy, true);
|
||||
turns.observeSpeech({ type: 'output_audio_buffer.cleared' });
|
||||
assert.equal(o.snapshot().outputBusy, false);
|
||||
o.destroy();
|
||||
assert.equal(turns.observeSpeech, original);
|
||||
});
|
||||
test('method receiver/args/result and throw are exact; abort restores wrappers', () => {
|
||||
const abort = new AbortController();
|
||||
const result = Promise.resolve();
|
||||
let seen;
|
||||
const error = Error('original');
|
||||
const input = {
|
||||
setMicrophoneEnabled(...args) {
|
||||
seen = [this, args];
|
||||
if (args[0] === 'throw') throw error;
|
||||
return result;
|
||||
},
|
||||
};
|
||||
const original = input.setMicrophoneEnabled;
|
||||
const c = {
|
||||
input,
|
||||
createAudio() {
|
||||
return {};
|
||||
},
|
||||
handleMessage() {},
|
||||
};
|
||||
const o = observeNativeActivity({
|
||||
adapter: { controller: c },
|
||||
nativeProvider: 'gemini',
|
||||
signal: abort.signal,
|
||||
onChange() {
|
||||
throw Error('observe');
|
||||
},
|
||||
});
|
||||
assert.equal(input.setMicrophoneEnabled(1, 2), result);
|
||||
assert.equal(seen[0], input);
|
||||
assert.deepEqual(seen[1], [1, 2]);
|
||||
assert.throws(
|
||||
() => input.setMicrophoneEnabled('throw'),
|
||||
(e) => e === error,
|
||||
);
|
||||
abort.abort();
|
||||
assert.equal(input.setMicrophoneEnabled, original);
|
||||
assert.equal(o.snapshot().known, false);
|
||||
});
|
||||
|
||||
test('options-method callback receiver maps to original options, explicit receiver is preserved', () => {
|
||||
const f = gemini();
|
||||
let receiver;
|
||||
const options = {
|
||||
onDrain() {
|
||||
receiver = this;
|
||||
return 17;
|
||||
},
|
||||
};
|
||||
f.c.audio = f.c.createAudio(options);
|
||||
assert.equal(f.callbacks[0].onDrain(), 17);
|
||||
assert.equal(receiver, options);
|
||||
const explicit = {};
|
||||
f.callbacks[0].onDrain.call(explicit);
|
||||
assert.equal(receiver, explicit);
|
||||
});
|
||||
test('actual native RealtimeTurns output lifecycle remains busy beyond generation', () => {
|
||||
const turns = new RealtimeTurns({});
|
||||
const observer = observeNativeActivity({
|
||||
adapter: { controller: { _turns: turns, _input: {} } },
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
observer.handle({ type: 'state', state: 'listening' });
|
||||
turns.observeSpeech({ type: 'output_audio_buffer.started' });
|
||||
turns.responseActive = false;
|
||||
observer.handle({ type: 'completion' });
|
||||
assert.equal(observer.snapshot().outputBusy, true);
|
||||
turns.observeSpeech({ type: 'output_audio_buffer.stopped' });
|
||||
assert.equal(observer.snapshot().outputBusy, false);
|
||||
turns.observeSpeech({ type: 'response.created' });
|
||||
turns.observeSpeech({ type: 'output_audio_buffer.stopped' });
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
observer.destroy();
|
||||
});
|
||||
|
||||
test('verified clean native mount can arm initial idle, eventless response invalidates it', () => {
|
||||
const turns = new RealtimeTurns({});
|
||||
const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
|
||||
const observer = observeNativeActivity({
|
||||
adapter: { controller },
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
let armed = 0,
|
||||
cleared = 0;
|
||||
const idle = createVoiceInactivity({
|
||||
readActivity: observer.snapshot,
|
||||
readPending: () => 0,
|
||||
setTimer() {
|
||||
armed++;
|
||||
return 1;
|
||||
},
|
||||
clearTimer() {
|
||||
cleared++;
|
||||
},
|
||||
});
|
||||
assert.equal(observer.snapshot().known, true);
|
||||
assert.equal(armed, 0);
|
||||
controller.status = 'listening';
|
||||
controller.dc = { readyState: 'open' };
|
||||
observer.handle({ type: 'state', state: 'listening' });
|
||||
idle.update();
|
||||
assert.equal(armed, 1);
|
||||
turns.observeSpeech({ type: 'response.created' });
|
||||
idle.update();
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
assert.equal(cleared, 1);
|
||||
turns.responseActive = false;
|
||||
observer.handle({ type: 'completion' });
|
||||
idle.update();
|
||||
assert.equal(armed, 1);
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
controller.status = 'idle';
|
||||
controller.dc = null;
|
||||
turns.reset();
|
||||
observer.handle({ type: 'state', state: 'idle' });
|
||||
assert.equal(observer.snapshot().known, true);
|
||||
idle.destroy();
|
||||
observer.destroy();
|
||||
});
|
||||
test('attaching to already active native transport never assumes empty playback', () => {
|
||||
const turns = new RealtimeTurns({});
|
||||
for (const controller of [
|
||||
{ status: 'listening', dc: {}, _turns: turns, _input: {} },
|
||||
{ status: 'idle', dc: {}, _turns: turns, _input: {} },
|
||||
]) {
|
||||
const observer = observeNativeActivity({
|
||||
adapter: { controller },
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
observer.destroy();
|
||||
}
|
||||
turns.responseActive = true;
|
||||
const observer = observeNativeActivity({
|
||||
adapter: {
|
||||
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
|
||||
},
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
observer.destroy();
|
||||
});
|
||||
|
||||
test('Gemini queue-only work blocks idle, queue drain rearms, expiry rereads delayed queue', () => {
|
||||
const f = gemini();
|
||||
f.start();
|
||||
f.c.controlQueue = [];
|
||||
let callback,
|
||||
arms = 0,
|
||||
clears = 0,
|
||||
expired = 0;
|
||||
const idle = createVoiceInactivity({
|
||||
readActivity: f.observer.snapshot,
|
||||
readPending: () => 0,
|
||||
onExpire() {
|
||||
expired++;
|
||||
},
|
||||
setTimer(fn) {
|
||||
callback = fn;
|
||||
return ++arms;
|
||||
},
|
||||
clearTimer() {
|
||||
clears++;
|
||||
},
|
||||
});
|
||||
assert.equal(arms, 1);
|
||||
f.c.controlQueue.push({});
|
||||
assert.equal(f.observer.snapshot().generationBusy, true);
|
||||
idle.update();
|
||||
assert.equal(clears, 1);
|
||||
f.c.controlQueue.shift();
|
||||
assert.equal(f.observer.snapshot().generationBusy, false);
|
||||
idle.update();
|
||||
assert.equal(arms, 2);
|
||||
// A queue may change after the last event-driven snapshot and before the deadline.
|
||||
f.c.controlQueue.push({});
|
||||
callback();
|
||||
assert.equal(expired, 0);
|
||||
f.c.controlQueue = [];
|
||||
idle.update();
|
||||
assert.equal(arms, 3);
|
||||
callback();
|
||||
assert.equal(expired, 1);
|
||||
idle.destroy();
|
||||
f.observer.destroy();
|
||||
});
|
||||
|
||||
test('Gemini presentation-only pending does not block idle; retired/deferred work still does', () => {
|
||||
const f = gemini();
|
||||
f.start();
|
||||
f.c.userTurnPending = true;
|
||||
assert.equal(f.observer.snapshot().generationBusy, false);
|
||||
f.c.retiredReplyPending = true;
|
||||
assert.equal(f.observer.snapshot().generationBusy, true);
|
||||
f.c.retiredReplyPending = false;
|
||||
f.c.deferredInput = {};
|
||||
assert.equal(f.observer.snapshot().generationBusy, true);
|
||||
f.c.deferredInput = null;
|
||||
assert.equal(f.observer.snapshot().generationBusy, false);
|
||||
f.observer.destroy();
|
||||
});
|
||||
|
||||
test('actual Gemini startup phases keep playback observation and permit quiet open-mic expiry', async () => {
|
||||
const { harness } = await import('../../testSupport/geminiNativeHarness.mjs');
|
||||
let observer,
|
||||
idle,
|
||||
deadline,
|
||||
expiries = 0;
|
||||
const h = harness({ emit: (event) => observer?.handle(event) });
|
||||
observer = observeNativeActivity({
|
||||
adapter: { controller: h.c },
|
||||
nativeProvider: 'gemini',
|
||||
onChange: () => idle?.update(),
|
||||
});
|
||||
idle = createVoiceInactivity({
|
||||
readActivity: observer.snapshot,
|
||||
readPending: () => 0,
|
||||
setTimer(fn) {
|
||||
deadline = fn;
|
||||
return fn;
|
||||
},
|
||||
clearTimer() {},
|
||||
onExpire() {
|
||||
expiries++;
|
||||
},
|
||||
});
|
||||
try {
|
||||
await h.c.start();
|
||||
assert.equal(h.c.userTurnPending, true);
|
||||
assert.equal(idle.snapshot().eligible, true);
|
||||
const beforeOutput = deadline;
|
||||
h.sockets[0].message({
|
||||
serverContent: {
|
||||
modelTurn: {
|
||||
parts: [
|
||||
{
|
||||
inlineData: { data: 'output', mimeType: 'audio/pcm;rate=24000' },
|
||||
},
|
||||
],
|
||||
},
|
||||
turnComplete: true,
|
||||
},
|
||||
});
|
||||
assert.equal(observer.snapshot().outputBusy, true);
|
||||
assert.equal(idle.snapshot().eligible, false);
|
||||
beforeOutput();
|
||||
assert.equal(expiries, 0);
|
||||
h.captures[0].drain();
|
||||
assert.equal(observer.snapshot().outputBusy, false);
|
||||
assert.equal(idle.snapshot().eligible, true);
|
||||
deadline();
|
||||
assert.equal(expiries, 1);
|
||||
} finally {
|
||||
idle.destroy();
|
||||
observer.destroy();
|
||||
h.c.stop({ removeUi: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('native typed input observation preserves receiver/result, refuses failed submissions and retires old wrapper', () => {
|
||||
let inputs = 0;
|
||||
const controller = {
|
||||
input: {},
|
||||
handleMessage() {},
|
||||
createAudio() {},
|
||||
sendTextCommand(text) {
|
||||
assert.equal(this, controller);
|
||||
if (text === 'throw') throw new Error('rejected');
|
||||
return text === 'reject' ? false : undefined;
|
||||
},
|
||||
};
|
||||
const original = controller.sendTextCommand;
|
||||
const observer = observeNativeActivity({
|
||||
adapter: { controller },
|
||||
nativeProvider: 'gemini',
|
||||
onInput: () => inputs++,
|
||||
});
|
||||
const old = controller.sendTextCommand;
|
||||
assert.equal(controller.sendTextCommand('typed request'), undefined);
|
||||
controller.sendTextCommand('reject');
|
||||
controller.sendTextCommand('');
|
||||
assert.throws(() => controller.sendTextCommand('throw'), /rejected/);
|
||||
assert.equal(inputs, 1);
|
||||
observer.destroy();
|
||||
assert.equal(controller.sendTextCommand, original);
|
||||
old.call(controller, 'late');
|
||||
assert.equal(inputs, 1);
|
||||
});
|
||||
|
||||
test('explicit silent OpenAI completion rearms expiry, including cancelled and failed responses', () => {
|
||||
for (const status of ['completed', 'cancelled', 'failed', 'incomplete']) {
|
||||
const turns = new RealtimeTurns({});
|
||||
const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
|
||||
let idle;
|
||||
let timer;
|
||||
let expired = 0;
|
||||
const observer = observeNativeActivity({
|
||||
adapter: { controller },
|
||||
nativeProvider: 'openai',
|
||||
onChange: () => idle?.update(),
|
||||
});
|
||||
idle = createVoiceInactivity({
|
||||
readActivity: observer.snapshot,
|
||||
readPending: () => false,
|
||||
setTimer(callback) {
|
||||
timer = callback;
|
||||
return 1;
|
||||
},
|
||||
clearTimer() {
|
||||
timer = null;
|
||||
},
|
||||
onExpire: () => expired++,
|
||||
});
|
||||
controller.status = 'listening';
|
||||
controller.dc = {};
|
||||
observer.handle({ type: 'state', state: 'listening' });
|
||||
assert.equal(typeof timer, 'function');
|
||||
turns.responseActive = true;
|
||||
turns.observeSpeech({
|
||||
type: 'response.created',
|
||||
response: { id: 'silent' },
|
||||
});
|
||||
assert.equal(timer, null);
|
||||
turns.observeSpeech({
|
||||
type: 'response.done',
|
||||
response: {
|
||||
id: 'silent',
|
||||
status,
|
||||
output:
|
||||
status === 'completed'
|
||||
? [{ type: 'function_call', name: 'set_panel_open' }]
|
||||
: [],
|
||||
},
|
||||
});
|
||||
// The provider updates generation state after forwarding the raw event.
|
||||
assert.equal(timer, null);
|
||||
turns.responseActive = false;
|
||||
observer.handle({ type: 'completion' });
|
||||
assert.equal(observer.snapshot().known, true);
|
||||
assert.equal(typeof timer, 'function');
|
||||
timer();
|
||||
assert.equal(expired, 1);
|
||||
idle.destroy();
|
||||
observer.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('silent OpenAI replies cannot clear unresolved older audio or incomplete output metadata', () => {
|
||||
const turns = new RealtimeTurns({});
|
||||
const observer = observeNativeActivity({
|
||||
adapter: {
|
||||
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
|
||||
},
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
const created = (id) =>
|
||||
turns.observeSpeech({ type: 'response.created', response: { id } });
|
||||
const done = (id, output) =>
|
||||
turns.observeSpeech({
|
||||
type: 'response.done',
|
||||
response: { id, status: 'completed', output },
|
||||
});
|
||||
created('audio');
|
||||
turns.observeSpeech({
|
||||
type: 'response.output_audio.delta',
|
||||
response_id: 'audio',
|
||||
});
|
||||
created('silent');
|
||||
done('silent', []);
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
done('audio', []); // Even contradictory completion metadata cannot erase audio evidence.
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
turns.observeSpeech({
|
||||
type: 'output_audio_buffer.started',
|
||||
response_id: 'audio',
|
||||
});
|
||||
assert.equal(observer.snapshot().outputBusy, true);
|
||||
done('silent', []);
|
||||
assert.equal(observer.snapshot().outputBusy, true);
|
||||
turns.observeSpeech({
|
||||
type: 'output_audio_buffer.stopped',
|
||||
response_id: 'audio',
|
||||
});
|
||||
assert.equal(observer.snapshot().outputBusy, false);
|
||||
created('unknown');
|
||||
done('unknown', undefined);
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
done('unobserved', []);
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
observer.destroy();
|
||||
});
|
||||
|
||||
test('overlapping silent replies resolve independently and audio output items remain unknown until playback', () => {
|
||||
const turns = new RealtimeTurns({});
|
||||
const observer = observeNativeActivity({
|
||||
adapter: {
|
||||
controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
|
||||
},
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
for (const id of ['a', 'b'])
|
||||
turns.observeSpeech({ type: 'response.created', response: { id } });
|
||||
for (const id of ['a', 'b']) {
|
||||
turns.observeSpeech({
|
||||
type: 'response.done',
|
||||
response: {
|
||||
id,
|
||||
status: 'completed',
|
||||
output: [
|
||||
{ type: 'message', content: [{ type: 'text', text: 'Done' }] },
|
||||
],
|
||||
},
|
||||
});
|
||||
assert.equal(observer.snapshot().known, id === 'b');
|
||||
}
|
||||
turns.observeSpeech({ type: 'response.created', response: { id: 'audio' } });
|
||||
turns.observeSpeech({
|
||||
type: 'response.done',
|
||||
response: {
|
||||
id: 'audio',
|
||||
status: 'completed',
|
||||
output: [{ type: 'message', content: [{ type: 'audio' }] }],
|
||||
},
|
||||
});
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
observer.destroy();
|
||||
});
|
||||
|
||||
test('silent completion cannot establish output knowledge when observer attached mid-session', () => {
|
||||
const turns = new RealtimeTurns({});
|
||||
const observer = observeNativeActivity({
|
||||
adapter: {
|
||||
controller: { status: 'listening', dc: {}, _turns: turns, _input: {} },
|
||||
},
|
||||
nativeProvider: 'openai',
|
||||
});
|
||||
turns.observeSpeech({ type: 'response.created', response: { id: 'new' } });
|
||||
turns.observeSpeech({
|
||||
type: 'response.done',
|
||||
response: { id: 'new', status: 'completed', output: [] },
|
||||
});
|
||||
assert.equal(observer.snapshot().known, false);
|
||||
observer.destroy();
|
||||
});
|
||||
|
||||
test('Gemini restores each retired audio immediately across repeated same-mount restarts', () => {
|
||||
const originals = new Map();
|
||||
const controller = {
|
||||
input: {},
|
||||
createAudio() {
|
||||
const audio = { pending: 0, play() {}, clearPlayback() {}, close() {} };
|
||||
originals.set(audio, { ...audio });
|
||||
return audio;
|
||||
},
|
||||
handleMessage() {},
|
||||
};
|
||||
const observer = observeNativeActivity({
|
||||
adapter: { controller },
|
||||
nativeProvider: 'gemini',
|
||||
});
|
||||
const assertRestored = (audio) => {
|
||||
for (const key of ['play', 'clearPlayback', 'close'])
|
||||
assert.equal(audio[key], originals.get(audio)[key]);
|
||||
};
|
||||
for (let i = 0; i < 5; i++) {
|
||||
observer.handle({ type: 'state', state: 'connecting' });
|
||||
controller.audio = controller.createAudio({});
|
||||
const audio = controller.audio;
|
||||
observer.handle({ type: 'state', state: 'listening' });
|
||||
assert.notEqual(audio.close, originals.get(audio).close);
|
||||
audio.close();
|
||||
assertRestored(audio);
|
||||
controller.audio = null;
|
||||
observer.handle({ type: 'state', state: 'idle' });
|
||||
}
|
||||
controller.audio = controller.createAudio({});
|
||||
const replaced = controller.audio;
|
||||
controller.audio = controller.createAudio({});
|
||||
assertRestored(replaced);
|
||||
const current = controller.audio;
|
||||
const replacement = () => {};
|
||||
current.play = replacement;
|
||||
observer.handle({ type: 'state', state: 'error' });
|
||||
assert.equal(current.play, replacement);
|
||||
assert.equal(current.close, originals.get(current).close);
|
||||
observer.destroy();
|
||||
});
|
||||
|
||||
test('actual OpenAI failed-response error/listening recovery preserves only proven no-audio knowledge', async (t) => {
|
||||
const { GevRealtimeController } = await import('../realtimeController.js');
|
||||
t.mock.method(console, 'error', () => {});
|
||||
for (const [output, known] of [
|
||||
[[], true],
|
||||
[[{ type: 'message', content: [{ type: 'audio' }] }], false],
|
||||
[undefined, false],
|
||||
]) {
|
||||
let observer, idle, timer;
|
||||
const states = [];
|
||||
let expired = 0;
|
||||
const controller = new GevRealtimeController({
|
||||
runner: async () => ({ ok: true }),
|
||||
backend: {},
|
||||
debugSink: null,
|
||||
ui: {
|
||||
root: {
|
||||
dataset: {},
|
||||
classList: { remove() {} },
|
||||
querySelectorAll: () => [],
|
||||
},
|
||||
status: {},
|
||||
detail: {},
|
||||
},
|
||||
onSessionEvent(event) {
|
||||
if (event.type === 'state') states.push(event.state);
|
||||
observer?.handle(event);
|
||||
},
|
||||
});
|
||||
observer = observeNativeActivity({
|
||||
adapter: { controller },
|
||||
nativeProvider: 'openai',
|
||||
onChange: () => idle?.update(),
|
||||
});
|
||||
idle = createVoiceInactivity({
|
||||
readActivity: observer.snapshot,
|
||||
readPending: () => false,
|
||||
setTimer(callback) {
|
||||
timer = callback;
|
||||
return 1;
|
||||
},
|
||||
clearTimer() {
|
||||
timer = null;
|
||||
},
|
||||
onExpire() {
|
||||
expired++;
|
||||
},
|
||||
});
|
||||
try {
|
||||
controller.dc = {
|
||||
readyState: 'open',
|
||||
send() {},
|
||||
close() {
|
||||
this.readyState = 'closed';
|
||||
},
|
||||
};
|
||||
controller.setStatus('listening', 'Ready');
|
||||
const event = (payload) =>
|
||||
controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
|
||||
await event({
|
||||
type: 'response.created',
|
||||
response: { id: 'failed-reply' },
|
||||
});
|
||||
await event({
|
||||
type: 'response.done',
|
||||
response: {
|
||||
id: 'failed-reply',
|
||||
status: 'failed',
|
||||
output,
|
||||
status_details: { error: { message: 'Synthetic failed reply' } },
|
||||
},
|
||||
});
|
||||
assert.deepEqual(states.slice(-2), ['error', 'listening']);
|
||||
assert.equal(controller.dc.readyState, 'open');
|
||||
assert.equal(observer.snapshot().known, known);
|
||||
assert.equal(idle.snapshot().eligible, known);
|
||||
if (known) {
|
||||
timer();
|
||||
assert.equal(expired, 1);
|
||||
} else assert.equal(timer, null);
|
||||
// Fatal teardown can preserve the error status; the next connection must
|
||||
// still retire the old generation rather than inherit unresolved output.
|
||||
controller.setStatus('error', 'Transport failed');
|
||||
controller.stop({ preserveStatus: true });
|
||||
controller.setStatus('connecting', 'Reconnecting');
|
||||
assert.equal(observer.snapshot().connected, false);
|
||||
assert.equal(observer.snapshot().known, true);
|
||||
controller.stop();
|
||||
assert.equal(observer.snapshot().connected, false);
|
||||
assert.equal(observer.snapshot().known, true);
|
||||
} finally {
|
||||
idle.destroy();
|
||||
observer.destroy();
|
||||
controller.stop();
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,194 @@
|
||||
export const VOICE_PREVIEW_STORAGE_KEY = 'godsEyeView.voice.previewEnabled';
|
||||
export const VOICE_TIMEOUT_STORAGE_KEY = 'godsEyeView.voice.inactivityMinutes';
|
||||
const PRESETS = ['3', '5', '10', '15'];
|
||||
|
||||
function parseMinutes(value) {
|
||||
if (typeof value !== 'number' && typeof value !== 'string') return undefined;
|
||||
if (typeof value === 'string' && !/^\d+$/.test(value)) return undefined;
|
||||
const number = Number(value);
|
||||
return Number.isInteger(number) && number >= 0 && number <= 60
|
||||
? number
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Outer, browser-local live state. Construction never writes storage. */
|
||||
export function createVoicePreferences({ storage } = {}) {
|
||||
const listeners = new Set();
|
||||
const target = () =>
|
||||
storage === undefined ? globalThis.localStorage : storage;
|
||||
const read = (key) => {
|
||||
try {
|
||||
return target()?.getItem?.(key);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
const rawMinutes = read(VOICE_TIMEOUT_STORAGE_KEY);
|
||||
let state = {
|
||||
previewEnabled: read(VOICE_PREVIEW_STORAGE_KEY) !== 'false',
|
||||
minutes: rawMinutes === 'none' ? 0 : (parseMinutes(rawMinutes) ?? 5),
|
||||
previewPersisted: null,
|
||||
timeoutPersisted: null,
|
||||
};
|
||||
const snapshot = () => ({ ...state });
|
||||
function apply(field, status, key, value, serialized) {
|
||||
const changed = state[field] !== value;
|
||||
let persisted = false;
|
||||
state = { ...state, [field]: value };
|
||||
try {
|
||||
const destination = target();
|
||||
if (
|
||||
typeof destination?.setItem === 'function' &&
|
||||
typeof destination?.getItem === 'function'
|
||||
) {
|
||||
destination.setItem(key, serialized);
|
||||
persisted = destination.getItem(key) === serialized;
|
||||
}
|
||||
} catch {
|
||||
/* The applied live value remains authoritative across mounts. */
|
||||
}
|
||||
state = { ...state, [status]: persisted };
|
||||
const result = {
|
||||
ok: true,
|
||||
applied: true,
|
||||
changed,
|
||||
persisted,
|
||||
...snapshot(),
|
||||
};
|
||||
for (const listener of [...listeners]) listener(snapshot());
|
||||
return result;
|
||||
}
|
||||
const invalid = () => ({
|
||||
ok: false,
|
||||
applied: false,
|
||||
changed: false,
|
||||
persisted: null,
|
||||
reason: 'invalid-value',
|
||||
...snapshot(),
|
||||
});
|
||||
return {
|
||||
snapshot,
|
||||
setPreviewEnabled(value) {
|
||||
return typeof value === 'boolean'
|
||||
? apply(
|
||||
'previewEnabled',
|
||||
'previewPersisted',
|
||||
VOICE_PREVIEW_STORAGE_KEY,
|
||||
value,
|
||||
String(value),
|
||||
)
|
||||
: invalid();
|
||||
},
|
||||
setMinutes(value) {
|
||||
const parsed = parseMinutes(value);
|
||||
return parsed === undefined
|
||||
? invalid()
|
||||
: apply(
|
||||
'minutes',
|
||||
'timeoutPersisted',
|
||||
VOICE_TIMEOUT_STORAGE_KEY,
|
||||
parsed,
|
||||
parsed === 0 ? 'none' : String(parsed),
|
||||
);
|
||||
},
|
||||
subscribe(listener) {
|
||||
if (typeof listener !== 'function')
|
||||
throw new TypeError('Preference listener required');
|
||||
listeners.add(listener);
|
||||
return () => listeners.delete(listener);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Mount-local controls, backed by an independently owned preference model. */
|
||||
export function bindPreferenceControls({ ui, preferences, copy, onChange }) {
|
||||
const {
|
||||
voicePreviewInput: preview,
|
||||
voicePreviewNote: previewNote,
|
||||
voiceTimeoutSelect: select,
|
||||
voiceTimeoutInput: input,
|
||||
voiceTimeoutNote: timeoutNote,
|
||||
} = ui;
|
||||
let disposed = false;
|
||||
const initial = preferences.snapshot();
|
||||
const displayed = parseMinutes(input?.value);
|
||||
let custom =
|
||||
!PRESETS.includes(String(initial.minutes)) && initial.minutes > 0
|
||||
? initial.minutes
|
||||
: displayed > 0
|
||||
? displayed
|
||||
: 5;
|
||||
function render(state) {
|
||||
if (disposed) return;
|
||||
if (preview) preview.checked = state.previewEnabled;
|
||||
if (previewNote)
|
||||
previewNote.textContent = `${copy.previewNote}${state.previewPersisted === false ? ` ${copy.notSaved}` : ''}`;
|
||||
if (timeoutNote)
|
||||
timeoutNote.textContent = `${copy.timeoutNote}${state.timeoutPersisted === false ? ` ${copy.notSaved}` : ''}`;
|
||||
const value = String(state.minutes);
|
||||
const customSelected = state.minutes > 0 && !PRESETS.includes(value);
|
||||
if (customSelected) custom = state.minutes;
|
||||
if (select)
|
||||
select.value =
|
||||
state.minutes === 0 ? '0' : customSelected ? 'custom' : value;
|
||||
if (input) {
|
||||
input.value = String(custom);
|
||||
input.hidden = !customSelected;
|
||||
}
|
||||
}
|
||||
const unsubscribe = preferences.subscribe((state) => {
|
||||
render(state);
|
||||
onChange?.(state);
|
||||
});
|
||||
const previewChange = () => {
|
||||
if (!disposed && !preferences.setPreviewEnabled(preview.checked).ok)
|
||||
render(preferences.snapshot());
|
||||
};
|
||||
const selectChange = () => {
|
||||
if (disposed) return;
|
||||
const value = select.value;
|
||||
if (value === 'custom') {
|
||||
const parsed = parseMinutes(input?.value);
|
||||
if (parsed === undefined || parsed === 0) {
|
||||
render(preferences.snapshot());
|
||||
return;
|
||||
}
|
||||
custom = parsed;
|
||||
preferences.setMinutes(parsed);
|
||||
// A custom value may equal a preset: retain the user's explicit editor choice.
|
||||
select.value = 'custom';
|
||||
if (input) input.hidden = false;
|
||||
} else if (value === '0' || PRESETS.includes(value))
|
||||
preferences.setMinutes(value);
|
||||
else render(preferences.snapshot());
|
||||
};
|
||||
const inputChange = () => {
|
||||
if (disposed) return;
|
||||
const parsed = parseMinutes(input.value);
|
||||
if (parsed === undefined || parsed === 0) {
|
||||
render(preferences.snapshot());
|
||||
return;
|
||||
}
|
||||
custom = parsed;
|
||||
preferences.setMinutes(parsed);
|
||||
if (select) select.value = 'custom';
|
||||
input.hidden = false;
|
||||
};
|
||||
const bindings = [
|
||||
[preview, previewChange],
|
||||
[select, selectChange],
|
||||
[input, inputChange],
|
||||
];
|
||||
for (const [element, listener] of bindings)
|
||||
element?.addEventListener('change', listener);
|
||||
render(initial);
|
||||
return {
|
||||
dispose() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
unsubscribe();
|
||||
for (const [element, listener] of bindings)
|
||||
element?.removeEventListener('change', listener);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
createVoicePreferences,
|
||||
bindPreferenceControls,
|
||||
VOICE_PREVIEW_STORAGE_KEY as P,
|
||||
VOICE_TIMEOUT_STORAGE_KEY as T,
|
||||
} from './preferences.js';
|
||||
|
||||
function storage(entries = {}, mode = 'normal') {
|
||||
const values = new Map(Object.entries(entries));
|
||||
const writes = [];
|
||||
return {
|
||||
writes,
|
||||
values,
|
||||
getItem(key) {
|
||||
if (mode === 'readthrow') throw Error('blocked');
|
||||
return values.get(key) ?? null;
|
||||
},
|
||||
setItem(key, value) {
|
||||
writes.push([key, value]);
|
||||
if (mode === 'throw') throw Error('blocked');
|
||||
if (mode !== 'noop' && mode !== 'readthrow')
|
||||
values.set(key, mode === 'mismatch' ? `${value}!` : value);
|
||||
},
|
||||
};
|
||||
}
|
||||
class Element {
|
||||
value = '';
|
||||
checked = false;
|
||||
textContent = '';
|
||||
hidden = false;
|
||||
listeners = new Set();
|
||||
addEventListener(type, listener) {
|
||||
assert.equal(type, 'change');
|
||||
this.listeners.add(listener);
|
||||
}
|
||||
removeEventListener(type, listener) {
|
||||
assert.equal(type, 'change');
|
||||
this.listeners.delete(listener);
|
||||
}
|
||||
change(value) {
|
||||
if (typeof value === 'boolean') this.checked = value;
|
||||
else this.value = value;
|
||||
for (const listener of [...this.listeners]) listener();
|
||||
}
|
||||
}
|
||||
function controls(custom = '7') {
|
||||
const ui = Object.fromEntries(
|
||||
[
|
||||
'voicePreviewInput',
|
||||
'voicePreviewNote',
|
||||
'voiceTimeoutSelect',
|
||||
'voiceTimeoutInput',
|
||||
'voiceTimeoutNote',
|
||||
].map((key) => [key, new Element()]),
|
||||
);
|
||||
ui.voiceTimeoutInput.value = custom;
|
||||
return ui;
|
||||
}
|
||||
const copy = {
|
||||
previewNote: 'Preview local',
|
||||
timeoutNote: 'Provider limits still apply',
|
||||
notSaved: 'Not saved',
|
||||
};
|
||||
function bind(ui, preferences, onChange) {
|
||||
return bindPreferenceControls({ ui, preferences, copy, onChange });
|
||||
}
|
||||
|
||||
test('defaults and restore are passive; legacy None and stored custom restore', () => {
|
||||
for (const [entries, minutes, previewEnabled] of [
|
||||
[{}, 5, true],
|
||||
[{ [P]: 'false', [T]: 'none' }, 0, false],
|
||||
[{ [T]: '17' }, 17, true],
|
||||
[{ [T]: 'broken' }, 5, true],
|
||||
]) {
|
||||
const disk = storage(entries);
|
||||
const prefs = createVoicePreferences({ storage: disk });
|
||||
assert.deepEqual(prefs.snapshot(), {
|
||||
previewEnabled,
|
||||
minutes,
|
||||
previewPersisted: null,
|
||||
timeoutPersisted: null,
|
||||
});
|
||||
const ui = controls();
|
||||
bind(ui, prefs).dispose();
|
||||
assert.deepEqual(disk.writes, []);
|
||||
}
|
||||
});
|
||||
|
||||
test('exact valid values, presets, None and serialized readback', () => {
|
||||
const disk = storage();
|
||||
const prefs = createVoicePreferences({ storage: disk });
|
||||
for (const value of [3, 5, 10, 15, '1', '60', 0]) {
|
||||
const result = prefs.setMinutes(value);
|
||||
assert.equal(result.ok, true);
|
||||
assert.equal(result.applied, true);
|
||||
assert.equal(result.persisted, true);
|
||||
assert.equal(result.minutes, Number(value));
|
||||
assert.deepEqual(disk.writes.at(-1), [
|
||||
T,
|
||||
Number(value) === 0 ? 'none' : String(Number(value)),
|
||||
]);
|
||||
}
|
||||
assert.equal(prefs.setPreviewEnabled(false).previewEnabled, false);
|
||||
assert.deepEqual(disk.writes.at(-1), [P, 'false']);
|
||||
});
|
||||
|
||||
test('fractional, malformed, coercible and out-of-range inputs never write or notify', () => {
|
||||
const disk = storage();
|
||||
const prefs = createVoicePreferences({ storage: disk });
|
||||
let notices = 0;
|
||||
prefs.subscribe(() => notices++);
|
||||
const before = prefs.snapshot();
|
||||
for (const value of [
|
||||
null,
|
||||
undefined,
|
||||
true,
|
||||
{},
|
||||
[],
|
||||
'',
|
||||
' ',
|
||||
' 3',
|
||||
'3 ',
|
||||
'3.0',
|
||||
'1e1',
|
||||
'0x10',
|
||||
'none',
|
||||
-1,
|
||||
61,
|
||||
3.2,
|
||||
NaN,
|
||||
Infinity,
|
||||
]) {
|
||||
const result = prefs.setMinutes(value);
|
||||
assert.equal(result.ok, false, String(value));
|
||||
assert.equal(result.applied, false);
|
||||
assert.equal(result.changed, false);
|
||||
assert.deepEqual(prefs.snapshot(), before);
|
||||
}
|
||||
for (const value of [0, 'true', null])
|
||||
assert.equal(prefs.setPreviewEnabled(value).ok, false);
|
||||
assert.equal(notices, 0);
|
||||
assert.deepEqual(disk.writes, []);
|
||||
});
|
||||
|
||||
for (const mode of ['throw', 'noop', 'mismatch', 'readthrow', 'missing']) {
|
||||
test(`storage ${mode} preserves live value and exposes unsaved state across A → B`, () => {
|
||||
const disk = mode === 'missing' ? null : storage({}, mode);
|
||||
const prefs = createVoicePreferences({ storage: disk });
|
||||
const a = controls();
|
||||
const bindingA = bind(a, prefs);
|
||||
a.voicePreviewInput.change(false);
|
||||
a.voiceTimeoutSelect.change('custom');
|
||||
a.voiceTimeoutInput.change('19');
|
||||
assert.deepEqual(prefs.snapshot(), {
|
||||
previewEnabled: false,
|
||||
minutes: 19,
|
||||
previewPersisted: false,
|
||||
timeoutPersisted: false,
|
||||
});
|
||||
bindingA.dispose();
|
||||
const writeCount = disk?.writes.length;
|
||||
const b = controls();
|
||||
const bindingB = bind(b, prefs);
|
||||
assert.equal(b.voicePreviewInput.checked, false);
|
||||
assert.equal(b.voiceTimeoutInput.value, '19');
|
||||
assert.match(b.voicePreviewNote.textContent, /Not saved/);
|
||||
assert.match(b.voiceTimeoutNote.textContent, /Not saved/);
|
||||
assert.equal(disk?.writes.length, writeCount);
|
||||
a.voicePreviewInput.change(true);
|
||||
a.voiceTimeoutSelect.change('3');
|
||||
assert.equal(prefs.snapshot().minutes, 19);
|
||||
assert.equal(prefs.snapshot().previewEnabled, false);
|
||||
bindingB.dispose();
|
||||
});
|
||||
}
|
||||
|
||||
test('same-value retry really writes and clears warnings after storage recovers', () => {
|
||||
const disk = storage();
|
||||
const original = disk.setItem;
|
||||
disk.setItem = () => {
|
||||
throw Error('blocked');
|
||||
};
|
||||
const prefs = createVoicePreferences({ storage: disk });
|
||||
const ui = controls();
|
||||
bind(ui, prefs);
|
||||
prefs.setPreviewEnabled(false);
|
||||
prefs.setMinutes(12);
|
||||
assert.match(ui.voiceTimeoutNote.textContent, /Not saved/);
|
||||
disk.setItem = original;
|
||||
const timeout = prefs.setMinutes(12);
|
||||
const preview = prefs.setPreviewEnabled(false);
|
||||
assert.equal(timeout.changed, false);
|
||||
assert.equal(preview.changed, false);
|
||||
assert.equal(timeout.persisted, true);
|
||||
assert.equal(preview.persisted, true);
|
||||
assert.deepEqual(disk.writes, [
|
||||
[T, '12'],
|
||||
[P, 'false'],
|
||||
]);
|
||||
assert.equal(ui.voiceTimeoutNote.textContent, copy.timeoutNote);
|
||||
assert.equal(ui.voicePreviewNote.textContent, copy.previewNote);
|
||||
});
|
||||
|
||||
test('Custom commits displayed valid value immediately, even when equal to preset', () => {
|
||||
const disk = storage();
|
||||
const prefs = createVoicePreferences({ storage: disk });
|
||||
const ui = controls('10');
|
||||
bind(ui, prefs);
|
||||
assert.equal(disk.writes.length, 0);
|
||||
ui.voiceTimeoutSelect.change('custom');
|
||||
assert.equal(prefs.snapshot().minutes, 10);
|
||||
assert.equal(ui.voiceTimeoutSelect.value, 'custom');
|
||||
assert.equal(ui.voiceTimeoutInput.hidden, false);
|
||||
assert.deepEqual(disk.writes, [[T, '10']]);
|
||||
ui.voiceTimeoutInput.change('23');
|
||||
ui.voiceTimeoutSelect.change('5');
|
||||
assert.equal(ui.voiceTimeoutInput.value, '23');
|
||||
assert.equal(ui.voiceTimeoutInput.hidden, true);
|
||||
ui.voiceTimeoutSelect.change('custom');
|
||||
assert.equal(prefs.snapshot().minutes, 23);
|
||||
assert.equal(ui.voiceTimeoutInput.value, '23');
|
||||
});
|
||||
|
||||
test('invalid control edits restore valid state without writes; None remains distinct', () => {
|
||||
const disk = storage({ [T]: '17' });
|
||||
const prefs = createVoicePreferences({ storage: disk });
|
||||
const ui = controls();
|
||||
bind(ui, prefs);
|
||||
assert.equal(ui.voiceTimeoutInput.value, '17');
|
||||
for (const value of ['1.5', '', '61', '0']) {
|
||||
ui.voiceTimeoutInput.change(value);
|
||||
assert.equal(ui.voiceTimeoutInput.value, '17');
|
||||
assert.equal(prefs.snapshot().minutes, 17);
|
||||
}
|
||||
ui.voiceTimeoutSelect.change('malformed');
|
||||
assert.equal(ui.voiceTimeoutSelect.value, 'custom');
|
||||
assert.equal(disk.writes.length, 0);
|
||||
ui.voiceTimeoutSelect.change('0');
|
||||
assert.equal(prefs.snapshot().minutes, 0);
|
||||
assert.equal(ui.voiceTimeoutInput.hidden, true);
|
||||
ui.voiceTimeoutInput.value = 'bad';
|
||||
ui.voiceTimeoutSelect.change('custom');
|
||||
assert.equal(ui.voiceTimeoutSelect.value, '0');
|
||||
assert.equal(prefs.snapshot().minutes, 0);
|
||||
assert.deepEqual(disk.writes, [[T, 'none']]);
|
||||
});
|
||||
|
||||
test('model snapshots cannot mutate state; unsubscribe and binder disposal release ownership', () => {
|
||||
const prefs = createVoicePreferences({ storage: storage() });
|
||||
const ui = controls();
|
||||
let calls = 0;
|
||||
const binding = bind(ui, prefs, () => calls++);
|
||||
const snap = prefs.snapshot();
|
||||
snap.minutes = 41;
|
||||
assert.equal(prefs.snapshot().minutes, 5);
|
||||
const unsub = prefs.subscribe((state) => {
|
||||
state.minutes = 22;
|
||||
});
|
||||
prefs.setMinutes(11);
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(prefs.snapshot().minutes, 11);
|
||||
unsub();
|
||||
binding.dispose();
|
||||
binding.dispose();
|
||||
prefs.setMinutes(12);
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(ui.voiceTimeoutInput.value, '11');
|
||||
for (const element of Object.values(ui))
|
||||
assert.equal(element.listeners.size, 0);
|
||||
});
|
||||
@@ -0,0 +1,204 @@
|
||||
// Minimal DOM fixture for actual control/card/binder composition, not layout proof.
|
||||
class Element {
|
||||
constructor(doc, tag = 'div') {
|
||||
Object.assign(this, {
|
||||
ownerDocument: doc,
|
||||
tagName: tag.toUpperCase(),
|
||||
children: [],
|
||||
attributes: new Map(),
|
||||
dataset: {},
|
||||
hidden: false,
|
||||
value: '',
|
||||
checked: false,
|
||||
listeners: new Map(),
|
||||
style: { setProperty() {}, removeProperty() {} },
|
||||
});
|
||||
this.classList = {
|
||||
add: (...names) => names.forEach((n) => this.classes.add(n)),
|
||||
remove: (...names) => names.forEach((n) => this.classes.delete(n)),
|
||||
contains: (name) => this.classes.has(name),
|
||||
};
|
||||
this.classes = new Set();
|
||||
}
|
||||
set id(value) {
|
||||
this.attributes.set('id', value);
|
||||
}
|
||||
get id() {
|
||||
return this.attributes.get('id') || '';
|
||||
}
|
||||
set className(value) {
|
||||
this.classes = new Set(value.split(/\s+/).filter(Boolean));
|
||||
}
|
||||
get className() {
|
||||
return [...this.classes].join(' ');
|
||||
}
|
||||
setAttribute(name, value) {
|
||||
this.attributes.set(name, String(value));
|
||||
if (name === 'class') this.className = value;
|
||||
if (name === 'hidden') this.hidden = true;
|
||||
if (name === 'value') this.value = value;
|
||||
if (name.startsWith('data-'))
|
||||
this.dataset[
|
||||
name.slice(5).replace(/-([a-z])/g, (_, c) => c.toUpperCase())
|
||||
] = value;
|
||||
}
|
||||
getAttribute(name) {
|
||||
return this.attributes.get(name) ?? null;
|
||||
}
|
||||
get isConnected() {
|
||||
return (
|
||||
this === this.ownerDocument.body || Boolean(this.parentNode?.isConnected)
|
||||
);
|
||||
}
|
||||
get parentElement() {
|
||||
return this.parentNode;
|
||||
}
|
||||
set innerHTML(html) {
|
||||
this.replaceChildren();
|
||||
const stack = [this];
|
||||
const voids = new Set(['INPUT', 'IMG', 'BR', 'HR']);
|
||||
for (const match of html.matchAll(/<(\/?)([a-z][\w-]*)\b([^>]*)>/gi)) {
|
||||
if (match[1]) {
|
||||
if (stack.length > 1) stack.pop();
|
||||
continue;
|
||||
}
|
||||
const node = new Element(this.ownerDocument, match[2]);
|
||||
for (const attr of match[3].matchAll(/([^\s=\/]+)(?:="([^"]*)")?/g))
|
||||
node.setAttribute(attr[1], attr[2] ?? '');
|
||||
stack.at(-1).appendChild(node);
|
||||
if (!voids.has(node.tagName)) stack.push(node);
|
||||
}
|
||||
}
|
||||
set textContent(value) {
|
||||
this.text = String(value);
|
||||
}
|
||||
get textContent() {
|
||||
return this.text ?? '';
|
||||
}
|
||||
appendChild(node) {
|
||||
node.remove();
|
||||
this.children.push(node);
|
||||
node.parentNode = this;
|
||||
return node;
|
||||
}
|
||||
append(...nodes) {
|
||||
nodes.forEach((node) => this.appendChild(node));
|
||||
}
|
||||
insertBefore(node, before) {
|
||||
node.remove();
|
||||
const at = this.children.indexOf(before);
|
||||
this.children.splice(at < 0 ? this.children.length : at, 0, node);
|
||||
node.parentNode = this;
|
||||
}
|
||||
replaceChildren(...nodes) {
|
||||
for (const child of this.children) child.parentNode = null;
|
||||
this.children = [];
|
||||
this.append(...nodes);
|
||||
}
|
||||
remove() {
|
||||
if (this.parentNode)
|
||||
this.parentNode.children = this.parentNode.children.filter(
|
||||
(n) => n !== this,
|
||||
);
|
||||
this.parentNode = null;
|
||||
}
|
||||
contains(node) {
|
||||
return node === this || this.children.some((child) => child.contains(node));
|
||||
}
|
||||
matches(selector) {
|
||||
if (selector.startsWith('#')) return this.id === selector.slice(1);
|
||||
if (selector.startsWith('.')) return this.classes.has(selector.slice(1));
|
||||
return this.tagName === selector.toUpperCase();
|
||||
}
|
||||
querySelectorAll(selector) {
|
||||
return this.children.flatMap((n) => [
|
||||
...(n.matches(selector) ? [n] : []),
|
||||
...n.querySelectorAll(selector),
|
||||
]);
|
||||
}
|
||||
querySelector(selector) {
|
||||
return this.querySelectorAll(selector)[0] || null;
|
||||
}
|
||||
checkVisibility() {
|
||||
return (
|
||||
this.isConnected &&
|
||||
!this.hidden &&
|
||||
(!this.parentNode || this.parentNode.checkVisibility())
|
||||
);
|
||||
}
|
||||
focus() {
|
||||
this.ownerDocument.activeElement = this;
|
||||
}
|
||||
addEventListener(type, listener, options = {}) {
|
||||
if (options.signal?.aborted) return;
|
||||
if (!this.listeners.has(type)) this.listeners.set(type, new Set());
|
||||
this.listeners.get(type).add(listener);
|
||||
if (options.signal)
|
||||
options.signal.addEventListener(
|
||||
'abort',
|
||||
() => this.removeEventListener(type, listener),
|
||||
{ once: true },
|
||||
);
|
||||
}
|
||||
removeEventListener(type, listener) {
|
||||
this.listeners.get(type)?.delete(listener);
|
||||
}
|
||||
fire(type, fields = {}) {
|
||||
const event = {
|
||||
type,
|
||||
target: this,
|
||||
preventDefault() {
|
||||
this.defaultPrevented = true;
|
||||
},
|
||||
...fields,
|
||||
};
|
||||
for (const listener of [...(this.listeners.get(type) || [])])
|
||||
listener.call(this, event);
|
||||
return event;
|
||||
}
|
||||
}
|
||||
|
||||
const installed = new WeakMap();
|
||||
export function installPanelDom(t) {
|
||||
if (installed.has(t)) return installed.get(t);
|
||||
const previous = {
|
||||
window: globalThis.window,
|
||||
document: globalThis.document,
|
||||
Element: globalThis.Element,
|
||||
};
|
||||
const doc = new EventTarget();
|
||||
doc.body = new Element(doc, 'body');
|
||||
doc.documentElement = new Element(doc, 'html');
|
||||
doc.documentElement.lang = 'en';
|
||||
doc.activeElement = doc.body;
|
||||
doc.createElement = (tag) => new Element(doc, tag);
|
||||
doc.getElementById = (id) => doc.body.querySelector(`#${id}`);
|
||||
doc.querySelector = (selector) => doc.body.querySelector(selector);
|
||||
const win = new EventTarget();
|
||||
globalThis.window = win;
|
||||
globalThis.document = doc;
|
||||
globalThis.Element = Element;
|
||||
installed.set(t, doc);
|
||||
t.after(() => Object.assign(globalThis, previous));
|
||||
return doc;
|
||||
}
|
||||
|
||||
export function controlledClock() {
|
||||
let now = 0,
|
||||
next = 0;
|
||||
const tasks = new Map();
|
||||
return {
|
||||
now: () => now,
|
||||
setTimer(fn, ms) {
|
||||
const id = ++next;
|
||||
tasks.set(id, { fn, at: now + ms });
|
||||
return id;
|
||||
},
|
||||
clearTimer: (id) => tasks.delete(id),
|
||||
advance(ms) {
|
||||
now += ms;
|
||||
for (const [id, task] of [...tasks])
|
||||
if (task.at <= now && tasks.delete(id)) task.fn();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -20,15 +20,15 @@
|
||||
*/
|
||||
|
||||
/** Movement below this many CSS pixels is still the same dwell. */
|
||||
export const POINTER_DWELL_PX = 12;
|
||||
const POINTER_DWELL_PX = 12;
|
||||
/** Stillness that makes a dwell point. */
|
||||
export const POINTER_DWELL_MS = 250;
|
||||
/** A resting pointer stays a referent this long after it last moved. */
|
||||
export const POINTER_FRESH_MS = 30000;
|
||||
/** Pixels from the canvas edge that count as "parked", never a referent. */
|
||||
export const POINTER_EDGE_PX = 8;
|
||||
const POINTER_EDGE_PX = 8;
|
||||
/** A dwell pick is reused by a snapshot only while this recent (contacts move). */
|
||||
export const POINTER_DWELL_REUSE_MS = 1000;
|
||||
const POINTER_DWELL_REUSE_MS = 1000;
|
||||
/**
|
||||
* Open mic has no gesture, so the pointer only counts when it moved or
|
||||
* settled on the map this recently: a cursor resting there for minutes is
|
||||
@@ -76,7 +76,7 @@ export function cameraPoseKey(viewer) {
|
||||
}
|
||||
|
||||
/** Bound and de-control a label before it goes to the model or the card. */
|
||||
export function pointerLabel(value, max = LABEL_MAX) {
|
||||
function pointerLabel(value, max = LABEL_MAX) {
|
||||
const clean = String(value ?? '')
|
||||
.replace(/[\p{Cc}\p{Cf}]+/gu, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
@@ -182,6 +182,63 @@ export function pointerChip(snapshot) {
|
||||
};
|
||||
}
|
||||
|
||||
/** Dedupe key of the clearing pointer_context item. */
|
||||
export const POINTER_NONE_KEY = 'none';
|
||||
|
||||
/**
|
||||
* The voice-card event for a turn's pointer: a gesture reveals its target at
|
||||
* once; open-mic and typed turns reveal it only through announcePointer.
|
||||
* @param {object|null} snapshot
|
||||
* @param {boolean} gesture
|
||||
* @returns {{announced: object|null, pointer: object|null}}
|
||||
*/
|
||||
export function pointerTurnEvent(snapshot, gesture) {
|
||||
const chip = gesture ? pointerChip(snapshot) : null;
|
||||
return chip
|
||||
? { announced: snapshot, pointer: { ...chip, screenPx: snapshot.screenPx } }
|
||||
: { announced: null, pointer: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* The pointer_context item a turn should send, or null when there is nothing
|
||||
* new. Without a gesture the model learns only what kind of thing is pointed
|
||||
* at, never which. Losing a sent pointer produces one clearing item.
|
||||
* @param {object|null} snapshot
|
||||
* @param {{at: string, gesture: boolean, sentKey: string|null}} turn
|
||||
* @returns {{item: object, key: string}|null}
|
||||
*/
|
||||
export function pointerTurnContext(snapshot, { at, gesture, sentKey }) {
|
||||
const chip = pointerChip(snapshot);
|
||||
const item = !chip
|
||||
? sentKey && sentKey !== POINTER_NONE_KEY
|
||||
? { type: 'pointer_context', at, target: 'none' }
|
||||
: null
|
||||
: gesture
|
||||
? pointerContextItem(snapshot)
|
||||
: {
|
||||
type: 'pointer_context',
|
||||
at,
|
||||
pointing: snapshot.entity?.kind || 'ground',
|
||||
};
|
||||
if (!item) return null;
|
||||
const { at: _at, ...content } = item;
|
||||
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
|
||||
return key === sentKey ? null : { item, key };
|
||||
}
|
||||
|
||||
/**
|
||||
* The voice-card event revealing a pointer a tool resolved, once per snapshot.
|
||||
* @param {object|null} snapshot
|
||||
* @param {object|null} announced Snapshot already shown.
|
||||
* @returns {object|null}
|
||||
*/
|
||||
export function pointerAnnouncement(snapshot, announced) {
|
||||
const chip = pointerChip(snapshot);
|
||||
return chip && snapshot !== announced
|
||||
? { ...chip, screenPx: snapshot.screenPx }
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Track the pointer over one element (the globe canvas).
|
||||
* @param {object} options
|
||||
@@ -330,7 +387,14 @@ export function createPointerTracker({
|
||||
* (open mic only when it moved recently: `recentMs`). The result becomes
|
||||
* the turn's referent (null when stale).
|
||||
*/
|
||||
beginTurn(at, { recentMs = POINTER_FRESH_MS } = {}) {
|
||||
beginTurn(at, { recentMs = POINTER_FRESH_MS, capturedSnapshot } = {}) {
|
||||
// Delayed admitted audio keeps its capture's pointer without replacing a
|
||||
// newer physical hold. Undefined preserves existing caller behavior.
|
||||
if (capturedSnapshot !== undefined) {
|
||||
const snapshot = destroyed ? { fresh: false, at } : capturedSnapshot;
|
||||
active = snapshot?.fresh ? snapshot : null;
|
||||
return snapshot;
|
||||
}
|
||||
const reuse =
|
||||
at !== 'text' && held && now() - held.heldAt <= POINTER_HOLD_MS;
|
||||
const snapshot = reuse ? held.snapshot : capture(at, { recentMs });
|
||||
|
||||
@@ -6,8 +6,13 @@ import {
|
||||
POINTER_HOLD_MS,
|
||||
buildPointerSnapshot,
|
||||
createPointerTracker,
|
||||
POINTER_NONE_KEY,
|
||||
formatPointerContext,
|
||||
pointerAnnouncement,
|
||||
pointerChip,
|
||||
pointerContextItem,
|
||||
pointerTurnContext,
|
||||
pointerTurnEvent,
|
||||
} from './pointerContext.js';
|
||||
|
||||
function harness({ pick } = {}) {
|
||||
@@ -151,6 +156,51 @@ test('destroy removes listeners and ignores later moves', () => {
|
||||
assert.equal(h.tracker.capture('keydown').fresh, false);
|
||||
});
|
||||
|
||||
test('delayed speech uses its released capture without replacing a newer physical hold', () => {
|
||||
const h = harness();
|
||||
h.move(700, 250);
|
||||
const capturedA = h.tracker.hold();
|
||||
h.tracker.releaseHold();
|
||||
h.move(200, 100);
|
||||
h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA });
|
||||
assert.equal(h.tracker.activeSnapshot(), capturedA);
|
||||
const heldB = h.tracker.hold();
|
||||
h.tracker.beginTurn('speech_start', { capturedSnapshot: capturedA });
|
||||
assert.equal(h.tracker.activeSnapshot(), capturedA);
|
||||
assert.equal(h.tracker.isHeld(), true);
|
||||
assert.equal(h.tracker.beginTurn('speech_start'), heldB);
|
||||
h.tracker.destroy();
|
||||
});
|
||||
|
||||
test('explicit stale captures clear the referent while a silent hold leaves it alone', () => {
|
||||
const h = harness();
|
||||
h.move(700, 250);
|
||||
const current = h.tracker.beginTurn('text');
|
||||
h.move(200, 100);
|
||||
h.tracker.hold();
|
||||
h.tracker.releaseHold();
|
||||
assert.equal(h.tracker.activeSnapshot(), current);
|
||||
for (const capturedSnapshot of [null, { fresh: false, at: 'keydown' }]) {
|
||||
h.tracker.beginTurn('text');
|
||||
h.tracker.beginTurn('speech_start', { capturedSnapshot });
|
||||
assert.equal(h.tracker.activeSnapshot(), null);
|
||||
}
|
||||
h.tracker.destroy();
|
||||
});
|
||||
|
||||
test('a capture arriving after destruction cannot restore an active referent', () => {
|
||||
const h = harness();
|
||||
h.move(700, 250);
|
||||
const capturedSnapshot = h.tracker.hold();
|
||||
assert.equal(capturedSnapshot.fresh, true);
|
||||
h.tracker.destroy();
|
||||
const result = h.tracker.beginTurn('speech_start', { capturedSnapshot });
|
||||
assert.equal(result.fresh, false);
|
||||
assert.equal(h.tracker.activeSnapshot(), null);
|
||||
assert.equal(h.tracker.isHeld(), false);
|
||||
assert.equal(h.tracker.beginTurn('speech_start').fresh, false);
|
||||
});
|
||||
|
||||
test('format: the model item is compact inert JSON; labels are bounded data', () => {
|
||||
const snapshot = buildPointerSnapshot({
|
||||
at: 'speech_start',
|
||||
@@ -182,3 +232,65 @@ test('chip: "this" names an entity, "here" gives the ground coordinate', () => {
|
||||
assert.deepEqual(pointerChip(ground), { kind: 'here', label: '30.27, -97.74' });
|
||||
assert.equal(pointerChip(null), null);
|
||||
});
|
||||
|
||||
test('pointer turn helpers reveal gesture targets, describe implicit ones by kind and dedupe repeats', () => {
|
||||
const snapshot = {
|
||||
fresh: true,
|
||||
at: 'keydown',
|
||||
entity: { kind: 'aircraft', layerId: 'flights', label: 'UAL1' },
|
||||
lat: 10,
|
||||
lon: 20,
|
||||
screen: { x: 5, y: 6 },
|
||||
screenPx: { x: 5, y: 6 },
|
||||
};
|
||||
const shown = pointerTurnEvent(snapshot, true);
|
||||
assert.equal(shown.announced, snapshot);
|
||||
assert.deepEqual(shown.pointer, {
|
||||
kind: 'this',
|
||||
label: 'UAL1',
|
||||
screenPx: { x: 5, y: 6 },
|
||||
});
|
||||
assert.deepEqual(pointerTurnEvent(snapshot, false), {
|
||||
announced: null,
|
||||
pointer: null,
|
||||
});
|
||||
|
||||
const gesture = pointerTurnContext(snapshot, {
|
||||
at: 'keydown',
|
||||
gesture: true,
|
||||
sentKey: null,
|
||||
});
|
||||
assert.deepEqual(gesture.item, pointerContextItem(snapshot));
|
||||
assert.equal(
|
||||
pointerTurnContext(snapshot, {
|
||||
at: 'keydown',
|
||||
gesture: true,
|
||||
sentKey: gesture.key,
|
||||
}),
|
||||
null,
|
||||
);
|
||||
assert.deepEqual(
|
||||
pointerTurnContext(snapshot, { at: 'text', gesture: false, sentKey: null })
|
||||
.item,
|
||||
{ type: 'pointer_context', at: 'text', pointing: 'aircraft' },
|
||||
);
|
||||
|
||||
const cleared = pointerTurnContext(null, {
|
||||
at: 'text',
|
||||
gesture: false,
|
||||
sentKey: gesture.key,
|
||||
});
|
||||
assert.deepEqual(cleared, {
|
||||
item: { type: 'pointer_context', at: 'text', target: 'none' },
|
||||
key: POINTER_NONE_KEY,
|
||||
});
|
||||
for (const sentKey of [null, POINTER_NONE_KEY])
|
||||
assert.equal(
|
||||
pointerTurnContext(null, { at: 'text', gesture: false, sentKey }),
|
||||
null,
|
||||
);
|
||||
|
||||
assert.deepEqual(pointerAnnouncement(snapshot, null), shown.pointer);
|
||||
assert.equal(pointerAnnouncement(snapshot, snapshot), null);
|
||||
assert.equal(pointerAnnouncement(null, null), null);
|
||||
});
|
||||
@@ -4,7 +4,7 @@ import { cameraPoseKey } from './pointerContext.js';
|
||||
import * as defaultAnnotationResolver from '../annotations/annotationResolver.js';
|
||||
|
||||
/** "Around here" radius as a share of camera height, clamped to sane bounds. */
|
||||
export function pointerRadiusKm(cameraHeightM) {
|
||||
function pointerRadiusKm(cameraHeightM) {
|
||||
const heightKm = Number(cameraHeightM) / 1000;
|
||||
if (!Number.isFinite(heightKm) || heightKm <= 0) return 50;
|
||||
return Math.round(Math.min(400, Math.max(5, heightKm * 0.75)));
|
||||
|
||||
@@ -14,7 +14,7 @@ export const RETICLE_MAX_MS = 10000;
|
||||
export const RETICLE_LINGER_MS = 1500;
|
||||
|
||||
/** Tools that move the camera away from the pointed spot. */
|
||||
export const CAMERA_TOOLS = new Set([
|
||||
const CAMERA_TOOLS = new Set([
|
||||
'fly_to_location',
|
||||
'track_entity',
|
||||
'zoom_to_globe',
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
export const VOICE_PROVIDERS = Object.freeze({
|
||||
openai: Object.freeze({ label: 'OpenAI' }), // i18n-ignore: provider product name
|
||||
gemini: Object.freeze({ label: 'Gemini' }), // i18n-ignore: provider product name
|
||||
});
|
||||
|
||||
/** Optional providers require an explicit host opt-in and an available factory. */
|
||||
export function supportedVoiceProviders(
|
||||
requested = ['openai'],
|
||||
factories = {},
|
||||
) {
|
||||
if (!Array.isArray(requested))
|
||||
throw new TypeError('supportedProviders must be an array');
|
||||
const selected = [...new Set(requested)].filter(
|
||||
(name) =>
|
||||
Object.hasOwn(VOICE_PROVIDERS, name) &&
|
||||
typeof factories[name] === 'function',
|
||||
);
|
||||
if (!selected.length)
|
||||
throw new Error('No supported voice provider is configured');
|
||||
return Object.freeze(selected);
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
import { supportedVoiceProviders } from './providerCapabilities.js';
|
||||
|
||||
const STORAGE_KEY = 'gev.voice.provider';
|
||||
|
||||
export function readVoiceProvider(storage, supported = ['openai', 'gemini']) {
|
||||
let saved;
|
||||
try {
|
||||
saved = (storage ?? globalThis.localStorage)?.getItem(STORAGE_KEY);
|
||||
} catch {
|
||||
/* Storage is optional. */
|
||||
}
|
||||
return supported.includes(saved)
|
||||
? saved
|
||||
: supported.includes('openai')
|
||||
? 'openai'
|
||||
: supported[0];
|
||||
}
|
||||
|
||||
/** Keep one public controller identity while replacing each provider's lifetime. */
|
||||
export function createProviderCommands(options, { bind, factories, storage }) {
|
||||
const supportedProviders = supportedVoiceProviders(
|
||||
options.supportedProviders ?? Object.keys(factories),
|
||||
factories,
|
||||
);
|
||||
window.__gevVoiceCommands?.stop?.({ removeUi: true });
|
||||
let provider = readVoiceProvider(storage, supportedProviders);
|
||||
let current;
|
||||
let disposed = false;
|
||||
const lifetime = new AbortController();
|
||||
let mountGeneration = 0;
|
||||
let voiceEnabled = false;
|
||||
const stored = () => {
|
||||
try {
|
||||
return storage ?? globalThis.localStorage;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const stop = (settings = {}) => {
|
||||
if (disposed) return;
|
||||
voiceEnabled = false;
|
||||
if (settings.removeUi) {
|
||||
disposed = true;
|
||||
options.signal?.removeEventListener('abort', destroy);
|
||||
}
|
||||
try {
|
||||
current?.session.stop(settings);
|
||||
} finally {
|
||||
if (settings.removeUi) {
|
||||
lifetime.abort();
|
||||
options.runner.dispose?.();
|
||||
}
|
||||
}
|
||||
};
|
||||
const destroy = () => stop({ removeUi: true });
|
||||
const selectProvider = (next) => {
|
||||
if (disposed || !supportedProviders.includes(next) || next === provider)
|
||||
return false;
|
||||
const resume = voiceEnabled && current.session.isActive();
|
||||
// Carry the input mode, never a held key, microphone or conversation.
|
||||
const pushToTalk = Boolean(
|
||||
current.session.adapter.getStartOptions?.().pushToTalk,
|
||||
);
|
||||
const generation = ++mountGeneration;
|
||||
voiceEnabled = false;
|
||||
current.session.destroy();
|
||||
// Disposal suppresses the retired adapter's idle event. Clear the shared
|
||||
// conversation explicitly before the new mount can use these owners.
|
||||
if (disposed || generation !== mountGeneration) return false;
|
||||
options.pointer?.clear();
|
||||
options.referents?.clear();
|
||||
options.runner.resetConversation?.();
|
||||
if (disposed || generation !== mountGeneration) return false;
|
||||
provider = next;
|
||||
try {
|
||||
stored()?.setItem(STORAGE_KEY, provider);
|
||||
} catch {
|
||||
/* Storage is optional. */
|
||||
}
|
||||
mount();
|
||||
if (resume && !disposed && provider === next)
|
||||
void current.session.start({ pushToTalk });
|
||||
globalThis.document?.getElementById('gev-voice-provider')?.focus();
|
||||
return true;
|
||||
};
|
||||
|
||||
const handleSessionEvent = (event) => {
|
||||
if (event?.type === 'state' && event.state === 'connecting')
|
||||
voiceEnabled = true;
|
||||
if (event?.type === 'state' && ['idle', 'error'].includes(event.state)) {
|
||||
voiceEnabled = false;
|
||||
}
|
||||
// The host still observes the mounted provider's neutral session events.
|
||||
options.onSessionEvent?.(event);
|
||||
};
|
||||
|
||||
const sendTextCommand = (text) => {
|
||||
if (disposed) return false;
|
||||
return current.session.sendText(text);
|
||||
};
|
||||
|
||||
const api = {
|
||||
// The shared action surface is independent of an active voice connection.
|
||||
runner: options.runner,
|
||||
supportedProviders,
|
||||
signal: lifetime.signal,
|
||||
get provider() {
|
||||
return provider;
|
||||
},
|
||||
get session() {
|
||||
return current?.session;
|
||||
},
|
||||
get status() {
|
||||
return current?.status ?? current?.session.state ?? 'idle';
|
||||
},
|
||||
get state() {
|
||||
return current?.session.state ?? 'idle';
|
||||
},
|
||||
get disposed() {
|
||||
return disposed;
|
||||
},
|
||||
start: (settings) => !disposed && current.session.start(settings),
|
||||
stop,
|
||||
destroy,
|
||||
isActive: () => !disposed && current.session.isActive(),
|
||||
sendTextCommand,
|
||||
sendText: sendTextCommand,
|
||||
notifyMapEvent: (event) => !disposed && current.session.sendMapEvent(event),
|
||||
};
|
||||
// Existing diagnostic and controller accessors continue to reflect the active provider.
|
||||
const facade = new Proxy(api, {
|
||||
get(target, name, receiver) {
|
||||
if (Reflect.has(target, name)) return Reflect.get(target, name, receiver);
|
||||
const value = current?.[name];
|
||||
return typeof value === 'function' ? value.bind(current) : value;
|
||||
},
|
||||
});
|
||||
function mount() {
|
||||
voiceEnabled = false;
|
||||
const generation = ++mountGeneration;
|
||||
current = bind({
|
||||
...options,
|
||||
signal: lifetime.signal,
|
||||
resetExisting: false,
|
||||
disposeRunner: false,
|
||||
provider,
|
||||
supportedProviders,
|
||||
onProviderChange: selectProvider,
|
||||
onSessionEvent: (event) => {
|
||||
if (!disposed && generation === mountGeneration)
|
||||
handleSessionEvent(event);
|
||||
},
|
||||
createSession: factories[provider],
|
||||
});
|
||||
window.__gevVoiceCommands = facade;
|
||||
}
|
||||
mount();
|
||||
options.signal?.addEventListener('abort', destroy, { once: true });
|
||||
if (options.signal?.aborted) destroy();
|
||||
return facade;
|
||||
}
|
||||
@@ -0,0 +1,888 @@
|
||||
// Imports intentionally target the candidate, never the immutable reference.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
createProviderCommands,
|
||||
readVoiceProvider,
|
||||
} from './providerCommands.js';
|
||||
import { createVoiceCommands as bindVoiceCommands } from './sessionCommands.js';
|
||||
import { createVoiceCommands as createCommands } from './commands.js';
|
||||
import { createGevActionRunner } from './gevActions.js';
|
||||
import { createPointerTracker } from './pointerContext.js';
|
||||
import { createReferentRegistry } from './referents.js';
|
||||
import { describeUnsavedWork, t as translate } from '../i18n/index.js';
|
||||
|
||||
const PROVIDER_KEY = 'gev.voice.provider';
|
||||
const settle = () => new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
let reject;
|
||||
const promise = new Promise((yes, no) => {
|
||||
resolve = yes;
|
||||
reject = no;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
// Listener counts observe the actual binder's registrations/removals.
|
||||
class Element extends EventTarget {
|
||||
listeners = new Map();
|
||||
attributes = new Map();
|
||||
addEventListener(type, handler, options) {
|
||||
if (!this.listeners.has(type)) this.listeners.set(type, new Set());
|
||||
this.listeners.get(type).add(handler);
|
||||
super.addEventListener(type, handler, options);
|
||||
}
|
||||
removeEventListener(type, handler, options) {
|
||||
this.listeners.get(type)?.delete(handler);
|
||||
super.removeEventListener(type, handler, options);
|
||||
}
|
||||
listenerCount(type) {
|
||||
return this.listeners.get(type)?.size || 0;
|
||||
}
|
||||
setAttribute(name, value) {
|
||||
this.attributes.set(name, String(value));
|
||||
}
|
||||
focus() {
|
||||
this.focusCount = (this.focusCount || 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
for (const custom of [false, true]) {
|
||||
test(`public composition preserves the ${custom ? 'custom adapter' : 'default OpenAI adapter'} path`, async (t) => {
|
||||
const prior = globalThis.window;
|
||||
globalThis.window = new EventTarget();
|
||||
let controls;
|
||||
t.after(() => {
|
||||
controls?.session.destroy();
|
||||
globalThis.window = prior;
|
||||
});
|
||||
const ui = {
|
||||
root: { dataset: {}, remove() {} },
|
||||
button: new Element(),
|
||||
status: {},
|
||||
detail: {},
|
||||
providerField: {},
|
||||
};
|
||||
let defaultCreated = 0;
|
||||
let customCreated = 0;
|
||||
const options = {
|
||||
runner: async () => ({ ok: true }),
|
||||
createControl: () => ui,
|
||||
createController: ({ onSessionEvent }) => {
|
||||
defaultCreated++;
|
||||
return {
|
||||
ui,
|
||||
start({ pushToTalk }) {
|
||||
this.pushToTalkMode = pushToTalk;
|
||||
onSessionEvent({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop() {},
|
||||
syncCostUi() {},
|
||||
bindPushToTalkShortcut() {},
|
||||
sendTextCommand: (text) => `default:${text}`,
|
||||
};
|
||||
},
|
||||
};
|
||||
if (custom) {
|
||||
options.supportedProviders = ['gemini'];
|
||||
options.createSession = ({ emit }) => {
|
||||
customCreated++;
|
||||
return {
|
||||
capabilities: { pushToTalk: false },
|
||||
start() {
|
||||
emit({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop() {},
|
||||
sendText: (text) => `custom:${text}`,
|
||||
sendMapEvent: () => true,
|
||||
};
|
||||
};
|
||||
}
|
||||
controls = createCommands(options);
|
||||
await controls.session.start({ pushToTalk: true });
|
||||
assert.equal(defaultCreated, custom ? 0 : 1);
|
||||
assert.equal(customCreated, custom ? 1 : 0);
|
||||
assert.equal(
|
||||
controls.session.sendText('test'),
|
||||
`${custom ? 'custom' : 'default'}:test`,
|
||||
);
|
||||
assert.equal(ui.providerField.hidden, true);
|
||||
assert.equal(ui.root.dataset.providerChoice, 'false');
|
||||
if (!custom)
|
||||
assert.deepEqual(controls.session.adapter.getStartOptions(), {
|
||||
pushToTalk: true,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** Real facade -> real binder -> real common session and real action runner.
|
||||
* Adapters and passive UI/viewer boundaries are fixtures. Wrappers only count
|
||||
* calls to real pointer/referent/conversation cleanup; they do not replace it.
|
||||
*/
|
||||
function compositionFixture(
|
||||
t,
|
||||
{
|
||||
supportedProviders = ['openai', 'gemini'],
|
||||
factoryProviders = ['openai', 'gemini'],
|
||||
savedProvider = null,
|
||||
storageDenied = false,
|
||||
gateStarts = () => false,
|
||||
onSessionEvent = () => {},
|
||||
} = {},
|
||||
) {
|
||||
const original = new Map(
|
||||
['window', 'document'].map((key) => [
|
||||
key,
|
||||
Object.getOwnPropertyDescriptor(globalThis, key),
|
||||
]),
|
||||
);
|
||||
const lifetime = new AbortController();
|
||||
const views = [];
|
||||
const instances = [];
|
||||
const mounts = [];
|
||||
const events = [];
|
||||
const order = [];
|
||||
const annotations = new Set();
|
||||
const storageWrites = [];
|
||||
const counters = { pointer: 0, referents: 0, conversation: 0, dispose: 0 };
|
||||
const state = { height: 1000, zooms: 0, renders: 0 };
|
||||
let facade;
|
||||
let pointer;
|
||||
globalThis.window = new EventTarget();
|
||||
globalThis.document = {
|
||||
getElementById: (id) =>
|
||||
id === 'gev-voice-provider' ? views.at(-1)?.providerSelect : null,
|
||||
};
|
||||
t.after(async () => {
|
||||
try {
|
||||
facade?.destroy();
|
||||
pointer?.destroy();
|
||||
for (const instance of instances)
|
||||
for (const gate of instance.gates) gate.resolve();
|
||||
await Promise.allSettled(instances.flatMap((instance) => instance.jobs));
|
||||
} finally {
|
||||
for (const [key, descriptor] of original) {
|
||||
if (descriptor) Object.defineProperty(globalThis, key, descriptor);
|
||||
else delete globalThis[key];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const canvas = new EventTarget();
|
||||
Object.assign(canvas, { clientWidth: 800, clientHeight: 600 });
|
||||
const position = { x: 6379137, y: 0, z: 0 };
|
||||
const viewer = {
|
||||
clock: { onTick: { addEventListener: () => () => {} } },
|
||||
camera: {
|
||||
positionWC: position,
|
||||
positionCartographic: { height: state.height },
|
||||
moveEnd: { addEventListener() {} },
|
||||
cancelFlight() {},
|
||||
zoomIn(metres) {
|
||||
position.x -= metres;
|
||||
state.height -= metres;
|
||||
this.positionCartographic.height = state.height;
|
||||
state.zooms++;
|
||||
},
|
||||
zoomOut(metres) {
|
||||
position.x += metres;
|
||||
state.height += metres;
|
||||
this.positionCartographic.height = state.height;
|
||||
state.zooms++;
|
||||
},
|
||||
},
|
||||
scene: {
|
||||
canvas,
|
||||
requestRender() {
|
||||
state.renders++;
|
||||
},
|
||||
},
|
||||
};
|
||||
const dataManager = { layers: new Map() };
|
||||
pointer = createPointerTracker({ element: null, now: () => 1000 });
|
||||
const referents = createReferentRegistry();
|
||||
const runner = createGevActionRunner({
|
||||
viewer,
|
||||
dataManager,
|
||||
styleManager: {},
|
||||
deixis: { pointer, referents },
|
||||
});
|
||||
for (const [object, method, counter] of [
|
||||
[pointer, 'clear', 'pointer'],
|
||||
[referents, 'clear', 'referents'],
|
||||
[runner, 'resetConversation', 'conversation'],
|
||||
[runner, 'dispose', 'dispose'],
|
||||
]) {
|
||||
const originalMethod = object[method];
|
||||
object[method] = (...args) => {
|
||||
counters[counter]++;
|
||||
order.push(counter);
|
||||
return originalMethod.apply(object, args);
|
||||
};
|
||||
}
|
||||
|
||||
const factories = Object.fromEntries(
|
||||
factoryProviders.map((provider) => [
|
||||
provider,
|
||||
(hooks) => {
|
||||
const id = instances.length;
|
||||
let generation = 0;
|
||||
const instance = {
|
||||
id,
|
||||
provider,
|
||||
hooks,
|
||||
starts: [],
|
||||
stops: [],
|
||||
jobs: [],
|
||||
gates: [],
|
||||
acquired: 0,
|
||||
microphoneEnabled: false,
|
||||
bound: 0,
|
||||
input: { pushToTalk: false, held: false },
|
||||
texts: [],
|
||||
mapEvents: [],
|
||||
};
|
||||
instances.push(instance);
|
||||
order.push(`factory:${id}`);
|
||||
const adapter = {
|
||||
identity: `${provider}:${id}`,
|
||||
capabilities: {
|
||||
pushToTalk: true,
|
||||
costControls: provider === 'openai',
|
||||
},
|
||||
controller: {
|
||||
providerDiagnostic() {
|
||||
return this.session.adapter.identity;
|
||||
},
|
||||
},
|
||||
start(settings = {}) {
|
||||
const attempt = ++generation;
|
||||
instance.starts.push({ ...settings });
|
||||
instance.input.pushToTalk = Boolean(settings.pushToTalk);
|
||||
hooks.emit({
|
||||
type: 'state',
|
||||
state: 'connecting',
|
||||
detail: `adapter:${provider}`,
|
||||
});
|
||||
const job = (async () => {
|
||||
if (attempt !== generation || hooks.signal.aborted) return;
|
||||
if (gateStarts(instance)) {
|
||||
const gate = deferred();
|
||||
instance.gates.push(gate);
|
||||
await gate.promise;
|
||||
}
|
||||
// Adapters own cancellation of their asynchronous acquisition.
|
||||
if (attempt !== generation || hooks.signal.aborted) return;
|
||||
instance.acquired++;
|
||||
instance.microphoneEnabled =
|
||||
!instance.input.pushToTalk || instance.input.held;
|
||||
hooks.emit({
|
||||
type: 'state',
|
||||
state: 'listening',
|
||||
detail: provider,
|
||||
});
|
||||
})();
|
||||
instance.jobs.push(job);
|
||||
return job;
|
||||
},
|
||||
stop(settings = {}) {
|
||||
generation++;
|
||||
instance.stops.push({ ...settings });
|
||||
order.push(`stop:${id}`);
|
||||
instance.input.pushToTalk = instance.input.held = false;
|
||||
instance.microphoneEnabled = false;
|
||||
},
|
||||
getStartOptions: () => ({ pushToTalk: instance.input.pushToTalk }),
|
||||
bindControls() {
|
||||
instance.bound++;
|
||||
},
|
||||
sendText(text) {
|
||||
instance.texts.push(text);
|
||||
return `${provider}:${text}`;
|
||||
},
|
||||
sendMapEvent(event) {
|
||||
instance.mapEvents.push(event);
|
||||
return true;
|
||||
},
|
||||
};
|
||||
instance.adapter = adapter;
|
||||
return adapter;
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
function createControl() {
|
||||
const id = views.length;
|
||||
order.push(`control:${id}`);
|
||||
const providerSelect = new Element();
|
||||
providerSelect.ownerDocument = { createElement: () => ({}) };
|
||||
providerSelect.replaceChildren = (...options) => {
|
||||
providerSelect.options = options;
|
||||
};
|
||||
const view = {
|
||||
root: {
|
||||
dataset: {},
|
||||
classList: { remove() {} },
|
||||
removeCount: 0,
|
||||
remove() {
|
||||
this.removeCount++;
|
||||
order.push(`remove:${id}`);
|
||||
},
|
||||
},
|
||||
button: new Element(),
|
||||
providerSelect,
|
||||
providerField: {},
|
||||
status: {},
|
||||
detail: {},
|
||||
errorDetail: {},
|
||||
tierButton: {},
|
||||
costValue: {},
|
||||
helpDetail: {},
|
||||
card: {},
|
||||
};
|
||||
views.push(view);
|
||||
return view;
|
||||
}
|
||||
function passiveView() {
|
||||
return {
|
||||
events: [],
|
||||
outlines: [],
|
||||
destroyed: 0,
|
||||
handle(event) {
|
||||
this.events.push(event);
|
||||
},
|
||||
outline(event) {
|
||||
this.outlines.push(event);
|
||||
},
|
||||
destroy() {
|
||||
this.destroyed++;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
facade = createProviderCommands(
|
||||
{
|
||||
runner,
|
||||
pointer,
|
||||
referents,
|
||||
dataManager,
|
||||
signal: lifetime.signal,
|
||||
supportedProviders,
|
||||
createControl,
|
||||
createCard() {
|
||||
const card = passiveView();
|
||||
views.at(-1).cardView = card;
|
||||
return card;
|
||||
},
|
||||
createReticle() {
|
||||
const reticle = passiveView();
|
||||
views.at(-1).reticle = reticle;
|
||||
return reticle;
|
||||
},
|
||||
annotations: {
|
||||
onOutlineEvent(callback) {
|
||||
annotations.add(callback);
|
||||
return () => annotations.delete(callback);
|
||||
},
|
||||
},
|
||||
onSessionEvent(event) {
|
||||
events.push(event);
|
||||
onSessionEvent(event, facade);
|
||||
},
|
||||
},
|
||||
{
|
||||
factories,
|
||||
storage: {
|
||||
getItem() {
|
||||
if (storageDenied) throw Error('Storage denied');
|
||||
return savedProvider;
|
||||
},
|
||||
setItem(key, value) {
|
||||
if (storageDenied) throw Error('Storage denied');
|
||||
storageWrites.push([key, value]);
|
||||
savedProvider = value;
|
||||
},
|
||||
},
|
||||
bind(options) {
|
||||
// Keep the real callback so stale-event suppression tests exercise the
|
||||
// facade's mount generation as well as the common session's disposal.
|
||||
const mount = { options, callback: options.onSessionEvent };
|
||||
mounts.push(mount);
|
||||
mount.controls = bindVoiceCommands(options);
|
||||
return mount.controls;
|
||||
},
|
||||
},
|
||||
);
|
||||
return {
|
||||
facade,
|
||||
runner,
|
||||
pointer,
|
||||
referents,
|
||||
lifetime,
|
||||
instances,
|
||||
views,
|
||||
mounts,
|
||||
events,
|
||||
order,
|
||||
annotations,
|
||||
counters,
|
||||
storageWrites,
|
||||
state,
|
||||
select(provider) {
|
||||
const select = views.at(-1).providerSelect;
|
||||
select.value = provider;
|
||||
select.dispatchEvent(new Event('change'));
|
||||
},
|
||||
seedConversation() {
|
||||
pointer.beginTurn('text', {
|
||||
capturedSnapshot: { fresh: true, at: 'text' },
|
||||
});
|
||||
referents.recordResult('frame_overhead', {
|
||||
ok: true,
|
||||
display: { title: 'One aircraft' },
|
||||
referents: [
|
||||
{ n: 1, id: 'aaa111', label: 'Fixture', layerId: 'flights' },
|
||||
],
|
||||
});
|
||||
assert.ok(pointer.activeSnapshot());
|
||||
assert.equal(referents.list().length, 1);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('provider preference uses only the supplied host capability list', () => {
|
||||
const stored = (value) => ({ getItem: () => value });
|
||||
assert.equal(readVoiceProvider(stored('gemini'), ['openai']), 'openai');
|
||||
assert.equal(readVoiceProvider(stored('openai'), ['gemini']), 'gemini');
|
||||
assert.equal(
|
||||
readVoiceProvider(stored('unknown'), ['gemini', 'openai']),
|
||||
'openai',
|
||||
);
|
||||
assert.equal(
|
||||
readVoiceProvider(stored('gemini'), ['openai', 'gemini']),
|
||||
'gemini',
|
||||
);
|
||||
assert.equal(
|
||||
readVoiceProvider(
|
||||
{
|
||||
getItem() {
|
||||
throw Error('denied');
|
||||
},
|
||||
},
|
||||
['gemini'],
|
||||
),
|
||||
'gemini',
|
||||
);
|
||||
});
|
||||
|
||||
for (const scenario of [
|
||||
{
|
||||
title: 'OpenAI-only host ignores saved Gemini',
|
||||
requested: ['openai'],
|
||||
saved: 'gemini',
|
||||
expected: ['openai'],
|
||||
provider: 'openai',
|
||||
},
|
||||
{
|
||||
title: 'Gemini-only host falls back from saved OpenAI',
|
||||
requested: ['gemini'],
|
||||
saved: 'openai',
|
||||
expected: ['gemini'],
|
||||
provider: 'gemini',
|
||||
},
|
||||
{
|
||||
title: 'dual host restores Gemini',
|
||||
requested: ['openai', 'gemini'],
|
||||
saved: 'gemini',
|
||||
expected: ['openai', 'gemini'],
|
||||
provider: 'gemini',
|
||||
},
|
||||
{
|
||||
title: 'unknown, duplicate and unavailable factories are excluded',
|
||||
requested: ['unknown', 'gemini', 'openai', 'gemini'],
|
||||
factories: ['gemini'],
|
||||
saved: 'openai',
|
||||
expected: ['gemini'],
|
||||
provider: 'gemini',
|
||||
},
|
||||
]) {
|
||||
test(scenario.title, (t) => {
|
||||
const f = compositionFixture(t, {
|
||||
supportedProviders: scenario.requested,
|
||||
factoryProviders: scenario.factories,
|
||||
savedProvider: scenario.saved,
|
||||
});
|
||||
assert.deepEqual(f.facade.supportedProviders, scenario.expected);
|
||||
assert.equal(Object.isFrozen(f.facade.supportedProviders), true);
|
||||
assert.equal(f.facade.provider, scenario.provider);
|
||||
assert.deepEqual(
|
||||
f.views[0].providerSelect.options.map((option) => option.value),
|
||||
scenario.expected,
|
||||
);
|
||||
assert.equal(
|
||||
f.views[0].providerField.hidden,
|
||||
scenario.expected.length === 1,
|
||||
);
|
||||
assert.equal(
|
||||
f.views[0].root.dataset.providerChoice,
|
||||
String(scenario.expected.length > 1),
|
||||
);
|
||||
assert.equal(f.instances.length, 1);
|
||||
assert.equal(
|
||||
f.instances[0].starts.length,
|
||||
0,
|
||||
'discovery must not start a provider',
|
||||
);
|
||||
assert.equal(f.facade.state, 'idle');
|
||||
});
|
||||
}
|
||||
|
||||
test('invalid or empty host support fails before mounting a control', () => {
|
||||
let mounts = 0;
|
||||
const dependencies = {
|
||||
bind() {
|
||||
mounts++;
|
||||
},
|
||||
factories: { openai() {} },
|
||||
};
|
||||
assert.throws(
|
||||
() =>
|
||||
createProviderCommands({ supportedProviders: 'gemini' }, dependencies),
|
||||
/must be an array/,
|
||||
);
|
||||
assert.throws(
|
||||
() =>
|
||||
createProviderCommands(
|
||||
{ supportedProviders: ['gemini', 'unknown'] },
|
||||
dependencies,
|
||||
),
|
||||
/No supported voice provider/,
|
||||
);
|
||||
assert.equal(mounts, 0);
|
||||
});
|
||||
|
||||
test('active switch keeps PTT mode, retires the held key, and disposes before mounting', async (t) => {
|
||||
const f = compositionFixture(t);
|
||||
const firstSession = f.facade.session;
|
||||
const outerSignal = f.facade.signal;
|
||||
const first = f.instances[0];
|
||||
// PTT controls can activate the adapter directly; the facade must learn
|
||||
// activity from its common events, not only from facade.start().
|
||||
first.input.held = true;
|
||||
await first.adapter.start({ pushToTalk: true });
|
||||
assert.equal(first.microphoneEnabled, true);
|
||||
assert.equal(f.facade.isActive(), true);
|
||||
f.seedConversation();
|
||||
const before = { ...f.counters };
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
const second = f.instances[1];
|
||||
assert.equal(window.__gevVoiceCommands, f.facade);
|
||||
assert.equal(f.facade.runner, f.runner);
|
||||
assert.equal(f.facade.signal, outerSignal);
|
||||
assert.equal(outerSignal.aborted, false);
|
||||
assert.equal(firstSession.disposed, true);
|
||||
assert.equal(first.hooks.signal.aborted, true);
|
||||
assert.deepEqual(first.stops, [{ removeUi: true }]);
|
||||
assert.equal(first.input.held, false);
|
||||
assert.equal(first.microphoneEnabled, false);
|
||||
assert.deepEqual(second.starts, [{ pushToTalk: true }]);
|
||||
assert.equal(second.input.held, false);
|
||||
assert.equal(
|
||||
second.microphoneEnabled,
|
||||
false,
|
||||
'a held key must never migrate to the new adapter',
|
||||
);
|
||||
assert.equal(f.facade.state, 'listening');
|
||||
assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('control:1'));
|
||||
assert.ok(f.order.indexOf('remove:0') < f.order.indexOf('control:1'));
|
||||
assert.ok(f.order.indexOf('stop:0') < f.order.indexOf('factory:1'));
|
||||
assert.equal(f.views[0].button.listenerCount('click'), 0);
|
||||
assert.equal(f.views[0].providerSelect.listenerCount('change'), 0);
|
||||
assert.equal(f.views[1].providerSelect.listenerCount('change'), 1);
|
||||
assert.equal(f.views[1].providerSelect.focusCount, 1);
|
||||
assert.equal(f.views[1].tierButton.hidden, true);
|
||||
assert.equal(f.views[1].costValue.hidden, true);
|
||||
assert.equal(f.facade.providerDiagnostic(), 'gemini:1');
|
||||
assert.equal(f.pointer.activeSnapshot(), null);
|
||||
assert.deepEqual(f.referents.list(), []);
|
||||
for (const key of ['pointer', 'referents', 'conversation'])
|
||||
assert.equal(
|
||||
f.counters[key],
|
||||
before[key] + 1,
|
||||
`one explicit replacement reset: ${key}`,
|
||||
);
|
||||
assert.equal(
|
||||
f.counters.dispose,
|
||||
0,
|
||||
'provider replacement keeps the page runner',
|
||||
);
|
||||
assert.deepEqual(f.storageWrites, [[PROVIDER_KEY, 'gemini']]);
|
||||
assert.equal(f.facade.sendTextCommand('hello'), 'gemini:hello');
|
||||
assert.deepEqual(first.texts, []);
|
||||
assert.deepEqual(second.texts, ['hello']);
|
||||
const result = await f.facade.runner('adjust_camera_zoom', {
|
||||
direction: 'in',
|
||||
amount: 'little',
|
||||
});
|
||||
assert.equal(
|
||||
result.ok,
|
||||
true,
|
||||
'the actual shared action runner remains usable after switching',
|
||||
);
|
||||
assert.equal(f.state.zooms, 1);
|
||||
});
|
||||
|
||||
for (const initialState of ['idle', 'error']) {
|
||||
test(`${initialState} selection never autostarts; manual start still works`, async (t) => {
|
||||
const f = compositionFixture(t, { storageDenied: true });
|
||||
if (initialState === 'error') {
|
||||
await f.facade.start({ pushToTalk: true });
|
||||
f.instances[0].adapter.stop({ preserveStatus: true });
|
||||
f.instances[0].hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: 'Fixture failure',
|
||||
});
|
||||
}
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
assert.equal(
|
||||
f.facade.provider,
|
||||
'gemini',
|
||||
'denied storage cannot block a supported selection',
|
||||
);
|
||||
assert.equal(f.facade.state, 'idle');
|
||||
assert.deepEqual(f.instances[1].starts, []);
|
||||
f.select('gemini');
|
||||
f.select('unknown');
|
||||
assert.equal(
|
||||
f.instances.length,
|
||||
2,
|
||||
'same and unsupported selections cannot remount',
|
||||
);
|
||||
assert.equal(f.facade.provider, 'gemini');
|
||||
await f.facade.start({ pushToTalk: false });
|
||||
assert.deepEqual(f.instances[1].starts, [{ pushToTalk: false }]);
|
||||
assert.equal(f.instances[1].microphoneEnabled, true);
|
||||
});
|
||||
}
|
||||
|
||||
for (const outcome of ['resolve', 'reject']) {
|
||||
test(`retired startup ${outcome} and stale events cannot affect the current mount`, async (t) => {
|
||||
const f = compositionFixture(t, {
|
||||
gateStarts: (instance) => instance.id === 0,
|
||||
});
|
||||
const oldStart = f.facade.start({ pushToTalk: false });
|
||||
const old = f.instances[0];
|
||||
assert.equal(f.facade.state, 'connecting');
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
assert.equal(f.facade.state, 'listening');
|
||||
const current = f.instances[1];
|
||||
const eventCount = f.events.length;
|
||||
const currentStops = current.stops.length;
|
||||
const currentCardCount = f.views[1].cardView.events.length;
|
||||
old.gates[0][outcome](new Error('Late retired startup failure'));
|
||||
await oldStart;
|
||||
for (const state of ['idle', 'error', 'connecting']) {
|
||||
old.hooks.emit({ type: 'state', state, detail: 'retired adapter' });
|
||||
f.mounts[0].callback({
|
||||
type: 'state',
|
||||
state,
|
||||
detail: 'retired forward callback',
|
||||
});
|
||||
}
|
||||
assert.equal(f.events.length, eventCount);
|
||||
assert.equal(f.views[1].cardView.events.length, currentCardCount);
|
||||
assert.equal(f.facade.state, 'listening');
|
||||
assert.equal(old.acquired, 0);
|
||||
assert.equal(current.stops.length, currentStops);
|
||||
assert.equal(current.starts.length, 1);
|
||||
f.facade.stop();
|
||||
f.mounts[0].callback({ type: 'state', state: 'connecting' });
|
||||
f.select('openai');
|
||||
assert.deepEqual(
|
||||
f.instances[2].starts,
|
||||
[],
|
||||
'a retired connecting callback cannot revive voice intent',
|
||||
);
|
||||
assert.equal(f.facade.state, 'idle');
|
||||
});
|
||||
}
|
||||
|
||||
test('Stop during replacement startup defeats late settlement and further automatic resume', async (t) => {
|
||||
const f = compositionFixture(t, {
|
||||
gateStarts: (instance) => instance.id === 1,
|
||||
});
|
||||
await f.facade.start();
|
||||
f.select('gemini');
|
||||
const replacement = f.instances[1];
|
||||
assert.equal(f.facade.state, 'connecting');
|
||||
f.facade.stop();
|
||||
replacement.gates[0].resolve();
|
||||
await Promise.all(replacement.jobs);
|
||||
assert.equal(replacement.acquired, 0);
|
||||
assert.equal(f.facade.state, 'idle');
|
||||
f.select('openai');
|
||||
assert.deepEqual(f.instances[2].starts, []);
|
||||
await f.facade.start();
|
||||
assert.equal(
|
||||
f.facade.state,
|
||||
'listening',
|
||||
'a later explicit start remains healthy',
|
||||
);
|
||||
});
|
||||
|
||||
for (const emitsIdle of [false, true]) {
|
||||
test(`Stop and error clear pointer/referents/conversation once: adapter emitsIdle=${emitsIdle}`, async (t) => {
|
||||
const f = compositionFixture(t);
|
||||
if (emitsIdle) {
|
||||
const instance = f.instances[0];
|
||||
const originalStop = instance.adapter.stop;
|
||||
instance.adapter.stop = function (settings = {}) {
|
||||
originalStop.call(this, settings);
|
||||
if (!settings.preserveStatus && !settings.removeUi)
|
||||
instance.hooks.emit({ type: 'state', state: 'idle' });
|
||||
};
|
||||
}
|
||||
for (const end of ['stop', 'error']) {
|
||||
await f.facade.start();
|
||||
f.seedConversation();
|
||||
const before = { ...f.counters };
|
||||
const generation = f.referents.generation();
|
||||
if (end === 'stop') f.facade.stop();
|
||||
else {
|
||||
f.instances[0].adapter.stop({ preserveStatus: true });
|
||||
f.instances[0].hooks.emit({
|
||||
type: 'state',
|
||||
state: 'error',
|
||||
detail: 'Fixture failure',
|
||||
});
|
||||
}
|
||||
assert.equal(f.pointer.activeSnapshot(), null);
|
||||
assert.deepEqual(f.referents.list(), []);
|
||||
assert.equal(f.referents.generation(), generation + 1);
|
||||
for (const key of ['pointer', 'referents', 'conversation'])
|
||||
assert.equal(f.counters[key], before[key] + 1, `${end}: ${key}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('outer destruction disposes the runner once and removes each provider binding', async (t) => {
|
||||
const priorUnsaved = describeUnsavedWork();
|
||||
const f = compositionFixture(t);
|
||||
await f.facade.start();
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
assert.equal(f.annotations.size, 1);
|
||||
assert.ok(
|
||||
describeUnsavedWork().includes(translate('the active voice session')),
|
||||
);
|
||||
const oldCardEvents = f.views[0].cardView.events.length;
|
||||
for (const callback of f.annotations) callback({ id: 'fixture-outline' });
|
||||
assert.equal(f.views[0].cardView.outlines.length, 0);
|
||||
assert.equal(f.views[1].cardView.outlines.length, 1);
|
||||
assert.equal(f.instances[1].mapEvents.length, 1);
|
||||
f.facade.destroy();
|
||||
f.facade.destroy();
|
||||
f.facade.stop({ removeUi: true });
|
||||
f.lifetime.abort();
|
||||
assert.equal(f.counters.dispose, 1);
|
||||
assert.equal(f.facade.signal.aborted, true);
|
||||
assert.equal(f.facade.disposed, true);
|
||||
assert.equal(f.annotations.size, 0);
|
||||
assert.deepEqual(describeUnsavedWork(), priorUnsaved);
|
||||
const finalEvents = f.events.length;
|
||||
const starts = f.instances.map((instance) => instance.starts.length);
|
||||
for (const [index, view] of f.views.entries()) {
|
||||
assert.equal(view.root.removeCount, 1);
|
||||
assert.equal(view.cardView.destroyed, 1);
|
||||
assert.equal(view.reticle.destroyed, 1);
|
||||
assert.equal(view.button.listenerCount('click'), 0);
|
||||
assert.equal(view.providerSelect.listenerCount('change'), 0);
|
||||
assert.equal(f.instances[index].hooks.signal.aborted, true);
|
||||
view.button.dispatchEvent(new Event('click'));
|
||||
view.providerSelect.value = 'openai';
|
||||
view.providerSelect.dispatchEvent(new Event('change'));
|
||||
f.instances[index].hooks.emit({ type: 'state', state: 'connecting' });
|
||||
f.mounts[index].callback({ type: 'state', state: 'connecting' });
|
||||
}
|
||||
assert.equal(f.events.length, finalEvents);
|
||||
assert.equal(f.views[0].cardView.events.length, oldCardEvents);
|
||||
assert.deepEqual(
|
||||
f.instances.map((instance) => instance.starts.length),
|
||||
starts,
|
||||
);
|
||||
assert.equal(f.instances.length, 2);
|
||||
assert.equal(f.facade.start(), false);
|
||||
assert.equal(f.facade.sendText('after destroy'), false);
|
||||
assert.equal(f.facade.notifyMapEvent({}), false);
|
||||
const result = await f.facade.runner('adjust_camera_zoom', {
|
||||
direction: 'in',
|
||||
amount: 'little',
|
||||
});
|
||||
assert.equal(result.cancelled, true);
|
||||
assert.equal(
|
||||
f.state.zooms,
|
||||
0,
|
||||
'the actual runner cannot move the camera after outer disposal',
|
||||
);
|
||||
});
|
||||
|
||||
for (const action of ['stop', 'destroy']) {
|
||||
for (const phase of ['common', 'adapter']) {
|
||||
test(`${action} from replacement ${phase} connecting observer prevents resume`, async (t) => {
|
||||
let intercepted = 0;
|
||||
const f = compositionFixture(t, {
|
||||
onSessionEvent(event, facade) {
|
||||
if (
|
||||
facade?.provider !== 'gemini' ||
|
||||
event.type !== 'state' ||
|
||||
event.state !== 'connecting'
|
||||
)
|
||||
return;
|
||||
const matching =
|
||||
phase === 'common'
|
||||
? event.detail === undefined
|
||||
: event.detail === 'adapter:gemini';
|
||||
if (matching && intercepted++ === 0) facade[action]();
|
||||
},
|
||||
});
|
||||
await f.facade.start({ pushToTalk: true });
|
||||
f.select('gemini');
|
||||
await settle();
|
||||
const replacement = f.instances[1];
|
||||
assert.equal(intercepted, 1);
|
||||
assert.equal(replacement.starts.length, phase === 'common' ? 0 : 1);
|
||||
assert.equal(replacement.acquired, 0);
|
||||
assert.equal(replacement.microphoneEnabled, false);
|
||||
assert.equal(f.facade.isActive(), false);
|
||||
assert.equal(f.facade.state, 'idle');
|
||||
assert.equal(
|
||||
f.events.filter(
|
||||
(event) => event.state === 'listening' && event.detail === 'gemini',
|
||||
).length,
|
||||
0,
|
||||
);
|
||||
assert.equal(f.facade.disposed, action === 'destroy');
|
||||
assert.equal(f.counters.dispose, action === 'destroy' ? 1 : 0);
|
||||
if (action === 'stop') {
|
||||
f.select('openai');
|
||||
assert.deepEqual(
|
||||
f.instances[2].starts,
|
||||
[],
|
||||
'Stop cleared the intent to resume',
|
||||
);
|
||||
await f.facade.start({ pushToTalk: false });
|
||||
assert.equal(f.facade.state, 'listening');
|
||||
} else {
|
||||
f.select('openai');
|
||||
assert.equal(f.instances.length, 2);
|
||||
assert.equal(f.facade.start(), false);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
|
||||
const DISCONNECT_GRACE_MS = 6000;
|
||||
|
||||
function releaseStartResources({ localStream = null, localPc = null } = {}) {
|
||||
@@ -59,7 +61,7 @@ export class RealtimeConnection {
|
||||
this.input.pushToTalkKeyHeld = pushToTalkKeyHeld;
|
||||
this.input.spaceKeyHeld = spaceKeyHeld;
|
||||
if (!window.RTCPeerConnection || !navigator.mediaDevices?.getUserMedia) {
|
||||
this.setStatus('error', 'WebRTC microphone support unavailable');
|
||||
this.setStatus('error', t('WebRTC microphone support unavailable'));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -78,7 +80,7 @@ export class RealtimeConnection {
|
||||
// — this is what "applies next session" means.
|
||||
this.cost.prepareSession();
|
||||
this.syncCostUi();
|
||||
this.setStatus('connecting', 'Requesting microphone');
|
||||
this.setStatus('connecting', t('Requesting microphone'));
|
||||
this.debugLog('session.starting', {
|
||||
epoch,
|
||||
tier: this.cost.voiceTier,
|
||||
@@ -179,9 +181,9 @@ export class RealtimeConnection {
|
||||
if (!ownsChannel()) return;
|
||||
const detail = this.input.pushToTalkMode
|
||||
? this.input.pushToTalkKeyHeld
|
||||
? 'Release Space to send'
|
||||
: 'Hold Space to talk'
|
||||
: 'Ask or command';
|
||||
? t('Release Space to send')
|
||||
: t('Hold Space to talk')
|
||||
: t('Ask or command');
|
||||
this.setStatus('listening', detail);
|
||||
this.debugLog('data_channel.open', {
|
||||
connection: this.connectionDiagnostics(dataChannel),
|
||||
|
||||
@@ -10,14 +10,13 @@ import { TurnMetrics } from './turnMetrics.js';
|
||||
import { createNarrationScheduler } from './narration.js';
|
||||
import { createEarcon } from './earcon.js';
|
||||
import {
|
||||
POINTER_NONE_KEY,
|
||||
POINTER_OPEN_MIC_RECENT_MS,
|
||||
pointerChip,
|
||||
pointerContextItem,
|
||||
pointerAnnouncement,
|
||||
pointerTurnContext,
|
||||
pointerTurnEvent,
|
||||
} from './pointerContext.js';
|
||||
|
||||
/** Dedupe key of the clearing pointer_context item. */
|
||||
const POINTER_NONE_KEY = 'none';
|
||||
|
||||
import { shouldPauseRadioForVoice } from './realtimeProtocol.js';
|
||||
import { postDebugLog } from './realtimeDiagnostics.js';
|
||||
|
||||
@@ -55,13 +54,17 @@ export {
|
||||
|
||||
import { createRealtimeBackend } from './realtimeBackend.js';
|
||||
|
||||
const STATUS = {
|
||||
idle: 'OFF',
|
||||
connecting: 'CONNECTING',
|
||||
listening: 'LISTENING',
|
||||
executing: 'EXECUTING',
|
||||
error: 'ERROR',
|
||||
};
|
||||
import { t } from '../i18n/index.js';
|
||||
import { voiceStatusLabel } from './control.js';
|
||||
|
||||
// Machine states (dataset.status, CSS); voiceStatusLabel() translates labels.
|
||||
const STATUS_STATES = new Set([
|
||||
'idle',
|
||||
'connecting',
|
||||
'listening',
|
||||
'executing',
|
||||
'error',
|
||||
]);
|
||||
|
||||
/** Compose voice state owners and coordinate ordered session startup/teardown. */
|
||||
export class GevRealtimeController extends RealtimeFacade {
|
||||
@@ -283,31 +286,18 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
} catch {
|
||||
snapshot = null;
|
||||
}
|
||||
const chip = pointerChip(snapshot);
|
||||
this.pointerAnnounced = gesture && chip ? snapshot : null;
|
||||
this.emitSessionEvent({
|
||||
type: 'pointer',
|
||||
pointer:
|
||||
gesture && chip ? { ...chip, screenPx: snapshot.screenPx } : null,
|
||||
const event = pointerTurnEvent(snapshot, gesture);
|
||||
this.pointerAnnounced = event.announced;
|
||||
this.emitSessionEvent({ type: 'pointer', pointer: event.pointer });
|
||||
const context = pointerTurnContext(snapshot, {
|
||||
at,
|
||||
gesture,
|
||||
sentKey: this.pointerSentKey,
|
||||
});
|
||||
const item = !chip
|
||||
? this.pointerSentKey && this.pointerSentKey !== POINTER_NONE_KEY
|
||||
? { type: 'pointer_context', at, target: 'none' }
|
||||
: null
|
||||
: gesture
|
||||
? pointerContextItem(snapshot)
|
||||
: // What kind of thing, never which: no label or coordinates.
|
||||
{
|
||||
type: 'pointer_context',
|
||||
at,
|
||||
pointing: snapshot.entity?.kind || 'ground',
|
||||
};
|
||||
if (!item) return snapshot;
|
||||
const { at: _at, ...content } = item;
|
||||
const key = chip ? JSON.stringify(content) : POINTER_NONE_KEY;
|
||||
if (key === this.pointerSentKey) return snapshot;
|
||||
if (this._turns.notifyMapEvent(item)) {
|
||||
this.pointerSentKey = key;
|
||||
if (!context) return snapshot;
|
||||
if (this._turns.notifyMapEvent(context.item)) {
|
||||
this.pointerSentKey = context.key;
|
||||
const chip = context.key !== POINTER_NONE_KEY;
|
||||
this.debugLog('pointer.context', {
|
||||
at,
|
||||
target: chip ? snapshot.target : 'none',
|
||||
@@ -323,13 +313,10 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
*/
|
||||
announcePointer() {
|
||||
const snapshot = this.pointer?.activeSnapshot?.() || null;
|
||||
const chip = pointerChip(snapshot);
|
||||
if (!chip || snapshot === this.pointerAnnounced) return false;
|
||||
const pointer = pointerAnnouncement(snapshot, this.pointerAnnounced);
|
||||
if (!pointer) return false;
|
||||
this.pointerAnnounced = snapshot;
|
||||
this.emitSessionEvent({
|
||||
type: 'pointer',
|
||||
pointer: { ...chip, screenPx: snapshot.screenPx },
|
||||
});
|
||||
this.emitSessionEvent({ type: 'pointer', pointer });
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -412,7 +399,7 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
this.ui.root.remove();
|
||||
}
|
||||
if (!preserveStatus && !removeUi) {
|
||||
this.setStatus('idle', 'Voice off');
|
||||
this.setStatus('idle', t('Voice off'));
|
||||
}
|
||||
this.setRadioVoiceDucking(false);
|
||||
if (removeUi) this.emitSessionEvent({ type: 'disposed' });
|
||||
@@ -432,24 +419,25 @@ export class GevRealtimeController extends RealtimeFacade {
|
||||
this.ui.root.dataset.status = status;
|
||||
if (status === 'error') this.ui.root.classList.remove('error-dismissed');
|
||||
this.updateVoiceButtonLabel();
|
||||
this.ui.status.textContent = STATUS[status] || STATUS.idle;
|
||||
const shownStatus = STATUS_STATES.has(status) ? status : 'idle';
|
||||
this.ui.status.textContent = voiceStatusLabel(shownStatus);
|
||||
const resolvedDetail =
|
||||
status === 'listening' && this.pushToTalkMode
|
||||
? this.pushToTalkKeyHeld
|
||||
? 'Release Space to send'
|
||||
: 'Hold Space to talk'
|
||||
? t('Release Space to send')
|
||||
: t('Hold Space to talk')
|
||||
: detail;
|
||||
const primaryDetail =
|
||||
status === 'error'
|
||||
? 'VOICE UNAVAILABLE'
|
||||
? t('VOICE UNAVAILABLE')
|
||||
: resolvedDetail ||
|
||||
(status === 'idle' ? 'VOICE STANDBY' : 'VOICE ACTIVE');
|
||||
(status === 'idle' ? t('VOICE STANDBY') : t('VOICE ACTIVE'));
|
||||
this.ui.detail.textContent = primaryDetail;
|
||||
this.ui.detail.title = primaryDetail;
|
||||
if (this.ui.errorDetail) {
|
||||
this.ui.errorDetail.textContent =
|
||||
status === 'error'
|
||||
? resolvedDetail || 'Voice session could not be started.'
|
||||
? resolvedDetail || t('Voice session could not be started.')
|
||||
: '';
|
||||
}
|
||||
if (status === 'idle' || status === 'connecting' || status === 'error') {
|
||||
|
||||
+66
-17
@@ -1,3 +1,4 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
import {
|
||||
readStoredVoiceTier,
|
||||
readStoredVoiceLimits,
|
||||
@@ -45,10 +46,21 @@ export class RealtimeCost {
|
||||
*/
|
||||
syncCostUi() {
|
||||
const state = this.costTracker.state();
|
||||
const syncWarning = () => {
|
||||
const warning = this.ui?.costWarning;
|
||||
const value = this.ui?.costValue;
|
||||
if (!warning || !value) return;
|
||||
warning.hidden = !['warn', 'cap', 'unknown'].includes(
|
||||
value.dataset.level,
|
||||
);
|
||||
warning.textContent = warning.hidden ? '' : value.textContent;
|
||||
warning.title = value.title;
|
||||
warning.dataset.level = value.dataset.level;
|
||||
};
|
||||
const pendingTier = resolveVoiceModel(this.voiceTier).tier;
|
||||
const isMini = pendingTier === 'mini';
|
||||
if (this.ui?.tierButton) {
|
||||
this.ui.tierButton.textContent = isMini ? 'MINI' : 'STD';
|
||||
this.ui.tierButton.textContent = isMini ? t('MINI') : t('STD');
|
||||
this.ui.tierButton.setAttribute(
|
||||
'aria-pressed',
|
||||
isMini ? 'true' : 'false',
|
||||
@@ -56,33 +68,65 @@ export class RealtimeCost {
|
||||
const pendingId = resolveVoiceModel(pendingTier).id;
|
||||
this.ui.tierButton.title =
|
||||
this.isActive() && state.modelId !== pendingId
|
||||
? `Next session: ${pendingId} — this session stays on ${state.modelId}`
|
||||
: `Voice model: ${pendingId} — click to switch to ${
|
||||
isMini ? 'standard' : 'mini'
|
||||
}; applies next session`;
|
||||
? t('Next session: {pendingId} — this session stays on {modelId}', {
|
||||
pendingId,
|
||||
modelId: state.modelId,
|
||||
})
|
||||
: t(
|
||||
'Voice model: {pendingId} — click to switch to {target}; applies next session',
|
||||
{
|
||||
pendingId,
|
||||
target: isMini ? t('standard') : t('mini'),
|
||||
},
|
||||
);
|
||||
}
|
||||
if (this.ui?.costValue) {
|
||||
const liveAuth = this.sessionCloudVoiceAuth || this.cloudVoiceAuth;
|
||||
if (liveAuth === 'oauth') {
|
||||
const { responses, input, output, captions } = this.oauthUsage;
|
||||
this.ui.costValue.hidden = false;
|
||||
this.ui.costValue.textContent = 'COST UNKNOWN';
|
||||
this.ui.costValue.textContent = t('COST UNKNOWN');
|
||||
this.ui.costValue.dataset.level = 'unknown';
|
||||
this.ui.costValue.title =
|
||||
`ChatGPT OAuth session: ${responses} response(s), ${input} input and ${output} output tokens, ${captions} caption transcription(s) reported. ` +
|
||||
'USD cost and the API spend cap are unavailable for this auth mode.' +
|
||||
t(
|
||||
'ChatGPT OAuth session: {responses} response(s), {input} input and {output} output tokens, {captions} caption transcription(s) reported.',
|
||||
{ responses, input, output, captions },
|
||||
) +
|
||||
' ' +
|
||||
t(
|
||||
'USD cost and the API spend cap are unavailable for this auth mode.',
|
||||
) +
|
||||
(state.incomplete
|
||||
? ' Usage is incomplete because a response was still in flight when the session ended.'
|
||||
? ' ' +
|
||||
t(
|
||||
'Usage is incomplete because a response was still in flight when the session ended.',
|
||||
)
|
||||
: '');
|
||||
syncWarning();
|
||||
return;
|
||||
}
|
||||
this.ui.costValue.hidden = false;
|
||||
this.ui.costValue.textContent = state.display;
|
||||
this.ui.costValue.dataset.level = state.level;
|
||||
this.ui.costValue.title =
|
||||
`Estimated session cost on ${state.modelId} — ${state.responses} response(s). ` +
|
||||
`Warns at ${formatCostUsd(state.warnUsd)}, ends the session at ${formatCostUsd(state.capUsd)}.` +
|
||||
(state.note ? ` ${state.note}` : '');
|
||||
t(
|
||||
'Estimated session cost on {modelId} — {responses} response(s). Warns at {warn}, ends the session at {cap}.',
|
||||
{
|
||||
modelId: state.modelId,
|
||||
responses: state.responses,
|
||||
warn: formatCostUsd(state.warnUsd),
|
||||
cap: formatCostUsd(state.capUsd),
|
||||
},
|
||||
) +
|
||||
// `state.note` is the tracker's English diagnostic; the tooltip says
|
||||
// the same in the interface language.
|
||||
(state.incomplete
|
||||
? ' ' +
|
||||
t(
|
||||
'Estimate is incomplete — a response was still in flight when the session ended, so its usage was never reported.',
|
||||
)
|
||||
: '');
|
||||
syncWarning();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,10 +167,12 @@ export class RealtimeCost {
|
||||
}
|
||||
this.syncCostUi();
|
||||
if (this.isActive() && this.ui?.detail) {
|
||||
this.setStatus(
|
||||
this.status,
|
||||
`${this.voiceTier.toUpperCase()} applies next session`,
|
||||
);
|
||||
// The tier as the interface names it; the stored value stays English.
|
||||
const tier =
|
||||
resolveVoiceModel(this.voiceTier).tier === 'mini'
|
||||
? t('MINI')
|
||||
: t('STANDARD');
|
||||
this.setStatus(this.status, t('{tier} applies next session', { tier }));
|
||||
}
|
||||
return this.voiceTier;
|
||||
}
|
||||
@@ -240,7 +286,10 @@ export class RealtimeCost {
|
||||
try {
|
||||
this.stop({ preserveStatus: true });
|
||||
} finally {
|
||||
this.setStatus('idle', `Session ended — cost cap ${state.display}`);
|
||||
this.setStatus(
|
||||
'idle',
|
||||
t('Session ended — cost cap {cost}', { cost: state.display }),
|
||||
);
|
||||
this.syncCostUi();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
// The API-key cost tooltip states an incomplete estimate in the
|
||||
// interface language; the tracker keeps its English note for diagnostics.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { setLocale } from '../i18n/index.js';
|
||||
import { RealtimeCost } from './realtimeCost.js';
|
||||
import { createVoiceCostTracker } from './voiceCost.js';
|
||||
|
||||
for (const locale of ['es', 'ru']) {
|
||||
test(`API-key cost tooltip for an incomplete session has no English (${locale})`, () => {
|
||||
setLocale(locale);
|
||||
try {
|
||||
const costTracker = createVoiceCostTracker({ tier: 'standard' });
|
||||
costTracker.markIncomplete();
|
||||
const host = {
|
||||
costTracker,
|
||||
voiceTier: 'standard',
|
||||
cloudVoiceAuth: 'api-key',
|
||||
sessionCloudVoiceAuth: 'api-key',
|
||||
isActive: () => false,
|
||||
ui: {
|
||||
costValue: { hidden: true, dataset: {}, title: '', textContent: '' },
|
||||
},
|
||||
};
|
||||
RealtimeCost.prototype.syncCostUi.call(host);
|
||||
const title = host.ui.costValue.title;
|
||||
assert.ok(title.length > 0);
|
||||
assert.doesNotMatch(
|
||||
title,
|
||||
/Estimate is incomplete|still in flight/,
|
||||
title,
|
||||
);
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('API-key cost tooltip in English still ends with the tracker note', () => {
|
||||
const costTracker = createVoiceCostTracker({ tier: 'standard' });
|
||||
costTracker.markIncomplete();
|
||||
const host = {
|
||||
costTracker,
|
||||
voiceTier: 'standard',
|
||||
cloudVoiceAuth: 'api-key',
|
||||
sessionCloudVoiceAuth: 'api-key',
|
||||
isActive: () => false,
|
||||
ui: {
|
||||
costValue: { hidden: true, dataset: {}, title: '', textContent: '' },
|
||||
},
|
||||
};
|
||||
RealtimeCost.prototype.syncCostUi.call(host);
|
||||
assert.ok(host.ui.costValue.title.endsWith(` ${costTracker.state().note}`));
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createVoiceControl } from './control.js';
|
||||
import { RealtimeCost } from './realtimeCost.js';
|
||||
import { createVoiceCostTracker, resolveVoiceModel } from './voiceCost.js';
|
||||
import { installPanelDom } from './panel/testDom.mjs';
|
||||
|
||||
function harness(t) {
|
||||
installPanelDom(t);
|
||||
const ui = createVoiceControl({ panel: true });
|
||||
const stops = [];
|
||||
const statuses = [];
|
||||
const cost = new RealtimeCost({
|
||||
readUi: () => ui,
|
||||
readStatus: () => 'listening',
|
||||
operations: {
|
||||
isActive: () => true,
|
||||
stop: (options) => stops.push(options),
|
||||
setStatus: (...args) => statuses.push(args),
|
||||
debugLog() {},
|
||||
},
|
||||
});
|
||||
cost.cloudVoiceAuth = 'api-key';
|
||||
cost.costTracker = createVoiceCostTracker({
|
||||
tier: 'standard',
|
||||
limits: { warnUsd: 2, capUsd: 5 },
|
||||
});
|
||||
const recordThreeDollars = () => {
|
||||
const tokens = Math.ceil(
|
||||
(3 * 1_000_000) / resolveVoiceModel('standard').rates.textOutput,
|
||||
);
|
||||
const state = cost.costTracker.record({
|
||||
output_tokens: tokens,
|
||||
output_token_details: { text_tokens: tokens },
|
||||
});
|
||||
cost.applyCost(state);
|
||||
};
|
||||
t.mock.method(console, 'warn', () => {});
|
||||
return { cost, ui, stops, statuses, recordThreeDollars };
|
||||
}
|
||||
|
||||
test('soft cost warning is visible outside closed Settings, while ordinary spend stays inside', (t) => {
|
||||
const { cost, ui, recordThreeDollars } = harness(t);
|
||||
cost.syncCostUi();
|
||||
assert.equal(ui.voicePanel.hidden, true);
|
||||
assert.equal(ui.costValue.checkVisibility(), false);
|
||||
assert.equal(ui.costWarning.hidden, true);
|
||||
assert.equal(ui.voicePanel.contains(ui.costWarning), false);
|
||||
assert.equal(ui.root.querySelectorAll('#gev-voice-cost-value').length, 1);
|
||||
|
||||
recordThreeDollars();
|
||||
assert.equal(ui.costWarning.checkVisibility(), true);
|
||||
assert.equal(ui.costWarning.dataset.level, 'warn');
|
||||
assert.equal(ui.costWarning.textContent, ui.costValue.textContent);
|
||||
assert.equal(ui.costWarning.title, ui.costValue.title);
|
||||
assert.match(ui.costWarning.title, /Warns at/);
|
||||
assert.equal(
|
||||
ui.voicePanel.hidden,
|
||||
true,
|
||||
'warning never forces Settings open',
|
||||
);
|
||||
|
||||
cost.prepareSession();
|
||||
cost.syncCostUi();
|
||||
assert.equal(
|
||||
ui.costWarning.hidden,
|
||||
true,
|
||||
'new session clears the old warning',
|
||||
);
|
||||
assert.equal(ui.costWarning.textContent, '');
|
||||
});
|
||||
|
||||
test('closed Settings cannot hide the hard cap or suppress its ordinary stop and status', (t) => {
|
||||
const { ui, stops, statuses, recordThreeDollars } = harness(t);
|
||||
recordThreeDollars();
|
||||
assert.equal(stops.length, 0);
|
||||
recordThreeDollars();
|
||||
assert.equal(ui.costWarning.checkVisibility(), true);
|
||||
assert.equal(ui.costWarning.dataset.level, 'cap');
|
||||
assert.deepEqual(stops, [{ preserveStatus: true }]);
|
||||
assert.equal(statuses.at(-1)[0], 'idle');
|
||||
assert.match(statuses.at(-1)[1], /Session ended — cost cap/);
|
||||
});
|
||||
|
||||
test('unknown OAuth cost remains visible outside Settings and clears on a fresh API meter', (t) => {
|
||||
const { cost, ui } = harness(t);
|
||||
cost.sessionCloudVoiceAuth = 'oauth';
|
||||
cost.syncCostUi();
|
||||
assert.equal(ui.costWarning.checkVisibility(), true);
|
||||
assert.equal(ui.costWarning.dataset.level, 'unknown');
|
||||
assert.equal(ui.costWarning.textContent, 'COST UNKNOWN');
|
||||
assert.match(
|
||||
ui.costWarning.title,
|
||||
/USD cost and the API spend cap are unavailable/,
|
||||
);
|
||||
assert.equal(ui.voicePanel.hidden, true);
|
||||
cost.sessionCloudVoiceAuth = 'api-key';
|
||||
cost.syncCostUi();
|
||||
assert.equal(ui.costWarning.hidden, true);
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
// The status after a tier switch names the tier in the interface
|
||||
// language, as the tier button does; the stored tier value stays English.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { setLocale, t } from '../i18n/index.js';
|
||||
import { RealtimeCost } from './realtimeCost.js';
|
||||
|
||||
function harness() {
|
||||
const button = { textContent: '', title: '', setAttribute() {} };
|
||||
const ui = { tierButton: button, detail: {} };
|
||||
const statusCalls = [];
|
||||
const cost = new RealtimeCost({
|
||||
readUi: () => ui,
|
||||
readStatus: () => 'listening',
|
||||
operations: {
|
||||
isActive: () => true,
|
||||
isVoiceSessionSettled: () => false,
|
||||
setStatus: (status, detail) => statusCalls.push(detail),
|
||||
},
|
||||
});
|
||||
return { cost, button, statusCalls };
|
||||
}
|
||||
|
||||
const EXPECTED = {
|
||||
en: {
|
||||
mini: 'MINI applies next session',
|
||||
standard: 'STANDARD applies next session',
|
||||
},
|
||||
ru: {
|
||||
mini: 'МИНИ — со следующей сессии',
|
||||
standard: 'СТАНДАРТ — со следующей сессии',
|
||||
},
|
||||
es: {
|
||||
mini: 'MINI desde la próxima sesión',
|
||||
standard: 'ESTÁNDAR desde la próxima sesión',
|
||||
},
|
||||
};
|
||||
|
||||
for (const locale of Object.keys(EXPECTED)) {
|
||||
test(`${locale}: tier status names the tier in the interface language`, (ctx) => {
|
||||
setLocale(locale);
|
||||
ctx.after(() => setLocale('en'));
|
||||
for (const tier of ['mini', 'standard']) {
|
||||
const { cost, button, statusCalls } = harness();
|
||||
assert.equal(cost.setVoiceTier(tier), tier, 'stored tier stays English');
|
||||
assert.equal(statusCalls.at(-1), EXPECTED[locale][tier]);
|
||||
assert.equal(button.textContent, tier === 'mini' ? t('MINI') : t('STD'));
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
export const ERROR_LOG_LIMIT = 30;
|
||||
const ERROR_LOG_LIMIT = 30;
|
||||
|
||||
export const ERROR_STORAGE_KEY = 'gev-realtime-errors';
|
||||
const ERROR_STORAGE_KEY = 'gev-realtime-errors';
|
||||
|
||||
export const DEBUG_LOG_URL = '/api/realtime/debug-log';
|
||||
const DEBUG_LOG_URL = '/api/realtime/debug-log';
|
||||
|
||||
export function createDebugSessionId() {
|
||||
function createDebugSessionId() {
|
||||
const randomPart = Math.random().toString(36).slice(2, 10);
|
||||
return `gev-${Date.now().toString(36)}-${randomPart}`;
|
||||
}
|
||||
@@ -47,7 +47,7 @@ export function sanitizeDebugValue(value, depth = 0) {
|
||||
return output;
|
||||
}
|
||||
|
||||
export function sanitizeDebugString(value) {
|
||||
function sanitizeDebugString(value) {
|
||||
if (value.startsWith('data:image/')) {
|
||||
return `[Redacted image data URL, ${value.length} chars]`;
|
||||
}
|
||||
@@ -65,13 +65,13 @@ export function sanitizeDebugString(value) {
|
||||
: redacted;
|
||||
}
|
||||
|
||||
export function isSecretLikeKey(key) {
|
||||
function isSecretLikeKey(key) {
|
||||
return /(?:api[_-]?key|authorization|bearer|client[_-]?secret|token|secret|password)/i.test(
|
||||
key,
|
||||
);
|
||||
}
|
||||
|
||||
export function createErrorRecord(source, error, extra = {}) {
|
||||
function createErrorRecord(source, error, extra = {}) {
|
||||
const rtcError = error?.error || error;
|
||||
return {
|
||||
timestamp: new Date().toISOString(),
|
||||
@@ -90,7 +90,7 @@ export function createErrorRecord(source, error, extra = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
export function formatErrorForDisplay(record) {
|
||||
function formatErrorForDisplay(record) {
|
||||
const primary = [record.source, record.message].filter(Boolean).join(': ');
|
||||
const state = [
|
||||
record.errorDetail && `detail=${record.errorDetail}`,
|
||||
@@ -105,7 +105,7 @@ export function formatErrorForDisplay(record) {
|
||||
return state ? `${primary}\n${state}` : primary;
|
||||
}
|
||||
|
||||
export function removeEmptyValues(value) {
|
||||
function removeEmptyValues(value) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value || {}).filter(
|
||||
([, item]) => item !== null && item !== undefined && item !== '',
|
||||
@@ -113,14 +113,7 @@ export function removeEmptyValues(value) {
|
||||
);
|
||||
}
|
||||
|
||||
export function compactText(value, maxLength) {
|
||||
const text = String(value || '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
return text.length <= maxLength ? text : `${text.slice(0, maxLength - 1)}…`;
|
||||
}
|
||||
|
||||
export function loadStoredErrors() {
|
||||
function loadStoredErrors() {
|
||||
try {
|
||||
const value = JSON.parse(localStorage.getItem(ERROR_STORAGE_KEY) || '[]');
|
||||
return Array.isArray(value) ? value.slice(0, ERROR_LOG_LIMIT) : [];
|
||||
@@ -129,7 +122,7 @@ export function loadStoredErrors() {
|
||||
}
|
||||
}
|
||||
|
||||
export function storeErrors(errors) {
|
||||
function storeErrors(errors) {
|
||||
try {
|
||||
localStorage.setItem(
|
||||
ERROR_STORAGE_KEY,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
import {
|
||||
PUSH_TO_TALK_HOLD_DELAY_MS,
|
||||
MICROPHONE_VISUALIZER_GATE,
|
||||
@@ -118,7 +119,7 @@ export class RealtimeInput {
|
||||
this.ui.root.dataset.pushToTalk = 'held';
|
||||
this.setMicrophoneEnabled(true);
|
||||
if (this.status === 'listening')
|
||||
this.setStatus('listening', 'Release Space to send');
|
||||
this.setStatus('listening', t('Release Space to send'));
|
||||
} else {
|
||||
this.start({ pushToTalk: true });
|
||||
// start() performs a controlled stop() before connecting. Restore the
|
||||
@@ -194,7 +195,7 @@ export class RealtimeInput {
|
||||
if (!this.pushToTalkMode) return;
|
||||
this.setMicrophoneEnabled(false);
|
||||
if (this.status === 'listening')
|
||||
this.setStatus('listening', 'Hold Space to talk');
|
||||
this.setStatus('listening', t('Hold Space to talk'));
|
||||
else this.updateVoiceButtonLabel();
|
||||
}
|
||||
|
||||
@@ -363,7 +364,8 @@ export class RealtimeInput {
|
||||
*/
|
||||
updateVoiceButtonLabel() {
|
||||
if (!this.ui.buttonLabel) return;
|
||||
this.ui.buttonLabel.textContent = 'MIC';
|
||||
// The mic button's caption; the first-run tip names it the same way.
|
||||
this.ui.buttonLabel.textContent = t('MIC');
|
||||
if (this.ui.helpDetail) {
|
||||
this.ui.helpDetail.textContent = resolveVoiceControlHint(
|
||||
this.pushToTalkMode,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
|
||||
export const PUSH_TO_TALK_HOLD_DELAY_MS = 500;
|
||||
|
||||
// The input meter is intentionally stricter than the assistant-output meter:
|
||||
@@ -150,8 +152,8 @@ export function resolveVoiceVisualizerSpeaker(
|
||||
*/
|
||||
export function resolveVoiceControlHint(pushToTalkMode, pushToTalkKeyHeld) {
|
||||
return pushToTalkMode && pushToTalkKeyHeld
|
||||
? 'Release Space to send'
|
||||
: 'Hold Space to speak · tap Space to activate focused controls';
|
||||
? t('Release Space to send')
|
||||
: t('Hold Space to speak · tap Space to activate focused controls');
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -135,7 +135,7 @@ test('an old progress completion leaves the active in-band response alone', asyn
|
||||
});
|
||||
|
||||
test('user speech stops an audible progress line: cancel and clear its audio', async () => {
|
||||
const { controller, sent, event, progressLine } = controllerFixture();
|
||||
const { sent, event, progressLine } = controllerFixture();
|
||||
await progressLine('resp_p');
|
||||
await event({ type: 'output_audio_buffer.started', response_id: 'resp_p' });
|
||||
sent.length = 0;
|
||||
|
||||
@@ -8,12 +8,12 @@ import {
|
||||
|
||||
// Voice cost control (repo-wide `godsEyeView.<feature>.<field>` convention;
|
||||
// the neighbouring ERROR_STORAGE_KEY predates it).
|
||||
export const VOICE_TIER_STORAGE_KEY = 'godsEyeView.voiceCost.tier';
|
||||
const VOICE_TIER_STORAGE_KEY = 'godsEyeView.voiceCost.tier';
|
||||
|
||||
export const VOICE_LIMITS_STORAGE_KEY = 'godsEyeView.voiceCost.limits';
|
||||
const VOICE_LIMITS_STORAGE_KEY = 'godsEyeView.voiceCost.limits';
|
||||
|
||||
/** Best-effort localStorage handle; absent in tests and locked-down browsers. */
|
||||
export function voiceStorage(storage) {
|
||||
function voiceStorage(storage) {
|
||||
if (storage) return storage;
|
||||
try {
|
||||
return typeof localStorage === 'undefined' ? null : localStorage;
|
||||
|
||||
@@ -99,6 +99,18 @@ export function hasStructuredViewIdentity(result) {
|
||||
);
|
||||
}
|
||||
|
||||
export function isGlobeViewRefusal(result) {
|
||||
return (
|
||||
result?.action === 'move_camera' &&
|
||||
result.ok === false &&
|
||||
[
|
||||
'globe-view-required',
|
||||
'globe-reset-unavailable',
|
||||
'globe-reset-failed',
|
||||
].includes(result.reason)
|
||||
);
|
||||
}
|
||||
|
||||
export function responseInstructionForToolResult(result) {
|
||||
if (result?.action === 'control_radio' && result.radioPlaybackSuppressed) {
|
||||
if (result.audioState === 'paused') {
|
||||
@@ -123,6 +135,20 @@ export function responseInstructionForToolResult(result) {
|
||||
: 'Be concise: useful names, place, layer and one or two notable properties.',
|
||||
].join(' ');
|
||||
}
|
||||
if (isGlobeViewRefusal(result)) {
|
||||
// A refusal is not a completed action. Keep tool text out of instructions.
|
||||
if (result.reason === 'globe-reset-failed')
|
||||
return 'The globe did not rotate after the Globe View reset attempt. Say so briefly. Do not retry, offer or perform another reset for this request.';
|
||||
if (result.reason === 'globe-reset-unavailable')
|
||||
return 'The globe did not rotate. Ask the user to zoom out further; the Globe View preset does not fit this viewport. Do not offer or perform a reset.';
|
||||
return 'The globe did not rotate. Say "Globe rotation isn’t available from this view. You can say ‘Reset to Globe View and rotate the globe.’" The suggested command is not a user request. Do not ask a question, perform a reset, or call tools in this reply.';
|
||||
}
|
||||
if (
|
||||
(result?.action ?? result?.tool) === 'zoom_to_globe' &&
|
||||
(result.ok !== true || result.cancelled)
|
||||
) {
|
||||
return 'The Globe View reset did not finish. Say so briefly; do not claim Globe View was reached. Do not retry the reset or rotate the globe for this request.';
|
||||
}
|
||||
if (result?.action === 'adjust_camera_zoom') {
|
||||
return result.ok
|
||||
? `Confirm once that the camera zoomed ${result.direction}. Do not claim any other change.`
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { t } from '../i18n/index.js';
|
||||
import {
|
||||
silenceRadioForVoice,
|
||||
startPreparedRadioAfterPlaybackReady,
|
||||
@@ -246,7 +247,9 @@ export class RealtimeRadio {
|
||||
handoffChannel?.readyState === 'open',
|
||||
},
|
||||
);
|
||||
const stillCurrent = handoffEpoch === this.radioHandoffEpoch;
|
||||
const stillCurrent =
|
||||
handoffEpoch === this.radioHandoffEpoch &&
|
||||
this.radioHandoffAttemptId === handoffAttemptId;
|
||||
if (this.radioHandoffAttemptId === handoffAttemptId) {
|
||||
this.radioHandoffInFlight = false;
|
||||
this.radioHandoffAttemptId = null;
|
||||
@@ -258,13 +261,32 @@ export class RealtimeRadio {
|
||||
if (radioHandoff.result?.ok || radioHandoff.cancelled || !stillCurrent)
|
||||
return;
|
||||
if (this.dc?.readyState === 'open' && !this.userTurnPending) {
|
||||
this.setStatus('listening', 'Radio did not start');
|
||||
this.setStatus('listening', t('Radio did not start'));
|
||||
this.queueResponseCreate(
|
||||
'Say exactly one short correction: “The Radio station could not start. Voice is still on.”',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Revoke one call's prepared/preflight result, preserving other owners. */
|
||||
cancelPlaybackResult(result) {
|
||||
if (!result) return;
|
||||
if (this.pendingRadioPlaybackResult === result) {
|
||||
this.pendingRadioPlaybackResult = null;
|
||||
this.radioHandoffDeferredByReservation = false;
|
||||
}
|
||||
if (this.radioHandoffInFlightResult !== result) return;
|
||||
const attemptId = this.radioHandoffAttemptId;
|
||||
const shouldStopPlayback = this.radioHandoffInFlight;
|
||||
this.radioHandoffInFlightResult = null;
|
||||
this.radioHandoffInFlight = false;
|
||||
this.radioHandoffAttemptId = null;
|
||||
// Keep the group epoch and sibling reservations; release ownership before
|
||||
// Stop can synchronously notify playback observers.
|
||||
if (shouldStopPlayback)
|
||||
this.radioLayer?.stopPlayback?.({ origin: 'voice-cleanup', attemptId });
|
||||
}
|
||||
|
||||
/** Invalidate delayed Radio work inside the requested authority scope. */
|
||||
abortRadioSiblingTools({ responseId = null, scope = 'all' } = {}) {
|
||||
for (const [controller, metadata] of this.activeRadioToolControllers) {
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { GevRealtimeController } from './realtimeController.js';
|
||||
|
||||
const tick = () => new Promise((resolve) => setImmediate(resolve));
|
||||
const playback = (id) => ({
|
||||
ok: true,
|
||||
action: 'control_radio',
|
||||
radioAction: 'play',
|
||||
radioPlaybackRequested: true,
|
||||
id,
|
||||
});
|
||||
|
||||
function fixture(t) {
|
||||
const attempts = [],
|
||||
stopped = [],
|
||||
corrections = [],
|
||||
states = [];
|
||||
const controller = new GevRealtimeController({
|
||||
ui: {
|
||||
root: {
|
||||
dataset: {},
|
||||
classList: { remove() {} },
|
||||
querySelectorAll: () => [],
|
||||
remove() {},
|
||||
},
|
||||
status: {},
|
||||
detail: {},
|
||||
errorDetail: {},
|
||||
},
|
||||
runner: async () => ({ ok: false }),
|
||||
debugSink: null,
|
||||
onSessionEvent: (event) => states.push(event),
|
||||
radioLayer: {
|
||||
setVoiceDucked() {},
|
||||
playForVoice: ({ attemptId }) =>
|
||||
new Promise((resolve, reject) =>
|
||||
attempts.push({ attemptId, resolve, reject }),
|
||||
),
|
||||
stopPlayback: (options) => stopped.push(options),
|
||||
},
|
||||
});
|
||||
controller.queueResponseCreate = (message) => corrections.push(message);
|
||||
controller.dc = { readyState: 'open', send() {}, close() {} };
|
||||
t.after(() => controller.stop({ removeUi: true }));
|
||||
return { controller, attempts, stopped, corrections, states };
|
||||
}
|
||||
|
||||
test('default OpenAI reservation silences rejected frozen preflight and resumes retained result after failure', async (t) => {
|
||||
const h = fixture(t),
|
||||
c = h.controller;
|
||||
const a = playback('A');
|
||||
c.pendingRadioPlaybackResult = a;
|
||||
const pending = c.startPendingRadioHandoff();
|
||||
const reservation = c.reserveRadioToolHandoff();
|
||||
assert.equal(c.pendingRadioPlaybackResult, a);
|
||||
assert.equal(c.radioHandoffInFlight, false);
|
||||
h.attempts[0].reject(new Error('obsolete frozen preflight'));
|
||||
await pending;
|
||||
assert.deepEqual(h.corrections, []);
|
||||
assert.equal(
|
||||
h.states.some((event) => event.detail === 'Radio did not start'),
|
||||
false,
|
||||
);
|
||||
c.settleRadioToolHandoffReservation(reservation, { commit: false });
|
||||
await tick();
|
||||
assert.equal(h.attempts.length, 2);
|
||||
assert.equal(c.radioHandoffInFlightResult, a);
|
||||
assert.notEqual(h.attempts[0].attemptId, h.attempts[1].attemptId);
|
||||
h.attempts[1].reject(new Error('current preflight failed'));
|
||||
await tick();
|
||||
assert.equal(h.corrections.length, 1);
|
||||
assert.equal(c.status, 'listening');
|
||||
});
|
||||
|
||||
test('default OpenAI group cancellation silences rejected preflight without blocking the next failure correction', async (t) => {
|
||||
const h = fixture(t),
|
||||
c = h.controller;
|
||||
c.pendingRadioPlaybackResult = playback('A');
|
||||
const old = c.startPendingRadioHandoff();
|
||||
c.cancelRadioHandoff();
|
||||
h.attempts[0].reject(new Error('cancelled preflight'));
|
||||
await old;
|
||||
assert.deepEqual(h.corrections, []);
|
||||
c.pendingRadioPlaybackResult = playback('B');
|
||||
const current = c.startPendingRadioHandoff();
|
||||
h.attempts[1].resolve(false);
|
||||
await current;
|
||||
assert.equal(h.corrections.length, 1);
|
||||
});
|
||||
|
||||
test('per-result cancellation retains sibling result and epoch and ignores late failure or duplicate cancellation', async (t) => {
|
||||
const h = fixture(t),
|
||||
c = h.controller;
|
||||
const a = playback('A'),
|
||||
b = playback('B');
|
||||
c.pendingRadioPlaybackResult = a;
|
||||
const old = c.startPendingRadioHandoff();
|
||||
const siblingEpoch = c.radioHandoffEpoch;
|
||||
c.pendingRadioPlaybackResult = b;
|
||||
c._radio.cancelPlaybackResult(a);
|
||||
c._radio.cancelPlaybackResult(a);
|
||||
c._radio.cancelPlaybackResult(playback('unknown'));
|
||||
assert.equal(c.radioHandoffEpoch, siblingEpoch);
|
||||
assert.equal(c.pendingRadioPlaybackResult, b);
|
||||
assert.equal(h.stopped.length, 1);
|
||||
h.attempts[0].reject(new Error('late cancelled preflight'));
|
||||
await old;
|
||||
assert.deepEqual(h.corrections, []);
|
||||
const current = c.startPendingRadioHandoff();
|
||||
const reservation = c.reserveRadioToolHandoff();
|
||||
c._radio.cancelPlaybackResult(b);
|
||||
assert.equal(c._radio.radioToolHandoffReservations.has(reservation), true);
|
||||
h.attempts[1].resolve(false);
|
||||
await current;
|
||||
assert.deepEqual(h.corrections, []);
|
||||
c.settleRadioToolHandoffReservation(reservation, { commit: false });
|
||||
await tick();
|
||||
assert.equal(h.attempts.length, 2, 'a cancelled result cannot be resumed');
|
||||
});
|
||||
@@ -16,6 +16,7 @@ export function createRealtimeSession({
|
||||
controller,
|
||||
capabilities: { costControls: true, pushToTalk: true },
|
||||
start: (settings) => controller.start(settings),
|
||||
getStartOptions: () => ({ pushToTalk: Boolean(controller.pushToTalkMode) }),
|
||||
stop: (settings) => controller.stop(settings),
|
||||
sendText: (text) => controller.sendTextCommand(text),
|
||||
sendMapEvent: (event) => controller.notifyMapEvent(event),
|
||||
|
||||
+236
-16
@@ -1,10 +1,12 @@
|
||||
import { realtimeSessionEvent } from './realtimeEvents.js';
|
||||
import { t } from '../i18n/index.js';
|
||||
import { readLayerLifecycleSummary } from './layerSummary.js';
|
||||
import {
|
||||
CALL_DEDUPE_MS,
|
||||
SUPERSEDED_RESPONSE_MEMORY,
|
||||
shouldStopVoiceAfterRadioTool,
|
||||
responseInstructionForToolResult,
|
||||
isGlobeViewRefusal,
|
||||
extractFunctionCalls,
|
||||
callDedupeKeys,
|
||||
parseArguments,
|
||||
@@ -12,7 +14,7 @@ import {
|
||||
import { narrationLabel } from './speech.js';
|
||||
|
||||
/** Out-of-band progress responses carry this metadata tag. */
|
||||
export const PROGRESS_RESPONSE_TAG = 'progress';
|
||||
const PROGRESS_RESPONSE_TAG = 'progress';
|
||||
|
||||
/**
|
||||
* Speaks one code-authored progress line out of band. The line travels as
|
||||
@@ -65,6 +67,11 @@ export class RealtimeTurns {
|
||||
this.pendingUserTextResponse = false;
|
||||
this.activeResponseId = null;
|
||||
this.supersededResponseIds = new Set();
|
||||
this.globeRefusals = new Map();
|
||||
this.globeAdmissionGates = new Map();
|
||||
this.globeResets = new Map();
|
||||
this.globeResponseRequests = new Map();
|
||||
this.globeRequest = { failed: false, pending: new Set() };
|
||||
this.activeToolAbortControllers = new Set();
|
||||
this.narration = null;
|
||||
this.narrationSequence = 0;
|
||||
@@ -136,7 +143,8 @@ export class RealtimeTurns {
|
||||
content: [{ type: 'input_text', text: cleanText }],
|
||||
},
|
||||
};
|
||||
this.sendRealtimeEvent(itemEvent, 'client.user_text');
|
||||
if (!this.sendRealtimeEvent(itemEvent, 'client.user_text')) return;
|
||||
this.globeRequest = { failed: false, pending: new Set() };
|
||||
this.requestUserTextResponse();
|
||||
}
|
||||
|
||||
@@ -277,7 +285,7 @@ export class RealtimeTurns {
|
||||
eventId: payload.event_id,
|
||||
activeResponseMessage: payload.error?.message || null,
|
||||
});
|
||||
this.setStatus('listening', 'Ask or command');
|
||||
this.setStatus('listening', t('Ask or command'));
|
||||
return;
|
||||
}
|
||||
// A conversation.item.delete for a stale viewport screenshot can land
|
||||
@@ -329,6 +337,7 @@ export class RealtimeTurns {
|
||||
this.userTurnPending = true;
|
||||
this.pendingResponseInstructions = null;
|
||||
this.cancelRadioHandoff({ abortTools: true });
|
||||
this.globeRequest = { failed: false, pending: new Set() };
|
||||
this.setVoiceSpeaker('user');
|
||||
// Point-and-ask: read the pointer as speech starts (open mic), or reuse
|
||||
// the push-to-talk keydown snapshot, before the user audio commits.
|
||||
@@ -383,6 +392,27 @@ export class RealtimeTurns {
|
||||
}
|
||||
|
||||
const toolResponseId = payload.response_id || payload.response?.id || null;
|
||||
// Follow-up responses retain reset ownership until fresh admitted input.
|
||||
const arrivalRequest = this.globeRequest;
|
||||
const globeRequest =
|
||||
this.globeResponseRequests.get(toolResponseId) ?? arrivalRequest;
|
||||
// Later siblings must wait for globe admission before they can mutate state.
|
||||
// Other tool responses keep their existing concurrency.
|
||||
let heldForAdmission = false;
|
||||
while (toolResponseId && this.globeAdmissionGates.has(toolResponseId)) {
|
||||
// Drop claimed duplicates before every wait, before their keys can expire.
|
||||
if (
|
||||
calls.every((call) =>
|
||||
callDedupeKeys(call).some((key) => this.processedCalls.has(key)),
|
||||
)
|
||||
)
|
||||
return;
|
||||
heldForAdmission = true;
|
||||
await this.globeAdmissionGates.get(toolResponseId).promise;
|
||||
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
|
||||
return;
|
||||
}
|
||||
|
||||
// A newer typed command superseded the response these calls belong to.
|
||||
// They are stale intent — dispatching one would let the old turn mutate
|
||||
// the map after the operator asked for something else.
|
||||
@@ -393,7 +423,10 @@ export class RealtimeTurns {
|
||||
// hazard `callDedupeKeys` is written to avoid). So each refused call gets
|
||||
// a terminal output saying plainly that the turn moved on. No
|
||||
// `response.create` follows — the deferred typed turn is the single answer.
|
||||
if (this.isSupersededResponse(toolResponseId)) {
|
||||
if (
|
||||
this.isSupersededResponse(toolResponseId) ||
|
||||
(heldForAdmission && arrivalRequest !== this.globeRequest)
|
||||
) {
|
||||
this.pruneProcessedCalls();
|
||||
for (const call of calls) {
|
||||
const keys = callDedupeKeys(call);
|
||||
@@ -414,7 +447,7 @@ export class RealtimeTurns {
|
||||
return;
|
||||
}
|
||||
|
||||
this.setStatus('executing', 'Running command');
|
||||
this.setStatus('executing', t('Running command'));
|
||||
this.pruneProcessedCalls();
|
||||
let sentOutput = false;
|
||||
let lastResult = null;
|
||||
@@ -428,7 +461,19 @@ export class RealtimeTurns {
|
||||
if (keys.some((key) => this.processedCalls.has(key))) continue;
|
||||
keys.forEach((key) => this.processedCalls.set(key, performance.now()));
|
||||
|
||||
if (this.globeRefusals.get(toolResponseId)?.blocking) {
|
||||
this.sendToolOutput(call.call_id || call.id, {
|
||||
ok: false,
|
||||
action: call.name,
|
||||
skipped: true,
|
||||
error: 'Not run: this response must wait for the Globe View reply.',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let result;
|
||||
let globeAdmissionGate = null;
|
||||
let globeResetGate = null;
|
||||
const resultChannel = this.dc;
|
||||
let radioHandoffEpochAtStart = this.radio.radioHandoffEpoch;
|
||||
let toolController = null;
|
||||
@@ -442,6 +487,80 @@ export class RealtimeTurns {
|
||||
this.narration?.preamble();
|
||||
try {
|
||||
const parsedArguments = parseArguments(call.arguments);
|
||||
if (
|
||||
toolResponseId &&
|
||||
call.name === 'move_camera' &&
|
||||
String(parsedArguments.motion || '').toLowerCase() === 'rotate_globe'
|
||||
) {
|
||||
let resolve;
|
||||
const promise = new Promise((settle) => {
|
||||
resolve = settle;
|
||||
});
|
||||
globeAdmissionGate = { promise, resolve };
|
||||
this.globeAdmissionGates.set(toolResponseId, globeAdmissionGate);
|
||||
// OpenAI delivers sibling calls concurrently. A reset dispatched
|
||||
// first must finish before rotation evaluates the camera's fit.
|
||||
// Hold later siblings at admission while this rotation waits.
|
||||
const resets = this.globeResets.get(toolResponseId);
|
||||
if (
|
||||
resets ||
|
||||
globeRequest.failed ||
|
||||
globeRequest.pending.size ||
|
||||
arrivalRequest !== this.globeRequest
|
||||
) {
|
||||
while (resets?.pending.size || globeRequest.pending.size)
|
||||
await Promise.all(
|
||||
[
|
||||
...new Set([
|
||||
...(resets?.pending || []),
|
||||
...globeRequest.pending,
|
||||
]),
|
||||
].map((gate) => gate.promise),
|
||||
);
|
||||
if (!this.ownsConversation(eventChannel) || this.isSessionEnding())
|
||||
return;
|
||||
keys.forEach((key) =>
|
||||
this.processedCalls.set(key, performance.now()),
|
||||
);
|
||||
const superseded =
|
||||
this.isSupersededResponse(toolResponseId) ||
|
||||
arrivalRequest !== this.globeRequest;
|
||||
if (superseded || resets?.failed || globeRequest.failed) {
|
||||
result = {
|
||||
ok: false,
|
||||
action: call.name,
|
||||
reason: 'globe-reset-failed',
|
||||
skipped: true,
|
||||
...(superseded ? { superseded: true } : {}),
|
||||
error: superseded
|
||||
? 'Superseded by a newer command from the operator — this call was not run.'
|
||||
: 'Not run: the Globe View reset/rotation sequence failed or was cancelled.',
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
if (toolResponseId && call.name === 'zoom_to_globe') {
|
||||
let resolve;
|
||||
const promise = new Promise((settle) => {
|
||||
resolve = settle;
|
||||
});
|
||||
let resets = this.globeResets.get(toolResponseId);
|
||||
if (!resets) {
|
||||
resets = { pending: new Set(), failed: false };
|
||||
this.globeResets.set(toolResponseId, resets);
|
||||
// Retain settled outcomes for later siblings, without evicting an
|
||||
// active reset that another response still needs to await.
|
||||
for (const [id, record] of this.globeResets) {
|
||||
if (this.globeResets.size <= SUPERSEDED_RESPONSE_MEMORY) break;
|
||||
if (!record.pending.size && id !== toolResponseId)
|
||||
this.globeResets.delete(id);
|
||||
}
|
||||
}
|
||||
globeResetGate = { promise, resolve, resets, request: globeRequest };
|
||||
resets.pending.add(globeResetGate);
|
||||
globeRequest.pending.add(globeResetGate);
|
||||
}
|
||||
|
||||
const isRadioControlCall = call.name === 'control_radio';
|
||||
const isRadioVisibilityCall =
|
||||
call.name === 'set_layer_visibility' &&
|
||||
@@ -498,7 +617,7 @@ export class RealtimeTurns {
|
||||
authorityDomain: radioAuthorityDomain,
|
||||
});
|
||||
}
|
||||
result = await (this.actionExecutor || this.runner)(
|
||||
result ??= await (this.actionExecutor || this.runner)(
|
||||
call.name,
|
||||
parsedArguments,
|
||||
{
|
||||
@@ -514,7 +633,28 @@ export class RealtimeTurns {
|
||||
radioHandoffEpochAtStart === this.radio.radioHandoffEpoch),
|
||||
},
|
||||
);
|
||||
if (
|
||||
globeAdmissionGate &&
|
||||
this.globeResets.has(toolResponseId) &&
|
||||
result?.reason === 'globe-view-required'
|
||||
)
|
||||
result = { ...result, reason: 'globe-reset-failed' };
|
||||
if (!this.ownsConversation(eventChannel)) return;
|
||||
if (
|
||||
isGlobeViewRefusal(result) &&
|
||||
toolResponseId &&
|
||||
arrivalRequest === this.globeRequest &&
|
||||
!this.isSupersededResponse(toolResponseId)
|
||||
) {
|
||||
this.globeRefusals.set(toolResponseId, {
|
||||
result,
|
||||
followupQueued: false,
|
||||
blocking: !result.skipped,
|
||||
});
|
||||
while (this.globeRefusals.size > SUPERSEDED_RESPONSE_MEMORY) {
|
||||
this.globeRefusals.delete(this.globeRefusals.keys().next().value);
|
||||
}
|
||||
}
|
||||
if (result?.ok && result.radioPlaybackRequested) {
|
||||
const sessionIsCurrent =
|
||||
this.activeToolAbortControllers.has(toolController) &&
|
||||
@@ -594,6 +734,23 @@ export class RealtimeTurns {
|
||||
: {}),
|
||||
};
|
||||
} finally {
|
||||
if (globeResetGate) {
|
||||
const { resets, request } = globeResetGate;
|
||||
if (result?.ok !== true || result.cancelled) {
|
||||
resets.failed = true;
|
||||
request.failed = true;
|
||||
}
|
||||
resets.pending.delete(globeResetGate);
|
||||
request.pending.delete(globeResetGate);
|
||||
globeResetGate.resolve();
|
||||
}
|
||||
if (globeAdmissionGate) {
|
||||
if (
|
||||
this.globeAdmissionGates.get(toolResponseId) === globeAdmissionGate
|
||||
)
|
||||
this.globeAdmissionGates.delete(toolResponseId);
|
||||
globeAdmissionGate.resolve();
|
||||
}
|
||||
if (toolController) {
|
||||
this.activeToolAbortControllers.delete(toolController);
|
||||
this.radio.releaseTool(toolController);
|
||||
@@ -670,15 +827,40 @@ export class RealtimeTurns {
|
||||
});
|
||||
}
|
||||
if (!this.ownsConversation(eventChannel)) return;
|
||||
// Keep the Radio handoff wording authoritative even when another tool
|
||||
// result follows Radio in the same multi-intent response.
|
||||
this.queueResponseCreate(
|
||||
responseInstructionForToolResult(
|
||||
this.radio.pendingRadioPlaybackResult || lastResult,
|
||||
),
|
||||
);
|
||||
const refusal = this.globeRefusals.get(toolResponseId);
|
||||
// A newer turn may arrive during admission or the viewport-image await.
|
||||
// Preserve its reply; neither the refusal map nor lastResult may revive
|
||||
// an old refusal follow-up after supersession.
|
||||
if (
|
||||
(this.isSupersededResponse(toolResponseId) ||
|
||||
arrivalRequest !== this.globeRequest) &&
|
||||
(refusal || isGlobeViewRefusal(lastResult) || lastResult?.superseded)
|
||||
) {
|
||||
// Complete an idle turn below, but leave a newer running tool's status
|
||||
// authoritative while suppressing this obsolete refusal follow-up.
|
||||
if (this.activeToolAbortControllers.size) return;
|
||||
} else if (refusal) {
|
||||
// Even an earlier sibling finishing late cannot replace the reset instruction.
|
||||
if (!refusal.followupQueued) {
|
||||
refusal.followupQueued = true;
|
||||
this.queueResponseCreate({
|
||||
instructions: responseInstructionForToolResult(refusal.result),
|
||||
tool_choice: 'none',
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Keep Radio handoff wording authoritative for ordinary multi-tool replies.
|
||||
const followupResult =
|
||||
this.radio.pendingRadioPlaybackResult || lastResult;
|
||||
const instructions = responseInstructionForToolResult(followupResult);
|
||||
this.queueResponseCreate(
|
||||
isGlobeViewRefusal(followupResult)
|
||||
? { instructions, tool_choice: 'none' }
|
||||
: instructions,
|
||||
);
|
||||
}
|
||||
}
|
||||
this.setStatus('listening', 'Ask or command');
|
||||
this.setStatus('listening', t('Ask or command'));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -965,6 +1147,22 @@ export class RealtimeTurns {
|
||||
this.userTurnPending = false;
|
||||
this.activeResponseId =
|
||||
payload.response?.id || payload.response_id || null;
|
||||
// Late calls retain only their globe reset/request ownership. Ordinary
|
||||
// tool admission and concurrency continue to use their existing policy.
|
||||
if (
|
||||
this.activeResponseId &&
|
||||
!this.globeResponseRequests.has(this.activeResponseId)
|
||||
) {
|
||||
this.globeResponseRequests.set(
|
||||
this.activeResponseId,
|
||||
this.globeRequest,
|
||||
);
|
||||
while (this.globeResponseRequests.size > SUPERSEDED_RESPONSE_MEMORY)
|
||||
this.globeResponseRequests.delete(
|
||||
this.globeResponseRequests.keys().next().value,
|
||||
);
|
||||
}
|
||||
|
||||
this.setVoiceSpeaker('ai');
|
||||
return;
|
||||
}
|
||||
@@ -997,7 +1195,7 @@ export class RealtimeTurns {
|
||||
// connection is still live. Recover to listening so the user can retry
|
||||
// (mirrors the transient-blip philosophy, H8).
|
||||
if (this.dc?.readyState === 'open') {
|
||||
this.setStatus('listening', 'Ask or command');
|
||||
this.setStatus('listening', t('Ask or command'));
|
||||
}
|
||||
}
|
||||
if (!this.radio.pendingRadioPlaybackResult) {
|
||||
@@ -1044,7 +1242,8 @@ export class RealtimeTurns {
|
||||
const sent = this.sendRealtimeEvent(
|
||||
{
|
||||
type: 'response.create',
|
||||
response: { instructions },
|
||||
response:
|
||||
typeof instructions === 'string' ? { instructions } : instructions,
|
||||
},
|
||||
'client.response_create.tool_followup',
|
||||
);
|
||||
@@ -1056,6 +1255,20 @@ export class RealtimeTurns {
|
||||
controller.abort();
|
||||
this.activeToolAbortControllers.clear();
|
||||
this.radio.clearTools();
|
||||
this.releaseGlobeResets();
|
||||
}
|
||||
|
||||
releaseGlobeResets() {
|
||||
for (const resets of this.globeResets.values()) {
|
||||
if (!resets.pending.size) continue;
|
||||
resets.failed = true;
|
||||
for (const gate of resets.pending) {
|
||||
gate.request.failed = true;
|
||||
gate.request.pending.delete(gate);
|
||||
gate.resolve();
|
||||
}
|
||||
resets.pending.clear();
|
||||
}
|
||||
}
|
||||
|
||||
reset() {
|
||||
@@ -1067,6 +1280,13 @@ export class RealtimeTurns {
|
||||
this.pendingUserTextResponse = false;
|
||||
this.activeResponseId = null;
|
||||
this.supersededResponseIds.clear();
|
||||
this.globeRefusals.clear();
|
||||
for (const gate of this.globeAdmissionGates.values()) gate.resolve();
|
||||
this.globeAdmissionGates.clear();
|
||||
this.releaseGlobeResets();
|
||||
this.globeResets.clear();
|
||||
this.globeResponseRequests.clear();
|
||||
this.globeRequest = { failed: false, pending: new Set() };
|
||||
this.narration?.cancel();
|
||||
this.assistantAudioPlaying = false;
|
||||
this.speechResponseIds.clear();
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { GevRealtimeController } from './realtimeController.js';
|
||||
|
||||
function fixture({ pointer = null, onUserSend = () => {} } = {}) {
|
||||
const sent = [];
|
||||
const executed = [];
|
||||
const rejectedTexts = new Set();
|
||||
let rejectResponse = false;
|
||||
const controller = new GevRealtimeController({
|
||||
runner: async (name) => {
|
||||
executed.push(name);
|
||||
return { ok: true };
|
||||
},
|
||||
backend: {},
|
||||
debugSink: null,
|
||||
pointer,
|
||||
ui: {
|
||||
root: {
|
||||
dataset: {},
|
||||
classList: { remove() {} },
|
||||
querySelectorAll: () => [],
|
||||
},
|
||||
status: {},
|
||||
detail: {},
|
||||
},
|
||||
});
|
||||
controller.dc = {
|
||||
readyState: 'open',
|
||||
send(body) {
|
||||
const value = JSON.parse(body);
|
||||
const text = value.item?.role === 'user' && value.item.content?.[0]?.text;
|
||||
if (value.item?.role === 'user') onUserSend();
|
||||
if (
|
||||
rejectedTexts.has(text) ||
|
||||
(rejectResponse && value.type === 'response.create')
|
||||
)
|
||||
throw new Error('Synthetic channel rejection');
|
||||
sent.push(value);
|
||||
},
|
||||
close() {
|
||||
this.readyState = 'closed';
|
||||
},
|
||||
};
|
||||
controller.status = 'listening';
|
||||
controller._turns.narration = null;
|
||||
const event = (payload) =>
|
||||
controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
|
||||
const created = (id) => event({ type: 'response.created', response: { id } });
|
||||
const done = (id) =>
|
||||
event({
|
||||
type: 'response.done',
|
||||
response: { id, status: 'completed', output: [] },
|
||||
});
|
||||
const creates = () =>
|
||||
sent.filter(
|
||||
(x) =>
|
||||
x.type === 'response.create' &&
|
||||
x.response?.metadata?.gev !== 'progress',
|
||||
);
|
||||
const texts = () =>
|
||||
sent
|
||||
.filter((x) => x.item?.role === 'user')
|
||||
.map((x) => x.item.content[0].text);
|
||||
return {
|
||||
controller,
|
||||
sent,
|
||||
executed,
|
||||
rejectedTexts,
|
||||
event,
|
||||
created,
|
||||
done,
|
||||
creates,
|
||||
texts,
|
||||
rejectResponses(value) {
|
||||
rejectResponse = value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('rejected idle text schedules no answer, and a successful successor answers once', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
f.rejectedTexts.add('rejected');
|
||||
assert.equal(f.controller.sendTextCommand('rejected'), undefined);
|
||||
assert.equal(f.creates().length, 0);
|
||||
assert.equal(f.controller.pendingUserTextResponse, false);
|
||||
assert.equal(f.controller.responseCreatePending, false);
|
||||
assert.equal(f.controller.sendTextCommand('successor'), undefined);
|
||||
assert.deepEqual(f.texts(), ['successor']);
|
||||
assert.equal(f.creates().length, 1);
|
||||
await f.created('successor');
|
||||
await f.done('successor');
|
||||
assert.equal(f.creates().length, 1);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejected text behind an active response cannot queue an answer', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
await f.created('active');
|
||||
f.controller.queueResponseCreate('Old tool follow-up');
|
||||
assert.equal(
|
||||
f.controller.pendingResponseInstructions,
|
||||
'Old tool follow-up',
|
||||
);
|
||||
const runningTool = new AbortController();
|
||||
f.controller._turns.activeToolAbortControllers.add(runningTool);
|
||||
f.rejectedTexts.add('rejected');
|
||||
f.controller.sendTextCommand('rejected');
|
||||
assert.equal(f.controller.pendingUserTextResponse, false);
|
||||
assert.equal(f.controller.pendingResponseInstructions, null);
|
||||
assert.equal(f.controller._turns.isSupersededResponse('active'), true);
|
||||
assert.equal(runningTool.signal.aborted, true);
|
||||
const staleCall = {
|
||||
type: 'response.function_call_arguments.done',
|
||||
response_id: 'active',
|
||||
call_id: 'stale-call',
|
||||
name: 'fly_to_location',
|
||||
arguments: '{"query":"Paris"}',
|
||||
};
|
||||
await f.event(staleCall);
|
||||
await f.event(staleCall);
|
||||
assert.deepEqual(f.executed, []);
|
||||
const outputs = f.sent.filter(
|
||||
(x) => x.item?.type === 'function_call_output',
|
||||
);
|
||||
assert.equal(outputs.length, 1);
|
||||
assert.equal(outputs[0].item.call_id, 'stale-call');
|
||||
assert.equal(JSON.parse(outputs[0].item.output).superseded, true);
|
||||
await f.done('active');
|
||||
assert.equal(f.creates().length, 0);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejected text still prepares narration and pointer context before sending', () => {
|
||||
const steps = [];
|
||||
const f = fixture({
|
||||
pointer: {
|
||||
beginTurn() {
|
||||
steps.push('pointer');
|
||||
return null;
|
||||
},
|
||||
clear() {},
|
||||
},
|
||||
onUserSend() {
|
||||
steps.push('send');
|
||||
},
|
||||
});
|
||||
f.controller._turns.narration = {
|
||||
cancel() {
|
||||
steps.push('cancel');
|
||||
},
|
||||
userTurnEnded() {
|
||||
steps.push('user-turn-ended');
|
||||
},
|
||||
};
|
||||
try {
|
||||
f.rejectedTexts.add('rejected');
|
||||
f.controller.sendTextCommand('rejected');
|
||||
assert.deepEqual(steps, ['cancel', 'user-turn-ended', 'pointer', 'send']);
|
||||
assert.equal(f.creates().length, 0);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('earlier accepted deferred input survives a newer failed send and answers once', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
await f.created('active');
|
||||
f.controller.sendTextCommand('accepted');
|
||||
f.rejectedTexts.add('rejected');
|
||||
f.controller.sendTextCommand('rejected');
|
||||
assert.deepEqual(f.texts(), ['accepted']);
|
||||
assert.equal(f.controller.pendingUserTextResponse, true);
|
||||
assert.equal(f.creates().length, 0);
|
||||
await f.done('active');
|
||||
assert.equal(f.creates().length, 1);
|
||||
await f.done('active');
|
||||
assert.equal(f.creates().length, 1);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('rejected text preserves an outstanding create without adding deferred input', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
f.controller.sendTextCommand('accepted');
|
||||
f.rejectedTexts.add('rejected');
|
||||
f.controller.sendTextCommand('rejected');
|
||||
assert.equal(f.controller.responseCreatePending, true);
|
||||
assert.equal(f.controller.pendingUserTextResponse, false);
|
||||
await f.created('accepted');
|
||||
await f.done('accepted');
|
||||
assert.equal(f.creates().length, 1);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
for (const acceptedFirst of [false, true]) {
|
||||
test(`progress completion answers only accepted deferred input (accepted first=${acceptedFirst})`, async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
assert.equal(
|
||||
f.controller._turns.speakProgressLine('Checking the map.'),
|
||||
true,
|
||||
);
|
||||
await f.event({
|
||||
type: 'response.created',
|
||||
response: { id: 'progress', metadata: { gev: 'progress' } },
|
||||
});
|
||||
if (acceptedFirst) f.controller.sendTextCommand('accepted');
|
||||
f.rejectedTexts.add('rejected');
|
||||
f.controller.sendTextCommand('rejected');
|
||||
assert.equal(f.controller.pendingUserTextResponse, acceptedFirst);
|
||||
await f.event({
|
||||
type: 'response.done',
|
||||
response: {
|
||||
id: 'progress',
|
||||
status: 'cancelled',
|
||||
metadata: { gev: 'progress' },
|
||||
},
|
||||
});
|
||||
assert.equal(f.creates().length, acceptedFirst ? 1 : 0);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('accepted text with a rejected response.create releases the pending latch for a successor', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
f.rejectResponses(true);
|
||||
f.controller.sendTextCommand('accepted without response');
|
||||
assert.equal(f.controller.responseCreatePending, false);
|
||||
assert.equal(f.creates().length, 0);
|
||||
f.rejectResponses(false);
|
||||
f.controller.sendTextCommand('successor');
|
||||
assert.deepEqual(f.texts(), ['accepted without response', 'successor']);
|
||||
assert.equal(f.creates().length, 1);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('successful deferred typed burst still retains both items and coalesces one answer', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
await f.created('active');
|
||||
f.controller.sendTextCommand('first');
|
||||
f.controller.sendTextCommand('second');
|
||||
assert.deepEqual(f.texts(), ['first', 'second']);
|
||||
assert.equal(f.creates().length, 0);
|
||||
await f.done('active');
|
||||
await f.done('active');
|
||||
assert.equal(f.creates().length, 1);
|
||||
} finally {
|
||||
f.controller.stop();
|
||||
}
|
||||
});
|
||||
@@ -5,17 +5,17 @@ import {
|
||||
import { cameraPoseKey } from './pointerContext.js';
|
||||
|
||||
/** The live viewer's camera pose identity (Cesium-free read). */
|
||||
export function readViewerCameraKey() {
|
||||
function readViewerCameraKey() {
|
||||
return cameraPoseKey(globalThis.window?.__godsEyeView?.viewer);
|
||||
}
|
||||
// Viewport-screenshot size guards (M13). The old code clamped WIDTH only, so a
|
||||
// tall portrait window produced an oversized capture whose dc.send could throw.
|
||||
// Cap total pixels (clamps both dimensions) and drop the image entirely if the
|
||||
// encoded data URL is still too big for the data channel.
|
||||
export const VIEWPORT_MAX_PIXELS = 1200 * 900;
|
||||
const VIEWPORT_MAX_PIXELS = 1200 * 900;
|
||||
|
||||
// ~1.08 MP, matches the old 1200px-wide landscape budget
|
||||
export const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
|
||||
const VIEWPORT_MAX_ENCODED_BYTES = 200 * 1024;
|
||||
|
||||
export async function captureViewportImage() {
|
||||
const viewer = window.__godsEyeView?.viewer;
|
||||
@@ -66,11 +66,11 @@ export async function captureViewportImage() {
|
||||
}
|
||||
|
||||
/** Side of the point-and-ask crop, in CSS pixels, and its output size. */
|
||||
export const POINTER_CROP_SIZE = 512;
|
||||
const POINTER_CROP_SIZE = 512;
|
||||
/** A pointer snapshot older than this no longer matches the frame. */
|
||||
export const POINTER_CROP_MAX_AGE_MS = 20000;
|
||||
const POINTER_CROP_MAX_AGE_MS = 20000;
|
||||
/** View scales where a crop shows buildings and terrain, not a smear. */
|
||||
export const POINTER_CROP_SCALES = new Set(['local', 'city']);
|
||||
const POINTER_CROP_SCALES = new Set(['local', 'city']);
|
||||
|
||||
/**
|
||||
* Square source rectangle around a canvas point, clamped inside the canvas,
|
||||
@@ -128,7 +128,7 @@ export function pointerCropRect({
|
||||
* @returns {Promise<{dataUrl: string, rect: object}|null>} JPEG data URL and
|
||||
* the crop's normalized bounds in the full canvas, or null.
|
||||
*/
|
||||
export async function capturePointerCrop(screenPx) {
|
||||
async function capturePointerCrop(screenPx) {
|
||||
const viewer = window.__godsEyeView?.viewer;
|
||||
const source = viewer?.scene?.canvas;
|
||||
if (!source?.width || !source?.height || !screenPx) return null;
|
||||
@@ -259,7 +259,7 @@ export async function renderFreshCesiumFrame(viewer) {
|
||||
}
|
||||
}
|
||||
|
||||
export function isNearlyBlackFrame(ctx, width, height) {
|
||||
function isNearlyBlackFrame(ctx, width, height) {
|
||||
const sampleWidth = Math.min(48, width);
|
||||
const sampleHeight = Math.min(32, height);
|
||||
if (!sampleWidth || !sampleHeight) return true;
|
||||
|
||||
+337
-12
@@ -9,11 +9,27 @@
|
||||
* referents the card lists (the referent registry reads the same answer, so
|
||||
* "the second one" always means the second row on screen).
|
||||
*
|
||||
* What the model receives stays English (`analystHeadline` feeds the spoken
|
||||
* answer); the card shows the same answer in the active language.
|
||||
*
|
||||
* Pure: no DOM, no Cesium.
|
||||
* @module voice/resultDisplay
|
||||
*/
|
||||
|
||||
import { voiceLayer } from './layerManifest.js';
|
||||
import {
|
||||
formatDate,
|
||||
formatNumber,
|
||||
getLocale,
|
||||
msg,
|
||||
t,
|
||||
tn,
|
||||
translateKnown,
|
||||
} from '../i18n/index.js';
|
||||
import { layerNameForId, sourceLabel } from '../ui/layerNames.js';
|
||||
import { localizePlaceName } from '../ui/placeNames.js';
|
||||
import { translateStatusLine } from '../ui/statusText.js';
|
||||
import { formatQuantity } from '../ui/unitLabels.js';
|
||||
|
||||
const LAYER_NOUNS = Object.freeze({
|
||||
flights: 'aircraft',
|
||||
@@ -43,7 +59,7 @@ function notesOf(...values) {
|
||||
];
|
||||
}
|
||||
|
||||
export function analystNounFor(layerKeys) {
|
||||
function analystNounFor(layerKeys) {
|
||||
const keys = [...new Set(layerKeys.filter(Boolean))];
|
||||
if (keys.length !== 1) return 'results';
|
||||
const key = keys[0];
|
||||
@@ -52,13 +68,17 @@ export function analystNounFor(layerKeys) {
|
||||
|
||||
/** The count/scope headline shared by the analyst card and spoken answer. */
|
||||
const ANALYST_HEADLINE_MAX = 64;
|
||||
// The card's headline wraps, and a translation runs longer than the English:
|
||||
// "Al menos 12 aeronaves militares en un radio de 40 km de Los Ángeles" is 67.
|
||||
// English keeps the spoken bound, so its card title still equals `say`.
|
||||
const SHOWN_HEADLINE_MAX = 120;
|
||||
|
||||
function boundedHeadline(value) {
|
||||
function boundedHeadline(value, max = ANALYST_HEADLINE_MAX) {
|
||||
const text = String(value || '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
if (text.length <= ANALYST_HEADLINE_MAX) return text;
|
||||
return `${text.slice(0, ANALYST_HEADLINE_MAX - 1).trimEnd()}…`;
|
||||
if (text.length <= max) return text;
|
||||
return `${text.slice(0, max - 1).trimEnd()}…`;
|
||||
}
|
||||
|
||||
export function analystHeadline(result) {
|
||||
@@ -72,6 +92,307 @@ export function analystHeadline(result) {
|
||||
);
|
||||
}
|
||||
|
||||
/* ---------------- the analyst card in the active language ---------------- */
|
||||
|
||||
/**
|
||||
* The count and its noun as the card shows them. The English matches
|
||||
* `analystNounFor`, one form for any count as the spoken headline has it.
|
||||
*/
|
||||
function shownCount(count, noun) {
|
||||
const values = { n: formatNumber(count) };
|
||||
switch (noun) {
|
||||
case 'aircraft':
|
||||
return tn(count, '{n} aircraft', '{n} aircraft', values);
|
||||
case 'military aircraft':
|
||||
return tn(
|
||||
count,
|
||||
'{n} military aircraft',
|
||||
'{n} military aircraft',
|
||||
values,
|
||||
);
|
||||
case 'ships':
|
||||
return tn(count, '{n} ships', '{n} ships', values);
|
||||
case 'satellites':
|
||||
return tn(count, '{n} satellites', '{n} satellites', values);
|
||||
case 'launches':
|
||||
return tn(count, '{n} launches', '{n} launches', values);
|
||||
case 'earthquakes':
|
||||
return tn(count, '{n} earthquakes', '{n} earthquakes', values);
|
||||
case 'fires':
|
||||
return tn(count, '{n} fires', '{n} fires', values);
|
||||
case 'fire perimeters':
|
||||
return tn(count, '{n} fire perimeters', '{n} fire perimeters', values);
|
||||
case 'cyclones':
|
||||
return tn(count, '{n} cyclones', '{n} cyclones', values);
|
||||
case 'buses':
|
||||
return tn(count, '{n} buses', '{n} buses', values);
|
||||
case 'bike share':
|
||||
return tn(count, '{n} bike share', '{n} bike share', values);
|
||||
case 'flock cameras':
|
||||
return tn(count, '{n} flock cameras', '{n} flock cameras', values);
|
||||
case 'military bases':
|
||||
return tn(count, '{n} military bases', '{n} military bases', values);
|
||||
case 'datacenters':
|
||||
return tn(count, '{n} datacenters', '{n} datacenters', values);
|
||||
case 'dams':
|
||||
return tn(count, '{n} dams', '{n} dams', values);
|
||||
case 'air quality':
|
||||
return tn(count, '{n} air quality', '{n} air quality', values);
|
||||
case 'results':
|
||||
return tn(count, '{n} results', '{n} results', values);
|
||||
default:
|
||||
return `${values.n} ${noun}`;
|
||||
}
|
||||
}
|
||||
|
||||
const shownRadius = (km) => formatQuantity(Number(km), 'km');
|
||||
|
||||
const FIXED_SCOPES = new Map(
|
||||
[msg('anywhere in the loaded data'), msg('in view')].map((text) => [
|
||||
text,
|
||||
text,
|
||||
]),
|
||||
);
|
||||
|
||||
/**
|
||||
* The engine's scope phrases (analystEngine, the Contacts window in
|
||||
* gevActions) as the card shows them. The English is exactly the engine's;
|
||||
* a phrase not listed shows as given.
|
||||
*/
|
||||
const SCOPE_PATTERNS = [
|
||||
[
|
||||
/^within (\d+(?:\.\d+)?) km of the view centre$/,
|
||||
([, km]) =>
|
||||
t('within {radius} of the view centre', { radius: shownRadius(km) }),
|
||||
],
|
||||
[
|
||||
/^within (\d+(?:\.\d+)?) km of (.+) \(Contacts window\)$/,
|
||||
([, km, place]) =>
|
||||
t('within {radius} of {place} (Contacts window)', {
|
||||
radius: shownRadius(km),
|
||||
place: localizePlaceName(place),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^within (\d+(?:\.\d+)?) km of (.+)$/,
|
||||
([, km, place]) =>
|
||||
t('within {radius} of {place}', {
|
||||
radius: shownRadius(km),
|
||||
place: localizePlaceName(place),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^within (\d+(?:\.\d+)?) km$/,
|
||||
([, km]) => t('within {radius}', { radius: shownRadius(km) }),
|
||||
],
|
||||
[/^over (.+)$/, ([, region]) => t('over {region}', { region })],
|
||||
];
|
||||
|
||||
function shownScope(text) {
|
||||
const scope = String(text || '').trim();
|
||||
if (!scope) return '';
|
||||
if (FIXED_SCOPES.has(scope)) return t(FIXED_SCOPES.get(scope));
|
||||
for (const [pattern, show] of SCOPE_PATTERNS) {
|
||||
const match = pattern.exec(scope);
|
||||
if (match) return show(match);
|
||||
}
|
||||
return translateKnown(scope);
|
||||
}
|
||||
|
||||
/** The analyst headline in the active language, for the card. */
|
||||
function shownAnalystHeadline(result) {
|
||||
const keys = (result?.coverage?.layersQueried || []).map(
|
||||
(layer) => layer.layerKey,
|
||||
);
|
||||
const count = Number(result?.count) || 0;
|
||||
const counted = shownCount(count, analystNounFor(keys));
|
||||
const items =
|
||||
result?.complete === false
|
||||
? t('At least {items}', { items: counted })
|
||||
: counted;
|
||||
const scope = shownScope(result?.scopeLabel);
|
||||
return boundedHeadline(
|
||||
scope ? t('{items} {scope}', { items, scope }) : items,
|
||||
getLocale() === 'en' ? ANALYST_HEADLINE_MAX : SHOWN_HEADLINE_MAX,
|
||||
);
|
||||
}
|
||||
|
||||
/** A layer key as the card names it. */
|
||||
function shownLayerKey(key) {
|
||||
const text = String(key || '').trim();
|
||||
return layerNameForId(text) || text;
|
||||
}
|
||||
|
||||
const shownLayerKeys = (list) =>
|
||||
String(list || '')
|
||||
.split(/,\s*/)
|
||||
.map(shownLayerKey)
|
||||
.filter(Boolean)
|
||||
.join(', ');
|
||||
|
||||
/** The feed windows layers declare in the voice manifest. */
|
||||
const FIXED_WINDOWS = new Map(
|
||||
[msg('last 24 h'), msg('past 30 d and upcoming')].map((text) => [text, text]),
|
||||
);
|
||||
|
||||
function shownWindow(text) {
|
||||
return String(text || '')
|
||||
.split(/;\s*/)
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean)
|
||||
.map((part) =>
|
||||
FIXED_WINDOWS.has(part)
|
||||
? t(FIXED_WINDOWS.get(part))
|
||||
: translateKnown(part),
|
||||
)
|
||||
.join('; ');
|
||||
}
|
||||
|
||||
const UTC_CLOCK = Object.freeze({
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit',
|
||||
hourCycle: 'h23',
|
||||
timeZone: 'UTC',
|
||||
});
|
||||
const UTC_DATE = Object.freeze({
|
||||
day: '2-digit',
|
||||
month: '2-digit',
|
||||
year: 'numeric',
|
||||
timeZone: 'UTC',
|
||||
});
|
||||
|
||||
/**
|
||||
* The note's ISO snapshot time as the card shows it: "14:32:05 UTC", dated
|
||||
* ("15.10.2026, 14:32:05 UTC") when not from today (UTC). English and
|
||||
* non-time values show as given.
|
||||
*/
|
||||
export function shownSnapshotTime(iso, now = Date.now()) {
|
||||
const time = Date.parse(iso);
|
||||
if (getLocale() === 'en' || !Number.isFinite(time)) return iso;
|
||||
const clock = `${formatDate(time, UTC_CLOCK)} UTC`;
|
||||
const today = new Date(now).toISOString().slice(0, 10);
|
||||
return new Date(time).toISOString().slice(0, 10) === today
|
||||
? clock
|
||||
: `${formatDate(time, UTC_DATE)}, ${clock}`;
|
||||
}
|
||||
|
||||
/** Coverage notes and caveats the engine and the action runner write. */
|
||||
const FIXED_NOTES = new Map(
|
||||
[
|
||||
msg('the flights layer loads by viewport'),
|
||||
msg('distance is ground distance, not overhead'),
|
||||
msg(
|
||||
'the count is a lower bound because at least one layer reached its retained limit.',
|
||||
),
|
||||
].map((text) => [text, text]),
|
||||
);
|
||||
|
||||
const LAYER_STATUS_NOTES = Object.freeze({
|
||||
off: (label) => t('{label} off', { label }),
|
||||
loading: (label) => t('{label} loading', { label }),
|
||||
error: (label) => t('{label} error', { label }),
|
||||
});
|
||||
|
||||
const NOTE_PATTERNS = [
|
||||
[
|
||||
/^counted the first ([\d,]+) loaded records$/,
|
||||
([, returned]) =>
|
||||
t('counted the first {returned} loaded records', {
|
||||
returned: formatNumber(Number(returned.replace(/,/g, ''))),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^counted ([\d,]+) of ([\d,]+) loaded records$/,
|
||||
([, returned, total]) =>
|
||||
t('counted {returned} of {total} loaded records', {
|
||||
returned: formatNumber(Number(returned.replace(/,/g, ''))),
|
||||
total: formatNumber(Number(total.replace(/,/g, ''))),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^(.+) just turned on — still loading, counts will rise$/,
|
||||
([, layers]) =>
|
||||
t('{layers} just turned on — still loading, counts will rise', {
|
||||
layers: shownLayerKeys(layers),
|
||||
}),
|
||||
],
|
||||
[
|
||||
// The note ends in a period unless a lower-bound clause follows it.
|
||||
/^Contacts panel snapshot(?: at (\S+))? — the same retained cohort supplies the card, speech, and follow-up memory(\.?)$/,
|
||||
([, time, period]) =>
|
||||
(time
|
||||
? t(
|
||||
'Contacts panel snapshot at {time} — the same retained cohort supplies the card, speech, and follow-up memory',
|
||||
{ time: shownSnapshotTime(time) },
|
||||
)
|
||||
: t(
|
||||
'Contacts panel snapshot — the same retained cohort supplies the card, speech, and follow-up memory',
|
||||
)) + period,
|
||||
],
|
||||
// Transit's empty note when no feed covers the view (transit/queries.js);
|
||||
// the model gets the English.
|
||||
[
|
||||
/^Fly below ([\d,]+) km to a covered region$/,
|
||||
([, km]) =>
|
||||
t('Fly below {km} km to a covered region', {
|
||||
km: formatNumber(Number(km.replace(/,/g, ''))),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^No feed here yet · ([\d,]+) regions available$/,
|
||||
([, regions]) => {
|
||||
const count = Number(regions.replace(/,/g, ''));
|
||||
return tn(
|
||||
count,
|
||||
'No feed here yet · {n} region available',
|
||||
'No feed here yet · {n} regions available',
|
||||
{ n: formatNumber(count) },
|
||||
);
|
||||
},
|
||||
],
|
||||
[
|
||||
/^([a-z0-9-]+) (off|loading|error)$/,
|
||||
([, key, status]) =>
|
||||
layerNameForId(key)
|
||||
? LAYER_STATUS_NOTES[status](shownLayerKey(key))
|
||||
: null,
|
||||
],
|
||||
[
|
||||
/^([a-z0-9-]+): (.+)$/,
|
||||
([, key, note]) =>
|
||||
layerNameForId(key) ? `${shownLayerKey(key)}: ${shownNote(note)}` : null,
|
||||
],
|
||||
];
|
||||
|
||||
function shownNote(text) {
|
||||
if (FIXED_NOTES.has(text)) return t(FIXED_NOTES.get(text));
|
||||
for (const [pattern, show] of NOTE_PATTERNS) {
|
||||
const match = pattern.exec(text);
|
||||
const shown = match ? show(match) : null;
|
||||
if (shown) return shown;
|
||||
}
|
||||
// A layer's empty note can be its status line, English pieces joined by
|
||||
// " · " ("feeds 1090, 978 stale · 0 heard", "TTC 0 (95s old)").
|
||||
const known = translateKnown(text);
|
||||
return known !== text ? known : translateStatusLine(text);
|
||||
}
|
||||
|
||||
/** Feed states as the card's chips show them; the values stay English. */
|
||||
const FEED_STATE_LABELS = Object.freeze({
|
||||
loading: msg('loading'),
|
||||
degraded: msg('degraded'),
|
||||
stale: msg('stale'),
|
||||
partial: msg('partial'),
|
||||
fallback: msg('fallback'),
|
||||
unavailable: msg('unavailable'),
|
||||
});
|
||||
|
||||
function shownFeedState(state) {
|
||||
const key = FEED_STATE_LABELS[state];
|
||||
return key ? t(key) : state;
|
||||
}
|
||||
|
||||
function analystCard(result) {
|
||||
if (!result?.ok) return null;
|
||||
const display = result.display || {};
|
||||
@@ -91,28 +412,32 @@ function analystCard(result) {
|
||||
].map((label) => ({ label }));
|
||||
return {
|
||||
display: {
|
||||
title: analystHeadline(result),
|
||||
title: shownAnalystHeadline(result),
|
||||
// The precise scope, unless the title already says exactly that.
|
||||
lines: [
|
||||
display.scope !== result.scopeLabel ? display.scope : null,
|
||||
display.window,
|
||||
display.scope !== result.scopeLabel ? shownScope(display.scope) : null,
|
||||
shownWindow(display.window),
|
||||
].filter(Boolean),
|
||||
chips: [
|
||||
...(result.feedState && result.feedState !== 'nominal'
|
||||
? [{ label: result.feedState }]
|
||||
? [{ label: shownFeedState(result.feedState) }]
|
||||
: []),
|
||||
...(result.partial ? [{ label: 'partial' }] : []),
|
||||
...(result.partial ? [{ label: t('partial') }] : []),
|
||||
],
|
||||
// Coverage qualifications lead the feed detail. The bounded card groups
|
||||
// overflow in its existing Notes disclosure rather than dropping them.
|
||||
preserveNotes: true,
|
||||
notes: [
|
||||
...notesOf(result.coverage?.note, display.caveat),
|
||||
...notesOf(result.coverage?.note, display.caveat).map(shownNote),
|
||||
...(unanswered.length
|
||||
? [`Not answered: ${unanswered.join(', ')}`]
|
||||
? [
|
||||
t('Not answered: {layers}', {
|
||||
layers: unanswered.map(shownLayerKey).join(', '),
|
||||
}),
|
||||
]
|
||||
: []),
|
||||
],
|
||||
sources,
|
||||
sources: sources.map(({ label }) => ({ label: sourceLabel(label) })),
|
||||
},
|
||||
referents: Array.isArray(result.items) ? result.items : [],
|
||||
resultSet: true,
|
||||
|
||||
@@ -147,8 +147,9 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
|
||||
},
|
||||
async start(options) {
|
||||
if (disposed || !['idle', 'error'].includes(state)) return;
|
||||
emit({ type: 'state', state: 'connecting' });
|
||||
const attempt = ++startAttempt;
|
||||
emit({ type: 'state', state: 'connecting' });
|
||||
if (disposed || attempt !== startAttempt) return;
|
||||
try {
|
||||
await adapter.start(options);
|
||||
} catch (error) {
|
||||
@@ -164,12 +165,17 @@ export function createVoiceSession({ createAdapter, runner, signal }) {
|
||||
stop(options = {}) {
|
||||
if (options.removeUi) return dispose();
|
||||
if (disposed) return;
|
||||
startAttempt++;
|
||||
const attempt = ++startAttempt;
|
||||
cancelActions();
|
||||
try {
|
||||
adapter.stop(options);
|
||||
} finally {
|
||||
if (!options.preserveStatus)
|
||||
if (
|
||||
!options.preserveStatus &&
|
||||
!disposed &&
|
||||
attempt === startAttempt &&
|
||||
state !== 'idle'
|
||||
)
|
||||
emit({ type: 'state', state: 'idle', detail: 'Voice off' });
|
||||
}
|
||||
},
|
||||
|
||||
+196
-1
@@ -6,8 +6,10 @@ import {
|
||||
voiceCardView,
|
||||
} from './voiceCardPresentation.js';
|
||||
import { createVoiceSession } from './session.js';
|
||||
import { createRealtimeSession } from './realtimeSession.js';
|
||||
import { realtimeSessionEvent } from './realtimeEvents.js';
|
||||
import { createVoiceCommands } from './sessionCommands.js';
|
||||
import { describeUnsavedWork } from '../i18n/index.js';
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
@@ -300,17 +302,24 @@ test('common button and annotation bindings work with an alternate adapter and c
|
||||
assert.equal(ui.helpDetail.textContent, 'Activate to toggle voice');
|
||||
assert.equal(ui.status.textContent, 'LISTENING');
|
||||
assert.equal(ui.detail.textContent, 'Ready');
|
||||
// Switching language (a reload) asks first during a session.
|
||||
assert.deepEqual(describeUnsavedWork(), ['the active voice session']);
|
||||
subscribed({ status: 'resolved' });
|
||||
assert.deepEqual(events, [
|
||||
{ type: 'map_annotation_outline', status: 'resolved' },
|
||||
]);
|
||||
button.dispatchEvent(new Event('click'));
|
||||
assert.equal(controls.state, 'idle');
|
||||
assert.deepEqual(describeUnsavedWork(), []);
|
||||
button.dispatchEvent(new Event('click'));
|
||||
await new Promise((done) => setTimeout(done, 0));
|
||||
assert.deepEqual(describeUnsavedWork(), ['the active voice session']);
|
||||
lifetime.abort();
|
||||
assert.deepEqual(describeUnsavedWork(), [], 'disposal unregisters');
|
||||
assert.equal(subscribed, null);
|
||||
assert.equal(removed, 1);
|
||||
button.dispatchEvent(new Event('click'));
|
||||
assert.equal(calls, 1);
|
||||
assert.equal(calls, 2);
|
||||
} finally {
|
||||
globalThis.window = previous;
|
||||
}
|
||||
@@ -433,3 +442,189 @@ test('a late rejection cannot settle a newer turn card', async () => {
|
||||
);
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
function startReentrancyFixture() {
|
||||
let starts = 0;
|
||||
let stops = 0;
|
||||
const settings = [];
|
||||
const session = createVoiceSession({
|
||||
runner: async () => ({ ok: true }),
|
||||
createAdapter: ({ emit }) => ({
|
||||
async start(options) {
|
||||
starts++;
|
||||
settings.push(options);
|
||||
emit({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop() {
|
||||
stops++;
|
||||
},
|
||||
sendText() {},
|
||||
sendMapEvent() {},
|
||||
}),
|
||||
});
|
||||
return { session, settings, starts: () => starts, stops: () => stops };
|
||||
}
|
||||
|
||||
for (const action of ['stop', 'destroy']) {
|
||||
test(`initial connecting observer ${action} prevents adapter acquisition`, async () => {
|
||||
const f = startReentrancyFixture();
|
||||
const unsubscribe = f.session.subscribe((event) => {
|
||||
if (event.type === 'state' && event.state === 'connecting') {
|
||||
unsubscribe();
|
||||
f.session[action]();
|
||||
}
|
||||
});
|
||||
await f.session.start();
|
||||
assert.equal(
|
||||
f.starts(),
|
||||
0,
|
||||
'cancelled attempt must not call adapter.start',
|
||||
);
|
||||
assert.equal(f.stops(), 1);
|
||||
assert.equal(f.session.state, 'idle');
|
||||
if (action === 'stop') {
|
||||
await f.session.start();
|
||||
assert.equal(f.starts(), 1, 'later explicit Start remains usable');
|
||||
assert.equal(f.session.state, 'listening');
|
||||
}
|
||||
f.session.destroy();
|
||||
});
|
||||
}
|
||||
|
||||
test('preserveStatus Stop still prevents adapter acquisition', async () => {
|
||||
const f = startReentrancyFixture();
|
||||
const unsubscribe = f.session.subscribe((event) => {
|
||||
if (event.type === 'state' && event.state === 'connecting') {
|
||||
unsubscribe();
|
||||
f.session.stop({ preserveStatus: true });
|
||||
}
|
||||
});
|
||||
await f.session.start();
|
||||
assert.equal(f.starts(), 0);
|
||||
assert.equal(
|
||||
f.session.state,
|
||||
'connecting',
|
||||
'preserved state is not ownership',
|
||||
);
|
||||
f.session.stop();
|
||||
await f.session.start();
|
||||
assert.equal(f.starts(), 1);
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
test('Stop and explicit reentrant successor start acquire only for the successor', async () => {
|
||||
const f = startReentrancyFixture();
|
||||
let successor;
|
||||
const unsubscribe = f.session.subscribe((event) => {
|
||||
if (event.type === 'state' && event.state === 'connecting') {
|
||||
unsubscribe();
|
||||
f.session.stop();
|
||||
successor = f.session.start();
|
||||
}
|
||||
});
|
||||
await f.session.start();
|
||||
await successor;
|
||||
assert.equal(f.starts(), 1);
|
||||
assert.equal(f.session.state, 'listening');
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
test('ordinary start forwards caller settings once and remains stoppable', async () => {
|
||||
const f = startReentrancyFixture();
|
||||
const settings = { pushToTalk: true };
|
||||
await f.session.start(settings);
|
||||
assert.equal(f.settings.length, 1);
|
||||
assert.equal(f.settings[0], settings);
|
||||
assert.equal(f.starts(), 1);
|
||||
assert.equal(f.session.state, 'listening');
|
||||
f.session.stop();
|
||||
assert.equal(f.session.state, 'idle');
|
||||
f.session.destroy();
|
||||
});
|
||||
|
||||
for (const emitsIdle of [false, true]) {
|
||||
for (const outcome of ['normal', 'preserveStatus', 'throw']) {
|
||||
test(`stop preserves custom adapter contract: emitsIdle=${emitsIdle}, ${outcome}`, async (t) => {
|
||||
let adapter;
|
||||
const calls = [];
|
||||
const events = [];
|
||||
const failure = new Error('Controlled stop failure');
|
||||
const session = createVoiceSession({
|
||||
runner: async () => ({}),
|
||||
createAdapter: ({ emit }) =>
|
||||
(adapter = {
|
||||
async start() {
|
||||
emit({ type: 'state', state: 'listening' });
|
||||
},
|
||||
stop(settings = {}) {
|
||||
calls.push({ receiver: this, settings });
|
||||
if (settings.removeUi) return;
|
||||
if (emitsIdle && !settings.preserveStatus)
|
||||
emit({ type: 'state', state: 'idle' });
|
||||
if (outcome === 'throw') throw failure;
|
||||
return 'adapter-only return';
|
||||
},
|
||||
sendText() {},
|
||||
sendMapEvent() {},
|
||||
}),
|
||||
});
|
||||
t.after(() => session.destroy());
|
||||
session.subscribe((event) => events.push(event));
|
||||
await session.start();
|
||||
events.length = 0;
|
||||
const settings =
|
||||
outcome === 'preserveStatus' ? { preserveStatus: true } : {};
|
||||
if (outcome === 'throw')
|
||||
assert.throws(
|
||||
() => session.stop(settings),
|
||||
(error) => error === failure,
|
||||
);
|
||||
else assert.equal(session.stop(settings), undefined);
|
||||
assert.equal(calls[0].receiver, adapter);
|
||||
assert.equal(calls[0].settings, settings);
|
||||
assert.equal(
|
||||
session.state,
|
||||
outcome === 'preserveStatus' ? 'listening' : 'idle',
|
||||
);
|
||||
assert.equal(
|
||||
events.filter(
|
||||
(event) => event.type === 'state' && event.state === 'idle',
|
||||
).length,
|
||||
outcome === 'preserveStatus' ? 0 : 1,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
test('actual default OpenAI adapter stop publishes one common idle', (t) => {
|
||||
const events = [];
|
||||
const ui = {
|
||||
root: {
|
||||
dataset: {},
|
||||
querySelectorAll: () => [],
|
||||
classList: { remove() {} },
|
||||
remove() {},
|
||||
},
|
||||
status: {},
|
||||
detail: {},
|
||||
};
|
||||
const runner = async () => ({});
|
||||
const session = createVoiceSession({
|
||||
runner,
|
||||
createAdapter: (hooks) =>
|
||||
createRealtimeSession({ ...hooks, runner, ui, debugSink() {} }),
|
||||
});
|
||||
t.after(() => session.destroy());
|
||||
session.subscribe((event) => events.push(event));
|
||||
// Exercise the real stop path without acquiring microphone/WebRTC resources.
|
||||
session.adapter.controller.setStatus('listening', 'Controlled active state');
|
||||
events.length = 0;
|
||||
session.stop();
|
||||
assert.equal(
|
||||
events.filter((event) => event.type === 'state' && event.state === 'idle')
|
||||
.length,
|
||||
1,
|
||||
);
|
||||
assert.equal(session.state, 'idle');
|
||||
assert.equal(session.adapter.controller.status, 'idle');
|
||||
});
|
||||
+115
-35
@@ -1,7 +1,9 @@
|
||||
import { createVoiceControl } from './control.js';
|
||||
import { createVoiceControl, voiceStatusLabel } from './control.js';
|
||||
import { createVoiceSession } from './session.js';
|
||||
import { VoiceCardControls } from './voiceCard.js';
|
||||
import { PointerReticle } from './pointerReticle.js';
|
||||
import { VOICE_PROVIDERS } from './providerCapabilities.js';
|
||||
import { msg, registerUnsavedWork, t } from '../i18n/index.js';
|
||||
|
||||
function createDefaultReticle(pointer) {
|
||||
const element = pointer?.element;
|
||||
@@ -24,19 +26,34 @@ export function createVoiceCommands({
|
||||
signal,
|
||||
debugSink,
|
||||
createControl = createVoiceControl,
|
||||
resetExisting = true,
|
||||
disposeRunner = true,
|
||||
provider,
|
||||
supportedProviders,
|
||||
onProviderChange,
|
||||
onSessionEvent,
|
||||
createCard = (options) => new VoiceCardControls(options),
|
||||
pointer = null,
|
||||
referents = null,
|
||||
createReticle = createDefaultReticle,
|
||||
panelHost = null,
|
||||
nativePanelProvider = null,
|
||||
}) {
|
||||
window.__gevVoiceCommands?.stop?.({ removeUi: true });
|
||||
const ui = createControl({ reset: true });
|
||||
if (resetExisting) window.__gevVoiceCommands?.stop?.({ removeUi: true });
|
||||
const ui = createControl({
|
||||
reset: true,
|
||||
...(panelHost ? { panel: true } : {}),
|
||||
});
|
||||
let actionObserver;
|
||||
let panelMount;
|
||||
const session = createVoiceSession({
|
||||
runner,
|
||||
signal,
|
||||
createAdapter: (hooks) =>
|
||||
createSession({
|
||||
createAdapter: (hooks) => {
|
||||
if (panelHost) actionObserver = panelHost.observeActions(hooks);
|
||||
return createSession({
|
||||
...hooks,
|
||||
...(actionObserver ? { runAction: actionObserver.runAction } : {}),
|
||||
runner,
|
||||
ui,
|
||||
dataManager,
|
||||
@@ -45,41 +62,96 @@ export function createVoiceCommands({
|
||||
createController,
|
||||
pointer,
|
||||
radioLayer: dataManager?.layers?.get('radio')?.module || null,
|
||||
}),
|
||||
});
|
||||
},
|
||||
});
|
||||
const adapter = session.adapter;
|
||||
const capabilities = adapter.capabilities || {};
|
||||
const providerHandler = () => onProviderChange?.(ui.providerSelect.value);
|
||||
if (ui.providerSelect && onProviderChange) {
|
||||
if (supportedProviders && ui.providerSelect.replaceChildren) {
|
||||
const doc = ui.providerSelect.ownerDocument;
|
||||
ui.providerSelect.replaceChildren(
|
||||
...supportedProviders.map((name) => {
|
||||
const option = doc.createElement('option');
|
||||
option.value = name;
|
||||
option.textContent = VOICE_PROVIDERS[name].label;
|
||||
return option;
|
||||
}),
|
||||
);
|
||||
}
|
||||
ui.providerSelect.value = provider;
|
||||
ui.providerSelect.title = [
|
||||
t('Changing provider reconnects voice if it is on.'),
|
||||
capabilities.advisoryLimit,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ');
|
||||
ui.providerSelect.addEventListener('change', providerHandler);
|
||||
ui.root.dataset.provider = provider;
|
||||
}
|
||||
const hasProviderChoice = Boolean(
|
||||
onProviderChange && supportedProviders?.length > 1,
|
||||
);
|
||||
if (ui.providerField) ui.providerField.hidden = !hasProviderChoice;
|
||||
ui.root.dataset.providerChoice = String(hasProviderChoice);
|
||||
if (ui.tierButton) ui.tierButton.hidden = !capabilities.costControls;
|
||||
if (ui.costValue) ui.costValue.hidden = !capabilities.costControls;
|
||||
if (!capabilities.pushToTalk) {
|
||||
ui.button.setAttribute('aria-label', 'Toggle voice control');
|
||||
if (ui.helpDetail) ui.helpDetail.textContent = 'Activate to toggle voice';
|
||||
ui.button.setAttribute('aria-label', t('Toggle voice control'));
|
||||
if (ui.helpDetail)
|
||||
ui.helpDetail.textContent = t('Activate to toggle voice');
|
||||
}
|
||||
// Retain the existing controller's inspection surface for browser tools.
|
||||
const controls = adapter.controller || session;
|
||||
controls.session = session;
|
||||
controls.ui = ui;
|
||||
const updateStatus = session.subscribe((event) => {
|
||||
if (event.type !== 'state') return;
|
||||
ui.root.dataset.status = event.state;
|
||||
ui.status.textContent =
|
||||
event.state === 'idle' ? 'OFF' : event.state.toUpperCase();
|
||||
// States are machine values; only the label is translated.
|
||||
ui.status.textContent = voiceStatusLabel(event.state);
|
||||
// The portable session reports 'Voice off' in English: show the idle
|
||||
// label instead. Other details are provider messages, shown as they come.
|
||||
const idle = event.state === 'idle';
|
||||
const detail = event.detail === 'Voice off' ? '' : event.detail;
|
||||
ui.detail.textContent =
|
||||
event.detail || (event.state === 'idle' ? 'Voice off' : 'Voice active');
|
||||
detail || (idle ? t('Voice off') : t('Voice active'));
|
||||
ui.button.setAttribute('aria-pressed', String(session.isActive()));
|
||||
if (ui.errorDetail)
|
||||
ui.errorDetail.textContent =
|
||||
event.state === 'error'
|
||||
? event.detail || 'Voice could not be started.'
|
||||
? event.detail || t('Voice could not be started.')
|
||||
: '';
|
||||
if (event.state === 'error') ui.root.classList?.remove('error-dismissed');
|
||||
});
|
||||
// Captions, plan and result card nested in the control; it only reads
|
||||
// provider-neutral session events.
|
||||
const card = ui.card ? createCard({ root: ui.root }) : null;
|
||||
const updateCard = card
|
||||
? session.subscribe((event) => card.handle(event))
|
||||
const card = ui.card
|
||||
? createCard({
|
||||
root: ui.root,
|
||||
...(panelHost ? { onRender: () => panelMount?.panel.sync() } : {}),
|
||||
})
|
||||
: null;
|
||||
panelMount = panelHost?.mount({
|
||||
ui,
|
||||
card,
|
||||
session,
|
||||
observer: actionObserver,
|
||||
nativeProvider: nativePanelProvider,
|
||||
});
|
||||
const updatePanel = panelMount ? session.subscribe(panelMount.handle) : null;
|
||||
const updateCard =
|
||||
card && !panelMount
|
||||
? session.subscribe((event) => card.handle(event))
|
||||
: null;
|
||||
controls.voiceCard = card;
|
||||
if (panelMount) {
|
||||
controls.voicePanel = panelMount.panel;
|
||||
controls.captureVoicePanelOperation = panelMount.captureOperation;
|
||||
controls.getVoicePanelState = panelMount.snapshot;
|
||||
controls.getVoicePreferences = panelHost.snapshot;
|
||||
}
|
||||
// Point-and-ask: the reticle follows turn events; pointer and referents are
|
||||
// session-scoped, so a session that ends forgets both.
|
||||
const reticle = pointer ? createReticle(pointer) : null;
|
||||
@@ -95,42 +167,50 @@ export function createVoiceCommands({
|
||||
});
|
||||
controls.pointer = pointer;
|
||||
controls.referents = referents;
|
||||
// Forward last: a host observer may synchronously replace or stop this mount.
|
||||
const forwardEvents = onSessionEvent
|
||||
? session.subscribe(onSessionEvent)
|
||||
: null;
|
||||
const annotationUnsubscribe = annotations?.onOutlineEvent?.((event) => {
|
||||
card?.outline(event);
|
||||
session.sendMapEvent({ type: 'map_annotation_outline', ...event });
|
||||
});
|
||||
// A reload (the language switch) ends the conversation: ask first.
|
||||
const unregisterUnsavedWork = registerUnsavedWork(() =>
|
||||
session.isActive() ? msg('the active voice session') : null,
|
||||
);
|
||||
const buttonHandler = () => {
|
||||
if (adapter.ignoreButtonClick?.()) return;
|
||||
if (session.isActive()) session.stop();
|
||||
else void session.start({ pushToTalk: false });
|
||||
if (session.isActive()) {
|
||||
const ticket = panelMount?.stopTicket();
|
||||
session.stop();
|
||||
panelMount?.closeAfterStop(ticket);
|
||||
} else void session.start({ pushToTalk: false });
|
||||
};
|
||||
ui.button.addEventListener('click', buttonHandler);
|
||||
session.signal.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
ui.button.removeEventListener('click', buttonHandler);
|
||||
annotationUnsubscribe?.();
|
||||
updateStatus();
|
||||
updateCard?.();
|
||||
updatePointer();
|
||||
reticle?.destroy();
|
||||
card?.destroy();
|
||||
runner.dispose?.();
|
||||
ui.root.remove();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
if (session.disposed) {
|
||||
let cleaned = false;
|
||||
const cleanup = () => {
|
||||
if (cleaned) return;
|
||||
cleaned = true;
|
||||
ui.button.removeEventListener('click', buttonHandler);
|
||||
ui.providerSelect?.removeEventListener('change', providerHandler);
|
||||
unregisterUnsavedWork();
|
||||
annotationUnsubscribe?.();
|
||||
updateStatus();
|
||||
updateCard?.();
|
||||
updatePanel?.();
|
||||
panelMount?.destroy();
|
||||
actionObserver?.dispose();
|
||||
updatePointer();
|
||||
forwardEvents?.();
|
||||
reticle?.destroy();
|
||||
card?.destroy();
|
||||
runner.dispose?.();
|
||||
if (disposeRunner) runner.dispose?.();
|
||||
ui.root.remove();
|
||||
} else adapter.bindControls?.();
|
||||
};
|
||||
session.signal.addEventListener('abort', cleanup, { once: true });
|
||||
if (session.disposed) cleanup();
|
||||
else adapter.bindControls?.();
|
||||
window.__gevVoiceCommands = controls;
|
||||
return controls;
|
||||
}
|
||||
+482
-63
@@ -16,9 +16,29 @@
|
||||
*
|
||||
* Builders are small pure functions keyed by tool name. Tools without one are
|
||||
* returned unchanged.
|
||||
*
|
||||
* Only `display` and plan/progress labels are UI text (t()). `say`,
|
||||
* `identityLine`, spoken progress lines and the tool result stay English for
|
||||
* the model; the card shows them through the shown* helpers.
|
||||
*/
|
||||
|
||||
import { analystHeadline } from './resultDisplay.js';
|
||||
import {
|
||||
formatNumber,
|
||||
getLocale,
|
||||
msg,
|
||||
t,
|
||||
tn,
|
||||
translateKnown,
|
||||
} from '../i18n/index.js';
|
||||
import {
|
||||
layerNameForId,
|
||||
shownLayerName,
|
||||
sourceLabel,
|
||||
} from '../ui/layerNames.js';
|
||||
import { localizePlaceName } from '../ui/placeNames.js';
|
||||
import { CITY_POIS } from '../cityPois.js';
|
||||
import { formatQuantity } from '../ui/unitLabels.js';
|
||||
|
||||
const LAYER_NOUNS = Object.freeze({
|
||||
flights: 'aircraft',
|
||||
@@ -53,17 +73,180 @@ function listPhrase(items) {
|
||||
return `${list.slice(0, -1).join(', ')} and ${list.at(-1)}`;
|
||||
}
|
||||
|
||||
function feet(meters) {
|
||||
function feetRounded(meters) {
|
||||
const value = Number(meters);
|
||||
if (!Number.isFinite(value)) return null;
|
||||
const rounded = Math.round((value * 3.28084) / 100) * 100;
|
||||
return Number.isFinite(value)
|
||||
? Math.round((value * 3.28084) / 100) * 100
|
||||
: null;
|
||||
}
|
||||
|
||||
function feet(meters) {
|
||||
const rounded = feetRounded(meters);
|
||||
if (rounded === null) return null;
|
||||
return `${rounded.toLocaleString('en-US')} feet`;
|
||||
}
|
||||
|
||||
function kilometers(km) {
|
||||
/** The card's altitude, in the active language. */
|
||||
function shownFeet(meters) {
|
||||
const rounded = feetRounded(meters);
|
||||
if (rounded === null) return null;
|
||||
return t('{value} feet', { value: formatNumber(rounded) });
|
||||
}
|
||||
|
||||
function roundedKm(km) {
|
||||
const value = Number(km);
|
||||
if (!Number.isFinite(value)) return null;
|
||||
return `${value < 10 ? Number(value.toFixed(1)) : Math.round(value)} km`;
|
||||
return value < 10 ? Number(value.toFixed(1)) : Math.round(value);
|
||||
}
|
||||
|
||||
function kilometers(km) {
|
||||
const value = roundedKm(km);
|
||||
return value === null ? null : `${value} km`;
|
||||
}
|
||||
|
||||
/** The card's distance, in the active language: "2,5 км". */
|
||||
function shownKm(km) {
|
||||
const value = roundedKm(km);
|
||||
return value === null ? null : formatQuantity(value, 'km');
|
||||
}
|
||||
|
||||
/** Sources as the card lists them. */
|
||||
function shownSources(source) {
|
||||
return source ? [{ label: sourceLabel(source) }] : [];
|
||||
}
|
||||
|
||||
// Entity labels that layers register as English data (firms/, alpr/); keep in
|
||||
// step with the producers. Other names (callsigns, places) show as they came.
|
||||
const ENTITY_LABELS = new Set([msg('ALPR camera')]);
|
||||
const FIRE_LABEL = /^Fire · FRP (\S+) MW$/;
|
||||
|
||||
/** An entity label as the card shows it, in the active language. */
|
||||
export function shownEntityLabel(label) {
|
||||
if (ENTITY_LABELS.has(label)) return t(label);
|
||||
const fire = FIRE_LABEL.exec(label ?? '');
|
||||
return fire ? t('Fire · FRP {frp} MW', { frp: fire[1] }) : label;
|
||||
}
|
||||
|
||||
/*
|
||||
* Tool errors stay English in the result (the model reads them). The card
|
||||
* translates the known ones; the English must match gevActions exactly.
|
||||
*/
|
||||
const KNOWN_ERRORS = new Map(
|
||||
[
|
||||
msg('The flights layer is not enabled'),
|
||||
msg('The military layer is not enabled'),
|
||||
msg('The satellites layer is not enabled'),
|
||||
msg('The ais-live-vessels layer is not enabled'),
|
||||
msg('Could not enable the requested layer'),
|
||||
msg('Could not disable the requested layer'),
|
||||
msg('Radio layer unavailable'),
|
||||
msg('Camera navigation is unavailable in the current view'),
|
||||
msg('Camera navigation policy unavailable'),
|
||||
msg('Nearest-aircraft selection supports Flights or Military Flights only'),
|
||||
msg(
|
||||
'Nearest-aircraft selection needs a preset, place name, or latitude and longitude',
|
||||
),
|
||||
msg('Could not arrive at the requested place'),
|
||||
msg('The nearest airborne aircraft could not be selected'),
|
||||
].map((text) => [text, text]),
|
||||
);
|
||||
|
||||
function noneWithinView(target, radius) {
|
||||
switch (target) {
|
||||
case 'flights':
|
||||
return t('No flights within {radius} of the current view', { radius });
|
||||
case 'military':
|
||||
return t('No military within {radius} of the current view', { radius });
|
||||
case 'satellites':
|
||||
return t('No satellites within {radius} of the current view', {
|
||||
radius,
|
||||
});
|
||||
case 'vessels':
|
||||
return t('No vessels within {radius} of the current view', { radius });
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const ERROR_PATTERNS = [
|
||||
[
|
||||
/^(.+) is enabled on the (.+) feed, but no airborne aircraft is loaded in the (.+) view yet$/,
|
||||
([, layer, source, place]) =>
|
||||
t(
|
||||
'{layer} is enabled on the {source} feed, but no airborne aircraft is loaded in the {place} view yet',
|
||||
{
|
||||
layer: shownLayerName(layer),
|
||||
source: sourceLabel(source),
|
||||
place: localizePlaceName(place),
|
||||
},
|
||||
),
|
||||
],
|
||||
[
|
||||
/^(.+) is enabled, but no airborne aircraft is loaded in the (.+) view yet$/,
|
||||
([, layer, place]) =>
|
||||
t(
|
||||
'{layer} is enabled, but no airborne aircraft is loaded in the {place} view yet',
|
||||
{ layer: shownLayerName(layer), place: localizePlaceName(place) },
|
||||
),
|
||||
],
|
||||
[
|
||||
/^(.+) is enabled, but its aircraft feed is unavailable$/,
|
||||
([, layer]) =>
|
||||
t('{layer} is enabled, but its aircraft feed is unavailable', {
|
||||
layer: shownLayerName(layer),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^(.+) is enabled, but its destination refresh did not complete$/,
|
||||
([, layer]) =>
|
||||
t('{layer} is enabled, but its destination refresh did not complete', {
|
||||
layer: shownLayerName(layer),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^(.+) is enabled, but (.+) is unavailable$/,
|
||||
([, layer, source]) =>
|
||||
t('{layer} is enabled, but {source} is unavailable', {
|
||||
layer: shownLayerName(layer),
|
||||
source: sourceLabel(source),
|
||||
}),
|
||||
],
|
||||
[
|
||||
/^Could not arrive at (.+)$/,
|
||||
([, place]) =>
|
||||
t('Could not arrive at {place}', { place: localizePlaceName(place) }),
|
||||
],
|
||||
[
|
||||
/^(.+) is no longer loaded$/,
|
||||
([, name]) => t('{name} is no longer loaded', { name }),
|
||||
],
|
||||
[
|
||||
/^Unknown target layer: (.*)$/,
|
||||
([, target]) => t('Unknown target layer: {target}', { target }),
|
||||
],
|
||||
[
|
||||
/^No (\S+) within (\d+(?:\.\d+)?) km of the current view$/,
|
||||
([, target, km]) => noneWithinView(target, shownKm(km)),
|
||||
],
|
||||
];
|
||||
|
||||
/** A tool error as the card shows it. */
|
||||
function shownError(error) {
|
||||
if (typeof error !== 'string' || !error) return null;
|
||||
const known = KNOWN_ERRORS.get(error);
|
||||
if (known) return t(known);
|
||||
for (const [pattern, show] of ERROR_PATTERNS) {
|
||||
const match = pattern.exec(error);
|
||||
const shown = match ? show(match) : null;
|
||||
if (shown) return shown;
|
||||
}
|
||||
return translateKnown(error);
|
||||
}
|
||||
|
||||
// The card wraps a failed result's lines rather than cutting them: a
|
||||
// translated error runs longer than the English (voiceCardPresentation).
|
||||
function errorLines(result) {
|
||||
return [shownError(result?.error)].filter(Boolean);
|
||||
}
|
||||
|
||||
function sentence(text) {
|
||||
@@ -76,9 +259,110 @@ function feedTag(state) {
|
||||
return MATERIAL_FEED_TAGS[state] || null;
|
||||
}
|
||||
|
||||
/** Feed states as the card shows them; the state values stay English. */
|
||||
const FEED_STATE_LABELS = Object.freeze({
|
||||
nominal: msg('nominal'),
|
||||
loading: msg('loading'),
|
||||
degraded: msg('degraded'),
|
||||
stale: msg('stale'),
|
||||
partial: msg('partial'),
|
||||
fallback: msg('fallback'),
|
||||
unavailable: msg('unavailable'),
|
||||
});
|
||||
|
||||
function feedStateLabel(state) {
|
||||
const key = FEED_STATE_LABELS[state];
|
||||
return key ? t(key) : state;
|
||||
}
|
||||
|
||||
/** A feed chip only when the state is worth seeing; nominal is the default. */
|
||||
function feedChips(state) {
|
||||
return state && state !== 'nominal' ? [{ label: state }] : [];
|
||||
return state && state !== 'nominal' ? [{ label: feedStateLabel(state) }] : [];
|
||||
}
|
||||
|
||||
/** '{label} on' / '{label} off' as the card shows it. */
|
||||
function switchedTitle(label, on) {
|
||||
return on ? t('{label} on', { label }) : t('{label} off', { label });
|
||||
}
|
||||
|
||||
// Card titles name the layer's things in whole phrases, so each language can
|
||||
// inflect the noun; the English matches LAYER_NOUNS.
|
||||
function frameTitle(layerId) {
|
||||
switch (layerId) {
|
||||
case 'flights':
|
||||
return t('Frame aircraft');
|
||||
case 'military':
|
||||
return t('Frame military aircraft');
|
||||
case 'ais-live-vessels':
|
||||
return t('Frame ships');
|
||||
case 'satellites':
|
||||
return t('Frame satellites');
|
||||
default:
|
||||
return t('Frame contacts');
|
||||
}
|
||||
}
|
||||
|
||||
function inFrameTitle(layerId, count) {
|
||||
switch (layerId) {
|
||||
case 'flights':
|
||||
return tn(
|
||||
count,
|
||||
'{count} aircraft in frame',
|
||||
'{count} aircraft in frame',
|
||||
);
|
||||
case 'military':
|
||||
return tn(
|
||||
count,
|
||||
'{count} military aircraft in frame',
|
||||
'{count} military aircraft in frame',
|
||||
);
|
||||
case 'ais-live-vessels':
|
||||
return tn(count, '{count} ships in frame', '{count} ships in frame');
|
||||
case 'satellites':
|
||||
return tn(
|
||||
count,
|
||||
'{count} satellites in frame',
|
||||
'{count} satellites in frame',
|
||||
);
|
||||
default:
|
||||
return tn(
|
||||
count,
|
||||
'{count} contacts in frame',
|
||||
'{count} contacts in frame',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function nearestTitle(layerId) {
|
||||
switch (layerId) {
|
||||
case 'military':
|
||||
return t('Nearest military aircraft');
|
||||
case 'ais-live-vessels':
|
||||
return t('Nearest ships');
|
||||
case 'satellites':
|
||||
return t('Nearest satellites');
|
||||
default:
|
||||
return t('Nearest aircraft');
|
||||
}
|
||||
}
|
||||
|
||||
/** Two shown items joined, or the one item. */
|
||||
function shownPair(items) {
|
||||
return items.length === 2
|
||||
? t('{first} and {second}', { first: items[0], second: items[1] })
|
||||
: items[0] || '';
|
||||
}
|
||||
|
||||
/** A preset city id (`sf`, `nyc`) as its English name; any other value unchanged. */
|
||||
function presetPlaceName(value) {
|
||||
const id = typeof value === 'string' ? value.trim() : '';
|
||||
return Object.hasOwn(CITY_POIS, id) ? CITY_POIS[id].name : value;
|
||||
}
|
||||
|
||||
// A place as the card shows it: a preset city in the interface language,
|
||||
// whether the call named it ("Tokyo") or gave its id ("tokyo", "nyc").
|
||||
function shownPlace(value, max) {
|
||||
return localizePlaceName(displayName(presetPlaceName(value), max));
|
||||
}
|
||||
|
||||
/** Ids such as `austin` or `local-firms` read as names on screen. */
|
||||
@@ -96,6 +380,12 @@ function layerLabel(result) {
|
||||
return spokenLabel(result?.label || result?.layerId || 'Layer');
|
||||
}
|
||||
|
||||
/** The layer as the card names it, in the active language. */
|
||||
function shownLayerLabel(result) {
|
||||
const name = shownLayerName(result?.label, result?.layerId);
|
||||
return spokenLabel(name || t('Layer'));
|
||||
}
|
||||
|
||||
/* ---------------- per-tool builders ---------------- */
|
||||
|
||||
function setLayerVisibility(result, args = {}) {
|
||||
@@ -105,7 +395,7 @@ function setLayerVisibility(result, args = {}) {
|
||||
const wanted = args.enabled === false ? 'off' : 'on';
|
||||
return {
|
||||
say: `Couldn't turn ${wanted} ${label}.`,
|
||||
display: { title: label, lines: [result?.error].filter(Boolean) },
|
||||
display: { title: shownLayerLabel(result), lines: errorLines(result) },
|
||||
};
|
||||
}
|
||||
const on = result.enabled !== false;
|
||||
@@ -117,28 +407,35 @@ function setLayerVisibility(result, args = {}) {
|
||||
return {
|
||||
say,
|
||||
display: {
|
||||
title: `${label} ${on ? 'on' : 'off'}`,
|
||||
title: switchedTitle(shownLayerLabel(result), on),
|
||||
chips: on ? feedChips(result.feedState) : [],
|
||||
sources: result.source ? [{ label: result.source }] : [],
|
||||
sources: shownSources(result.source),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function flyToLocation(result) {
|
||||
const label = spokenLabel(result?.label || result?.query || 'there');
|
||||
const title =
|
||||
spokenLabel(localizePlaceName(result?.label || result?.query)) ||
|
||||
t('there');
|
||||
if (result?.cancelled) return null;
|
||||
if (!result?.ok) {
|
||||
return {
|
||||
say: sentence(`Couldn't find ${label}`),
|
||||
display: { title: label, lines: [result?.error].filter(Boolean) },
|
||||
display: { title, lines: errorLines(result) },
|
||||
};
|
||||
}
|
||||
return {
|
||||
say: sentence(result.arrived ? `Over ${label}` : `Flying to ${label}`),
|
||||
display: {
|
||||
title: label,
|
||||
title,
|
||||
lines: result.rangeM
|
||||
? [`Camera range ${kilometers(result.rangeM / 1000)}`]
|
||||
? [
|
||||
t('Camera range {range}', {
|
||||
range: shownKm(result.rangeM / 1000),
|
||||
}),
|
||||
]
|
||||
: [],
|
||||
},
|
||||
};
|
||||
@@ -162,8 +459,8 @@ function frameOverhead(result) {
|
||||
return {
|
||||
say,
|
||||
display: {
|
||||
title: `Frame ${noun}`,
|
||||
lines: [result?.error].filter(Boolean),
|
||||
title: frameTitle(result?.layerId),
|
||||
lines: errorLines(result),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -181,12 +478,15 @@ function frameOverhead(result) {
|
||||
return {
|
||||
say: sentence(`Framed ${countText} ${noun} within ${radius}`),
|
||||
display: {
|
||||
title: `${count} ${noun} in frame`,
|
||||
title: inFrameTitle(result.layerId, count),
|
||||
lines: [
|
||||
`Within ${radius} of the view`,
|
||||
result.detectionEnabled ? 'Labels on' : null,
|
||||
t('Within {radius} of the view', {
|
||||
radius: shownKm(result?.radiusKm),
|
||||
}),
|
||||
result.detectionEnabled ? t('Labels on') : null,
|
||||
].filter(Boolean),
|
||||
notes: count >= 80 ? ['Framing uses the nearest 80 loaded contacts'] : [],
|
||||
notes:
|
||||
count >= 80 ? [t('Framing uses the nearest 80 loaded contacts')] : [],
|
||||
},
|
||||
referents,
|
||||
};
|
||||
@@ -197,8 +497,12 @@ function selectNearestAircraft(result) {
|
||||
result?.location?.label || result?.location || 'there',
|
||||
);
|
||||
const noun = LAYER_NOUNS[result?.layerId] || 'aircraft';
|
||||
const shownLocation =
|
||||
spokenLabel(
|
||||
localizePlaceName(result?.location?.label || result?.location),
|
||||
) || t('there');
|
||||
const feed = result?.feed || {};
|
||||
const sources = feed.source ? [{ label: feed.source }] : [];
|
||||
const sources = shownSources(feed.source);
|
||||
if (result?.cancelled) return null;
|
||||
if (!result?.ok) {
|
||||
let say;
|
||||
@@ -212,8 +516,8 @@ function selectNearestAircraft(result) {
|
||||
return {
|
||||
say,
|
||||
display: {
|
||||
title: `Nearest ${noun}`,
|
||||
lines: [result?.error].filter(Boolean),
|
||||
title: nearestTitle(result?.layerId),
|
||||
lines: errorLines(result),
|
||||
chips: feedChips(feed.state),
|
||||
sources,
|
||||
},
|
||||
@@ -239,8 +543,17 @@ function selectNearestAircraft(result) {
|
||||
display: {
|
||||
title: name,
|
||||
lines: [
|
||||
distance ? `${distance} from ${location}` : null,
|
||||
altitude ? `Altitude ${altitude}` : null,
|
||||
distance
|
||||
? t('{distance} from {location}', {
|
||||
distance: shownKm(aircraft.distanceKm),
|
||||
location: shownLocation,
|
||||
})
|
||||
: null,
|
||||
altitude
|
||||
? t('Altitude {altitude}', {
|
||||
altitude: shownFeet(aircraft.altitudeM),
|
||||
})
|
||||
: null,
|
||||
].filter(Boolean),
|
||||
chips: feedChips(feed.state),
|
||||
sources,
|
||||
@@ -251,14 +564,30 @@ function selectNearestAircraft(result) {
|
||||
};
|
||||
}
|
||||
|
||||
function routeText(properties) {
|
||||
const SPOKEN_ROUTE = Object.freeze({
|
||||
between: (origin, destination) => `${origin} to ${destination}`,
|
||||
from: (origin) => `from ${origin}`,
|
||||
to: (destination) => `to ${destination}`,
|
||||
});
|
||||
|
||||
const SHOWN_ROUTE = Object.freeze({
|
||||
between: (origin, destination) =>
|
||||
t('{origin} to {destination}', { origin, destination }),
|
||||
from: (origin) => t('from {origin}', { origin }),
|
||||
to: (destination) => t('to {destination}', { destination }),
|
||||
});
|
||||
|
||||
function routeText(properties, words = SPOKEN_ROUTE) {
|
||||
const route = spokenLabel(properties.route || '', 40);
|
||||
if (route) return route.replace(/\s*(?:-|→|>)\s*/g, ' to ');
|
||||
if (route)
|
||||
return route
|
||||
.split(/\s*(?:-|→|>)\s*/)
|
||||
.reduce((origin, destination) => words.between(origin, destination));
|
||||
const origin = spokenLabel(properties.routeOrigin || '', 12);
|
||||
const destination = spokenLabel(properties.routeDestination || '', 12);
|
||||
if (origin && destination) return `${origin} to ${destination}`;
|
||||
if (origin) return `from ${origin}`;
|
||||
if (destination) return `to ${destination}`;
|
||||
if (origin && destination) return words.between(origin, destination);
|
||||
if (origin) return words.from(origin);
|
||||
if (destination) return words.to(destination);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -275,9 +604,13 @@ function getEntityContext(result) {
|
||||
return {
|
||||
say: null,
|
||||
display: {
|
||||
title,
|
||||
lines: [selected.layerName].filter(Boolean),
|
||||
sources: selected.source ? [{ label: selected.source }] : [],
|
||||
title: spokenLabel(shownEntityLabel(title)),
|
||||
lines: [
|
||||
selected.layerName
|
||||
? shownLayerName(selected.layerName, selected.layerId)
|
||||
: null,
|
||||
].filter(Boolean),
|
||||
sources: shownSources(selected.source),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -301,11 +634,15 @@ function getEntityContext(result) {
|
||||
title,
|
||||
lines: [
|
||||
properties.registration
|
||||
? `Registration ${spokenLabel(properties.registration, 16)}`
|
||||
? t('Registration {registration}', {
|
||||
registration: spokenLabel(properties.registration, 16),
|
||||
})
|
||||
: null,
|
||||
route
|
||||
? t('Route {route}', { route: routeText(properties, SHOWN_ROUTE) })
|
||||
: null,
|
||||
route ? `Route ${route}` : null,
|
||||
].filter(Boolean),
|
||||
sources: selected.source ? [{ label: selected.source }] : [],
|
||||
sources: shownSources(selected.source),
|
||||
},
|
||||
referents: [
|
||||
{
|
||||
@@ -350,10 +687,10 @@ function getCurrentViewState(result) {
|
||||
.map((part) => sentence(part[0].toUpperCase() + part.slice(1)))
|
||||
.join(' '),
|
||||
display: {
|
||||
title: 'View state',
|
||||
title: t('View state'),
|
||||
lines: enabled.map(
|
||||
(layer) =>
|
||||
`${spokenLabel(layer.name || layer.id, 32)} · ${layer.count ?? 0} · ${layer.feedState || 'nominal'}`,
|
||||
`${spokenLabel(shownLayerName(layer.name, layer.id), 32)} · ${layer.count ?? 0} · ${feedStateLabel(layer.feedState || 'nominal')}`,
|
||||
),
|
||||
},
|
||||
schedule: 'silent',
|
||||
@@ -387,13 +724,15 @@ function annotateMap(result) {
|
||||
say: parts.length ? parts.join(' ') : null,
|
||||
display: {
|
||||
title: result?.ok
|
||||
? `Marked ${drawn.length} ${drawn.length === 1 ? 'place' : 'places'}`
|
||||
: 'Nothing marked',
|
||||
? tn(drawn.length, 'Marked {count} place', 'Marked {count} places')
|
||||
: t('Nothing marked'),
|
||||
// The numbered referents list the marks; lines stay for extra facts.
|
||||
lines: [],
|
||||
notes: [
|
||||
...(failed.length ? [`Not found: ${failed.join(', ')}`] : []),
|
||||
...(result?.outlinePending ? ['Tracing outlines'] : []),
|
||||
...(failed.length
|
||||
? [t('Not found: {places}', { places: failed.join(', ') })]
|
||||
: []),
|
||||
...(result?.outlinePending ? [t('Tracing outlines')] : []),
|
||||
],
|
||||
},
|
||||
referents,
|
||||
@@ -487,29 +826,99 @@ function annotationTargets(args) {
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
// frame_overhead targets as the schema names them.
|
||||
function frameStepLabel(target) {
|
||||
switch (target) {
|
||||
case 'flights':
|
||||
return t('Frame flights');
|
||||
case 'military':
|
||||
return t('Frame military');
|
||||
case 'satellites':
|
||||
return t('Frame satellites');
|
||||
case 'vessels':
|
||||
return t('Frame vessels');
|
||||
default:
|
||||
return t('Frame {target}', { target });
|
||||
}
|
||||
}
|
||||
|
||||
// Plan-step labels for the other tools in actionSchemas. A tool missing here
|
||||
// shows its raw name in English only.
|
||||
const STEP_TITLES = Object.freeze({
|
||||
zoom_to_globe: msg('Zoom out to globe'),
|
||||
show_data_layers_menu: msg('Open data layers'),
|
||||
set_panel_open: msg('Open or close a panel'),
|
||||
set_voice_inactivity_timeout: msg('Voice inactivity timeout'),
|
||||
set_context_mode: msg('Change context mode'),
|
||||
control_cockpit: msg('Control cockpit'),
|
||||
set_visual_style: msg('Change visual style'),
|
||||
set_hud: msg('Adjust HUD'),
|
||||
set_cyber_sonar: msg('Adjust cyber sonar'),
|
||||
set_detection: msg('Adjust detection overlay'),
|
||||
set_map_stack: msg('Change map stack'),
|
||||
set_post_processing: msg('Adjust post-processing'),
|
||||
control_scene: msg('Control scene'),
|
||||
control_cctv: msg('Control CCTV'),
|
||||
control_radio: msg('Control radio'),
|
||||
track_entity: msg('Track contact'),
|
||||
stop_tracking: msg('Stop tracking'),
|
||||
clear_annotations: msg('Clear marks'),
|
||||
move_camera: msg('Move camera'),
|
||||
fly_route: msg('Fly route'),
|
||||
analyst_query: msg('Analyst query'),
|
||||
next_iss_pass: msg('Next ISS pass'),
|
||||
next_satellite_pass: msg('Next satellite pass'),
|
||||
});
|
||||
|
||||
/** A short on-screen label for one tool call in the voice card's plan. */
|
||||
export function planStepLabel(name, args = {}) {
|
||||
if (Object.hasOwn(STEP_TITLES, name)) return t(STEP_TITLES[name]);
|
||||
switch (name) {
|
||||
case 'annotate_map': {
|
||||
const targets = annotationTargets(args);
|
||||
return targets.length
|
||||
? `Mark ${listPhrase(targets.slice(0, 2))}${targets.length > 2 ? ` +${targets.length - 2}` : ''}`
|
||||
: 'Mark the map';
|
||||
if (!targets.length) return t('Mark the map');
|
||||
const shown = shownPair(targets.slice(0, 2));
|
||||
return targets.length > 2
|
||||
? t('Mark {targets} +{more}', {
|
||||
targets: shown,
|
||||
more: targets.length - 2,
|
||||
})
|
||||
: t('Mark {targets}', { targets: shown });
|
||||
}
|
||||
case 'fly_to_location':
|
||||
return `Fly to ${displayName(args.query || args.locationId || 'location', 40)}`;
|
||||
case 'select_nearest_aircraft':
|
||||
return `Nearest aircraft to ${displayName(args.locationQuery || args.locationId || 'here', 32)}`;
|
||||
case 'set_layer_visibility':
|
||||
return `${displayName(args.layerId || 'Layer', 32)} ${args.enabled === false ? 'off' : 'on'}`;
|
||||
case 'fly_to_location': {
|
||||
const place = args.query || args.locationId;
|
||||
return place
|
||||
? t('Fly to {place}', { place: shownPlace(place, 40) })
|
||||
: t('Fly to Location');
|
||||
}
|
||||
case 'select_nearest_aircraft': {
|
||||
const place = args.locationQuery || args.locationId;
|
||||
return place
|
||||
? t('Nearest aircraft to {place}', { place: shownPlace(place, 32) })
|
||||
: t('Nearest aircraft to Here');
|
||||
}
|
||||
case 'set_layer_visibility': {
|
||||
const on = args.enabled !== false;
|
||||
const label =
|
||||
layerNameForId(args.layerId) ||
|
||||
(args.layerId ? displayName(args.layerId, 32) : '');
|
||||
// "Layer" alone is a noun the languages inflect the state for.
|
||||
if (!label) return on ? t('Layer on') : t('Layer off');
|
||||
return switchedTitle(label, on);
|
||||
}
|
||||
case 'adjust_camera_zoom':
|
||||
return args.direction === 'out' ? t('Zoom out') : t('Zoom in');
|
||||
case 'get_entity_context':
|
||||
return 'Read the view';
|
||||
return t('Read the view');
|
||||
case 'get_current_view_state':
|
||||
return 'Check view state';
|
||||
return t('Check view state');
|
||||
case 'frame_overhead':
|
||||
return `Frame ${spokenLabel(args.target || 'flights', 24)}`;
|
||||
return frameStepLabel(spokenLabel(args.target || 'flights', 24));
|
||||
default:
|
||||
return spokenLabel(String(name || 'Action').replace(/_/g, ' '), 40);
|
||||
// A tool with no label: its name in English, "Action" elsewhere.
|
||||
return getLocale() === 'en'
|
||||
? spokenLabel(String(name || 'Action').replace(/_/g, ' '), 40)
|
||||
: t('Action');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -522,9 +931,15 @@ export function narrationLabel(name, args = {}) {
|
||||
case 'annotate_map':
|
||||
return annotationTargets(args)[0] || '';
|
||||
case 'fly_to_location':
|
||||
return displayName(args.query || args.locationId || '', 32);
|
||||
return displayName(
|
||||
presetPlaceName(args.query || args.locationId || ''),
|
||||
32,
|
||||
);
|
||||
case 'select_nearest_aircraft':
|
||||
return displayName(args.locationQuery || args.locationId || '', 32);
|
||||
return displayName(
|
||||
presetPlaceName(args.locationQuery || args.locationId || ''),
|
||||
32,
|
||||
);
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
@@ -532,18 +947,22 @@ export function narrationLabel(name, args = {}) {
|
||||
|
||||
/** On-screen labels for progress steps. */
|
||||
const STEP_LABELS = Object.freeze({
|
||||
resolve: 'Finding places',
|
||||
outline: 'Tracing outline',
|
||||
search: 'Looking up place',
|
||||
fly: 'Flying',
|
||||
layer: 'Turning on layer',
|
||||
refresh: 'Loading aircraft',
|
||||
nearest: 'Picking nearest',
|
||||
resolve: msg('Finding places'),
|
||||
outline: msg('Tracing outline'),
|
||||
search: msg('Looking up place'),
|
||||
fly: msg('Flying'),
|
||||
layer: msg('Turning on layer'),
|
||||
refresh: msg('Loading aircraft'),
|
||||
nearest: msg('Picking nearest'),
|
||||
});
|
||||
|
||||
/** On-screen label for a progress step. */
|
||||
export function progressStepLabel(step, label) {
|
||||
const base = STEP_LABELS[step] || spokenLabel(step || 'Working', 24);
|
||||
const base = STEP_LABELS[step]
|
||||
? t(STEP_LABELS[step])
|
||||
: step
|
||||
? spokenLabel(step, 24)
|
||||
: t('Working');
|
||||
return label ? `${base}: ${spokenLabel(label, 32)}` : base;
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* Phrases that add length without adding an answer. Tool numbers are spoken
|
||||
* verbatim, so approximations are hedges too.
|
||||
*/
|
||||
export const HEDGE_PHRASES = Object.freeze([
|
||||
const HEDGE_PHRASES = Object.freeze([
|
||||
'keep in mind',
|
||||
"it's worth noting",
|
||||
'it is worth noting',
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { countWords, findHedges } from './speechLint.js';
|
||||
|
||||
/** Quiet period after the last audio or response before a turn is closed. */
|
||||
export const TURN_SETTLE_MS = 2500;
|
||||
const TURN_SETTLE_MS = 2500;
|
||||
|
||||
/**
|
||||
* Per-turn latency and speech spans for one voice session.
|
||||
|
||||
+57
-4
@@ -5,7 +5,7 @@ import {
|
||||
} from './voiceCardPresentation.js';
|
||||
|
||||
/** Hide an idle card this long after its last update. */
|
||||
export const VOICE_CARD_IDLE_MS = 14000;
|
||||
const VOICE_CARD_IDLE_MS = 14000;
|
||||
|
||||
const ELEMENT_IDS = Object.freeze({
|
||||
card: 'gev-voice-card',
|
||||
@@ -13,6 +13,7 @@ const ELEMENT_IDS = Object.freeze({
|
||||
dismiss: 'gev-voice-card-dismiss',
|
||||
user: 'gev-voice-card-user',
|
||||
userText: 'gev-voice-card-user-text',
|
||||
userWho: 'gev-voice-card-user-who',
|
||||
pointer: 'gev-voice-card-pointer',
|
||||
pointerKind: 'gev-voice-card-pointer-kind',
|
||||
pointerText: 'gev-voice-card-pointer-text',
|
||||
@@ -70,8 +71,9 @@ export class VoiceCardControls {
|
||||
idleMs = VOICE_CARD_IDLE_MS,
|
||||
setTimer = (fn, ms) => setTimeout(fn, ms),
|
||||
clearTimer = (id) => clearTimeout(id),
|
||||
onRender,
|
||||
}) {
|
||||
Object.assign(this, { idleMs, setTimer, clearTimer });
|
||||
Object.assign(this, { idleMs, setTimer, clearTimer, onRender });
|
||||
this.elements = Object.fromEntries(
|
||||
Object.entries(ELEMENT_IDS).map(([key, id]) => [
|
||||
key,
|
||||
@@ -79,6 +81,10 @@ export class VoiceCardControls {
|
||||
]),
|
||||
);
|
||||
this.state = initialVoiceCardState();
|
||||
this.enabled = true;
|
||||
this.presented = true;
|
||||
this.announcementRevision = 0;
|
||||
this.consumedPresentationAnnouncement = -1;
|
||||
this.hideTimer = null;
|
||||
this.destroyed = false;
|
||||
this.listeners = new AbortController();
|
||||
@@ -96,17 +102,53 @@ export class VoiceCardControls {
|
||||
if (this.destroyed || !this.elements.card) return;
|
||||
const next = reduceVoiceCard(this.state, event);
|
||||
if (next === this.state) return;
|
||||
if (
|
||||
next.announce !== this.state.announce ||
|
||||
(next.announce &&
|
||||
((event.type === 'transcript' &&
|
||||
event.role === 'assistant' &&
|
||||
event.final) ||
|
||||
(event.type === 'action-result' &&
|
||||
next.result !== this.state.result) ||
|
||||
(event.type === 'pointer' &&
|
||||
next.pointer &&
|
||||
next.pointer !== this.state.pointer)))
|
||||
)
|
||||
this.announcementRevision++;
|
||||
this.state = next;
|
||||
this.render();
|
||||
if (this.state.visible) this.scheduleIdle();
|
||||
else this.cancelIdle();
|
||||
}
|
||||
|
||||
/** A local submission allows future output after dismissal; keep its content. */
|
||||
rearm() {
|
||||
if (this.destroyed || !this.elements.card) return;
|
||||
this.state = { ...this.state, dismissed: false };
|
||||
}
|
||||
|
||||
/** An annotation outline resolved or failed after its tool result. */
|
||||
outline(event) {
|
||||
this.handle({ type: 'outline', id: event?.id, status: event?.status });
|
||||
}
|
||||
|
||||
/** Change presentation only; session events and the idle deadline continue. */
|
||||
setEnabled(enabled) {
|
||||
if (this.destroyed || !this.elements.card) return;
|
||||
this.enabled = Boolean(enabled);
|
||||
this.render();
|
||||
}
|
||||
|
||||
/** View selection changes presentation without changing the saved preference. */
|
||||
setPresented(presented) {
|
||||
if (this.destroyed || !this.elements.card || this.presented === presented)
|
||||
return;
|
||||
if (!presented)
|
||||
this.consumedPresentationAnnouncement = this.announcementRevision;
|
||||
this.presented = presented;
|
||||
this.render();
|
||||
}
|
||||
|
||||
scheduleIdle() {
|
||||
this.cancelIdle();
|
||||
this.hideTimer = this.setTimer(() => {
|
||||
@@ -129,12 +171,13 @@ export class VoiceCardControls {
|
||||
|
||||
render() {
|
||||
const el = this.elements;
|
||||
const view = voiceCardView(this.state);
|
||||
const view = this.snapshot();
|
||||
setHidden(el.card, !view.visible);
|
||||
el.card.dataset.busy = String(view.busy);
|
||||
setText(el.phase, view.phase);
|
||||
setHidden(el.user, !view.user.visible);
|
||||
setText(el.userText, view.user.text);
|
||||
setText(el.userWho, view.user.label);
|
||||
setHidden(el.pointer, !view.pointer.visible);
|
||||
setText(el.pointerKind, view.pointer.kind);
|
||||
setText(el.pointerText, view.pointer.text);
|
||||
@@ -186,11 +229,21 @@ export class VoiceCardControls {
|
||||
node.textContent = note;
|
||||
});
|
||||
setText(el.live, view.announce);
|
||||
this.onRender?.();
|
||||
}
|
||||
|
||||
/** Test/diagnostic view of what the card shows. */
|
||||
snapshot() {
|
||||
return voiceCardView(this.state);
|
||||
const view = voiceCardView(this.state);
|
||||
return this.enabled && this.presented
|
||||
? {
|
||||
...view,
|
||||
announce:
|
||||
this.consumedPresentationAnnouncement === this.announcementRevision
|
||||
? ''
|
||||
: view.announce,
|
||||
}
|
||||
: { ...view, visible: false, announce: '' };
|
||||
}
|
||||
|
||||
/** Idempotent: removes listeners and the idle timer; markup stays owned by the control. */
|
||||
|
||||
+465
-47
@@ -3,13 +3,17 @@ import { readFileSync } from 'node:fs';
|
||||
import test from 'node:test';
|
||||
import { readStylesheet } from '../testSupport/readStylesheet.mjs';
|
||||
import { VoiceCardControls } from './voiceCard.js';
|
||||
import { getLocale, setLocale } from '../i18n/index.js';
|
||||
import {
|
||||
initialVoiceCardState,
|
||||
reduceVoiceCard,
|
||||
voiceCardView,
|
||||
} from './voiceCardPresentation.js';
|
||||
|
||||
const controlSource = readFileSync(new URL('./control.js', import.meta.url), 'utf8');
|
||||
const controlSource = readFileSync(
|
||||
new URL('./control.js', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
const css = readStylesheet(new URL('../../style.css', import.meta.url));
|
||||
|
||||
function fold(events) {
|
||||
@@ -19,11 +23,37 @@ function fold(events) {
|
||||
test('presentation: a slow annotate turn shows captions, plan steps and the result card', () => {
|
||||
const state = fold([
|
||||
{ type: 'interruption' },
|
||||
{ type: 'transcript', role: 'user', text: 'Annotate the Texas State Capitol', final: true },
|
||||
{ type: 'transcript', role: 'assistant', text: 'Finding the Capitol, ', responseId: 'r1' },
|
||||
{ type: 'transcript', role: 'assistant', text: 'then outlining it.', responseId: 'r1' },
|
||||
{ type: 'action-call', name: 'annotate_map', callId: 'call_a', arguments: { annotations: [{ target: 'Texas State Capitol' }] } },
|
||||
{ type: 'progress', name: 'annotate_map', callId: 'call_a', step: 'resolve', label: 'Texas State Capitol' },
|
||||
{
|
||||
type: 'transcript',
|
||||
role: 'user',
|
||||
text: 'Annotate the Texas State Capitol',
|
||||
final: true,
|
||||
},
|
||||
{
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'Finding the Capitol, ',
|
||||
responseId: 'r1',
|
||||
},
|
||||
{
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'then outlining it.',
|
||||
responseId: 'r1',
|
||||
},
|
||||
{
|
||||
type: 'action-call',
|
||||
name: 'annotate_map',
|
||||
callId: 'call_a',
|
||||
arguments: { annotations: [{ target: 'Texas State Capitol' }] },
|
||||
},
|
||||
{
|
||||
type: 'progress',
|
||||
name: 'annotate_map',
|
||||
callId: 'call_a',
|
||||
step: 'resolve',
|
||||
label: 'Texas State Capitol',
|
||||
},
|
||||
]);
|
||||
let view = voiceCardView(state);
|
||||
assert.equal(view.visible, true);
|
||||
@@ -31,9 +61,16 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
|
||||
assert.equal(view.phase, 'WORKING');
|
||||
assert.equal(view.user.text, 'Annotate the Texas State Capitol');
|
||||
assert.equal(view.assistant.text, 'Finding the Capitol, then outlining it.');
|
||||
assert.deepEqual(view.steps.map(({ label, status, detail }) => ({ label, status, detail })), [
|
||||
{ label: 'Mark Texas State Capitol', status: 'running', detail: 'Finding places: Texas State Capitol' },
|
||||
]);
|
||||
assert.deepEqual(
|
||||
view.steps.map(({ label, status, detail }) => ({ label, status, detail })),
|
||||
[
|
||||
{
|
||||
label: 'Mark Texas State Capitol',
|
||||
status: 'running',
|
||||
detail: 'Finding places: Texas State Capitol',
|
||||
},
|
||||
],
|
||||
);
|
||||
const done = [
|
||||
{
|
||||
type: 'action-result',
|
||||
@@ -43,7 +80,12 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
|
||||
ok: true,
|
||||
outlinePending: true,
|
||||
items: [{ ok: true, id: 'anno-1', outlinePending: true }],
|
||||
display: { title: 'Marked 1 place', lines: ['Texas State Capitol'], notes: ['Tracing outlines'], sources: [{ label: 'OpenStreetMap' }] },
|
||||
display: {
|
||||
title: 'Marked 1 place',
|
||||
lines: ['Texas State Capitol'],
|
||||
notes: ['Tracing outlines'],
|
||||
sources: [{ label: 'OpenStreetMap' }],
|
||||
},
|
||||
referents: [{ n: 1, id: 'anno-1', label: 'Texas State Capitol' }],
|
||||
},
|
||||
},
|
||||
@@ -53,35 +95,80 @@ test('presentation: a slow annotate turn shows captions, plan steps and the resu
|
||||
assert.equal(view.phase, 'DONE');
|
||||
assert.equal(view.steps[0].detail, 'Outline traced');
|
||||
assert.equal(view.result.title, 'Marked 1 place');
|
||||
assert.deepEqual(view.result.referents, [{ n: 1, label: 'Texas State Capitol' }]);
|
||||
assert.deepEqual(view.result.notes, ['Tracing outlines', 'Source: OpenStreetMap']);
|
||||
assert.equal(view.announce, 'Marked 1 place', 'the live region carries the result title');
|
||||
assert.deepEqual(view.result.referents, [
|
||||
{ n: 1, label: 'Texas State Capitol' },
|
||||
]);
|
||||
assert.deepEqual(view.result.notes, [
|
||||
'Tracing outlines',
|
||||
'Source: OpenStreetMap',
|
||||
]);
|
||||
assert.equal(
|
||||
view.announce,
|
||||
'Marked 1 place',
|
||||
'the live region carries the result title',
|
||||
);
|
||||
});
|
||||
|
||||
test('presentation: silent lookups, dismissal and session end', () => {
|
||||
const lookup = fold([
|
||||
{ type: 'interruption' },
|
||||
{ type: 'action-call', name: 'get_current_view_state', arguments: {} },
|
||||
{ type: 'action-result', name: 'get_current_view_state', result: { ok: true, schedule: 'silent', display: { title: 'View state' } } },
|
||||
{
|
||||
type: 'action-result',
|
||||
name: 'get_current_view_state',
|
||||
result: {
|
||||
ok: true,
|
||||
schedule: 'silent',
|
||||
display: { title: 'View state' },
|
||||
},
|
||||
},
|
||||
]);
|
||||
assert.equal(voiceCardView(lookup).result.visible, false, 'a silent lookup does not take the result card');
|
||||
assert.equal(
|
||||
voiceCardView(lookup).result.visible,
|
||||
false,
|
||||
'a silent lookup does not take the result card',
|
||||
);
|
||||
const dismissed = reduceVoiceCard(lookup, { type: 'dismiss' });
|
||||
assert.equal(voiceCardView(dismissed).visible, false);
|
||||
const stillDismissed = reduceVoiceCard(dismissed, { type: 'transcript', role: 'assistant', text: 'Hi', final: true });
|
||||
assert.equal(voiceCardView(stillDismissed).visible, false, 'dismissed until the next turn');
|
||||
assert.equal(voiceCardView(reduceVoiceCard(stillDismissed, { type: 'interruption' })).visible, true);
|
||||
assert.deepEqual(reduceVoiceCard(lookup, { type: 'state', state: 'idle' }), initialVoiceCardState());
|
||||
assert.equal(reduceVoiceCard(lookup, { type: 'turn-metrics' }), lookup, 'unrelated events are no-ops');
|
||||
const stillDismissed = reduceVoiceCard(dismissed, {
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'Hi',
|
||||
final: true,
|
||||
});
|
||||
assert.equal(
|
||||
voiceCardView(stillDismissed).visible,
|
||||
false,
|
||||
'dismissed until the next turn',
|
||||
);
|
||||
assert.equal(
|
||||
voiceCardView(reduceVoiceCard(stillDismissed, { type: 'interruption' }))
|
||||
.visible,
|
||||
true,
|
||||
);
|
||||
assert.deepEqual(
|
||||
reduceVoiceCard(lookup, { type: 'state', state: 'idle' }),
|
||||
initialVoiceCardState(),
|
||||
);
|
||||
assert.equal(
|
||||
reduceVoiceCard(lookup, { type: 'turn-metrics' }),
|
||||
lookup,
|
||||
'unrelated events are no-ops',
|
||||
);
|
||||
});
|
||||
|
||||
test('markup: the card nests inside the voice control with every element and a polite live region', () => {
|
||||
const root = controlSource.indexOf("root.id = 'gev-voice-control'");
|
||||
const card = controlSource.indexOf('id="gev-voice-card"');
|
||||
assert.ok(root >= 0 && card > root, 'the card is part of the voice control markup');
|
||||
assert.ok(
|
||||
root >= 0 && card > root,
|
||||
'the card is part of the voice control markup',
|
||||
);
|
||||
for (const id of [
|
||||
'gev-voice-card-phase',
|
||||
'gev-voice-card-dismiss',
|
||||
'gev-voice-card-user-text',
|
||||
'gev-voice-card-user-who',
|
||||
'gev-voice-card-assistant-text',
|
||||
'gev-voice-card-plan',
|
||||
'gev-voice-card-result-title',
|
||||
@@ -92,11 +179,22 @@ test('markup: the card nests inside the voice control with every element and a p
|
||||
]) {
|
||||
assert.match(controlSource, new RegExp(`id="${id}"`), `${id} is missing`);
|
||||
}
|
||||
assert.match(controlSource, /id="gev-voice-card-live"[^>]*aria-live="polite"/);
|
||||
assert.match(controlSource, /aria-label="Dismiss voice card"/);
|
||||
assert.match(
|
||||
controlSource,
|
||||
/id="gev-voice-card-live"[^>]*aria-live="polite"/,
|
||||
);
|
||||
assert.match(controlSource, /aria-label="\$\{t\('Dismiss voice card'\)\}"/);
|
||||
assert.match(css, /#gev-voice-control \.gev-voice-card \{/);
|
||||
assert.match(css, /body\.recording-mode #gev-voice-control/, 'recording mode hides the control, card included');
|
||||
assert.match(css, /body\.ui-clean-view #command-dock/, 'Clean-UI hides the dock, card included');
|
||||
assert.match(
|
||||
css,
|
||||
/body\.recording-mode #gev-voice-control/,
|
||||
'recording mode hides the control, card included',
|
||||
);
|
||||
assert.match(
|
||||
css,
|
||||
/body\.ui-clean-view #command-dock/,
|
||||
'Clean-UI hides the dock, card included',
|
||||
);
|
||||
});
|
||||
|
||||
function fakeRoot() {
|
||||
@@ -111,7 +209,12 @@ function fakeRoot() {
|
||||
ownerDocument: null,
|
||||
addEventListener(type, handler, options) {
|
||||
listeners.set(type, handler);
|
||||
options?.signal?.addEventListener('abort', () => listeners.delete(type));
|
||||
options?.signal?.addEventListener('abort', () =>
|
||||
listeners.delete(type),
|
||||
);
|
||||
},
|
||||
removeEventListener(type, handler) {
|
||||
if (listeners.get(type) === handler) listeners.delete(type);
|
||||
},
|
||||
fire: (type) => listeners.get(type)?.({}),
|
||||
listenerCount: () => listeners.size,
|
||||
@@ -128,6 +231,7 @@ function fakeRoot() {
|
||||
const nodes = new Map();
|
||||
const doc = { createElement: (tag) => ({ ...make(tag), tag }) };
|
||||
const root = {
|
||||
...make('gev-voice-control'),
|
||||
querySelector(selector) {
|
||||
const id = selector.slice(1);
|
||||
if (!nodes.has(id)) {
|
||||
@@ -160,12 +264,26 @@ test('controls: render, idle hide, dismiss and idempotent teardown', () => {
|
||||
});
|
||||
assert.equal(nodes.get('gev-voice-card').hidden, true, 'starts hidden');
|
||||
card.handle({ type: 'interruption' });
|
||||
card.handle({ type: 'action-call', name: 'fly_to_location', arguments: { query: 'Tokyo' } });
|
||||
card.handle({ type: 'action-result', name: 'fly_to_location', result: { ok: true, display: { title: 'Tokyo', lines: [], chips: [{ label: 'stale' }] } } });
|
||||
card.handle({
|
||||
type: 'action-call',
|
||||
name: 'fly_to_location',
|
||||
arguments: { query: 'Tokyo' },
|
||||
});
|
||||
card.handle({
|
||||
type: 'action-result',
|
||||
name: 'fly_to_location',
|
||||
result: {
|
||||
ok: true,
|
||||
display: { title: 'Tokyo', lines: [], chips: [{ label: 'stale' }] },
|
||||
},
|
||||
});
|
||||
assert.equal(nodes.get('gev-voice-card').hidden, false);
|
||||
assert.equal(nodes.get('gev-voice-card-result-title').textContent, 'Tokyo');
|
||||
assert.equal(nodes.get('gev-voice-card-plan').children.length, 1);
|
||||
assert.equal(nodes.get('gev-voice-card-chips').children[0].textContent, 'stale');
|
||||
assert.equal(
|
||||
nodes.get('gev-voice-card-chips').children[0].textContent,
|
||||
'stale',
|
||||
);
|
||||
assert.equal(timers.size, 1, 'one idle timer at a time');
|
||||
[...timers.values()][0]();
|
||||
assert.equal(nodes.get('gev-voice-card').hidden, true, 'hides when idle');
|
||||
@@ -177,7 +295,11 @@ test('controls: render, idle hide, dismiss and idempotent teardown', () => {
|
||||
assert.equal(timers.size, 0);
|
||||
assert.equal(nodes.get('gev-voice-card-dismiss').listenerCount(), 0);
|
||||
card.handle({ type: 'interruption' });
|
||||
assert.equal(nodes.get('gev-voice-card').hidden, true, 'a destroyed card ignores late events');
|
||||
assert.equal(
|
||||
nodes.get('gev-voice-card').hidden,
|
||||
true,
|
||||
'a destroyed card ignores late events',
|
||||
);
|
||||
});
|
||||
|
||||
test('controls: a control without card markup is inert', () => {
|
||||
@@ -189,17 +311,41 @@ test('controls: a control without card markup is inert', () => {
|
||||
test('presentation: concurrent calls of one tool update their own steps', () => {
|
||||
const state = fold([
|
||||
{ type: 'interruption' },
|
||||
{ type: 'action-call', name: 'fly_to_location', callId: 'c1', arguments: { query: 'Tokyo' } },
|
||||
{ type: 'action-call', name: 'fly_to_location', callId: 'c2', arguments: { query: 'Paris' } },
|
||||
{
|
||||
type: 'action-call',
|
||||
name: 'fly_to_location',
|
||||
callId: 'c1',
|
||||
arguments: { query: 'Tokyo' },
|
||||
},
|
||||
{
|
||||
type: 'action-call',
|
||||
name: 'fly_to_location',
|
||||
callId: 'c2',
|
||||
arguments: { query: 'Paris' },
|
||||
},
|
||||
// Out of order: the first call finishes last.
|
||||
{ type: 'action-result', name: 'fly_to_location', callId: 'c2', result: { ok: false, error: 'x' } },
|
||||
{ type: 'progress', name: 'fly_to_location', callId: 'c1', step: 'search', label: 'Tokyo' },
|
||||
{
|
||||
type: 'action-result',
|
||||
name: 'fly_to_location',
|
||||
callId: 'c2',
|
||||
result: { ok: false, error: 'x' },
|
||||
},
|
||||
{
|
||||
type: 'progress',
|
||||
name: 'fly_to_location',
|
||||
callId: 'c1',
|
||||
step: 'search',
|
||||
label: 'Tokyo',
|
||||
},
|
||||
]);
|
||||
const steps = voiceCardView(state).steps;
|
||||
assert.deepEqual(steps.map(({ label, status }) => [label, status]), [
|
||||
['Fly to Tokyo', 'running'],
|
||||
['Fly to Paris', 'failed'],
|
||||
]);
|
||||
assert.deepEqual(
|
||||
steps.map(({ label, status }) => [label, status]),
|
||||
[
|
||||
['Fly to Tokyo', 'running'],
|
||||
['Fly to Paris', 'failed'],
|
||||
],
|
||||
);
|
||||
assert.equal(steps[0].detail, 'Looking up place: Tokyo');
|
||||
});
|
||||
|
||||
@@ -207,27 +353,299 @@ test('presentation: stale and unmatched events never reopen the card', () => {
|
||||
const ended = fold([
|
||||
{ type: 'interruption' },
|
||||
{ type: 'action-call', name: 'annotate_map', callId: 'c1', arguments: {} },
|
||||
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, outlinePending: true, items: [{ ok: true, id: 'anno-9', outlinePending: true }] } },
|
||||
{
|
||||
type: 'action-result',
|
||||
name: 'annotate_map',
|
||||
callId: 'c1',
|
||||
result: {
|
||||
ok: true,
|
||||
outlinePending: true,
|
||||
items: [{ ok: true, id: 'anno-9', outlinePending: true }],
|
||||
},
|
||||
},
|
||||
{ type: 'state', state: 'idle' },
|
||||
]);
|
||||
for (const late of [
|
||||
{ type: 'outline', id: 'anno-9', status: 'resolved' },
|
||||
{ type: 'outline', status: 'resolved' },
|
||||
{ type: 'progress', name: 'annotate_map', callId: 'c1', step: 'resolve' },
|
||||
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, display: { title: 'Marked 1 place' } } },
|
||||
{
|
||||
type: 'action-result',
|
||||
name: 'annotate_map',
|
||||
callId: 'c1',
|
||||
result: { ok: true, display: { title: 'Marked 1 place' } },
|
||||
},
|
||||
]) {
|
||||
assert.equal(reduceVoiceCard(ended, late), ended, `${late.type} after stop is a no-op`);
|
||||
assert.equal(
|
||||
reduceVoiceCard(ended, late),
|
||||
ended,
|
||||
`${late.type} after stop is a no-op`,
|
||||
);
|
||||
}
|
||||
const live = fold([
|
||||
{ type: 'interruption' },
|
||||
{ type: 'action-call', name: 'annotate_map', callId: 'c1', arguments: {} },
|
||||
{ type: 'action-result', name: 'annotate_map', callId: 'c1', result: { ok: true, outlinePending: true, items: [{ ok: true, id: 'anno-1', outlinePending: true }, { ok: true, id: 'anno-2', outlinePending: true }] } },
|
||||
{
|
||||
type: 'action-result',
|
||||
name: 'annotate_map',
|
||||
callId: 'c1',
|
||||
result: {
|
||||
ok: true,
|
||||
outlinePending: true,
|
||||
items: [
|
||||
{ ok: true, id: 'anno-1', outlinePending: true },
|
||||
{ ok: true, id: 'anno-2', outlinePending: true },
|
||||
],
|
||||
},
|
||||
},
|
||||
{ type: 'dismiss' },
|
||||
]);
|
||||
assert.equal(reduceVoiceCard(live, { type: 'outline', id: 'anno-7', status: 'resolved' }), live, 'another mark is ignored');
|
||||
const first = reduceVoiceCard(live, { type: 'outline', id: 'anno-1', status: 'resolved' });
|
||||
assert.equal(voiceCardView(first).steps[0].detail, 'Tracing outline', 'one of two outlines still pending');
|
||||
assert.equal(voiceCardView(first).visible, false, 'a dismissed card stays hidden');
|
||||
const both = reduceVoiceCard(first, { type: 'outline', id: 'anno-2', status: 'failed' });
|
||||
assert.equal(
|
||||
reduceVoiceCard(live, {
|
||||
type: 'outline',
|
||||
id: 'anno-7',
|
||||
status: 'resolved',
|
||||
}),
|
||||
live,
|
||||
'another mark is ignored',
|
||||
);
|
||||
const first = reduceVoiceCard(live, {
|
||||
type: 'outline',
|
||||
id: 'anno-1',
|
||||
status: 'resolved',
|
||||
});
|
||||
assert.equal(
|
||||
voiceCardView(first).steps[0].detail,
|
||||
'Tracing outline',
|
||||
'one of two outlines still pending',
|
||||
);
|
||||
assert.equal(
|
||||
voiceCardView(first).visible,
|
||||
false,
|
||||
'a dismissed card stays hidden',
|
||||
);
|
||||
const both = reduceVoiceCard(first, {
|
||||
type: 'outline',
|
||||
id: 'anno-2',
|
||||
status: 'failed',
|
||||
});
|
||||
assert.equal(voiceCardView(both).steps[0].detail, 'Outline unavailable');
|
||||
});
|
||||
|
||||
test('presentation controls preserve real reducer content and the existing 14-second idle deadline', () => {
|
||||
const { root, nodes } = fakeRoot();
|
||||
const timers = new Map();
|
||||
let next = 0,
|
||||
renders = 0;
|
||||
const card = new VoiceCardControls({
|
||||
root,
|
||||
setTimer(fn, ms) {
|
||||
const id = ++next;
|
||||
timers.set(id, { fn, ms });
|
||||
return id;
|
||||
},
|
||||
clearTimer(id) {
|
||||
timers.delete(id);
|
||||
},
|
||||
onRender() {
|
||||
renders++;
|
||||
},
|
||||
});
|
||||
card.handle({ type: 'interruption' });
|
||||
card.handle({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'A current answer',
|
||||
final: true,
|
||||
responseId: 'r1',
|
||||
});
|
||||
const content = card.state;
|
||||
const timer = [...timers.keys()][0];
|
||||
assert.equal(timers.get(timer).ms, 14000);
|
||||
card.setPresented(false);
|
||||
assert.equal(card.snapshot().visible, false);
|
||||
assert.equal(nodes.get('gev-voice-card-live').textContent, '');
|
||||
card.setPresented(true);
|
||||
assert.equal(card.snapshot().visible, true);
|
||||
assert.equal(card.snapshot().assistant.text, 'A current answer');
|
||||
assert.equal(
|
||||
card.snapshot().announce,
|
||||
'',
|
||||
'returning to a tab does not repeat an already presented announcement',
|
||||
);
|
||||
assert.equal(card.state, content);
|
||||
assert.equal(
|
||||
[...timers.keys()][0],
|
||||
timer,
|
||||
'presentation switches do not reset idle',
|
||||
);
|
||||
card.setEnabled(false);
|
||||
card.handle({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'A later answer',
|
||||
final: true,
|
||||
responseId: 'r2',
|
||||
});
|
||||
assert.equal(card.snapshot().visible, false);
|
||||
card.setEnabled(true);
|
||||
assert.equal(card.snapshot().assistant.text, 'A later answer');
|
||||
assert.equal(card.snapshot().announce, 'A later answer');
|
||||
assert.ok(renders > 4);
|
||||
const idle = [...timers.values()][0];
|
||||
idle.fn();
|
||||
assert.equal(
|
||||
card.snapshot().visible,
|
||||
false,
|
||||
'original idle still ends content presentation',
|
||||
);
|
||||
card.destroy();
|
||||
const lastRenders = renders;
|
||||
card.setPresented(false);
|
||||
card.setEnabled(false);
|
||||
assert.equal(renders, lastRenders);
|
||||
});
|
||||
|
||||
test('a new hidden reply with identical words is still a fresh announcement', () => {
|
||||
const { root } = fakeRoot();
|
||||
const card = new VoiceCardControls({
|
||||
root,
|
||||
setTimer: () => 1,
|
||||
clearTimer() {},
|
||||
});
|
||||
card.handle({ type: 'interruption' });
|
||||
card.handle({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'Same words',
|
||||
final: true,
|
||||
responseId: 'r1',
|
||||
});
|
||||
card.setPresented(false);
|
||||
card.handle({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'Same words',
|
||||
final: true,
|
||||
responseId: 'r2',
|
||||
});
|
||||
card.setPresented(true);
|
||||
assert.equal(card.snapshot().announce, 'Same words');
|
||||
card.destroy();
|
||||
});
|
||||
|
||||
test('additive presentation gates retain the current localized reducer labels', () => {
|
||||
const previous = getLocale();
|
||||
try {
|
||||
setLocale('es');
|
||||
const { root } = fakeRoot();
|
||||
const card = new VoiceCardControls({
|
||||
root,
|
||||
setTimer: () => 1,
|
||||
clearTimer() {},
|
||||
});
|
||||
card.handle({ type: 'interruption' });
|
||||
card.handle({
|
||||
type: 'action-call',
|
||||
name: 'annotate_map',
|
||||
callId: 'a',
|
||||
arguments: {},
|
||||
});
|
||||
assert.equal(card.snapshot().phase, 'EN CURSO');
|
||||
const steps = card.state.steps;
|
||||
const result = card.state.result;
|
||||
const who = root.querySelector('#gev-voice-card-user-who');
|
||||
card.handle({ type: 'transcript', role: 'user', text: 'Same words', attribution: 'unverified' });
|
||||
assert.equal(card.state.userAttribution, 'unverified');
|
||||
assert.equal(card.snapshot().user.label, 'Habla reciente');
|
||||
assert.equal(who.textContent, 'Habla reciente');
|
||||
card.handle({ type: 'transcript', role: 'user', text: 'Same words' });
|
||||
assert.equal(card.state.userAttribution, null);
|
||||
assert.equal(card.snapshot().user.label, 'TÚ');
|
||||
assert.equal(who.textContent, 'TÚ');
|
||||
assert.equal(card.state.steps, steps);
|
||||
assert.equal(card.state.result, result);
|
||||
|
||||
card.setPresented(false);
|
||||
card.setPresented(true);
|
||||
assert.equal(card.snapshot().phase, 'EN CURSO');
|
||||
card.setEnabled(false);
|
||||
card.setEnabled(true);
|
||||
assert.equal(card.snapshot().phase, 'EN CURSO');
|
||||
card.destroy();
|
||||
} finally {
|
||||
setLocale(previous);
|
||||
}
|
||||
});
|
||||
|
||||
test('manual panel and preference compose with the actual card without changing voice content or enabling Preview', async () => {
|
||||
const { createManualVoicePanel } = await import('./panel/manualPanel.js');
|
||||
const { createVoicePreferences, bindPreferenceControls } =
|
||||
await import('./panel/preferences.js');
|
||||
const { root } = fakeRoot();
|
||||
let panel;
|
||||
const card = new VoiceCardControls({
|
||||
root,
|
||||
setTimer: () => 1,
|
||||
clearTimer() {},
|
||||
onRender: () => panel?.sync(),
|
||||
});
|
||||
const ui = {
|
||||
root,
|
||||
card: root.querySelector('#gev-voice-card'),
|
||||
voicePanel: root.querySelector('#panel'),
|
||||
previewPanel: root.querySelector('#preview-panel'),
|
||||
voiceSettingsPanel: root.querySelector('#settings-panel'),
|
||||
previewEmpty: root.querySelector('#empty'),
|
||||
voicePreviewInput: root.querySelector('#preview-input'),
|
||||
voicePreviewNote: root.querySelector('#preview-note'),
|
||||
};
|
||||
const preference = createVoicePreferences({ storage: null });
|
||||
const fields = bindPreferenceControls({
|
||||
ui,
|
||||
preferences: preference,
|
||||
copy: {
|
||||
previewNote: 'Mostrar respuestas.',
|
||||
timeoutNote: '',
|
||||
notSaved: 'No guardado.',
|
||||
},
|
||||
onChange: (state) => {
|
||||
card.setEnabled(state.previewEnabled);
|
||||
panel?.sync();
|
||||
},
|
||||
});
|
||||
card.setEnabled(preference.snapshot().previewEnabled);
|
||||
panel = createManualVoicePanel({
|
||||
ui,
|
||||
card,
|
||||
getPreviewEnabled: () => preference.snapshot().previewEnabled,
|
||||
copy: { previewEmpty: 'Sin respuesta.', previewDisabled: 'Desactivada.' },
|
||||
});
|
||||
card.handle({ type: 'interruption' });
|
||||
card.handle({
|
||||
type: 'transcript',
|
||||
role: 'assistant',
|
||||
text: 'Actual current answer',
|
||||
final: true,
|
||||
});
|
||||
const content = card.state;
|
||||
assert.equal(
|
||||
card.snapshot().visible,
|
||||
false,
|
||||
'manual panel stays closed despite content',
|
||||
);
|
||||
panel.select('preview');
|
||||
assert.equal(card.snapshot().assistant.text, 'Actual current answer');
|
||||
assert.equal(card.snapshot().visible, true);
|
||||
preference.setPreviewEnabled(false);
|
||||
assert.equal(card.snapshot().visible, false);
|
||||
assert.equal(ui.previewEmpty.textContent, 'Desactivada.');
|
||||
panel.select('settings');
|
||||
panel.select('preview');
|
||||
assert.equal(preference.snapshot().previewEnabled, false);
|
||||
assert.equal(card.snapshot().visible, false);
|
||||
assert.equal(card.state, content);
|
||||
fields.dispose();
|
||||
panel.destroy();
|
||||
card.destroy();
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
// A failed tool step: the card shows the tool error in the UI language; the
|
||||
// result keeps gevActions' English for the model.
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { setLocale } from '../i18n/index.js';
|
||||
import { attachVoiceResult, spokenLabel } from './speech.js';
|
||||
import {
|
||||
initialVoiceCardState,
|
||||
reduceVoiceCard,
|
||||
voiceCardView,
|
||||
} from './voiceCardPresentation.js';
|
||||
|
||||
// Error texts as gevActions.js builds them (frameOverhead, select_nearest_aircraft).
|
||||
const CASES = [
|
||||
{
|
||||
name: 'frame_overhead',
|
||||
args: { target: 'military' },
|
||||
result: {
|
||||
ok: false,
|
||||
action: 'frame_overhead',
|
||||
layerId: 'military',
|
||||
error: 'The military layer is not enabled',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'select_nearest_aircraft',
|
||||
args: { locationQuery: 'Madrid' },
|
||||
result: {
|
||||
ok: false,
|
||||
action: 'select_nearest_aircraft',
|
||||
stage: 'nearest',
|
||||
cancelled: false,
|
||||
error:
|
||||
'Live Flights is enabled, but no airborne aircraft is loaded in the Madrid view yet',
|
||||
location: { label: 'Madrid' },
|
||||
layer: { label: 'Live Flights' },
|
||||
feed: { state: 'nominal', source: 'OpenSky' },
|
||||
count: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'frame_overhead',
|
||||
args: { target: 'flights' },
|
||||
result: {
|
||||
ok: false,
|
||||
action: 'frame_overhead',
|
||||
layerId: 'flights',
|
||||
radiusKm: 150,
|
||||
count: 0,
|
||||
error: 'No flights within 150 km of the current view',
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
function cardFor(locale, { name, args, result }) {
|
||||
setLocale(locale);
|
||||
let state = reduceVoiceCard(initialVoiceCardState(), {
|
||||
type: 'action-call',
|
||||
id: 'c1',
|
||||
name,
|
||||
arguments: args,
|
||||
});
|
||||
state = reduceVoiceCard(state, {
|
||||
type: 'action-result',
|
||||
id: 'c1',
|
||||
name,
|
||||
result: attachVoiceResult(name, result, args),
|
||||
});
|
||||
return voiceCardView(state);
|
||||
}
|
||||
|
||||
for (const locale of ['es', 'ru']) {
|
||||
for (const c of CASES) {
|
||||
test(`${locale}: ${c.name} failure shows the card lines in the UI language`, () => {
|
||||
const view = cardFor(locale, c);
|
||||
try {
|
||||
assert.ok(view.result.visible, 'card shows a result');
|
||||
assert.notEqual(view.result.title, '', 'card has a title');
|
||||
// Title and step status are translated.
|
||||
assert.notEqual(
|
||||
view.steps[0].statusText,
|
||||
'failed',
|
||||
'step status translated',
|
||||
);
|
||||
// The error line is translated, not the raw English.
|
||||
for (const line of view.result.lines) {
|
||||
assert.ok(
|
||||
!/\b(is not enabled|no airborne aircraft|of the current view)\b/.test(
|
||||
line,
|
||||
),
|
||||
`English tool error shown under ${locale} title "${view.result.title}" / step "${view.steps[0].label}: ${view.steps[0].statusText}": "${line}"`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
test('en: the card shows the tool error as the model got it', () => {
|
||||
for (const c of CASES) {
|
||||
const view = cardFor('en', c);
|
||||
// Whole, not cut at 80 characters: the card wraps an error line.
|
||||
assert.deepEqual(view.result.lines, [
|
||||
spokenLabel(c.result.error, Infinity),
|
||||
]);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
// The voice card names layers, sources and distances in the interface
|
||||
// language; what the model receives stays English.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { setLocale } from '../i18n/index.js';
|
||||
import {
|
||||
attachVoiceResult,
|
||||
planStepLabel,
|
||||
shownEntityLabel,
|
||||
} from './speech.js';
|
||||
import {
|
||||
initialVoiceCardState,
|
||||
reduceVoiceCard,
|
||||
voiceCardView,
|
||||
} from './voiceCardPresentation.js';
|
||||
|
||||
for (const locale of ['ru', 'es']) {
|
||||
test(`voice card is in ${locale} for a layer switch`, (t) => {
|
||||
t.after(() => setLocale('en'));
|
||||
setLocale(locale);
|
||||
const out = attachVoiceResult(
|
||||
'set_layer_visibility',
|
||||
{
|
||||
ok: true,
|
||||
layerId: 'flights',
|
||||
label: 'Live Flights',
|
||||
enabled: true,
|
||||
source: 'Flights',
|
||||
},
|
||||
{ layerId: 'flights', enabled: true },
|
||||
);
|
||||
const view = voiceCardView({ steps: [], result: out });
|
||||
assert.doesNotMatch(out.display.title, /Live Flights/);
|
||||
assert.doesNotMatch(JSON.stringify(view), /Flights/);
|
||||
});
|
||||
test(`voice card distance is in ${locale}`, (t) => {
|
||||
t.after(() => setLocale('en'));
|
||||
setLocale(locale);
|
||||
const out = attachVoiceResult('fly_to_location', {
|
||||
ok: true,
|
||||
label: 'Paris',
|
||||
rangeM: 2500,
|
||||
});
|
||||
assert.doesNotMatch(out.display.lines[0], /2\.5 km/);
|
||||
});
|
||||
test(`plan label is in ${locale}`, (t) => {
|
||||
t.after(() => setLocale('en'));
|
||||
setLocale(locale);
|
||||
const a = planStepLabel('select_nearest_aircraft', {});
|
||||
const b = planStepLabel('set_layer_visibility', {
|
||||
layerId: 'earthquakes',
|
||||
enabled: true,
|
||||
});
|
||||
assert.doesNotMatch(a, /Here/);
|
||||
assert.doesNotMatch(b, /Earthquakes/);
|
||||
});
|
||||
}
|
||||
|
||||
test('en: layer switch, distance and plan labels in English', () => {
|
||||
const out = attachVoiceResult(
|
||||
'set_layer_visibility',
|
||||
{
|
||||
ok: true,
|
||||
layerId: 'flights',
|
||||
label: 'Live Flights',
|
||||
enabled: true,
|
||||
source: 'Flights',
|
||||
},
|
||||
{ layerId: 'flights', enabled: true },
|
||||
);
|
||||
assert.equal(out.say, 'Live Flights on.');
|
||||
assert.equal(out.display.title, 'Live Flights on');
|
||||
assert.deepEqual(out.display.sources, [{ label: 'Flights' }]);
|
||||
const fly = attachVoiceResult('fly_to_location', {
|
||||
ok: true,
|
||||
label: 'Paris',
|
||||
rangeM: 2500,
|
||||
});
|
||||
assert.equal(fly.display.lines[0].replace(/\s/g, ' '), 'Camera range 2.5 km');
|
||||
assert.equal(
|
||||
planStepLabel('select_nearest_aircraft', {}),
|
||||
'Nearest aircraft to Here',
|
||||
);
|
||||
assert.equal(
|
||||
planStepLabel('frame_overhead', { target: 'vessels' }),
|
||||
'Frame vessels',
|
||||
);
|
||||
// The plan names the layer as its result card does, not by its id.
|
||||
assert.equal(
|
||||
planStepLabel('set_layer_visibility', {
|
||||
layerId: 'earthquakes',
|
||||
enabled: true,
|
||||
}),
|
||||
'Earthquakes (24h) on',
|
||||
);
|
||||
});
|
||||
|
||||
// FIRMS and ALPR register English entity labels: the model reads them as data
|
||||
// (get_entity_context, pointer_context), the card shows them translated.
|
||||
const ENTITY_LABELS = {
|
||||
ru: { fire: 'Пожар · FRP 12 МВт', alpr: 'Камера ALPR' },
|
||||
es: { fire: 'Incendio · FRP 12 MW', alpr: 'Cámara ALPR' },
|
||||
};
|
||||
|
||||
for (const [locale, shown] of Object.entries(ENTITY_LABELS)) {
|
||||
test(`entity labels stay English in the result and show in ${locale}`, (t) => {
|
||||
t.after(() => setLocale('en'));
|
||||
setLocale(locale);
|
||||
const out = attachVoiceResult('get_entity_context', {
|
||||
ok: true,
|
||||
selected: {
|
||||
id: 'fire-1',
|
||||
name: 'Fire · FRP 12 MW',
|
||||
layerId: 'local-firms',
|
||||
layerName: 'NASA FIRMS',
|
||||
},
|
||||
});
|
||||
assert.equal(out.selected.name, 'Fire · FRP 12 MW');
|
||||
assert.equal(out.display.title, shown.fire);
|
||||
|
||||
const state = reduceVoiceCard(initialVoiceCardState(), {
|
||||
type: 'pointer',
|
||||
pointer: { kind: 'this', label: 'ALPR camera' },
|
||||
});
|
||||
assert.equal(state.pointer.label, 'ALPR camera');
|
||||
assert.equal(voiceCardView(state).pointer.text, shown.alpr);
|
||||
assert.equal(state.announce, `This: ${shown.alpr}`);
|
||||
});
|
||||
}
|
||||
|
||||
test('only the registered entity labels are translated', () => {
|
||||
assert.equal(shownEntityLabel('Fire · FRP 4.9 MW'), 'Fire · FRP 4.9 MW');
|
||||
assert.equal(shownEntityLabel('ALPR camera'), 'ALPR camera');
|
||||
setLocale('ru');
|
||||
try {
|
||||
assert.equal(shownEntityLabel('Fire · FRP 4.9 MW'), 'Пожар · FRP 4.9 МВт');
|
||||
assert.equal(shownEntityLabel('UPS793'), 'UPS793');
|
||||
assert.equal(shownEntityLabel('Fire Station 5'), 'Fire Station 5');
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
// The voice card names a preset city in the interface language in the
|
||||
// fly-to headline and the nearest-aircraft distance line; what the model
|
||||
// says stays English.
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { setLocale } from '../i18n/index.js';
|
||||
import { SPEECH_BUILDERS } from './speech.js';
|
||||
|
||||
function inLocale(locale, run) {
|
||||
setLocale(locale);
|
||||
try {
|
||||
return run();
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
}
|
||||
|
||||
const FLY = { ok: true, arrived: true, label: 'Tokyo' };
|
||||
const NEAREST = {
|
||||
ok: true,
|
||||
layerId: 'flights',
|
||||
location: { label: 'Tokyo' },
|
||||
aircraft: { callsign: 'JAL123', distanceKm: 12, altitudeM: 3000 },
|
||||
feed: {},
|
||||
};
|
||||
|
||||
test('the fly-to headline shows the city in the interface language', () => {
|
||||
assert.equal(SPEECH_BUILDERS.fly_to_location(FLY).display.title, 'Tokyo');
|
||||
const ru = inLocale('ru', () => SPEECH_BUILDERS.fly_to_location(FLY));
|
||||
assert.equal(ru.display.title, 'Токио');
|
||||
assert.match(ru.say, /Tokyo/);
|
||||
const es = inLocale('es', () => SPEECH_BUILDERS.fly_to_location(FLY));
|
||||
assert.equal(es.display.title, 'Tokio');
|
||||
});
|
||||
|
||||
test('the nearest-aircraft distance line shows the city in the interface language', () => {
|
||||
const en = SPEECH_BUILDERS.select_nearest_aircraft(NEAREST);
|
||||
assert.match(en.display.lines[0], /from Tokyo$/);
|
||||
const ru = inLocale('ru', () =>
|
||||
SPEECH_BUILDERS.select_nearest_aircraft(NEAREST),
|
||||
);
|
||||
assert.match(ru.display.lines[0], /Токио/);
|
||||
assert.match(ru.say, /Tokyo/);
|
||||
const es = inLocale('es', () =>
|
||||
SPEECH_BUILDERS.select_nearest_aircraft(NEAREST),
|
||||
);
|
||||
assert.match(es.display.lines[0], /Tokio/);
|
||||
});
|
||||
@@ -0,0 +1,282 @@
|
||||
// Voice card in the UI language: every tool's plan step, the bare "Layer"
|
||||
// step, whole error lines and long headlines, the Contacts snapshot clock.
|
||||
// What the model receives stays English.
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { setLocale } from '../i18n/index.js';
|
||||
import { GEV_ACTION_SCHEMAS } from './actionSchemas.js';
|
||||
import { presentResult, shownSnapshotTime } from './resultDisplay.js';
|
||||
import { CITY_POIS } from '../cityPois.js';
|
||||
import { attachVoiceResult, narrationLabel, planStepLabel } from './speech.js';
|
||||
import {
|
||||
initialVoiceCardState,
|
||||
reduceVoiceCard,
|
||||
voiceCardView,
|
||||
} from './voiceCardPresentation.js';
|
||||
|
||||
const TOOLS = GEV_ACTION_SCHEMAS.map((schema) => schema.name);
|
||||
|
||||
function inLocale(locale, run) {
|
||||
setLocale(locale);
|
||||
try {
|
||||
return run();
|
||||
} finally {
|
||||
setLocale('en');
|
||||
}
|
||||
}
|
||||
|
||||
test('every tool has a plan label, never its raw name', () => {
|
||||
for (const name of TOOLS) {
|
||||
const label = planStepLabel(name, {});
|
||||
assert.notEqual(label, name.replace(/_/g, ' '), name);
|
||||
assert.doesNotMatch(label, /_/, name);
|
||||
}
|
||||
assert.equal(planStepLabel('analyst_query', {}), 'Analyst query');
|
||||
assert.equal(
|
||||
planStepLabel('adjust_camera_zoom', { direction: 'in' }),
|
||||
'Zoom in',
|
||||
);
|
||||
assert.equal(
|
||||
planStepLabel('adjust_camera_zoom', { direction: 'out' }),
|
||||
'Zoom out',
|
||||
);
|
||||
// A tool the card does not know keeps its name in English.
|
||||
assert.equal(planStepLabel('brand_new_tool', {}), 'brand new tool');
|
||||
});
|
||||
|
||||
for (const locale of ['ru', 'es']) {
|
||||
test(`${locale} plan labels are all translated`, () => {
|
||||
const english = Object.fromEntries(
|
||||
TOOLS.map((name) => [name, planStepLabel(name, { direction: 'out' })]),
|
||||
);
|
||||
inLocale(locale, () => {
|
||||
for (const name of TOOLS) {
|
||||
// A layer switch without a layer is checked below.
|
||||
const label = planStepLabel(name, { direction: 'out' });
|
||||
assert.notEqual(label, english[name], `${name}: "${label}"`);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test('analyst query and an unknown tool in ru and es', () => {
|
||||
inLocale('ru', () => {
|
||||
assert.equal(planStepLabel('analyst_query', {}), 'Аналитический запрос');
|
||||
assert.equal(planStepLabel('brand_new_tool', {}), 'Действие');
|
||||
});
|
||||
inLocale('es', () => {
|
||||
assert.equal(planStepLabel('analyst_query', {}), 'Consulta analítica');
|
||||
assert.equal(planStepLabel('brand_new_tool', {}), 'Acción');
|
||||
});
|
||||
});
|
||||
|
||||
test('a layer step with no layer agrees with "Capa"', () => {
|
||||
const step = (enabled) => planStepLabel('set_layer_visibility', { enabled });
|
||||
assert.equal(step(true), 'Layer on');
|
||||
assert.equal(step(false), 'Layer off');
|
||||
inLocale('es', () => {
|
||||
assert.equal(step(true), 'Capa: activada');
|
||||
assert.equal(step(false), 'Capa: desactivada');
|
||||
// A named layer keeps the shared "{label} on" wording.
|
||||
assert.match(
|
||||
planStepLabel('set_layer_visibility', {
|
||||
layerId: 'earthquakes',
|
||||
enabled: true,
|
||||
}),
|
||||
/: activado$/,
|
||||
);
|
||||
});
|
||||
inLocale('ru', () => {
|
||||
assert.equal(step(true), 'Слой: вкл.');
|
||||
assert.equal(step(false), 'Слой: выкл.');
|
||||
});
|
||||
});
|
||||
|
||||
function cardAfter(name, result, args = {}) {
|
||||
let state = reduceVoiceCard(initialVoiceCardState(), {
|
||||
type: 'action-call',
|
||||
id: 'c1',
|
||||
name,
|
||||
arguments: args,
|
||||
});
|
||||
state = reduceVoiceCard(state, {
|
||||
type: 'action-result',
|
||||
id: 'c1',
|
||||
name,
|
||||
result: attachVoiceResult(name, result, args),
|
||||
});
|
||||
return voiceCardView(state);
|
||||
}
|
||||
|
||||
const NO_PLACE = {
|
||||
ok: false,
|
||||
action: 'select_nearest_aircraft',
|
||||
stage: 'location',
|
||||
error:
|
||||
'Nearest-aircraft selection needs a preset, place name, or latitude and longitude',
|
||||
feed: {},
|
||||
};
|
||||
|
||||
test('an error line shows whole, in every language', () => {
|
||||
for (const [locale, expected] of [
|
||||
['en', NO_PLACE.error],
|
||||
[
|
||||
'ru',
|
||||
'Для выбора ближайшего самолёта нужен пресет, название места или широта и долгота',
|
||||
],
|
||||
[
|
||||
'es',
|
||||
'Para seleccionar la aeronave más cercana se necesita un preajuste, un nombre de lugar o la latitud y la longitud',
|
||||
],
|
||||
]) {
|
||||
const view = inLocale(locale, () =>
|
||||
cardAfter('select_nearest_aircraft', NO_PLACE),
|
||||
);
|
||||
assert.deepEqual(view.result.lines, [expected], locale);
|
||||
}
|
||||
});
|
||||
|
||||
test('a successful line keeps its 80-character cap', () => {
|
||||
const long = 'x'.repeat(120);
|
||||
const view = voiceCardView({
|
||||
...initialVoiceCardState(),
|
||||
result: { display: { title: 'T', lines: [long] } },
|
||||
});
|
||||
assert.equal(view.result.lines[0].length, 80);
|
||||
});
|
||||
|
||||
test('a translated analyst headline fits the card title whole', () => {
|
||||
const title =
|
||||
'Al menos 12 aeronaves militares en un radio de 40 km de Los Angeles';
|
||||
const card = () =>
|
||||
voiceCardView({
|
||||
...initialVoiceCardState(),
|
||||
result: { display: { title, lines: [] } },
|
||||
}).result.title;
|
||||
assert.equal(inLocale('es', card), title);
|
||||
// English keeps its 64-character title, equal to the spoken headline.
|
||||
assert.equal(card().length, 64);
|
||||
});
|
||||
|
||||
test('the snapshot time reads as a UTC clock, dated when not today', () => {
|
||||
const iso = '2026-10-15T14:32:05.000Z';
|
||||
const sameDay = Date.parse('2026-10-15T23:00:00Z');
|
||||
const nextDay = Date.parse('2026-10-16T00:10:00Z');
|
||||
assert.equal(shownSnapshotTime(iso, sameDay), iso, 'English keeps ISO');
|
||||
inLocale('ru', () => {
|
||||
assert.equal(shownSnapshotTime(iso, sameDay), '14:32:05 UTC');
|
||||
assert.equal(shownSnapshotTime(iso, nextDay), '15.10.2026, 14:32:05 UTC');
|
||||
});
|
||||
inLocale('es', () => {
|
||||
assert.equal(shownSnapshotTime(iso, sameDay), '14:32:05 UTC');
|
||||
assert.equal(shownSnapshotTime(iso, nextDay), '15/10/2026, 14:32:05 UTC');
|
||||
assert.equal(shownSnapshotTime('not a time'), 'not a time');
|
||||
});
|
||||
});
|
||||
|
||||
test('the Contacts note shows the clock; the model note stays ISO', () => {
|
||||
const evaluatedAt = new Date();
|
||||
evaluatedAt.setUTCHours(14, 32, 5, 0);
|
||||
const iso = evaluatedAt.toISOString();
|
||||
const note = `Contacts panel snapshot at ${iso} — the same retained cohort supplies the card, speech, and follow-up memory.`;
|
||||
const result = {
|
||||
ok: true,
|
||||
count: 12,
|
||||
complete: true,
|
||||
scopeLabel: 'in view',
|
||||
display: { scope: 'in view' },
|
||||
coverage: { note },
|
||||
items: [],
|
||||
};
|
||||
const notes = inLocale('es', () =>
|
||||
presentResult('analyst_query', result).display.notes.join(' | '),
|
||||
);
|
||||
assert.match(notes, /\(14:32:05 UTC\)/);
|
||||
assert.doesNotMatch(notes, /T14:32/);
|
||||
assert.equal(result.coverage.note, note, 'the result is not changed');
|
||||
});
|
||||
|
||||
test('plan steps and errors name a preset place in the UI language', () => {
|
||||
const failed = {
|
||||
ok: false,
|
||||
action: 'select_nearest_aircraft',
|
||||
stage: 'nearest',
|
||||
error:
|
||||
'Live Flights is enabled, but no airborne aircraft is loaded in the Tokyo view yet',
|
||||
location: { label: 'Tokyo' },
|
||||
feed: { state: 'nominal' },
|
||||
};
|
||||
assert.equal(
|
||||
planStepLabel('fly_to_location', { query: 'Tokyo' }),
|
||||
'Fly to Tokyo',
|
||||
);
|
||||
inLocale('es', () => {
|
||||
assert.match(planStepLabel('fly_to_location', { query: 'Tokyo' }), /Tokio/);
|
||||
assert.match(
|
||||
planStepLabel('select_nearest_aircraft', { locationId: 'tokyo' }),
|
||||
/Tokio/,
|
||||
);
|
||||
const view = cardAfter('select_nearest_aircraft', failed);
|
||||
assert.match(view.result.lines[0], /Tokio/);
|
||||
});
|
||||
inLocale('ru', () => {
|
||||
assert.match(planStepLabel('fly_to_location', { query: 'Tokyo' }), /Токио/);
|
||||
});
|
||||
assert.match(failed.error, /Tokyo view/, 'the result stays English');
|
||||
});
|
||||
|
||||
test('the analyst scope names Los Angeles in the UI language', () => {
|
||||
const result = {
|
||||
ok: true,
|
||||
count: 12,
|
||||
complete: false,
|
||||
scopeLabel: 'within 40 km of Los Angeles',
|
||||
display: { scope: 'within 40 km of Los Angeles' },
|
||||
coverage: { layersQueried: [{ layerKey: 'military' }] },
|
||||
items: [],
|
||||
};
|
||||
const title = inLocale(
|
||||
'es',
|
||||
() => presentResult('analyst_query', result).display.title,
|
||||
);
|
||||
assert.match(title, /Los Ángeles$/, 'the headline is not cut at 64');
|
||||
assert.equal(result.scopeLabel, 'within 40 km of Los Angeles');
|
||||
});
|
||||
|
||||
// A preset city id from the tool schema (sf, nyc, dc) reads as its city name.
|
||||
test('a preset city id names the city in plan steps and narration', () => {
|
||||
const step = (id, action = 'fly_to_location') =>
|
||||
planStepLabel(action, { locationId: id });
|
||||
assert.equal(step('sf'), 'Fly to San Francisco');
|
||||
assert.equal(
|
||||
narrationLabel('fly_to_location', { locationId: 'dc' }),
|
||||
'Washington DC',
|
||||
);
|
||||
assert.equal(
|
||||
planStepLabel('fly_to_location', { query: 'Ulaanbaatar' }),
|
||||
'Fly to Ulaanbaatar',
|
||||
'other places are shown as given',
|
||||
);
|
||||
inLocale('es', () => {
|
||||
assert.equal(step('tokyo'), 'Volar a Tokio');
|
||||
assert.equal(step('nyc'), 'Volar a Nueva York');
|
||||
for (const id of ['sf', 'nyc', 'dc'])
|
||||
assert.doesNotMatch(step(id), /\b(Sf|Nyc|Dc)$/, id);
|
||||
});
|
||||
inLocale('ru', () => {
|
||||
assert.match(step('dc', 'select_nearest_aircraft'), /Вашингтон/);
|
||||
});
|
||||
});
|
||||
|
||||
test('every locationId the voice tools accept is a preset city', () => {
|
||||
const ids = new Set();
|
||||
const visit = (node) => {
|
||||
if (!node || typeof node !== 'object') return;
|
||||
if (Array.isArray(node.locationId?.enum))
|
||||
node.locationId.enum.forEach((id) => ids.add(id));
|
||||
Object.values(node).forEach(visit);
|
||||
};
|
||||
visit(GEV_ACTION_SCHEMAS);
|
||||
assert.ok(ids.size > 0, 'found the locationId enums');
|
||||
for (const id of ids) assert.ok(Object.hasOwn(CITY_POIS, id), id);
|
||||
});
|
||||
@@ -1,7 +1,13 @@
|
||||
import { planStepLabel, progressStepLabel, spokenLabel } from './speech.js';
|
||||
import {
|
||||
planStepLabel,
|
||||
progressStepLabel,
|
||||
shownEntityLabel,
|
||||
spokenLabel,
|
||||
} from './speech.js';
|
||||
import { presentResult } from './resultDisplay.js';
|
||||
import { MAX_DISPLAYED_REFERENTS, normalizeReferents } from './referents.js';
|
||||
|
||||
import { getLocale, msg, t } from '../i18n/index.js';
|
||||
/**
|
||||
* Pure presentation of the voice card nested in the voice control: captions,
|
||||
* the current plan, the latest result and its numbered referents.
|
||||
@@ -16,12 +22,20 @@ const MAX_STEPS = 6;
|
||||
const MAX_LINES = 6;
|
||||
const MAX_NOTES = 6;
|
||||
const MAX_CAPTION = 220;
|
||||
// Titles and lines wrap; the caps only stop runaway values. Translations run
|
||||
// longer (resultDisplay.js), so they cap at 120. Error lines are never cut.
|
||||
const MAX_TITLE = 64;
|
||||
const MAX_TRANSLATED_TITLE = 120;
|
||||
const MAX_LINE = 80;
|
||||
|
||||
const titleMax = () =>
|
||||
getLocale() === 'en' ? MAX_TITLE : MAX_TRANSLATED_TITLE;
|
||||
|
||||
const STATUS_TEXT = Object.freeze({
|
||||
running: 'in progress',
|
||||
done: 'done',
|
||||
failed: 'failed',
|
||||
cancelled: 'cancelled',
|
||||
running: msg('in progress'),
|
||||
done: msg('done'),
|
||||
failed: msg('failed'),
|
||||
cancelled: msg('cancelled'),
|
||||
});
|
||||
|
||||
/** An empty, hidden card. */
|
||||
@@ -30,6 +44,7 @@ export function initialVoiceCardState() {
|
||||
visible: false,
|
||||
dismissed: false,
|
||||
user: null,
|
||||
userAttribution: null,
|
||||
pointer: null,
|
||||
assistant: null,
|
||||
assistantResponse: null,
|
||||
@@ -55,6 +70,7 @@ function newTurn(state) {
|
||||
visible: true,
|
||||
dismissed: false,
|
||||
user: '',
|
||||
userAttribution: null,
|
||||
pointer: null,
|
||||
assistant: null,
|
||||
assistantResponse: null,
|
||||
@@ -123,13 +139,20 @@ export function reduceVoiceCard(state, event) {
|
||||
return reveal({
|
||||
...state,
|
||||
pointer,
|
||||
announce: `${kind}: ${spokenLabel(pointer.label, 32)}`,
|
||||
announce: `${kind}: ${spokenLabel(shownEntityLabel(pointer.label), 32)}`,
|
||||
});
|
||||
}
|
||||
case 'transcript': {
|
||||
const text = event.text || '';
|
||||
if (event.role === 'user') {
|
||||
return text ? reveal({ ...state, user: caption(text) }) : state;
|
||||
return text
|
||||
? reveal({
|
||||
...state,
|
||||
user: caption(text),
|
||||
userAttribution:
|
||||
event.attribution === 'unverified' ? 'unverified' : null,
|
||||
})
|
||||
: state;
|
||||
}
|
||||
if (event.role !== 'assistant' || !text) return state;
|
||||
const sameResponse =
|
||||
@@ -190,7 +213,7 @@ export function reduceVoiceCard(state, event) {
|
||||
: result.ok
|
||||
? 'done'
|
||||
: 'failed',
|
||||
detail: result.outlinePending ? 'Tracing outline' : '',
|
||||
detail: result.outlinePending ? t('Tracing outline') : '',
|
||||
annotationIds: result.outlinePending ? annotationIds(result) : [],
|
||||
}),
|
||||
);
|
||||
@@ -204,12 +227,14 @@ export function reduceVoiceCard(state, event) {
|
||||
...next,
|
||||
result: {
|
||||
display: shown.display,
|
||||
// A failed result's lines are its error: shown whole, wrapped.
|
||||
wholeLines: result.ok === false,
|
||||
referents: normalizeReferents(
|
||||
shown.referents,
|
||||
MAX_DISPLAYED_REFERENTS,
|
||||
),
|
||||
},
|
||||
announce: spokenLabel(shown.display.title, 64),
|
||||
announce: spokenLabel(shown.display.title, titleMax()),
|
||||
});
|
||||
}
|
||||
case 'outline': {
|
||||
@@ -224,10 +249,10 @@ export function reduceVoiceCard(state, event) {
|
||||
...step,
|
||||
annotationIds: remaining,
|
||||
detail: remaining.length
|
||||
? 'Tracing outline'
|
||||
? t('Tracing outline')
|
||||
: event.status === 'resolved'
|
||||
? 'Outline traced'
|
||||
: 'Outline unavailable',
|
||||
? t('Outline traced')
|
||||
: t('Outline unavailable'),
|
||||
};
|
||||
return reveal({ ...state, steps });
|
||||
}
|
||||
@@ -244,7 +269,9 @@ export function voiceCardView(state) {
|
||||
const display = state.result?.display || null;
|
||||
const allNotes = [
|
||||
...(display?.notes || []),
|
||||
...(display?.sources || []).map((source) => `Source: ${source.label}`),
|
||||
...(display?.sources || []).map((source) =>
|
||||
t('Source: {label}', { label: source.label }),
|
||||
),
|
||||
];
|
||||
// Analyst notes carry count/scope/provenance qualifications. Preserve their
|
||||
// full sanitized text in at most six existing rows, joining overflow into
|
||||
@@ -267,15 +294,17 @@ export function voiceCardView(state) {
|
||||
return {
|
||||
visible: Boolean(state.visible),
|
||||
busy: running,
|
||||
phase: running ? 'WORKING' : state.steps.length ? 'DONE' : '',
|
||||
phase: running ? t('WORKING') : state.steps.length ? t('DONE') : '',
|
||||
user: {
|
||||
label:
|
||||
state.userAttribution === 'unverified' ? t('Recent speech') : t('YOU'),
|
||||
visible: state.user !== null,
|
||||
text: state.user || '…',
|
||||
},
|
||||
pointer: {
|
||||
visible: Boolean(state.pointer),
|
||||
kind: state.pointer?.kind === 'here' ? 'HERE' : 'THIS',
|
||||
text: spokenLabel(state.pointer?.label || '', 32),
|
||||
kind: state.pointer?.kind === 'here' ? t('HERE') : t('THIS'),
|
||||
text: spokenLabel(shownEntityLabel(state.pointer?.label || ''), 32),
|
||||
},
|
||||
assistant: {
|
||||
visible: Boolean(state.assistant),
|
||||
@@ -285,15 +314,17 @@ export function voiceCardView(state) {
|
||||
key: String(step.id),
|
||||
label: step.label,
|
||||
status: step.status,
|
||||
statusText: STATUS_TEXT[step.status] || '',
|
||||
statusText: STATUS_TEXT[step.status] ? t(STATUS_TEXT[step.status]) : '',
|
||||
detail: step.detail || '',
|
||||
})),
|
||||
result: {
|
||||
visible: Boolean(display?.title),
|
||||
title: spokenLabel(display?.title || '', 64),
|
||||
title: spokenLabel(display?.title || '', titleMax()),
|
||||
chips: (display?.chips || []).map((chip) => spokenLabel(chip.label, 24)),
|
||||
lines: (display?.lines || [])
|
||||
.map((line) => spokenLabel(line, 80))
|
||||
.map((line) =>
|
||||
spokenLabel(line, state.result?.wholeLines ? Infinity : MAX_LINE),
|
||||
)
|
||||
.slice(0, MAX_LINES),
|
||||
referents: (state.result?.referents || [])
|
||||
.slice(0, MAX_DISPLAYED_REFERENTS)
|
||||
|
||||
@@ -50,7 +50,7 @@ export const VOICE_MODELS = Object.freeze({
|
||||
standard: Object.freeze({
|
||||
tier: 'standard',
|
||||
id: 'gpt-realtime-2',
|
||||
label: 'STANDARD',
|
||||
label: 'STANDARD', // i18n-ignore: model data; the tier button has its own text
|
||||
/** USD per 1M tokens — gpt-realtime-2. */
|
||||
rates: Object.freeze({
|
||||
textInput: 4,
|
||||
@@ -66,7 +66,7 @@ export const VOICE_MODELS = Object.freeze({
|
||||
mini: Object.freeze({
|
||||
tier: 'mini',
|
||||
id: 'gpt-realtime-2.1-mini',
|
||||
label: 'MINI',
|
||||
label: 'MINI', // i18n-ignore: model data; the tier button has its own text
|
||||
/** USD per 1M tokens — gpt-realtime-2.1-mini (~3.2× cheaper on audio). */
|
||||
rates: Object.freeze({
|
||||
textInput: 0.6,
|
||||
|
||||
Reference in new issue
Block a user